Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

modify default asym 4bits auto-round format to awq, fix save folder typo for mllm (#575)

Weiweic1c8148e4c8a
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index b527f16bf..faf9a8f7c 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -170,9 +170,11 @@ class AutoRound(object):
self.model_orig_dtype = model.dtype
self.seed = seed
set_seed(self.seed)
- assert not unsupport_meta_device(model), (
- "AutoRound does not support for params on meta device."
- " Please use more gpus by setting `--device 0,1,2,3` or just use one gpu")
+ if unsupport_meta_device(model):
+ raise RuntimeError(
+ "AutoRound does not support parameters on meta device. "
+ "Please use more GPUs by setting `--device 0,1,2,3` or just use one GPU."
+ )
## important tuning hype-parameters
self.amp = amp
@@ -384,19 +386,28 @@ class AutoRound(object):
"""Checks if the configurations are valid.
Raises:
- AssertionError: If any of the configurations are invalid.
+ ValueError, TypeError: If any of the configurations are invalid.
"""
- assert isinstance(self.model, torch.nn.Module)
- assert self.bits > 0, "bits must be positive"
- assert self.act_bits > 0, "bits must be positive"
- assert self.group_size == -1 or self.group_size >= 1, "only supports positive group_size or -1(per channel)"
- assert self.act_group_size == -1 or self.act_group_size >= 1, \
- "only supports positive group_size or -1(per channel)"
- assert self.batch_size > 0, "batch size must be positive"
- assert self.iters >= 0, "iters must be non-negative"
- assert self.seqlen > 0, "seqlen must be positive"
- assert self.nblocks > 0, "nblocks must be positive"
- assert self.gradient_accumulate_steps > 0, "gradient accumulate step must be positive"
+ if not isinstance(self.model, torch.nn.Module):
+ raise TypeError("model must be an instance of torch.nn.Module")
+ if self.bits <= 0:
+ raise ValueError("bits must be positive")
+ if self.act_bits <= 0:
+ raise ValueError("act_bits must be positive")
+ if not (self.group_size == -1 or self.group_size >= 1):
+ raise ValueError("group_size must be -1 (per channel) or a positive integer")
+ if not (self.act_group_size == -1 or self.act_group_size >= 1):
+ raise ValueError("act_group_size must be -1 (per channel) or a positive integer")
+ if self.batch_size <= 0:
+ raise ValueError("batch_size must be positive")
+ if self.iters < 0:
+ raise ValueError("iters must be non-negative")
+ if self.seqlen <= 0:
+ raise ValueError("seqlen must be positive")
+ if self.nblocks <= 0:
+ raise ValueError("nblocks must be positive")
+ if self.gradient_accumulate_steps <= 0:
+ raise ValueError("gradient_accumulate_steps must be positive")
# assert self.tokenizer != None or self.dataloader != None
if self.act_bits <= 8:
logger.warning(
@@ -493,10 +504,18 @@ class AutoRound(object):
# Adjust format settings based on compatibility
for index in range(len(formats)):
format = formats[index]
- if "auto_round" in format:
- if (self.sym and ("gptq" not in format and "awq" not in format)) or self.bits == 3:
+ if format=="auto_round":
+ if self.sym or self.bits == 3:
format = format.replace('auto_round', 'auto_round:auto_gptq')
formats[index] = format
+ if self.bits == 4 and not self.sym:
+ enable_awq = all(
+ config["bits"] == self.bits or config["bits"] >= 16
+ for config in self.layer_config.values()
+ )
+ if enable_awq:
+ formats[index] = format.replace("auto_round", "auto_round:auto_awq")
+
# Remove duplicates from formats list
def remove_duplicates(lst):
@@ -622,8 +641,11 @@ class AutoRound(object):
device=self.device,
pbar=pbar
)
- if self.is_packing_immediate:
- assert len(self.formats) == 1
+ if self.is_packing_immediate and len(self.formats) != 1:
+ raise ValueError(
+ f"Expected exactly one packing format when 'is_packing_immediate' is True, "
+ f"but got {len(self.formats)} formats."
+ )
self.quant_layers(layer_names, all_inputs) ##TODO pack layer immediately
@@ -2182,3 +2204,4 @@ class AutoRoundAdam(AutoRoundOPT):
super_group_size=super_group_size,
**kwargs,
)
+
diff --git a/auto_round/calib_dataset.py b/auto_round/calib_dataset.py
index 33d16c998..f2827419b 100644
--- a/auto_round/calib_dataset.py
+++ b/auto_round/calib_dataset.py
@@ -408,7 +408,8 @@ def get_local_dataset(tokenizer, seqlen, dataset_name="./tmp.json", split=None,
text = data["text"]
elif isinstance(data, dict) and "input_ids" in data.keys():
text = data["input_ids"]
- assert isinstance(text, str), "data must be string"
+ if not isinstance(text, str):
+ raise TypeError("data must be a string")
text = text.rstrip()
text = text.rstrip("\n")
samples.append({"text": text})
@@ -696,3 +697,4 @@ def get_dataloader(
calib_dataloader = DataLoader(dataset_final, batch_size=bs, shuffle=False, collate_fn=collate_batch)
return calib_dataloader
+
diff --git a/auto_round/data_type/utils.py b/auto_round/data_type/utils.py
index 8c8ee0c70..87133e4b5 100644
--- a/auto_round/data_type/utils.py
+++ b/auto_round/data_type/utils.py
@@ -83,7 +83,7 @@ def get_quant_func(dtype, bits, sym):
This function returns the appropriate quantization function from the QUANT_FUNC_WITH_DTYPE
dictionary based on the provided data type (`dtype`), bit width (`bits`), and whether
- the quantization is symmetric (`sym`). If the function does not exist, it asserts False.
+ the quantization is symmetric (`sym`). If the function does not exist, raise ValueError.
Args:
dtype (str): The data type for the quantization (e.g., 'int', 'mxfp4').
@@ -130,7 +130,7 @@ def get_quant_func(dtype, bits, sym):
if key in QUANT_FUNC_WITH_DTYPE.keys():
return QUANT_FUNC_WITH_DTYPE[key], key
- assert False, f"{dtype} is not supported"
+ raise ValueError(f"{dtype} is not supported")
def round_ste(x: torch.Tensor):
@@ -206,3 +206,4 @@ def get_gaudi_fp8_ste_func():
+
diff --git a/auto_round/export/export_to_autoround/export.py b/auto_round/export/export_to_autoround/export.py
index 2d39eb48b..e50fcd5cb 100644
--- a/auto_round/export/export_to_autoround/export.py
+++ b/auto_round/export/export_to_autoround/export.py
@@ -69,7 +69,7 @@ def dynamic_import_quant_linear_for_packing(backend, bits, group_size, sym, act_
class: The dynamically imported QuantLinear class configured according to the specified parameters.
Raises:
- AssertionError: If the backend is not supported.
+ ValueError: If the backend is not supported.
"""
if "auto_round" in backend and "awq" not in backend and "gptq" not in backend:
if act_bits <= 8: ##easily have bug for other configuration, need to refine code later
@@ -88,7 +88,10 @@ def dynamic_import_quant_linear_for_packing(backend, bits, group_size, sym, act_
elif "gptq" in backend and not "gptqmodel" in backend: ## have g_idx
return get_autogptq_packing_qlinear(backend, bits, group_size, sym)
else:
- assert False, f"only support auto_gptq, auto_awq and auto_round backend"
+ raise ValueError(
+ f"Unsupported backend: '{backend}'. "
+ "Only 'auto_round', 'auto_round:auto_awq', 'auto_round:auto_gptq', 'awq', and 'gptq' are supported."
+ )
def pack_qact_layer(name, model):
@@ -247,7 +250,7 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:ex
None
Raises:
- AssertionError: If the backend is not supported.
+ ValueError: If the backend is not supported.
"""
##if using sym, we change to gptq sym kernel to avoid compiling from auto_round source
@@ -367,3 +370,4 @@ def save(model: nn.Module, save_dir: str, max_shard_size: str = "5GB", safe_seri
if hasattr(model, "config") and hasattr(model.config, "quantization_config"):
with open(os.path.join(save_dir, config_file), "w", encoding="utf-8") as f:
json.dump(model.config.quantization_config, f, indent=2)
+
diff --git a/auto_round/export/export_to_awq/utils.py b/auto_round/export/export_to_awq/utils.py
index 40107b802..48b00ccc1 100644
--- a/auto_round/export/export_to_awq/utils.py
+++ b/auto_round/export/export_to_awq/utils.py
@@ -148,8 +148,11 @@ class WQLinear_GEMM(nn.Module):
self.training = training
# quick sanity check (make sure alignment)
- assert self.in_features % self.group_size == 0
- assert out_features % (32 // self.w_bit) == 0
+ if self.in_features % self.group_size != 0:
+ raise ValueError(f"in_features ({self.in_features}) shape mismatch")
+
+ if out_features % (32 // self.w_bit) != 0:
+ raise ValueError(f"out_features ({out_features}) shape mismatch")
self.register_buffer(
"qweight",
@@ -203,7 +206,8 @@ class WQLinear_GEMM(nn.Module):
return awq_linear
# need scales and zeros info for real quantization
- assert scales is not None and zeros is not None
+ if scales is None or zeros is None:
+ raise ValueError("Both 'scales' and 'zeros' must be provided (not None)")
awq_linear.scales = scales.clone().half()
if linear.bias is not None:
@@ -312,3 +316,4 @@ def clear_memory(weight=None):
del weight
gc.collect()
torch.cuda.empty_cache()
+
diff --git a/auto_round/export/export_to_gguf/export.py b/auto_round/export/export_to_gguf/export.py
index 5d6d3459c..8c63e2c72 100644
--- a/auto_round/export/export_to_gguf/export.py
+++ b/auto_round/export/export_to_gguf/export.py
@@ -74,7 +74,8 @@ def save_quantized_as_gguf(output_dir, backend="gguf:q4_0", **kwargs):
model_name = model_name[-1]
output_type = backend.split(":")[-1]
- assert output_type.lower() in FTYPE_MAP, f"{output_type} is not supported"
+ if output_type.lower() not in FTYPE_MAP:
+ raise TypeError(f"{output_type} type is not supported")
output_type = FTYPE_MAP.get(output_type.lower())
@@ -96,3 +97,4 @@ def save_quantized_as_gguf(output_dir, backend="gguf:q4_0", **kwargs):
shutil.rmtree(tmp_work_dir, ignore_errors=True)
return model
+
diff --git a/auto_round/export/export_to_itrex/config.py b/auto_round/export/export_to_itrex/config.py
index 355b95a56..c212dbe82 100644
--- a/auto_round/export/export_to_itrex/config.py
+++ b/auto_round/export/export_to_itrex/config.py
@@ -249,7 +249,7 @@ class QuantConfig(PretrainedConfig):
self._set_token_in_kwargs(kwargs)
if os.path.isfile(save_directory):
- raise AssertionError(f"Provided path ({save_directory}) should be a directory, not a file")
+ raise OSError(f"Provided path ({save_directory}) should be a directory, not a file")
os.makedirs(save_directory, exist_ok=True)
@@ -301,3 +301,4 @@ class QuantConfig(PretrainedConfig):
for parameter in remove_parameters:
if hasattr(self, parameter):
delattr(self, parameter)
+
diff --git a/auto_round/inference/backend.py b/auto_round/inference/backend.py
index c2d589280..127d76e1b 100644
--- a/auto_round/inference/backend.py
+++ b/auto_round/inference/backend.py
@@ -597,15 +597,16 @@ def get_layer_backend(device, backend, orig_backend, bits, group_size, sym, in_f
The selected backend that is compatible with the layer configuration.
Raises:
- AssertionError:
- If the specified backend is not supported.
ValueError:
+ If the specified backend is not supported.
If no compatible backend is found for the given layer configuration.
"""
# Check if the provided backend is in BackendInfos
backend = find_backend(backend)
- assert backend in BackendInfos.keys(), \
- f"Unsupported backend {backend}, please set it to `auto` to try automatic selection"
+ if backend not in BackendInfos.keys():
+ raise ValueError(
+ f"Unsupported backend '{backend}'. Please set it to 'auto' to enable automatic selection."
+ )
packing_format = BackendInfos[orig_backend].packing_format
@@ -735,3 +736,4 @@ def process_requirement(requirements: list, target_device="cuda", logger_level="
log(joined_cmds)
if logger_level == "error":
exit(-1)
+
diff --git a/auto_round/mllm/eval.py b/auto_round/mllm/eval.py
index 44de86c56..997dea72d 100644
--- a/auto_round/mllm/eval.py
+++ b/auto_round/mllm/eval.py
@@ -286,7 +286,8 @@ def mllm_eval(
if mode == 'all':
eval_results = dataset.evaluate(result_file, **judge_kwargs)
if eval_results is not None:
- assert isinstance(eval_results, dict) or isinstance(eval_results, pd.DataFrame)
+ if not isinstance(eval_results, dict) and not isinstance(eval_results, pd.DataFrame):
+ raise TypeError("Unsupported eval result type")
logger.info(f'The evaluation of model {model_name} x dataset {dataset_name} has finished! ')
logger.info('Evaluation Results:')
if isinstance(eval_results, dict):
@@ -372,7 +373,8 @@ def lmms_eval(
config = AutoConfig.from_pretrained(model, trust_remote_code=True)
model_type = config.model_type
- assert model_type in MODEL_TYPE_TO_LMMS_MODEL, f"{model_type} is not support by lmms."
+ if model_type not in MODEL_TYPE_TO_LMMS_MODEL:
+ raise ValueError(f"{model_type} is not support by lmms.")
if MODEL_TYPE_TO_LMMS_MODEL[model_type] == "phi3v":
model_args = f"model_id_name={model}"
@@ -410,3 +412,4 @@ def lmms_eval(
return results
+
diff --git a/auto_round/mllm/template.py b/auto_round/mllm/template.py
index 88ba62593..0782fd09c 100644
--- a/auto_round/mllm/template.py
+++ b/auto_round/mllm/template.py
@@ -127,15 +127,17 @@ def load_template(path: str):
if "model_type" not in data:
data["model_type"] = "user_define"
if "replace_tokens" in data and data["replace_tokens"] is not None:
- assert len(data["replace_tokens"]) % 2 == 0, \
- "the format of replace_tokens should be [old_tag1, replace_tag1, old_tag2, replace_tag2]"
+ if len(data["replace_tokens"]) % 2 != 0:
+ raise ValueError("the format of replace_tokens should be " \
+ "[old_tag1, replace_tag1, old_tag2, replace_tag2]")
temp = []
for i in range(0, len(data["replace_tokens"]), 2):
temp.append((data["replace_tokens"][i], data["replace_tokens"][i + 1]))
data["replace_tokens"] = temp
if "processor" in data:
- assert data["processor"] in PROCESSORS.keys(), \
- "{} is not supported, current support: {}".format(data["processor"], ",".join(PROCESSORS.keys()))
+ if data["processor"] not in PROCESSORS.keys():
+ raise ValueError(f"{data['processor']} is not supported, current support: " \
+ "{','.join(PROCESSORS.keys())}")
data["processor"] = PROCESSORS[data["processor"]]
template = _register_template(
**data
@@ -177,3 +179,4 @@ def get_template(
model=model, tokenizer=tokenizer, processor=processor, image_processor=image_processor, use_rtn=use_rtn)
return template
+
diff --git a/auto_round/script/llm.py b/auto_round/script/llm.py
index f6046c64f..af68b1175 100644
--- a/auto_round/script/llm.py
+++ b/auto_round/script/llm.py
@@ -355,7 +355,7 @@ def tune(args):
"It is recommended to use sym quantization or --format='auto_round'")
if "marlin" in args.format and args.asym is True:
- assert False, "marlin backend only supports sym quantization, please remove --asym"
+ raise RuntimeError("marlin backend only supports sym quantization, please remove --asym")
##must set this before import torch
set_cuda_visible_devices(args.device)
@@ -768,3 +768,4 @@ def eval_task_by_task(
print(make_table(res_all))
print("total eval time:", time.time() - st)
+
diff --git a/auto_round/script/mllm.py b/auto_round/script/mllm.py
index 212fb902a..7f76d3204 100644
--- a/auto_round/script/mllm.py
+++ b/auto_round/script/mllm.py
@@ -458,7 +458,8 @@ def tune(args):
format_list = args.format.replace(' ', '').split(',')
inplace = False if len(format_list) > 1 else True
for format_ in format_list:
- eval_folder = f'{export_dir}-{format_}'
+ save_format_ = format_.replace(":", "-").replace("_", "-")
+ eval_folder = os.path.join(export_dir, save_format_) if len(formats) > 1 else export_dir
safe_serialization = True
if hasattr(autoround.model, "config") and hasattr(autoround.model.config, "model_type"):
if "phi3_v" in autoround.model.config.model_type:
@@ -562,3 +563,4 @@ def lmms_eval(args):
)
return results
+
diff --git a/auto_round/utils.py b/auto_round/utils.py
index 8ddeb3ce0..61c15a52c 100644
--- a/auto_round/utils.py
+++ b/auto_round/utils.py
@@ -642,7 +642,7 @@ def convert_dtype_str2torch(str_dtype):
torch.dtype: The PyTorch dtype.
Raises:
- AssertionError: If the input str_dtype is unsupported.
+ ValueError: If the input str_dtype is unsupported.
"""
if isinstance(str_dtype, torch.dtype) or str_dtype is None:
return str_dtype
@@ -655,7 +655,7 @@ def convert_dtype_str2torch(str_dtype):
elif str_dtype == "bf16" or str_dtype == "bfloat16":
return torch.bfloat16
else:
- assert False, "Unsupported str dtype {} to torch dtype".format(str_dtype)
+ raise ValueError(f"Unsupported string dtype '{str_dtype}' for conversion to torch dtype.")
def convert_dtype_torch2str(dtype):
@@ -668,7 +668,7 @@ def convert_dtype_torch2str(dtype):
str: The string representation of the dtype.
Raises:
- AssertionError: If the input dtype is unsupported.
+ ValueError: If the input dtype is unsupported.
"""
if isinstance(dtype, str) or dtype is None:
return dtype
@@ -683,7 +683,7 @@ def convert_dtype_torch2str(dtype):
elif isinstance(dtype, str) and dtype in ["int8", "fp32", "fp16", "bf16"]:
return dtype
else:
- assert False, "Unsupported pytorch dtype {} to str dtype".format(dtype)
+ raise ValueError(f"Unsupported PyTorch dtype '{dtype}' for conversion to string dtype.")
def convert_dtype_torch2str_hf(dtype):
@@ -696,7 +696,7 @@ def convert_dtype_torch2str_hf(dtype):
str: The string representation of the dtype.
Raises:
- AssertionError: If the input str_dtype is unsupported.
+ ValueError: If the input str_dtype is unsupported.
"""
if dtype is None:
return dtype
@@ -707,7 +707,7 @@ def convert_dtype_torch2str_hf(dtype):
return dtype
str_dtype = str(dtype)
if "." not in str_dtype:
- assert False, "Unsupported pytorch dtype {} to huggingface str dtype".format(dtype)
+ raise ValueError(f"Unsupported pytorch dtype '{dtype}' for conversion to huggingface str dtype")
str_dtype = str_dtype.split(".")[1]
return str_dtype
@@ -1512,3 +1512,4 @@ def filter_quantization_config(quantization_config):
quantization_config.pop("act_dynamic", None)
quantization_config.pop("act_sym", None)
quantization_config.pop("act_group_size", None)
+