PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
modify default asym 4bits auto-round format to awq, fix save folder typo for mllm (#575)
Weiweic1c8148e4c8a
diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex b527f16bf..faf9a8f7c 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -170,9 +170,11 @@ class AutoRound(object):self.model_orig_dtype = model.dtypeself.seed = seedset_seed(self.seed)- assert not unsupport_meta_device(model), (- "AutoRound does not support for params on meta device."- " Please use more gpus by setting `--device 0,1,2,3` or just use one gpu")+ if unsupport_meta_device(model):+ raise RuntimeError(+ "AutoRound does not support parameters on meta device. "+ "Please use more GPUs by setting `--device 0,1,2,3` or just use one GPU."+ )## important tuning hype-parametersself.amp = amp@@ -384,19 +386,28 @@ class AutoRound(object):"""Checks if the configurations are valid.Raises:- AssertionError: If any of the configurations are invalid.+ ValueError, TypeError: If any of the configurations are invalid."""- assert isinstance(self.model, torch.nn.Module)- assert self.bits > 0, "bits must be positive"- assert self.act_bits > 0, "bits must be positive"- assert self.group_size == -1 or self.group_size >= 1, "only supports positive group_size or -1(per channel)"- assert self.act_group_size == -1 or self.act_group_size >= 1, \- "only supports positive group_size or -1(per channel)"- assert self.batch_size > 0, "batch size must be positive"- assert self.iters >= 0, "iters must be non-negative"- assert self.seqlen > 0, "seqlen must be positive"- assert self.nblocks > 0, "nblocks must be positive"- assert self.gradient_accumulate_steps > 0, "gradient accumulate step must be positive"+ if not isinstance(self.model, torch.nn.Module):+ raise TypeError("model must be an instance of torch.nn.Module")+ if self.bits <= 0:+ raise ValueError("bits must be positive")+ if self.act_bits <= 0:+ raise ValueError("act_bits must be positive")+ if not (self.group_size == -1 or self.group_size >= 1):+ raise ValueError("group_size must be -1 (per channel) or a positive integer")+ if not (self.act_group_size == -1 or self.act_group_size >= 1):+ raise ValueError("act_group_size must be -1 (per channel) or a positive integer")+ if self.batch_size <= 0:+ raise ValueError("batch_size must be positive")+ if self.iters < 0:+ raise ValueError("iters must be non-negative")+ if self.seqlen <= 0:+ raise ValueError("seqlen must be positive")+ if self.nblocks <= 0:+ raise ValueError("nblocks must be positive")+ if self.gradient_accumulate_steps <= 0:+ raise ValueError("gradient_accumulate_steps must be positive")# assert self.tokenizer != None or self.dataloader != Noneif self.act_bits <= 8:logger.warning(@@ -493,10 +504,18 @@ class AutoRound(object):# Adjust format settings based on compatibilityfor index in range(len(formats)):format = formats[index]- if "auto_round" in format:- if (self.sym and ("gptq" not in format and "awq" not in format)) or self.bits == 3:+ if format=="auto_round":+ if self.sym or self.bits == 3:format = format.replace('auto_round', 'auto_round:auto_gptq')formats[index] = format+ if self.bits == 4 and not self.sym:+ enable_awq = all(+ config["bits"] == self.bits or config["bits"] >= 16+ for config in self.layer_config.values()+ )+ if enable_awq:+ formats[index] = format.replace("auto_round", "auto_round:auto_awq")+# Remove duplicates from formats listdef remove_duplicates(lst):@@ -622,8 +641,11 @@ class AutoRound(object):device=self.device,pbar=pbar)- if self.is_packing_immediate:- assert len(self.formats) == 1+ if self.is_packing_immediate and len(self.formats) != 1:+ raise ValueError(+ f"Expected exactly one packing format when 'is_packing_immediate' is True, "+ f"but got {len(self.formats)} formats."+ )self.quant_layers(layer_names, all_inputs) ##TODO pack layer immediately@@ -2182,3 +2204,4 @@ class AutoRoundAdam(AutoRoundOPT):super_group_size=super_group_size,**kwargs,)+diff --git a/auto_round/calib_dataset.py b/auto_round/calib_dataset.pyindex 33d16c998..f2827419b 100644--- a/auto_round/calib_dataset.py+++ b/auto_round/calib_dataset.py@@ -408,7 +408,8 @@ def get_local_dataset(tokenizer, seqlen, dataset_name="./tmp.json", split=None,text = data["text"]elif isinstance(data, dict) and "input_ids" in data.keys():text = data["input_ids"]- assert isinstance(text, str), "data must be string"+ if not isinstance(text, str):+ raise TypeError("data must be a string")text = text.rstrip()text = text.rstrip("\n")samples.append({"text": text})@@ -696,3 +697,4 @@ def get_dataloader(calib_dataloader = DataLoader(dataset_final, batch_size=bs, shuffle=False, collate_fn=collate_batch)return calib_dataloader+diff --git a/auto_round/data_type/utils.py b/auto_round/data_type/utils.pyindex 8c8ee0c70..87133e4b5 100644--- a/auto_round/data_type/utils.py+++ b/auto_round/data_type/utils.py@@ -83,7 +83,7 @@ def get_quant_func(dtype, bits, sym):This function returns the appropriate quantization function from the QUANT_FUNC_WITH_DTYPEdictionary based on the provided data type (`dtype`), bit width (`bits`), and whether- the quantization is symmetric (`sym`). If the function does not exist, it asserts False.+ the quantization is symmetric (`sym`). If the function does not exist, raise ValueError.Args:dtype (str): The data type for the quantization (e.g., 'int', 'mxfp4').@@ -130,7 +130,7 @@ def get_quant_func(dtype, bits, sym):if key in QUANT_FUNC_WITH_DTYPE.keys():return QUANT_FUNC_WITH_DTYPE[key], key- assert False, f"{dtype} is not supported"+ raise ValueError(f"{dtype} is not supported")def round_ste(x: torch.Tensor):@@ -206,3 +206,4 @@ def get_gaudi_fp8_ste_func():+diff --git a/auto_round/export/export_to_autoround/export.py b/auto_round/export/export_to_autoround/export.pyindex 2d39eb48b..e50fcd5cb 100644--- a/auto_round/export/export_to_autoround/export.py+++ b/auto_round/export/export_to_autoround/export.py@@ -69,7 +69,7 @@ def dynamic_import_quant_linear_for_packing(backend, bits, group_size, sym, act_class: The dynamically imported QuantLinear class configured according to the specified parameters.Raises:- AssertionError: If the backend is not supported.+ ValueError: If the backend is not supported."""if "auto_round" in backend and "awq" not in backend and "gptq" not in backend:if act_bits <= 8: ##easily have bug for other configuration, need to refine code later@@ -88,7 +88,10 @@ def dynamic_import_quant_linear_for_packing(backend, bits, group_size, sym, act_elif "gptq" in backend and not "gptqmodel" in backend: ## have g_idxreturn get_autogptq_packing_qlinear(backend, bits, group_size, sym)else:- assert False, f"only support auto_gptq, auto_awq and auto_round backend"+ raise ValueError(+ f"Unsupported backend: '{backend}'. "+ "Only 'auto_round', 'auto_round:auto_awq', 'auto_round:auto_gptq', 'awq', and 'gptq' are supported."+ )def pack_qact_layer(name, model):@@ -247,7 +250,7 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:exNoneRaises:- AssertionError: If the backend is not supported.+ ValueError: If the backend is not supported."""##if using sym, we change to gptq sym kernel to avoid compiling from auto_round source@@ -367,3 +370,4 @@ def save(model: nn.Module, save_dir: str, max_shard_size: str = "5GB", safe_seriif hasattr(model, "config") and hasattr(model.config, "quantization_config"):with open(os.path.join(save_dir, config_file), "w", encoding="utf-8") as f:json.dump(model.config.quantization_config, f, indent=2)+diff --git a/auto_round/export/export_to_awq/utils.py b/auto_round/export/export_to_awq/utils.pyindex 40107b802..48b00ccc1 100644--- a/auto_round/export/export_to_awq/utils.py+++ b/auto_round/export/export_to_awq/utils.py@@ -148,8 +148,11 @@ class WQLinear_GEMM(nn.Module):self.training = training# quick sanity check (make sure alignment)- assert self.in_features % self.group_size == 0- assert out_features % (32 // self.w_bit) == 0+ if self.in_features % self.group_size != 0:+ raise ValueError(f"in_features ({self.in_features}) shape mismatch")++ if out_features % (32 // self.w_bit) != 0:+ raise ValueError(f"out_features ({out_features}) shape mismatch")self.register_buffer("qweight",@@ -203,7 +206,8 @@ class WQLinear_GEMM(nn.Module):return awq_linear# need scales and zeros info for real quantization- assert scales is not None and zeros is not None+ if scales is None or zeros is None:+ raise ValueError("Both 'scales' and 'zeros' must be provided (not None)")awq_linear.scales = scales.clone().half()if linear.bias is not None:@@ -312,3 +316,4 @@ def clear_memory(weight=None):del weightgc.collect()torch.cuda.empty_cache()+diff --git a/auto_round/export/export_to_gguf/export.py b/auto_round/export/export_to_gguf/export.pyindex 5d6d3459c..8c63e2c72 100644--- a/auto_round/export/export_to_gguf/export.py+++ b/auto_round/export/export_to_gguf/export.py@@ -74,7 +74,8 @@ def save_quantized_as_gguf(output_dir, backend="gguf:q4_0", **kwargs):model_name = model_name[-1]output_type = backend.split(":")[-1]- assert output_type.lower() in FTYPE_MAP, f"{output_type} is not supported"+ if output_type.lower() not in FTYPE_MAP:+ raise TypeError(f"{output_type} type is not supported")output_type = FTYPE_MAP.get(output_type.lower())@@ -96,3 +97,4 @@ def save_quantized_as_gguf(output_dir, backend="gguf:q4_0", **kwargs):shutil.rmtree(tmp_work_dir, ignore_errors=True)return model+diff --git a/auto_round/export/export_to_itrex/config.py b/auto_round/export/export_to_itrex/config.pyindex 355b95a56..c212dbe82 100644--- a/auto_round/export/export_to_itrex/config.py+++ b/auto_round/export/export_to_itrex/config.py@@ -249,7 +249,7 @@ class QuantConfig(PretrainedConfig):self._set_token_in_kwargs(kwargs)if os.path.isfile(save_directory):- raise AssertionError(f"Provided path ({save_directory}) should be a directory, not a file")+ raise OSError(f"Provided path ({save_directory}) should be a directory, not a file")os.makedirs(save_directory, exist_ok=True)@@ -301,3 +301,4 @@ class QuantConfig(PretrainedConfig):for parameter in remove_parameters:if hasattr(self, parameter):delattr(self, parameter)+diff --git a/auto_round/inference/backend.py b/auto_round/inference/backend.pyindex c2d589280..127d76e1b 100644--- a/auto_round/inference/backend.py+++ b/auto_round/inference/backend.py@@ -597,15 +597,16 @@ def get_layer_backend(device, backend, orig_backend, bits, group_size, sym, in_fThe selected backend that is compatible with the layer configuration.Raises:- AssertionError:- If the specified backend is not supported.ValueError:+ If the specified backend is not supported.If no compatible backend is found for the given layer configuration."""# Check if the provided backend is in BackendInfosbackend = find_backend(backend)- assert backend in BackendInfos.keys(), \- f"Unsupported backend {backend}, please set it to `auto` to try automatic selection"+ if backend not in BackendInfos.keys():+ raise ValueError(+ f"Unsupported backend '{backend}'. Please set it to 'auto' to enable automatic selection."+ )packing_format = BackendInfos[orig_backend].packing_format@@ -735,3 +736,4 @@ def process_requirement(requirements: list, target_device="cuda", logger_level="log(joined_cmds)if logger_level == "error":exit(-1)+diff --git a/auto_round/mllm/eval.py b/auto_round/mllm/eval.pyindex 44de86c56..997dea72d 100644--- a/auto_round/mllm/eval.py+++ b/auto_round/mllm/eval.py@@ -286,7 +286,8 @@ def mllm_eval(if mode == 'all':eval_results = dataset.evaluate(result_file, **judge_kwargs)if eval_results is not None:- assert isinstance(eval_results, dict) or isinstance(eval_results, pd.DataFrame)+ if not isinstance(eval_results, dict) and not isinstance(eval_results, pd.DataFrame):+ raise TypeError("Unsupported eval result type")logger.info(f'The evaluation of model {model_name} x dataset {dataset_name} has finished! ')logger.info('Evaluation Results:')if isinstance(eval_results, dict):@@ -372,7 +373,8 @@ def lmms_eval(config = AutoConfig.from_pretrained(model, trust_remote_code=True)model_type = config.model_type- assert model_type in MODEL_TYPE_TO_LMMS_MODEL, f"{model_type} is not support by lmms."+ if model_type not in MODEL_TYPE_TO_LMMS_MODEL:+ raise ValueError(f"{model_type} is not support by lmms.")if MODEL_TYPE_TO_LMMS_MODEL[model_type] == "phi3v":model_args = f"model_id_name={model}"@@ -410,3 +412,4 @@ def lmms_eval(return results+diff --git a/auto_round/mllm/template.py b/auto_round/mllm/template.pyindex 88ba62593..0782fd09c 100644--- a/auto_round/mllm/template.py+++ b/auto_round/mllm/template.py@@ -127,15 +127,17 @@ def load_template(path: str):if "model_type" not in data:data["model_type"] = "user_define"if "replace_tokens" in data and data["replace_tokens"] is not None:- assert len(data["replace_tokens"]) % 2 == 0, \- "the format of replace_tokens should be [old_tag1, replace_tag1, old_tag2, replace_tag2]"+ if len(data["replace_tokens"]) % 2 != 0:+ raise ValueError("the format of replace_tokens should be " \+ "[old_tag1, replace_tag1, old_tag2, replace_tag2]")temp = []for i in range(0, len(data["replace_tokens"]), 2):temp.append((data["replace_tokens"][i], data["replace_tokens"][i + 1]))data["replace_tokens"] = tempif "processor" in data:- assert data["processor"] in PROCESSORS.keys(), \- "{} is not supported, current support: {}".format(data["processor"], ",".join(PROCESSORS.keys()))+ if data["processor"] not in PROCESSORS.keys():+ raise ValueError(f"{data['processor']} is not supported, current support: " \+ "{','.join(PROCESSORS.keys())}")data["processor"] = PROCESSORS[data["processor"]]template = _register_template(**data@@ -177,3 +179,4 @@ def get_template(model=model, tokenizer=tokenizer, processor=processor, image_processor=image_processor, use_rtn=use_rtn)return template+diff --git a/auto_round/script/llm.py b/auto_round/script/llm.pyindex f6046c64f..af68b1175 100644--- a/auto_round/script/llm.py+++ b/auto_round/script/llm.py@@ -355,7 +355,7 @@ def tune(args):"It is recommended to use sym quantization or --format='auto_round'")if "marlin" in args.format and args.asym is True:- assert False, "marlin backend only supports sym quantization, please remove --asym"+ raise RuntimeError("marlin backend only supports sym quantization, please remove --asym")##must set this before import torchset_cuda_visible_devices(args.device)@@ -768,3 +768,4 @@ def eval_task_by_task(print(make_table(res_all))print("total eval time:", time.time() - st)+diff --git a/auto_round/script/mllm.py b/auto_round/script/mllm.pyindex 212fb902a..7f76d3204 100644--- a/auto_round/script/mllm.py+++ b/auto_round/script/mllm.py@@ -458,7 +458,8 @@ def tune(args):format_list = args.format.replace(' ', '').split(',')inplace = False if len(format_list) > 1 else Truefor format_ in format_list:- eval_folder = f'{export_dir}-{format_}'+ save_format_ = format_.replace(":", "-").replace("_", "-")+ eval_folder = os.path.join(export_dir, save_format_) if len(formats) > 1 else export_dirsafe_serialization = Trueif hasattr(autoround.model, "config") and hasattr(autoround.model.config, "model_type"):if "phi3_v" in autoround.model.config.model_type:@@ -562,3 +563,4 @@ def lmms_eval(args):)return results+diff --git a/auto_round/utils.py b/auto_round/utils.pyindex 8ddeb3ce0..61c15a52c 100644--- a/auto_round/utils.py+++ b/auto_round/utils.py@@ -642,7 +642,7 @@ def convert_dtype_str2torch(str_dtype):torch.dtype: The PyTorch dtype.Raises:- AssertionError: If the input str_dtype is unsupported.+ ValueError: If the input str_dtype is unsupported."""if isinstance(str_dtype, torch.dtype) or str_dtype is None:return str_dtype@@ -655,7 +655,7 @@ def convert_dtype_str2torch(str_dtype):elif str_dtype == "bf16" or str_dtype == "bfloat16":return torch.bfloat16else:- assert False, "Unsupported str dtype {} to torch dtype".format(str_dtype)+ raise ValueError(f"Unsupported string dtype '{str_dtype}' for conversion to torch dtype.")def convert_dtype_torch2str(dtype):@@ -668,7 +668,7 @@ def convert_dtype_torch2str(dtype):str: The string representation of the dtype.Raises:- AssertionError: If the input dtype is unsupported.+ ValueError: If the input dtype is unsupported."""if isinstance(dtype, str) or dtype is None:return dtype@@ -683,7 +683,7 @@ def convert_dtype_torch2str(dtype):elif isinstance(dtype, str) and dtype in ["int8", "fp32", "fp16", "bf16"]:return dtypeelse:- assert False, "Unsupported pytorch dtype {} to str dtype".format(dtype)+ raise ValueError(f"Unsupported PyTorch dtype '{dtype}' for conversion to string dtype.")def convert_dtype_torch2str_hf(dtype):@@ -696,7 +696,7 @@ def convert_dtype_torch2str_hf(dtype):str: The string representation of the dtype.Raises:- AssertionError: If the input str_dtype is unsupported.+ ValueError: If the input str_dtype is unsupported."""if dtype is None:return dtype@@ -707,7 +707,7 @@ def convert_dtype_torch2str_hf(dtype):return dtypestr_dtype = str(dtype)if "." not in str_dtype:- assert False, "Unsupported pytorch dtype {} to huggingface str dtype".format(dtype)+ raise ValueError(f"Unsupported pytorch dtype '{dtype}' for conversion to huggingface str dtype")str_dtype = str_dtype.split(".")[1]return str_dtype@@ -1512,3 +1512,4 @@ def filter_quantization_config(quantization_config):quantization_config.pop("act_dynamic", None)quantization_config.pop("act_sym", None)quantization_config.pop("act_group_size", None)+