PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
fix cuda ut fail: gptqmodel and gguf format with fp8 input model (#1399)
Heng Guo4dc04f4b9b84
diff --git a/auto_round/export/export_to_gguf/convert.py b/auto_round/export/export_to_gguf/convert.pyindex 6a22a13e2..1f93bbba6 100644--- a/auto_round/export/export_to_gguf/convert.py+++ b/auto_round/export/export_to_gguf/convert.py@@ -159,7 +159,7 @@ def get_tensors(cls) -> Iterator[tuple[str, Tensor]]:yield name, tensordef is_extra_tensor(tensor_name):- if is_quantized_input_module(cls.model) and "scale" in tensor_name.split(".")[-1]:+ if getattr(cls.model, "_is_quantized_input_module", False) and "scale" in tensor_name.split(".")[-1]:return Falseif tensor_name not in cls.model.tensor_name_list:return Truediff --git a/auto_round/inference/convert_model.py b/auto_round/inference/convert_model.pyindex 096836df1..3f4acd1bf 100644--- a/auto_round/inference/convert_model.py+++ b/auto_round/inference/convert_model.py@@ -504,30 +504,18 @@ def post_init(model: torch.nn.Module, used_backends: list[str]) -> None:# GPTQModel post-initif need_gptqmodel_init:- from gptqmodel.quantization import METHOD # pylint: disable=E0401- from gptqmodel.utils.importer import hf_select_quant_linear_v2 # pylint: disable=E0401- from gptqmodel.utils.model import hf_convert_gptq_v1_to_v2_format # pylint: disable=E0401+ from gptqmodel import __version__ as gptqmodel_version # pylint: disable=E0401from gptqmodel.utils.model import hf_gptqmodel_post_init as gptq_post_init # pylint: disable=E0401+ from packaging import version- quant_linear = hf_select_quant_linear_v2(- bits=model.config.quantization_config.bits,- group_size=model.config.quantization_config.group_size,- desc_act=False,- sym=model.config.quantization_config.sym,- format="gptq_v2",- quant_method=METHOD.GPTQ,- backend="EXLLAMA_V2",- pack=True,- device_map="auto",- )- model, _ = hf_convert_gptq_v1_to_v2_format(- model,- bits=model.config.quantization_config.bits,- qlinear_kernel=quant_linear,- checkpoint_format="gptq",- meta=None,- )- model = gptq_post_init(model, use_act_order=False)+ if version.parse(gptqmodel_version) <= version.parse("5.6.0"):+ model = gptq_post_init(model, use_act_order=False)+ else:+ # for new version of gptqmodel, use validate_once to import kernels+ for n, m in model.named_modules():+ if hasattr(m, "validate_once"):+ m.validate_once()+ model = gptq_post_init(model, use_act_order=False)# IPEX post-initif need_ipex_init:diff --git a/auto_round/utils/weight_handler.py b/auto_round/utils/weight_handler.pyindex ccdf44c3d..ec7c08e2c 100644--- a/auto_round/utils/weight_handler.py+++ b/auto_round/utils/weight_handler.py@@ -297,6 +297,7 @@ def check_and_mark_quantized_module(model: torch.nn.Module) -> Set[ModuleWeightTfor weight_type, handler in _WEIGHT_TYPE_HANDLERS.items():# Check model itself firstif handler.detect_layer(model):+ model._is_quantized_input_module = Truemodel.quantized_weight_type = weight_typedetected_types.add(weight_type)@@ -306,6 +307,10 @@ def check_and_mark_quantized_module(model: torch.nn.Module) -> Set[ModuleWeightTif handler.detect_layer(m):# Mark the layer itselfm.quantized_weight_type = weight_type+ # for gguf format, gguf format need to mark the quantized input module+ # in order to skip weight_scale_inv, etc. (extra tensor from file)+ if not getattr(model, "_is_quantized_input_module", False):+ model._is_quantized_input_module = True# Record detected typesdetected_types.add(weight_type)