Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

fix cuda ut fail: gptqmodel and gguf format with fp8 input model (#1399)

Heng Guo4dc04f4b9b84
diff --git a/auto_round/export/export_to_gguf/convert.py b/auto_round/export/export_to_gguf/convert.py
index 6a22a13e2..1f93bbba6 100644
--- a/auto_round/export/export_to_gguf/convert.py
+++ b/auto_round/export/export_to_gguf/convert.py
@@ -159,7 +159,7 @@ def get_tensors(cls) -> Iterator[tuple[str, Tensor]]:
yield name, tensor
def is_extra_tensor(tensor_name):
- if is_quantized_input_module(cls.model) and "scale" in tensor_name.split(".")[-1]:
+ if getattr(cls.model, "_is_quantized_input_module", False) and "scale" in tensor_name.split(".")[-1]:
return False
if tensor_name not in cls.model.tensor_name_list:
return True
diff --git a/auto_round/inference/convert_model.py b/auto_round/inference/convert_model.py
index 096836df1..3f4acd1bf 100644
--- a/auto_round/inference/convert_model.py
+++ b/auto_round/inference/convert_model.py
@@ -504,30 +504,18 @@ def post_init(model: torch.nn.Module, used_backends: list[str]) -> None:
# GPTQModel post-init
if need_gptqmodel_init:
- from gptqmodel.quantization import METHOD # pylint: disable=E0401
- from gptqmodel.utils.importer import hf_select_quant_linear_v2 # pylint: disable=E0401
- from gptqmodel.utils.model import hf_convert_gptq_v1_to_v2_format # pylint: disable=E0401
+ from gptqmodel import __version__ as gptqmodel_version # pylint: disable=E0401
from gptqmodel.utils.model import hf_gptqmodel_post_init as gptq_post_init # pylint: disable=E0401
+ from packaging import version
- quant_linear = hf_select_quant_linear_v2(
- bits=model.config.quantization_config.bits,
- group_size=model.config.quantization_config.group_size,
- desc_act=False,
- sym=model.config.quantization_config.sym,
- format="gptq_v2",
- quant_method=METHOD.GPTQ,
- backend="EXLLAMA_V2",
- pack=True,
- device_map="auto",
- )
- model, _ = hf_convert_gptq_v1_to_v2_format(
- model,
- bits=model.config.quantization_config.bits,
- qlinear_kernel=quant_linear,
- checkpoint_format="gptq",
- meta=None,
- )
- model = gptq_post_init(model, use_act_order=False)
+ if version.parse(gptqmodel_version) <= version.parse("5.6.0"):
+ model = gptq_post_init(model, use_act_order=False)
+ else:
+ # for new version of gptqmodel, use validate_once to import kernels
+ for n, m in model.named_modules():
+ if hasattr(m, "validate_once"):
+ m.validate_once()
+ model = gptq_post_init(model, use_act_order=False)
# IPEX post-init
if need_ipex_init:
diff --git a/auto_round/utils/weight_handler.py b/auto_round/utils/weight_handler.py
index ccdf44c3d..ec7c08e2c 100644
--- a/auto_round/utils/weight_handler.py
+++ b/auto_round/utils/weight_handler.py
@@ -297,6 +297,7 @@ def check_and_mark_quantized_module(model: torch.nn.Module) -> Set[ModuleWeightT
for weight_type, handler in _WEIGHT_TYPE_HANDLERS.items():
# Check model itself first
if handler.detect_layer(model):
+ model._is_quantized_input_module = True
model.quantized_weight_type = weight_type
detected_types.add(weight_type)
@@ -306,6 +307,10 @@ def check_and_mark_quantized_module(model: torch.nn.Module) -> Set[ModuleWeightT
if handler.detect_layer(m):
# Mark the layer itself
m.quantized_weight_type = weight_type
+ # for gguf format, gguf format need to mark the quantized input module
+ # in order to skip weight_scale_inv, etc. (extra tensor from file)
+ if not getattr(model, "_is_quantized_input_module", False):
+ model._is_quantized_input_module = True
# Record detected types
detected_types.add(weight_type)