Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

update logic for compatibility with gptqmodel:exllamav2 backend (#1438)

Xin He86231c44ff49
diff --git a/auto_round/inference/convert_model.py b/auto_round/inference/convert_model.py
index e660ef762..8c8b8782f 100644
--- a/auto_round/inference/convert_model.py
+++ b/auto_round/inference/convert_model.py
@@ -469,7 +469,7 @@ def infer_target_device(device_map: Union[dict, int, str, None] = None) -> str:
return get_available_devices()[0]
-def process_gptq_qzero(model: nn.Module):
+def convert_gptq_v1_to_v2_format(model: nn.Module):
"""Convert gptq v1 to v2 format to ensure compatible with gptqmodel:exllamav2 backend."""
from gptqmodel.nn_modules.qlinear.exllamav2 import ExllamaV2QuantLinear # pylint: disable=E0401
@@ -519,7 +519,15 @@ def post_init(model: torch.nn.Module, used_backends: list[str]) -> None:
from gptqmodel.utils.model import hf_gptqmodel_post_init as gptq_post_init # pylint: disable=E0401
from packaging import version
- process_gptq_qzero(model) # Handle qzero layers if present
+ packing_format = None
+ if hasattr(model, "config") and hasattr(model.config, "quantization_config"):
+ quant_cfg = model.config.quantization_config
+ if hasattr(quant_cfg, "packing_format"):
+ packing_format = quant_cfg.packing_format
+
+ if packing_format == "auto_round:gptq":
+ # v1: auto_round:gptq; v2: auto_round:gptqmodel
+ convert_gptq_v1_to_v2_format(model) # Handle qzero layers if present
if version.parse(gptqmodel_version) <= version.parse("5.6.0"):
model = gptq_post_init(model, use_act_order=False)
else: