PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
update logic for compatibility with gptqmodel:exllamav2 backend (#1438)
Xin He86231c44ff49
diff --git a/auto_round/inference/convert_model.py b/auto_round/inference/convert_model.pyindex e660ef762..8c8b8782f 100644--- a/auto_round/inference/convert_model.py+++ b/auto_round/inference/convert_model.py@@ -469,7 +469,7 @@ def infer_target_device(device_map: Union[dict, int, str, None] = None) -> str:return get_available_devices()[0]-def process_gptq_qzero(model: nn.Module):+def convert_gptq_v1_to_v2_format(model: nn.Module):"""Convert gptq v1 to v2 format to ensure compatible with gptqmodel:exllamav2 backend."""from gptqmodel.nn_modules.qlinear.exllamav2 import ExllamaV2QuantLinear # pylint: disable=E0401@@ -519,7 +519,15 @@ def post_init(model: torch.nn.Module, used_backends: list[str]) -> None:from gptqmodel.utils.model import hf_gptqmodel_post_init as gptq_post_init # pylint: disable=E0401from packaging import version- process_gptq_qzero(model) # Handle qzero layers if present+ packing_format = None+ if hasattr(model, "config") and hasattr(model.config, "quantization_config"):+ quant_cfg = model.config.quantization_config+ if hasattr(quant_cfg, "packing_format"):+ packing_format = quant_cfg.packing_format++ if packing_format == "auto_round:gptq":+ # v1: auto_round:gptq; v2: auto_round:gptqmodel+ convert_gptq_v1_to_v2_format(model) # Handle qzero layers if presentif version.parse(gptqmodel_version) <= version.parse("5.6.0"):model = gptq_post_init(model, use_act_order=False)else: