PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
Update BackendInfos for AutoGPTQ based on transformer version and save g_idx for gptqmodel backend (#1429)
Xin Hea1876107ff22
diff --git a/auto_round/export/export_to_autoround/export.py b/auto_round/export/export_to_autoround/export.pyindex 8d78d0bee..ac4701dd3 100644--- a/auto_round/export/export_to_autoround/export.py+++ b/auto_round/export/export_to_autoround/export.py@@ -14,6 +14,7 @@import copy+import functoolsimport inspectimport jsonimport os@@ -79,7 +80,7 @@ def dynamic_import_quant_linear_for_packing(backend, bits, group_size, sym, act_elif "gptqmodel" in backend:from auto_round_extension.torch.qlinear_torch import QuantLinear- return QuantLinear+ return functools.partial(QuantLinear, g_idx=True)elif "auto_round" in backend and "gptq" in backend and "gptqmodel" not in backend:from auto_round_extension.torch.qlinear_torch_zp import QuantLineardiff --git a/auto_round/inference/backend.py b/auto_round/inference/backend.pyindex e024b65f0..f1955b769 100644--- a/auto_round/inference/backend.py+++ b/auto_round/inference/backend.py@@ -17,6 +17,7 @@ import platformfrom dataclasses import dataclass, fieldfrom typing import TYPE_CHECKING, Any, Optional+from packaging import versionfrom transformers.utils.versions import require_versionimport auto_round_extension.cuda.gptqmodel_marlin@@ -26,6 +27,8 @@ from auto_round.logger import loggerfrom auto_round.schemes import QuantizationSchemefrom auto_round.utils import get_library_version+is_transformers_below_v5 = version.parse(get_library_version("transformers")) < version.parse("5.0.0")+BackendInfos = {}import cpuinfo@@ -198,53 +201,55 @@ AWQ_FORMAT = ["auto_round:auto_awq"]LLM_COMPRESSOR_FORMAT = ["auto_round:llm_compressor"]WOQ_DEFAULT_ACT_BITS = [None, 16, 32]-BackendInfos["auto_gptq:exllamav2"] = BackendInfo(- device=["cuda"],- sym=[True, False],- packing_format=GPTQ_FORMAT,- bits=[4],- priority=5,- compute_dtype=["float16"],- data_type=["int"],- act_bits=WOQ_DEFAULT_ACT_BITS,- # 16, 384,768 accuracy issue- group_size=[-1, 32, 64, 128, 256, 512, 1024, 2048],- checkers=[compatible_exllamav2_feature_checker],- alias=["gptq", "auto_gptq", "exllamav2", "gptq:exllamav2", "auto_gptq:exllamav2"],- requirements=["torch<2.6.0", "auto-gptq>=0.7.1"],-)+if is_transformers_below_v5:+ # AutoGPTQ is no longer maintained, supports transformers < 5.0.0+ BackendInfos["auto_gptq:exllamav2"] = BackendInfo(+ device=["cuda"],+ sym=[True, False],+ packing_format=GPTQ_FORMAT,+ bits=[4],+ priority=5,+ compute_dtype=["float16"],+ data_type=["int"],+ act_bits=WOQ_DEFAULT_ACT_BITS,+ # 16, 384,768 accuracy issue+ group_size=[-1, 32, 64, 128, 256, 512, 1024, 2048],+ checkers=[compatible_exllamav2_feature_checker],+ alias=["gptq", "auto_gptq", "exllamav2", "gptq:exllamav2", "auto_gptq:exllamav2"],+ requirements=["torch<2.6.0", "auto-gptq>=0.7.1"],+ )-BackendInfos["auto_gptq:tritonv2"] = BackendInfo(- device=["cuda"],- sym=[True, False],- packing_format=GPTQ_FORMAT,- bits=[2, 4, 8],- group_size=None,- compute_dtype=["float16"],- data_type=["int"],- act_bits=WOQ_DEFAULT_ACT_BITS,- priority=0,- checkers=[exllamav2_feature_checker],- alias=["auto_gptq:tritonv2"],- requirements=["auto-gptq>=0.7.1", "triton>=2.0"],-)+ BackendInfos["auto_gptq:tritonv2"] = BackendInfo(+ device=["cuda"],+ sym=[True, False],+ packing_format=GPTQ_FORMAT,+ bits=[2, 4, 8],+ group_size=None,+ compute_dtype=["float16"],+ data_type=["int"],+ act_bits=WOQ_DEFAULT_ACT_BITS,+ priority=0,+ checkers=[exllamav2_feature_checker],+ alias=["auto_gptq:tritonv2"],+ requirements=["auto-gptq>=0.7.1", "triton>=2.0"],+ )-BackendInfos["auto_gptq:cuda"] = BackendInfo(- device=["cuda"],- sym=[True, False],- packing_format=GPTQ_FORMAT,- bits=[2, 3, 4, 8],- group_size=None,- priority=1,- checkers=[exllamav2_feature_checker],- compute_dtype=["float16"],- data_type=["int"],- act_bits=WOQ_DEFAULT_ACT_BITS,- alias=["auto_gptq:cuda"],- requirements=[- "auto-gptq>=0.7.1",- ],-)+ BackendInfos["auto_gptq:cuda"] = BackendInfo(+ device=["cuda"],+ sym=[True, False],+ packing_format=GPTQ_FORMAT,+ bits=[2, 3, 4, 8],+ group_size=None,+ priority=1,+ checkers=[exllamav2_feature_checker],+ compute_dtype=["float16"],+ data_type=["int"],+ act_bits=WOQ_DEFAULT_ACT_BITS,+ alias=["auto_gptq:cuda"],+ requirements=[+ "auto-gptq>=0.7.1",+ ],+ )# FP8 static quant# Weight: FP8, per-channel, may be extended to per-tensor in futurediff --git a/auto_round_extension/torch/qlinear_torch.py b/auto_round_extension/torch/qlinear_torch.pyindex d5389cfa6..953162c95 100644--- a/auto_round_extension/torch/qlinear_torch.py+++ b/auto_round_extension/torch/qlinear_torch.py@@ -51,7 +51,7 @@ class QuantLinear(nn.Module):QUANT_TYPE = "torch"- def __init__(self, bits, group_size, infeatures, outfeatures, bias, trainable=False, **kwargs):+ def __init__(self, bits, group_size, infeatures, outfeatures, bias, trainable=False, g_idx=False, **kwargs):super().__init__()if bits not in [2, 3, 4, 8]:raise NotImplementedError("Only 2,3,4,8 bits are supported.")@@ -87,6 +87,11 @@ class QuantLinear(nn.Module):else:self.bias = None+ if g_idx:+ self.register_buffer(+ "g_idx", torch.tensor([i // self.group_size for i in range(infeatures)], dtype=torch.int32)+ )+self.trainable = trainable# is performed by unpacking the weights and using torch.matmuldiff --git a/docs/step_by_step.md b/docs/step_by_step.mdindex b5027dc11..1f2d6d7ad 100644--- a/docs/step_by_step.md+++ b/docs/step_by_step.md@@ -718,8 +718,8 @@ print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50, do_sample=Fal| ipex | cpu/xpu | 4 | BF16/FP16 | 5 | gptq_zp+-1/awq | intel-extension-for-pytorch || marlin | cuda | 4,8 | BF16/FP16 | 6 | gptq/gptq_zp+-1 | gptqmodel || exllamav2 or<br/>gptqmodel:exllamav2 | cuda | 4 | BF16/FP16 | 5 | gptq | gptqmodel |-| exllamav2 or<br/>gptq:exllamav2 | cuda | 4 | FP16 | 5 | gptq_zp+-1 | auto-gptq |-| gptq:cuda | cuda | 2,3,4,8 | FP16 | 1 | gptq_zp+-1 | auto-gptq <br/> |+| exllamav2 or<br/>gptq:exllamav2 | cuda | 4 | FP16 | 5 | gptq_zp+-1 | auto-gptq<br/>transformers<5.0.0 |+| gptq:cuda | cuda | 2,3,4,8 | FP16 | 1 | gptq_zp+-1 | auto-gptq<br/>transformers<5.0.0 || triton | xpu/cuda | 2,4,8 | BF16/FP16 | 2 | gptq/gptq_zp+-1 | <br/>auto-round || awq | cuda | 4 | FP16 | 5 | awq | auto-awq || hpu | hpu | 4 | BF16 | 0 | gptq/gptq_zp+-1 | auto-round |diff --git a/docs/step_by_step_CN.md b/docs/step_by_step_CN.mdindex 23aa960e2..845fe594b 100644--- a/docs/step_by_step_CN.md+++ b/docs/step_by_step_CN.md@@ -678,8 +678,8 @@ print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50, do_sample=Fal| ipex | cpu/xpu | 4 | BF16/FP16 | 5 | gptq_zp+-1/awq | intel-extension-for-pytorch || marlin | cuda | 4、8 | BF16/FP16 | 6 | gptq/gptq_zp+-1 | gptqmodel || exllamav2/<br/>gptqmodel:exllamav2 | cuda | 4 | BF16/FP16 | 5 | gptq | gptqmodel |-| exllamav2/<br/>gptq:exllamav2 | cuda | 4 | FP16 | 5 | gptq_zp+-1 | auto-gptq |-| gptq:cuda | cuda | 2、3、4、8 | FP16 | 1 | gptq_zp+-1 | auto-gptq |+| exllamav2/<br/>gptq:exllamav2 | cuda | 4 | FP16 | 5 | gptq_zp+-1 | auto-gptq<br/>transformers<5.0.0 |+| gptq:cuda | cuda | 2、3、4、8 | FP16 | 1 | gptq_zp+-1 | auto-gptq<br/>transformers<5.0.0 || triton | xpu/cuda | 2、4、8 | BF16/FP16 | 2 | gptq/gptq_zp+-1 | auto-round || awq | cuda | 4 | FP16 | 5 | awq | auto-awq || hpu | hpu | 4 | BF16 | 0 | gptq/gptq_zp+-1 | auto-round |