Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

Update BackendInfos for AutoGPTQ based on transformer version and save g_idx for gptqmodel backend (#1429)

Xin Hea1876107ff22
diff --git a/auto_round/export/export_to_autoround/export.py b/auto_round/export/export_to_autoround/export.py
index 8d78d0bee..ac4701dd3 100644
--- a/auto_round/export/export_to_autoround/export.py
+++ b/auto_round/export/export_to_autoround/export.py
@@ -14,6 +14,7 @@
import copy
+import functools
import inspect
import json
import os
@@ -79,7 +80,7 @@ def dynamic_import_quant_linear_for_packing(backend, bits, group_size, sym, act_
elif "gptqmodel" in backend:
from auto_round_extension.torch.qlinear_torch import QuantLinear
- return QuantLinear
+ return functools.partial(QuantLinear, g_idx=True)
elif "auto_round" in backend and "gptq" in backend and "gptqmodel" not in backend:
from auto_round_extension.torch.qlinear_torch_zp import QuantLinear
diff --git a/auto_round/inference/backend.py b/auto_round/inference/backend.py
index e024b65f0..f1955b769 100644
--- a/auto_round/inference/backend.py
+++ b/auto_round/inference/backend.py
@@ -17,6 +17,7 @@ import platform
from dataclasses import dataclass, field
from typing import TYPE_CHECKING, Any, Optional
+from packaging import version
from transformers.utils.versions import require_version
import auto_round_extension.cuda.gptqmodel_marlin
@@ -26,6 +27,8 @@ from auto_round.logger import logger
from auto_round.schemes import QuantizationScheme
from auto_round.utils import get_library_version
+is_transformers_below_v5 = version.parse(get_library_version("transformers")) < version.parse("5.0.0")
+
BackendInfos = {}
import cpuinfo
@@ -198,53 +201,55 @@ AWQ_FORMAT = ["auto_round:auto_awq"]
LLM_COMPRESSOR_FORMAT = ["auto_round:llm_compressor"]
WOQ_DEFAULT_ACT_BITS = [None, 16, 32]
-BackendInfos["auto_gptq:exllamav2"] = BackendInfo(
- device=["cuda"],
- sym=[True, False],
- packing_format=GPTQ_FORMAT,
- bits=[4],
- priority=5,
- compute_dtype=["float16"],
- data_type=["int"],
- act_bits=WOQ_DEFAULT_ACT_BITS,
- # 16, 384,768 accuracy issue
- group_size=[-1, 32, 64, 128, 256, 512, 1024, 2048],
- checkers=[compatible_exllamav2_feature_checker],
- alias=["gptq", "auto_gptq", "exllamav2", "gptq:exllamav2", "auto_gptq:exllamav2"],
- requirements=["torch<2.6.0", "auto-gptq>=0.7.1"],
-)
+if is_transformers_below_v5:
+ # AutoGPTQ is no longer maintained, supports transformers < 5.0.0
+ BackendInfos["auto_gptq:exllamav2"] = BackendInfo(
+ device=["cuda"],
+ sym=[True, False],
+ packing_format=GPTQ_FORMAT,
+ bits=[4],
+ priority=5,
+ compute_dtype=["float16"],
+ data_type=["int"],
+ act_bits=WOQ_DEFAULT_ACT_BITS,
+ # 16, 384,768 accuracy issue
+ group_size=[-1, 32, 64, 128, 256, 512, 1024, 2048],
+ checkers=[compatible_exllamav2_feature_checker],
+ alias=["gptq", "auto_gptq", "exllamav2", "gptq:exllamav2", "auto_gptq:exllamav2"],
+ requirements=["torch<2.6.0", "auto-gptq>=0.7.1"],
+ )
-BackendInfos["auto_gptq:tritonv2"] = BackendInfo(
- device=["cuda"],
- sym=[True, False],
- packing_format=GPTQ_FORMAT,
- bits=[2, 4, 8],
- group_size=None,
- compute_dtype=["float16"],
- data_type=["int"],
- act_bits=WOQ_DEFAULT_ACT_BITS,
- priority=0,
- checkers=[exllamav2_feature_checker],
- alias=["auto_gptq:tritonv2"],
- requirements=["auto-gptq>=0.7.1", "triton>=2.0"],
-)
+ BackendInfos["auto_gptq:tritonv2"] = BackendInfo(
+ device=["cuda"],
+ sym=[True, False],
+ packing_format=GPTQ_FORMAT,
+ bits=[2, 4, 8],
+ group_size=None,
+ compute_dtype=["float16"],
+ data_type=["int"],
+ act_bits=WOQ_DEFAULT_ACT_BITS,
+ priority=0,
+ checkers=[exllamav2_feature_checker],
+ alias=["auto_gptq:tritonv2"],
+ requirements=["auto-gptq>=0.7.1", "triton>=2.0"],
+ )
-BackendInfos["auto_gptq:cuda"] = BackendInfo(
- device=["cuda"],
- sym=[True, False],
- packing_format=GPTQ_FORMAT,
- bits=[2, 3, 4, 8],
- group_size=None,
- priority=1,
- checkers=[exllamav2_feature_checker],
- compute_dtype=["float16"],
- data_type=["int"],
- act_bits=WOQ_DEFAULT_ACT_BITS,
- alias=["auto_gptq:cuda"],
- requirements=[
- "auto-gptq>=0.7.1",
- ],
-)
+ BackendInfos["auto_gptq:cuda"] = BackendInfo(
+ device=["cuda"],
+ sym=[True, False],
+ packing_format=GPTQ_FORMAT,
+ bits=[2, 3, 4, 8],
+ group_size=None,
+ priority=1,
+ checkers=[exllamav2_feature_checker],
+ compute_dtype=["float16"],
+ data_type=["int"],
+ act_bits=WOQ_DEFAULT_ACT_BITS,
+ alias=["auto_gptq:cuda"],
+ requirements=[
+ "auto-gptq>=0.7.1",
+ ],
+ )
# FP8 static quant
# Weight: FP8, per-channel, may be extended to per-tensor in future
diff --git a/auto_round_extension/torch/qlinear_torch.py b/auto_round_extension/torch/qlinear_torch.py
index d5389cfa6..953162c95 100644
--- a/auto_round_extension/torch/qlinear_torch.py
+++ b/auto_round_extension/torch/qlinear_torch.py
@@ -51,7 +51,7 @@ class QuantLinear(nn.Module):
QUANT_TYPE = "torch"
- def __init__(self, bits, group_size, infeatures, outfeatures, bias, trainable=False, **kwargs):
+ def __init__(self, bits, group_size, infeatures, outfeatures, bias, trainable=False, g_idx=False, **kwargs):
super().__init__()
if bits not in [2, 3, 4, 8]:
raise NotImplementedError("Only 2,3,4,8 bits are supported.")
@@ -87,6 +87,11 @@ class QuantLinear(nn.Module):
else:
self.bias = None
+ if g_idx:
+ self.register_buffer(
+ "g_idx", torch.tensor([i // self.group_size for i in range(infeatures)], dtype=torch.int32)
+ )
+
self.trainable = trainable
# is performed by unpacking the weights and using torch.matmul
diff --git a/docs/step_by_step.md b/docs/step_by_step.md
index b5027dc11..1f2d6d7ad 100644
--- a/docs/step_by_step.md
+++ b/docs/step_by_step.md
@@ -718,8 +718,8 @@ print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50, do_sample=Fal
| ipex | cpu/xpu | 4 | BF16/FP16 | 5 | gptq_zp+-1/awq | intel-extension-for-pytorch |
| marlin | cuda | 4,8 | BF16/FP16 | 6 | gptq/gptq_zp+-1 | gptqmodel |
| exllamav2 or<br/>gptqmodel:exllamav2 | cuda | 4 | BF16/FP16 | 5 | gptq | gptqmodel |
-| exllamav2 or<br/>gptq:exllamav2 | cuda | 4 | FP16 | 5 | gptq_zp+-1 | auto-gptq |
-| gptq:cuda | cuda | 2,3,4,8 | FP16 | 1 | gptq_zp+-1 | auto-gptq <br/> |
+| exllamav2 or<br/>gptq:exllamav2 | cuda | 4 | FP16 | 5 | gptq_zp+-1 | auto-gptq<br/>transformers<5.0.0 |
+| gptq:cuda | cuda | 2,3,4,8 | FP16 | 1 | gptq_zp+-1 | auto-gptq<br/>transformers<5.0.0 |
| triton | xpu/cuda | 2,4,8 | BF16/FP16 | 2 | gptq/gptq_zp+-1 | <br/>auto-round |
| awq | cuda | 4 | FP16 | 5 | awq | auto-awq |
| hpu | hpu | 4 | BF16 | 0 | gptq/gptq_zp+-1 | auto-round |
diff --git a/docs/step_by_step_CN.md b/docs/step_by_step_CN.md
index 23aa960e2..845fe594b 100644
--- a/docs/step_by_step_CN.md
+++ b/docs/step_by_step_CN.md
@@ -678,8 +678,8 @@ print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50, do_sample=Fal
| ipex | cpu/xpu | 4 | BF16/FP16 | 5 | gptq_zp+-1/awq | intel-extension-for-pytorch |
| marlin | cuda | 4、8 | BF16/FP16 | 6 | gptq/gptq_zp+-1 | gptqmodel |
| exllamav2/<br/>gptqmodel:exllamav2 | cuda | 4 | BF16/FP16 | 5 | gptq | gptqmodel |
-| exllamav2/<br/>gptq:exllamav2 | cuda | 4 | FP16 | 5 | gptq_zp+-1 | auto-gptq |
-| gptq:cuda | cuda | 2、3、4、8 | FP16 | 1 | gptq_zp+-1 | auto-gptq |
+| exllamav2/<br/>gptq:exllamav2 | cuda | 4 | FP16 | 5 | gptq_zp+-1 | auto-gptq<br/>transformers<5.0.0 |
+| gptq:cuda | cuda | 2、3、4、8 | FP16 | 1 | gptq_zp+-1 | auto-gptq<br/>transformers<5.0.0 |
| triton | xpu/cuda | 2、4、8 | BF16/FP16 | 2 | gptq/gptq_zp+-1 | auto-round |
| awq | cuda | 4 | FP16 | 5 | awq | auto-awq |
| hpu | hpu | 4 | BF16 | 0 | gptq/gptq_zp+-1 | auto-round |