Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

add gptqmodel 6.0 compatible change and MTP quantization skip warning. (#1663)

Xin Heac7aa288a4e1
diff --git a/auto_round/inference/backend.py b/auto_round/inference/backend.py
index d98545679..c5b10c4dc 100644
--- a/auto_round/inference/backend.py
+++ b/auto_round/inference/backend.py
@@ -861,17 +861,29 @@ def dynamic_import_inference_linear(backend, config):
raise ValueError(f"unsupported backend {backend}, please set it to `auto` and retry")
-def get_gptqmodel_awq_infer_linear(backend):
- """Returns the appropriate gptqmodel AWQ QuantLinear class for inference."""
-
+def safe_import_gptqmodel():
+ """Safely import gptqmodel on CPU to avoid meta device issues."""
dtype = torch.get_default_dtype()
if dtype != torch.float32:
torch.set_default_dtype(torch.float32)
try:
- import gptqmodel # pylint: disable=E0401
+ # When loaded via the "meta" device, `gptqmodel==6.0.3` raises an error (since the internal
+ # loading process within the `transformers` library defaults to the "meta" device mode).
+ # Therefore, it is necessary to first switch to the CPU to bypass this error, and then
+ # switch back to the original data type once the loading process is complete.
+ with torch.device("cpu"):
+ import gptqmodel # pylint: disable=E0401
+
+ return gptqmodel
finally:
torch.set_default_dtype(dtype)
+
+def get_gptqmodel_awq_infer_linear(backend):
+ """Returns the appropriate gptqmodel AWQ QuantLinear class for inference."""
+
+ gptqmodel = safe_import_gptqmodel()
+
# Select AWQ kernel based on the BackendInfos key
if "marlin" in backend:
from gptqmodel.nn_modules.qlinear.marlin_awq import AwqMarlinQuantLinear # pylint: disable=E0401
@@ -894,14 +906,7 @@ def get_gptqmodel_awq_infer_linear(backend):
def get_gptqmodel_infer_linear(backend, bits=4, group_size=128, sym=False):
- dtype = torch.get_default_dtype()
- if dtype != torch.float32:
- torch.set_default_dtype(torch.float32)
-
- try:
- import gptqmodel # pylint: disable=E0401
- finally:
- torch.set_default_dtype(dtype)
+ gptqmodel = safe_import_gptqmodel()
if "marlin" in backend:
return auto_round_extension.cuda.gptqmodel_marlin.get_marlin_layer()
diff --git a/auto_round/utils/missing_tensors.py b/auto_round/utils/missing_tensors.py
index bc05dbe01..94f3d7be7 100644
--- a/auto_round/utils/missing_tensors.py
+++ b/auto_round/utils/missing_tensors.py
@@ -682,6 +682,15 @@ def _woq_quantize_missing_tensors(target_dir: str, missing_tensors_dict: dict) -
layer_name = weight_key[: -len(".weight")]
layer_cfg = _resolve_layer_cfg(layer_name)
bits = layer_cfg["bits"]
+ if bits not in [4, 8]:
+ if bits < 4:
+ bits = 4
+ elif 4 < bits < 8:
+ bits = 8
+ logger.warning(
+ f"Unsupported bits={layer_cfg['bits']} for layer '{layer_name}' in extra_config, "
+ f"falling back to bits={bits} for quantization."
+ )
group_size = layer_cfg["group_size"]
sym = layer_cfg["sym"]
diff --git a/auto_round_extension/cuda/gptqmodel_marlin.py b/auto_round_extension/cuda/gptqmodel_marlin.py
index 16949cb0a..144209b08 100644
--- a/auto_round_extension/cuda/gptqmodel_marlin.py
+++ b/auto_round_extension/cuda/gptqmodel_marlin.py
@@ -24,12 +24,22 @@ import torch
def get_marlin_layer(): ##use an ugly wrapper to import gptqmodel on demand
- import gptqmodel
+
+ # When loaded via the "meta" device, `gptqmodel==6.0.3` raises an error (since the internal
+ # loading process within the `transformers` library defaults to the "meta" device mode).
+ # Therefore, it is necessary to first switch to the CPU to bypass this error, and then
+ # switch back to the original data type once the loading process is complete.
+ with torch.device("cpu"):
+ import gptqmodel
+
from packaging.version import Version
NEW_VERSION = False
if Version(gptqmodel.__version__) >= Version("5.0.0"):
NEW_VERSION = True
+ NEW_VERSION_6_0 = False
+ if Version(gptqmodel.__version__) >= Version("6.0.0"):
+ NEW_VERSION_6_0 = True
from gptqmodel.models._const import DEVICE, PLATFORM # pylint: disable=E0401
from gptqmodel.nn_modules.qlinear import BaseQuantLinear # pylint: disable=E0401
from gptqmodel.utils.backend import BACKEND # pylint: disable=E0401
@@ -244,20 +254,59 @@ def get_marlin_layer(): ##use an ugly wrapper to import gptqmodel on demand
# (since we have only one group per output channel)
desc_act = False
- super().__init__(
- bits=bits,
- group_size=group_size,
- sym=sym,
- desc_act=desc_act,
- in_features=in_features,
- out_features=out_features,
- bias=bias,
- pack_dtype=pack_dtype,
- backend=kwargs.pop("backend", BACKEND.MARLIN),
- adapter=None,
- register_buffers=False,
- **kwargs,
- )
+ backend = kwargs.pop("backend", BACKEND.MARLIN)
+ if NEW_VERSION_6_0:
+ # gptqmodel >= 6.0.0: BaseQuantLinear no longer accepts group_size/sym/desc_act/pack_dtype
+ # directly; they must be passed via validate_kwargs. Attributes are also set manually.
+ super().__init__(
+ bits=bits,
+ in_features=in_features,
+ out_features=out_features,
+ bias=bias,
+ backend=backend,
+ adapter=None,
+ register_buffers=False,
+ validate_kwargs={
+ "group_size": group_size,
+ "desc_act": desc_act,
+ "sym": sym,
+ "pack_dtype": pack_dtype,
+ },
+ **kwargs,
+ )
+ # Set attributes that intermediate classes (PackedQuantLinear /
+ # GPTQQuantLinear) would have set in the old API.
+ self.pack_dtype = pack_dtype
+ if pack_dtype == torch.int8:
+ self.pack_dtype_bits = 8
+ elif pack_dtype == torch.int16:
+ self.pack_dtype_bits = 16
+ elif pack_dtype == torch.int32:
+ self.pack_dtype_bits = 32
+ elif pack_dtype == torch.int64:
+ self.pack_dtype_bits = 64
+ else:
+ raise ValueError(f"Unsupported pack_dtype: {pack_dtype}")
+ self.pack_factor = self.pack_dtype_bits // bits
+ self.group_size = group_size if group_size != -1 else in_features
+ self.requested_group_size = group_size
+ self.desc_act = desc_act
+ self.sym = sym
+ else:
+ super().__init__(
+ bits=bits,
+ group_size=group_size,
+ sym=sym,
+ desc_act=desc_act,
+ in_features=in_features,
+ out_features=out_features,
+ bias=bias,
+ pack_dtype=pack_dtype,
+ backend=backend,
+ adapter=None,
+ register_buffers=False,
+ **kwargs,
+ )
# toggle fp32 mode depending on MARLIN or MARLIN_FP16 backend
self.fp32 = True if self.backend in [BACKEND.MARLIN, BACKEND.AUTO] else False