PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
add gptqmodel 6.0 compatible change and MTP quantization skip warning. (#1663)
Xin Heac7aa288a4e1
diff --git a/auto_round/inference/backend.py b/auto_round/inference/backend.pyindex d98545679..c5b10c4dc 100644--- a/auto_round/inference/backend.py+++ b/auto_round/inference/backend.py@@ -861,17 +861,29 @@ def dynamic_import_inference_linear(backend, config):raise ValueError(f"unsupported backend {backend}, please set it to `auto` and retry")-def get_gptqmodel_awq_infer_linear(backend):- """Returns the appropriate gptqmodel AWQ QuantLinear class for inference."""-+def safe_import_gptqmodel():+ """Safely import gptqmodel on CPU to avoid meta device issues."""dtype = torch.get_default_dtype()if dtype != torch.float32:torch.set_default_dtype(torch.float32)try:- import gptqmodel # pylint: disable=E0401+ # When loaded via the "meta" device, `gptqmodel==6.0.3` raises an error (since the internal+ # loading process within the `transformers` library defaults to the "meta" device mode).+ # Therefore, it is necessary to first switch to the CPU to bypass this error, and then+ # switch back to the original data type once the loading process is complete.+ with torch.device("cpu"):+ import gptqmodel # pylint: disable=E0401++ return gptqmodelfinally:torch.set_default_dtype(dtype)++def get_gptqmodel_awq_infer_linear(backend):+ """Returns the appropriate gptqmodel AWQ QuantLinear class for inference."""++ gptqmodel = safe_import_gptqmodel()+# Select AWQ kernel based on the BackendInfos keyif "marlin" in backend:from gptqmodel.nn_modules.qlinear.marlin_awq import AwqMarlinQuantLinear # pylint: disable=E0401@@ -894,14 +906,7 @@ def get_gptqmodel_awq_infer_linear(backend):def get_gptqmodel_infer_linear(backend, bits=4, group_size=128, sym=False):- dtype = torch.get_default_dtype()- if dtype != torch.float32:- torch.set_default_dtype(torch.float32)-- try:- import gptqmodel # pylint: disable=E0401- finally:- torch.set_default_dtype(dtype)+ gptqmodel = safe_import_gptqmodel()if "marlin" in backend:return auto_round_extension.cuda.gptqmodel_marlin.get_marlin_layer()diff --git a/auto_round/utils/missing_tensors.py b/auto_round/utils/missing_tensors.pyindex bc05dbe01..94f3d7be7 100644--- a/auto_round/utils/missing_tensors.py+++ b/auto_round/utils/missing_tensors.py@@ -682,6 +682,15 @@ def _woq_quantize_missing_tensors(target_dir: str, missing_tensors_dict: dict) -layer_name = weight_key[: -len(".weight")]layer_cfg = _resolve_layer_cfg(layer_name)bits = layer_cfg["bits"]+ if bits not in [4, 8]:+ if bits < 4:+ bits = 4+ elif 4 < bits < 8:+ bits = 8+ logger.warning(+ f"Unsupported bits={layer_cfg['bits']} for layer '{layer_name}' in extra_config, "+ f"falling back to bits={bits} for quantization."+ )group_size = layer_cfg["group_size"]sym = layer_cfg["sym"]diff --git a/auto_round_extension/cuda/gptqmodel_marlin.py b/auto_round_extension/cuda/gptqmodel_marlin.pyindex 16949cb0a..144209b08 100644--- a/auto_round_extension/cuda/gptqmodel_marlin.py+++ b/auto_round_extension/cuda/gptqmodel_marlin.py@@ -24,12 +24,22 @@ import torchdef get_marlin_layer(): ##use an ugly wrapper to import gptqmodel on demand- import gptqmodel++ # When loaded via the "meta" device, `gptqmodel==6.0.3` raises an error (since the internal+ # loading process within the `transformers` library defaults to the "meta" device mode).+ # Therefore, it is necessary to first switch to the CPU to bypass this error, and then+ # switch back to the original data type once the loading process is complete.+ with torch.device("cpu"):+ import gptqmodel+from packaging.version import VersionNEW_VERSION = Falseif Version(gptqmodel.__version__) >= Version("5.0.0"):NEW_VERSION = True+ NEW_VERSION_6_0 = False+ if Version(gptqmodel.__version__) >= Version("6.0.0"):+ NEW_VERSION_6_0 = Truefrom gptqmodel.models._const import DEVICE, PLATFORM # pylint: disable=E0401from gptqmodel.nn_modules.qlinear import BaseQuantLinear # pylint: disable=E0401from gptqmodel.utils.backend import BACKEND # pylint: disable=E0401@@ -244,20 +254,59 @@ def get_marlin_layer(): ##use an ugly wrapper to import gptqmodel on demand# (since we have only one group per output channel)desc_act = False- super().__init__(- bits=bits,- group_size=group_size,- sym=sym,- desc_act=desc_act,- in_features=in_features,- out_features=out_features,- bias=bias,- pack_dtype=pack_dtype,- backend=kwargs.pop("backend", BACKEND.MARLIN),- adapter=None,- register_buffers=False,- **kwargs,- )+ backend = kwargs.pop("backend", BACKEND.MARLIN)+ if NEW_VERSION_6_0:+ # gptqmodel >= 6.0.0: BaseQuantLinear no longer accepts group_size/sym/desc_act/pack_dtype+ # directly; they must be passed via validate_kwargs. Attributes are also set manually.+ super().__init__(+ bits=bits,+ in_features=in_features,+ out_features=out_features,+ bias=bias,+ backend=backend,+ adapter=None,+ register_buffers=False,+ validate_kwargs={+ "group_size": group_size,+ "desc_act": desc_act,+ "sym": sym,+ "pack_dtype": pack_dtype,+ },+ **kwargs,+ )+ # Set attributes that intermediate classes (PackedQuantLinear /+ # GPTQQuantLinear) would have set in the old API.+ self.pack_dtype = pack_dtype+ if pack_dtype == torch.int8:+ self.pack_dtype_bits = 8+ elif pack_dtype == torch.int16:+ self.pack_dtype_bits = 16+ elif pack_dtype == torch.int32:+ self.pack_dtype_bits = 32+ elif pack_dtype == torch.int64:+ self.pack_dtype_bits = 64+ else:+ raise ValueError(f"Unsupported pack_dtype: {pack_dtype}")+ self.pack_factor = self.pack_dtype_bits // bits+ self.group_size = group_size if group_size != -1 else in_features+ self.requested_group_size = group_size+ self.desc_act = desc_act+ self.sym = sym+ else:+ super().__init__(+ bits=bits,+ group_size=group_size,+ sym=sym,+ desc_act=desc_act,+ in_features=in_features,+ out_features=out_features,+ bias=bias,+ pack_dtype=pack_dtype,+ backend=backend,+ adapter=None,+ register_buffers=False,+ **kwargs,+ )# toggle fp32 mode depending on MARLIN or MARLIN_FP16 backendself.fp32 = True if self.backend in [BACKEND.MARLIN, BACKEND.AUTO] else False