Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

fix layer config (#1373)

Wenhua Cheng4064c4f1bd79
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py
index 34418dc62..22149ddbc 100644
--- a/auto_round/compressors/base.py
+++ b/auto_round/compressors/base.py
@@ -25,13 +25,11 @@ from typing import Any, Callable, Optional, Union
import accelerate
import torch
-import transformers
from accelerate.big_modeling import dispatch_model, infer_auto_device_map
from accelerate.utils import get_balanced_memory, get_max_memory
-from packaging import version
from torch import autocast
from tqdm import tqdm
-from transformers import AutoConfig, set_seed
+from transformers import set_seed
from auto_round import envs
from auto_round.auto_scheme.gen_auto_scheme import AutoScheme
@@ -253,6 +251,16 @@ class BaseCompressor(object):
... # ...
... }
"""
+ scheme_fields = [f.name for f in fields(QuantizationScheme)]
+
+ # 1. Pre-extract user-specified overrides from kwargs
+ # This ensures we know exactly what the user wants to "force"
+ self.user_scheme_overrides = {}
+ for k in scheme_fields:
+ if k in kwargs:
+ value = kwargs.pop(k)
+ if value is not None:
+ self.user_scheme_overrides[k] = value
# Model related
model_dtype = kwargs.pop("model_dtype", None)
@@ -307,8 +315,9 @@ class BaseCompressor(object):
# should be set after loading model and set layer_config, cause some special scheme need these.
# Preserve the original, unparsed scheme for later use in auto scheme generation
# within `configure_layer_config` (which may need the raw value instead of `self.scheme`).
+ default_scheme, self.is_auto_scheme = self._parse_and_set_scheme(scheme, self.user_scheme_overrides)
self.orig_scheme = copy.deepcopy(scheme)
- self.scheme, self.is_auto_scheme = self._parse_and_set_scheme(scheme, kwargs)
+ self.scheme = default_scheme
gguf_scheme_name = get_gguf_scheme(self.scheme)
# GGUF uses fp32 scale dtype as default
@@ -606,127 +615,152 @@ class BaseCompressor(object):
else:
raise TypeError(f"device_map should be [str, torch.device, int, dict], but got {type(device_map)}")
- def _parse_and_set_scheme(
- self, scheme: Union[str, dict, QuantizationScheme], kwargs
- ) -> tuple[QuantizationScheme, bool]:
- """Parse and set the quantization scheme."""
-
- def _parse_and_set(scheme, kwargs):
- if kwargs.get("data_type", None) and kwargs["data_type"].endswith("_dq") and not scheme.startswith("gguf"):
- if "bits" not in kwargs:
- data_type = kwargs["data_type"]
- raise KeyError(
- f"please set bits when setting data_type={data_type}, or using scheme as an alternative."
- )
- bits = kwargs["bits"]
- scheme = f"gguf:q{bits}_k" if bits == 6 else f"gguf:q{bits}_k_s"
- res = None
- if isinstance(scheme, QuantizationScheme):
- scheme = asdict(scheme)
- elif isinstance(scheme, dict):
- scheme = scheme
- elif isinstance(scheme, str):
- # We’d better keep the string scheme instead of the dict config,
- # since GGUF uses different mixed-bit strategies for q4_k_s and q4_k_m
- # even though they share the same scheme dict.
- scheme = scheme.strip("'\" ")
- res = scheme
- scheme = scheme.upper()
- self.layer_config = _handle_special_schemes(
- scheme,
- self.layer_config,
- self.model,
- supported_types=self.supported_types,
- inner_supported_types=self.inner_supported_types,
- quant_lm_head=self.quant_lm_head,
- mllm=getattr(self, "mllm", False),
- )
- scheme = asdict(preset_name_to_scheme(scheme))
- scheme_keys = [f.name for f in fields(QuantizationScheme)]
- for key in scheme_keys:
- if key in kwargs and kwargs[key] is not None:
- setattr(self, key, kwargs[key])
- else:
- setattr(self, key, scheme.get(key, None))
- # kwargs.pop(key, None)
- if self.act_dynamic is None:
- self.act_dynamic = True
+ def _reconcile_bits_and_dtype(self, config: dict, prefix: str = ""):
+ """
+ Harmonizes 'bits' and 'data_type' for weights or activations.
+ Ensures internal consistency by prioritizing data_type inference.
+ """
+ dt_key = f"{prefix}data_type"
+ bits_key = f"{prefix}bits"
+
+ if config.get(dt_key) is None:
+ return
- tmp_bits = infer_bits_by_data_type(self.data_type)
- if tmp_bits is not None and tmp_bits < 16 and tmp_bits != self.bits:
+ # Infer the correct bit-width based on the data_type string
+ inferred_bits = infer_bits_by_data_type(config[dt_key])
+
+ if inferred_bits is not None and inferred_bits < 16:
+ # Check for conflict between user-specified bits and inferred bits
+ if inferred_bits != config.get(bits_key):
logger.warning(
- f"'data_type' do not match the specified 'bits' setting. Resetting 'bits' to {tmp_bits}."
+ f"'{dt_key}' does not match '{bits_key}'. " f"Resetting '{bits_key}' to {inferred_bits}."
)
- self.bits = tmp_bits
- if tmp_bits is not None and tmp_bits < 16:
- for (
- supported_dtype
- ) in SUPPORTED_DTYPES: # to easily handle dtype mx_fp4 and layer_config={xxx:{bits:8}}
- if self.data_type.startswith(supported_dtype):
- if supported_dtype + str(tmp_bits) == self.data_type: # could not replace FP8_e4m3
- self.data_type = supported_dtype
- break
+ config[bits_key] = inferred_bits
- self.act_group_size = self.act_group_size if self.act_group_size is not None else self.group_size
- self.act_bits = self.act_bits if self.act_bits is not None else 16
- self.act_sym = self.act_sym if self.act_sym is not None else self.sym
+ # Normalize data_type (e.g., 'mx_fp4' -> 'mx')
+ for supported in SUPPORTED_DTYPES:
+ if config[dt_key] == f"{supported}{inferred_bits}":
+ config[dt_key] = supported
+ break
- if self.act_data_type is None:
- if self.data_type in SUPPORTED_DTYPES and self.act_bits < 16:
- self.act_data_type = self.data_type
- logger.info(f"activation adopts {self.data_type}")
- else:
- self.act_data_type = "float"
- tmp_act_bits = infer_bits_by_data_type(self.act_data_type)
- if tmp_act_bits is not None and tmp_act_bits < 16 and tmp_act_bits != self.act_bits:
- self.act_bits = tmp_act_bits
- logger.warning(
- f"`act_data_type` do not"
- f" match the specified 'act_bits' setting. Resetting 'act_bits' to {tmp_act_bits}."
+ def _override_scheme_with_user_specify(
+ self, scheme: Union[str, dict, QuantizationScheme], user_scheme_overrides: dict[str, Any], return_str=True
+ ) -> Union[str, QuantizationScheme]:
+ """
+ Updates a base quantization scheme with user-provided overrides.
+ Handles GGUF formatting and synchronizes weight/activation parameters.
+ """
+ # 1. GGUF special handling: map data_type suffix to GGUF scheme names
+ dt_override = user_scheme_overrides.get("data_type", "")
+ if (
+ isinstance(scheme, QuantizationScheme) or (isinstance(scheme, str) and not scheme.startswith("gguf"))
+ ) and dt_override.endswith("_dq"):
+ if "bits" not in user_scheme_overrides:
+ raise KeyError(f"Must specify 'bits' when using data_type={dt_override}")
+
+ bits = user_scheme_overrides["bits"]
+ suffix = "k" if bits == 6 else "k_s"
+ scheme = f"gguf:q{bits}_{suffix}"
+
+ # 2. Convert input scheme to a dictionary for processing
+ if isinstance(scheme, QuantizationScheme):
+ scheme_dict = asdict(scheme)
+ elif isinstance(scheme, str):
+ normalized_name = scheme.strip("'\" ").upper()
+ if normalized_name.startswith("GGUF") and len(user_scheme_overrides) > 0:
+ logger.warning_once(
+ "When using GGUF scheme, user-specified overrides will be ignored to ensure format compatibility."
)
- if tmp_act_bits is not None and tmp_act_bits < 16:
- for (
- supported_dtype
- ) in SUPPORTED_DTYPES: # To easily handle dtype mx_fp4 and layer_config={xxx:{bits:8}}
- if self.act_data_type.startswith(supported_dtype):
- if supported_dtype + str(tmp_act_bits) == self.act_data_type: # Could not replace FP8_e4m3
- self.act_data_type = supported_dtype
- break
- for key in scheme_keys:
- scheme[key] = getattr(self, key)
- if res and QuantizationScheme.from_dict(scheme) == preset_name_to_scheme(res):
- return res
+ user_scheme_overrides = {}
+ # If no overrides exist, return the normalized string immediately
+ if not user_scheme_overrides and return_str:
+ return normalized_name
+ scheme_dict = asdict(preset_name_to_scheme(normalized_name))
+ else:
+ scheme_dict = scheme.copy()
+
+ # 3. Apply overrides and define default behaviors
+ scheme_dict.update(user_scheme_overrides)
+
+ if scheme_dict.get("act_dynamic") is None:
+ scheme_dict["act_dynamic"] = True
+
+ # 4. Reconcile weight settings (bits vs data_type)
+ self._reconcile_bits_and_dtype(scheme_dict)
+
+ # 5. Fallback logic: Inherit activation settings from weight settings
+ scheme_dict["act_group_size"] = (
+ scheme_dict.get("act_group_size")
+ if scheme_dict.get("act_group_size") is not None
+ else scheme_dict.get("group_size")
+ )
+ scheme_dict["act_bits"] = scheme_dict.get("act_bits") or 16
+ scheme_dict["act_sym"] = (
+ scheme_dict.get("act_sym") if scheme_dict.get("act_sym") is not None else scheme_dict.get("sym")
+ )
+
+ # 6. Activation data_type logic
+ if scheme_dict.get("act_data_type") is None:
+ is_supported = scheme_dict["data_type"] in SUPPORTED_DTYPES
+ if is_supported and scheme_dict["act_bits"] < 16:
+ scheme_dict["act_data_type"] = scheme_dict["data_type"]
+ logger.info(f"Activation adopting weight data_type: {scheme_dict['data_type']}")
+ else:
+ scheme_dict["act_data_type"] = "float"
+
+ # 7. Reconcile activation settings
+ self._reconcile_bits_and_dtype(scheme_dict, prefix="act_")
+
+ return QuantizationScheme.from_dict(scheme_dict)
+
+ def _parse_and_set_scheme(
+ self, scheme: Union[str, dict, QuantizationScheme, AutoScheme], user_scheme_overrides: dict[str, Any]
+ ) -> tuple[Union[str, QuantizationScheme], bool]:
+ """
+ Parses the final scheme and binds all resulting attributes to 'self'.
+ """
+
+ is_auto_scheme = isinstance(scheme, AutoScheme)
+
+ if is_auto_scheme:
+ if not scheme.options:
+ raise ValueError("AutoScheme options cannot be empty")
else:
- return QuantizationScheme.from_dict(scheme)
-
- if isinstance(scheme, AutoScheme):
- if len(scheme.options) <= 0:
- raise ValueError("options of AutoScheme must not be empty")
- options = []
- for option in scheme.options:
- new_option = _parse_and_set(option, kwargs)
- options.append(new_option)
- scheme.options = options
- for opt in options:
- if isinstance(opt, str) and opt == "BF16":
+ for option in scheme.options:
+ if isinstance(option, str):
+ if "mixed" in option:
+ raise ValueError(f"Mixed option {option} is not supported")
+
+ # Map user overrides across all auto-scheme options
+ scheme.options = [
+ self._override_scheme_with_user_specify(opt, user_scheme_overrides) for opt in scheme.options
+ ]
+
+ # Select the primary scheme for attribute binding (skipping BF16)
+ default_scheme = scheme.options[0]
+ for opt in scheme.options:
+ if opt == "BF16":
continue
if isinstance(opt, QuantizationScheme):
- if opt.bits >= 16 and (opt.act_bits is None or opt.act_bits >= 16):
- continue
- self.scheme = opt # Choose the first one that not 16 bits
- break
- # apply scheme to set default bits
- scheme = _parse_and_set(self.scheme, kwargs)
- is_auto_scheme = True
+ if opt.bits < 16 or (opt.act_bits and opt.act_bits < 16):
+ default_scheme = opt
+ break
else:
- scheme = _parse_and_set(scheme, kwargs)
- is_auto_scheme = False
+ default_scheme = self._override_scheme_with_user_specify(scheme, user_scheme_overrides)
- scheme_keys = [f.name for f in fields(QuantizationScheme)]
- for key in scheme_keys:
- kwargs.pop(key, None)
+ # Extract attributes from the chosen default_scheme
+ if isinstance(default_scheme, str):
+ final_attrs = self._override_scheme_with_user_specify(
+ default_scheme, user_scheme_overrides, return_str=False
+ )
+ else:
+ final_attrs = asdict(default_scheme)
- return scheme, is_auto_scheme
+ # Bind attributes to self for easy instance-level access
+ for key, value in final_attrs.items():
+ setattr(self, key, value)
+
+ return default_scheme, is_auto_scheme
def _adjust_torch_compile(self, enable_torch_compile: bool) -> None:
"""Sets the torch compile configuration for the tuning."""
@@ -1542,6 +1576,16 @@ class BaseCompressor(object):
if self.is_auto_scheme:
self.layer_config = self._gen_auto_scheme()
+ else:
+ self.layer_config = _handle_special_schemes(
+ self.orig_scheme,
+ self.layer_config,
+ self.model,
+ supported_types=self.supported_types,
+ inner_supported_types=self.inner_supported_types,
+ quant_lm_head=self.quant_lm_head,
+ mllm=getattr(self, "mllm", False),
+ )
fill_default_value = True
if self.is_auto_scheme:
@@ -2432,9 +2476,8 @@ class BaseCompressor(object):
else:
act_max = act_max.to(module.act_max.device)
if is_nv_fp(self.act_data_type): ## for nvfp per-tensor input_global_scale calculation usage
- module.act_max = torch.max(
- torch.tensor([act_max.max(), module.act_max.max()], device=act_max.device)
- )
+ max_val = torch.max(act_max.max(), module.act_max.max())
+ module.act_max = max_val.unsqueeze(0) if max_val.dim() == 0 else max_val
else:
module.act_max = torch.max(act_max, module.act_max)
diff --git a/auto_round/export/export_to_autoround/export_to_nvfp_mxfp.py b/auto_round/export/export_to_autoround/export_to_nvfp_mxfp.py
index 186eea053..b7382a317 100644
--- a/auto_round/export/export_to_autoround/export_to_nvfp_mxfp.py
+++ b/auto_round/export/export_to_autoround/export_to_nvfp_mxfp.py
@@ -73,7 +73,8 @@ def pack_layer(name, model, backend, device=None):
sym = layer.sym
if is_nv_fp(act_data_type) and act_bits <= 8:
- if not getattr(layer, "input_global_scale", None):
+ input_global_scale = getattr(layer, "input_global_scale", None)
+ if input_global_scale is None:
assert hasattr(layer, "act_max")
from auto_round.data_type.nvfp import calculate_gparam
diff --git a/auto_round/export/export_to_llmcompressor/export_to_fp.py b/auto_round/export/export_to_llmcompressor/export_to_fp.py
index e7835309f..05c01ee3a 100644
--- a/auto_round/export/export_to_llmcompressor/export_to_fp.py
+++ b/auto_round/export/export_to_llmcompressor/export_to_fp.py
@@ -72,7 +72,8 @@ def pack_layer(name, model, device=None):
sym = layer.sym
if is_nv_fp(act_data_type) and act_bits <= 8:
- if not getattr(layer, "input_global_scale", None):
+ input_global_scale = getattr(layer, "input_global_scale", None)
+ if input_global_scale is None:
assert hasattr(layer, "act_max")
from auto_round.data_type.nvfp import calculate_gparam
diff --git a/auto_round/schemes.py b/auto_round/schemes.py
index 54824270e..e9026450d 100644
--- a/auto_round/schemes.py
+++ b/auto_round/schemes.py
@@ -176,6 +176,7 @@ MXFP4 = QuantizationScheme.from_dict(
"act_data_type": "mx_fp",
"act_group_size": 32,
"act_sym": True,
+ "act_dynamic": True,
}
)
@@ -188,6 +189,7 @@ MXFP4_RCEIL = QuantizationScheme.from_dict(
"act_data_type": "mx_fp_rceil",
"act_group_size": 32,
"act_sym": True,
+ "act_dynamic": True,
}
)
@@ -201,6 +203,7 @@ MXFP8 = QuantizationScheme.from_dict(
"act_data_type": "mx_fp",
"act_group_size": 32,
"act_sym": True,
+ "act_dynamic": True,
}
)
@@ -213,6 +216,7 @@ MXFP8_RCEIL = QuantizationScheme.from_dict(
"act_data_type": "mx_fp_rceil",
"act_group_size": 32,
"act_sym": True,
+ "act_dynamic": True,
}
)
@@ -226,6 +230,7 @@ NVFP4 = QuantizationScheme.from_dict(
"act_data_type": "nv_fp4_with_static_gs",
"act_group_size": 16,
"act_sym": True,
+ "act_dynamic": True,
}
)
@@ -312,6 +317,8 @@ def _handle_special_schemes(
Provide some special auto_round recipes.
"""
+ if not isinstance(scheme_name, str):
+ return layer_config
if layer_config is None:
layer_config = {}
if scheme_name.lower() == "gguf:q2_k_mixed":
@@ -340,13 +347,13 @@ def _handle_special_schemes(
continue
if type(m) in supported_types or type(m) in inner_supported_types:
if "expert" in n and "shared" not in n:
- layer_config[n] = {"bits": 4}
+ layer_config[n] = {"bits": 4, "data_type": "int"}
elif n != lm_head_name and mllm:
layer_config[n] = {"bits": 16}
elif n != lm_head_name:
- layer_config[n] = {"bits": 8}
+ layer_config[n] = {"bits": 8, "data_type": "int"}
elif n == lm_head_name and quant_lm_head:
- layer_config[n] = {"bits": 8}
+ layer_config[n] = {"bits": 8, "data_type": "int"}
return layer_config
diff --git a/auto_round/utils/model.py b/auto_round/utils/model.py
index 7ce80a5cd..fe416cd97 100644
--- a/auto_round/utils/model.py
+++ b/auto_round/utils/model.py
@@ -1156,11 +1156,11 @@ def set_amax_for_uncalibrated_experts(
f"This may indicate calibration hooks were not attached to expert layers."
)
return uncalibrated_experts
- else:
- # Flatten all tensors to 1D before concatenation
- flat_values = [t.reshape(-1) for t in amax_values]
- all_values = torch.cat(flat_values)
- set_amax_value = torch.max(all_values)
+ # Flatten all tensors to 1D before concatenation
+ flat_values = [t.reshape(-1) for t in amax_values]
+ all_values = torch.cat(flat_values)
+ set_amax_value = torch.max(all_values)
+ set_amax_value = set_amax_value.unsqueeze(0) if set_amax_value.dim() == 0 else set_amax_value
for module in experts:
current_amax = _get_amax_value(module)
diff --git a/test/test_cpu/schemes/test_scheme.py b/test/test_cpu/schemes/test_scheme.py
index e2b0c15c3..dcafb3ca8 100644
--- a/test/test_cpu/schemes/test_scheme.py
+++ b/test/test_cpu/schemes/test_scheme.py
@@ -159,3 +159,30 @@ class TestAutoRound:
assert device_list == ["cuda:0", "cuda:1", "cpu"]
device_list = parse_available_devices("0,1")
assert len(device_list) == 1 and "cpu" in device_list
+
+ def test_set_scheme(self, tiny_qwen_model_path):
+ ar = AutoRound(
+ tiny_qwen_model_path,
+ scheme="gguf:q2_k_s",
+ data_type="fp",
+ nsamples=1,
+ disable_opt_rtn=True,
+ iters=0,
+ seqlen=2,
+ )
+ ar.quantize()
+
+ from auto_round.schemes import QuantizationScheme
+
+ qs = QuantizationScheme.from_dict({"bits": 4, "group_size": 64})
+ ar = AutoRound(
+ tiny_qwen_model_path,
+ scheme=qs,
+ bits=2,
+ data_type="int_asym_dq",
+ nsamples=1,
+ iters=0,
+ disable_opt_rtn=True,
+ seqlen=2,
+ )
+ ar.quantize()