PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
fix layer config (#1373)
Wenhua Cheng4064c4f1bd79
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.pyindex 34418dc62..22149ddbc 100644--- a/auto_round/compressors/base.py+++ b/auto_round/compressors/base.py@@ -25,13 +25,11 @@ from typing import Any, Callable, Optional, Unionimport accelerateimport torch-import transformersfrom accelerate.big_modeling import dispatch_model, infer_auto_device_mapfrom accelerate.utils import get_balanced_memory, get_max_memory-from packaging import versionfrom torch import autocastfrom tqdm import tqdm-from transformers import AutoConfig, set_seed+from transformers import set_seedfrom auto_round import envsfrom auto_round.auto_scheme.gen_auto_scheme import AutoScheme@@ -253,6 +251,16 @@ class BaseCompressor(object):... # ...... }"""+ scheme_fields = [f.name for f in fields(QuantizationScheme)]++ # 1. Pre-extract user-specified overrides from kwargs+ # This ensures we know exactly what the user wants to "force"+ self.user_scheme_overrides = {}+ for k in scheme_fields:+ if k in kwargs:+ value = kwargs.pop(k)+ if value is not None:+ self.user_scheme_overrides[k] = value# Model relatedmodel_dtype = kwargs.pop("model_dtype", None)@@ -307,8 +315,9 @@ class BaseCompressor(object):# should be set after loading model and set layer_config, cause some special scheme need these.# Preserve the original, unparsed scheme for later use in auto scheme generation# within `configure_layer_config` (which may need the raw value instead of `self.scheme`).+ default_scheme, self.is_auto_scheme = self._parse_and_set_scheme(scheme, self.user_scheme_overrides)self.orig_scheme = copy.deepcopy(scheme)- self.scheme, self.is_auto_scheme = self._parse_and_set_scheme(scheme, kwargs)+ self.scheme = default_schemegguf_scheme_name = get_gguf_scheme(self.scheme)# GGUF uses fp32 scale dtype as default@@ -606,127 +615,152 @@ class BaseCompressor(object):else:raise TypeError(f"device_map should be [str, torch.device, int, dict], but got {type(device_map)}")- def _parse_and_set_scheme(- self, scheme: Union[str, dict, QuantizationScheme], kwargs- ) -> tuple[QuantizationScheme, bool]:- """Parse and set the quantization scheme."""-- def _parse_and_set(scheme, kwargs):- if kwargs.get("data_type", None) and kwargs["data_type"].endswith("_dq") and not scheme.startswith("gguf"):- if "bits" not in kwargs:- data_type = kwargs["data_type"]- raise KeyError(- f"please set bits when setting data_type={data_type}, or using scheme as an alternative."- )- bits = kwargs["bits"]- scheme = f"gguf:q{bits}_k" if bits == 6 else f"gguf:q{bits}_k_s"- res = None- if isinstance(scheme, QuantizationScheme):- scheme = asdict(scheme)- elif isinstance(scheme, dict):- scheme = scheme- elif isinstance(scheme, str):- # We’d better keep the string scheme instead of the dict config,- # since GGUF uses different mixed-bit strategies for q4_k_s and q4_k_m- # even though they share the same scheme dict.- scheme = scheme.strip("'\" ")- res = scheme- scheme = scheme.upper()- self.layer_config = _handle_special_schemes(- scheme,- self.layer_config,- self.model,- supported_types=self.supported_types,- inner_supported_types=self.inner_supported_types,- quant_lm_head=self.quant_lm_head,- mllm=getattr(self, "mllm", False),- )- scheme = asdict(preset_name_to_scheme(scheme))- scheme_keys = [f.name for f in fields(QuantizationScheme)]- for key in scheme_keys:- if key in kwargs and kwargs[key] is not None:- setattr(self, key, kwargs[key])- else:- setattr(self, key, scheme.get(key, None))- # kwargs.pop(key, None)- if self.act_dynamic is None:- self.act_dynamic = True+ def _reconcile_bits_and_dtype(self, config: dict, prefix: str = ""):+ """+ Harmonizes 'bits' and 'data_type' for weights or activations.+ Ensures internal consistency by prioritizing data_type inference.+ """+ dt_key = f"{prefix}data_type"+ bits_key = f"{prefix}bits"++ if config.get(dt_key) is None:+ return- tmp_bits = infer_bits_by_data_type(self.data_type)- if tmp_bits is not None and tmp_bits < 16 and tmp_bits != self.bits:+ # Infer the correct bit-width based on the data_type string+ inferred_bits = infer_bits_by_data_type(config[dt_key])++ if inferred_bits is not None and inferred_bits < 16:+ # Check for conflict between user-specified bits and inferred bits+ if inferred_bits != config.get(bits_key):logger.warning(- f"'data_type' do not match the specified 'bits' setting. Resetting 'bits' to {tmp_bits}."+ f"'{dt_key}' does not match '{bits_key}'. " f"Resetting '{bits_key}' to {inferred_bits}.")- self.bits = tmp_bits- if tmp_bits is not None and tmp_bits < 16:- for (- supported_dtype- ) in SUPPORTED_DTYPES: # to easily handle dtype mx_fp4 and layer_config={xxx:{bits:8}}- if self.data_type.startswith(supported_dtype):- if supported_dtype + str(tmp_bits) == self.data_type: # could not replace FP8_e4m3- self.data_type = supported_dtype- break+ config[bits_key] = inferred_bits- self.act_group_size = self.act_group_size if self.act_group_size is not None else self.group_size- self.act_bits = self.act_bits if self.act_bits is not None else 16- self.act_sym = self.act_sym if self.act_sym is not None else self.sym+ # Normalize data_type (e.g., 'mx_fp4' -> 'mx')+ for supported in SUPPORTED_DTYPES:+ if config[dt_key] == f"{supported}{inferred_bits}":+ config[dt_key] = supported+ break- if self.act_data_type is None:- if self.data_type in SUPPORTED_DTYPES and self.act_bits < 16:- self.act_data_type = self.data_type- logger.info(f"activation adopts {self.data_type}")- else:- self.act_data_type = "float"- tmp_act_bits = infer_bits_by_data_type(self.act_data_type)- if tmp_act_bits is not None and tmp_act_bits < 16 and tmp_act_bits != self.act_bits:- self.act_bits = tmp_act_bits- logger.warning(- f"`act_data_type` do not"- f" match the specified 'act_bits' setting. Resetting 'act_bits' to {tmp_act_bits}."+ def _override_scheme_with_user_specify(+ self, scheme: Union[str, dict, QuantizationScheme], user_scheme_overrides: dict[str, Any], return_str=True+ ) -> Union[str, QuantizationScheme]:+ """+ Updates a base quantization scheme with user-provided overrides.+ Handles GGUF formatting and synchronizes weight/activation parameters.+ """+ # 1. GGUF special handling: map data_type suffix to GGUF scheme names+ dt_override = user_scheme_overrides.get("data_type", "")+ if (+ isinstance(scheme, QuantizationScheme) or (isinstance(scheme, str) and not scheme.startswith("gguf"))+ ) and dt_override.endswith("_dq"):+ if "bits" not in user_scheme_overrides:+ raise KeyError(f"Must specify 'bits' when using data_type={dt_override}")++ bits = user_scheme_overrides["bits"]+ suffix = "k" if bits == 6 else "k_s"+ scheme = f"gguf:q{bits}_{suffix}"++ # 2. Convert input scheme to a dictionary for processing+ if isinstance(scheme, QuantizationScheme):+ scheme_dict = asdict(scheme)+ elif isinstance(scheme, str):+ normalized_name = scheme.strip("'\" ").upper()+ if normalized_name.startswith("GGUF") and len(user_scheme_overrides) > 0:+ logger.warning_once(+ "When using GGUF scheme, user-specified overrides will be ignored to ensure format compatibility.")- if tmp_act_bits is not None and tmp_act_bits < 16:- for (- supported_dtype- ) in SUPPORTED_DTYPES: # To easily handle dtype mx_fp4 and layer_config={xxx:{bits:8}}- if self.act_data_type.startswith(supported_dtype):- if supported_dtype + str(tmp_act_bits) == self.act_data_type: # Could not replace FP8_e4m3- self.act_data_type = supported_dtype- break- for key in scheme_keys:- scheme[key] = getattr(self, key)- if res and QuantizationScheme.from_dict(scheme) == preset_name_to_scheme(res):- return res+ user_scheme_overrides = {}+ # If no overrides exist, return the normalized string immediately+ if not user_scheme_overrides and return_str:+ return normalized_name+ scheme_dict = asdict(preset_name_to_scheme(normalized_name))+ else:+ scheme_dict = scheme.copy()++ # 3. Apply overrides and define default behaviors+ scheme_dict.update(user_scheme_overrides)++ if scheme_dict.get("act_dynamic") is None:+ scheme_dict["act_dynamic"] = True++ # 4. Reconcile weight settings (bits vs data_type)+ self._reconcile_bits_and_dtype(scheme_dict)++ # 5. Fallback logic: Inherit activation settings from weight settings+ scheme_dict["act_group_size"] = (+ scheme_dict.get("act_group_size")+ if scheme_dict.get("act_group_size") is not None+ else scheme_dict.get("group_size")+ )+ scheme_dict["act_bits"] = scheme_dict.get("act_bits") or 16+ scheme_dict["act_sym"] = (+ scheme_dict.get("act_sym") if scheme_dict.get("act_sym") is not None else scheme_dict.get("sym")+ )++ # 6. Activation data_type logic+ if scheme_dict.get("act_data_type") is None:+ is_supported = scheme_dict["data_type"] in SUPPORTED_DTYPES+ if is_supported and scheme_dict["act_bits"] < 16:+ scheme_dict["act_data_type"] = scheme_dict["data_type"]+ logger.info(f"Activation adopting weight data_type: {scheme_dict['data_type']}")+ else:+ scheme_dict["act_data_type"] = "float"++ # 7. Reconcile activation settings+ self._reconcile_bits_and_dtype(scheme_dict, prefix="act_")++ return QuantizationScheme.from_dict(scheme_dict)++ def _parse_and_set_scheme(+ self, scheme: Union[str, dict, QuantizationScheme, AutoScheme], user_scheme_overrides: dict[str, Any]+ ) -> tuple[Union[str, QuantizationScheme], bool]:+ """+ Parses the final scheme and binds all resulting attributes to 'self'.+ """++ is_auto_scheme = isinstance(scheme, AutoScheme)++ if is_auto_scheme:+ if not scheme.options:+ raise ValueError("AutoScheme options cannot be empty")else:- return QuantizationScheme.from_dict(scheme)-- if isinstance(scheme, AutoScheme):- if len(scheme.options) <= 0:- raise ValueError("options of AutoScheme must not be empty")- options = []- for option in scheme.options:- new_option = _parse_and_set(option, kwargs)- options.append(new_option)- scheme.options = options- for opt in options:- if isinstance(opt, str) and opt == "BF16":+ for option in scheme.options:+ if isinstance(option, str):+ if "mixed" in option:+ raise ValueError(f"Mixed option {option} is not supported")++ # Map user overrides across all auto-scheme options+ scheme.options = [+ self._override_scheme_with_user_specify(opt, user_scheme_overrides) for opt in scheme.options+ ]++ # Select the primary scheme for attribute binding (skipping BF16)+ default_scheme = scheme.options[0]+ for opt in scheme.options:+ if opt == "BF16":continueif isinstance(opt, QuantizationScheme):- if opt.bits >= 16 and (opt.act_bits is None or opt.act_bits >= 16):- continue- self.scheme = opt # Choose the first one that not 16 bits- break- # apply scheme to set default bits- scheme = _parse_and_set(self.scheme, kwargs)- is_auto_scheme = True+ if opt.bits < 16 or (opt.act_bits and opt.act_bits < 16):+ default_scheme = opt+ breakelse:- scheme = _parse_and_set(scheme, kwargs)- is_auto_scheme = False+ default_scheme = self._override_scheme_with_user_specify(scheme, user_scheme_overrides)- scheme_keys = [f.name for f in fields(QuantizationScheme)]- for key in scheme_keys:- kwargs.pop(key, None)+ # Extract attributes from the chosen default_scheme+ if isinstance(default_scheme, str):+ final_attrs = self._override_scheme_with_user_specify(+ default_scheme, user_scheme_overrides, return_str=False+ )+ else:+ final_attrs = asdict(default_scheme)- return scheme, is_auto_scheme+ # Bind attributes to self for easy instance-level access+ for key, value in final_attrs.items():+ setattr(self, key, value)++ return default_scheme, is_auto_schemedef _adjust_torch_compile(self, enable_torch_compile: bool) -> None:"""Sets the torch compile configuration for the tuning."""@@ -1542,6 +1576,16 @@ class BaseCompressor(object):if self.is_auto_scheme:self.layer_config = self._gen_auto_scheme()+ else:+ self.layer_config = _handle_special_schemes(+ self.orig_scheme,+ self.layer_config,+ self.model,+ supported_types=self.supported_types,+ inner_supported_types=self.inner_supported_types,+ quant_lm_head=self.quant_lm_head,+ mllm=getattr(self, "mllm", False),+ )fill_default_value = Trueif self.is_auto_scheme:@@ -2432,9 +2476,8 @@ class BaseCompressor(object):else:act_max = act_max.to(module.act_max.device)if is_nv_fp(self.act_data_type): ## for nvfp per-tensor input_global_scale calculation usage- module.act_max = torch.max(- torch.tensor([act_max.max(), module.act_max.max()], device=act_max.device)- )+ max_val = torch.max(act_max.max(), module.act_max.max())+ module.act_max = max_val.unsqueeze(0) if max_val.dim() == 0 else max_valelse:module.act_max = torch.max(act_max, module.act_max)diff --git a/auto_round/export/export_to_autoround/export_to_nvfp_mxfp.py b/auto_round/export/export_to_autoround/export_to_nvfp_mxfp.pyindex 186eea053..b7382a317 100644--- a/auto_round/export/export_to_autoround/export_to_nvfp_mxfp.py+++ b/auto_round/export/export_to_autoround/export_to_nvfp_mxfp.py@@ -73,7 +73,8 @@ def pack_layer(name, model, backend, device=None):sym = layer.symif is_nv_fp(act_data_type) and act_bits <= 8:- if not getattr(layer, "input_global_scale", None):+ input_global_scale = getattr(layer, "input_global_scale", None)+ if input_global_scale is None:assert hasattr(layer, "act_max")from auto_round.data_type.nvfp import calculate_gparamdiff --git a/auto_round/export/export_to_llmcompressor/export_to_fp.py b/auto_round/export/export_to_llmcompressor/export_to_fp.pyindex e7835309f..05c01ee3a 100644--- a/auto_round/export/export_to_llmcompressor/export_to_fp.py+++ b/auto_round/export/export_to_llmcompressor/export_to_fp.py@@ -72,7 +72,8 @@ def pack_layer(name, model, device=None):sym = layer.symif is_nv_fp(act_data_type) and act_bits <= 8:- if not getattr(layer, "input_global_scale", None):+ input_global_scale = getattr(layer, "input_global_scale", None)+ if input_global_scale is None:assert hasattr(layer, "act_max")from auto_round.data_type.nvfp import calculate_gparamdiff --git a/auto_round/schemes.py b/auto_round/schemes.pyindex 54824270e..e9026450d 100644--- a/auto_round/schemes.py+++ b/auto_round/schemes.py@@ -176,6 +176,7 @@ MXFP4 = QuantizationScheme.from_dict("act_data_type": "mx_fp","act_group_size": 32,"act_sym": True,+ "act_dynamic": True,})@@ -188,6 +189,7 @@ MXFP4_RCEIL = QuantizationScheme.from_dict("act_data_type": "mx_fp_rceil","act_group_size": 32,"act_sym": True,+ "act_dynamic": True,})@@ -201,6 +203,7 @@ MXFP8 = QuantizationScheme.from_dict("act_data_type": "mx_fp","act_group_size": 32,"act_sym": True,+ "act_dynamic": True,})@@ -213,6 +216,7 @@ MXFP8_RCEIL = QuantizationScheme.from_dict("act_data_type": "mx_fp_rceil","act_group_size": 32,"act_sym": True,+ "act_dynamic": True,})@@ -226,6 +230,7 @@ NVFP4 = QuantizationScheme.from_dict("act_data_type": "nv_fp4_with_static_gs","act_group_size": 16,"act_sym": True,+ "act_dynamic": True,})@@ -312,6 +317,8 @@ def _handle_special_schemes(Provide some special auto_round recipes."""+ if not isinstance(scheme_name, str):+ return layer_configif layer_config is None:layer_config = {}if scheme_name.lower() == "gguf:q2_k_mixed":@@ -340,13 +347,13 @@ def _handle_special_schemes(continueif type(m) in supported_types or type(m) in inner_supported_types:if "expert" in n and "shared" not in n:- layer_config[n] = {"bits": 4}+ layer_config[n] = {"bits": 4, "data_type": "int"}elif n != lm_head_name and mllm:layer_config[n] = {"bits": 16}elif n != lm_head_name:- layer_config[n] = {"bits": 8}+ layer_config[n] = {"bits": 8, "data_type": "int"}elif n == lm_head_name and quant_lm_head:- layer_config[n] = {"bits": 8}+ layer_config[n] = {"bits": 8, "data_type": "int"}return layer_configdiff --git a/auto_round/utils/model.py b/auto_round/utils/model.pyindex 7ce80a5cd..fe416cd97 100644--- a/auto_round/utils/model.py+++ b/auto_round/utils/model.py@@ -1156,11 +1156,11 @@ def set_amax_for_uncalibrated_experts(f"This may indicate calibration hooks were not attached to expert layers.")return uncalibrated_experts- else:- # Flatten all tensors to 1D before concatenation- flat_values = [t.reshape(-1) for t in amax_values]- all_values = torch.cat(flat_values)- set_amax_value = torch.max(all_values)+ # Flatten all tensors to 1D before concatenation+ flat_values = [t.reshape(-1) for t in amax_values]+ all_values = torch.cat(flat_values)+ set_amax_value = torch.max(all_values)+ set_amax_value = set_amax_value.unsqueeze(0) if set_amax_value.dim() == 0 else set_amax_valuefor module in experts:current_amax = _get_amax_value(module)diff --git a/test/test_cpu/schemes/test_scheme.py b/test/test_cpu/schemes/test_scheme.pyindex e2b0c15c3..dcafb3ca8 100644--- a/test/test_cpu/schemes/test_scheme.py+++ b/test/test_cpu/schemes/test_scheme.py@@ -159,3 +159,30 @@ class TestAutoRound:assert device_list == ["cuda:0", "cuda:1", "cpu"]device_list = parse_available_devices("0,1")assert len(device_list) == 1 and "cpu" in device_list++ def test_set_scheme(self, tiny_qwen_model_path):+ ar = AutoRound(+ tiny_qwen_model_path,+ scheme="gguf:q2_k_s",+ data_type="fp",+ nsamples=1,+ disable_opt_rtn=True,+ iters=0,+ seqlen=2,+ )+ ar.quantize()++ from auto_round.schemes import QuantizationScheme++ qs = QuantizationScheme.from_dict({"bits": 4, "group_size": 64})+ ar = AutoRound(+ tiny_qwen_model_path,+ scheme=qs,+ bits=2,+ data_type="int_asym_dq",+ nsamples=1,+ iters=0,+ disable_opt_rtn=True,+ seqlen=2,+ )+ ar.quantize()