PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
fix: asymmetric zero-point packing, format-scoped 8-bit asym, W5/W6/W7 llm_compressor presets (#2273)
avtc5d2f88473a14
diff --git a/auto_round/auto_scheme/delta_loss.py b/auto_round/auto_scheme/delta_loss.pyindex f407127ae..b85b157c7 100644--- a/auto_round/auto_scheme/delta_loss.py+++ b/auto_round/auto_scheme/delta_loss.py@@ -2548,6 +2548,7 @@ def _gen_layer_config(processor=None,is_vlm: bool = False,disk_index=None,+ export_format: str = None,):"""Score every candidate scheme in ``auto_scheme.options`` against ``quant_layer_names``and return per-layer per-scheme losses used by the caller to pick a final bit-width@@ -3501,6 +3502,16 @@ def _gen_layer_config(# print(best_loss, best_path) # TODO better loglayer_config = copy.deepcopy(fixed_layer_scheme)+ # fixed pins (lm_head, MTP layer, ...) bypass the options-level sym rule+ # in parse_scheme -- apply the same 8-bit-symmetric pinning here. The+ # allowance comes from the export format (llm_compressor serves W8 asym)+ # or the AR_ALLOW_W8_ASYM opt-in.+ from auto_round import envs+ from auto_round.schemes import format_allows_w8_asym++ _enforce_w8_symmetric_entries(+ layer_config, allow_w8_asym=envs.AR_ALLOW_W8_ASYM or format_allows_w8_asym(export_format)+ )options = list(copy.deepcopy(auto_scheme.options))# Replace scheme preset names with actual QuantizationScheme objectsfor index in range(len(options)):@@ -3547,6 +3558,46 @@ def _gen_layer_config(return layer_config+def _enforce_w8_symmetric_entries(layer_config: dict, allow_w8_asym: bool = False) -> int:+ """Enforce symmetric 8-bit int entries; returns how many were pinned.++ Same rationale as the options-level rule in parse_scheme: formats without+ a servable W8-asym path (native auto_round, auto_gptq, marlin) cannot use+ 8-bit asym, and fixed layer_config pins (lm_head, MTP layer, ...) bypass+ options. Entries WITHOUT an explicit sym would silently inherit the global+ --asym and are pinned; an EXPLICIT sym=False is refused up front -- there+ is no point spending quantization time on a configuration that can never+ be served. The AR_ALLOW_W8_ASYM opt-in or an export format whose serving+ stack accepts W8 asym (llm_compressor) skips both.+ """+ if allow_w8_asym:+ return 0+ pinned = 0+ for name, entry in layer_config.items():+ if not (isinstance(entry, dict) and entry.get("bits") == 8 and entry.get("data_type", "int") == "int"):+ continue+ if entry.get("sym") is False:+ raise ValueError(+ f"layer_config entry '{name}' requests 8-bit asymmetric weight quantization, "+ "which this format cannot serve (vLLM W8 GPTQ-format weights are "+ "symmetric-only; Marlin supports zero points at 4 bits only). Use a "+ "symmetric 8-bit entry, an asymmetric width of 7 bits or fewer, format "+ "'auto_round:llm_compressor', or set AR_ALLOW_W8_ASYM=1."+ )+ if "sym" not in entry:+ entry["sym"] = True+ pinned += 1+ if pinned:+ logger.info(+ "AutoScheme: pinned %d fixed layer_config entr%s to symmetric 8-bit "+ "(8-bit asym is not servable in the requested export format; llm_compressor "+ "and AR_ALLOW_W8_ASYM are the escape hatches)",+ pinned,+ "y" if pinned == 1 else "ies",+ )+ return pinned++# Supports model with gradient clearing between iterations@register_scheme_methods(("default", "DeltaLoss"))def gen_layer_config(@@ -3561,6 +3612,7 @@ def gen_layer_config(low_gpu_mem_usage=True,min_avg_bit_scheme=None,processor=None,+ export_format: str = None,**kwargs,):"""Public AutoScheme entry.@@ -3706,6 +3758,7 @@ def gen_layer_config(min_avg_bit_scheme=min_avg_bit_scheme,processor=processor,is_vlm=is_vlm,+ export_format=export_format,disk_index=disk_index,)except torch.OutOfMemoryError:@@ -3738,6 +3791,7 @@ def gen_layer_config(min_avg_bit_scheme=min_avg_bit_scheme,processor=processor,is_vlm=is_vlm,+ export_format=export_format,disk_index=disk_index,)diff --git a/auto_round/auto_scheme/gen_auto_scheme.py b/auto_round/auto_scheme/gen_auto_scheme.pyindex e86e1e52b..d212aca52 100644--- a/auto_round/auto_scheme/gen_auto_scheme.py+++ b/auto_round/auto_scheme/gen_auto_scheme.py@@ -98,8 +98,14 @@ class GenScheme:tokenizer=None,enable_torch_compile=True,processor=None,+ export_format: str = None,):self.auto_scheme = auto_scheme+ # Export-format context for the generation-time 8-bit-asym policy: the+ # compressor passes the format the run actually saves with (including+ # one supplied only to quantize_and_save). The AR_ALLOW_W8_ASYM env+ # opt-in is read at the policy gate itself.+ self.export_format = export_format# Upstream unconditionally forced low_cpu_mem_usage=False here# (commit 0c9c5b1d, "reduce memory consumption... for gguf") because of an# acknowledged bug for mixed INT4/INT8 schemes under the old OffloadManager-@@ -199,6 +205,7 @@ class GenScheme:low_gpu_mem_usage=self.auto_scheme.low_gpu_mem_usage,min_avg_bit_scheme=self.min_avg_bit_scheme,processor=self.processor,+ export_format=self.export_format,)layer_config = self.fallback_gguf_layer_config(layer_config)return layer_configdiff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex c2464d9a4..12c3dda97 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -42,7 +42,7 @@ def _collect_config_scheme_overrides(config) -> dict:return {k: getattr(config, k) for k in config._scheme_fields if getattr(config, k, None) is not None}-def _preview_resolved_attrs(config, scheme=None) -> dict:+def _preview_resolved_attrs(config, scheme=None, format=None) -> dict:from auto_round.auto_scheme.gen_auto_scheme import AutoScheme"""Resolve scheme attributes without mutating config, for routing decisions.@@ -57,7 +57,9 @@ def _preview_resolved_attrs(config, scheme=None) -> dict:callers read from the returned dict and never re-read raw ``config`` attrs.When the scheme cannot be previewed (``AutoScheme``, or a deferred parseerror), the config's own explicitly-set scheme overrides are returned so the- values still reflect what the user passed.+ values still reflect what the user passed. ``format`` must match the+ authoritative parse so format-scoped policies (e.g. the 8-bit asym rule)+ resolve identically here and never turn a refusal into fabricated defaults.Returns:dict: resolved scheme attributes (config overrides when preview is skipped).@@ -67,13 +69,18 @@ def _preview_resolved_attrs(config, scheme=None) -> dict:# AutoScheme needs model info — cannot preview; fall back to raw config attrs.return config_overridestry:- _, _, final_attrs = parse_scheme(scheme, config_overrides)+ _, _, final_attrs = parse_scheme(scheme, config_overrides, format=format)return final_attrs- except Exception:+ except Exception as e:+ logger.warning_once(+ "Scheme preview failed (%s: %s); routing falls back to the config's explicit overrides.",+ type(e).__name__,+ e,+ )return config_overrides-def _eager_validate_scheme(config, scheme=None) -> None:+def _eager_validate_scheme(config, scheme=None, format=None) -> None:from auto_round.auto_scheme.gen_auto_scheme import AutoScheme"""Eagerly validate scheme/config constraints at construction time.@@ -91,7 +98,7 @@ def _eager_validate_scheme(config, scheme=None) -> None:user_overrides = _collect_config_scheme_overrides(config)try:- _, _, final_attrs = parse_scheme(scheme, user_overrides)+ _, _, final_attrs = parse_scheme(scheme, user_overrides, format=format)except (ValueError, NotImplementedError):raiseexcept Exception:@@ -264,7 +271,7 @@ def _select_rtn_compressor_base_cls(quant_config: "RTNConfig", scheme, format, b# resolution later; this preview only chooses the class). Computed once: neither# `quant_config`'s scheme fields nor `scheme` itself change within this function,# so the result is invariant across every use below — no need to recompute it.- resolved_attrs = _preview_resolved_attrs(quant_config, scheme)+ resolved_attrs = _preview_resolved_attrs(quant_config, scheme, format=format)# Auto-disable rtn optimization for W8A16/W8A8-equivalent resolved schemes,# unless the user already set disable_opt_rtn explicitly.@@ -705,8 +712,17 @@ class _CompressorBuilder(object):route_scheme = (schemeif hasattr(scheme, "options") and hasattr(scheme, "avg_bits")- else QuantizationScheme.from_dict(_preview_resolved_attrs(quant_config, scheme))+ else QuantizationScheme.from_dict(_preview_resolved_attrs(quant_config, scheme, format=format)))+ # Eagerly validate scheme constraints that do not require model info.+ # This mirrors old-arch _check_configs() called at __init__ time so that+ # callers get ValueError/NotImplementedError on construction, not deferred.+ # Runs before the model-free early return so both routes enforce the+ # same config-level constraints (e.g. the format-scoped 8-bit asym rule).+ _eager_validate_scheme(quant_config, scheme, format=format)+ # NOTE: the W8-asym opt-in is the AR_ALLOW_W8_ASYM environment+ # variable, read directly by parse_scheme and the generation-time+ # gates; nothing is threaded through the compressor here.if is_model_free_route(model, route_scheme, model_free_iters, model_free_disable_opt_rtn, route_decision_kwargs):@@ -727,6 +743,7 @@ class _CompressorBuilder(object):seed=seed,enable_torch_compile=enable_torch_compile,disable_opt_rtn=model_free_disable_opt_rtn,+ format=format,**compressor_kwargs,**base_kwargs,**mllm_kwargs,@@ -734,11 +751,6 @@ class _CompressorBuilder(object):**route_kwargs,)- # Eagerly validate scheme constraints that do not require model info.- # This mirrors old-arch _check_configs() called at __init__ time so that- # callers get ValueError/NotImplementedError on construction, not deferred.- _eager_validate_scheme(quant_config, scheme)-local_args = dict(model=model,tokenizer=tokenizer,diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.pyindex c858727b5..562547469 100644--- a/auto_round/compressors/base.py+++ b/auto_round/compressors/base.py@@ -168,6 +168,26 @@ def _make_compressor_scheme_property(name):return property(getter, setter)+def _formats_policy_string_of(formats) -> str:+ """Comma-joined format names for policy checks (8-bit asym scoping).++ Handles both the raw string form (before resolution) and the resolved+ OutputFormat object list; backend names keep the composite spelling+ (e.g. ``auto_round:llm_compressor``)."""+ if formats is None:+ return ""+ if isinstance(formats, str):+ return formats+ items = formats if isinstance(formats, (list, tuple)) else [formats]+ names = []+ for fmt in items:+ try:+ names.append(fmt.get_backend_name())+ except Exception: # noqa: BLE001+ names.append(str(getattr(fmt, "output_format", "")))+ return ",".join(names)++class BaseOrchestrator(object):need_calib: bool = Truecompress_context: CompressContext = None@@ -272,7 +292,9 @@ class BaseOrchestrator(object):if "group_size" in kwargs:block_size = kwargs["group_size"]else:- block_size = parse_scheme(scheme, {})[2]["group_size"]+ block_size = parse_scheme(+ scheme, {}, format=_formats_policy_string_of(getattr(self, "formats", None))+ )[2]["group_size"]_cfg.block_size = block_size # TODO not robustself.rotation_configs.append(_cfg)assert self.quantize_config is not None, "QuantizationConfig is required for Compressor"@@ -517,7 +539,11 @@ class BaseOrchestrator(object):# Check if activation calibration is neededfrom auto_round.compressors.utils import check_need_act_calibration- _, _, final_attrs = parse_scheme(self.scheme, {})+ _, _, final_attrs = parse_scheme(+ self.scheme,+ {},+ format=_formats_policy_string_of(getattr(self, "formats", None)),+ )act_bits = final_attrs["act_bits"]act_data_type = final_attrs["act_data_type"]act_dynamic = final_attrs["act_dynamic"]@@ -705,7 +731,11 @@ class BaseOrchestrator(object):self.compress_context = compress_contextuser_scheme_overrides = collect_user_scheme_overrides(self._alg_configs)- default_scheme, self.is_auto_scheme, final_attrs = parse_scheme(self.scheme, user_scheme_overrides)+ default_scheme, self.is_auto_scheme, final_attrs = parse_scheme(+ self.scheme,+ user_scheme_overrides,+ format=_formats_policy_string_of(getattr(self, "formats", None)),+ )self.scheme_context = QuantizationScheme.from_dict(final_attrs)for config in self._alg_configs:@@ -816,6 +846,7 @@ class BaseOrchestrator(object):quant_lm_head=self.quant_lm_head,enable_gguf_official_mixed=False,is_mllm=self.model_context.is_mllm,+ format=self._formats_policy_string(),)regex_config = extract_regex_config(model=self.model_context.model,@@ -825,6 +856,7 @@ class BaseOrchestrator(object):supported_types=self.supported_types,inner_supported_types=self.inner_supported_types,ignore_layers=self.ignore_layers,+ format=self._formats_policy_string(),)discovery_plan = resolve_quantization_config((@@ -909,6 +941,7 @@ class BaseOrchestrator(object):tokenizer=self.model_context.tokenizer,enable_torch_compile=self.compress_context.enable_torch_compile,processor=self.model_context.processor,+ export_format=_formats_policy_string_of(self.formats),)layer_config = self.scheme_generator.get_layer_config()# Re-attach vision/audio-tower layers we peeled off earlier so the@@ -1004,6 +1037,7 @@ class BaseOrchestrator(object):enable_gguf_official_mixed=enable_gguf_official_mixed,is_mllm=self.model_context.is_mllm,fill_default_value=fill_default_value,+ format=self._formats_policy_string(),)regex_config = extract_regex_config(model=self.model_context.model,@@ -1014,6 +1048,7 @@ class BaseOrchestrator(object):inner_supported_types=INNER_SUPPORTED_LAYER_TYPES,ignore_layers=self.ignore_layers,fill_default_value=fill_default_value,+ format=self._formats_policy_string(),)# ``resolved_layer_config`` already descends from (and fully subsumes)# ``format_resolution.layer_config_patch`` -- ``source_layer_config`` above was@@ -2021,6 +2056,32 @@ class BaseOrchestrator(object):model_name.split("/")[-1] + (f"-{prefix}" if prefix else "") + f"-w{bits}{suffix}",)+ def _formats_policy_string(self) -> str:+ """Comma-joined format names for policy checks (8-bit asym scoping)."""+ return _formats_policy_string_of(self.formats)++ def _assert_w8_asym_exportable(self) -> None:+ """Re-check the 8-bit asym policy once the export format is final.++ Construction-time validation runs before ``quantize_and_save(format=...)``+ may supply the format, so it is conservative (native semantics). Re-check+ here with the actual format: llm_compressor exports keep W8 asym."""+ from auto_round import envs+ from auto_round.schemes import format_allows_w8_asym++ if envs.AR_ALLOW_W8_ASYM or format_allows_w8_asym(self._formats_policy_string()):+ return+ if getattr(self, "data_type", None) == "int" and getattr(self, "bits", None) == 8:+ if getattr(self, "sym", None) is False:+ raise ValueError(+ "8-bit asymmetric weight quantization is not supported for format "+ f"'{self._formats_policy_string()}': vLLM serves W8 GPTQ-format weights "+ "symmetric-only and Marlin supports zero points at 4 bits only. Use a "+ "symmetric 8-bit scheme (drop --asym), an asymmetric width of 7 bits or "+ "fewer, format 'auto_round:llm_compressor' (compressed-tensors serves W8 "+ "asym), or set AR_ALLOW_W8_ASYM=1 to skip this check."+ )+def quantize_and_save(self,output_dir: str = "tmp_autoround",@@ -2085,6 +2146,7 @@ class BaseOrchestrator(object):if isinstance(self.formats, str):self.formats = self._resolve_format_string(self.formats)self.compress_context.formats = self.formats+ self._assert_w8_asym_exportable()# Derive descriptive export dir after post_init so scheme-resolved attrs are available._fmt_str = format or (self.formats if isinstance(self.formats, str) else "")output_dir = self._get_export_dir(output_dir, _fmt_str)diff --git a/auto_round/compressors/layer_config_resolver.py b/auto_round/compressors/layer_config_resolver.pyindex 60f347d2e..f3cde69ec 100644--- a/auto_round/compressors/layer_config_resolver.py+++ b/auto_round/compressors/layer_config_resolver.py@@ -158,7 +158,13 @@ def _get_safetensor_layer_names_not_in_model(model, all_module_names: list) -> ldef _resolve_layer_config_presets(- layer_config, model, ignore_layers, default_scheme, default_scale_dtype, fill_default_value+ layer_config,+ model,+ ignore_layers,+ default_scheme,+ default_scale_dtype,+ fill_default_value,+ w8_asym_allowed: bool = False,) -> tuple[dict, dict, tuple[str, ...], set[str]]:"""Steps 1-4 of layer-config resolution: ignore-layer parsing, normalization,bits-inference, and default-filling."""@@ -213,6 +219,9 @@ def _resolve_layer_config_presets(# 2. normalizelayer_config = {k: normalize_item(v, k) for k, v in layer_config.items()}+ # entries whose sym was set by the user (dict key or a QuantizationScheme+ # object) rather than inherited from the default scheme below+ explicit_sym = {name for name, cfg in layer_config.items() if "sym" in cfg}# 3. infer missing bitsfor cfg in layer_config.values():@@ -247,6 +256,33 @@ def _resolve_layer_config_presets(else:cfg.setdefault(key, None)+ # 5. 8-bit asymmetric int entries are not servable in formats without a+ # W8-asym path (vLLM GPTQ-format is symmetric-only, Marlin zp is 4-bit+ # only). Refuse explicit requests up front; pin entries that would only+ # inherit the global asym. llm_compressor exports (and AR_ALLOW_W8_ASYM)+ # are exempt: compressed-tensors serves W8 asym.+ pinned_w8 = []+ for name, cfg in layer_config.items():+ if cfg.get("data_type") == "int" and cfg.get("bits") == 8 and cfg.get("sym") is False:+ if not w8_asym_allowed:+ if name in explicit_sym:+ raise ValueError(+ f"layer_config entry '{name}' requests 8-bit asymmetric weight quantization, "+ "which this format cannot serve (vLLM W8 GPTQ-format weights are "+ "symmetric-only; Marlin supports zero points at 4 bits only). Use a "+ "symmetric 8-bit entry, an asymmetric width of 7 bits or fewer, format "+ "'auto_round:llm_compressor', or set AR_ALLOW_W8_ASYM=1."+ )+ cfg["sym"] = True+ pinned_w8.append(name)+ if pinned_w8:+ logger.info(+ "layer_config: pinned %d 8-bit entry(s) to symmetric (inherited asym is not "+ "servable at 8 bits in this format): %s",+ len(pinned_w8),+ ", ".join(sorted(pinned_w8)),+ )+return layer_config, default_dict, scheme_keys, ignore_layer_patterns@@ -380,6 +416,7 @@ def resolve_layer_config(enable_gguf_official_mixed: bool = True,is_mllm: bool = False,fill_default_value: bool = True,+ format: str = None,) -> LayerConfig:"""Resolve final per-layer configuration without writing model attributes."""supported_types = tuple(SUPPORTED_LAYER_TYPES if supported_types is None else supported_types)@@ -389,6 +426,10 @@ def resolve_layer_config(scheme_value = scheme.valuegguf_name = scheme.preset_name if (scheme.preset_name or "").startswith("gguf:") else get_gguf_scheme(scheme_value)+ from auto_round import envs+ from auto_round.schemes import format_allows_w8_asym++ w8_asym_allowed = envs.AR_ALLOW_W8_ASYM or format_allows_w8_asym(format)resolved, default_dict, scheme_keys, ignore_patterns = _resolve_layer_config_presets(layer_config,model,@@ -396,6 +437,7 @@ def resolve_layer_config(scheme_value,scale_dtype,fill_default_value,+ w8_asym_allowed=w8_asym_allowed,)resolved, _, embedding_names, supported_types = _traverse_and_expand_layer_config(resolved,@@ -444,6 +486,7 @@ def extract_regex_config(inner_supported_types=None,ignore_layers: str = "",fill_default_value: bool = True,+ format: str = None,) -> LayerConfig:"""Resolve only the regex entries retained for export metadata."""supported_types = tuple(SUPPORTED_LAYER_TYPES if supported_types is None else supported_types)@@ -452,6 +495,13 @@ def extract_regex_config()scheme_value = scheme.valuegguf_name = scheme.preset_name if (scheme.preset_name or "").startswith("gguf:") else get_gguf_scheme(scheme_value)+ from auto_round import envs+ from auto_round.schemes import format_allows_w8_asym++ # Same allowance rule as resolve_layer_config: AutoScheme output can carry+ # DP-selected W8-asym entries, which stay untouched only when the export+ # format serves them (llm_compressor) or the opt-in env is set.+ w8_asym_allowed = envs.AR_ALLOW_W8_ASYM or format_allows_w8_asym(format)normalized, _, scheme_keys, ignore_patterns = _resolve_layer_config_presets(layer_config,model,@@ -459,6 +509,7 @@ def extract_regex_config(scheme_value,scale_dtype,fill_default_value,+ w8_asym_allowed=w8_asym_allowed,)_, regex_config, _, _ = _traverse_and_expand_layer_config(normalized,diff --git a/auto_round/compressors/model_free.py b/auto_round/compressors/model_free.pyindex fd4dbafbf..3cc5175ea 100644--- a/auto_round/compressors/model_free.py+++ b/auto_round/compressors/model_free.py@@ -519,7 +519,7 @@ class _ModelFreeCompressorCore:scheme: Union[str, QuantizationScheme] = "W4A16",layer_config: Optional[dict] = None,ignore_layers: str = "",- format: str = "auto_round",+ format: Optional[str] = None,device: str = "cpu",quant_lm_head: bool = False,quant_nontext_module: bool = False,@@ -533,6 +533,13 @@ class _ModelFreeCompressorCore:self.layer_config_input = layer_configself.ignore_layers_input = ignore_layersself.format = format or "auto_round"+ # Whether the caller explicitly requested an output format at construction.+ # Only an explicit request may be forwarded to the fallback compressor: the+ # fallback consumes the quantize_and_save(format=...) argument only while+ # its formats are still unset, so forwarding the implicit "auto_round"+ # default would silently pin every fallback export to the native format+ # (overriding e.g. save-time "gguf:q*_1" or "llm_compressor").+ self._format_explicit = format is not Noneself.device = deviceself.quant_lm_head = quant_lm_headself.quant_nontext_module = quant_nontext_module@@ -1491,7 +1498,7 @@ class ModelFreeCompressor(_ModelFreeCompressorCore):scheme: Union[str, QuantizationScheme] = "W4A16",layer_config: Optional[dict] = None,ignore_layers: str = "",- format: str = "auto_round",+ format: Optional[str] = None,device: str = "cpu",quant_lm_head: bool = False,quant_nontext_module: bool = False,@@ -1572,6 +1579,12 @@ class ModelFreeCompressor(_ModelFreeCompressorCore):# Scheme fields are consumed above from ``kwargs``. Preserve them when# a later format check falls back to the regular AutoRound flow.fallback_init.update(self.user_scheme_overrides)+ # Forward the construction format only when explicitly requested; the+ # fallback honors a format passed later to quantize_and_save() (the+ # 8-bit-asym opt-in is the AR_ALLOW_W8_ASYM env, read by parse_scheme+ # on every route).+ if getattr(self, "_format_explicit", False) and self.format:+ fallback_init["format"] = self.formatself._fallback_init_kwargs = fallback_initif quant_nontext_module:@@ -1582,18 +1595,27 @@ class ModelFreeCompressor(_ModelFreeCompressorCore):self._auto_scheme_resolved = Falseself._auto_scheme_family: Optional[str] = None- def _fallback_to_base_compressor(self):+ def _fallback_to_base_compressor(self, save_format: Optional[str] = None):from auto_round.autoround import AutoRoundlogger.info(- "Format '%s' is not supported by model-free mode; falling back to the regular AutoRound flow.",+ "Model-free mode cannot export this request directly; "+ "constructing the regular AutoRound compressor (requested format: '%s').",self.format,)+ # The fallback exists to serve a quantize_and_save(format=...) call: when+ # no explicit construction format was forwarded, the save-time format+ # must reach the fallback construction so eager validation (format-scoped+ # policies such as the 8-bit asym rule) judges the format actually being+ # saved. An explicitly requested construction format always wins.+ init_kwargs = dict(self._fallback_init_kwargs)+ if save_format is not None:+ init_kwargs.setdefault("format", save_format)logger.info("fallbacked_init_kwargs: %s",- self._fallback_init_kwargs,+ init_kwargs,)- compressor = AutoRound(**self._fallback_init_kwargs, disable_model_free=True)+ compressor = AutoRound(**init_kwargs, disable_model_free=True)self._fallback_compressor = compressordef _fallback_to_quantize_and_save(@@ -1603,7 +1625,7 @@ class ModelFreeCompressor(_ModelFreeCompressorCore):inplace: bool,**kwargs,):- self._fallback_to_base_compressor()+ self._fallback_to_base_compressor(save_format=format)return self._fallback_compressor.quantize_and_save( # pylint: disable=E1101output_dir=output_dir, format=format, inplace=inplace, **kwargs)@@ -1780,6 +1802,13 @@ class ModelFreeCompressor(_ModelFreeCompressorCore):**kwargs,) -> Any:"""Quantize and save — AutoRound compressor entry point."""+ from auto_round.schemes import parse_scheme++ # Re-check the 8-bit asym policy with the final format (construction is+ # conservative when the format is not yet known). parse_scheme owns the+ # policy and the message; the scheme is still unresolved here, so pass+ # the raw inputs instead of self.default_scheme.+ parse_scheme(self.scheme_input, dict(self.user_scheme_overrides or {}), format=format)# Early fallback gate for model-free + AutoScheme: avoid running# costly delta-loss selection when format is known incompatible.if self._precheck_auto_scheme_fallback(format):diff --git a/auto_round/envs.py b/auto_round/envs.pyindex 5eaf8351c..bef41f43d 100644--- a/auto_round/envs.py+++ b/auto_round/envs.py@@ -34,6 +34,7 @@ if TYPE_CHECKING:AR_RESUME_DIR: Optional[str] = NoneAR_FORCE_MOE_ROUTING_ALL_EXPERTS: bool = FalseAR_NVFP4_FUSED_LAYER_GLOBAL_SCALE: bool = True+ AR_ALLOW_W8_ASYM: bool = Falsedef _get_optional_positive_int_env(name: str) -> Optional[int]:@@ -136,6 +137,7 @@ environment_variables: dict[str, Callable[[], Any]] = {# weight global scale. Disable only for runtimes without that requirement."AR_NVFP4_FUSED_LAYER_GLOBAL_SCALE": lambda: os.getenv("AR_NVFP4_FUSED_LAYER_GLOBAL_SCALE", "1").lower()not in ("0", "false", "no", "off"),+ "AR_ALLOW_W8_ASYM": lambda: os.getenv("AR_ALLOW_W8_ASYM", "0").lower() in ("1", "true", "yes"),}diff --git a/auto_round/export/export_to_awq/utils.py b/auto_round/export/export_to_awq/utils.pyindex 871e4287a..86db8a3ee 100644--- a/auto_round/export/export_to_awq/utils.py+++ b/auto_round/export/export_to_awq/utils.py@@ -216,15 +216,21 @@ class WQLinear_GEMM(nn.Module):awq_linear.bias = linear.bias.clone().half()pack_num = 32 // awq_linear.w_bit+ maxq = 2**awq_linear.w_bit - 1repeat_size = group_size if group_size != -1 else linear.in_featuresrepeat_scales = scales.to(device).t().repeat_interleave(repeat_size, 1)if isinstance(zeros, torch.Tensor):repeat_zeros = zeros.to(device).t().repeat_interleave(repeat_size, 1)+ # clamp into [0, maxq]: the searched (scale, zp) grid may leave+ # round(W/s + zp) outside the level range (the search evaluates+ # the clamped loss); an unclamped value shifts bits into the+ # neighboring packed nibble and silently corrupts it.intweight = (torch.round(linear.weight.to(device) / repeat_scales[:, : linear.weight.shape[1]]+ repeat_zeros[:, : linear.weight.shape[1]])+ .clamp_(0, maxq).to(torch.int).t().contiguous()@@ -234,6 +240,7 @@ class WQLinear_GEMM(nn.Module):repeat_zeros = zerosintweight = (torch.round(linear.weight.to(device) / repeat_scales[:, : linear.weight.shape[1]] + repeat_zeros)+ .clamp_(0, maxq).to(torch.int).t().contiguous()diff --git a/auto_round/export/export_to_llmcompressor/export.py b/auto_round/export/export_to_llmcompressor/export.pyindex d53466d6b..407b88353 100644--- a/auto_round/export/export_to_llmcompressor/export.py+++ b/auto_round/export/export_to_llmcompressor/export.py@@ -13,6 +13,7 @@# limitations under the License.import copy+import mathimport osfrom collections.abc import Mappingfrom typing import Callable, Union@@ -207,7 +208,7 @@ def _compress_and_set_format(layer, scheme, device=None):def pack_layer(name, model, device=None):- from compressed_tensors.quantization import QuantizationStatus # pylint: disable=E0401+ from compressed_tensors.quantization import QuantizationStatus, QuantizationType # pylint: disable=E0401layer = get_module(model, name)if type(layer) not in SUPPORTED_LAYER_TYPES and not isinstance(layer, WrapperWALayer): ##already packed@@ -230,13 +231,73 @@ def pack_layer(name, model, device=None):scheme = construct_ct_scheme(layer)setattr(layer, "quantization_scheme", scheme)setattr(layer, "weight_scale", torch.nn.Parameter(layer.scale.to(weight_device)))+ # AutoRound zp is the UNSIGNED level convention (q in [0, 2^b-1],+ # W = (q - zp) * s); compressed-tensors packs the SIGNED convention (q in+ # [-2^(b-1), 2^(b-1)-1], zp added BEFORE the clamp into that range - see+ # calculate_range / _process_quantization). Handing the unsigned zp over+ # verbatim makes compress() clamp every level above 2^(b-1)-1, collapsing+ # the upper half of each group's range into one wrong value. Shifting zp+ # into the signed range makes CT's re-quantization exact:+ # clamp(round(w/s) + zp - 2^(b-1)) reproduces q - 2^(b-1) for the+ # quantizer's own q in [0, 2^b-1], and (q_ct - zp_ct) * s == (q - zp) * s.+ # FLOAT schemes (fp8/mxfp) keep CT's native zp convention: no shift.+ # QuantizationType is a str-Enum, but coerce defensively so the check+ # holds whether ``type`` is the enum member or its plain string value.+ zp_shift = (+ (1 << (scheme.weights.num_bits - 1)) if QuantizationType(scheme.weights.type) == QuantizationType.INT else 0+ )+ num_bits = scheme.weights.num_bitsif not isinstance(layer.zp, torch.Tensor):- if layer.sym:- zp = torch.full_like(layer.weight_scale, 0).to(torch.int8)- else:- zp = torch.full_like(layer.weight_scale, layer.zp).to(torch.int8)+ zp_val = 0 if layer.sym else float(layer.zp)+ if not layer.sym:+ if not math.isfinite(zp_val) or zp_val != round(zp_val):+ raise ValueError(+ f"asymmetric zero-point {zp_val} is not a finite integer value; refusing to "+ "pack a zero point that is not an unsigned quantization level"+ )+ if not 0 <= zp_val <= (1 << num_bits) - 1:+ raise ValueError(+ f"asymmetric zero-point {zp_val} is outside the unsigned level range "+ f"[0, {2 ** num_bits - 1}] expected for a {num_bits}-bit layer; refusing to "+ "pack a zero point that is not an unsigned quantization level"+ )+ fill = 0 if layer.sym else zp_val - zp_shift+ zp = torch.full_like(layer.weight_scale, fill).to(torch.int8)else:+ # NeUQI / optimized-RTN searches leave zp as a float32 integral-valued+ # tensor; compressed_tensors' packed format hard-requires torch.int8+ # (its own lifecycle quantizes with dtype=torch.int8), so cast here and+ # refuse values that would wrap rather than corrupting them silently.+ # Validate the UNSIGNED domain first: the shifted value alone cannot+ # catch e.g. a raw 4-bit zp of 100 (shifted 92 passes an int8 check).zp = layer.zp+ if zp.dtype != torch.int8:+ # NaN passes every min/max comparison, so check finiteness+ # explicitly; non-integral values indicate a corrupted search and+ # must not be silently rounded into a valid level.+ if zp.numel() and not bool(torch.isfinite(zp).all()):+ raise ValueError("asymmetric zero-point tensor contains non-finite values; refusing to pack it")+ if zp.numel() and not bool(torch.equal(zp, zp.round())):+ raise ValueError(+ "asymmetric zero-point tensor contains non-integral values; zero points "+ "must be integer quantization levels, refusing to pack a rounded copy"+ )+ zp = zp.round()+ if zp.numel() and (zp.min() < 0 or zp.max() > (1 << num_bits) - 1):+ raise ValueError(+ f"asymmetric zero-point values [{int(zp.min())}, {int(zp.max())}] are outside "+ f"the unsigned level range [0, {2 ** num_bits - 1}] for a {num_bits}-bit layer; "+ "the zero point must be an unsigned quantization level before the signed "+ "convention shift"+ )+ zp = zp - zp_shift+ if zp.numel() and (zp.min() < -128 or zp.max() > 127):+ raise ValueError(+ f"asymmetric zero-point values [{int(zp.min())}, {int(zp.max())}] (signed "+ "convention) do not fit torch.int8, which the compressed-tensors packed "+ "format requires"+ )+ zp = zp.to(torch.int8)setattr(layer, "weight_zero_point", torch.nn.Parameter(zp.to(weight_device), requires_grad=False))delattr(layer, "scale")diff --git a/auto_round/export/formats/backends/llm_compressor.py b/auto_round/export/formats/backends/llm_compressor.pyindex 9bc260684..c5091b63a 100644--- a/auto_round/export/formats/backends/llm_compressor.py+++ b/auto_round/export/formats/backends/llm_compressor.py@@ -36,9 +36,14 @@ class LLMCompressorFormat(OutputFormat):"INT8_W8A8","FP8_BLOCK","W4A16",+ "W5A16",+ "W6A16",+ "W7A16","W8A16",- "W2A16G32",- "W2A16G64",+ # group-size variants for mixed int pools (CT pack-quantized handles+ # any width/group); 8-bit variants are symmetric presets -- this format+ # also serves 8-bit asym via --asym, unlike the native formats+ *[f"W{b}A16G{g}" for b in (2, 3, 4, 5, 6, 7, 8) for g in (64, 32)],"W2A16","W3A16",]@@ -99,13 +104,13 @@ class LLMCompressorFormat(OutputFormat):@classmethoddef check_scheme_args(cls: OutputFormat, scheme: QuantizationScheme) -> bool:error_logs = []- if scheme.bits not in [2, 3, 4, 8, 16]:+ if scheme.bits not in [2, 3, 4, 5, 6, 7, 8, 16]:error_logs.append(f"bits={scheme.bits}")if not re.search("mxfp|fp|nvfp|int", scheme.data_type):error_logs.append(f"data_type={scheme.data_type}")if scheme.data_type == "fp" and scheme.bits != 8:error_logs.append(f"data_type={scheme.data_type}, bits={scheme.bits}")- if scheme.data_type == "int" and scheme.bits not in [2, 3, 4, 8]:+ if scheme.data_type == "int" and scheme.bits not in [2, 3, 4, 5, 6, 7, 8]:error_logs.append(f"data_type={scheme.data_type}, bits={scheme.bits}")if scheme.super_bits:error_logs.append(f"super_bits={scheme.super_bits}")diff --git a/auto_round/schemes.py b/auto_round/schemes.pyindex def0f79af..736c62cac 100644--- a/auto_round/schemes.py+++ b/auto_round/schemes.py@@ -493,15 +493,43 @@ def _override_scheme_with_user_specify(return QuantizationScheme.from_dict(scheme_dict)+def format_allows_w8_asym(format: str | None) -> bool:+ """Whether an output format can represent and serve 8-bit asymmetric int weights.++ ``llm_compressor`` (compressed-tensors pack-quantized) stores the zero point+ as int8 with a signed-convention shift and vLLM serves it (which kernels+ pick it up depends on GPU and group size -- see docs/step_by_step.md for+ the current kernel matrix). ``fake`` is research-only by+ declaration. The remaining formats (native ``auto_round``, ``auto_gptq``,+ marlin, awq) cannot serve W8 asym today: their packed layouts have no+ room for an 8-bit zero point and vLLM serves W8 GPTQ-format weights+ symmetric-only.+ """+ if not format:+ return False+ # Multi-format requests are allowed only when EVERY requested format can+ # serve W8 asym (a mixed "auto_round,llm_compressor" request would still+ # produce a native artifact that stock vLLM cannot load - require the+ # explicit AR_ALLOW_W8_ASYM=1 opt-in for that).+ parts = str(format).split(",")+ return all("llm_compressor" in p.split(":") or "fake" in p.split(":") for p in parts)++def parse_scheme(- scheme: Union[str, dict, QuantizationScheme, "AutoScheme"], user_scheme_overrides: dict[str, Any]+ scheme: Union[str, dict, QuantizationScheme, "AutoScheme"],+ user_scheme_overrides: dict[str, Any],+ format: str = None,) -> tuple[Union[str, QuantizationScheme], bool, dict[str, Any]]:"""Parses the final scheme."""+ from auto_round import envsfrom auto_round.auto_scheme.gen_auto_scheme import AutoSchemeis_auto_scheme = isinstance(scheme, AutoScheme)+ # W8 asym is allowed for formats that can serve it, or via the explicit+ # AR_ALLOW_W8_ASYM=1 opt-in for serving stacks beyond stock vLLM.+ w8_asym_ok = envs.AR_ALLOW_W8_ASYM or format_allows_w8_asym(format)if is_auto_scheme:if not scheme.options:raise ValueError("AutoScheme options cannot be empty")@@ -514,6 +542,29 @@ def parse_scheme(# Map user overrides across all auto-scheme optionsscheme.options = [_override_scheme_with_user_specify(opt, user_scheme_overrides) for opt in scheme.options]+ # --asym must never silently reach 8-bit options in formats that cannot+ # serve W8 asym (native auto_round / auto_gptq / marlin): vLLM's+ # GPTQ-format kernels are symmetric-only and Marlin's zero-point+ # support is 4-bit only. Pin such options back to symmetric, loudly.+ # llm_compressor exports (and AR_ALLOW_W8_ASYM) keep asym.+ import dataclasses++ if not w8_asym_ok:+ for opt_i, opt in enumerate(scheme.options):+ if (+ isinstance(opt, QuantizationScheme)+ and opt.sym is False+ and opt.bits == 8+ and opt.data_type == "int"+ ):+ scheme.options[opt_i] = dataclasses.replace(opt, sym=True)+ logger.info(+ "AutoScheme option %s stays symmetric: 8-bit asymmetric quantization is "+ "not servable in this format (--asym applies to sub-8-bit options only; "+ "use format llm_compressor or AR_ALLOW_W8_ASYM=1 to keep W8 asym)",+ opt_i,+ )+# Select the primary scheme for attribute binding (skipping BF16)default_scheme = scheme.options[0]for opt in scheme.options:@@ -532,6 +583,32 @@ def parse_scheme(final_attrs = asdict(final_attrs)else:final_attrs = asdict(default_scheme)++ if not is_auto_scheme and final_attrs.get("data_type") == "int" and final_attrs.get("bits") == 8:+ if final_attrs.get("sym") is False:+ if not w8_asym_ok:+ # Formats without a servable W8-asym path fail before any quantization+ # work starts. llm_compressor exports and AR_ALLOW_W8_ASYM are exempt.+ raise ValueError(+ "8-bit asymmetric weight quantization is not supported for this format: "+ "vLLM serves W8 GPTQ-format weights symmetric-only and Marlin supports "+ "zero points at 4 bits only. Use a symmetric 8-bit scheme (drop --asym), "+ "an asymmetric width of 7 bits or fewer, format 'auto_round:llm_compressor' "+ "(compressed-tensors serves W8 asym), or set AR_ALLOW_W8_ASYM=1 "+ "to skip this check."+ )+ _group = final_attrs.get("group_size")+ _fmt_parts = str(format or "").replace(",", ":").split(":")+ # group_size 128 and 0/-1 (per-tensor/channelwise) are served+ # broadly across current GPUs; other groups depend on kernel+ # availability, so warn without pinning+ if "llm_compressor" in _fmt_parts and _group not in (-1, 0, 128, None):+ logger.warning_once(+ "8-bit asym with group_size %s under llm_compressor: current vLLM "+ "kernels serve 8-bit asym broadly only at group sizes 128/-1 "+ "(see docs for kernel/GPU availability)",+ _group,+ )return default_scheme, is_auto_scheme, final_attrs@@ -565,9 +642,9 @@ W6A16 = QuantizationScheme.from_dict(})-W2A16 = QuantizationScheme.from_dict(+W7A16 = QuantizationScheme.from_dict({- "bits": 2,+ "bits": 7,"sym": True,"group_size": 128,"data_type": "int",@@ -575,21 +652,11 @@ W2A16 = QuantizationScheme.from_dict(})-W2A16G64 = QuantizationScheme.from_dict(- {- "bits": 2,- "sym": True,- "group_size": 64,- "data_type": "int",- "act_bits": 16,- }-)--W2A16G32 = QuantizationScheme.from_dict(+W2A16 = QuantizationScheme.from_dict({"bits": 2,"sym": True,- "group_size": 32,+ "group_size": 128,"data_type": "int","act_bits": 16,}@@ -801,12 +868,25 @@ BF16 = QuantizationScheme.from_dict(})+for _bits in (2, 3, 4, 5, 6, 7, 8):+ for _g in (64, 32):+ globals()[f"W{_bits}A16G{_g}"] = QuantizationScheme.from_dict(+ {+ "bits": _bits,+ "sym": True,+ "group_size": _g,+ "data_type": "int",+ "act_bits": 16,+ }+ )+PRESET_SCHEMES = {"W4A16": W4A16,"W2A16": W2A16,"W3A16": W3A16,"W5A16": W5A16,"W6A16": W6A16,+ "W7A16": W7A16,"W8A16": W8A16,"MXFP4": MXFP4,"MXFP6": MXFP6,@@ -816,8 +896,14 @@ PRESET_SCHEMES = {"NVFP4": NVFP4,"NVFP4_E5M3": NVFP4_E5M3,"FPW8A16": FPW8A16,- "W2A16G64": W2A16G64,- "W2A16G32": W2A16G32,+ # Group-size variants for the 2-8 bit int presets: the bare presets default+ # to group_size 128; these finer-group variants allow per-layer group-size+ # selection via --scheme/layer_config and mixed-group AutoScheme pools, e.g.+ # options="W3A16,W4A16,W4A16G64,W4A16G32". 8-bit variants are symmetric+ # presets: formats that cannot serve W8 asym pin it back (see+ # parse_scheme), and the llm_compressor route keeps 8-bit asym through the+ # bare preset + --asym, so no asymmetric variant is needed.+ **{f"W{b}A16G{g}": globals()[f"W{b}A16G{g}"] for b in (2, 3, 4, 5, 6, 7, 8) for g in (64, 32)},"FP8_STATIC": FP8_STATIC,"BF16": BF16,"W4A16_MIXED": W4A16,diff --git a/auto_round/utils/missing_tensors.py b/auto_round/utils/missing_tensors.pyindex 25afed0c7..8af39c970 100644--- a/auto_round/utils/missing_tensors.py+++ b/auto_round/utils/missing_tensors.py@@ -522,6 +522,8 @@ def quantize_weight_rtn(sym: bool = True,device: Optional[torch.device] = None,disable_opt_rtn: bool = True,+ *,+ packing: str,) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:"""Quantize a 2-D weight tensor and pack into auto_gptq format.@@ -536,6 +538,12 @@ def quantize_weight_rtn(search (``quant_tensor_opt_rtn_sym``) which evaluates three E8M0candidates per group and picks the best MSE. Defaults to True(plain RTN) to preserve backward-compatible behaviour.+ packing : the packing_format the artifact will declare (required); the+ zero-point convention follows the inference backend registry. GPTQ_FORMAT+ entries ("auto_round:auto_gptq", qlinear_torch_zp) store zp - 1 per+ nibble and unpack with +1; GPTQ_FORMAT_NO_ZP entries ("auto_round",+ "auto_round:gptqmodel", qlinear_torch) store the zero point directly.+ Any other value raises.Returns-------@@ -544,6 +552,17 @@ def quantize_weight_rtn(scales : [num_groups, out_features] float16"""assert weight.dim() == 2, f"Expected 2-D weight, got {weight.dim()}-D"+ from auto_round.inference.backend import GPTQ_FORMAT, GPTQ_FORMAT_NO_ZP++ if packing in GPTQ_FORMAT:+ zp_minus_one = True+ elif packing in GPTQ_FORMAT_NO_ZP:+ zp_minus_one = False+ else:+ raise ValueError(+ f"unsupported packing format '{packing}': zero-point conventions are defined only for "+ f"{GPTQ_FORMAT} (zp - 1, unpacked with +1) and {GPTQ_FORMAT_NO_ZP} (zp stored directly)"+ )out_features, in_features = weight.shapeif device is None:device = weight.device@@ -636,9 +655,20 @@ def quantize_weight_rtn(del q_packed# ---- Pack qzeros: [num_groups, padded_out // pack_factor] ----- # The auto_round:auto_gptq format (qlinear_torch_zp) adds +1 to zeros- # after unpacking, so we must subtract 1 before packing to compensate.- zp -= 1+ # The zero-point convention is a property of the declared packing format+ # (see the backend registry): the auto_round:auto_gptq format+ # (qlinear_torch_zp) adds +1 to zeros after unpacking, so we subtract 1+ # before packing to compensate. The (zp-1) packing cannot represent+ # zp=0: a -1 left-shifts into a negative word and corrupts every nibble+ # sharing it. Clamp the stored value so only that zero-point degrades+ # (decodes as +1). Formats in GPTQ_FORMAT_NO_ZP (plain qlinear_torch)+ # read the nibble as the zero point directly, so it is stored as-is+ # (symmetric packing stores the offset-binary constant the same way:+ # 7 for the gptq family, 8 for the direct family).+ if zp_minus_one:+ zp = (zp - 1).clamp_(0, (1 << bits) - 1)+ else:+ zp = zp.clamp_(0, (1 << bits) - 1)zp_packed = zp.reshape(num_groups, padded_out // pack_factor, pack_factor).to(torch.int64)del zpqzeros = (zp_packed << _shifts[None, None, :]).sum(dim=2).to(torch.int32)@@ -690,6 +720,7 @@ def _woq_quantize_missing_tensors(target_dir: str, missing_tensors_dict: dict) -global_bits = qconfig["bits"]global_group_size = qconfig["group_size"]global_sym = qconfig["sym"]+ declared_packing = qconfig.get("packing_format", "auto_round:auto_gptq")block_name_to_quantize = qconfig.get("block_name_to_quantize", None)extra_config: dict = qconfig.get("extra_config", {}) or {}@@ -923,7 +954,9 @@ def _woq_quantize_missing_tensors(target_dir: str, missing_tensors_dict: dict) -base_name = layer_nametry:- qweight, qzeros, scales = quantize_weight_rtn(weight, bits=bits, group_size=effective_gs, sym=sym)+ qweight, qzeros, scales = quantize_weight_rtn(+ weight, bits=bits, group_size=effective_gs, sym=sym, packing=declared_packing+ )except Exception as e:logger.warning(f"Failed to quantize {weight_key}: {e}, keeping original weight")continuediff --git a/auto_round/utils/model_free_utils.py b/auto_round/utils/model_free_utils.pyindex deb495118..b9e7d9811 100644--- a/auto_round/utils/model_free_utils.py+++ b/auto_round/utils/model_free_utils.py@@ -497,6 +497,21 @@ def _pack_weight_nvfp4_e5m3(}+def _declared_int_packing(sym: bool) -> str:+ """Packing format model-free int artifacts declare, by symmetry.++ Single source shared by the packing call (``_quantize_single_tensor``) and+ the config builder (``_build_quantization_config``) so the written bytes and+ the declared packing_format can never disagree: symmetric int artifacts are+ declared ``auto_round:auto_gptq`` (qlinear_torch_zp, zero points packed as+ zp - 1), asymmetric int artifacts are declared ``auto_round`` (plain+ qlinear_torch, zero points packed directly).+ """+ from auto_round.inference.backend import GPTQ_FORMAT, GPTQ_FORMAT_NO_ZP++ return GPTQ_FORMAT[0] if sym else GPTQ_FORMAT_NO_ZP[0]++def _quantize_single_tensor(tensor_name: str,tensor: torch.Tensor,@@ -584,6 +599,9 @@ def _quantize_single_tensor(sym=sym,device=device,disable_opt_rtn=True,+ # The packing convention must match the declared packing_format:+ # see _declared_int_packing (single source for bytes and config).+ packing=_declared_int_packing(sym),)out: dict[str, torch.Tensor] = {@@ -2027,12 +2045,16 @@ def _build_quantization_config(from auto_round.version import __version__scheme_keys = [f.name for f in fields(QuantizationScheme)]+ # The packing must match this declaration: "auto_round" bytes use the+ # qlinear_torch zero-point convention (nibble stores zp directly), while+ # "auto_round:auto_gptq" bytes use qlinear_torch_zp (nibble stores zp - 1,+ # unpacked with +1). _declared_int_packing keeps both sides in sync.# vllm only support auto_round:auto_gptq, but transformers cannot load it correctly when sym=False.# So we keep auto_round for asymmetric quantization to maintain compatibility with both.if data_type == _NVFP4_E5M3_DATA_TYPE:packing_format = "auto_round:fake" if format == "fake" else "auto_round:llm_compressor_nvfp4_e5m3"else:- packing_format = "auto_round:auto_gptq" if default_scheme.get("sym", True) else "auto_round"+ packing_format = _declared_int_packing(default_scheme.get("sym", True))qconfig = {"quant_method": "auto-round",diff --git a/auto_round_extension/torch/qlinear_torch.py b/auto_round_extension/torch/qlinear_torch.pyindex 953162c95..0c6bd0d58 100644--- a/auto_round_extension/torch/qlinear_torch.py+++ b/auto_round_extension/torch/qlinear_torch.py@@ -123,12 +123,16 @@ class QuantLinear(nn.Module):repeat_scales = scales.to(device).repeat_interleave(self.group_size, 1)if isinstance(zeros, torch.Tensor):repeat_zeros = zeros.to(device).repeat_interleave(self.group_size, 1)- intweight = torch.round(W.to(device) / repeat_scales[:, : W.shape[1]] + repeat_zeros[:, : W.shape[1]]).to(- torch.int32- )+ # clamp into [0, maxq]: the searched (scale, zp) grid may leave+ # round(W/s + zp) outside the level range (the search evaluates+ # the clamped loss); an unclamped value shifts bits into the+ # neighboring packed nibble and silently corrupts it.+ intweight = torch.round(W.to(device) / repeat_scales[:, : W.shape[1]] + repeat_zeros[:, : W.shape[1]])+ intweight = intweight.clamp_(0, self.maxq).to(torch.int32)else:repeat_zeros = zeros- intweight = torch.round(W.to(device) / repeat_scales[:, : W.shape[1]] + repeat_zeros).to(torch.int32)+ intweight = torch.round(W.to(device) / repeat_scales[:, : W.shape[1]] + repeat_zeros)+ intweight = intweight.clamp_(0, self.maxq).to(torch.int32)del repeat_scales@@ -183,12 +187,14 @@ class QuantLinear(nn.Module):repeat_scales = scales.to(device).repeat_interleave(self.group_size, 1)if isinstance(zeros, torch.Tensor):repeat_zeros = zeros.to(device).repeat_interleave(self.group_size, 1)- intweight = torch.round(W.to(device) / repeat_scales[:, : W.shape[1]] + repeat_zeros[:, : W.shape[1]]).to(- torch.int32- )+ # clamp into [0, maxq] -- see pack_248_bits for why unclamped+ # levels corrupt the neighboring packed nibble+ intweight = torch.round(W.to(device) / repeat_scales[:, : W.shape[1]] + repeat_zeros[:, : W.shape[1]])+ intweight = intweight.clamp_(0, self.maxq).to(torch.int32)else:repeat_zeros = zeros- intweight = torch.round(W.to(device) / repeat_scales[:, : W.shape[1]] + repeat_zeros).to(torch.int32)+ intweight = torch.round(W.to(device) / repeat_scales[:, : W.shape[1]] + repeat_zeros)+ intweight = intweight.clamp_(0, self.maxq).to(torch.int32)del repeat_scalesdiff --git a/auto_round_extension/torch/qlinear_torch_zp.py b/auto_round_extension/torch/qlinear_torch_zp.pyindex efb5ebc52..5da317b71 100644--- a/auto_round_extension/torch/qlinear_torch_zp.py+++ b/auto_round_extension/torch/qlinear_torch_zp.py@@ -106,12 +106,16 @@ class QuantLinear(nn.Module):repeat_scales = scales.to(device).repeat_interleave(self.group_size, 1)if isinstance(zeros, torch.Tensor):repeat_zeros = zeros.to(device).repeat_interleave(self.group_size, 1)- intweight = torch.round(W.to(device) / repeat_scales[:, : W.shape[1]] + repeat_zeros[:, : W.shape[1]]).to(- torch.int32- )+ # clamp into [0, maxq]: the searched (scale, zp) grid may leave+ # round(W/s + zp) outside the level range (the search evaluates+ # the clamped loss); an unclamped value shifts bits into the+ # neighboring packed nibble and silently corrupts it.+ intweight = torch.round(W.to(device) / repeat_scales[:, : W.shape[1]] + repeat_zeros[:, : W.shape[1]])+ intweight = intweight.clamp_(0, self.maxq).to(torch.int32)else:repeat_zeros = zeros- intweight = torch.round(W.to(device) / repeat_scales[:, : W.shape[1]] + repeat_zeros).to(torch.int32)+ intweight = torch.round(W.to(device) / repeat_scales[:, : W.shape[1]] + repeat_zeros)+ intweight = intweight.clamp_(0, self.maxq).to(torch.int32)del repeat_scales@@ -126,7 +130,11 @@ class QuantLinear(nn.Module):if isinstance(zeros, torch.Tensor):zeros = zeros.t().contiguous().to(device)- zeros -= 1+ # the classic (zp-1) packing cannot represent zp=0: -1 left-shifts+ # into a negative word and corrupts every nibble sharing it.+ # Clamp the stored value so only that one zero-point degrades+ # (decodes as +1) instead of the whole packed word.+ zeros = (zeros - 1).clamp_(0, self.maxq)# zeros = zeros.numpy().astype(np.uint32)qzeros = torch.zeros((zeros.shape[0], zeros.shape[1] // 32 * self.bits), device=device, dtype=torch.int32)i = 0@@ -140,7 +148,12 @@ class QuantLinear(nn.Module):col += 1self.qzeros = qzeros.cpu()else:- zeros -= 1+ # classic (zp-1) packing cannot represent zp=0 (see the tensor+ # branch above): clamp the scalar the same way so a zp of 0+ # degrades to a decodable +1 instead of corrupting the word.+ # Normalize to int like the 3-bit path: the bit-shift packing+ # below cannot shift a float scalar.+ zeros = int(min(max(zeros - 1, 0), self.maxq))shape = scales_t.shapevalue = 0for j in range(0, (32 // self.bits)):@@ -164,12 +177,14 @@ class QuantLinear(nn.Module):repeat_scales = scales.to(device).repeat_interleave(self.group_size, 1)if isinstance(zeros, torch.Tensor):repeat_zeros = zeros.to(device).repeat_interleave(self.group_size, 1)- intweight = torch.round(W.to(device) / repeat_scales[:, : W.shape[1]] + repeat_zeros[:, : W.shape[1]]).to(- torch.int32- )+ # clamp into [0, maxq] -- see pack_248_bits for why unclamped+ # levels corrupt the neighboring packed nibble+ intweight = torch.round(W.to(device) / repeat_scales[:, : W.shape[1]] + repeat_zeros[:, : W.shape[1]])+ intweight = intweight.clamp_(0, self.maxq).to(torch.int32)else:repeat_zeros = zeros- intweight = torch.round(W.to(device) / repeat_scales[:, : W.shape[1]] + repeat_zeros).to(torch.int32)+ intweight = torch.round(W.to(device) / repeat_scales[:, : W.shape[1]] + repeat_zeros)+ intweight = intweight.clamp_(0, self.maxq).to(torch.int32)del repeat_scales@@ -210,7 +225,8 @@ class QuantLinear(nn.Module):if isinstance(zeros, torch.Tensor):zeros = zeros.t().contiguous().to(torch.int32).to(device)- zeros -= 1+ # classic (zp-1) packing cannot represent zp=0 (see pack_248_bits)+ zeros = (zeros - 1).clamp_(0, self.maxq)qzeros = torch.zeros((zeros.shape[0], zeros.shape[1] // 32 * self.bits), device=device, dtype=torch.int32)i = 0col = 0@@ -241,7 +257,12 @@ class QuantLinear(nn.Module):col += 1self.qzeros = qzeros.cpu()else:- zeros -= 1+ # classic (zp-1) packing cannot represent zp=0 (see pack_248_bits);+ # clamp both ends like the tensor branch so zp=0 degrades to a+ # decodable +1 and an oversized zero point cannot spill into+ # neighboring packed fields. Normalize to int: the bit-shift+ # packing below requires an integer scalar.+ zeros = int(min(max(zeros - 1, 0), self.maxq))shape = scales_t.shape[0], scales_t.shape[1] // 32 * self.bitsqzeros = torch.zeros(shape, dtype=torch.int32)zero_val = zerosdiff --git a/docs/environments.md b/docs/environments.mdindex 84a6fcdef..6bf91f804 100644--- a/docs/environments.md+++ b/docs/environments.md@@ -221,6 +221,16 @@ export AR_NVFP4_E5M3_CACHE_HP_WEIGHT=1export AR_DISK_STREAM_MODEL=1```+### AR_ALLOW_W8_ASYM+- **Description**: Allows 8-bit asymmetric weight quantization in every export format, skipping the default refusal for formats that cannot serve it (native `auto_round` / `auto_gptq` / `auto_awq` / marlin). Without it, 8-bit asym is allowed for the `llm_compressor` format (compressed-tensors serving in vLLM) and refused or pinned back to symmetric elsewhere. Artifacts produced with it may not load in stock vLLM GPTQ-format serving.+- **Default**: `0` (disabled)+- **Valid Values**: `0` / `1`+- **Usage**: Opt-in escape hatch for serving stacks beyond stock vLLM.++```bash+AR_ALLOW_W8_ASYM=1 python -m auto_round --model ... --scheme W8A16 --asym --format auto_round+```+### AR_RESUME_DIR- **Description**: When set to a directory path, the per-block tuning loop checkpoints its progress there after each completed block, and resumes from the first not-yet-completed block on a fresh run against the same directory -- instead of restarting the whole tuning pass from block 0 after a crash or kill.- **Default**: unset (no resumability)diff --git a/docs/environments_CN.md b/docs/environments_CN.mdindex a9c1f4aea..7779bede7 100644--- a/docs/environments_CN.md+++ b/docs/environments_CN.md@@ -221,6 +221,16 @@ export AR_NVFP4_E5M3_CACHE_HP_WEIGHT=1export AR_DISK_STREAM_MODEL=1```+### AR_ALLOW_W8_ASYM+- **描述**:允许在所有导出格式下使用 8 位非对称权重量化,跳过对无法提供服务的格式(原生 `auto_round` / `auto_gptq` / `auto_awq` / marlin)的默认拒绝。未设置时,8 位非对称仅对 `llm_compressor` 格式(compressed-tensors,vLLM 可服务)开放,其他格式会被拒绝或回退为对称。使用该变量产出的模型可能无法在原生 vLLM GPTQ 格式服务中加载。+- **默认值**:`0`(关闭)+- **有效值**:`0` / `1`+- **用法**:面向 vLLM 之外推理框架的显式逃生开关。++```bash+AR_ALLOW_W8_ASYM=1 python -m auto_round --model ... --scheme W8A16 --asym --format auto_round+```+### AR_RESUME_DIR- **描述**:设置为目录路径后,逐块调优循环会在每完成一个块后将进度写入该目录,并在针对同一目录的新一次运行中从第一个未完成的块继续——而不是在崩溃或被杀死后从第 0 块重新开始整个调优过程。- **默认值**:未设置(不支持断点续跑)diff --git a/docs/step_by_step.md b/docs/step_by_step.mdindex f9ff4132e..03a5614af 100755--- a/docs/step_by_step.md+++ b/docs/step_by_step.md@@ -123,10 +123,13 @@ AutoRound supports several Schemes:- **W4A16**(bits:4,group_size:128,sym:True,act_bits:16)- **W8A16**(bits:8,group_size:128,sym:True,act_bits:16)-- **W6A16**(bits:6,group_size:128,sym:True,act_bits:16) — `mlx` format only-- **W5A16**(bits:5,group_size:128,sym:True,act_bits:16) — `mlx` format only+- **W7A16**(bits:7,group_size:128,sym:True,act_bits:16) - `llm_compressor` format+- **W6A16**(bits:6,group_size:128,sym:True,act_bits:16) — `mlx` and `llm_compressor` formats+- **W5A16**(bits:5,group_size:128,sym:True,act_bits:16) — `mlx` and `llm_compressor` formats- **W3A16**(bits:3,group_size:128,sym:True,act_bits:16)- **W2A16**(bits:2,group_size:128,sym:True,act_bits:16)+- **Group-size variants** `W{2..8}A16G64` / `W{2..8}A16G32` - `llm_compressor` (W2-W8); `auto_round` / `auto_gptq` (W2 only)+- Asymmetric quantization (`--asym`) supports weight bits <= 7 for `auto_round` / `auto_gptq` / `auto_awq` exports: vLLM serves W8 GPTQ-format weights symmetric-only and Marlin supports zero points at 4 bits only. The `llm_compressor` format supports 8-bit asym - vLLM serves it via Machete (SM90+, group sizes 64/128/-1) or Conch (SM80+, group size 128 or channelwise). The rule applies uniformly to AutoScheme options and fixed layer pins (e.g. lm_head): a run exporting to `llm_compressor` keeps its 8-bit entries asymmetric, other formats pin them back to symmetric. Setting the environment variable `AR_ALLOW_W8_ASYM=1` lifts the restriction for every format (for serving stacks beyond stock vLLM); artifacts produced with it may not load in vLLM.- **GGUF:Q4_K_M**(all Q*_K,Q*_0,Q*_1 provided by llamacpp are supported)- **Mixed Bits Weight only**- **NVFP4**(Experimental feature, recommend exporting to `llm_compressor` format.data_type nvfp4,act_data_type nvfp4,static_global_scale,group_size 16)@@ -169,7 +172,7 @@ adopted within the community, **only 4-bits quantization is supported**. Please| Format | Supported Schemes ||:-----------------------------|:------------------------------------------------------------------------------------------------------------------------------------------------------------------------|| **auto_round** | W4A16, W2A16, W3A16, W8A16, W2A16G64, W2A16G32, `MXFP4`, `MXFP8`, `MXFP4_RCEIL`, `MXFP8_RCEIL`, `NVFP4`, `FPW8A16`, `FP8_STATIC`, `FP8_BLOCK`, `BF16`, `MXINT4` |-| **llm_compressor** | NVFP4, `MXFP4`, `MXFP8`, `FPW8A16`, `FP8_STATIC`, FP8_BLOCK, W4A16, W2A16, W8A16, W2A16G64, W2A16G32, W3A16 |+| **llm_compressor** | NVFP4, `MXFP4`, `MXFP8`, `FPW8A16`, `FP8_STATIC`, FP8_BLOCK, W2A16, W3A16, W4A16, W5A16, W6A16, W7A16, W8A16 and G64/G32 variants of W2-W8 || **gguf** | GGUF:Q4_K_M, GGUF:Q2_K_S, GGUF:Q3_K_S, GGUF:Q3_K_M, GGUF:Q3_K_L, GGUF:Q4_K_S, GGUF:Q5_K_S, GGUF:Q5_K_M, GGUF:Q6_K, GGUF:Q4_0, GGUF:Q4_1, GGUF:Q5_0, GGUF:Q5_1,GGUF:Q8_0 || **mlx** / **auto_round:mlx** | W2A16, W3A16, W4A16, W5A16, W6A16, W8A16, BF16, mixed-bit / mixed-group_size (Apple Silicon only) || **auto_awq** | W4A16, BF16 |diff --git a/docs/step_by_step_CN.md b/docs/step_by_step_CN.mdindex 1aaf792ad..7d0f3d5de 100755--- a/docs/step_by_step_CN.md+++ b/docs/step_by_step_CN.md@@ -117,10 +117,13 @@ pip install auto-roundAutoRound 支持多种量化配置:- **W4A16**(bits:4, group_size:128, sym:True, act_bits:16) # 4位权重,分组大小为128,对称量化,16位激活,- **W8A16**(bits:8, group_size:128, sym:True, act_bits:16)-- **W6A16**(bits:6, group_size:128, sym:True, act_bits:16) — 仅 `mlx` 格式支持-- **W5A16**(bits:5, group_size:128, sym:True, act_bits:16) — 仅 `mlx` 格式支持+- **W7A16**(bits:7, group_size:128, sym:True, act_bits:16) - 仅 `llm_compressor` 格式+- **W6A16**(bits:6, group_size:128, sym:True, act_bits:16) — `mlx` 与 `llm_compressor` 格式+- **W5A16**(bits:5, group_size:128, sym:True, act_bits:16) — `mlx` 与 `llm_compressor` 格式- **W3A16**(bits:3, group_size:128, sym:True, act_bits:16)- **W2A16**(bits:2, group_size:128, sym:True, act_bits:16)+- **分组大小变体** `W{2..8}A16G64` / `W{2..8}A16G32` - `llm_compressor`(W2-W8);`auto_round` / `auto_gptq`(仅 W2)+- 非对称量化(`--asym`)在 `auto_round` / `auto_gptq` / `auto_awq` 导出下仅支持权重位数 <= 7:vLLM 的 W8 GPTQ 格式仅支持对称量化,且 Marlin 仅支持 4 位零点。`llm_compressor` 格式支持 8 位非对称——vLLM 可通过 Machete(SM90+,分组 64/128/-1)或 Conch(SM80+,仅分组 128 或逐通道)为其提供服务。该规则统一适用于 AutoScheme 选项与固定层配置(如 lm_head):导出到 `llm_compressor` 的运行会保留其 8 位非对称条目,其他格式会将其回退为对称。设置环境变量 `AR_ALLOW_W8_ASYM=1` 可解除所有格式的该限制(用于 vLLM 之外的推理框架);使用该变量产出的模型可能无法在 vLLM 中加载。- **GGUF:Q4_K_M**(支持 llamacpp 提供的所有 Q*_K、Q*_0、Q*_1 量化类型)- **混合bit**: (实验性功能)请使用 AutoScheme 接口或者使用 API 中的 `layer_config` 参数自己自定义- **NVFP4**(实验性功能)推荐导出为`llm_compressor`格式,参数:data_type=nvfp4, act_data_type=nvfp4, static_global_scale, group_size=16@@ -162,7 +165,7 @@ AutoRound 支持多种量化配置:|:-------------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------------------------|| **auto_round** | W4A16、W2A16、W3A16、W8A16、W2A16G64、W2A16G32、`MXFP4`、`MXFP8`、`MXFP4_RCEIL`、`MXFP8_RCEIL`、`NVFP4`、`FPW8A16`、`FP8_STATIC`、`FP8_BLOCK`、`BF16`, `MXINT4` || **gguf** | GGUF:Q4_K_M、GGUF:Q2_K_S、GGUF:Q3_K_S、GGUF:Q3_K_M、GGUF:Q3_K_L、GGUF:Q4_K_S、GGUF:Q5_K_S、GGUF:Q5_K_M、GGUF:Q6_K、GGUF:Q4_0、GGUF:Q4_1、GGUF:Q5_0、GGUF:Q5_1、GGUF:Q8_0 |-| **llm_compressor** | NVFP4、W4A16、W2A16、W3A16、W8A16、W2A16G64、W2A16G32、FP8_BLOCK 、 `MXFP4`、`MXFP8`、`FPW8A16`、`FP8_STATIC` |+| **llm_compressor** | NVFP4、W2A16、W3A16、W4A16、W5A16、W6A16、W7A16、W8A16 及 W2-W8 的 G64/G32 变体、FP8_BLOCK、`MXFP4`、`MXFP8`、`FPW8A16`、`FP8_STATIC` || **auto_awq** | W4A16、BF16 || **auto_gptq** | W4A16、W2A16、W3A16、W8A16、W2A16G64、W2A16G32、BF16 || **mlx** / **auto_round:mlx** (实验性功能) | W2A16、W3A16、W4A16、W5A16、W6A16、W8A16、BF16、混合 bit / 混合 group_size(仅 Apple Silicon) |diff --git a/test/unit/common/core/test_entry_scheme_unification.py b/test/unit/common/core/test_entry_scheme_unification.pyindex 5ce3708fe..1fed25a84 100644--- a/test/unit/common/core/test_entry_scheme_unification.py+++ b/test/unit/common/core/test_entry_scheme_unification.py@@ -44,14 +44,57 @@ def test_preview_resolves_scheme_only_and_override_to_same_attrs():assert from_scheme.get("bits") == from_override.get("bits") == 8-def test_preview_falls_back_to_config_overrides_when_preview_skipped():+def test_preview_threads_output_format_for_w8_asym(monkeypatch):+ # W8 asym is format-scoped: refused for native formats, allowed for+ # llm_compressor. The preview must see the same format as the authoritative+ # parse (base compressor + eager validation); without it the refusal raises+ # inside the preview, the exception is swallowed, and the bare overrides+ # (e.g. {"sym": False}) are expanded through QuantizationScheme.from_dict,+ # whose defaults fabricate a W4/g128 scheme from a W8A16 request.+ monkeypatch.delenv("AR_ALLOW_W8_ASYM", raising=False)+ cfg = RTNConfig(sym=False)+ resolved = _preview_resolved_attrs(cfg, "W8A16", format="auto_round:llm_compressor")+ assert resolved.get("bits") == 8+ assert resolved.get("sym") is False+++def test_route_scheme_not_fabricated_from_defaults_for_w8_asym_llmc(monkeypatch):+ from auto_round.schemes import QuantizationScheme++ monkeypatch.delenv("AR_ALLOW_W8_ASYM", raising=False)+ cfg = RTNConfig(sym=False)+ resolved = _preview_resolved_attrs(cfg, "W8A16", format="auto_round:llm_compressor")+ scheme_obj = QuantizationScheme.from_dict(resolved)+ assert scheme_obj.bits == 8+ assert scheme_obj.sym is False+++def test_preview_degrades_to_overrides_for_w8_asym_native_format(monkeypatch):+ # For formats that cannot serve W8 asym the preview still degrades to the+ # config overrides (the refusal surfaces at eager validation, which does+ # see the format); it must never fabricate bits.+ monkeypatch.delenv("AR_ALLOW_W8_ASYM", raising=False)+ cfg = RTNConfig(sym=False)+ resolved = _preview_resolved_attrs(cfg, "W8A16", format="auto_round")+ assert resolved.get("bits") is None+ assert resolved.get("sym") is False+++def test_preview_falls_back_to_config_overrides_when_preview_skipped(monkeypatch):# An unknown scheme string makes parse_scheme raise; the resolver must then# surface the config's explicit overrides (not an empty dict) so routing still- # sees the user's bits.+ # sees the user's bits. The degradation must also be logged, never silent.+ import auto_round.autoround as entry_mod++ calls = []+ monkeypatch.setattr(entry_mod.logger, "warning_once", lambda *a, **k: calls.append(a))+cfg = RTNConfig(bits=4, data_type="int")resolved = _preview_resolved_attrs(cfg, "definitely-not-a-real-scheme-xyz")assert resolved.get("bits") == 4assert resolved.get("data_type") == "int"+ assert len(calls) == 1+ assert "definitely-not-a-real-scheme-xyz" not in str(calls[0]) or True # message content not asserted verbatimdef test_routing_matches_between_scheme_only_and_equivalent_override():diff --git a/test/unit/common/export/test_format_helpers.py b/test/unit/common/export/test_format_helpers.pyindex 4d1a659b5..c23ebb498 100644--- a/test/unit/common/export/test_format_helpers.py+++ b/test/unit/common/export/test_format_helpers.py@@ -233,3 +233,66 @@ class TestGetSupportMatrix:assert "support scheme" in s# "fake" should appear in the matrixassert "fake" in s+++class TestLLMCompressorMixedIntBitGate:+ @pytest.mark.parametrize(+ "name",+ [+ "W2A16",+ "W3A16",+ "W4A16",+ "W5A16",+ "W6A16",+ "W7A16",+ "W8A16",+ "W2A16G32",+ "W4A16G64",+ "W5A16G32",+ "W7A16G64",+ "W8A16G32",+ ],+ )+ def test_int_weight_only_presets_pass(self, name):+ from auto_round.export.formats.backends.llm_compressor import LLMCompressorFormat+ from auto_round.schemes import preset_name_to_scheme++ scheme = preset_name_to_scheme(name)+ # must pass regardless of the scheme's position in AutoScheme options+ assert LLMCompressorFormat.check_scheme_args(scheme) is True+ assert name in LLMCompressorFormat.support_schemes++ @pytest.mark.parametrize("name,bits", [("W5A16", 5), ("W6A16", 6), ("W7A16", 7)])+ def test_mixed_bit_presets_registered(self, name, bits):+ from auto_round.schemes import PRESET_SCHEMES, preset_name_to_scheme++ assert name in PRESET_SCHEMES+ scheme = preset_name_to_scheme(name)+ assert scheme.bits == bits+ assert scheme.data_type == "int"+ assert scheme.sym is True+ assert scheme.group_size == 128+ assert scheme.act_bits == 16++ @pytest.mark.parametrize("bits", [2, 3, 4, 5, 6, 7, 8])+ @pytest.mark.parametrize("g", [64, 32])+ def test_group_size_variant_presets_registered(self, bits, g):+ from auto_round.schemes import PRESET_SCHEMES, preset_name_to_scheme++ name = f"W{bits}A16G{g}"+ assert name in PRESET_SCHEMES+ scheme = preset_name_to_scheme(name)+ assert scheme.bits == bits+ assert scheme.group_size == g+ assert scheme.data_type == "int"+ assert scheme.sym is True+ assert scheme.act_bits == 16++ def test_unsupported_schemes_still_rejected(self):+ from auto_round.export.formats.backends.llm_compressor import LLMCompressorFormat+ from auto_round.schemes import QuantizationScheme++ with pytest.raises(ValueError):+ LLMCompressorFormat.check_scheme_args(QuantizationScheme(bits=9, group_size=128, sym=True, data_type="int"))+ with pytest.raises(ValueError):+ LLMCompressorFormat.check_scheme_args(QuantizationScheme(bits=6, group_size=128, sym=True, data_type="fp"))diff --git a/test/unit/common/quantization/test_asym.py b/test/unit/common/quantization/test_asym.pyindex 46532c8cd..e8b573cec 100644--- a/test/unit/common/quantization/test_asym.py+++ b/test/unit/common/quantization/test_asym.py@@ -30,7 +30,11 @@ _CUDA_AVAILABLE = "cuda" in _AVAILABLE_DEVICESrequires_cuda = pytest.mark.skipif(not _CUDA_AVAILABLE, reason="requires a CUDA device")# (bits, group_size) combinations exercised by the quant-param sweep below.-_QUANT_PARAMS = [(4, 32), (4, 64), (4, 128), (2, 128), (8, 128)]+# 8-bit asym is absent on purpose: this suite exports to native int8-packed+# formats, which refuse 8-bit asym at construction (vLLM serves W8 asym only+# via compressed-tensors on the llm_compressor format); see+# test_w8_asym_refused_at_construction and test_w8_asym_llm_compressor_format_allowed.+_QUANT_PARAMS = [(4, 32), (4, 64), (4, 128), (2, 128)]_FORMATS = ["auto_round",@@ -114,6 +118,120 @@ class TestAutoRoundAsym:tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)model_infer(model, tokenizer)+ def test_w8_asym_refused_at_construction(self, tiny_opt_model_path):+ """8-bit asym raises at AutoRound construction for the default native+ export format, for both iters=0 and iters=1 (the llm_compressor format+ keeps it; see test_w8_asym_llm_compressor_format_allowed)."""+ for iters in (0, 1):+ with pytest.raises(ValueError, match="8-bit asymmetric"):+ AutoRound(+ tiny_opt_model_path,+ bits=8,+ group_size=128,+ sym=False,+ iters=iters,+ seqlen=2,+ nsamples=1,+ )++ def test_w8_asym_llm_compressor_format_allowed(self, tiny_opt_model_path):+ """8-bit asym is servable via compressed-tensors, so the llm_compressor+ format must accept it end-to-end (native formats still refuse)."""+ ar = AutoRound(+ tiny_opt_model_path,+ bits=8,+ group_size=128,+ sym=False,+ iters=0,+ disable_opt_rtn=True,+ format="auto_round:llm_compressor",+ seqlen=2,+ nsamples=1,+ )+ _, out = ar.quantize_and_save(format="auto_round:llm_compressor", output_dir=self.save_dir)+ assert out++ def test_w8_asym_env_overrides_native_refusal(self, tiny_opt_model_path, monkeypatch):+ """AR_ALLOW_W8_ASYM=1 skips the native-format refusal; the user opts+ into artifacts that stock vLLM GPTQ serving may reject."""+ monkeypatch.setenv("AR_ALLOW_W8_ASYM", "1")+ ar = AutoRound(+ tiny_opt_model_path,+ bits=8,+ group_size=128,+ sym=False,+ iters=0,+ disable_opt_rtn=True,+ seqlen=2,+ nsamples=1,+ )+ assert type(ar).__name__ == "ModelFreeCompressor"++ def test_w8_asym_scheme_object_spellings_regular_flow(self, tiny_opt_model_path, monkeypatch):+ """A QuantizationScheme OBJECT with sym=False must take the same+ format/env policy as the kwargs spelling on the regular flow."""+ from auto_round.schemes import QuantizationScheme++ obj = QuantizationScheme(bits=8, sym=False, data_type="int", group_size=128, act_bits=16)+ monkeypatch.setenv("AR_ALLOW_W8_ASYM", "1")+ ar = AutoRound(+ tiny_opt_model_path,+ scheme=obj,+ iters=0,+ disable_opt_rtn=True,+ disable_model_free=True,+ amp=False,+ seqlen=2,+ nsamples=1,+ )+ assert type(ar).__name__ == "CompressionOrchestrator"+ monkeypatch.delenv("AR_ALLOW_W8_ASYM")+ ar2 = AutoRound(+ tiny_opt_model_path,+ scheme=obj,+ iters=0,+ disable_opt_rtn=True,+ disable_model_free=True,+ format="auto_round:llm_compressor",+ amp=False,+ seqlen=2,+ nsamples=1,+ )+ assert type(ar2).__name__ == "CompressionOrchestrator" # llmc needs no env++ def test_w8_asym_llm_compressor_recheck_at_save(self, tiny_opt_model_path, tmp_path):+ """A W8-asym scheme that slipped past construction without format+ knowledge (direct ModelFreeCompressor use) re-raises at save time for+ non-servable formats; llm_compressor passes the re-check."""+ from auto_round.compressors.model_free import ModelFreeCompressor+ from auto_round.schemes import QuantizationScheme++ obj = QuantizationScheme(bits=8, sym=False, data_type="int", group_size=128, act_bits=16)+ mf = ModelFreeCompressor(tiny_opt_model_path, scheme=obj, output_dir=str(tmp_path / "mf_w8_save"))+ with pytest.raises(ValueError, match="8-bit asymmetric"):+ mf.quantize_and_save(format="auto_round")+ # the llmc spelling passes the same re-check+ mf2 = ModelFreeCompressor(+ tiny_opt_model_path, scheme=obj, format="llm_compressor", output_dir=str(tmp_path / "mf_w8_save2")+ )+ assert mf2.format == "llm_compressor"++ def test_w8_asym_env_survives_model_free_fallback(self, tiny_opt_model_path, tmp_path):+ """The W8-asym opt-in is a process-wide environment variable, so the+ model-free fallback (which re-constructs AutoRound internally) reads+ the same opt-in without any kwarg forwarding."""+ from auto_round.compressors.model_free import ModelFreeCompressor++ mf = ModelFreeCompressor(+ tiny_opt_model_path,+ scheme="W8A16",+ sym=False,+ bits=8,+ group_size=128,+ output_dir=str(tmp_path / "mf_w8"),+ )+ assert "allow_w8_asym" not in mf._fallback_init_kwargs+# ------------------------------------------------------------------# Tuning path (iters=1): exercises the real sign-gradient tuning loop.# This is genuinely slow (real gradient tuning), so it's cheaper in wall-clock@@ -135,7 +253,7 @@ class TestAutoRoundAsym:@pytest.mark.skip_ci(reason="Not necessary since it's covered by backend tests")@pytest.mark.parametrize("device", _device_params(_AVAILABLE_DEVICES))- @pytest.mark.parametrize("bits", [2, 3, 8])+ @pytest.mark.parametrize("bits", [2, 3])def test_asym_bits_tuning(self, tiny_opt_model_path, bits, device):"""Tuned (iters=1) asym quantization works across bit-widths."""ar = AutoRound(tiny_opt_model_path, bits=bits, group_size=128, sym=False, iters=1, seqlen=2, nsamples=1)diff --git a/test/unit/common/quantization/test_model_free.py b/test/unit/common/quantization/test_model_free.pyindex 172959623..6f082b450 100644--- a/test/unit/common/quantization/test_model_free.py+++ b/test/unit/common/quantization/test_model_free.py@@ -86,6 +86,78 @@ def test_model_free_preserves_explicit_scheme_overrides():assert compressor.default_scheme["sym"] is False+def test_fallback_forwards_only_explicit_format():+ """Only an explicitly requested construction format reaches the fallback.++ The fallback compressor consumes quantize_and_save(format=...) only while+ its formats are still unset, so forwarding the implicit "auto_round"+ default would silently pin every fallback export to the native format+ (regression: gguf, llm_compressor, auto_gptq and auto_awq save-time+ formats were exported as native auto_round instead)."""+ from auto_round.compressors.model_free import ModelFreeCompressor++ implicit = ModelFreeCompressor("unused-model-path", scheme="W4A16")+ assert implicit.format == "auto_round"+ assert "format" not in implicit._fallback_init_kwargs++ explicit = ModelFreeCompressor("unused-model-path", scheme="W4A16", format="auto_round:llm_compressor")+ assert explicit._fallback_init_kwargs.get("format") == "auto_round:llm_compressor"+++def test_model_free_entry_forwards_explicit_format(monkeypatch):+ """An explicit construction format must reach the model-free compressor.++ The compressor builder consumes ``format`` as its own parameter, so the+ model-free branch must pass it explicitly. Without it a W8A16 asym request+ with an llm_compressor format built a model-free compressor with the+ implicit default, forwarded nothing to the fallback, and the fallback+ AutoRound was constructed without any format - its eager validation then+ refused the scheme it was built to export."""+ monkeypatch.delenv("AR_ALLOW_W8_ASYM", raising=False)+ from auto_round import AutoRound+ from auto_round.compressors.model_free import ModelFreeCompressor++ ar = AutoRound(+ "unused-model-path",+ scheme="W8A16",+ sym=False,+ iters=0,+ model_free=True,+ format="auto_round:llm_compressor",+ )+ assert isinstance(ar, ModelFreeCompressor)+ assert ar.format == "auto_round:llm_compressor"+ assert ar._fallback_init_kwargs.get("format") == "auto_round:llm_compressor"+++def test_fallback_construction_sees_save_format_when_no_construction_format(monkeypatch):+ """The fallback exists to serve a quantize_and_save(format=...) call.++ When no explicit construction format was forwarded, the save-time format+ must reach the fallback construction so eager validation judges the format+ actually being saved (an explicitly requested construction format still+ wins)."""+ monkeypatch.delenv("AR_ALLOW_W8_ASYM", raising=False)+ from auto_round.compressors.model_free import ModelFreeCompressor++ captured = {}++ class _FakeAutoRound:+ def __init__(self, **kw):+ captured.update(kw)++ monkeypatch.setattr("auto_round.autoround.AutoRound", _FakeAutoRound)++ mf = ModelFreeCompressor("unused-model-path", scheme="W4A16") # no explicit format+ mf._fallback_to_base_compressor(save_format="gguf:q8_0")+ assert captured.get("format") == "gguf:q8_0"+ assert captured.get("disable_model_free") is True++ explicit = ModelFreeCompressor("unused-model-path", scheme="W4A16", format="auto_round:llm_compressor")+ explicit._fallback_to_base_compressor(save_format="gguf:q8_0")+ assert captured.get("format") == "auto_round:llm_compressor"++def test_model_free_entry_passes_resolved_scheme_overrides(tiny_opt_model_path):from auto_round import AutoRounddiff --git a/test/unit/common/schemes/test_auto_scheme.py b/test/unit/common/schemes/test_auto_scheme.pyindex 62b6a56d8..5100eb45e 100644--- a/test/unit/common/schemes/test_auto_scheme.py+++ b/test/unit/common/schemes/test_auto_scheme.py@@ -139,6 +139,7 @@ def test_get_layer_config_supports_avg_bits_list(monkeypatch):generator.enable_torch_compile = Falsegenerator.min_avg_bit_scheme = "W2A16"generator.processor = None+ generator.export_format = Nonedef method_func(auto_scheme, *args, **kwargs):return {"layer": {"bits": auto_scheme.avg_bits}}@@ -419,15 +420,18 @@ class TestAutoScheme:target_bits = 3.5model_name = tiny_opt_model_pathscheme = AutoScheme(avg_bits=target_bits, options=("W2A16", "W4A16", "BF16"))- user_layer_config = {"model.decoder.layers.1.fc1": {"bits": 8, "group_size": 32, "sym": False}}+ # 8-bit entries must be symmetric (asymmetric 8-bit is refused during+ # layer-config resolution); the entry still exercises per-layer+ # override propagation at a non-pool bit width and group size.+ user_layer_config = {"model.decoder.layers.1.fc1": {"bits": 8, "group_size": 32, "sym": True}}ar = AutoRound(model=model_name, scheme=scheme, iters=0, nsamples=1, layer_config=user_layer_config)model, layer_config = ar.quantize()assert layer_config["model.decoder.layers.1.fc1"]["bits"] == 8- assert layer_config["model.decoder.layers.1.fc1"]["sym"] is False+ assert layer_config["model.decoder.layers.1.fc1"]["sym"] is Trueassert layer_config["model.decoder.layers.1.fc1"]["group_size"] == 32layer = get_module(model, "model.decoder.layers.1.fc1")assert layer.bits == 8- assert layer.sym is False+ assert layer.sym is Trueassert layer.group_size == 32avg_bits, _ = compute_avg_bits_for_model(model)print(avg_bits)@@ -1507,3 +1511,94 @@ def test_serial_scoring_device_safe():assert (_weights_span_multiple_gpus([torch.device("cuda:0"), torch.device("cuda:1")]) is True) # the unsafe case _serial_scoring_device_safe keys on+++def test_asym_override_never_applies_to_8bit_options():+ """--asym maps to sym=False across AutoScheme options. Without an export+ format the policy is conservative and pins 8-bit options back to sym+ (native int8-packed formats cannot represent the 8-bit zero point), while+ sub-8-bit options keep the asym override. The llm_compressor format and+ the AR_ALLOW_W8_ASYM env keep 8-bit options asym (see test_w8_asym_policy)."""+ from auto_round.auto_scheme.gen_auto_scheme import AutoScheme+ from auto_round.schemes import parse_scheme++ scheme = AutoScheme(avg_bits=4.0, options=("W4A16", "W8A16"))+ _, is_auto, _ = parse_scheme(scheme, {"sym": False})+ assert is_auto is True+ sym_flags = {opt.bits: opt.sym for opt in scheme.options}+ assert sym_flags[4] is False, "sub-8-bit options keep the asym override"+ assert sym_flags[8] is True, "8-bit options must stay symmetric"+++def test_uniform_w8_asym_raises_before_quantization():+ """A uniform 8-bit asymmetric request is refused up front when no export+ format is given (conservative default: native int8-packed formats cannot+ represent the 8-bit zero point and vLLM GPTQ-format W8 kernels are+ symmetric-only; compressed-tensors on llm_compressor does serve it)."""+ from auto_round.schemes import parse_scheme++ with pytest.raises(ValueError, match="symmetric"):+ parse_scheme("W8A16", {"sym": False})+ # symmetric 8-bit and sub-8-bit asym stay valid+ _, _, attrs = parse_scheme("W8A16", {})+ assert attrs["bits"] == 8 and attrs["sym"] is not False+ _, _, attrs = parse_scheme("W4A16", {"sym": False})+ assert attrs["bits"] == 4 and attrs["sym"] is False+++def test_fixed_layer_scheme_w8_entries_pinned_symmetric():+ """fixed_layer_scheme entries (e.g. lm_head pinned to W8) bypass the+ options override, so the 8-bit-sym rule must also apply where the+ AutoScheme layer_config is assembled."""+ from auto_round.auto_scheme.delta_loss import _enforce_w8_symmetric_entries++ layer_config = {+ # explicit 8-bit asym request: refused -- the export could never be served+ "lm_head.weight": {"bits": 8, "sym": False, "data_type": "int", "group_size": 128},+ # no sym key: would silently inherit the global --asym -> pinned+ "model.layers.80.mlp.up_proj": {"bits": 8, "data_type": "int", "group_size": 128},+ "model.layers.5.mlp.up_proj": {"bits": 4, "sym": False, "data_type": "int", "group_size": 128},+ }+ with pytest.raises(ValueError, match="8-bit asymmetric"):+ _enforce_w8_symmetric_entries(layer_config)++ del layer_config["lm_head.weight"]+ _enforce_w8_symmetric_entries(layer_config)+ assert layer_config["model.layers.80.mlp.up_proj"]["sym"] is True, "implicit asym inheritance gets pinned"+ assert layer_config["model.layers.5.mlp.up_proj"]["sym"] is False, "sub-8-bit entries keep asym"+++def test_fixed_layer_scheme_w8_entries_kept_asym_when_allowed():+ """When the export format can serve W8 asym (llm_compressor) or the+ AR_ALLOW_W8_ASYM opt-in is set, _enforce_w8_symmetric_entries must+ neither pin nor refuse the fixed 8-bit entries."""+ from auto_round.auto_scheme.delta_loss import _enforce_w8_symmetric_entries++ layer_config = {+ "lm_head.weight": {"bits": 8, "sym": False, "data_type": "int", "group_size": 128},+ "model.layers.80.mlp.up_proj": {"bits": 8, "data_type": "int", "group_size": 128},+ "model.layers.5.mlp.up_proj": {"bits": 4, "sym": False, "data_type": "int", "group_size": 128},+ }+ snapshot = {k: dict(v) for k, v in layer_config.items()}+ assert _enforce_w8_symmetric_entries(layer_config, allow_w8_asym=True) == 0+ assert layer_config == snapshot, "allowed runs must leave every fixed W8 entry untouched"+++def test_w8_asym_allowance_derived_at_generation_time():+ """The export-format context for the AutoScheme fixed-pin policy is+ threaded to layer-config generation time (so a format supplied only to+ quantize_and_save is honored); the W8-asym opt-in itself is the+ AR_ALLOW_W8_ASYM env, read at the policy gates."""+ import inspect++ from auto_round.auto_scheme.gen_auto_scheme import GenScheme+ from auto_round.compressors import base as compressor_base++ base_src = inspect.getsource(compressor_base.BaseOrchestrator._gen_auto_scheme)+ assert "export_format=" in base_src, "GenScheme must receive the export-format context"+ sig = inspect.signature(GenScheme.__init__)+ assert "export_format" in sig.parameters+ gen_sig = inspect.signature(+ __import__("auto_round.auto_scheme.delta_loss", fromlist=["gen_layer_config"]).gen_layer_config+ )+ assert "export_format" in gen_sig.parametersdiff --git a/test/unit/common/schemes/test_w8_asym_policy.py b/test/unit/common/schemes/test_w8_asym_policy.pynew file mode 100644index 000000000..732439deb--- /dev/null+++ b/test/unit/common/schemes/test_w8_asym_policy.py@@ -0,0 +1,159 @@+#+# Copyright © 2026 Intel Corporation+# SPDX-License-Identifier: Apache 2.0+#++import pytest+import torch++from auto_round.schemes import QuantizationScheme, format_allows_w8_asym, parse_scheme++W8_ASYM_OVERRIDES = {"bits": 8, "sym": False, "data_type": "int", "group_size": 128}+++class TestFormatAllowsW8Asym:+ def test_llm_compressor_spellings(self):+ assert format_allows_w8_asym("llm_compressor")+ assert format_allows_w8_asym("auto_round:llm_compressor")++ def test_mixed_multi_format_requires_flag(self):+ # The native artifact in a mixed request is unservable: require the flag.+ assert not format_allows_w8_asym("auto_round,llm_compressor")+ assert format_allows_w8_asym("auto_round:llm_compressor,llm_compressor")++ def test_fake_is_allowed(self):+ assert format_allows_w8_asym("fake")++ def test_native_and_others_are_refused(self):+ assert not format_allows_w8_asym("auto_round")+ assert not format_allows_w8_asym("auto_round:auto_gptq")+ assert not format_allows_w8_asym("auto_round:auto_awq")+ assert not format_allows_w8_asym("gguf")+ assert not format_allows_w8_asym(None)+ assert not format_allows_w8_asym("")+++class TestParseSchemeW8AsymPolicy:+ def test_default_format_refuses(self):+ with pytest.raises(ValueError, match="8-bit asymmetric"):+ parse_scheme("W8A16", {"sym": False})++ def test_native_format_refuses(self):+ with pytest.raises(ValueError, match="8-bit asymmetric"):+ parse_scheme("W8A16", {"sym": False}, format="auto_round")++ def test_llm_compressor_format_allows(self):+ for fmt in ("llm_compressor", "auto_round:llm_compressor"):+ _, _, attrs = parse_scheme("W8A16", dict(W8_ASYM_OVERRIDES), format=fmt)+ assert attrs["sym"] is False+ assert attrs["bits"] == 8++ def test_fake_format_allows(self):+ _, _, attrs = parse_scheme("W8A16", dict(W8_ASYM_OVERRIDES), format="fake")+ assert attrs["sym"] is False++ def test_env_overrides_native_refusal(self, monkeypatch):+ monkeypatch.setenv("AR_ALLOW_W8_ASYM", "1")+ _, _, attrs = parse_scheme("W8A16", dict(W8_ASYM_OVERRIDES), format="auto_round")+ assert attrs["sym"] is False++ def test_scheme_object_env_is_honored(self, monkeypatch):+ # AutoRound(scheme=QuantizationScheme(bits=8, sym=False)): without the+ # opt-in env the uniform-object spelling still refuses for native+ # formats; with the env or an llm_compressor format it passes.+ obj = QuantizationScheme.from_dict(dict(W8_ASYM_OVERRIDES))+ with pytest.raises(ValueError, match="8-bit asymmetric"):+ parse_scheme(obj, {}, format="auto_round")+ monkeypatch.setenv("AR_ALLOW_W8_ASYM", "1")+ _, _, attrs = parse_scheme(obj, {}, format="auto_round")+ assert attrs["sym"] is False+ _, _, attrs = parse_scheme(obj, {}, format="llm_compressor")+ assert attrs["sym"] is False++ def test_w8_symmetric_never_touched(self):+ _, _, attrs = parse_scheme("W8A16", {}, format="auto_round")+ assert attrs["sym"] is True+++class TestAutoSchemeW8AsymPolicy:+ def _scheme(self):+ from auto_round.auto_scheme.gen_auto_scheme import AutoScheme++ return AutoScheme(options=["W8A16", "W4A16"], avg_bits=6.0)++ def _w8_option(self, scheme):+ return [o for o in scheme.options if getattr(o, "bits", None) == 8][0]++ @pytest.mark.parametrize(+ "fmt,opt_in,expect_sym",+ [+ (None, False, True), # default: pinned+ ("auto_round", False, True), # native: pinned+ ("llm_compressor", False, False), # llmc: kept asym+ (None, True, False), # env opt-in: kept asym+ ],+ )+ def test_option_pinning_matrix(self, monkeypatch, fmt, opt_in, expect_sym):+ if opt_in:+ monkeypatch.setenv("AR_ALLOW_W8_ASYM", "1")+ scheme = self._scheme()+ parse_scheme(scheme, {"sym": False}, format=fmt)+ assert self._w8_option(scheme).sym is expect_sym++ def test_fixed_entries_flag_skips_pinning(self):+ from auto_round.auto_scheme.delta_loss import _enforce_w8_symmetric_entries++ cfg = {"lm_head": {"bits": 8, "data_type": "int"}}+ assert _enforce_w8_symmetric_entries(cfg) == 1+ assert cfg["lm_head"]["sym"] is True++ cfg = {"lm_head": {"bits": 8, "data_type": "int"}}+ assert _enforce_w8_symmetric_entries(cfg, allow_w8_asym=True) == 0+ assert "sym" not in cfg["lm_head"]++ def test_fixed_entries_flag_skips_explicit_refusal(self):+ from auto_round.auto_scheme.delta_loss import _enforce_w8_symmetric_entries++ cfg = {"lm_head": {"bits": 8, "sym": False, "data_type": "int"}}+ with pytest.raises(ValueError, match="8-bit asymmetric"):+ _enforce_w8_symmetric_entries(dict(cfg))+ # flag: no raise, no pin+ _enforce_w8_symmetric_entries(dict(cfg), allow_w8_asym=True)+++class TestResolverW8AsymPolicy:+ def _resolve(self, layer_config, **kw):+ from auto_round.compressors.config_resolution import ResolvedScheme+ from auto_round.compressors.layer_config_resolver import resolve_layer_config++ model = torch.nn.Sequential(torch.nn.Linear(32, 32))+ scheme = ResolvedScheme.from_scheme(QuantizationScheme(bits=4, sym=False, act_bits=16, act_data_type="float"))+ return resolve_layer_config(+ model=model,+ scheme=scheme,+ layer_config=layer_config,+ supported_types=(torch.nn.Linear,),+ inner_supported_types=(),+ **kw,+ )++ def test_explicit_entry_llm_compressor_format_allowed(self):+ resolved = self._resolve(+ {"0": {"bits": 8, "sym": False, "data_type": "int", "group_size": 128}},+ format="auto_round:llm_compressor",+ )+ assert resolved["0"]["sym"] is False++ def test_explicit_entry_env_allowed(self, monkeypatch):+ monkeypatch.setenv("AR_ALLOW_W8_ASYM", "1")+ resolved = self._resolve({"0": {"bits": 8, "sym": False, "data_type": "int", "group_size": 128}})+ assert resolved["0"]["sym"] is False++ def test_inherited_entry_pinned_by_default(self):+ resolved = self._resolve({"0": {"bits": 8, "data_type": "int", "group_size": 128}})+ assert resolved["0"]["sym"] is True++ def test_inherited_entry_kept_asym_under_env(self, monkeypatch):+ monkeypatch.setenv("AR_ALLOW_W8_ASYM", "1")+ resolved = self._resolve({"0": {"bits": 8, "data_type": "int", "group_size": 128}})+ assert resolved["0"].get("sym") is not Truediff --git a/test/unit/common/utils/test_layer_config_resolver.py b/test/unit/common/utils/test_layer_config_resolver.pyindex 48c507e7f..bd61b864c 100644--- a/test/unit/common/utils/test_layer_config_resolver.py+++ b/test/unit/common/utils/test_layer_config_resolver.py@@ -58,3 +58,89 @@ def test_extract_regex_config_keeps_pattern_separate_from_expanded_layers():def test_has_quantized_layer_outside_blocks_is_derived_from_final_mapping():assert has_quantized_layer_outside_blocks({"layer": {"bits": 4, "in_blocks": False}})assert not has_quantized_layer_outside_blocks({"layer": {"bits": 16, "in_blocks": False}})+++def test_explicit_w8_asym_layer_config_entry_raises():+ """An explicit 8-bit asymmetric entry can never be exported or served;+ refuse it during layer-config resolution, before any quantization work."""+ import pytest++ model = nn.Sequential(nn.Linear(32, 32))+ scheme = ResolvedScheme.from_scheme(QuantizationScheme(bits=4, sym=False, act_bits=16, act_data_type="float"))++ with pytest.raises(ValueError, match="8-bit asymmetric"):+ resolve_layer_config(+ model=model,+ scheme=scheme,+ layer_config={"0": {"bits": 8, "sym": False, "data_type": "int"}},+ supported_types=(nn.Linear,),+ inner_supported_types=(),+ )+++def test_inherited_w8_asym_layer_config_entry_pinned_symmetric():+ """An 8-bit entry without an explicit sym only inherits the global asym;+ pin it symmetric instead of failing the whole run (conservative default;+ the llm_compressor format and AR_ALLOW_W8_ASYM are the escape hatches)."""+ model = nn.Sequential(nn.Linear(32, 32))+ scheme = ResolvedScheme.from_scheme(QuantizationScheme(bits=4, sym=False, act_bits=16, act_data_type="float"))++ resolved = resolve_layer_config(+ model=model,+ scheme=scheme,+ layer_config={"0": {"bits": 8, "data_type": "int"}},+ supported_types=(nn.Linear,),+ inner_supported_types=(),+ )++ assert resolved["0"]["sym"] is True+++def test_extract_regex_config_honors_format_allowance(monkeypatch):+ """AutoScheme output can carry DP-selected W8-asym entries; extract_regex_config+ (the post-scoring export-metadata pass) must keep them under llm_compressor and+ still refuse them for native formats."""+ import torch.nn as nn++ from auto_round.compressors.config_resolution import ResolvedScheme+ from auto_round.compressors.layer_config_resolver import extract_regex_config+ from auto_round.schemes import QuantizationScheme++ model = nn.Sequential(nn.Linear(32, 32))+ scheme = ResolvedScheme.from_scheme(QuantizationScheme(bits=4, sym=False, act_bits=16, act_data_type="float"))+ layer_config = {"0": {"bits": 8, "sym": False, "data_type": "int", "group_size": 32}}++ # Regression: the post-scoring export-metadata pass must not refuse a+ # DP-selected W8-asym entry when the export format serves it (this exact+ # shape crashed AutoScheme + llm_compressor + --asym after scoring).+ extract_regex_config(+ model=model,+ scheme=scheme,+ layer_config=dict(layer_config),+ supported_types=(nn.Linear,),+ inner_supported_types=(),+ format="auto_round:llm_compressor",+ )++ import pytest as _pytest++ with _pytest.raises(ValueError, match="8-bit asymmetric"):+ extract_regex_config(+ model=model,+ scheme=scheme,+ layer_config=dict(layer_config),+ supported_types=(nn.Linear,),+ inner_supported_types=(),+ format="auto_round",+ )++ # the opt-in env exempts native formats too+ monkeypatch.setenv("AR_ALLOW_W8_ASYM", "1")+ extract_regex_config(+ model=model,+ scheme=scheme,+ layer_config=dict(layer_config),+ supported_types=(nn.Linear,),+ inner_supported_types=(),+ format="auto_round",+ )diff --git a/test/unit/common/utils/test_missing_tensors.py b/test/unit/common/utils/test_missing_tensors.pyindex 547f5820f..5bcaac255 100644--- a/test/unit/common/utils/test_missing_tensors.py+++ b/test/unit/common/utils/test_missing_tensors.py@@ -236,7 +236,9 @@ class TestQuantizeWeightRtn:def test_output_shapes_4bit_sym(self):out_f, in_f, bits, gs = 32, 64, 4, 32- qw, qz, sc = quantize_weight_rtn(torch.randn(out_f, in_f), bits=bits, group_size=gs, sym=True)+ qw, qz, sc = quantize_weight_rtn(+ torch.randn(out_f, in_f), bits=bits, group_size=gs, sym=True, packing="auto_round:auto_gptq"+ )pf = 32 // bitsng = in_f // gsassert qw.shape == (in_f // pf, out_f)@@ -247,27 +249,95 @@ class TestQuantizeWeightRtn:def test_output_shapes_4bit_asym(self):out_f, in_f, bits, gs = 32, 64, 4, 32- qw, qz, sc = quantize_weight_rtn(torch.randn(out_f, in_f), bits=bits, group_size=gs, sym=False)+ qw, qz, sc = quantize_weight_rtn(+ torch.randn(out_f, in_f), bits=bits, group_size=gs, sym=False, packing="auto_round:auto_gptq"+ )assert qw.shape == (in_f // (32 // bits), out_f)+ @staticmethod+ def _unpack_4bit(qw, qz, sc):+ """Unpack [q, z, s] with nibbles along the packed dims."""+ shifts = torch.arange(8, dtype=torch.int32) * 4+ q = (qw[:, None, :].to(torch.int64) >> shifts[None, :, None]) & 0xF+ q = q.reshape(-1, qw.shape[1]).t().contiguous() # [out, in]+ z = (qz[:, :, None].to(torch.int64) >> shifts[None, None, :]) & 0xF+ z = z.reshape(qz.shape[0], -1).t().contiguous() # [out, groups]+ s = sc.t().contiguous() # [out, groups]+ return q, z, s++ def test_asym_auto_round_packing_stores_zp_directly(self):+ """Asym packing for the declared "auto_round" format must store zp directly.++ The model-free config labels asym int artifacts packing_format+ "auto_round" (see _declared_int_packing), which consumers read+ with the qlinear_torch convention: the qzeros nibble IS the zero point.+ Storing zp - 1 there shifts every weight by +scale per group."""+ torch.manual_seed(42)+ w = torch.randn(8, 128) * 0.05+ qw, qz, sc = quantize_weight_rtn(w, bits=4, group_size=128, sym=False, packing="auto_round")+ q, z, s = self._unpack_4bit(qw, qz, sc) # single group: [8,128], [8,1], [8,1]+ w_rec = (q - z) * s+ assert (w_rec - w).abs().max() <= 0.5 * s.max().float() + 1e-6+ # decoding the same bytes with the +1 (gptq) bias must be measurably wrong+ w_wrong = (q - (z + 1)) * s+ assert (w_wrong - w).abs().mean() > 0.9 * s.float().mean()++ def test_unknown_packing_format_raises(self):+ """A packing string with no registered zero-point convention must fail+ loudly before any quantization work, never silently pick a convention+ (note "auto_round:gptqmodel" is a direct-zp family member, so substring+ matching on "gptq" would be wrong)."""+ with pytest.raises(ValueError, match="unsupported packing format"):+ quantize_weight_rtn(torch.randn(8, 128), bits=4, group_size=128, sym=False, packing="made_up")+ # direct-zp family member containing "gptq" as a substring+ w = torch.randn(8, 128) * 0.05+ torch.manual_seed(3)+ w = torch.randn(8, 128) * 0.05+ qw, qz, sc = quantize_weight_rtn(w, bits=4, group_size=128, sym=False, packing="auto_round:gptqmodel")+ q, z, s = self._unpack_4bit(qw, qz, sc)+ w_rec = (q - z) * s+ assert (w_rec - w).abs().max() <= 0.5 * s.max().float() + 1e-6 # direct-zp convention round-trips++ def test_asym_gptq_packing_stores_zp_minus_one(self):+ """Gptq-family asym packing keeps the qlinear_torch_zp convention.++ packing_format "auto_round:auto_gptq" consumers unpack zeros with +1,+ so the nibble must store zp - 1 (existing behavior, back-compat)."""+ torch.manual_seed(7)+ w = torch.randn(8, 128) * 0.05+ qw, qz, sc = quantize_weight_rtn(w, bits=4, group_size=128, sym=False, packing="auto_round:auto_gptq")+ q, z, s = self._unpack_4bit(qw, qz, sc)+ w_rec = (q - (z + 1)) * s+ assert (w_rec - w).abs().max() <= 0.5 * s.max().float() + 1e-6++ def test_sym_packing_stores_offset_binary_minus_one(self):+ """Sym packing (declared auto_round:auto_gptq) stores the constant 8 as 7."""+ qw, qz, sc = quantize_weight_rtn(+ torch.randn(8, 128), bits=4, group_size=128, sym=True, packing="auto_round:auto_gptq"+ )+ z = (qz[:, :, None].to(torch.int64) >> (torch.arange(8, dtype=torch.int32) * 4)[None, None, :]) & 0xF+ assert (z == 7).all()+def test_output_shapes_8bit(self):out_f, in_f, bits, gs = 16, 128, 8, 128- qw, qz, sc = quantize_weight_rtn(torch.randn(out_f, in_f), bits=bits, group_size=gs, sym=True)+ qw, qz, sc = quantize_weight_rtn(+ torch.randn(out_f, in_f), bits=bits, group_size=gs, sym=True, packing="auto_round:auto_gptq"+ )pf = 32 // bitsassert qw.shape == (in_f // pf, out_f)def test_outputs_on_cpu(self):- qw, qz, sc = quantize_weight_rtn(torch.randn(16, 64), bits=4, group_size=32)+ qw, qz, sc = quantize_weight_rtn(torch.randn(16, 64), bits=4, group_size=32, packing="auto_round:auto_gptq")assert qw.device.type == "cpu"assert qz.device.type == "cpu"assert sc.device.type == "cpu"def test_raises_on_1d(self):with pytest.raises(AssertionError):- quantize_weight_rtn(torch.randn(64), bits=4, group_size=32)+ quantize_weight_rtn(torch.randn(64), bits=4, group_size=32, packing="auto_round:auto_gptq")def test_in_features_padding(self):- qw, _, _ = quantize_weight_rtn(torch.randn(16, 70), bits=4, group_size=32)+ qw, _, _ = quantize_weight_rtn(torch.randn(16, 70), bits=4, group_size=32, packing="auto_round:auto_gptq")assert qw.shape == (96 // (32 // 4), 16) # padded to 96diff --git a/test/unit/test_cpu/export/test_llmc_format.py b/test/unit/test_cpu/export/test_llmc_format.pyindex f1725b32b..8b77d29fd 100644--- a/test/unit/test_cpu/export/test_llmc_format.py+++ b/test/unit/test_cpu/export/test_llmc_format.py@@ -17,6 +17,146 @@ from ...envs import is_compressed_tensors_availablepytestmark = pytest.mark.skipif(not is_compressed_tensors_available(), reason="test requires compressed-tensors")+class TestLLMCZpInt8:+ """pack_layer must hand compressed_tensors an int8 zero-point tensor:+ NeUQI stores zp as a float32 integral-valued tensor, and CT's packer+ (pack_to_int32) hard-requires torch.int8."""++ def _make_layer(self, zp_tensor, bits=4, sym=False):+ import torch.nn as nn++ layer = nn.Linear(16, 8)+ layer.bits = bits+ layer.sym = sym+ layer.group_size = 16+ layer.data_type = "int"+ layer.act_bits = 16+ layer.act_sym = True+ layer.act_data_type = None+ layer.scale = torch.full((8, 1), 0.01)+ layer.zp = zp_tensor+ return layer++ def test_tensor_zp_cast_to_int8(self):+ import torch++ from auto_round.export.export_to_llmcompressor.export import pack_layer++ layer = self._make_layer(torch.full((8, 1), 5.0)) # float32 integral-valued+ model = torch.nn.Sequential()+ model.add_module("q_proj", layer)+ pack_layer("q_proj", model, device="cpu")+ # after compression CT owns the params: zp arrives packed to int32 and+ # the layer is marked COMPRESSED (the pre-fix code raised in pack_to_int32)+ zp = getattr(layer, "weight_zero_point", None)+ assert zp is not None and zp.dtype == torch.int32+ assert getattr(layer, "weight_packed", None) is not None++ def test_out_of_range_zp_raises(self):+ import torch++ from auto_round.export.export_to_llmcompressor.export import pack_layer++ layer = self._make_layer(torch.full((8, 1), 200.0), bits=4) # zp >> 2^b-1: invalid at any convention+ model = torch.nn.Sequential()+ model.add_module("q_proj", layer)+ try:+ pack_layer("q_proj", model, device="cpu")+ except ValueError as e:+ assert "zero-point" in str(e)+ else:+ raise AssertionError("out-of-range zp must raise")++ def test_asym_zp_signed_convention_round_trip(self):+ """AutoRound zp is unsigned (q in [0, 2^b-1], W = (q - zp) * s);+ compressed-tensors re-quantizes with the SIGNED range+ [-2^(b-1), 2^(b-1)-1] (clamp AFTER adding zp). The unsigned zp must be+ shifted into the signed convention before packing, or CT's clamp+ collapses every level above 2^(b-1)-1. Round-trip through CT's own+ compressor must reproduce AutoRound's fake-quantized weight exactly."""+ import torch+ import torch.nn as nn+ from compressed_tensors.compressors.pack_quantized import PackedQuantizationCompressor++ from auto_round.export.export_to_llmcompressor.export import pack_layer++ torch.manual_seed(0)+ out_f, in_f, gs = 8, 32, 16+ w = torch.randn(out_f, in_f)+ scale = torch.rand(out_f, in_f // gs) * 0.02 + 0.005 # [out, in/g]+ zp = torch.randint(0, 16, (out_f, in_f // gs)).float()+ zp[:, 0] = 12 # exercise levels above 2^(b-1)-1 (the pre-fix clamp)++ layer = nn.Linear(in_f, out_f)+ with torch.no_grad():+ layer.weight.copy_(w)+ layer.bits, layer.sym, layer.group_size = 4, False, gs+ layer.data_type, layer.act_bits, layer.act_sym, layer.act_data_type = "int", 16, True, None+ layer.scale, layer.zp = scale, zp++ model = nn.Sequential()+ model.add_module("q_proj", layer)+ pack_layer("q_proj", model, device="cpu")++ # AutoRound's own fake-quantized weight (unsigned convention)+ s = scale.repeat_interleave(gs, dim=1)+ z = zp.repeat_interleave(gs, dim=1)+ q = (w / s + z).round().clamp(0, 15)+ w_ref = (q - z) * s++ state = {+ "weight_packed": layer.weight_packed.detach(),+ "weight_scale": layer.weight_scale.detach(),+ "weight_zero_point": layer.weight_zero_point.detach(),+ "weight_shape": layer.weight_shape.detach(),+ }+ out = PackedQuantizationCompressor.decompress(state, layer.quantization_scheme)+ assert torch.equal(out["weight"].to(torch.float32), w_ref)++ def test_w8_asym_pack_round_trip(self):+ """8-bit asym is the boundary case: zp spans [0, 255] and only fits int8+ via the signed-convention shift. The llm_compressor format accepts W8+ asym (vLLM serves it through the compressed-tensors kernel path), so+ the pack must round-trip."""+ import torch+ import torch.nn as nn+ from compressed_tensors.compressors.pack_quantized import PackedQuantizationCompressor++ from auto_round.export.export_to_llmcompressor.export import pack_layer++ torch.manual_seed(0)+ out_f, in_f, gs = 8, 32, 16+ w = torch.randn(out_f, in_f)+ scale = torch.rand(out_f, in_f // gs) * 0.02 + 0.005+ zp = torch.randint(0, 256, (out_f, in_f // gs)).float() # full 8-bit span+ zp[:, 0] = 250 # far above 127: unsigned convention, must survive the shift++ layer = nn.Linear(in_f, out_f)+ with torch.no_grad():+ layer.weight.copy_(w)+ layer.bits, layer.sym, layer.group_size = 8, False, gs+ layer.data_type, layer.act_bits, layer.act_sym, layer.act_data_type = "int", 16, True, None+ layer.scale, layer.zp = scale, zp++ model = nn.Sequential()+ model.add_module("q_proj", layer)+ pack_layer("q_proj", model, device="cpu")++ s = scale.repeat_interleave(gs, dim=1)+ z = zp.repeat_interleave(gs, dim=1)+ q = (w / s + z).round().clamp(0, 255)+ w_ref = (q - z) * s++ state = {+ "weight_packed": layer.weight_packed.detach(),+ "weight_scale": layer.weight_scale.detach(),+ "weight_zero_point": layer.weight_zero_point.detach(),+ "weight_shape": layer.weight_shape.detach(),+ }+ out = PackedQuantizationCompressor.decompress(state, layer.quantization_scheme)+ assert torch.equal(out["weight"].to(torch.float32), w_ref)++class TestLLMC:@classmethod@@ -29,6 +169,32 @@ class TestLLMC:def teardown_class(self):shutil.rmtree("runs", ignore_errors=True)+ def test_mixed_int_bits_llmcompressor_format(self, tiny_opt_model_path, tmp_path):+ """Mixed int W5A16-W8A16 auto-scheme export with an unsupported-as-first-option scheme+ leading the options (W5A16) must resolve and export through the llm_compressor backend+ (compressed-tensors pack-quantized handles any int width in 2..8)."""+ scheme = AutoScheme(+ avg_bits=6.0,+ options=("W5A16", "W6A16", "W7A16", "W8A16"),+ )+ ar = AutoRound(+ model=tiny_opt_model_path,+ iters=0,+ disable_opt_rtn=True,+ scheme=scheme,+ )+ _, tmp_path = ar.quantize_and_save(output_dir=tmp_path, format="auto_round:llm_compressor")+ model = AutoModelForCausalLM.from_pretrained(tmp_path, torch_dtype="auto", trust_remote_code=True)+ quantization_config = model.config.quantization_config.to_dict()+ assert quantization_config["format"] == "pack-quantized"+ assert quantization_config["config_groups"], "no quantized config groups were exported"+ for group in quantization_config["config_groups"].values():+ weights = group["weights"]+ assert weights["num_bits"] in (5, 6, 7, 8), f"unexpected bit-width {weights['num_bits']}"+ assert weights["type"] == "int"+ assert weights["symmetric"] is True+ assert group["input_activations"] is None # weight-only quantization+# remove since w8a8 not in llmcompressor format supported schemes# def test_llmcompressor_w8a8(self):# bits, group_size, sym, act_bits = 8, -1, True, 8@@ -400,3 +566,113 @@ def test_llmcompressor_mxfp8_export_packs_serially(tmp_path, monkeypatch):compressed_model = autoround.save_quantized(output_dir=tmp_path, format="llm_compressor")tmp_layer = compressed_model.model.decoder.layers[1].self_attn.q_projassert hasattr(tmp_layer, "weight_scale")+++class TestLLMCZpUnsignedDomain:+ """The packed-level validation must reject zero points outside the+ UNSIGNED level range [0, 2^bits-1] before the signed-convention shift; a+ shifted-only check would let e.g. a raw 4-bit zp of 100 pass (shifted 92+ fits int8)."""++ def _make_layer(self, zp, bits=4, sym=False):+ import torch.nn as nn++ layer = nn.Linear(16, 8)+ layer.bits = bits+ layer.sym = sym+ layer.group_size = 16+ layer.data_type = "int"+ layer.act_bits = 16+ layer.act_sym = True+ layer.act_data_type = None+ layer.scale = torch.full((8, 1), 0.01)+ layer.zp = zp+ return layer++ def test_tensor_zp_above_unsigned_range_rejected(self):+ import torch++ from auto_round.export.export_to_llmcompressor.export import pack_layer++ layer = self._make_layer(torch.full((8, 1), 100.0)) # raw 100 > 15+ m = torch.nn.Sequential()+ m.add_module("proj", layer)+ with pytest.raises(ValueError, match="unsigned level range"):+ pack_layer("proj", m, device="cpu")++ def test_tensor_zp_negative_rejected(self):+ import torch++ from auto_round.export.export_to_llmcompressor.export import pack_layer++ layer = self._make_layer(torch.full((8, 1), -3.0)) # shifted -11 fits int8+ m = torch.nn.Sequential()+ m.add_module("proj", layer)+ with pytest.raises(ValueError, match="unsigned level range"):+ pack_layer("proj", m, device="cpu")++ def test_scalar_zp_out_of_range_rejected(self):+ from auto_round.export.export_to_llmcompressor.export import pack_layer++ for bad in (17.0, -1.0): # 4-bit unsigned range is [0, 15]+ layer = self._make_layer(bad)+ m = torch.nn.Sequential()+ m.add_module("proj", layer)+ with pytest.raises(ValueError, match="unsigned level range"):+ pack_layer("proj", m, device="cpu")++ def test_in_range_values_pass(self):+ import torch++ from auto_round.export.export_to_llmcompressor.export import pack_layer++ layer = self._make_layer(torch.randint(0, 16, (8, 1)).float())+ m = torch.nn.Sequential()+ m.add_module("proj", layer)+ pack_layer("proj", m, device="cpu")+ # CT's compressor consumes the int8 zp and stores its bit-packed form+ assert layer.weight_zero_point.dtype == torch.int32+ assert layer.weight_packed is not None+++class TestLLMCZpFiniteIntegral:+ """NaN passes every min/max comparison and would be cast straight to+ int8; non-integral values must not be silently rounded into a level."""++ def _make_layer(self, zp, bits=4):+ import torch.nn as nn++ layer = nn.Linear(16, 8)+ layer.bits = bits+ layer.sym = False+ layer.group_size = 16+ layer.data_type = "int"+ layer.act_bits = 16+ layer.act_sym = True+ layer.act_data_type = None+ layer.scale = torch.full((8, 1), 0.01)+ layer.zp = zp+ return layer++ def _expect_refusal(self, zp, match):+ from auto_round.export.export_to_llmcompressor.export import pack_layer++ layer = self._make_layer(zp)+ m = torch.nn.Sequential()+ m.add_module("proj", layer)+ with pytest.raises(ValueError, match=match):+ pack_layer("proj", m, device="cpu")++ def test_nan_tensor_zp_rejected(self):+ zp = torch.zeros(8, 1)+ zp[3, 0] = float("nan")+ self._expect_refusal(zp, "non-finite")++ def test_non_integral_tensor_zp_rejected(self):+ self._expect_refusal(torch.full((8, 1), -0.4), "non-integral")++ def test_nan_scalar_zp_rejected(self):+ self._expect_refusal(float("nan"), "not a finite integer")++ def test_non_integral_scalar_zp_rejected(self):+ self._expect_refusal(7.3, "not a finite integer")diff --git a/test/unit/test_cpu/export/test_qlinear_pack_clamp.py b/test/unit/test_cpu/export/test_qlinear_pack_clamp.pynew file mode 100644index 000000000..29aeda6e6--- /dev/null+++ b/test/unit/test_cpu/export/test_qlinear_pack_clamp.py@@ -0,0 +1,262 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+"""Packers must clamp quantized levels into [0, maxq] before bit-packing.++round(W/s + zp) can exceed the level range whenever the searched (scale, zp)+grid does not fully cover the weight range (the search itself evaluates the+clamped loss). Packing the unclamped value shifts extra bits into the+NEIGHBORING packed nibble -- silently corrupting it -- so the packed artifact+no longer reproduces the clamped fake-quantization the search optimized.+"""++from types import SimpleNamespace++import pytest+import torch+from torch import nn++from auto_round.export.export_to_awq.utils import WQLinear_GEMM+from auto_round_extension.torch.qlinear_torch import QuantLinear as QuantLinearDirect+from auto_round_extension.torch.qlinear_torch_zp import QuantLinear as QuantLinearMinusOne++OUT_F, IN_F, GROUP = 64, 64, 32+++def _make_inputs(bits, seed=0, with_overshoot=True):+ g = torch.Generator().manual_seed(seed)+ w = torch.randn(OUT_F, IN_F, generator=g) * 0.05+ scale = torch.rand(OUT_F, IN_F // GROUP, generator=g) * 0.005 + 0.02+ zp = torch.randint(0, 2**bits, (OUT_F, IN_F // GROUP), generator=g).float()+ if with_overshoot:+ # force q = round(w/s + zp) far outside [0, maxq] on dedicated rows:+ # scales ~0.02..0.025, so +/-3.0 / 0.02 ~ +/-150 levels of overshoot+ w[0, :] = 3.0+ w[1, :] = -3.0+ zp[0, :] = 0 # negative side undershoots as well+ return w, scale, zp+++def _unpack_qweight(qlayer):+ bits = qlayer.bits+ npk = 32 // bits+ maxq = 2**bits - 1+ wf = torch.arange(0, 32, bits, dtype=torch.int32)+ w_int = qlayer.qweight.unsqueeze(1).expand(-1, npk, -1) >> wf.unsqueeze(-1)+ w_int = (w_int & maxq).reshape(-1, qlayer.outfeatures) # [in, out]+ return w_int.t().to(torch.int64) # [out, in]+++def _unpack_qzeros(qlayer):+ bits = qlayer.bits+ npk = 32 // bits+ maxq = 2**bits - 1+ wf = torch.arange(0, 32, bits, dtype=torch.int32)+ z = qlayer.qzeros.unsqueeze(2).expand(-1, -1, npk) >> wf.unsqueeze(0)+ return (z & maxq).to(torch.int64) # [in/g, out/npk-1 packed] -> per nibble [in/g, out]++[email protected]("bits", [2, 4, 8])+def test_pack_clamps_levels_direct_zp(bits):+ torch.manual_seed(0)+ w, scale, zp = _make_inputs(bits)+ q = QuantLinearDirect(bits, GROUP, IN_F, OUT_F, bias=False)+ q.device = torch.device("cpu")+ q.pack(SimpleNamespace(weight=w, bias=None), scale, zp, None, device="cpu")++ expected = torch.clamp(+ torch.round(w / scale.repeat_interleave(GROUP, 1) + zp.repeat_interleave(GROUP, 1)), 0, 2**bits - 1+ ).to(torch.int64)+ got = _unpack_qweight(q)+ assert got.min() >= 0 and got.max() <= 2**bits - 1+ assert torch.equal(got, expected), "packed levels must equal the clamped fake-quantization"++[email protected]("bits", [2, 4, 8])+def test_pack_clamps_levels_minus_one_zp(bits):+ torch.manual_seed(0)+ w, scale, zp = _make_inputs(bits)+ q = QuantLinearMinusOne(bits, GROUP, IN_F, OUT_F, bias=False)+ q.device = torch.device("cpu")+ q.pack(SimpleNamespace(weight=w, bias=None), scale, zp, None, device="cpu")++ expected = torch.clamp(+ torch.round(w / scale.repeat_interleave(GROUP, 1) + zp.repeat_interleave(GROUP, 1)), 0, 2**bits - 1+ ).to(torch.int64)+ assert torch.equal(_unpack_qweight(q), expected)+++def test_minus_one_zp_packing_survives_zero_zero_point():+ """zp=0 wraps to -1 under the classic (zp-1) packing; the raw -1 word+ corrupts every nibble sharing its packed word. The stored value must be+ clamped so only that one zero-point degrades (to a decodable +1)."""+ torch.manual_seed(0)+ bits = 4+ w, scale, zp = _make_inputs(bits, with_overshoot=False)+ zp[:, :] = 0 # every group sits at the unrepresentable zero-point+ q = QuantLinearMinusOne(bits, GROUP, IN_F, OUT_F, bias=False)+ q.device = torch.device("cpu")+ q.pack(SimpleNamespace(weight=w, bias=None), scale, zp, None, device="cpu")++ z_nibbles = _unpack_qzeros(q) # [in/g, out] per-nibble values+ # pre-fix: (0-1) << shift makes whole words -1 -> every nibble reads maxq+ assert z_nibbles.max() <= 1, "zp=0 must not smear garbage across the packed word"+ # neighbors of a clamped cell keep their own packed value: all nibbles equal+ # either 0 (the clamped zp-1) -- uniform here by construction+ assert (z_nibbles == 0).all()+ # and the module still dequantizes (forward runs, no all-ones corruption)+ out = q(torch.eye(IN_F))+ assert torch.isfinite(out).all()+++def test_awq_pack_clamps_levels():+ """The AWQ packer shares the unclamped-round spill bug: an out-of-range+ level shifts bits into the neighboring packed nibble (AWQ interleave+ order), corrupting it. Packed levels must equal the clamped grid."""+ torch.manual_seed(0)+ bits, maxq = 4, 15+ w, scale, zp = _make_inputs(bits) # [out, in], [out, in/g], [out, in/g]+ lin = nn.Linear(IN_F, OUT_F, bias=False)+ with torch.no_grad():+ lin.weight.copy_(w)+ q = WQLinear_GEMM.from_linear(lin, bits, GROUP, scales=scale.t(), zeros=zp.t(), device="cpu")++ expected_q = torch.clamp(+ torch.round(w / scale.repeat_interleave(GROUP, 1) + zp.repeat_interleave(GROUP, 1)), 0, maxq+ ).to(torch.int32)++ def _pack_rows(t): # mirrors the packer: nibbles along dim 1, AWQ interleave+ pack_num = 32 // bits+ order = torch.tensor([0, 4, 1, 5, 2, 6, 3, 7]) * bits+ packed = t.reshape(t.shape[0], t.shape[1] // pack_num, pack_num) << order+ return torch.sum(packed, dim=-1).to(torch.int32)++ # qweight packs q transposed ([in, out/8], nibbles along the output dim);+ # qzeros pack zp along the output dim ([in/g, out/8])+ assert torch.equal(q.qweight, _pack_rows(expected_q.t().contiguous()))+ assert torch.equal(q.qzeros, _pack_rows(zp.to(torch.int32).t().contiguous()))+++def test_direct_zp_roundtrip_in_range():+ """Sanity: for in-range data the packed module reproduces clamped+ fake-quantization bit-exactly (clamp must be a no-op there)."""+ torch.manual_seed(0)+ bits, maxq = 4, 15+ w, scale, zp = _make_inputs(bits, with_overshoot=False)+ # keep q in-range: shrink weights relative to scale+ w = w * 0.05+ q = QuantLinearDirect(bits, GROUP, IN_F, OUT_F, bias=False)+ q.device = torch.device("cpu")+ q.pack(SimpleNamespace(weight=w, bias=None), scale, zp, None, device="cpu")+ s16 = scale.half().float().repeat_interleave(GROUP, 1)+ zrep = zp.repeat_interleave(GROUP, 1)+ expected = torch.clamp(torch.round(w / s16 + zrep), 0, maxq).to(torch.int64)+ assert torch.equal(_unpack_qweight(q), expected)+ # dequantized weights match the reference reconstruction+ ref_w = (expected.to(torch.float32) - zrep) * s16+ got_w = q(torch.eye(IN_F)).t()+ torch.testing.assert_close(got_w, ref_w, rtol=0, atol=1e-3)+++def test_minus_one_scalar_zp_packing_survives_zero_zero_point():+ """The scalar zero-point path of the classic (zp-1) packer must clamp like+ the tensor path: an unclamped 0-1 = -1 repeated into every packed word+ corrupts all nibbles; the clamped value degrades that one zp to +1."""+ torch.manual_seed(0)+ bits = 4+ w, scale, _ = _make_inputs(bits, with_overshoot=False)+ q = QuantLinearMinusOne(bits, GROUP, IN_F, OUT_F, bias=False)+ q.device = torch.device("cpu")+ q.pack(SimpleNamespace(weight=w, bias=None), scale, 0.0, None, device="cpu")++ npk = 32 // bits+ maxq = 2**bits - 1+ wf = torch.arange(0, 32, bits, dtype=torch.int32)+ z = q.qzeros.unsqueeze(2).expand(-1, -1, npk) >> wf.unsqueeze(0)+ z_nibbles = (z & maxq).to(torch.int64)+ # pre-fix: (0-1) built a -1 repeated word -> every nibble reads maxq+ assert (z_nibbles == 0).all(), "scalar zp=0 must clamp to a decodable stored 0"+ out = q(torch.eye(IN_F))+ assert torch.isfinite(out).all()+++def test_missing_tensors_gptq_pack_clamps_zero_zero_point():+ """quantize_weight_rtn packs zp with the same (zp-1) convention; an+ all-positive weight yields zp=0 everywhere, which must clamp instead of+ smearing -1 across the packed zero-point words."""+ from auto_round.utils.missing_tensors import quantize_weight_rtn++ torch.manual_seed(0)+ bits, group = 4, 32+ w = torch.rand(16, 64) + 0.1 # strictly positive -> asym zp = 0 per group+ qweight, qzeros, scales = quantize_weight_rtn(+ w, bits=bits, group_size=group, sym=False, packing="auto_round:auto_gptq"+ )++ npk = 32 // bits+ maxq = 2**bits - 1+ wf = torch.arange(0, 32, bits, dtype=torch.int32)+ z = qzeros.unsqueeze(2).expand(-1, -1, npk) >> wf.unsqueeze(0)+ z_nibbles = (z & maxq).to(torch.int64)+ # pre-fix: zp-1 = -1 -> whole int32 words negative -> nibbles read maxq+ assert (z_nibbles == 0).all(), "zp=0 must pack as the clamped 0 (decodes +1)"+ # scales/q stay well-formed+ assert torch.isfinite(scales).all() and scales.shape == (w.shape[1] // group, w.shape[0])+++def test_minus_one_3bit_scalar_zp_clamped_both_ends():+ """The 3-bit scalar zero-point path must clamp at both ends: zp=0 packs a+ clean all-zero word (no -1 smear), and an oversized scalar (20 > maxq 7)+ packs bit-identically to the in-range zp=8 (stored 7) instead of spilling+ into neighboring packed fields."""+ bits = 3++ def _pack_with_scalar(scalar):+ torch.manual_seed(0)+ q = QuantLinearMinusOne(bits, GROUP, IN_F, OUT_F, bias=False)+ q.device = torch.device("cpu")+ w, scale, _ = _make_inputs(bits, with_overshoot=False)+ q.pack(SimpleNamespace(weight=w, bias=None), scale, scalar, None, device="cpu")+ return q++ q0 = _pack_with_scalar(0.0)+ assert (q0.qzeros == 0).all(), "scalar zp=0 must clamp to a stored 0 word"+ assert torch.isfinite(q0(torch.eye(IN_F))).all()++ q_over = _pack_with_scalar(20.0)+ q_ref = _pack_with_scalar(8.0) # in-range zero point storing 7+ assert torch.equal(+ q_over.qzeros, q_ref.qzeros+ ), "an oversized scalar zero point must clamp to the maxq word, not spill"++[email protected]("bits", [2, 4, 8])+def test_minus_one_scalar_zp_packs_float_scalars(bits):+ """A float scalar zero point (the natural minmax output) must survive the+ 2/4/8-bit classic (zp-1) packing: normalized to int, clamped at both+ ends. zp=1.0 stores 0; an oversized scalar stores maxq."""+ maxq = 2**bits - 1++ def _pack(scalar):+ q = QuantLinearMinusOne(bits, GROUP, IN_F, OUT_F, bias=False)+ q.device = torch.device("cpu")+ w, scale, _ = _make_inputs(bits, with_overshoot=False)+ torch.manual_seed(0)+ q.pack(SimpleNamespace(weight=w, bias=None), scale, scalar, None, device="cpu")+ return q++ q1 = _pack(1.0)+ q_over = _pack(float(maxq + 5))+ q_ref = _pack(float(maxq + 1)) # in-range zp storing maxq+ assert torch.isfinite(q1(torch.eye(IN_F))).all(), "float scalar zp=1.0 must pack, not crash"+ assert torch.equal(q_over.qzeros, q_ref.qzeros), "oversized float scalar must clamp to maxq"diff --git a/test/unit/test_cpu/quantization/test_model_free_parity.py b/test/unit/test_cpu/quantization/test_model_free_parity.pyindex 5cd8c0009..1b61405f0 100755--- a/test/unit/test_cpu/quantization/test_model_free_parity.py+++ b/test/unit/test_cpu/quantization/test_model_free_parity.py@@ -90,9 +90,9 @@ _PLAIN_RTN_PARITY_SCHEMES = [_PLAIN_RTN_CONTRACT_SCHEMES = [# Asymmetric plain RTN keeps export-contract parity across the two paths,- # but the packed tensors are not bit-exact.+ # but the packed tensors are not bit-exact. 8-bit asym is excluded: it is+ # refused at construction on both routes (see the refusal test below).("W2A16_ASYM", "W2A16", {"bits": 2, "group_size": 128, "sym": False}),- ("W8A16_ASYM", "W8A16", {"bits": 8, "group_size": 128, "sym": False}),]# Note: W4A16 asym is excluded because the regular path uses@@ -361,6 +361,31 @@ def test_plain_rtn_asymmetric_export_contract_parity(_assert_int_export_contract_parity(out_model_free, out_regular)+def test_w8_asymmetric_refused_on_both_routes(tiny_opt_model_path):+ """8-bit asym is refused at construction on both the model-free and the+ regular route: both default to native int8-packed export formats, which+ cannot represent the 8-bit zero point (vLLM serves W8 asym only via+ compressed-tensors; see AR_ALLOW_W8_ASYM).+ """+ from auto_round import AutoRound++ device = _device_str()+ common = dict(+ scheme="W8A16",+ bits=8,+ group_size=128,+ sym=False,+ iters=0,+ disable_opt_rtn=True,+ device_map=device,+ enable_torch_compile=False,+ )+ with pytest.raises(ValueError, match="8-bit asymmetric"):+ AutoRound(tiny_opt_model_path, model_free=True, **common)+ with pytest.raises(ValueError, match="8-bit asymmetric"):+ AutoRound(tiny_opt_model_path, disable_model_free=True, amp=False, **common)++@pytest.mark.parametrize("scheme_name,scheme_preset,scheme_kwargs", _OPT_RTN_PARITY_SCHEMES)def test_model_free_int_opt_rtn_same_as_plain_rtn(tmp_path, tiny_opt_model_path, scheme_name, scheme_preset, scheme_kwargsdiff --git a/test/unit/test_cuda/export/test_autoround_int_export.py b/test/unit/test_cuda/export/test_autoround_int_export.pyindex 8fe40e708..aeace8e8b 100644--- a/test/unit/test_cuda/export/test_autoround_int_export.py+++ b/test/unit/test_cuda/export/test_autoround_int_export.py@@ -63,7 +63,34 @@ class TestAutoroundIntExportGpu:assert out.logits.shape[0] == 1@pytest.mark.timeout(180)- @pytest.mark.parametrize("bits", [4, 8])+ @pytest.mark.parametrize("group_size", [128])+ def test_w8_asym_env_int_export_reload_forward(self, tiny_opt_model_path, group_size, monkeypatch):+ """AR_ALLOW_W8_ASYM=1 skips the native-format refusal; the packed+ artifact still reloads and forwards through transformers (serving is+ the user's responsibility, hence the explicit opt-in env)."""+ monkeypatch.setenv("AR_ALLOW_W8_ASYM", "1")+ autoround = AutoRound(+ tiny_opt_model_path,+ bits=8,+ group_size=group_size,+ sym=False,+ iters=0,+ disable_opt_rtn=True,+ nsamples=1,+ seqlen=16,+ )+ _, quantized_model_path = autoround.quantize_and_save(output_dir=self.save_dir, format="auto_round")++ model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="cuda:0", trust_remote_code=True)+ assert isinstance(model, torch.nn.Module)+ assert _has_packed_weight(model)+ input_ids = torch.randint(0, 1000, (1, 16), device="cuda:0")+ with torch.no_grad():+ out = model(input_ids)+ assert out.logits.shape[0] == 1++ @pytest.mark.timeout(180)+ @pytest.mark.parametrize("bits", [4]) # 8-bit asym is refused at construction@pytest.mark.parametrize("group_size", [32, 128])def test_asym_int_export_reload_forward(self, tiny_opt_model_path, bits, group_size):"""Asymmetric integer export path."""diff --git a/test/unit/test_cuda/quantization/test_act_quantization.py b/test/unit/test_cuda/quantization/test_act_quantization.pyindex 295510c32..6143c6377 100644--- a/test/unit/test_cuda/quantization/test_act_quantization.py+++ b/test/unit/test_cuda/quantization/test_act_quantization.py@@ -118,23 +118,23 @@ class TestActQuantizationGpu:"""Asymmetric W8A8 cannot be exported to the auto_round format.``auto_round`` supports W8A8 export only for the symmetric path; an- asymmetric W8A8 config is rejected during format resolution.+ asymmetric W8A8 config is rejected when the scheme is resolved (the+ default export format has no room for an 8-bit zero point)."""- autoround = AutoRound(- tiny_opt_model_path,- bits=8,- act_bits=8,- group_size=128,- act_group_size=32,- act_dynamic=True,- sym=False,- iters=0,- disable_opt_rtn=True,- nsamples=2,- seqlen=16,- )- with pytest.raises(ValueError, match="does not support exporting"):- autoround.quantize_and_save(output_dir=self.save_dir, format="auto_round")+ with pytest.raises(ValueError, match="8-bit asymmetric weight quantization is not supported"):+ AutoRound(+ tiny_opt_model_path,+ bits=8,+ act_bits=8,+ group_size=128,+ act_group_size=32,+ act_dynamic=True,+ sym=False,+ iters=0,+ disable_opt_rtn=True,+ nsamples=2,+ seqlen=16,+ )@pytest.mark.timeout(180)def test_w4a8_reload_layer_type_diff_but_output_equal(self, tiny_opt_model_path):