Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

fix: asymmetric zero-point packing, format-scoped 8-bit asym, W5/W6/W7 llm_compressor presets (#2273)

avtc5d2f88473a14
diff --git a/auto_round/auto_scheme/delta_loss.py b/auto_round/auto_scheme/delta_loss.py
index f407127ae..b85b157c7 100644
--- a/auto_round/auto_scheme/delta_loss.py
+++ b/auto_round/auto_scheme/delta_loss.py
@@ -2548,6 +2548,7 @@ def _gen_layer_config(
processor=None,
is_vlm: bool = False,
disk_index=None,
+ export_format: str = None,
):
"""Score every candidate scheme in ``auto_scheme.options`` against ``quant_layer_names``
and return per-layer per-scheme losses used by the caller to pick a final bit-width
@@ -3501,6 +3502,16 @@ def _gen_layer_config(
# print(best_loss, best_path) # TODO better log
layer_config = copy.deepcopy(fixed_layer_scheme)
+ # fixed pins (lm_head, MTP layer, ...) bypass the options-level sym rule
+ # in parse_scheme -- apply the same 8-bit-symmetric pinning here. The
+ # allowance comes from the export format (llm_compressor serves W8 asym)
+ # or the AR_ALLOW_W8_ASYM opt-in.
+ from auto_round import envs
+ from auto_round.schemes import format_allows_w8_asym
+
+ _enforce_w8_symmetric_entries(
+ layer_config, allow_w8_asym=envs.AR_ALLOW_W8_ASYM or format_allows_w8_asym(export_format)
+ )
options = list(copy.deepcopy(auto_scheme.options))
# Replace scheme preset names with actual QuantizationScheme objects
for index in range(len(options)):
@@ -3547,6 +3558,46 @@ def _gen_layer_config(
return layer_config
+def _enforce_w8_symmetric_entries(layer_config: dict, allow_w8_asym: bool = False) -> int:
+ """Enforce symmetric 8-bit int entries; returns how many were pinned.
+
+ Same rationale as the options-level rule in parse_scheme: formats without
+ a servable W8-asym path (native auto_round, auto_gptq, marlin) cannot use
+ 8-bit asym, and fixed layer_config pins (lm_head, MTP layer, ...) bypass
+ options. Entries WITHOUT an explicit sym would silently inherit the global
+ --asym and are pinned; an EXPLICIT sym=False is refused up front -- there
+ is no point spending quantization time on a configuration that can never
+ be served. The AR_ALLOW_W8_ASYM opt-in or an export format whose serving
+ stack accepts W8 asym (llm_compressor) skips both.
+ """
+ if allow_w8_asym:
+ return 0
+ pinned = 0
+ for name, entry in layer_config.items():
+ if not (isinstance(entry, dict) and entry.get("bits") == 8 and entry.get("data_type", "int") == "int"):
+ continue
+ if entry.get("sym") is False:
+ raise ValueError(
+ f"layer_config entry '{name}' requests 8-bit asymmetric weight quantization, "
+ "which this format cannot serve (vLLM W8 GPTQ-format weights are "
+ "symmetric-only; Marlin supports zero points at 4 bits only). Use a "
+ "symmetric 8-bit entry, an asymmetric width of 7 bits or fewer, format "
+ "'auto_round:llm_compressor', or set AR_ALLOW_W8_ASYM=1."
+ )
+ if "sym" not in entry:
+ entry["sym"] = True
+ pinned += 1
+ if pinned:
+ logger.info(
+ "AutoScheme: pinned %d fixed layer_config entr%s to symmetric 8-bit "
+ "(8-bit asym is not servable in the requested export format; llm_compressor "
+ "and AR_ALLOW_W8_ASYM are the escape hatches)",
+ pinned,
+ "y" if pinned == 1 else "ies",
+ )
+ return pinned
+
+
# Supports model with gradient clearing between iterations
@register_scheme_methods(("default", "DeltaLoss"))
def gen_layer_config(
@@ -3561,6 +3612,7 @@ def gen_layer_config(
low_gpu_mem_usage=True,
min_avg_bit_scheme=None,
processor=None,
+ export_format: str = None,
**kwargs,
):
"""Public AutoScheme entry.
@@ -3706,6 +3758,7 @@ def gen_layer_config(
min_avg_bit_scheme=min_avg_bit_scheme,
processor=processor,
is_vlm=is_vlm,
+ export_format=export_format,
disk_index=disk_index,
)
except torch.OutOfMemoryError:
@@ -3738,6 +3791,7 @@ def gen_layer_config(
min_avg_bit_scheme=min_avg_bit_scheme,
processor=processor,
is_vlm=is_vlm,
+ export_format=export_format,
disk_index=disk_index,
)
diff --git a/auto_round/auto_scheme/gen_auto_scheme.py b/auto_round/auto_scheme/gen_auto_scheme.py
index e86e1e52b..d212aca52 100644
--- a/auto_round/auto_scheme/gen_auto_scheme.py
+++ b/auto_round/auto_scheme/gen_auto_scheme.py
@@ -98,8 +98,14 @@ class GenScheme:
tokenizer=None,
enable_torch_compile=True,
processor=None,
+ export_format: str = None,
):
self.auto_scheme = auto_scheme
+ # Export-format context for the generation-time 8-bit-asym policy: the
+ # compressor passes the format the run actually saves with (including
+ # one supplied only to quantize_and_save). The AR_ALLOW_W8_ASYM env
+ # opt-in is read at the policy gate itself.
+ self.export_format = export_format
# Upstream unconditionally forced low_cpu_mem_usage=False here
# (commit 0c9c5b1d, "reduce memory consumption... for gguf") because of an
# acknowledged bug for mixed INT4/INT8 schemes under the old OffloadManager-
@@ -199,6 +205,7 @@ class GenScheme:
low_gpu_mem_usage=self.auto_scheme.low_gpu_mem_usage,
min_avg_bit_scheme=self.min_avg_bit_scheme,
processor=self.processor,
+ export_format=self.export_format,
)
layer_config = self.fallback_gguf_layer_config(layer_config)
return layer_config
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index c2464d9a4..12c3dda97 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -42,7 +42,7 @@ def _collect_config_scheme_overrides(config) -> dict:
return {k: getattr(config, k) for k in config._scheme_fields if getattr(config, k, None) is not None}
-def _preview_resolved_attrs(config, scheme=None) -> dict:
+def _preview_resolved_attrs(config, scheme=None, format=None) -> dict:
from auto_round.auto_scheme.gen_auto_scheme import AutoScheme
"""Resolve scheme attributes without mutating config, for routing decisions.
@@ -57,7 +57,9 @@ def _preview_resolved_attrs(config, scheme=None) -> dict:
callers read from the returned dict and never re-read raw ``config`` attrs.
When the scheme cannot be previewed (``AutoScheme``, or a deferred parse
error), the config's own explicitly-set scheme overrides are returned so the
- values still reflect what the user passed.
+ values still reflect what the user passed. ``format`` must match the
+ authoritative parse so format-scoped policies (e.g. the 8-bit asym rule)
+ resolve identically here and never turn a refusal into fabricated defaults.
Returns:
dict: resolved scheme attributes (config overrides when preview is skipped).
@@ -67,13 +69,18 @@ def _preview_resolved_attrs(config, scheme=None) -> dict:
# AutoScheme needs model info — cannot preview; fall back to raw config attrs.
return config_overrides
try:
- _, _, final_attrs = parse_scheme(scheme, config_overrides)
+ _, _, final_attrs = parse_scheme(scheme, config_overrides, format=format)
return final_attrs
- except Exception:
+ except Exception as e:
+ logger.warning_once(
+ "Scheme preview failed (%s: %s); routing falls back to the config's explicit overrides.",
+ type(e).__name__,
+ e,
+ )
return config_overrides
-def _eager_validate_scheme(config, scheme=None) -> None:
+def _eager_validate_scheme(config, scheme=None, format=None) -> None:
from auto_round.auto_scheme.gen_auto_scheme import AutoScheme
"""Eagerly validate scheme/config constraints at construction time.
@@ -91,7 +98,7 @@ def _eager_validate_scheme(config, scheme=None) -> None:
user_overrides = _collect_config_scheme_overrides(config)
try:
- _, _, final_attrs = parse_scheme(scheme, user_overrides)
+ _, _, final_attrs = parse_scheme(scheme, user_overrides, format=format)
except (ValueError, NotImplementedError):
raise
except Exception:
@@ -264,7 +271,7 @@ def _select_rtn_compressor_base_cls(quant_config: "RTNConfig", scheme, format, b
# resolution later; this preview only chooses the class). Computed once: neither
# `quant_config`'s scheme fields nor `scheme` itself change within this function,
# so the result is invariant across every use below — no need to recompute it.
- resolved_attrs = _preview_resolved_attrs(quant_config, scheme)
+ resolved_attrs = _preview_resolved_attrs(quant_config, scheme, format=format)
# Auto-disable rtn optimization for W8A16/W8A8-equivalent resolved schemes,
# unless the user already set disable_opt_rtn explicitly.
@@ -705,8 +712,17 @@ class _CompressorBuilder(object):
route_scheme = (
scheme
if hasattr(scheme, "options") and hasattr(scheme, "avg_bits")
- else QuantizationScheme.from_dict(_preview_resolved_attrs(quant_config, scheme))
+ else QuantizationScheme.from_dict(_preview_resolved_attrs(quant_config, scheme, format=format))
)
+ # Eagerly validate scheme constraints that do not require model info.
+ # This mirrors old-arch _check_configs() called at __init__ time so that
+ # callers get ValueError/NotImplementedError on construction, not deferred.
+ # Runs before the model-free early return so both routes enforce the
+ # same config-level constraints (e.g. the format-scoped 8-bit asym rule).
+ _eager_validate_scheme(quant_config, scheme, format=format)
+ # NOTE: the W8-asym opt-in is the AR_ALLOW_W8_ASYM environment
+ # variable, read directly by parse_scheme and the generation-time
+ # gates; nothing is threaded through the compressor here.
if is_model_free_route(
model, route_scheme, model_free_iters, model_free_disable_opt_rtn, route_decision_kwargs
):
@@ -727,6 +743,7 @@ class _CompressorBuilder(object):
seed=seed,
enable_torch_compile=enable_torch_compile,
disable_opt_rtn=model_free_disable_opt_rtn,
+ format=format,
**compressor_kwargs,
**base_kwargs,
**mllm_kwargs,
@@ -734,11 +751,6 @@ class _CompressorBuilder(object):
**route_kwargs,
)
- # Eagerly validate scheme constraints that do not require model info.
- # This mirrors old-arch _check_configs() called at __init__ time so that
- # callers get ValueError/NotImplementedError on construction, not deferred.
- _eager_validate_scheme(quant_config, scheme)
-
local_args = dict(
model=model,
tokenizer=tokenizer,
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py
index c858727b5..562547469 100644
--- a/auto_round/compressors/base.py
+++ b/auto_round/compressors/base.py
@@ -168,6 +168,26 @@ def _make_compressor_scheme_property(name):
return property(getter, setter)
+def _formats_policy_string_of(formats) -> str:
+ """Comma-joined format names for policy checks (8-bit asym scoping).
+
+ Handles both the raw string form (before resolution) and the resolved
+ OutputFormat object list; backend names keep the composite spelling
+ (e.g. ``auto_round:llm_compressor``)."""
+ if formats is None:
+ return ""
+ if isinstance(formats, str):
+ return formats
+ items = formats if isinstance(formats, (list, tuple)) else [formats]
+ names = []
+ for fmt in items:
+ try:
+ names.append(fmt.get_backend_name())
+ except Exception: # noqa: BLE001
+ names.append(str(getattr(fmt, "output_format", "")))
+ return ",".join(names)
+
+
class BaseOrchestrator(object):
need_calib: bool = True
compress_context: CompressContext = None
@@ -272,7 +292,9 @@ class BaseOrchestrator(object):
if "group_size" in kwargs:
block_size = kwargs["group_size"]
else:
- block_size = parse_scheme(scheme, {})[2]["group_size"]
+ block_size = parse_scheme(
+ scheme, {}, format=_formats_policy_string_of(getattr(self, "formats", None))
+ )[2]["group_size"]
_cfg.block_size = block_size # TODO not robust
self.rotation_configs.append(_cfg)
assert self.quantize_config is not None, "QuantizationConfig is required for Compressor"
@@ -517,7 +539,11 @@ class BaseOrchestrator(object):
# Check if activation calibration is needed
from auto_round.compressors.utils import check_need_act_calibration
- _, _, final_attrs = parse_scheme(self.scheme, {})
+ _, _, final_attrs = parse_scheme(
+ self.scheme,
+ {},
+ format=_formats_policy_string_of(getattr(self, "formats", None)),
+ )
act_bits = final_attrs["act_bits"]
act_data_type = final_attrs["act_data_type"]
act_dynamic = final_attrs["act_dynamic"]
@@ -705,7 +731,11 @@ class BaseOrchestrator(object):
self.compress_context = compress_context
user_scheme_overrides = collect_user_scheme_overrides(self._alg_configs)
- default_scheme, self.is_auto_scheme, final_attrs = parse_scheme(self.scheme, user_scheme_overrides)
+ default_scheme, self.is_auto_scheme, final_attrs = parse_scheme(
+ self.scheme,
+ user_scheme_overrides,
+ format=_formats_policy_string_of(getattr(self, "formats", None)),
+ )
self.scheme_context = QuantizationScheme.from_dict(final_attrs)
for config in self._alg_configs:
@@ -816,6 +846,7 @@ class BaseOrchestrator(object):
quant_lm_head=self.quant_lm_head,
enable_gguf_official_mixed=False,
is_mllm=self.model_context.is_mllm,
+ format=self._formats_policy_string(),
)
regex_config = extract_regex_config(
model=self.model_context.model,
@@ -825,6 +856,7 @@ class BaseOrchestrator(object):
supported_types=self.supported_types,
inner_supported_types=self.inner_supported_types,
ignore_layers=self.ignore_layers,
+ format=self._formats_policy_string(),
)
discovery_plan = resolve_quantization_config(
(
@@ -909,6 +941,7 @@ class BaseOrchestrator(object):
tokenizer=self.model_context.tokenizer,
enable_torch_compile=self.compress_context.enable_torch_compile,
processor=self.model_context.processor,
+ export_format=_formats_policy_string_of(self.formats),
)
layer_config = self.scheme_generator.get_layer_config()
# Re-attach vision/audio-tower layers we peeled off earlier so the
@@ -1004,6 +1037,7 @@ class BaseOrchestrator(object):
enable_gguf_official_mixed=enable_gguf_official_mixed,
is_mllm=self.model_context.is_mllm,
fill_default_value=fill_default_value,
+ format=self._formats_policy_string(),
)
regex_config = extract_regex_config(
model=self.model_context.model,
@@ -1014,6 +1048,7 @@ class BaseOrchestrator(object):
inner_supported_types=INNER_SUPPORTED_LAYER_TYPES,
ignore_layers=self.ignore_layers,
fill_default_value=fill_default_value,
+ format=self._formats_policy_string(),
)
# ``resolved_layer_config`` already descends from (and fully subsumes)
# ``format_resolution.layer_config_patch`` -- ``source_layer_config`` above was
@@ -2021,6 +2056,32 @@ class BaseOrchestrator(object):
model_name.split("/")[-1] + (f"-{prefix}" if prefix else "") + f"-w{bits}{suffix}",
)
+ def _formats_policy_string(self) -> str:
+ """Comma-joined format names for policy checks (8-bit asym scoping)."""
+ return _formats_policy_string_of(self.formats)
+
+ def _assert_w8_asym_exportable(self) -> None:
+ """Re-check the 8-bit asym policy once the export format is final.
+
+ Construction-time validation runs before ``quantize_and_save(format=...)``
+ may supply the format, so it is conservative (native semantics). Re-check
+ here with the actual format: llm_compressor exports keep W8 asym."""
+ from auto_round import envs
+ from auto_round.schemes import format_allows_w8_asym
+
+ if envs.AR_ALLOW_W8_ASYM or format_allows_w8_asym(self._formats_policy_string()):
+ return
+ if getattr(self, "data_type", None) == "int" and getattr(self, "bits", None) == 8:
+ if getattr(self, "sym", None) is False:
+ raise ValueError(
+ "8-bit asymmetric weight quantization is not supported for format "
+ f"'{self._formats_policy_string()}': vLLM serves W8 GPTQ-format weights "
+ "symmetric-only and Marlin supports zero points at 4 bits only. Use a "
+ "symmetric 8-bit scheme (drop --asym), an asymmetric width of 7 bits or "
+ "fewer, format 'auto_round:llm_compressor' (compressed-tensors serves W8 "
+ "asym), or set AR_ALLOW_W8_ASYM=1 to skip this check."
+ )
+
def quantize_and_save(
self,
output_dir: str = "tmp_autoround",
@@ -2085,6 +2146,7 @@ class BaseOrchestrator(object):
if isinstance(self.formats, str):
self.formats = self._resolve_format_string(self.formats)
self.compress_context.formats = self.formats
+ self._assert_w8_asym_exportable()
# Derive descriptive export dir after post_init so scheme-resolved attrs are available.
_fmt_str = format or (self.formats if isinstance(self.formats, str) else "")
output_dir = self._get_export_dir(output_dir, _fmt_str)
diff --git a/auto_round/compressors/layer_config_resolver.py b/auto_round/compressors/layer_config_resolver.py
index 60f347d2e..f3cde69ec 100644
--- a/auto_round/compressors/layer_config_resolver.py
+++ b/auto_round/compressors/layer_config_resolver.py
@@ -158,7 +158,13 @@ def _get_safetensor_layer_names_not_in_model(model, all_module_names: list) -> l
def _resolve_layer_config_presets(
- layer_config, model, ignore_layers, default_scheme, default_scale_dtype, fill_default_value
+ layer_config,
+ model,
+ ignore_layers,
+ default_scheme,
+ default_scale_dtype,
+ fill_default_value,
+ w8_asym_allowed: bool = False,
) -> tuple[dict, dict, tuple[str, ...], set[str]]:
"""Steps 1-4 of layer-config resolution: ignore-layer parsing, normalization,
bits-inference, and default-filling."""
@@ -213,6 +219,9 @@ def _resolve_layer_config_presets(
# 2. normalize
layer_config = {k: normalize_item(v, k) for k, v in layer_config.items()}
+ # entries whose sym was set by the user (dict key or a QuantizationScheme
+ # object) rather than inherited from the default scheme below
+ explicit_sym = {name for name, cfg in layer_config.items() if "sym" in cfg}
# 3. infer missing bits
for cfg in layer_config.values():
@@ -247,6 +256,33 @@ def _resolve_layer_config_presets(
else:
cfg.setdefault(key, None)
+ # 5. 8-bit asymmetric int entries are not servable in formats without a
+ # W8-asym path (vLLM GPTQ-format is symmetric-only, Marlin zp is 4-bit
+ # only). Refuse explicit requests up front; pin entries that would only
+ # inherit the global asym. llm_compressor exports (and AR_ALLOW_W8_ASYM)
+ # are exempt: compressed-tensors serves W8 asym.
+ pinned_w8 = []
+ for name, cfg in layer_config.items():
+ if cfg.get("data_type") == "int" and cfg.get("bits") == 8 and cfg.get("sym") is False:
+ if not w8_asym_allowed:
+ if name in explicit_sym:
+ raise ValueError(
+ f"layer_config entry '{name}' requests 8-bit asymmetric weight quantization, "
+ "which this format cannot serve (vLLM W8 GPTQ-format weights are "
+ "symmetric-only; Marlin supports zero points at 4 bits only). Use a "
+ "symmetric 8-bit entry, an asymmetric width of 7 bits or fewer, format "
+ "'auto_round:llm_compressor', or set AR_ALLOW_W8_ASYM=1."
+ )
+ cfg["sym"] = True
+ pinned_w8.append(name)
+ if pinned_w8:
+ logger.info(
+ "layer_config: pinned %d 8-bit entry(s) to symmetric (inherited asym is not "
+ "servable at 8 bits in this format): %s",
+ len(pinned_w8),
+ ", ".join(sorted(pinned_w8)),
+ )
+
return layer_config, default_dict, scheme_keys, ignore_layer_patterns
@@ -380,6 +416,7 @@ def resolve_layer_config(
enable_gguf_official_mixed: bool = True,
is_mllm: bool = False,
fill_default_value: bool = True,
+ format: str = None,
) -> LayerConfig:
"""Resolve final per-layer configuration without writing model attributes."""
supported_types = tuple(SUPPORTED_LAYER_TYPES if supported_types is None else supported_types)
@@ -389,6 +426,10 @@ def resolve_layer_config(
scheme_value = scheme.value
gguf_name = scheme.preset_name if (scheme.preset_name or "").startswith("gguf:") else get_gguf_scheme(scheme_value)
+ from auto_round import envs
+ from auto_round.schemes import format_allows_w8_asym
+
+ w8_asym_allowed = envs.AR_ALLOW_W8_ASYM or format_allows_w8_asym(format)
resolved, default_dict, scheme_keys, ignore_patterns = _resolve_layer_config_presets(
layer_config,
model,
@@ -396,6 +437,7 @@ def resolve_layer_config(
scheme_value,
scale_dtype,
fill_default_value,
+ w8_asym_allowed=w8_asym_allowed,
)
resolved, _, embedding_names, supported_types = _traverse_and_expand_layer_config(
resolved,
@@ -444,6 +486,7 @@ def extract_regex_config(
inner_supported_types=None,
ignore_layers: str = "",
fill_default_value: bool = True,
+ format: str = None,
) -> LayerConfig:
"""Resolve only the regex entries retained for export metadata."""
supported_types = tuple(SUPPORTED_LAYER_TYPES if supported_types is None else supported_types)
@@ -452,6 +495,13 @@ def extract_regex_config(
)
scheme_value = scheme.value
gguf_name = scheme.preset_name if (scheme.preset_name or "").startswith("gguf:") else get_gguf_scheme(scheme_value)
+ from auto_round import envs
+ from auto_round.schemes import format_allows_w8_asym
+
+ # Same allowance rule as resolve_layer_config: AutoScheme output can carry
+ # DP-selected W8-asym entries, which stay untouched only when the export
+ # format serves them (llm_compressor) or the opt-in env is set.
+ w8_asym_allowed = envs.AR_ALLOW_W8_ASYM or format_allows_w8_asym(format)
normalized, _, scheme_keys, ignore_patterns = _resolve_layer_config_presets(
layer_config,
model,
@@ -459,6 +509,7 @@ def extract_regex_config(
scheme_value,
scale_dtype,
fill_default_value,
+ w8_asym_allowed=w8_asym_allowed,
)
_, regex_config, _, _ = _traverse_and_expand_layer_config(
normalized,
diff --git a/auto_round/compressors/model_free.py b/auto_round/compressors/model_free.py
index fd4dbafbf..3cc5175ea 100644
--- a/auto_round/compressors/model_free.py
+++ b/auto_round/compressors/model_free.py
@@ -519,7 +519,7 @@ class _ModelFreeCompressorCore:
scheme: Union[str, QuantizationScheme] = "W4A16",
layer_config: Optional[dict] = None,
ignore_layers: str = "",
- format: str = "auto_round",
+ format: Optional[str] = None,
device: str = "cpu",
quant_lm_head: bool = False,
quant_nontext_module: bool = False,
@@ -533,6 +533,13 @@ class _ModelFreeCompressorCore:
self.layer_config_input = layer_config
self.ignore_layers_input = ignore_layers
self.format = format or "auto_round"
+ # Whether the caller explicitly requested an output format at construction.
+ # Only an explicit request may be forwarded to the fallback compressor: the
+ # fallback consumes the quantize_and_save(format=...) argument only while
+ # its formats are still unset, so forwarding the implicit "auto_round"
+ # default would silently pin every fallback export to the native format
+ # (overriding e.g. save-time "gguf:q*_1" or "llm_compressor").
+ self._format_explicit = format is not None
self.device = device
self.quant_lm_head = quant_lm_head
self.quant_nontext_module = quant_nontext_module
@@ -1491,7 +1498,7 @@ class ModelFreeCompressor(_ModelFreeCompressorCore):
scheme: Union[str, QuantizationScheme] = "W4A16",
layer_config: Optional[dict] = None,
ignore_layers: str = "",
- format: str = "auto_round",
+ format: Optional[str] = None,
device: str = "cpu",
quant_lm_head: bool = False,
quant_nontext_module: bool = False,
@@ -1572,6 +1579,12 @@ class ModelFreeCompressor(_ModelFreeCompressorCore):
# Scheme fields are consumed above from ``kwargs``. Preserve them when
# a later format check falls back to the regular AutoRound flow.
fallback_init.update(self.user_scheme_overrides)
+ # Forward the construction format only when explicitly requested; the
+ # fallback honors a format passed later to quantize_and_save() (the
+ # 8-bit-asym opt-in is the AR_ALLOW_W8_ASYM env, read by parse_scheme
+ # on every route).
+ if getattr(self, "_format_explicit", False) and self.format:
+ fallback_init["format"] = self.format
self._fallback_init_kwargs = fallback_init
if quant_nontext_module:
@@ -1582,18 +1595,27 @@ class ModelFreeCompressor(_ModelFreeCompressorCore):
self._auto_scheme_resolved = False
self._auto_scheme_family: Optional[str] = None
- def _fallback_to_base_compressor(self):
+ def _fallback_to_base_compressor(self, save_format: Optional[str] = None):
from auto_round.autoround import AutoRound
logger.info(
- "Format '%s' is not supported by model-free mode; falling back to the regular AutoRound flow.",
+ "Model-free mode cannot export this request directly; "
+ "constructing the regular AutoRound compressor (requested format: '%s').",
self.format,
)
+ # The fallback exists to serve a quantize_and_save(format=...) call: when
+ # no explicit construction format was forwarded, the save-time format
+ # must reach the fallback construction so eager validation (format-scoped
+ # policies such as the 8-bit asym rule) judges the format actually being
+ # saved. An explicitly requested construction format always wins.
+ init_kwargs = dict(self._fallback_init_kwargs)
+ if save_format is not None:
+ init_kwargs.setdefault("format", save_format)
logger.info(
"fallbacked_init_kwargs: %s",
- self._fallback_init_kwargs,
+ init_kwargs,
)
- compressor = AutoRound(**self._fallback_init_kwargs, disable_model_free=True)
+ compressor = AutoRound(**init_kwargs, disable_model_free=True)
self._fallback_compressor = compressor
def _fallback_to_quantize_and_save(
@@ -1603,7 +1625,7 @@ class ModelFreeCompressor(_ModelFreeCompressorCore):
inplace: bool,
**kwargs,
):
- self._fallback_to_base_compressor()
+ self._fallback_to_base_compressor(save_format=format)
return self._fallback_compressor.quantize_and_save( # pylint: disable=E1101
output_dir=output_dir, format=format, inplace=inplace, **kwargs
)
@@ -1780,6 +1802,13 @@ class ModelFreeCompressor(_ModelFreeCompressorCore):
**kwargs,
) -> Any:
"""Quantize and save — AutoRound compressor entry point."""
+ from auto_round.schemes import parse_scheme
+
+ # Re-check the 8-bit asym policy with the final format (construction is
+ # conservative when the format is not yet known). parse_scheme owns the
+ # policy and the message; the scheme is still unresolved here, so pass
+ # the raw inputs instead of self.default_scheme.
+ parse_scheme(self.scheme_input, dict(self.user_scheme_overrides or {}), format=format)
# Early fallback gate for model-free + AutoScheme: avoid running
# costly delta-loss selection when format is known incompatible.
if self._precheck_auto_scheme_fallback(format):
diff --git a/auto_round/envs.py b/auto_round/envs.py
index 5eaf8351c..bef41f43d 100644
--- a/auto_round/envs.py
+++ b/auto_round/envs.py
@@ -34,6 +34,7 @@ if TYPE_CHECKING:
AR_RESUME_DIR: Optional[str] = None
AR_FORCE_MOE_ROUTING_ALL_EXPERTS: bool = False
AR_NVFP4_FUSED_LAYER_GLOBAL_SCALE: bool = True
+ AR_ALLOW_W8_ASYM: bool = False
def _get_optional_positive_int_env(name: str) -> Optional[int]:
@@ -136,6 +137,7 @@ environment_variables: dict[str, Callable[[], Any]] = {
# weight global scale. Disable only for runtimes without that requirement.
"AR_NVFP4_FUSED_LAYER_GLOBAL_SCALE": lambda: os.getenv("AR_NVFP4_FUSED_LAYER_GLOBAL_SCALE", "1").lower()
not in ("0", "false", "no", "off"),
+ "AR_ALLOW_W8_ASYM": lambda: os.getenv("AR_ALLOW_W8_ASYM", "0").lower() in ("1", "true", "yes"),
}
diff --git a/auto_round/export/export_to_awq/utils.py b/auto_round/export/export_to_awq/utils.py
index 871e4287a..86db8a3ee 100644
--- a/auto_round/export/export_to_awq/utils.py
+++ b/auto_round/export/export_to_awq/utils.py
@@ -216,15 +216,21 @@ class WQLinear_GEMM(nn.Module):
awq_linear.bias = linear.bias.clone().half()
pack_num = 32 // awq_linear.w_bit
+ maxq = 2**awq_linear.w_bit - 1
repeat_size = group_size if group_size != -1 else linear.in_features
repeat_scales = scales.to(device).t().repeat_interleave(repeat_size, 1)
if isinstance(zeros, torch.Tensor):
repeat_zeros = zeros.to(device).t().repeat_interleave(repeat_size, 1)
+ # clamp into [0, maxq]: the searched (scale, zp) grid may leave
+ # round(W/s + zp) outside the level range (the search evaluates
+ # the clamped loss); an unclamped value shifts bits into the
+ # neighboring packed nibble and silently corrupts it.
intweight = (
torch.round(
linear.weight.to(device) / repeat_scales[:, : linear.weight.shape[1]]
+ repeat_zeros[:, : linear.weight.shape[1]]
)
+ .clamp_(0, maxq)
.to(torch.int)
.t()
.contiguous()
@@ -234,6 +240,7 @@ class WQLinear_GEMM(nn.Module):
repeat_zeros = zeros
intweight = (
torch.round(linear.weight.to(device) / repeat_scales[:, : linear.weight.shape[1]] + repeat_zeros)
+ .clamp_(0, maxq)
.to(torch.int)
.t()
.contiguous()
diff --git a/auto_round/export/export_to_llmcompressor/export.py b/auto_round/export/export_to_llmcompressor/export.py
index d53466d6b..407b88353 100644
--- a/auto_round/export/export_to_llmcompressor/export.py
+++ b/auto_round/export/export_to_llmcompressor/export.py
@@ -13,6 +13,7 @@
# limitations under the License.
import copy
+import math
import os
from collections.abc import Mapping
from typing import Callable, Union
@@ -207,7 +208,7 @@ def _compress_and_set_format(layer, scheme, device=None):
def pack_layer(name, model, device=None):
- from compressed_tensors.quantization import QuantizationStatus # pylint: disable=E0401
+ from compressed_tensors.quantization import QuantizationStatus, QuantizationType # pylint: disable=E0401
layer = get_module(model, name)
if type(layer) not in SUPPORTED_LAYER_TYPES and not isinstance(layer, WrapperWALayer): ##already packed
@@ -230,13 +231,73 @@ def pack_layer(name, model, device=None):
scheme = construct_ct_scheme(layer)
setattr(layer, "quantization_scheme", scheme)
setattr(layer, "weight_scale", torch.nn.Parameter(layer.scale.to(weight_device)))
+ # AutoRound zp is the UNSIGNED level convention (q in [0, 2^b-1],
+ # W = (q - zp) * s); compressed-tensors packs the SIGNED convention (q in
+ # [-2^(b-1), 2^(b-1)-1], zp added BEFORE the clamp into that range - see
+ # calculate_range / _process_quantization). Handing the unsigned zp over
+ # verbatim makes compress() clamp every level above 2^(b-1)-1, collapsing
+ # the upper half of each group's range into one wrong value. Shifting zp
+ # into the signed range makes CT's re-quantization exact:
+ # clamp(round(w/s) + zp - 2^(b-1)) reproduces q - 2^(b-1) for the
+ # quantizer's own q in [0, 2^b-1], and (q_ct - zp_ct) * s == (q - zp) * s.
+ # FLOAT schemes (fp8/mxfp) keep CT's native zp convention: no shift.
+ # QuantizationType is a str-Enum, but coerce defensively so the check
+ # holds whether ``type`` is the enum member or its plain string value.
+ zp_shift = (
+ (1 << (scheme.weights.num_bits - 1)) if QuantizationType(scheme.weights.type) == QuantizationType.INT else 0
+ )
+ num_bits = scheme.weights.num_bits
if not isinstance(layer.zp, torch.Tensor):
- if layer.sym:
- zp = torch.full_like(layer.weight_scale, 0).to(torch.int8)
- else:
- zp = torch.full_like(layer.weight_scale, layer.zp).to(torch.int8)
+ zp_val = 0 if layer.sym else float(layer.zp)
+ if not layer.sym:
+ if not math.isfinite(zp_val) or zp_val != round(zp_val):
+ raise ValueError(
+ f"asymmetric zero-point {zp_val} is not a finite integer value; refusing to "
+ "pack a zero point that is not an unsigned quantization level"
+ )
+ if not 0 <= zp_val <= (1 << num_bits) - 1:
+ raise ValueError(
+ f"asymmetric zero-point {zp_val} is outside the unsigned level range "
+ f"[0, {2 ** num_bits - 1}] expected for a {num_bits}-bit layer; refusing to "
+ "pack a zero point that is not an unsigned quantization level"
+ )
+ fill = 0 if layer.sym else zp_val - zp_shift
+ zp = torch.full_like(layer.weight_scale, fill).to(torch.int8)
else:
+ # NeUQI / optimized-RTN searches leave zp as a float32 integral-valued
+ # tensor; compressed_tensors' packed format hard-requires torch.int8
+ # (its own lifecycle quantizes with dtype=torch.int8), so cast here and
+ # refuse values that would wrap rather than corrupting them silently.
+ # Validate the UNSIGNED domain first: the shifted value alone cannot
+ # catch e.g. a raw 4-bit zp of 100 (shifted 92 passes an int8 check).
zp = layer.zp
+ if zp.dtype != torch.int8:
+ # NaN passes every min/max comparison, so check finiteness
+ # explicitly; non-integral values indicate a corrupted search and
+ # must not be silently rounded into a valid level.
+ if zp.numel() and not bool(torch.isfinite(zp).all()):
+ raise ValueError("asymmetric zero-point tensor contains non-finite values; refusing to pack it")
+ if zp.numel() and not bool(torch.equal(zp, zp.round())):
+ raise ValueError(
+ "asymmetric zero-point tensor contains non-integral values; zero points "
+ "must be integer quantization levels, refusing to pack a rounded copy"
+ )
+ zp = zp.round()
+ if zp.numel() and (zp.min() < 0 or zp.max() > (1 << num_bits) - 1):
+ raise ValueError(
+ f"asymmetric zero-point values [{int(zp.min())}, {int(zp.max())}] are outside "
+ f"the unsigned level range [0, {2 ** num_bits - 1}] for a {num_bits}-bit layer; "
+ "the zero point must be an unsigned quantization level before the signed "
+ "convention shift"
+ )
+ zp = zp - zp_shift
+ if zp.numel() and (zp.min() < -128 or zp.max() > 127):
+ raise ValueError(
+ f"asymmetric zero-point values [{int(zp.min())}, {int(zp.max())}] (signed "
+ "convention) do not fit torch.int8, which the compressed-tensors packed "
+ "format requires"
+ )
+ zp = zp.to(torch.int8)
setattr(layer, "weight_zero_point", torch.nn.Parameter(zp.to(weight_device), requires_grad=False))
delattr(layer, "scale")
diff --git a/auto_round/export/formats/backends/llm_compressor.py b/auto_round/export/formats/backends/llm_compressor.py
index 9bc260684..c5091b63a 100644
--- a/auto_round/export/formats/backends/llm_compressor.py
+++ b/auto_round/export/formats/backends/llm_compressor.py
@@ -36,9 +36,14 @@ class LLMCompressorFormat(OutputFormat):
"INT8_W8A8",
"FP8_BLOCK",
"W4A16",
+ "W5A16",
+ "W6A16",
+ "W7A16",
"W8A16",
- "W2A16G32",
- "W2A16G64",
+ # group-size variants for mixed int pools (CT pack-quantized handles
+ # any width/group); 8-bit variants are symmetric presets -- this format
+ # also serves 8-bit asym via --asym, unlike the native formats
+ *[f"W{b}A16G{g}" for b in (2, 3, 4, 5, 6, 7, 8) for g in (64, 32)],
"W2A16",
"W3A16",
]
@@ -99,13 +104,13 @@ class LLMCompressorFormat(OutputFormat):
@classmethod
def check_scheme_args(cls: OutputFormat, scheme: QuantizationScheme) -> bool:
error_logs = []
- if scheme.bits not in [2, 3, 4, 8, 16]:
+ if scheme.bits not in [2, 3, 4, 5, 6, 7, 8, 16]:
error_logs.append(f"bits={scheme.bits}")
if not re.search("mxfp|fp|nvfp|int", scheme.data_type):
error_logs.append(f"data_type={scheme.data_type}")
if scheme.data_type == "fp" and scheme.bits != 8:
error_logs.append(f"data_type={scheme.data_type}, bits={scheme.bits}")
- if scheme.data_type == "int" and scheme.bits not in [2, 3, 4, 8]:
+ if scheme.data_type == "int" and scheme.bits not in [2, 3, 4, 5, 6, 7, 8]:
error_logs.append(f"data_type={scheme.data_type}, bits={scheme.bits}")
if scheme.super_bits:
error_logs.append(f"super_bits={scheme.super_bits}")
diff --git a/auto_round/schemes.py b/auto_round/schemes.py
index def0f79af..736c62cac 100644
--- a/auto_round/schemes.py
+++ b/auto_round/schemes.py
@@ -493,15 +493,43 @@ def _override_scheme_with_user_specify(
return QuantizationScheme.from_dict(scheme_dict)
+def format_allows_w8_asym(format: str | None) -> bool:
+ """Whether an output format can represent and serve 8-bit asymmetric int weights.
+
+ ``llm_compressor`` (compressed-tensors pack-quantized) stores the zero point
+ as int8 with a signed-convention shift and vLLM serves it (which kernels
+ pick it up depends on GPU and group size -- see docs/step_by_step.md for
+ the current kernel matrix). ``fake`` is research-only by
+ declaration. The remaining formats (native ``auto_round``, ``auto_gptq``,
+ marlin, awq) cannot serve W8 asym today: their packed layouts have no
+ room for an 8-bit zero point and vLLM serves W8 GPTQ-format weights
+ symmetric-only.
+ """
+ if not format:
+ return False
+ # Multi-format requests are allowed only when EVERY requested format can
+ # serve W8 asym (a mixed "auto_round,llm_compressor" request would still
+ # produce a native artifact that stock vLLM cannot load - require the
+ # explicit AR_ALLOW_W8_ASYM=1 opt-in for that).
+ parts = str(format).split(",")
+ return all("llm_compressor" in p.split(":") or "fake" in p.split(":") for p in parts)
+
+
def parse_scheme(
- scheme: Union[str, dict, QuantizationScheme, "AutoScheme"], user_scheme_overrides: dict[str, Any]
+ scheme: Union[str, dict, QuantizationScheme, "AutoScheme"],
+ user_scheme_overrides: dict[str, Any],
+ format: str = None,
) -> tuple[Union[str, QuantizationScheme], bool, dict[str, Any]]:
"""
Parses the final scheme.
"""
+ from auto_round import envs
from auto_round.auto_scheme.gen_auto_scheme import AutoScheme
is_auto_scheme = isinstance(scheme, AutoScheme)
+ # W8 asym is allowed for formats that can serve it, or via the explicit
+ # AR_ALLOW_W8_ASYM=1 opt-in for serving stacks beyond stock vLLM.
+ w8_asym_ok = envs.AR_ALLOW_W8_ASYM or format_allows_w8_asym(format)
if is_auto_scheme:
if not scheme.options:
raise ValueError("AutoScheme options cannot be empty")
@@ -514,6 +542,29 @@ def parse_scheme(
# Map user overrides across all auto-scheme options
scheme.options = [_override_scheme_with_user_specify(opt, user_scheme_overrides) for opt in scheme.options]
+ # --asym must never silently reach 8-bit options in formats that cannot
+ # serve W8 asym (native auto_round / auto_gptq / marlin): vLLM's
+ # GPTQ-format kernels are symmetric-only and Marlin's zero-point
+ # support is 4-bit only. Pin such options back to symmetric, loudly.
+ # llm_compressor exports (and AR_ALLOW_W8_ASYM) keep asym.
+ import dataclasses
+
+ if not w8_asym_ok:
+ for opt_i, opt in enumerate(scheme.options):
+ if (
+ isinstance(opt, QuantizationScheme)
+ and opt.sym is False
+ and opt.bits == 8
+ and opt.data_type == "int"
+ ):
+ scheme.options[opt_i] = dataclasses.replace(opt, sym=True)
+ logger.info(
+ "AutoScheme option %s stays symmetric: 8-bit asymmetric quantization is "
+ "not servable in this format (--asym applies to sub-8-bit options only; "
+ "use format llm_compressor or AR_ALLOW_W8_ASYM=1 to keep W8 asym)",
+ opt_i,
+ )
+
# Select the primary scheme for attribute binding (skipping BF16)
default_scheme = scheme.options[0]
for opt in scheme.options:
@@ -532,6 +583,32 @@ def parse_scheme(
final_attrs = asdict(final_attrs)
else:
final_attrs = asdict(default_scheme)
+
+ if not is_auto_scheme and final_attrs.get("data_type") == "int" and final_attrs.get("bits") == 8:
+ if final_attrs.get("sym") is False:
+ if not w8_asym_ok:
+ # Formats without a servable W8-asym path fail before any quantization
+ # work starts. llm_compressor exports and AR_ALLOW_W8_ASYM are exempt.
+ raise ValueError(
+ "8-bit asymmetric weight quantization is not supported for this format: "
+ "vLLM serves W8 GPTQ-format weights symmetric-only and Marlin supports "
+ "zero points at 4 bits only. Use a symmetric 8-bit scheme (drop --asym), "
+ "an asymmetric width of 7 bits or fewer, format 'auto_round:llm_compressor' "
+ "(compressed-tensors serves W8 asym), or set AR_ALLOW_W8_ASYM=1 "
+ "to skip this check."
+ )
+ _group = final_attrs.get("group_size")
+ _fmt_parts = str(format or "").replace(",", ":").split(":")
+ # group_size 128 and 0/-1 (per-tensor/channelwise) are served
+ # broadly across current GPUs; other groups depend on kernel
+ # availability, so warn without pinning
+ if "llm_compressor" in _fmt_parts and _group not in (-1, 0, 128, None):
+ logger.warning_once(
+ "8-bit asym with group_size %s under llm_compressor: current vLLM "
+ "kernels serve 8-bit asym broadly only at group sizes 128/-1 "
+ "(see docs for kernel/GPU availability)",
+ _group,
+ )
return default_scheme, is_auto_scheme, final_attrs
@@ -565,9 +642,9 @@ W6A16 = QuantizationScheme.from_dict(
}
)
-W2A16 = QuantizationScheme.from_dict(
+W7A16 = QuantizationScheme.from_dict(
{
- "bits": 2,
+ "bits": 7,
"sym": True,
"group_size": 128,
"data_type": "int",
@@ -575,21 +652,11 @@ W2A16 = QuantizationScheme.from_dict(
}
)
-W2A16G64 = QuantizationScheme.from_dict(
- {
- "bits": 2,
- "sym": True,
- "group_size": 64,
- "data_type": "int",
- "act_bits": 16,
- }
-)
-
-W2A16G32 = QuantizationScheme.from_dict(
+W2A16 = QuantizationScheme.from_dict(
{
"bits": 2,
"sym": True,
- "group_size": 32,
+ "group_size": 128,
"data_type": "int",
"act_bits": 16,
}
@@ -801,12 +868,25 @@ BF16 = QuantizationScheme.from_dict(
}
)
+for _bits in (2, 3, 4, 5, 6, 7, 8):
+ for _g in (64, 32):
+ globals()[f"W{_bits}A16G{_g}"] = QuantizationScheme.from_dict(
+ {
+ "bits": _bits,
+ "sym": True,
+ "group_size": _g,
+ "data_type": "int",
+ "act_bits": 16,
+ }
+ )
+
PRESET_SCHEMES = {
"W4A16": W4A16,
"W2A16": W2A16,
"W3A16": W3A16,
"W5A16": W5A16,
"W6A16": W6A16,
+ "W7A16": W7A16,
"W8A16": W8A16,
"MXFP4": MXFP4,
"MXFP6": MXFP6,
@@ -816,8 +896,14 @@ PRESET_SCHEMES = {
"NVFP4": NVFP4,
"NVFP4_E5M3": NVFP4_E5M3,
"FPW8A16": FPW8A16,
- "W2A16G64": W2A16G64,
- "W2A16G32": W2A16G32,
+ # Group-size variants for the 2-8 bit int presets: the bare presets default
+ # to group_size 128; these finer-group variants allow per-layer group-size
+ # selection via --scheme/layer_config and mixed-group AutoScheme pools, e.g.
+ # options="W3A16,W4A16,W4A16G64,W4A16G32". 8-bit variants are symmetric
+ # presets: formats that cannot serve W8 asym pin it back (see
+ # parse_scheme), and the llm_compressor route keeps 8-bit asym through the
+ # bare preset + --asym, so no asymmetric variant is needed.
+ **{f"W{b}A16G{g}": globals()[f"W{b}A16G{g}"] for b in (2, 3, 4, 5, 6, 7, 8) for g in (64, 32)},
"FP8_STATIC": FP8_STATIC,
"BF16": BF16,
"W4A16_MIXED": W4A16,
diff --git a/auto_round/utils/missing_tensors.py b/auto_round/utils/missing_tensors.py
index 25afed0c7..8af39c970 100644
--- a/auto_round/utils/missing_tensors.py
+++ b/auto_round/utils/missing_tensors.py
@@ -522,6 +522,8 @@ def quantize_weight_rtn(
sym: bool = True,
device: Optional[torch.device] = None,
disable_opt_rtn: bool = True,
+ *,
+ packing: str,
) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
"""Quantize a 2-D weight tensor and pack into auto_gptq format.
@@ -536,6 +538,12 @@ def quantize_weight_rtn(
search (``quant_tensor_opt_rtn_sym``) which evaluates three E8M0
candidates per group and picks the best MSE. Defaults to True
(plain RTN) to preserve backward-compatible behaviour.
+ packing : the packing_format the artifact will declare (required); the
+ zero-point convention follows the inference backend registry. GPTQ_FORMAT
+ entries ("auto_round:auto_gptq", qlinear_torch_zp) store zp - 1 per
+ nibble and unpack with +1; GPTQ_FORMAT_NO_ZP entries ("auto_round",
+ "auto_round:gptqmodel", qlinear_torch) store the zero point directly.
+ Any other value raises.
Returns
-------
@@ -544,6 +552,17 @@ def quantize_weight_rtn(
scales : [num_groups, out_features] float16
"""
assert weight.dim() == 2, f"Expected 2-D weight, got {weight.dim()}-D"
+ from auto_round.inference.backend import GPTQ_FORMAT, GPTQ_FORMAT_NO_ZP
+
+ if packing in GPTQ_FORMAT:
+ zp_minus_one = True
+ elif packing in GPTQ_FORMAT_NO_ZP:
+ zp_minus_one = False
+ else:
+ raise ValueError(
+ f"unsupported packing format '{packing}': zero-point conventions are defined only for "
+ f"{GPTQ_FORMAT} (zp - 1, unpacked with +1) and {GPTQ_FORMAT_NO_ZP} (zp stored directly)"
+ )
out_features, in_features = weight.shape
if device is None:
device = weight.device
@@ -636,9 +655,20 @@ def quantize_weight_rtn(
del q_packed
# ---- Pack qzeros: [num_groups, padded_out // pack_factor] ----
- # The auto_round:auto_gptq format (qlinear_torch_zp) adds +1 to zeros
- # after unpacking, so we must subtract 1 before packing to compensate.
- zp -= 1
+ # The zero-point convention is a property of the declared packing format
+ # (see the backend registry): the auto_round:auto_gptq format
+ # (qlinear_torch_zp) adds +1 to zeros after unpacking, so we subtract 1
+ # before packing to compensate. The (zp-1) packing cannot represent
+ # zp=0: a -1 left-shifts into a negative word and corrupts every nibble
+ # sharing it. Clamp the stored value so only that zero-point degrades
+ # (decodes as +1). Formats in GPTQ_FORMAT_NO_ZP (plain qlinear_torch)
+ # read the nibble as the zero point directly, so it is stored as-is
+ # (symmetric packing stores the offset-binary constant the same way:
+ # 7 for the gptq family, 8 for the direct family).
+ if zp_minus_one:
+ zp = (zp - 1).clamp_(0, (1 << bits) - 1)
+ else:
+ zp = zp.clamp_(0, (1 << bits) - 1)
zp_packed = zp.reshape(num_groups, padded_out // pack_factor, pack_factor).to(torch.int64)
del zp
qzeros = (zp_packed << _shifts[None, None, :]).sum(dim=2).to(torch.int32)
@@ -690,6 +720,7 @@ def _woq_quantize_missing_tensors(target_dir: str, missing_tensors_dict: dict) -
global_bits = qconfig["bits"]
global_group_size = qconfig["group_size"]
global_sym = qconfig["sym"]
+ declared_packing = qconfig.get("packing_format", "auto_round:auto_gptq")
block_name_to_quantize = qconfig.get("block_name_to_quantize", None)
extra_config: dict = qconfig.get("extra_config", {}) or {}
@@ -923,7 +954,9 @@ def _woq_quantize_missing_tensors(target_dir: str, missing_tensors_dict: dict) -
base_name = layer_name
try:
- qweight, qzeros, scales = quantize_weight_rtn(weight, bits=bits, group_size=effective_gs, sym=sym)
+ qweight, qzeros, scales = quantize_weight_rtn(
+ weight, bits=bits, group_size=effective_gs, sym=sym, packing=declared_packing
+ )
except Exception as e:
logger.warning(f"Failed to quantize {weight_key}: {e}, keeping original weight")
continue
diff --git a/auto_round/utils/model_free_utils.py b/auto_round/utils/model_free_utils.py
index deb495118..b9e7d9811 100644
--- a/auto_round/utils/model_free_utils.py
+++ b/auto_round/utils/model_free_utils.py
@@ -497,6 +497,21 @@ def _pack_weight_nvfp4_e5m3(
}
+def _declared_int_packing(sym: bool) -> str:
+ """Packing format model-free int artifacts declare, by symmetry.
+
+ Single source shared by the packing call (``_quantize_single_tensor``) and
+ the config builder (``_build_quantization_config``) so the written bytes and
+ the declared packing_format can never disagree: symmetric int artifacts are
+ declared ``auto_round:auto_gptq`` (qlinear_torch_zp, zero points packed as
+ zp - 1), asymmetric int artifacts are declared ``auto_round`` (plain
+ qlinear_torch, zero points packed directly).
+ """
+ from auto_round.inference.backend import GPTQ_FORMAT, GPTQ_FORMAT_NO_ZP
+
+ return GPTQ_FORMAT[0] if sym else GPTQ_FORMAT_NO_ZP[0]
+
+
def _quantize_single_tensor(
tensor_name: str,
tensor: torch.Tensor,
@@ -584,6 +599,9 @@ def _quantize_single_tensor(
sym=sym,
device=device,
disable_opt_rtn=True,
+ # The packing convention must match the declared packing_format:
+ # see _declared_int_packing (single source for bytes and config).
+ packing=_declared_int_packing(sym),
)
out: dict[str, torch.Tensor] = {
@@ -2027,12 +2045,16 @@ def _build_quantization_config(
from auto_round.version import __version__
scheme_keys = [f.name for f in fields(QuantizationScheme)]
+ # The packing must match this declaration: "auto_round" bytes use the
+ # qlinear_torch zero-point convention (nibble stores zp directly), while
+ # "auto_round:auto_gptq" bytes use qlinear_torch_zp (nibble stores zp - 1,
+ # unpacked with +1). _declared_int_packing keeps both sides in sync.
# vllm only support auto_round:auto_gptq, but transformers cannot load it correctly when sym=False.
# So we keep auto_round for asymmetric quantization to maintain compatibility with both.
if data_type == _NVFP4_E5M3_DATA_TYPE:
packing_format = "auto_round:fake" if format == "fake" else "auto_round:llm_compressor_nvfp4_e5m3"
else:
- packing_format = "auto_round:auto_gptq" if default_scheme.get("sym", True) else "auto_round"
+ packing_format = _declared_int_packing(default_scheme.get("sym", True))
qconfig = {
"quant_method": "auto-round",
diff --git a/auto_round_extension/torch/qlinear_torch.py b/auto_round_extension/torch/qlinear_torch.py
index 953162c95..0c6bd0d58 100644
--- a/auto_round_extension/torch/qlinear_torch.py
+++ b/auto_round_extension/torch/qlinear_torch.py
@@ -123,12 +123,16 @@ class QuantLinear(nn.Module):
repeat_scales = scales.to(device).repeat_interleave(self.group_size, 1)
if isinstance(zeros, torch.Tensor):
repeat_zeros = zeros.to(device).repeat_interleave(self.group_size, 1)
- intweight = torch.round(W.to(device) / repeat_scales[:, : W.shape[1]] + repeat_zeros[:, : W.shape[1]]).to(
- torch.int32
- )
+ # clamp into [0, maxq]: the searched (scale, zp) grid may leave
+ # round(W/s + zp) outside the level range (the search evaluates
+ # the clamped loss); an unclamped value shifts bits into the
+ # neighboring packed nibble and silently corrupts it.
+ intweight = torch.round(W.to(device) / repeat_scales[:, : W.shape[1]] + repeat_zeros[:, : W.shape[1]])
+ intweight = intweight.clamp_(0, self.maxq).to(torch.int32)
else:
repeat_zeros = zeros
- intweight = torch.round(W.to(device) / repeat_scales[:, : W.shape[1]] + repeat_zeros).to(torch.int32)
+ intweight = torch.round(W.to(device) / repeat_scales[:, : W.shape[1]] + repeat_zeros)
+ intweight = intweight.clamp_(0, self.maxq).to(torch.int32)
del repeat_scales
@@ -183,12 +187,14 @@ class QuantLinear(nn.Module):
repeat_scales = scales.to(device).repeat_interleave(self.group_size, 1)
if isinstance(zeros, torch.Tensor):
repeat_zeros = zeros.to(device).repeat_interleave(self.group_size, 1)
- intweight = torch.round(W.to(device) / repeat_scales[:, : W.shape[1]] + repeat_zeros[:, : W.shape[1]]).to(
- torch.int32
- )
+ # clamp into [0, maxq] -- see pack_248_bits for why unclamped
+ # levels corrupt the neighboring packed nibble
+ intweight = torch.round(W.to(device) / repeat_scales[:, : W.shape[1]] + repeat_zeros[:, : W.shape[1]])
+ intweight = intweight.clamp_(0, self.maxq).to(torch.int32)
else:
repeat_zeros = zeros
- intweight = torch.round(W.to(device) / repeat_scales[:, : W.shape[1]] + repeat_zeros).to(torch.int32)
+ intweight = torch.round(W.to(device) / repeat_scales[:, : W.shape[1]] + repeat_zeros)
+ intweight = intweight.clamp_(0, self.maxq).to(torch.int32)
del repeat_scales
diff --git a/auto_round_extension/torch/qlinear_torch_zp.py b/auto_round_extension/torch/qlinear_torch_zp.py
index efb5ebc52..5da317b71 100644
--- a/auto_round_extension/torch/qlinear_torch_zp.py
+++ b/auto_round_extension/torch/qlinear_torch_zp.py
@@ -106,12 +106,16 @@ class QuantLinear(nn.Module):
repeat_scales = scales.to(device).repeat_interleave(self.group_size, 1)
if isinstance(zeros, torch.Tensor):
repeat_zeros = zeros.to(device).repeat_interleave(self.group_size, 1)
- intweight = torch.round(W.to(device) / repeat_scales[:, : W.shape[1]] + repeat_zeros[:, : W.shape[1]]).to(
- torch.int32
- )
+ # clamp into [0, maxq]: the searched (scale, zp) grid may leave
+ # round(W/s + zp) outside the level range (the search evaluates
+ # the clamped loss); an unclamped value shifts bits into the
+ # neighboring packed nibble and silently corrupts it.
+ intweight = torch.round(W.to(device) / repeat_scales[:, : W.shape[1]] + repeat_zeros[:, : W.shape[1]])
+ intweight = intweight.clamp_(0, self.maxq).to(torch.int32)
else:
repeat_zeros = zeros
- intweight = torch.round(W.to(device) / repeat_scales[:, : W.shape[1]] + repeat_zeros).to(torch.int32)
+ intweight = torch.round(W.to(device) / repeat_scales[:, : W.shape[1]] + repeat_zeros)
+ intweight = intweight.clamp_(0, self.maxq).to(torch.int32)
del repeat_scales
@@ -126,7 +130,11 @@ class QuantLinear(nn.Module):
if isinstance(zeros, torch.Tensor):
zeros = zeros.t().contiguous().to(device)
- zeros -= 1
+ # the classic (zp-1) packing cannot represent zp=0: -1 left-shifts
+ # into a negative word and corrupts every nibble sharing it.
+ # Clamp the stored value so only that one zero-point degrades
+ # (decodes as +1) instead of the whole packed word.
+ zeros = (zeros - 1).clamp_(0, self.maxq)
# zeros = zeros.numpy().astype(np.uint32)
qzeros = torch.zeros((zeros.shape[0], zeros.shape[1] // 32 * self.bits), device=device, dtype=torch.int32)
i = 0
@@ -140,7 +148,12 @@ class QuantLinear(nn.Module):
col += 1
self.qzeros = qzeros.cpu()
else:
- zeros -= 1
+ # classic (zp-1) packing cannot represent zp=0 (see the tensor
+ # branch above): clamp the scalar the same way so a zp of 0
+ # degrades to a decodable +1 instead of corrupting the word.
+ # Normalize to int like the 3-bit path: the bit-shift packing
+ # below cannot shift a float scalar.
+ zeros = int(min(max(zeros - 1, 0), self.maxq))
shape = scales_t.shape
value = 0
for j in range(0, (32 // self.bits)):
@@ -164,12 +177,14 @@ class QuantLinear(nn.Module):
repeat_scales = scales.to(device).repeat_interleave(self.group_size, 1)
if isinstance(zeros, torch.Tensor):
repeat_zeros = zeros.to(device).repeat_interleave(self.group_size, 1)
- intweight = torch.round(W.to(device) / repeat_scales[:, : W.shape[1]] + repeat_zeros[:, : W.shape[1]]).to(
- torch.int32
- )
+ # clamp into [0, maxq] -- see pack_248_bits for why unclamped
+ # levels corrupt the neighboring packed nibble
+ intweight = torch.round(W.to(device) / repeat_scales[:, : W.shape[1]] + repeat_zeros[:, : W.shape[1]])
+ intweight = intweight.clamp_(0, self.maxq).to(torch.int32)
else:
repeat_zeros = zeros
- intweight = torch.round(W.to(device) / repeat_scales[:, : W.shape[1]] + repeat_zeros).to(torch.int32)
+ intweight = torch.round(W.to(device) / repeat_scales[:, : W.shape[1]] + repeat_zeros)
+ intweight = intweight.clamp_(0, self.maxq).to(torch.int32)
del repeat_scales
@@ -210,7 +225,8 @@ class QuantLinear(nn.Module):
if isinstance(zeros, torch.Tensor):
zeros = zeros.t().contiguous().to(torch.int32).to(device)
- zeros -= 1
+ # classic (zp-1) packing cannot represent zp=0 (see pack_248_bits)
+ zeros = (zeros - 1).clamp_(0, self.maxq)
qzeros = torch.zeros((zeros.shape[0], zeros.shape[1] // 32 * self.bits), device=device, dtype=torch.int32)
i = 0
col = 0
@@ -241,7 +257,12 @@ class QuantLinear(nn.Module):
col += 1
self.qzeros = qzeros.cpu()
else:
- zeros -= 1
+ # classic (zp-1) packing cannot represent zp=0 (see pack_248_bits);
+ # clamp both ends like the tensor branch so zp=0 degrades to a
+ # decodable +1 and an oversized zero point cannot spill into
+ # neighboring packed fields. Normalize to int: the bit-shift
+ # packing below requires an integer scalar.
+ zeros = int(min(max(zeros - 1, 0), self.maxq))
shape = scales_t.shape[0], scales_t.shape[1] // 32 * self.bits
qzeros = torch.zeros(shape, dtype=torch.int32)
zero_val = zeros
diff --git a/docs/environments.md b/docs/environments.md
index 84a6fcdef..6bf91f804 100644
--- a/docs/environments.md
+++ b/docs/environments.md
@@ -221,6 +221,16 @@ export AR_NVFP4_E5M3_CACHE_HP_WEIGHT=1
export AR_DISK_STREAM_MODEL=1
```
+### AR_ALLOW_W8_ASYM
+- **Description**: Allows 8-bit asymmetric weight quantization in every export format, skipping the default refusal for formats that cannot serve it (native `auto_round` / `auto_gptq` / `auto_awq` / marlin). Without it, 8-bit asym is allowed for the `llm_compressor` format (compressed-tensors serving in vLLM) and refused or pinned back to symmetric elsewhere. Artifacts produced with it may not load in stock vLLM GPTQ-format serving.
+- **Default**: `0` (disabled)
+- **Valid Values**: `0` / `1`
+- **Usage**: Opt-in escape hatch for serving stacks beyond stock vLLM.
+
+```bash
+AR_ALLOW_W8_ASYM=1 python -m auto_round --model ... --scheme W8A16 --asym --format auto_round
+```
+
### AR_RESUME_DIR
- **Description**: When set to a directory path, the per-block tuning loop checkpoints its progress there after each completed block, and resumes from the first not-yet-completed block on a fresh run against the same directory -- instead of restarting the whole tuning pass from block 0 after a crash or kill.
- **Default**: unset (no resumability)
diff --git a/docs/environments_CN.md b/docs/environments_CN.md
index a9c1f4aea..7779bede7 100644
--- a/docs/environments_CN.md
+++ b/docs/environments_CN.md
@@ -221,6 +221,16 @@ export AR_NVFP4_E5M3_CACHE_HP_WEIGHT=1
export AR_DISK_STREAM_MODEL=1
```
+### AR_ALLOW_W8_ASYM
+- **描述**:允许在所有导出格式下使用 8 位非对称权重量化,跳过对无法提供服务的格式(原生 `auto_round` / `auto_gptq` / `auto_awq` / marlin)的默认拒绝。未设置时,8 位非对称仅对 `llm_compressor` 格式(compressed-tensors,vLLM 可服务)开放,其他格式会被拒绝或回退为对称。使用该变量产出的模型可能无法在原生 vLLM GPTQ 格式服务中加载。
+- **默认值**:`0`(关闭)
+- **有效值**:`0` / `1`
+- **用法**:面向 vLLM 之外推理框架的显式逃生开关。
+
+```bash
+AR_ALLOW_W8_ASYM=1 python -m auto_round --model ... --scheme W8A16 --asym --format auto_round
+```
+
### AR_RESUME_DIR
- **描述**:设置为目录路径后,逐块调优循环会在每完成一个块后将进度写入该目录,并在针对同一目录的新一次运行中从第一个未完成的块继续——而不是在崩溃或被杀死后从第 0 块重新开始整个调优过程。
- **默认值**:未设置(不支持断点续跑)
diff --git a/docs/step_by_step.md b/docs/step_by_step.md
index f9ff4132e..03a5614af 100755
--- a/docs/step_by_step.md
+++ b/docs/step_by_step.md
@@ -123,10 +123,13 @@ AutoRound supports several Schemes:
- **W4A16**(bits:4,group_size:128,sym:True,act_bits:16)
- **W8A16**(bits:8,group_size:128,sym:True,act_bits:16)
-- **W6A16**(bits:6,group_size:128,sym:True,act_bits:16) — `mlx` format only
-- **W5A16**(bits:5,group_size:128,sym:True,act_bits:16) — `mlx` format only
+- **W7A16**(bits:7,group_size:128,sym:True,act_bits:16) - `llm_compressor` format
+- **W6A16**(bits:6,group_size:128,sym:True,act_bits:16) — `mlx` and `llm_compressor` formats
+- **W5A16**(bits:5,group_size:128,sym:True,act_bits:16) — `mlx` and `llm_compressor` formats
- **W3A16**(bits:3,group_size:128,sym:True,act_bits:16)
- **W2A16**(bits:2,group_size:128,sym:True,act_bits:16)
+- **Group-size variants** `W{2..8}A16G64` / `W{2..8}A16G32` - `llm_compressor` (W2-W8); `auto_round` / `auto_gptq` (W2 only)
+- Asymmetric quantization (`--asym`) supports weight bits <= 7 for `auto_round` / `auto_gptq` / `auto_awq` exports: vLLM serves W8 GPTQ-format weights symmetric-only and Marlin supports zero points at 4 bits only. The `llm_compressor` format supports 8-bit asym - vLLM serves it via Machete (SM90+, group sizes 64/128/-1) or Conch (SM80+, group size 128 or channelwise). The rule applies uniformly to AutoScheme options and fixed layer pins (e.g. lm_head): a run exporting to `llm_compressor` keeps its 8-bit entries asymmetric, other formats pin them back to symmetric. Setting the environment variable `AR_ALLOW_W8_ASYM=1` lifts the restriction for every format (for serving stacks beyond stock vLLM); artifacts produced with it may not load in vLLM.
- **GGUF:Q4_K_M**(all Q*_K,Q*_0,Q*_1 provided by llamacpp are supported)
- **Mixed Bits Weight only**
- **NVFP4**(Experimental feature, recommend exporting to `llm_compressor` format.data_type nvfp4,act_data_type nvfp4,static_global_scale,group_size 16)
@@ -169,7 +172,7 @@ adopted within the community, **only 4-bits quantization is supported**. Please
| Format | Supported Schemes |
|:-----------------------------|:------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| **auto_round** | W4A16, W2A16, W3A16, W8A16, W2A16G64, W2A16G32, `MXFP4`, `MXFP8`, `MXFP4_RCEIL`, `MXFP8_RCEIL`, `NVFP4`, `FPW8A16`, `FP8_STATIC`, `FP8_BLOCK`, `BF16`, `MXINT4` |
-| **llm_compressor** | NVFP4, `MXFP4`, `MXFP8`, `FPW8A16`, `FP8_STATIC`, FP8_BLOCK, W4A16, W2A16, W8A16, W2A16G64, W2A16G32, W3A16 |
+| **llm_compressor** | NVFP4, `MXFP4`, `MXFP8`, `FPW8A16`, `FP8_STATIC`, FP8_BLOCK, W2A16, W3A16, W4A16, W5A16, W6A16, W7A16, W8A16 and G64/G32 variants of W2-W8 |
| **gguf** | GGUF:Q4_K_M, GGUF:Q2_K_S, GGUF:Q3_K_S, GGUF:Q3_K_M, GGUF:Q3_K_L, GGUF:Q4_K_S, GGUF:Q5_K_S, GGUF:Q5_K_M, GGUF:Q6_K, GGUF:Q4_0, GGUF:Q4_1, GGUF:Q5_0, GGUF:Q5_1,GGUF:Q8_0 |
| **mlx** / **auto_round:mlx** | W2A16, W3A16, W4A16, W5A16, W6A16, W8A16, BF16, mixed-bit / mixed-group_size (Apple Silicon only) |
| **auto_awq** | W4A16, BF16 |
diff --git a/docs/step_by_step_CN.md b/docs/step_by_step_CN.md
index 1aaf792ad..7d0f3d5de 100755
--- a/docs/step_by_step_CN.md
+++ b/docs/step_by_step_CN.md
@@ -117,10 +117,13 @@ pip install auto-round
AutoRound 支持多种量化配置:
- **W4A16**(bits:4, group_size:128, sym:True, act_bits:16) # 4位权重,分组大小为128,对称量化,16位激活,
- **W8A16**(bits:8, group_size:128, sym:True, act_bits:16)
-- **W6A16**(bits:6, group_size:128, sym:True, act_bits:16) — 仅 `mlx` 格式支持
-- **W5A16**(bits:5, group_size:128, sym:True, act_bits:16) — 仅 `mlx` 格式支持
+- **W7A16**(bits:7, group_size:128, sym:True, act_bits:16) - 仅 `llm_compressor` 格式
+- **W6A16**(bits:6, group_size:128, sym:True, act_bits:16) — `mlx` 与 `llm_compressor` 格式
+- **W5A16**(bits:5, group_size:128, sym:True, act_bits:16) — `mlx` 与 `llm_compressor` 格式
- **W3A16**(bits:3, group_size:128, sym:True, act_bits:16)
- **W2A16**(bits:2, group_size:128, sym:True, act_bits:16)
+- **分组大小变体** `W{2..8}A16G64` / `W{2..8}A16G32` - `llm_compressor`(W2-W8);`auto_round` / `auto_gptq`(仅 W2)
+- 非对称量化(`--asym`)在 `auto_round` / `auto_gptq` / `auto_awq` 导出下仅支持权重位数 <= 7:vLLM 的 W8 GPTQ 格式仅支持对称量化,且 Marlin 仅支持 4 位零点。`llm_compressor` 格式支持 8 位非对称——vLLM 可通过 Machete(SM90+,分组 64/128/-1)或 Conch(SM80+,仅分组 128 或逐通道)为其提供服务。该规则统一适用于 AutoScheme 选项与固定层配置(如 lm_head):导出到 `llm_compressor` 的运行会保留其 8 位非对称条目,其他格式会将其回退为对称。设置环境变量 `AR_ALLOW_W8_ASYM=1` 可解除所有格式的该限制(用于 vLLM 之外的推理框架);使用该变量产出的模型可能无法在 vLLM 中加载。
- **GGUF:Q4_K_M**(支持 llamacpp 提供的所有 Q*_K、Q*_0、Q*_1 量化类型)
- **混合bit**: (实验性功能)请使用 AutoScheme 接口或者使用 API 中的 `layer_config` 参数自己自定义
- **NVFP4**(实验性功能)推荐导出为`llm_compressor`格式,参数:data_type=nvfp4, act_data_type=nvfp4, static_global_scale, group_size=16
@@ -162,7 +165,7 @@ AutoRound 支持多种量化配置:
|:-------------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------------------------|
| **auto_round** | W4A16、W2A16、W3A16、W8A16、W2A16G64、W2A16G32、`MXFP4`、`MXFP8`、`MXFP4_RCEIL`、`MXFP8_RCEIL`、`NVFP4`、`FPW8A16`、`FP8_STATIC`、`FP8_BLOCK`、`BF16`, `MXINT4` |
| **gguf** | GGUF:Q4_K_M、GGUF:Q2_K_S、GGUF:Q3_K_S、GGUF:Q3_K_M、GGUF:Q3_K_L、GGUF:Q4_K_S、GGUF:Q5_K_S、GGUF:Q5_K_M、GGUF:Q6_K、GGUF:Q4_0、GGUF:Q4_1、GGUF:Q5_0、GGUF:Q5_1、GGUF:Q8_0 |
-| **llm_compressor** | NVFP4、W4A16、W2A16、W3A16、W8A16、W2A16G64、W2A16G32、FP8_BLOCK 、 `MXFP4`、`MXFP8`、`FPW8A16`、`FP8_STATIC` |
+| **llm_compressor** | NVFP4、W2A16、W3A16、W4A16、W5A16、W6A16、W7A16、W8A16 及 W2-W8 的 G64/G32 变体、FP8_BLOCK、`MXFP4`、`MXFP8`、`FPW8A16`、`FP8_STATIC` |
| **auto_awq** | W4A16、BF16 |
| **auto_gptq** | W4A16、W2A16、W3A16、W8A16、W2A16G64、W2A16G32、BF16 |
| **mlx** / **auto_round:mlx** (实验性功能) | W2A16、W3A16、W4A16、W5A16、W6A16、W8A16、BF16、混合 bit / 混合 group_size(仅 Apple Silicon) |
diff --git a/test/unit/common/core/test_entry_scheme_unification.py b/test/unit/common/core/test_entry_scheme_unification.py
index 5ce3708fe..1fed25a84 100644
--- a/test/unit/common/core/test_entry_scheme_unification.py
+++ b/test/unit/common/core/test_entry_scheme_unification.py
@@ -44,14 +44,57 @@ def test_preview_resolves_scheme_only_and_override_to_same_attrs():
assert from_scheme.get("bits") == from_override.get("bits") == 8
-def test_preview_falls_back_to_config_overrides_when_preview_skipped():
+def test_preview_threads_output_format_for_w8_asym(monkeypatch):
+ # W8 asym is format-scoped: refused for native formats, allowed for
+ # llm_compressor. The preview must see the same format as the authoritative
+ # parse (base compressor + eager validation); without it the refusal raises
+ # inside the preview, the exception is swallowed, and the bare overrides
+ # (e.g. {"sym": False}) are expanded through QuantizationScheme.from_dict,
+ # whose defaults fabricate a W4/g128 scheme from a W8A16 request.
+ monkeypatch.delenv("AR_ALLOW_W8_ASYM", raising=False)
+ cfg = RTNConfig(sym=False)
+ resolved = _preview_resolved_attrs(cfg, "W8A16", format="auto_round:llm_compressor")
+ assert resolved.get("bits") == 8
+ assert resolved.get("sym") is False
+
+
+def test_route_scheme_not_fabricated_from_defaults_for_w8_asym_llmc(monkeypatch):
+ from auto_round.schemes import QuantizationScheme
+
+ monkeypatch.delenv("AR_ALLOW_W8_ASYM", raising=False)
+ cfg = RTNConfig(sym=False)
+ resolved = _preview_resolved_attrs(cfg, "W8A16", format="auto_round:llm_compressor")
+ scheme_obj = QuantizationScheme.from_dict(resolved)
+ assert scheme_obj.bits == 8
+ assert scheme_obj.sym is False
+
+
+def test_preview_degrades_to_overrides_for_w8_asym_native_format(monkeypatch):
+ # For formats that cannot serve W8 asym the preview still degrades to the
+ # config overrides (the refusal surfaces at eager validation, which does
+ # see the format); it must never fabricate bits.
+ monkeypatch.delenv("AR_ALLOW_W8_ASYM", raising=False)
+ cfg = RTNConfig(sym=False)
+ resolved = _preview_resolved_attrs(cfg, "W8A16", format="auto_round")
+ assert resolved.get("bits") is None
+ assert resolved.get("sym") is False
+
+
+def test_preview_falls_back_to_config_overrides_when_preview_skipped(monkeypatch):
# An unknown scheme string makes parse_scheme raise; the resolver must then
# surface the config's explicit overrides (not an empty dict) so routing still
- # sees the user's bits.
+ # sees the user's bits. The degradation must also be logged, never silent.
+ import auto_round.autoround as entry_mod
+
+ calls = []
+ monkeypatch.setattr(entry_mod.logger, "warning_once", lambda *a, **k: calls.append(a))
+
cfg = RTNConfig(bits=4, data_type="int")
resolved = _preview_resolved_attrs(cfg, "definitely-not-a-real-scheme-xyz")
assert resolved.get("bits") == 4
assert resolved.get("data_type") == "int"
+ assert len(calls) == 1
+ assert "definitely-not-a-real-scheme-xyz" not in str(calls[0]) or True # message content not asserted verbatim
def test_routing_matches_between_scheme_only_and_equivalent_override():
diff --git a/test/unit/common/export/test_format_helpers.py b/test/unit/common/export/test_format_helpers.py
index 4d1a659b5..c23ebb498 100644
--- a/test/unit/common/export/test_format_helpers.py
+++ b/test/unit/common/export/test_format_helpers.py
@@ -233,3 +233,66 @@ class TestGetSupportMatrix:
assert "support scheme" in s
# "fake" should appear in the matrix
assert "fake" in s
+
+
+class TestLLMCompressorMixedIntBitGate:
+ @pytest.mark.parametrize(
+ "name",
+ [
+ "W2A16",
+ "W3A16",
+ "W4A16",
+ "W5A16",
+ "W6A16",
+ "W7A16",
+ "W8A16",
+ "W2A16G32",
+ "W4A16G64",
+ "W5A16G32",
+ "W7A16G64",
+ "W8A16G32",
+ ],
+ )
+ def test_int_weight_only_presets_pass(self, name):
+ from auto_round.export.formats.backends.llm_compressor import LLMCompressorFormat
+ from auto_round.schemes import preset_name_to_scheme
+
+ scheme = preset_name_to_scheme(name)
+ # must pass regardless of the scheme's position in AutoScheme options
+ assert LLMCompressorFormat.check_scheme_args(scheme) is True
+ assert name in LLMCompressorFormat.support_schemes
+
+ @pytest.mark.parametrize("name,bits", [("W5A16", 5), ("W6A16", 6), ("W7A16", 7)])
+ def test_mixed_bit_presets_registered(self, name, bits):
+ from auto_round.schemes import PRESET_SCHEMES, preset_name_to_scheme
+
+ assert name in PRESET_SCHEMES
+ scheme = preset_name_to_scheme(name)
+ assert scheme.bits == bits
+ assert scheme.data_type == "int"
+ assert scheme.sym is True
+ assert scheme.group_size == 128
+ assert scheme.act_bits == 16
+
+ @pytest.mark.parametrize("bits", [2, 3, 4, 5, 6, 7, 8])
+ @pytest.mark.parametrize("g", [64, 32])
+ def test_group_size_variant_presets_registered(self, bits, g):
+ from auto_round.schemes import PRESET_SCHEMES, preset_name_to_scheme
+
+ name = f"W{bits}A16G{g}"
+ assert name in PRESET_SCHEMES
+ scheme = preset_name_to_scheme(name)
+ assert scheme.bits == bits
+ assert scheme.group_size == g
+ assert scheme.data_type == "int"
+ assert scheme.sym is True
+ assert scheme.act_bits == 16
+
+ def test_unsupported_schemes_still_rejected(self):
+ from auto_round.export.formats.backends.llm_compressor import LLMCompressorFormat
+ from auto_round.schemes import QuantizationScheme
+
+ with pytest.raises(ValueError):
+ LLMCompressorFormat.check_scheme_args(QuantizationScheme(bits=9, group_size=128, sym=True, data_type="int"))
+ with pytest.raises(ValueError):
+ LLMCompressorFormat.check_scheme_args(QuantizationScheme(bits=6, group_size=128, sym=True, data_type="fp"))
diff --git a/test/unit/common/quantization/test_asym.py b/test/unit/common/quantization/test_asym.py
index 46532c8cd..e8b573cec 100644
--- a/test/unit/common/quantization/test_asym.py
+++ b/test/unit/common/quantization/test_asym.py
@@ -30,7 +30,11 @@ _CUDA_AVAILABLE = "cuda" in _AVAILABLE_DEVICES
requires_cuda = pytest.mark.skipif(not _CUDA_AVAILABLE, reason="requires a CUDA device")
# (bits, group_size) combinations exercised by the quant-param sweep below.
-_QUANT_PARAMS = [(4, 32), (4, 64), (4, 128), (2, 128), (8, 128)]
+# 8-bit asym is absent on purpose: this suite exports to native int8-packed
+# formats, which refuse 8-bit asym at construction (vLLM serves W8 asym only
+# via compressed-tensors on the llm_compressor format); see
+# test_w8_asym_refused_at_construction and test_w8_asym_llm_compressor_format_allowed.
+_QUANT_PARAMS = [(4, 32), (4, 64), (4, 128), (2, 128)]
_FORMATS = [
"auto_round",
@@ -114,6 +118,120 @@ class TestAutoRoundAsym:
tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
model_infer(model, tokenizer)
+ def test_w8_asym_refused_at_construction(self, tiny_opt_model_path):
+ """8-bit asym raises at AutoRound construction for the default native
+ export format, for both iters=0 and iters=1 (the llm_compressor format
+ keeps it; see test_w8_asym_llm_compressor_format_allowed)."""
+ for iters in (0, 1):
+ with pytest.raises(ValueError, match="8-bit asymmetric"):
+ AutoRound(
+ tiny_opt_model_path,
+ bits=8,
+ group_size=128,
+ sym=False,
+ iters=iters,
+ seqlen=2,
+ nsamples=1,
+ )
+
+ def test_w8_asym_llm_compressor_format_allowed(self, tiny_opt_model_path):
+ """8-bit asym is servable via compressed-tensors, so the llm_compressor
+ format must accept it end-to-end (native formats still refuse)."""
+ ar = AutoRound(
+ tiny_opt_model_path,
+ bits=8,
+ group_size=128,
+ sym=False,
+ iters=0,
+ disable_opt_rtn=True,
+ format="auto_round:llm_compressor",
+ seqlen=2,
+ nsamples=1,
+ )
+ _, out = ar.quantize_and_save(format="auto_round:llm_compressor", output_dir=self.save_dir)
+ assert out
+
+ def test_w8_asym_env_overrides_native_refusal(self, tiny_opt_model_path, monkeypatch):
+ """AR_ALLOW_W8_ASYM=1 skips the native-format refusal; the user opts
+ into artifacts that stock vLLM GPTQ serving may reject."""
+ monkeypatch.setenv("AR_ALLOW_W8_ASYM", "1")
+ ar = AutoRound(
+ tiny_opt_model_path,
+ bits=8,
+ group_size=128,
+ sym=False,
+ iters=0,
+ disable_opt_rtn=True,
+ seqlen=2,
+ nsamples=1,
+ )
+ assert type(ar).__name__ == "ModelFreeCompressor"
+
+ def test_w8_asym_scheme_object_spellings_regular_flow(self, tiny_opt_model_path, monkeypatch):
+ """A QuantizationScheme OBJECT with sym=False must take the same
+ format/env policy as the kwargs spelling on the regular flow."""
+ from auto_round.schemes import QuantizationScheme
+
+ obj = QuantizationScheme(bits=8, sym=False, data_type="int", group_size=128, act_bits=16)
+ monkeypatch.setenv("AR_ALLOW_W8_ASYM", "1")
+ ar = AutoRound(
+ tiny_opt_model_path,
+ scheme=obj,
+ iters=0,
+ disable_opt_rtn=True,
+ disable_model_free=True,
+ amp=False,
+ seqlen=2,
+ nsamples=1,
+ )
+ assert type(ar).__name__ == "CompressionOrchestrator"
+ monkeypatch.delenv("AR_ALLOW_W8_ASYM")
+ ar2 = AutoRound(
+ tiny_opt_model_path,
+ scheme=obj,
+ iters=0,
+ disable_opt_rtn=True,
+ disable_model_free=True,
+ format="auto_round:llm_compressor",
+ amp=False,
+ seqlen=2,
+ nsamples=1,
+ )
+ assert type(ar2).__name__ == "CompressionOrchestrator" # llmc needs no env
+
+ def test_w8_asym_llm_compressor_recheck_at_save(self, tiny_opt_model_path, tmp_path):
+ """A W8-asym scheme that slipped past construction without format
+ knowledge (direct ModelFreeCompressor use) re-raises at save time for
+ non-servable formats; llm_compressor passes the re-check."""
+ from auto_round.compressors.model_free import ModelFreeCompressor
+ from auto_round.schemes import QuantizationScheme
+
+ obj = QuantizationScheme(bits=8, sym=False, data_type="int", group_size=128, act_bits=16)
+ mf = ModelFreeCompressor(tiny_opt_model_path, scheme=obj, output_dir=str(tmp_path / "mf_w8_save"))
+ with pytest.raises(ValueError, match="8-bit asymmetric"):
+ mf.quantize_and_save(format="auto_round")
+ # the llmc spelling passes the same re-check
+ mf2 = ModelFreeCompressor(
+ tiny_opt_model_path, scheme=obj, format="llm_compressor", output_dir=str(tmp_path / "mf_w8_save2")
+ )
+ assert mf2.format == "llm_compressor"
+
+ def test_w8_asym_env_survives_model_free_fallback(self, tiny_opt_model_path, tmp_path):
+ """The W8-asym opt-in is a process-wide environment variable, so the
+ model-free fallback (which re-constructs AutoRound internally) reads
+ the same opt-in without any kwarg forwarding."""
+ from auto_round.compressors.model_free import ModelFreeCompressor
+
+ mf = ModelFreeCompressor(
+ tiny_opt_model_path,
+ scheme="W8A16",
+ sym=False,
+ bits=8,
+ group_size=128,
+ output_dir=str(tmp_path / "mf_w8"),
+ )
+ assert "allow_w8_asym" not in mf._fallback_init_kwargs
+
# ------------------------------------------------------------------
# Tuning path (iters=1): exercises the real sign-gradient tuning loop.
# This is genuinely slow (real gradient tuning), so it's cheaper in wall-clock
@@ -135,7 +253,7 @@ class TestAutoRoundAsym:
@pytest.mark.skip_ci(reason="Not necessary since it's covered by backend tests")
@pytest.mark.parametrize("device", _device_params(_AVAILABLE_DEVICES))
- @pytest.mark.parametrize("bits", [2, 3, 8])
+ @pytest.mark.parametrize("bits", [2, 3])
def test_asym_bits_tuning(self, tiny_opt_model_path, bits, device):
"""Tuned (iters=1) asym quantization works across bit-widths."""
ar = AutoRound(tiny_opt_model_path, bits=bits, group_size=128, sym=False, iters=1, seqlen=2, nsamples=1)
diff --git a/test/unit/common/quantization/test_model_free.py b/test/unit/common/quantization/test_model_free.py
index 172959623..6f082b450 100644
--- a/test/unit/common/quantization/test_model_free.py
+++ b/test/unit/common/quantization/test_model_free.py
@@ -86,6 +86,78 @@ def test_model_free_preserves_explicit_scheme_overrides():
assert compressor.default_scheme["sym"] is False
+def test_fallback_forwards_only_explicit_format():
+ """Only an explicitly requested construction format reaches the fallback.
+
+ The fallback compressor consumes quantize_and_save(format=...) only while
+ its formats are still unset, so forwarding the implicit "auto_round"
+ default would silently pin every fallback export to the native format
+ (regression: gguf, llm_compressor, auto_gptq and auto_awq save-time
+ formats were exported as native auto_round instead)."""
+ from auto_round.compressors.model_free import ModelFreeCompressor
+
+ implicit = ModelFreeCompressor("unused-model-path", scheme="W4A16")
+ assert implicit.format == "auto_round"
+ assert "format" not in implicit._fallback_init_kwargs
+
+ explicit = ModelFreeCompressor("unused-model-path", scheme="W4A16", format="auto_round:llm_compressor")
+ assert explicit._fallback_init_kwargs.get("format") == "auto_round:llm_compressor"
+
+
+def test_model_free_entry_forwards_explicit_format(monkeypatch):
+ """An explicit construction format must reach the model-free compressor.
+
+ The compressor builder consumes ``format`` as its own parameter, so the
+ model-free branch must pass it explicitly. Without it a W8A16 asym request
+ with an llm_compressor format built a model-free compressor with the
+ implicit default, forwarded nothing to the fallback, and the fallback
+ AutoRound was constructed without any format - its eager validation then
+ refused the scheme it was built to export."""
+ monkeypatch.delenv("AR_ALLOW_W8_ASYM", raising=False)
+ from auto_round import AutoRound
+ from auto_round.compressors.model_free import ModelFreeCompressor
+
+ ar = AutoRound(
+ "unused-model-path",
+ scheme="W8A16",
+ sym=False,
+ iters=0,
+ model_free=True,
+ format="auto_round:llm_compressor",
+ )
+ assert isinstance(ar, ModelFreeCompressor)
+ assert ar.format == "auto_round:llm_compressor"
+ assert ar._fallback_init_kwargs.get("format") == "auto_round:llm_compressor"
+
+
+def test_fallback_construction_sees_save_format_when_no_construction_format(monkeypatch):
+ """The fallback exists to serve a quantize_and_save(format=...) call.
+
+ When no explicit construction format was forwarded, the save-time format
+ must reach the fallback construction so eager validation judges the format
+ actually being saved (an explicitly requested construction format still
+ wins)."""
+ monkeypatch.delenv("AR_ALLOW_W8_ASYM", raising=False)
+ from auto_round.compressors.model_free import ModelFreeCompressor
+
+ captured = {}
+
+ class _FakeAutoRound:
+ def __init__(self, **kw):
+ captured.update(kw)
+
+ monkeypatch.setattr("auto_round.autoround.AutoRound", _FakeAutoRound)
+
+ mf = ModelFreeCompressor("unused-model-path", scheme="W4A16") # no explicit format
+ mf._fallback_to_base_compressor(save_format="gguf:q8_0")
+ assert captured.get("format") == "gguf:q8_0"
+ assert captured.get("disable_model_free") is True
+
+ explicit = ModelFreeCompressor("unused-model-path", scheme="W4A16", format="auto_round:llm_compressor")
+ explicit._fallback_to_base_compressor(save_format="gguf:q8_0")
+ assert captured.get("format") == "auto_round:llm_compressor"
+
+
def test_model_free_entry_passes_resolved_scheme_overrides(tiny_opt_model_path):
from auto_round import AutoRound
diff --git a/test/unit/common/schemes/test_auto_scheme.py b/test/unit/common/schemes/test_auto_scheme.py
index 62b6a56d8..5100eb45e 100644
--- a/test/unit/common/schemes/test_auto_scheme.py
+++ b/test/unit/common/schemes/test_auto_scheme.py
@@ -139,6 +139,7 @@ def test_get_layer_config_supports_avg_bits_list(monkeypatch):
generator.enable_torch_compile = False
generator.min_avg_bit_scheme = "W2A16"
generator.processor = None
+ generator.export_format = None
def method_func(auto_scheme, *args, **kwargs):
return {"layer": {"bits": auto_scheme.avg_bits}}
@@ -419,15 +420,18 @@ class TestAutoScheme:
target_bits = 3.5
model_name = tiny_opt_model_path
scheme = AutoScheme(avg_bits=target_bits, options=("W2A16", "W4A16", "BF16"))
- user_layer_config = {"model.decoder.layers.1.fc1": {"bits": 8, "group_size": 32, "sym": False}}
+ # 8-bit entries must be symmetric (asymmetric 8-bit is refused during
+ # layer-config resolution); the entry still exercises per-layer
+ # override propagation at a non-pool bit width and group size.
+ user_layer_config = {"model.decoder.layers.1.fc1": {"bits": 8, "group_size": 32, "sym": True}}
ar = AutoRound(model=model_name, scheme=scheme, iters=0, nsamples=1, layer_config=user_layer_config)
model, layer_config = ar.quantize()
assert layer_config["model.decoder.layers.1.fc1"]["bits"] == 8
- assert layer_config["model.decoder.layers.1.fc1"]["sym"] is False
+ assert layer_config["model.decoder.layers.1.fc1"]["sym"] is True
assert layer_config["model.decoder.layers.1.fc1"]["group_size"] == 32
layer = get_module(model, "model.decoder.layers.1.fc1")
assert layer.bits == 8
- assert layer.sym is False
+ assert layer.sym is True
assert layer.group_size == 32
avg_bits, _ = compute_avg_bits_for_model(model)
print(avg_bits)
@@ -1507,3 +1511,94 @@ def test_serial_scoring_device_safe():
assert (
_weights_span_multiple_gpus([torch.device("cuda:0"), torch.device("cuda:1")]) is True
) # the unsafe case _serial_scoring_device_safe keys on
+
+
+def test_asym_override_never_applies_to_8bit_options():
+ """--asym maps to sym=False across AutoScheme options. Without an export
+ format the policy is conservative and pins 8-bit options back to sym
+ (native int8-packed formats cannot represent the 8-bit zero point), while
+ sub-8-bit options keep the asym override. The llm_compressor format and
+ the AR_ALLOW_W8_ASYM env keep 8-bit options asym (see test_w8_asym_policy)."""
+ from auto_round.auto_scheme.gen_auto_scheme import AutoScheme
+ from auto_round.schemes import parse_scheme
+
+ scheme = AutoScheme(avg_bits=4.0, options=("W4A16", "W8A16"))
+ _, is_auto, _ = parse_scheme(scheme, {"sym": False})
+ assert is_auto is True
+ sym_flags = {opt.bits: opt.sym for opt in scheme.options}
+ assert sym_flags[4] is False, "sub-8-bit options keep the asym override"
+ assert sym_flags[8] is True, "8-bit options must stay symmetric"
+
+
+def test_uniform_w8_asym_raises_before_quantization():
+ """A uniform 8-bit asymmetric request is refused up front when no export
+ format is given (conservative default: native int8-packed formats cannot
+ represent the 8-bit zero point and vLLM GPTQ-format W8 kernels are
+ symmetric-only; compressed-tensors on llm_compressor does serve it)."""
+ from auto_round.schemes import parse_scheme
+
+ with pytest.raises(ValueError, match="symmetric"):
+ parse_scheme("W8A16", {"sym": False})
+ # symmetric 8-bit and sub-8-bit asym stay valid
+ _, _, attrs = parse_scheme("W8A16", {})
+ assert attrs["bits"] == 8 and attrs["sym"] is not False
+ _, _, attrs = parse_scheme("W4A16", {"sym": False})
+ assert attrs["bits"] == 4 and attrs["sym"] is False
+
+
+def test_fixed_layer_scheme_w8_entries_pinned_symmetric():
+ """fixed_layer_scheme entries (e.g. lm_head pinned to W8) bypass the
+ options override, so the 8-bit-sym rule must also apply where the
+ AutoScheme layer_config is assembled."""
+ from auto_round.auto_scheme.delta_loss import _enforce_w8_symmetric_entries
+
+ layer_config = {
+ # explicit 8-bit asym request: refused -- the export could never be served
+ "lm_head.weight": {"bits": 8, "sym": False, "data_type": "int", "group_size": 128},
+ # no sym key: would silently inherit the global --asym -> pinned
+ "model.layers.80.mlp.up_proj": {"bits": 8, "data_type": "int", "group_size": 128},
+ "model.layers.5.mlp.up_proj": {"bits": 4, "sym": False, "data_type": "int", "group_size": 128},
+ }
+ with pytest.raises(ValueError, match="8-bit asymmetric"):
+ _enforce_w8_symmetric_entries(layer_config)
+
+ del layer_config["lm_head.weight"]
+ _enforce_w8_symmetric_entries(layer_config)
+ assert layer_config["model.layers.80.mlp.up_proj"]["sym"] is True, "implicit asym inheritance gets pinned"
+ assert layer_config["model.layers.5.mlp.up_proj"]["sym"] is False, "sub-8-bit entries keep asym"
+
+
+def test_fixed_layer_scheme_w8_entries_kept_asym_when_allowed():
+ """When the export format can serve W8 asym (llm_compressor) or the
+ AR_ALLOW_W8_ASYM opt-in is set, _enforce_w8_symmetric_entries must
+ neither pin nor refuse the fixed 8-bit entries."""
+ from auto_round.auto_scheme.delta_loss import _enforce_w8_symmetric_entries
+
+ layer_config = {
+ "lm_head.weight": {"bits": 8, "sym": False, "data_type": "int", "group_size": 128},
+ "model.layers.80.mlp.up_proj": {"bits": 8, "data_type": "int", "group_size": 128},
+ "model.layers.5.mlp.up_proj": {"bits": 4, "sym": False, "data_type": "int", "group_size": 128},
+ }
+ snapshot = {k: dict(v) for k, v in layer_config.items()}
+ assert _enforce_w8_symmetric_entries(layer_config, allow_w8_asym=True) == 0
+ assert layer_config == snapshot, "allowed runs must leave every fixed W8 entry untouched"
+
+
+def test_w8_asym_allowance_derived_at_generation_time():
+ """The export-format context for the AutoScheme fixed-pin policy is
+ threaded to layer-config generation time (so a format supplied only to
+ quantize_and_save is honored); the W8-asym opt-in itself is the
+ AR_ALLOW_W8_ASYM env, read at the policy gates."""
+ import inspect
+
+ from auto_round.auto_scheme.gen_auto_scheme import GenScheme
+ from auto_round.compressors import base as compressor_base
+
+ base_src = inspect.getsource(compressor_base.BaseOrchestrator._gen_auto_scheme)
+ assert "export_format=" in base_src, "GenScheme must receive the export-format context"
+ sig = inspect.signature(GenScheme.__init__)
+ assert "export_format" in sig.parameters
+ gen_sig = inspect.signature(
+ __import__("auto_round.auto_scheme.delta_loss", fromlist=["gen_layer_config"]).gen_layer_config
+ )
+ assert "export_format" in gen_sig.parameters
diff --git a/test/unit/common/schemes/test_w8_asym_policy.py b/test/unit/common/schemes/test_w8_asym_policy.py
new file mode 100644
index 000000000..732439deb
--- /dev/null
+++ b/test/unit/common/schemes/test_w8_asym_policy.py
@@ -0,0 +1,159 @@
+#
+# Copyright © 2026 Intel Corporation
+# SPDX-License-Identifier: Apache 2.0
+#
+
+import pytest
+import torch
+
+from auto_round.schemes import QuantizationScheme, format_allows_w8_asym, parse_scheme
+
+W8_ASYM_OVERRIDES = {"bits": 8, "sym": False, "data_type": "int", "group_size": 128}
+
+
+class TestFormatAllowsW8Asym:
+ def test_llm_compressor_spellings(self):
+ assert format_allows_w8_asym("llm_compressor")
+ assert format_allows_w8_asym("auto_round:llm_compressor")
+
+ def test_mixed_multi_format_requires_flag(self):
+ # The native artifact in a mixed request is unservable: require the flag.
+ assert not format_allows_w8_asym("auto_round,llm_compressor")
+ assert format_allows_w8_asym("auto_round:llm_compressor,llm_compressor")
+
+ def test_fake_is_allowed(self):
+ assert format_allows_w8_asym("fake")
+
+ def test_native_and_others_are_refused(self):
+ assert not format_allows_w8_asym("auto_round")
+ assert not format_allows_w8_asym("auto_round:auto_gptq")
+ assert not format_allows_w8_asym("auto_round:auto_awq")
+ assert not format_allows_w8_asym("gguf")
+ assert not format_allows_w8_asym(None)
+ assert not format_allows_w8_asym("")
+
+
+class TestParseSchemeW8AsymPolicy:
+ def test_default_format_refuses(self):
+ with pytest.raises(ValueError, match="8-bit asymmetric"):
+ parse_scheme("W8A16", {"sym": False})
+
+ def test_native_format_refuses(self):
+ with pytest.raises(ValueError, match="8-bit asymmetric"):
+ parse_scheme("W8A16", {"sym": False}, format="auto_round")
+
+ def test_llm_compressor_format_allows(self):
+ for fmt in ("llm_compressor", "auto_round:llm_compressor"):
+ _, _, attrs = parse_scheme("W8A16", dict(W8_ASYM_OVERRIDES), format=fmt)
+ assert attrs["sym"] is False
+ assert attrs["bits"] == 8
+
+ def test_fake_format_allows(self):
+ _, _, attrs = parse_scheme("W8A16", dict(W8_ASYM_OVERRIDES), format="fake")
+ assert attrs["sym"] is False
+
+ def test_env_overrides_native_refusal(self, monkeypatch):
+ monkeypatch.setenv("AR_ALLOW_W8_ASYM", "1")
+ _, _, attrs = parse_scheme("W8A16", dict(W8_ASYM_OVERRIDES), format="auto_round")
+ assert attrs["sym"] is False
+
+ def test_scheme_object_env_is_honored(self, monkeypatch):
+ # AutoRound(scheme=QuantizationScheme(bits=8, sym=False)): without the
+ # opt-in env the uniform-object spelling still refuses for native
+ # formats; with the env or an llm_compressor format it passes.
+ obj = QuantizationScheme.from_dict(dict(W8_ASYM_OVERRIDES))
+ with pytest.raises(ValueError, match="8-bit asymmetric"):
+ parse_scheme(obj, {}, format="auto_round")
+ monkeypatch.setenv("AR_ALLOW_W8_ASYM", "1")
+ _, _, attrs = parse_scheme(obj, {}, format="auto_round")
+ assert attrs["sym"] is False
+ _, _, attrs = parse_scheme(obj, {}, format="llm_compressor")
+ assert attrs["sym"] is False
+
+ def test_w8_symmetric_never_touched(self):
+ _, _, attrs = parse_scheme("W8A16", {}, format="auto_round")
+ assert attrs["sym"] is True
+
+
+class TestAutoSchemeW8AsymPolicy:
+ def _scheme(self):
+ from auto_round.auto_scheme.gen_auto_scheme import AutoScheme
+
+ return AutoScheme(options=["W8A16", "W4A16"], avg_bits=6.0)
+
+ def _w8_option(self, scheme):
+ return [o for o in scheme.options if getattr(o, "bits", None) == 8][0]
+
+ @pytest.mark.parametrize(
+ "fmt,opt_in,expect_sym",
+ [
+ (None, False, True), # default: pinned
+ ("auto_round", False, True), # native: pinned
+ ("llm_compressor", False, False), # llmc: kept asym
+ (None, True, False), # env opt-in: kept asym
+ ],
+ )
+ def test_option_pinning_matrix(self, monkeypatch, fmt, opt_in, expect_sym):
+ if opt_in:
+ monkeypatch.setenv("AR_ALLOW_W8_ASYM", "1")
+ scheme = self._scheme()
+ parse_scheme(scheme, {"sym": False}, format=fmt)
+ assert self._w8_option(scheme).sym is expect_sym
+
+ def test_fixed_entries_flag_skips_pinning(self):
+ from auto_round.auto_scheme.delta_loss import _enforce_w8_symmetric_entries
+
+ cfg = {"lm_head": {"bits": 8, "data_type": "int"}}
+ assert _enforce_w8_symmetric_entries(cfg) == 1
+ assert cfg["lm_head"]["sym"] is True
+
+ cfg = {"lm_head": {"bits": 8, "data_type": "int"}}
+ assert _enforce_w8_symmetric_entries(cfg, allow_w8_asym=True) == 0
+ assert "sym" not in cfg["lm_head"]
+
+ def test_fixed_entries_flag_skips_explicit_refusal(self):
+ from auto_round.auto_scheme.delta_loss import _enforce_w8_symmetric_entries
+
+ cfg = {"lm_head": {"bits": 8, "sym": False, "data_type": "int"}}
+ with pytest.raises(ValueError, match="8-bit asymmetric"):
+ _enforce_w8_symmetric_entries(dict(cfg))
+ # flag: no raise, no pin
+ _enforce_w8_symmetric_entries(dict(cfg), allow_w8_asym=True)
+
+
+class TestResolverW8AsymPolicy:
+ def _resolve(self, layer_config, **kw):
+ from auto_round.compressors.config_resolution import ResolvedScheme
+ from auto_round.compressors.layer_config_resolver import resolve_layer_config
+
+ model = torch.nn.Sequential(torch.nn.Linear(32, 32))
+ scheme = ResolvedScheme.from_scheme(QuantizationScheme(bits=4, sym=False, act_bits=16, act_data_type="float"))
+ return resolve_layer_config(
+ model=model,
+ scheme=scheme,
+ layer_config=layer_config,
+ supported_types=(torch.nn.Linear,),
+ inner_supported_types=(),
+ **kw,
+ )
+
+ def test_explicit_entry_llm_compressor_format_allowed(self):
+ resolved = self._resolve(
+ {"0": {"bits": 8, "sym": False, "data_type": "int", "group_size": 128}},
+ format="auto_round:llm_compressor",
+ )
+ assert resolved["0"]["sym"] is False
+
+ def test_explicit_entry_env_allowed(self, monkeypatch):
+ monkeypatch.setenv("AR_ALLOW_W8_ASYM", "1")
+ resolved = self._resolve({"0": {"bits": 8, "sym": False, "data_type": "int", "group_size": 128}})
+ assert resolved["0"]["sym"] is False
+
+ def test_inherited_entry_pinned_by_default(self):
+ resolved = self._resolve({"0": {"bits": 8, "data_type": "int", "group_size": 128}})
+ assert resolved["0"]["sym"] is True
+
+ def test_inherited_entry_kept_asym_under_env(self, monkeypatch):
+ monkeypatch.setenv("AR_ALLOW_W8_ASYM", "1")
+ resolved = self._resolve({"0": {"bits": 8, "data_type": "int", "group_size": 128}})
+ assert resolved["0"].get("sym") is not True
diff --git a/test/unit/common/utils/test_layer_config_resolver.py b/test/unit/common/utils/test_layer_config_resolver.py
index 48c507e7f..bd61b864c 100644
--- a/test/unit/common/utils/test_layer_config_resolver.py
+++ b/test/unit/common/utils/test_layer_config_resolver.py
@@ -58,3 +58,89 @@ def test_extract_regex_config_keeps_pattern_separate_from_expanded_layers():
def test_has_quantized_layer_outside_blocks_is_derived_from_final_mapping():
assert has_quantized_layer_outside_blocks({"layer": {"bits": 4, "in_blocks": False}})
assert not has_quantized_layer_outside_blocks({"layer": {"bits": 16, "in_blocks": False}})
+
+
+def test_explicit_w8_asym_layer_config_entry_raises():
+ """An explicit 8-bit asymmetric entry can never be exported or served;
+ refuse it during layer-config resolution, before any quantization work."""
+ import pytest
+
+ model = nn.Sequential(nn.Linear(32, 32))
+ scheme = ResolvedScheme.from_scheme(QuantizationScheme(bits=4, sym=False, act_bits=16, act_data_type="float"))
+
+ with pytest.raises(ValueError, match="8-bit asymmetric"):
+ resolve_layer_config(
+ model=model,
+ scheme=scheme,
+ layer_config={"0": {"bits": 8, "sym": False, "data_type": "int"}},
+ supported_types=(nn.Linear,),
+ inner_supported_types=(),
+ )
+
+
+def test_inherited_w8_asym_layer_config_entry_pinned_symmetric():
+ """An 8-bit entry without an explicit sym only inherits the global asym;
+ pin it symmetric instead of failing the whole run (conservative default;
+ the llm_compressor format and AR_ALLOW_W8_ASYM are the escape hatches)."""
+ model = nn.Sequential(nn.Linear(32, 32))
+ scheme = ResolvedScheme.from_scheme(QuantizationScheme(bits=4, sym=False, act_bits=16, act_data_type="float"))
+
+ resolved = resolve_layer_config(
+ model=model,
+ scheme=scheme,
+ layer_config={"0": {"bits": 8, "data_type": "int"}},
+ supported_types=(nn.Linear,),
+ inner_supported_types=(),
+ )
+
+ assert resolved["0"]["sym"] is True
+
+
+def test_extract_regex_config_honors_format_allowance(monkeypatch):
+ """AutoScheme output can carry DP-selected W8-asym entries; extract_regex_config
+ (the post-scoring export-metadata pass) must keep them under llm_compressor and
+ still refuse them for native formats."""
+ import torch.nn as nn
+
+ from auto_round.compressors.config_resolution import ResolvedScheme
+ from auto_round.compressors.layer_config_resolver import extract_regex_config
+ from auto_round.schemes import QuantizationScheme
+
+ model = nn.Sequential(nn.Linear(32, 32))
+ scheme = ResolvedScheme.from_scheme(QuantizationScheme(bits=4, sym=False, act_bits=16, act_data_type="float"))
+ layer_config = {"0": {"bits": 8, "sym": False, "data_type": "int", "group_size": 32}}
+
+ # Regression: the post-scoring export-metadata pass must not refuse a
+ # DP-selected W8-asym entry when the export format serves it (this exact
+ # shape crashed AutoScheme + llm_compressor + --asym after scoring).
+ extract_regex_config(
+ model=model,
+ scheme=scheme,
+ layer_config=dict(layer_config),
+ supported_types=(nn.Linear,),
+ inner_supported_types=(),
+ format="auto_round:llm_compressor",
+ )
+
+ import pytest as _pytest
+
+ with _pytest.raises(ValueError, match="8-bit asymmetric"):
+ extract_regex_config(
+ model=model,
+ scheme=scheme,
+ layer_config=dict(layer_config),
+ supported_types=(nn.Linear,),
+ inner_supported_types=(),
+ format="auto_round",
+ )
+
+ # the opt-in env exempts native formats too
+ monkeypatch.setenv("AR_ALLOW_W8_ASYM", "1")
+ extract_regex_config(
+ model=model,
+ scheme=scheme,
+ layer_config=dict(layer_config),
+ supported_types=(nn.Linear,),
+ inner_supported_types=(),
+ format="auto_round",
+ )
diff --git a/test/unit/common/utils/test_missing_tensors.py b/test/unit/common/utils/test_missing_tensors.py
index 547f5820f..5bcaac255 100644
--- a/test/unit/common/utils/test_missing_tensors.py
+++ b/test/unit/common/utils/test_missing_tensors.py
@@ -236,7 +236,9 @@ class TestQuantizeWeightRtn:
def test_output_shapes_4bit_sym(self):
out_f, in_f, bits, gs = 32, 64, 4, 32
- qw, qz, sc = quantize_weight_rtn(torch.randn(out_f, in_f), bits=bits, group_size=gs, sym=True)
+ qw, qz, sc = quantize_weight_rtn(
+ torch.randn(out_f, in_f), bits=bits, group_size=gs, sym=True, packing="auto_round:auto_gptq"
+ )
pf = 32 // bits
ng = in_f // gs
assert qw.shape == (in_f // pf, out_f)
@@ -247,27 +249,95 @@ class TestQuantizeWeightRtn:
def test_output_shapes_4bit_asym(self):
out_f, in_f, bits, gs = 32, 64, 4, 32
- qw, qz, sc = quantize_weight_rtn(torch.randn(out_f, in_f), bits=bits, group_size=gs, sym=False)
+ qw, qz, sc = quantize_weight_rtn(
+ torch.randn(out_f, in_f), bits=bits, group_size=gs, sym=False, packing="auto_round:auto_gptq"
+ )
assert qw.shape == (in_f // (32 // bits), out_f)
+ @staticmethod
+ def _unpack_4bit(qw, qz, sc):
+ """Unpack [q, z, s] with nibbles along the packed dims."""
+ shifts = torch.arange(8, dtype=torch.int32) * 4
+ q = (qw[:, None, :].to(torch.int64) >> shifts[None, :, None]) & 0xF
+ q = q.reshape(-1, qw.shape[1]).t().contiguous() # [out, in]
+ z = (qz[:, :, None].to(torch.int64) >> shifts[None, None, :]) & 0xF
+ z = z.reshape(qz.shape[0], -1).t().contiguous() # [out, groups]
+ s = sc.t().contiguous() # [out, groups]
+ return q, z, s
+
+ def test_asym_auto_round_packing_stores_zp_directly(self):
+ """Asym packing for the declared "auto_round" format must store zp directly.
+
+ The model-free config labels asym int artifacts packing_format
+ "auto_round" (see _declared_int_packing), which consumers read
+ with the qlinear_torch convention: the qzeros nibble IS the zero point.
+ Storing zp - 1 there shifts every weight by +scale per group."""
+ torch.manual_seed(42)
+ w = torch.randn(8, 128) * 0.05
+ qw, qz, sc = quantize_weight_rtn(w, bits=4, group_size=128, sym=False, packing="auto_round")
+ q, z, s = self._unpack_4bit(qw, qz, sc) # single group: [8,128], [8,1], [8,1]
+ w_rec = (q - z) * s
+ assert (w_rec - w).abs().max() <= 0.5 * s.max().float() + 1e-6
+ # decoding the same bytes with the +1 (gptq) bias must be measurably wrong
+ w_wrong = (q - (z + 1)) * s
+ assert (w_wrong - w).abs().mean() > 0.9 * s.float().mean()
+
+ def test_unknown_packing_format_raises(self):
+ """A packing string with no registered zero-point convention must fail
+ loudly before any quantization work, never silently pick a convention
+ (note "auto_round:gptqmodel" is a direct-zp family member, so substring
+ matching on "gptq" would be wrong)."""
+ with pytest.raises(ValueError, match="unsupported packing format"):
+ quantize_weight_rtn(torch.randn(8, 128), bits=4, group_size=128, sym=False, packing="made_up")
+ # direct-zp family member containing "gptq" as a substring
+ w = torch.randn(8, 128) * 0.05
+ torch.manual_seed(3)
+ w = torch.randn(8, 128) * 0.05
+ qw, qz, sc = quantize_weight_rtn(w, bits=4, group_size=128, sym=False, packing="auto_round:gptqmodel")
+ q, z, s = self._unpack_4bit(qw, qz, sc)
+ w_rec = (q - z) * s
+ assert (w_rec - w).abs().max() <= 0.5 * s.max().float() + 1e-6 # direct-zp convention round-trips
+
+ def test_asym_gptq_packing_stores_zp_minus_one(self):
+ """Gptq-family asym packing keeps the qlinear_torch_zp convention.
+
+ packing_format "auto_round:auto_gptq" consumers unpack zeros with +1,
+ so the nibble must store zp - 1 (existing behavior, back-compat)."""
+ torch.manual_seed(7)
+ w = torch.randn(8, 128) * 0.05
+ qw, qz, sc = quantize_weight_rtn(w, bits=4, group_size=128, sym=False, packing="auto_round:auto_gptq")
+ q, z, s = self._unpack_4bit(qw, qz, sc)
+ w_rec = (q - (z + 1)) * s
+ assert (w_rec - w).abs().max() <= 0.5 * s.max().float() + 1e-6
+
+ def test_sym_packing_stores_offset_binary_minus_one(self):
+ """Sym packing (declared auto_round:auto_gptq) stores the constant 8 as 7."""
+ qw, qz, sc = quantize_weight_rtn(
+ torch.randn(8, 128), bits=4, group_size=128, sym=True, packing="auto_round:auto_gptq"
+ )
+ z = (qz[:, :, None].to(torch.int64) >> (torch.arange(8, dtype=torch.int32) * 4)[None, None, :]) & 0xF
+ assert (z == 7).all()
+
def test_output_shapes_8bit(self):
out_f, in_f, bits, gs = 16, 128, 8, 128
- qw, qz, sc = quantize_weight_rtn(torch.randn(out_f, in_f), bits=bits, group_size=gs, sym=True)
+ qw, qz, sc = quantize_weight_rtn(
+ torch.randn(out_f, in_f), bits=bits, group_size=gs, sym=True, packing="auto_round:auto_gptq"
+ )
pf = 32 // bits
assert qw.shape == (in_f // pf, out_f)
def test_outputs_on_cpu(self):
- qw, qz, sc = quantize_weight_rtn(torch.randn(16, 64), bits=4, group_size=32)
+ qw, qz, sc = quantize_weight_rtn(torch.randn(16, 64), bits=4, group_size=32, packing="auto_round:auto_gptq")
assert qw.device.type == "cpu"
assert qz.device.type == "cpu"
assert sc.device.type == "cpu"
def test_raises_on_1d(self):
with pytest.raises(AssertionError):
- quantize_weight_rtn(torch.randn(64), bits=4, group_size=32)
+ quantize_weight_rtn(torch.randn(64), bits=4, group_size=32, packing="auto_round:auto_gptq")
def test_in_features_padding(self):
- qw, _, _ = quantize_weight_rtn(torch.randn(16, 70), bits=4, group_size=32)
+ qw, _, _ = quantize_weight_rtn(torch.randn(16, 70), bits=4, group_size=32, packing="auto_round:auto_gptq")
assert qw.shape == (96 // (32 // 4), 16) # padded to 96
diff --git a/test/unit/test_cpu/export/test_llmc_format.py b/test/unit/test_cpu/export/test_llmc_format.py
index f1725b32b..8b77d29fd 100644
--- a/test/unit/test_cpu/export/test_llmc_format.py
+++ b/test/unit/test_cpu/export/test_llmc_format.py
@@ -17,6 +17,146 @@ from ...envs import is_compressed_tensors_available
pytestmark = pytest.mark.skipif(not is_compressed_tensors_available(), reason="test requires compressed-tensors")
+class TestLLMCZpInt8:
+ """pack_layer must hand compressed_tensors an int8 zero-point tensor:
+ NeUQI stores zp as a float32 integral-valued tensor, and CT's packer
+ (pack_to_int32) hard-requires torch.int8."""
+
+ def _make_layer(self, zp_tensor, bits=4, sym=False):
+ import torch.nn as nn
+
+ layer = nn.Linear(16, 8)
+ layer.bits = bits
+ layer.sym = sym
+ layer.group_size = 16
+ layer.data_type = "int"
+ layer.act_bits = 16
+ layer.act_sym = True
+ layer.act_data_type = None
+ layer.scale = torch.full((8, 1), 0.01)
+ layer.zp = zp_tensor
+ return layer
+
+ def test_tensor_zp_cast_to_int8(self):
+ import torch
+
+ from auto_round.export.export_to_llmcompressor.export import pack_layer
+
+ layer = self._make_layer(torch.full((8, 1), 5.0)) # float32 integral-valued
+ model = torch.nn.Sequential()
+ model.add_module("q_proj", layer)
+ pack_layer("q_proj", model, device="cpu")
+ # after compression CT owns the params: zp arrives packed to int32 and
+ # the layer is marked COMPRESSED (the pre-fix code raised in pack_to_int32)
+ zp = getattr(layer, "weight_zero_point", None)
+ assert zp is not None and zp.dtype == torch.int32
+ assert getattr(layer, "weight_packed", None) is not None
+
+ def test_out_of_range_zp_raises(self):
+ import torch
+
+ from auto_round.export.export_to_llmcompressor.export import pack_layer
+
+ layer = self._make_layer(torch.full((8, 1), 200.0), bits=4) # zp >> 2^b-1: invalid at any convention
+ model = torch.nn.Sequential()
+ model.add_module("q_proj", layer)
+ try:
+ pack_layer("q_proj", model, device="cpu")
+ except ValueError as e:
+ assert "zero-point" in str(e)
+ else:
+ raise AssertionError("out-of-range zp must raise")
+
+ def test_asym_zp_signed_convention_round_trip(self):
+ """AutoRound zp is unsigned (q in [0, 2^b-1], W = (q - zp) * s);
+ compressed-tensors re-quantizes with the SIGNED range
+ [-2^(b-1), 2^(b-1)-1] (clamp AFTER adding zp). The unsigned zp must be
+ shifted into the signed convention before packing, or CT's clamp
+ collapses every level above 2^(b-1)-1. Round-trip through CT's own
+ compressor must reproduce AutoRound's fake-quantized weight exactly."""
+ import torch
+ import torch.nn as nn
+ from compressed_tensors.compressors.pack_quantized import PackedQuantizationCompressor
+
+ from auto_round.export.export_to_llmcompressor.export import pack_layer
+
+ torch.manual_seed(0)
+ out_f, in_f, gs = 8, 32, 16
+ w = torch.randn(out_f, in_f)
+ scale = torch.rand(out_f, in_f // gs) * 0.02 + 0.005 # [out, in/g]
+ zp = torch.randint(0, 16, (out_f, in_f // gs)).float()
+ zp[:, 0] = 12 # exercise levels above 2^(b-1)-1 (the pre-fix clamp)
+
+ layer = nn.Linear(in_f, out_f)
+ with torch.no_grad():
+ layer.weight.copy_(w)
+ layer.bits, layer.sym, layer.group_size = 4, False, gs
+ layer.data_type, layer.act_bits, layer.act_sym, layer.act_data_type = "int", 16, True, None
+ layer.scale, layer.zp = scale, zp
+
+ model = nn.Sequential()
+ model.add_module("q_proj", layer)
+ pack_layer("q_proj", model, device="cpu")
+
+ # AutoRound's own fake-quantized weight (unsigned convention)
+ s = scale.repeat_interleave(gs, dim=1)
+ z = zp.repeat_interleave(gs, dim=1)
+ q = (w / s + z).round().clamp(0, 15)
+ w_ref = (q - z) * s
+
+ state = {
+ "weight_packed": layer.weight_packed.detach(),
+ "weight_scale": layer.weight_scale.detach(),
+ "weight_zero_point": layer.weight_zero_point.detach(),
+ "weight_shape": layer.weight_shape.detach(),
+ }
+ out = PackedQuantizationCompressor.decompress(state, layer.quantization_scheme)
+ assert torch.equal(out["weight"].to(torch.float32), w_ref)
+
+ def test_w8_asym_pack_round_trip(self):
+ """8-bit asym is the boundary case: zp spans [0, 255] and only fits int8
+ via the signed-convention shift. The llm_compressor format accepts W8
+ asym (vLLM serves it through the compressed-tensors kernel path), so
+ the pack must round-trip."""
+ import torch
+ import torch.nn as nn
+ from compressed_tensors.compressors.pack_quantized import PackedQuantizationCompressor
+
+ from auto_round.export.export_to_llmcompressor.export import pack_layer
+
+ torch.manual_seed(0)
+ out_f, in_f, gs = 8, 32, 16
+ w = torch.randn(out_f, in_f)
+ scale = torch.rand(out_f, in_f // gs) * 0.02 + 0.005
+ zp = torch.randint(0, 256, (out_f, in_f // gs)).float() # full 8-bit span
+ zp[:, 0] = 250 # far above 127: unsigned convention, must survive the shift
+
+ layer = nn.Linear(in_f, out_f)
+ with torch.no_grad():
+ layer.weight.copy_(w)
+ layer.bits, layer.sym, layer.group_size = 8, False, gs
+ layer.data_type, layer.act_bits, layer.act_sym, layer.act_data_type = "int", 16, True, None
+ layer.scale, layer.zp = scale, zp
+
+ model = nn.Sequential()
+ model.add_module("q_proj", layer)
+ pack_layer("q_proj", model, device="cpu")
+
+ s = scale.repeat_interleave(gs, dim=1)
+ z = zp.repeat_interleave(gs, dim=1)
+ q = (w / s + z).round().clamp(0, 255)
+ w_ref = (q - z) * s
+
+ state = {
+ "weight_packed": layer.weight_packed.detach(),
+ "weight_scale": layer.weight_scale.detach(),
+ "weight_zero_point": layer.weight_zero_point.detach(),
+ "weight_shape": layer.weight_shape.detach(),
+ }
+ out = PackedQuantizationCompressor.decompress(state, layer.quantization_scheme)
+ assert torch.equal(out["weight"].to(torch.float32), w_ref)
+
+
class TestLLMC:
@classmethod
@@ -29,6 +169,32 @@ class TestLLMC:
def teardown_class(self):
shutil.rmtree("runs", ignore_errors=True)
+ def test_mixed_int_bits_llmcompressor_format(self, tiny_opt_model_path, tmp_path):
+ """Mixed int W5A16-W8A16 auto-scheme export with an unsupported-as-first-option scheme
+ leading the options (W5A16) must resolve and export through the llm_compressor backend
+ (compressed-tensors pack-quantized handles any int width in 2..8)."""
+ scheme = AutoScheme(
+ avg_bits=6.0,
+ options=("W5A16", "W6A16", "W7A16", "W8A16"),
+ )
+ ar = AutoRound(
+ model=tiny_opt_model_path,
+ iters=0,
+ disable_opt_rtn=True,
+ scheme=scheme,
+ )
+ _, tmp_path = ar.quantize_and_save(output_dir=tmp_path, format="auto_round:llm_compressor")
+ model = AutoModelForCausalLM.from_pretrained(tmp_path, torch_dtype="auto", trust_remote_code=True)
+ quantization_config = model.config.quantization_config.to_dict()
+ assert quantization_config["format"] == "pack-quantized"
+ assert quantization_config["config_groups"], "no quantized config groups were exported"
+ for group in quantization_config["config_groups"].values():
+ weights = group["weights"]
+ assert weights["num_bits"] in (5, 6, 7, 8), f"unexpected bit-width {weights['num_bits']}"
+ assert weights["type"] == "int"
+ assert weights["symmetric"] is True
+ assert group["input_activations"] is None # weight-only quantization
+
# remove since w8a8 not in llmcompressor format supported schemes
# def test_llmcompressor_w8a8(self):
# bits, group_size, sym, act_bits = 8, -1, True, 8
@@ -400,3 +566,113 @@ def test_llmcompressor_mxfp8_export_packs_serially(tmp_path, monkeypatch):
compressed_model = autoround.save_quantized(output_dir=tmp_path, format="llm_compressor")
tmp_layer = compressed_model.model.decoder.layers[1].self_attn.q_proj
assert hasattr(tmp_layer, "weight_scale")
+
+
+class TestLLMCZpUnsignedDomain:
+ """The packed-level validation must reject zero points outside the
+ UNSIGNED level range [0, 2^bits-1] before the signed-convention shift; a
+ shifted-only check would let e.g. a raw 4-bit zp of 100 pass (shifted 92
+ fits int8)."""
+
+ def _make_layer(self, zp, bits=4, sym=False):
+ import torch.nn as nn
+
+ layer = nn.Linear(16, 8)
+ layer.bits = bits
+ layer.sym = sym
+ layer.group_size = 16
+ layer.data_type = "int"
+ layer.act_bits = 16
+ layer.act_sym = True
+ layer.act_data_type = None
+ layer.scale = torch.full((8, 1), 0.01)
+ layer.zp = zp
+ return layer
+
+ def test_tensor_zp_above_unsigned_range_rejected(self):
+ import torch
+
+ from auto_round.export.export_to_llmcompressor.export import pack_layer
+
+ layer = self._make_layer(torch.full((8, 1), 100.0)) # raw 100 > 15
+ m = torch.nn.Sequential()
+ m.add_module("proj", layer)
+ with pytest.raises(ValueError, match="unsigned level range"):
+ pack_layer("proj", m, device="cpu")
+
+ def test_tensor_zp_negative_rejected(self):
+ import torch
+
+ from auto_round.export.export_to_llmcompressor.export import pack_layer
+
+ layer = self._make_layer(torch.full((8, 1), -3.0)) # shifted -11 fits int8
+ m = torch.nn.Sequential()
+ m.add_module("proj", layer)
+ with pytest.raises(ValueError, match="unsigned level range"):
+ pack_layer("proj", m, device="cpu")
+
+ def test_scalar_zp_out_of_range_rejected(self):
+ from auto_round.export.export_to_llmcompressor.export import pack_layer
+
+ for bad in (17.0, -1.0): # 4-bit unsigned range is [0, 15]
+ layer = self._make_layer(bad)
+ m = torch.nn.Sequential()
+ m.add_module("proj", layer)
+ with pytest.raises(ValueError, match="unsigned level range"):
+ pack_layer("proj", m, device="cpu")
+
+ def test_in_range_values_pass(self):
+ import torch
+
+ from auto_round.export.export_to_llmcompressor.export import pack_layer
+
+ layer = self._make_layer(torch.randint(0, 16, (8, 1)).float())
+ m = torch.nn.Sequential()
+ m.add_module("proj", layer)
+ pack_layer("proj", m, device="cpu")
+ # CT's compressor consumes the int8 zp and stores its bit-packed form
+ assert layer.weight_zero_point.dtype == torch.int32
+ assert layer.weight_packed is not None
+
+
+class TestLLMCZpFiniteIntegral:
+ """NaN passes every min/max comparison and would be cast straight to
+ int8; non-integral values must not be silently rounded into a level."""
+
+ def _make_layer(self, zp, bits=4):
+ import torch.nn as nn
+
+ layer = nn.Linear(16, 8)
+ layer.bits = bits
+ layer.sym = False
+ layer.group_size = 16
+ layer.data_type = "int"
+ layer.act_bits = 16
+ layer.act_sym = True
+ layer.act_data_type = None
+ layer.scale = torch.full((8, 1), 0.01)
+ layer.zp = zp
+ return layer
+
+ def _expect_refusal(self, zp, match):
+ from auto_round.export.export_to_llmcompressor.export import pack_layer
+
+ layer = self._make_layer(zp)
+ m = torch.nn.Sequential()
+ m.add_module("proj", layer)
+ with pytest.raises(ValueError, match=match):
+ pack_layer("proj", m, device="cpu")
+
+ def test_nan_tensor_zp_rejected(self):
+ zp = torch.zeros(8, 1)
+ zp[3, 0] = float("nan")
+ self._expect_refusal(zp, "non-finite")
+
+ def test_non_integral_tensor_zp_rejected(self):
+ self._expect_refusal(torch.full((8, 1), -0.4), "non-integral")
+
+ def test_nan_scalar_zp_rejected(self):
+ self._expect_refusal(float("nan"), "not a finite integer")
+
+ def test_non_integral_scalar_zp_rejected(self):
+ self._expect_refusal(7.3, "not a finite integer")
diff --git a/test/unit/test_cpu/export/test_qlinear_pack_clamp.py b/test/unit/test_cpu/export/test_qlinear_pack_clamp.py
new file mode 100644
index 000000000..29aeda6e6
--- /dev/null
+++ b/test/unit/test_cpu/export/test_qlinear_pack_clamp.py
@@ -0,0 +1,262 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Packers must clamp quantized levels into [0, maxq] before bit-packing.
+
+round(W/s + zp) can exceed the level range whenever the searched (scale, zp)
+grid does not fully cover the weight range (the search itself evaluates the
+clamped loss). Packing the unclamped value shifts extra bits into the
+NEIGHBORING packed nibble -- silently corrupting it -- so the packed artifact
+no longer reproduces the clamped fake-quantization the search optimized.
+"""
+
+from types import SimpleNamespace
+
+import pytest
+import torch
+from torch import nn
+
+from auto_round.export.export_to_awq.utils import WQLinear_GEMM
+from auto_round_extension.torch.qlinear_torch import QuantLinear as QuantLinearDirect
+from auto_round_extension.torch.qlinear_torch_zp import QuantLinear as QuantLinearMinusOne
+
+OUT_F, IN_F, GROUP = 64, 64, 32
+
+
+def _make_inputs(bits, seed=0, with_overshoot=True):
+ g = torch.Generator().manual_seed(seed)
+ w = torch.randn(OUT_F, IN_F, generator=g) * 0.05
+ scale = torch.rand(OUT_F, IN_F // GROUP, generator=g) * 0.005 + 0.02
+ zp = torch.randint(0, 2**bits, (OUT_F, IN_F // GROUP), generator=g).float()
+ if with_overshoot:
+ # force q = round(w/s + zp) far outside [0, maxq] on dedicated rows:
+ # scales ~0.02..0.025, so +/-3.0 / 0.02 ~ +/-150 levels of overshoot
+ w[0, :] = 3.0
+ w[1, :] = -3.0
+ zp[0, :] = 0 # negative side undershoots as well
+ return w, scale, zp
+
+
+def _unpack_qweight(qlayer):
+ bits = qlayer.bits
+ npk = 32 // bits
+ maxq = 2**bits - 1
+ wf = torch.arange(0, 32, bits, dtype=torch.int32)
+ w_int = qlayer.qweight.unsqueeze(1).expand(-1, npk, -1) >> wf.unsqueeze(-1)
+ w_int = (w_int & maxq).reshape(-1, qlayer.outfeatures) # [in, out]
+ return w_int.t().to(torch.int64) # [out, in]
+
+
+def _unpack_qzeros(qlayer):
+ bits = qlayer.bits
+ npk = 32 // bits
+ maxq = 2**bits - 1
+ wf = torch.arange(0, 32, bits, dtype=torch.int32)
+ z = qlayer.qzeros.unsqueeze(2).expand(-1, -1, npk) >> wf.unsqueeze(0)
+ return (z & maxq).to(torch.int64) # [in/g, out/npk-1 packed] -> per nibble [in/g, out]
+
+
[email protected]("bits", [2, 4, 8])
+def test_pack_clamps_levels_direct_zp(bits):
+ torch.manual_seed(0)
+ w, scale, zp = _make_inputs(bits)
+ q = QuantLinearDirect(bits, GROUP, IN_F, OUT_F, bias=False)
+ q.device = torch.device("cpu")
+ q.pack(SimpleNamespace(weight=w, bias=None), scale, zp, None, device="cpu")
+
+ expected = torch.clamp(
+ torch.round(w / scale.repeat_interleave(GROUP, 1) + zp.repeat_interleave(GROUP, 1)), 0, 2**bits - 1
+ ).to(torch.int64)
+ got = _unpack_qweight(q)
+ assert got.min() >= 0 and got.max() <= 2**bits - 1
+ assert torch.equal(got, expected), "packed levels must equal the clamped fake-quantization"
+
+
[email protected]("bits", [2, 4, 8])
+def test_pack_clamps_levels_minus_one_zp(bits):
+ torch.manual_seed(0)
+ w, scale, zp = _make_inputs(bits)
+ q = QuantLinearMinusOne(bits, GROUP, IN_F, OUT_F, bias=False)
+ q.device = torch.device("cpu")
+ q.pack(SimpleNamespace(weight=w, bias=None), scale, zp, None, device="cpu")
+
+ expected = torch.clamp(
+ torch.round(w / scale.repeat_interleave(GROUP, 1) + zp.repeat_interleave(GROUP, 1)), 0, 2**bits - 1
+ ).to(torch.int64)
+ assert torch.equal(_unpack_qweight(q), expected)
+
+
+def test_minus_one_zp_packing_survives_zero_zero_point():
+ """zp=0 wraps to -1 under the classic (zp-1) packing; the raw -1 word
+ corrupts every nibble sharing its packed word. The stored value must be
+ clamped so only that one zero-point degrades (to a decodable +1)."""
+ torch.manual_seed(0)
+ bits = 4
+ w, scale, zp = _make_inputs(bits, with_overshoot=False)
+ zp[:, :] = 0 # every group sits at the unrepresentable zero-point
+ q = QuantLinearMinusOne(bits, GROUP, IN_F, OUT_F, bias=False)
+ q.device = torch.device("cpu")
+ q.pack(SimpleNamespace(weight=w, bias=None), scale, zp, None, device="cpu")
+
+ z_nibbles = _unpack_qzeros(q) # [in/g, out] per-nibble values
+ # pre-fix: (0-1) << shift makes whole words -1 -> every nibble reads maxq
+ assert z_nibbles.max() <= 1, "zp=0 must not smear garbage across the packed word"
+ # neighbors of a clamped cell keep their own packed value: all nibbles equal
+ # either 0 (the clamped zp-1) -- uniform here by construction
+ assert (z_nibbles == 0).all()
+ # and the module still dequantizes (forward runs, no all-ones corruption)
+ out = q(torch.eye(IN_F))
+ assert torch.isfinite(out).all()
+
+
+def test_awq_pack_clamps_levels():
+ """The AWQ packer shares the unclamped-round spill bug: an out-of-range
+ level shifts bits into the neighboring packed nibble (AWQ interleave
+ order), corrupting it. Packed levels must equal the clamped grid."""
+ torch.manual_seed(0)
+ bits, maxq = 4, 15
+ w, scale, zp = _make_inputs(bits) # [out, in], [out, in/g], [out, in/g]
+ lin = nn.Linear(IN_F, OUT_F, bias=False)
+ with torch.no_grad():
+ lin.weight.copy_(w)
+ q = WQLinear_GEMM.from_linear(lin, bits, GROUP, scales=scale.t(), zeros=zp.t(), device="cpu")
+
+ expected_q = torch.clamp(
+ torch.round(w / scale.repeat_interleave(GROUP, 1) + zp.repeat_interleave(GROUP, 1)), 0, maxq
+ ).to(torch.int32)
+
+ def _pack_rows(t): # mirrors the packer: nibbles along dim 1, AWQ interleave
+ pack_num = 32 // bits
+ order = torch.tensor([0, 4, 1, 5, 2, 6, 3, 7]) * bits
+ packed = t.reshape(t.shape[0], t.shape[1] // pack_num, pack_num) << order
+ return torch.sum(packed, dim=-1).to(torch.int32)
+
+ # qweight packs q transposed ([in, out/8], nibbles along the output dim);
+ # qzeros pack zp along the output dim ([in/g, out/8])
+ assert torch.equal(q.qweight, _pack_rows(expected_q.t().contiguous()))
+ assert torch.equal(q.qzeros, _pack_rows(zp.to(torch.int32).t().contiguous()))
+
+
+def test_direct_zp_roundtrip_in_range():
+ """Sanity: for in-range data the packed module reproduces clamped
+ fake-quantization bit-exactly (clamp must be a no-op there)."""
+ torch.manual_seed(0)
+ bits, maxq = 4, 15
+ w, scale, zp = _make_inputs(bits, with_overshoot=False)
+ # keep q in-range: shrink weights relative to scale
+ w = w * 0.05
+ q = QuantLinearDirect(bits, GROUP, IN_F, OUT_F, bias=False)
+ q.device = torch.device("cpu")
+ q.pack(SimpleNamespace(weight=w, bias=None), scale, zp, None, device="cpu")
+ s16 = scale.half().float().repeat_interleave(GROUP, 1)
+ zrep = zp.repeat_interleave(GROUP, 1)
+ expected = torch.clamp(torch.round(w / s16 + zrep), 0, maxq).to(torch.int64)
+ assert torch.equal(_unpack_qweight(q), expected)
+ # dequantized weights match the reference reconstruction
+ ref_w = (expected.to(torch.float32) - zrep) * s16
+ got_w = q(torch.eye(IN_F)).t()
+ torch.testing.assert_close(got_w, ref_w, rtol=0, atol=1e-3)
+
+
+def test_minus_one_scalar_zp_packing_survives_zero_zero_point():
+ """The scalar zero-point path of the classic (zp-1) packer must clamp like
+ the tensor path: an unclamped 0-1 = -1 repeated into every packed word
+ corrupts all nibbles; the clamped value degrades that one zp to +1."""
+ torch.manual_seed(0)
+ bits = 4
+ w, scale, _ = _make_inputs(bits, with_overshoot=False)
+ q = QuantLinearMinusOne(bits, GROUP, IN_F, OUT_F, bias=False)
+ q.device = torch.device("cpu")
+ q.pack(SimpleNamespace(weight=w, bias=None), scale, 0.0, None, device="cpu")
+
+ npk = 32 // bits
+ maxq = 2**bits - 1
+ wf = torch.arange(0, 32, bits, dtype=torch.int32)
+ z = q.qzeros.unsqueeze(2).expand(-1, -1, npk) >> wf.unsqueeze(0)
+ z_nibbles = (z & maxq).to(torch.int64)
+ # pre-fix: (0-1) built a -1 repeated word -> every nibble reads maxq
+ assert (z_nibbles == 0).all(), "scalar zp=0 must clamp to a decodable stored 0"
+ out = q(torch.eye(IN_F))
+ assert torch.isfinite(out).all()
+
+
+def test_missing_tensors_gptq_pack_clamps_zero_zero_point():
+ """quantize_weight_rtn packs zp with the same (zp-1) convention; an
+ all-positive weight yields zp=0 everywhere, which must clamp instead of
+ smearing -1 across the packed zero-point words."""
+ from auto_round.utils.missing_tensors import quantize_weight_rtn
+
+ torch.manual_seed(0)
+ bits, group = 4, 32
+ w = torch.rand(16, 64) + 0.1 # strictly positive -> asym zp = 0 per group
+ qweight, qzeros, scales = quantize_weight_rtn(
+ w, bits=bits, group_size=group, sym=False, packing="auto_round:auto_gptq"
+ )
+
+ npk = 32 // bits
+ maxq = 2**bits - 1
+ wf = torch.arange(0, 32, bits, dtype=torch.int32)
+ z = qzeros.unsqueeze(2).expand(-1, -1, npk) >> wf.unsqueeze(0)
+ z_nibbles = (z & maxq).to(torch.int64)
+ # pre-fix: zp-1 = -1 -> whole int32 words negative -> nibbles read maxq
+ assert (z_nibbles == 0).all(), "zp=0 must pack as the clamped 0 (decodes +1)"
+ # scales/q stay well-formed
+ assert torch.isfinite(scales).all() and scales.shape == (w.shape[1] // group, w.shape[0])
+
+
+def test_minus_one_3bit_scalar_zp_clamped_both_ends():
+ """The 3-bit scalar zero-point path must clamp at both ends: zp=0 packs a
+ clean all-zero word (no -1 smear), and an oversized scalar (20 > maxq 7)
+ packs bit-identically to the in-range zp=8 (stored 7) instead of spilling
+ into neighboring packed fields."""
+ bits = 3
+
+ def _pack_with_scalar(scalar):
+ torch.manual_seed(0)
+ q = QuantLinearMinusOne(bits, GROUP, IN_F, OUT_F, bias=False)
+ q.device = torch.device("cpu")
+ w, scale, _ = _make_inputs(bits, with_overshoot=False)
+ q.pack(SimpleNamespace(weight=w, bias=None), scale, scalar, None, device="cpu")
+ return q
+
+ q0 = _pack_with_scalar(0.0)
+ assert (q0.qzeros == 0).all(), "scalar zp=0 must clamp to a stored 0 word"
+ assert torch.isfinite(q0(torch.eye(IN_F))).all()
+
+ q_over = _pack_with_scalar(20.0)
+ q_ref = _pack_with_scalar(8.0) # in-range zero point storing 7
+ assert torch.equal(
+ q_over.qzeros, q_ref.qzeros
+ ), "an oversized scalar zero point must clamp to the maxq word, not spill"
+
+
[email protected]("bits", [2, 4, 8])
+def test_minus_one_scalar_zp_packs_float_scalars(bits):
+ """A float scalar zero point (the natural minmax output) must survive the
+ 2/4/8-bit classic (zp-1) packing: normalized to int, clamped at both
+ ends. zp=1.0 stores 0; an oversized scalar stores maxq."""
+ maxq = 2**bits - 1
+
+ def _pack(scalar):
+ q = QuantLinearMinusOne(bits, GROUP, IN_F, OUT_F, bias=False)
+ q.device = torch.device("cpu")
+ w, scale, _ = _make_inputs(bits, with_overshoot=False)
+ torch.manual_seed(0)
+ q.pack(SimpleNamespace(weight=w, bias=None), scale, scalar, None, device="cpu")
+ return q
+
+ q1 = _pack(1.0)
+ q_over = _pack(float(maxq + 5))
+ q_ref = _pack(float(maxq + 1)) # in-range zp storing maxq
+ assert torch.isfinite(q1(torch.eye(IN_F))).all(), "float scalar zp=1.0 must pack, not crash"
+ assert torch.equal(q_over.qzeros, q_ref.qzeros), "oversized float scalar must clamp to maxq"
diff --git a/test/unit/test_cpu/quantization/test_model_free_parity.py b/test/unit/test_cpu/quantization/test_model_free_parity.py
index 5cd8c0009..1b61405f0 100755
--- a/test/unit/test_cpu/quantization/test_model_free_parity.py
+++ b/test/unit/test_cpu/quantization/test_model_free_parity.py
@@ -90,9 +90,9 @@ _PLAIN_RTN_PARITY_SCHEMES = [
_PLAIN_RTN_CONTRACT_SCHEMES = [
# Asymmetric plain RTN keeps export-contract parity across the two paths,
- # but the packed tensors are not bit-exact.
+ # but the packed tensors are not bit-exact. 8-bit asym is excluded: it is
+ # refused at construction on both routes (see the refusal test below).
("W2A16_ASYM", "W2A16", {"bits": 2, "group_size": 128, "sym": False}),
- ("W8A16_ASYM", "W8A16", {"bits": 8, "group_size": 128, "sym": False}),
]
# Note: W4A16 asym is excluded because the regular path uses
@@ -361,6 +361,31 @@ def test_plain_rtn_asymmetric_export_contract_parity(
_assert_int_export_contract_parity(out_model_free, out_regular)
+def test_w8_asymmetric_refused_on_both_routes(tiny_opt_model_path):
+ """8-bit asym is refused at construction on both the model-free and the
+ regular route: both default to native int8-packed export formats, which
+ cannot represent the 8-bit zero point (vLLM serves W8 asym only via
+ compressed-tensors; see AR_ALLOW_W8_ASYM).
+ """
+ from auto_round import AutoRound
+
+ device = _device_str()
+ common = dict(
+ scheme="W8A16",
+ bits=8,
+ group_size=128,
+ sym=False,
+ iters=0,
+ disable_opt_rtn=True,
+ device_map=device,
+ enable_torch_compile=False,
+ )
+ with pytest.raises(ValueError, match="8-bit asymmetric"):
+ AutoRound(tiny_opt_model_path, model_free=True, **common)
+ with pytest.raises(ValueError, match="8-bit asymmetric"):
+ AutoRound(tiny_opt_model_path, disable_model_free=True, amp=False, **common)
+
+
@pytest.mark.parametrize("scheme_name,scheme_preset,scheme_kwargs", _OPT_RTN_PARITY_SCHEMES)
def test_model_free_int_opt_rtn_same_as_plain_rtn(
tmp_path, tiny_opt_model_path, scheme_name, scheme_preset, scheme_kwargs
diff --git a/test/unit/test_cuda/export/test_autoround_int_export.py b/test/unit/test_cuda/export/test_autoround_int_export.py
index 8fe40e708..aeace8e8b 100644
--- a/test/unit/test_cuda/export/test_autoround_int_export.py
+++ b/test/unit/test_cuda/export/test_autoround_int_export.py
@@ -63,7 +63,34 @@ class TestAutoroundIntExportGpu:
assert out.logits.shape[0] == 1
@pytest.mark.timeout(180)
- @pytest.mark.parametrize("bits", [4, 8])
+ @pytest.mark.parametrize("group_size", [128])
+ def test_w8_asym_env_int_export_reload_forward(self, tiny_opt_model_path, group_size, monkeypatch):
+ """AR_ALLOW_W8_ASYM=1 skips the native-format refusal; the packed
+ artifact still reloads and forwards through transformers (serving is
+ the user's responsibility, hence the explicit opt-in env)."""
+ monkeypatch.setenv("AR_ALLOW_W8_ASYM", "1")
+ autoround = AutoRound(
+ tiny_opt_model_path,
+ bits=8,
+ group_size=group_size,
+ sym=False,
+ iters=0,
+ disable_opt_rtn=True,
+ nsamples=1,
+ seqlen=16,
+ )
+ _, quantized_model_path = autoround.quantize_and_save(output_dir=self.save_dir, format="auto_round")
+
+ model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="cuda:0", trust_remote_code=True)
+ assert isinstance(model, torch.nn.Module)
+ assert _has_packed_weight(model)
+ input_ids = torch.randint(0, 1000, (1, 16), device="cuda:0")
+ with torch.no_grad():
+ out = model(input_ids)
+ assert out.logits.shape[0] == 1
+
+ @pytest.mark.timeout(180)
+ @pytest.mark.parametrize("bits", [4]) # 8-bit asym is refused at construction
@pytest.mark.parametrize("group_size", [32, 128])
def test_asym_int_export_reload_forward(self, tiny_opt_model_path, bits, group_size):
"""Asymmetric integer export path."""
diff --git a/test/unit/test_cuda/quantization/test_act_quantization.py b/test/unit/test_cuda/quantization/test_act_quantization.py
index 295510c32..6143c6377 100644
--- a/test/unit/test_cuda/quantization/test_act_quantization.py
+++ b/test/unit/test_cuda/quantization/test_act_quantization.py
@@ -118,23 +118,23 @@ class TestActQuantizationGpu:
"""Asymmetric W8A8 cannot be exported to the auto_round format.
``auto_round`` supports W8A8 export only for the symmetric path; an
- asymmetric W8A8 config is rejected during format resolution.
+ asymmetric W8A8 config is rejected when the scheme is resolved (the
+ default export format has no room for an 8-bit zero point).
"""
- autoround = AutoRound(
- tiny_opt_model_path,
- bits=8,
- act_bits=8,
- group_size=128,
- act_group_size=32,
- act_dynamic=True,
- sym=False,
- iters=0,
- disable_opt_rtn=True,
- nsamples=2,
- seqlen=16,
- )
- with pytest.raises(ValueError, match="does not support exporting"):
- autoround.quantize_and_save(output_dir=self.save_dir, format="auto_round")
+ with pytest.raises(ValueError, match="8-bit asymmetric weight quantization is not supported"):
+ AutoRound(
+ tiny_opt_model_path,
+ bits=8,
+ act_bits=8,
+ group_size=128,
+ act_group_size=32,
+ act_dynamic=True,
+ sym=False,
+ iters=0,
+ disable_opt_rtn=True,
+ nsamples=2,
+ seqlen=16,
+ )
@pytest.mark.timeout(180)
def test_w4a8_reload_layer_type_diff_but_output_equal(self, tiny_opt_model_path):