Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

refactor scheme,layer_config and formats (#2013)

Heng Guo77e8ea599361
diff --git a/auto_round/__init__.py b/auto_round/__init__.py
index 031e205ff..d3f87c2c4 100644
--- a/auto_round/__init__.py
+++ b/auto_round/__init__.py
@@ -11,10 +11,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-from auto_round.autoround import AutoRound
-
-# support for old api
-from auto_round.autoround import AutoRoundLLM, AutoRoundMLLM, AutoRoundAdam, AutoRoundDiffusion
+from auto_round.autoround import AutoRound, AutoRoundAdam, AutoRoundDiffusion, AutoRoundLLM, AutoRoundMLLM
from auto_round.algorithms.quantization.rtn.config import OptimizedRTNConfig, RTNConfig
from auto_round.algorithms.quantization.sign_round.config import (
AdamRoundConfig,
diff --git a/auto_round/algorithms/config_resolver.py b/auto_round/algorithms/config_resolver.py
index 329e30ba8..cccf8055b 100644
--- a/auto_round/algorithms/config_resolver.py
+++ b/auto_round/algorithms/config_resolver.py
@@ -148,3 +148,15 @@ def resolve_shared_config_values(configs: list[Any]) -> list[Any]:
if getattr(config, attr_name) is None:
setattr(config, attr_name, shared_value)
return configs
+
+
+def sync_shared_config_from(source_config: Any, target_configs: list[Any]) -> None:
+ """Propagate resolved source values to targets that already define matching attrs."""
+ source_attrs = _public_config_attrs(source_config)
+ for target in _quantization_configs(target_configs):
+ if target is source_config:
+ continue
+ target_attrs = _public_config_attrs(target)
+ for attr_name, source_value in source_attrs.items():
+ if attr_name in target_attrs and source_value is not None:
+ setattr(target, attr_name, source_value)
diff --git a/auto_round/algorithms/quantization/config.py b/auto_round/algorithms/quantization/config.py
index 9479f0952..055525c9b 100644
--- a/auto_round/algorithms/quantization/config.py
+++ b/auto_round/algorithms/quantization/config.py
@@ -11,7 +11,6 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-from enum import Enum
from typing import ClassVar
from auto_round.export.export_to_gguf.config import GGUF_INNER_CONFIG
@@ -19,14 +18,6 @@ from auto_round.logger import logger
from auto_round.schemes import QuantizationScheme
-class BackendDataType(str, Enum):
- STANDARD_FP = "fp"
- MX_FP = "mx_fp"
- NV_FP = "nv_fp"
- FP8_STATIC = "fp8_static"
- FP8 = "fp8"
-
-
class QuantizationConfig:
"""Common quantization configuration shared by block quantizers.
@@ -186,71 +177,44 @@ class QuantizationConfig:
@property
def is_act_quantize(self) -> bool:
- return self.act_bits is not None and self.act_bits <= 8
+ return self.scheme.is_act_quantize()
@property
def is_nv_fp(self) -> bool:
- return self.data_type is not None and BackendDataType.NV_FP in self.data_type
+ return self.scheme.is_nv_fp()
@property
def is_act_nv_fp(self) -> bool:
- return self.act_data_type is not None and BackendDataType.NV_FP in self.act_data_type
+ return self.scheme.is_act_nv_fp()
@property
def is_mx_fp(self) -> bool:
- return self.data_type is not None and BackendDataType.MX_FP in self.data_type
+ return self.scheme.is_mx_fp()
@property
def is_act_mx_fp(self) -> bool:
- return self.act_data_type is not None and BackendDataType.MX_FP in self.act_data_type
+ return self.scheme.is_act_mx_fp()
@property
def is_dynamic_wint8aint8(self) -> bool:
- if self.act_dynamic:
- return True
- if self.act_data_type is not None and self.data_type is not None:
- if ("int8" in self.act_data_type or ("int" in self.act_data_type and self.act_bits == 8)) and (
- "int8" in self.data_type or ("int" in self.data_type and self.bits == 8)
- ):
- return True
- return False
+ return self.scheme.is_dynamic_wint8aint8()
@property
def is_standard_fp(self) -> bool:
- return (
- self.data_type is not None
- and BackendDataType.STANDARD_FP in self.data_type
- and not self.is_mx_fp
- and not self.is_nv_fp
- )
+ return self.scheme.is_standard_fp()
@property
def is_act_standard_fp(self) -> bool:
- return (
- self.act_data_type is not None
- and BackendDataType.STANDARD_FP in self.act_data_type
- and not self.is_act_mx_fp
- and not self.is_act_nv_fp
- )
+ return self.scheme.is_act_standard_fp()
@property
def is_static_afp8(self) -> bool:
- return self.act_data_type is not None and BackendDataType.FP8_STATIC in self.act_data_type
+ return self.scheme.is_static_afp8()
@property
- def is_static_wfp8afp8(self) -> bool:
- return self.data_type is not None and BackendDataType.FP8_STATIC in self.data_type and self.is_static_afp8
+ def is_act_static(self) -> bool:
+ return self.scheme.is_act_static()
@property
def is_wfp8afp8(self) -> bool:
- if self.act_data_type is None or self.data_type is None:
- return False
- if (
- ("fp8" in self.act_data_type or ("fp" in self.act_data_type and self.act_bits == 8))
- and ("fp8" in self.data_type or ("fp" in self.data_type and self.bits == 8))
- and self.is_act_standard_fp
- and self.is_standard_fp
- ):
- return True
- else:
- return False
+ return self.scheme.is_wfp8afp8()
diff --git a/auto_round/algorithms/quantization/sign_round/config.py b/auto_round/algorithms/quantization/sign_round/config.py
index 823b10e05..c3fb94e4e 100644
--- a/auto_round/algorithms/quantization/sign_round/config.py
+++ b/auto_round/algorithms/quantization/sign_round/config.py
@@ -105,7 +105,7 @@ class SignRoundConfig(QuantizationConfig):
def finalize_scheme(self) -> None:
"""Resolve lr/minmax_lr once `bits` is known (low-bit schemes use a higher lr)."""
- if self.lr is None:
+ if self.lr is None and self.iters > 0:
# TODO need to check 4 bits lr setting for auto-round-best, 3bits only validate on small models
if self.iters >= 1000 and self.bits is not None and self.bits <= 3:
self.lr = 2.0 / self.iters
diff --git a/auto_round/algorithms/registry.py b/auto_round/algorithms/registry.py
index d56b0a497..081b09f0f 100644
--- a/auto_round/algorithms/registry.py
+++ b/auto_round/algorithms/registry.py
@@ -90,6 +90,41 @@ def resolve_algorithm_alias(alias: str) -> str | None:
return _ALIAS_TO_NAME.get(alias.strip().lower())
+def resolve_algorithm_names(algorithms, *, ignore_unknown: bool = False) -> list[str]:
+ """Resolve one algorithm string or sequence to ordered canonical names.
+
+ A comma-separated string and a sequence use exactly the same semantics.
+ Duplicate aliases are removed while preserving the first occurrence.
+ """
+ if isinstance(algorithms, str):
+ raw_names = algorithms.split(",")
+ else:
+ raw_names = list(algorithms)
+
+ canonical_names = []
+ seen = set()
+ for raw_name in raw_names:
+ if not isinstance(raw_name, str):
+ raise TypeError(f"Algorithm names must be strings, got {type(raw_name).__name__}.")
+ name = raw_name.strip()
+ if not name:
+ continue
+ canonical = resolve_algorithm_alias(name)
+ if canonical is None:
+ if ignore_unknown:
+ continue
+ supported_aliases = sorted(_ALIAS_TO_NAME.keys())
+ raise ValueError(
+ f"Unknown algorithm alias '{name}'. Supported aliases: {supported_aliases}. "
+ "If you are adding a new algorithm, register it via "
+ "auto_round.algorithms.registry.register_algorithm()."
+ )
+ if canonical not in seen:
+ canonical_names.append(canonical)
+ seen.add(canonical)
+ return canonical_names
+
+
def get_algorithm_entry(name: str) -> AlgRegistryEntry:
_ensure_builtin_algorithms_registered()
canonical = resolve_algorithm_alias(name)
diff --git a/auto_round/algorithms/transforms/awq/qdq.py b/auto_round/algorithms/transforms/awq/qdq.py
index ccdbf5db3..4c98acdbb 100644
--- a/auto_round/algorithms/transforms/awq/qdq.py
+++ b/auto_round/algorithms/transforms/awq/qdq.py
@@ -164,6 +164,7 @@ class QDQTool:
)
if init_scale is not None:
quant_kwargs["init_scale"] = init_scale
+ quant_kwargs["imatrix"] = imatrix if isinstance(imatrix, torch.Tensor) else torch.ones_like(weight)
active_quant_func = opt_quant_func
qdq_weight, _, _ = active_quant_func(weight, **quant_kwargs)
diff --git a/auto_round/algorithms/transforms/spinquant/apply.py b/auto_round/algorithms/transforms/spinquant/apply.py
index e8e4c8f68..108820bb5 100644
--- a/auto_round/algorithms/transforms/spinquant/apply.py
+++ b/auto_round/algorithms/transforms/spinquant/apply.py
@@ -32,7 +32,7 @@ Usage via the unified entry point::
Usage via AutoRound pipeline::
- from auto_round.compressors.entry import AutoRound
+ from auto_round import AutoRound
from auto_round.algorithms.transforms.spinquant import SpinQuantConfig
AutoRound(
diff --git a/auto_round/auto_scheme/delta_loss.py b/auto_round/auto_scheme/delta_loss.py
index e362687e3..035d9122b 100644
--- a/auto_round/auto_scheme/delta_loss.py
+++ b/auto_round/auto_scheme/delta_loss.py
@@ -1184,7 +1184,7 @@ def get_score_for_scheme(
# Unified mllm-aware forward (casts pixel_values/images to
# model.dtype, handles dict-with-text/str/tuple paths the same
- # way AutoRoundMLLM.calib does).
+ # way the multimodal compressor calibration does).
output, _prepared = model_forward(model, data_for_forward, labels=labels, use_cache=False)
output.loss.backward()
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index c6c4b48f9..c4c58c318 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -11,339 +11,776 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+
from __future__ import annotations
-from typing import TYPE_CHECKING, Optional, Union
+from typing import TYPE_CHECKING, Any, Optional, Union
import torch
from auto_round.logger import deprecated, logger
-from auto_round.schemes import QuantizationScheme
+from auto_round.schemes import QuantizationScheme, parse_scheme
from auto_round.utils.device_manager import normalize_default_device_map
if TYPE_CHECKING:
+ from auto_round.algorithms.quantization.config import QuantizationConfig
+ from auto_round.algorithms.quantization.rtn.config import RTNConfig
from auto_round.auto_scheme.gen_auto_scheme import AutoScheme
from auto_round.compressors.base import BaseOrchestrator as BaseCompressor
-_COMPAT_KWARGS = {
- "format",
- "bits",
- "group_size",
- "sym",
- "data_type",
- "act_bits",
- "act_group_size",
- "act_sym",
- "act_data_type",
- "act_dynamic",
- "super_bits",
- "super_group_size",
- "scale_dtype",
- "ignore_layers",
- "quant_lm_head",
- "to_quant_block_names",
- "model_free",
- "disable_model_free",
- "model_dtype",
- "trust_remote_code",
- "amp",
- "nblocks",
+def _collect_config_scheme_overrides(config) -> dict:
+ """Return the config's explicitly-set scheme fields as a ``{field: value}`` dict.
+
+ These are exactly the per-field overrides layered on top of ``scheme=`` — the
+ single mechanism through which ``bits`` / ``act_bits`` / ``data_type`` etc.
+ reach the resolved scheme. Fields left as ``None`` are omitted so the scheme's
+ own value wins.
+ """
+ return {k: getattr(config, k) for k in config._scheme_fields if getattr(config, k, None) is not None}
+
+
+def _preview_resolved_attrs(config, scheme=None) -> dict:
+ from auto_round.auto_scheme.gen_auto_scheme import AutoScheme
+
+ """Resolve scheme attributes without mutating config, for routing decisions.
+
+ Called in ``AutoRound.__new__`` before the concrete compressor class is
+ chosen. ``SchemeMixin.resolve_scheme()`` will do the authoritative
+ resolution later; this is just a lightweight preview so routing logic
+ (``enable_imatrix``, ``needs_act_calib``, etc.) can use the correct values
+ even when the user specified only ``scheme=`` without explicit bit/dtype args.
+
+ This is the single source of resolved scheme fields for entry-level routing:
+ callers read from the returned dict and never re-read raw ``config`` attrs.
+ When the scheme cannot be previewed (``AutoScheme``, or a deferred parse
+ error), the config's own explicitly-set scheme overrides are returned so the
+ values still reflect what the user passed.
+
+ Returns:
+ dict: resolved scheme attributes (config overrides when preview is skipped).
+ """
+ config_overrides = _collect_config_scheme_overrides(config)
+ if isinstance(scheme, AutoScheme):
+ # AutoScheme needs model info — cannot preview; fall back to raw config attrs.
+ return config_overrides
+ try:
+ _, _, final_attrs = parse_scheme(scheme, config_overrides)
+ return final_attrs
+ except Exception:
+ return config_overrides
+
+
+def _eager_validate_scheme(config, scheme=None) -> None:
+ from auto_round.auto_scheme.gen_auto_scheme import AutoScheme
+
+ """Eagerly validate scheme/config constraints at construction time.
+
+ Mirrors the old-arch ``_check_configs()`` call in ``BaseCompressor.__init__``.
+ Raises ``ValueError`` or ``NotImplementedError`` immediately if the scheme
+ contains config-only invalid combinations (e.g. tuple group_size with non-fp8
+ weight dtype) so that callers get a fast failure rather than a deferred error
+ buried inside ``post_init()``.
+
+ ``AutoScheme`` is skipped because it requires model information.
+ """
+ if isinstance(scheme, AutoScheme):
+ return
+
+ user_overrides = _collect_config_scheme_overrides(config)
+ try:
+ _, _, final_attrs = parse_scheme(scheme, user_overrides)
+ except (ValueError, NotImplementedError):
+ raise
+ except Exception:
+ return # Other parse errors are deferred to post_init
+
+ import copy
+
+ temp_config = copy.copy(config)
+ if hasattr(config, "scheme"):
+ temp_config.scheme = config.scheme.copy()
+ temp_config._user_set_scheme_fields = set(getattr(config, "_user_set_scheme_fields", set()))
+ for key, value in final_attrs.items():
+ setattr(temp_config, key, value)
+ temp_config.check_config() # raises ValueError / NotImplementedError if invalid
+
+
+# ---------------------------------------------------------------------------
+# Compressor-class registry
+# ---------------------------------------------------------------------------
+# Maps (model_type, base_class_name) → combined class, created lazily.
+_COMPRESSOR_REGISTRY: dict[tuple[str, str], type] = {}
+
+
+def _get_compressor_class(model_type: str, base_cls: type) -> type:
+ """Return the compressor class for *base_cls* wired with the right model-type Mixin.
+
+ For ``model_type == "llm"`` the bare *base_cls* is returned unchanged.
+ For ``"mllm"`` and ``"diffusion"`` the corresponding Mixin is prepended via
+ :func:`type` and the result is cached in ``_COMPRESSOR_REGISTRY`` so that
+ each ``(model_type, base_cls)`` pair is created at most once per process.
+ """
+ if model_type == "llm":
+ return base_cls
+ key = (model_type, base_cls.__name__)
+ if key in _COMPRESSOR_REGISTRY:
+ return _COMPRESSOR_REGISTRY[key]
+ if model_type == "mllm":
+ from auto_round.compressors.mllm_mixin import MLLMMixin
+
+ mixin = MLLMMixin
+ elif model_type == "diffusion":
+ from auto_round.compressors.diffusion_mixin import DiffusionMixin
+
+ mixin = DiffusionMixin
+ else:
+ return base_cls
+ combined = type(f"{model_type.capitalize()}{base_cls.__name__}", (mixin, base_cls), {})
+ _COMPRESSOR_REGISTRY[key] = combined
+ return combined
+
+
+def is_weight_scheme(scheme: Union[str, dict, object]) -> bool:
+ if isinstance(scheme, str):
+ return scheme.upper().startswith("W")
+ if isinstance(scheme, dict):
+ return all(isinstance(s, str) and s.upper().startswith("W") for s in scheme.values())
+ from auto_round.auto_scheme.gen_auto_scheme import AutoScheme
+
+ if isinstance(scheme, AutoScheme):
+ opts = scheme.options
+ if isinstance(opts, (list, tuple)):
+ return all(isinstance(s, str) and s.upper().startswith("W") for s in opts)
+ if isinstance(opts, str):
+ return opts.upper().startswith("W")
+ return False
+
+
+def is_gguf_k_target(value: Union[str, "AutoScheme", object]) -> bool:
+ from auto_round.auto_scheme.gen_auto_scheme import AutoScheme
+
+ if isinstance(value, str):
+ normalized = value.strip().lower()
+ return normalized.startswith("gguf:") and "_k" in normalized
+ if isinstance(value, AutoScheme):
+ opts = value.options
+ if isinstance(opts, str):
+ opts = [opts]
+ if isinstance(opts, (list, tuple)):
+ return any(isinstance(opt, str) and is_gguf_k_target(opt) for opt in opts)
+ return False
+
+
+def _resolve_quant_config_for_routing(alg_configs) -> tuple[list, list, "QuantizationConfig"]:
+ from auto_round.algorithms.config_resolver import split_quantization_configs
+ from auto_round.algorithms.quantization.config import QuantizationConfig
+ from auto_round.algorithms.quantization.rtn.config import RTNConfig
+
+ preprocessor_configs, block_quant_configs = split_quantization_configs(alg_configs)
+ if len(block_quant_configs) == 0 and preprocessor_configs:
+ from auto_round.algorithms.quantization.rtn.config import RTNConfig as _RTNConfig
+
+ return preprocessor_configs, block_quant_configs, _RTNConfig()
+ if len(block_quant_configs) > 1:
+ raise ValueError(
+ f"Only one block-quantization config is allowed, but got {len(block_quant_configs)}: "
+ f"{[type(c).__name__ for c in block_quant_configs]}"
+ )
+ if len(block_quant_configs) == 1:
+ return preprocessor_configs, block_quant_configs, block_quant_configs[0]
+ raise ValueError(
+ "At least one quantization algorithm config is required. "
+ "Pass a block quantizer such as RTNConfig or SignRoundConfig, "
+ "or a quantization preprocessor such as AWQConfig."
+ )
+
+
+def _build_model_free_compressor(
+ model,
+ scheme,
+ layer_config,
+ tokenizer,
+ device_map,
+ *,
+ announced_via_flag: bool,
+ **model_free_kwargs,
+):
+ """Construct a ``ModelFreeCompressor`` for the model-free auto-route.
+
+ Shared by the unified entry and the model-free route so the string-model
+ guard, the auto-routing info log, and the constructor call live in one place.
+ ``announced_via_flag`` is truthy when the caller explicitly passed
+ ``model_free=True`` (so the informational auto-routing message is suppressed).
+ """
+ from auto_round.compressors.model_free import ModelFreeCompressor
+
+ if not isinstance(model, str):
+ raise ValueError("model_free=True requires `model` to be a HuggingFace ID or local path string.")
+ if not announced_via_flag:
+ logger.info(
+ "Auto-routing to model-free quantization "
+ "(iters=0, disable_opt_rtn=True, supported scheme). "
+ "Pass disable_model_free=True to use the regular flow."
+ )
+ return ModelFreeCompressor(
+ model_name_or_path=model,
+ scheme=scheme,
+ layer_config=layer_config,
+ tokenizer=tokenizer,
+ device_map=device_map,
+ **model_free_kwargs,
+ )
+
+
+def _build_model_type_ctor_kwargs(model, base_kwargs, mllm_kwargs, diffusion_kwargs) -> tuple[str, dict[str, Any]]:
+ from auto_round.utils.model import detect_model_type
+
+ model_type = detect_model_type(model)
+ has_multimodal_assets = mllm_kwargs.get("processor") is not None or mllm_kwargs.get("image_processor") is not None
+ if has_multimodal_assets and model_type != "mllm":
+ model_type = "mllm"
+
+ ctor_kwargs = dict(base_kwargs)
+ if model_type == "mllm":
+ ctor_kwargs.update(mllm_kwargs)
+ if model_type == "diffusion":
+ ctor_kwargs.update(diffusion_kwargs)
+ return model_type, ctor_kwargs
+
+
+def _select_rtn_compressor_base_cls(quant_config: "RTNConfig", scheme, format, base_kwargs) -> type:
+ from auto_round.algorithms.quantization.rtn.config import OptimizedRTNConfig, RTNConfig
+ from auto_round.auto_scheme.gen_auto_scheme import AutoScheme
+ from auto_round.compressors.orchestrator import CompressionOrchestrator as Compressor
+ from auto_round.compressors.utils import check_need_act_calibration
+
+ enable_imatrix = False
+ disable_opt_rtn = getattr(quant_config, "disable_opt_rtn", False)
+
+ # Single resolved-scheme source for routing (SchemeMixin does the authoritative
+ # resolution later; this preview only chooses the class). Computed once: neither
+ # `quant_config`'s scheme fields nor `scheme` itself change within this function,
+ # so the result is invariant across every use below — no need to recompute it.
+ resolved_attrs = _preview_resolved_attrs(quant_config, scheme)
+
+ # Auto-disable rtn optimization for W8A16/W8A8-equivalent resolved schemes,
+ # unless the user already set disable_opt_rtn explicitly.
+ if getattr(quant_config, "orig_disable_opt_rtn", None) is None:
+ bits = resolved_attrs.get("bits")
+ act_bits = resolved_attrs.get("act_bits")
+ data_type = resolved_attrs.get("data_type")
+ if bits is not None and bits >= 8 and act_bits is not None and act_bits >= 8 and data_type == "int":
+ logger.warning("`disable_opt_rtn` is turned on for W8A16/W8A8 quantization to improve efficiency.")
+ disable_opt_rtn = True
+ quant_config.disable_opt_rtn = True
+
+ if not disable_opt_rtn:
+ has_gguf_k = is_gguf_k_target(format) or is_gguf_k_target(scheme)
+ if has_gguf_k:
+ enable_imatrix = True
+ else:
+ sym = resolved_attrs.get("sym")
+ data_type = resolved_attrs.get("data_type") or ""
+ bits = resolved_attrs.get("bits")
+ if sym is not None and sym is False:
+ enable_imatrix = False
+ elif data_type == "int" and (bits is None or bits < 8):
+ enable_imatrix = True
+ elif is_weight_scheme(scheme):
+ enable_imatrix = True
+
+ act_bits = resolved_attrs.get("act_bits")
+ act_data_type = resolved_attrs.get("act_data_type")
+ act_dynamic = resolved_attrs.get("act_dynamic")
+ is_act_quantize = act_bits is not None and act_bits <= 8
+ needs_act_calib = is_act_quantize and check_need_act_calibration(
+ act_dynamic,
+ act_data_type,
+ act_bits if act_bits is not None else 16,
+ static_kv_dtype=base_kwargs.get("static_kv_dtype"),
+ static_attention_dtype=base_kwargs.get("static_attention_dtype"),
+ )
+
+ # AutoScheme always requires calibration data for delta-loss based scheme
+ # selection, regardless of whether imatrix is needed.
+ quant_config.enable_imatrix = enable_imatrix
+ needs_optimized_rtn = enable_imatrix or needs_act_calib or isinstance(scheme, AutoScheme)
+ if needs_optimized_rtn:
+ if not isinstance(quant_config, OptimizedRTNConfig):
+ quant_config.__class__ = OptimizedRTNConfig
+ else:
+ # Pure zero-shot RTN: downgrade to basic RTNConfig
+ if isinstance(quant_config, OptimizedRTNConfig):
+ quant_config.__class__ = RTNConfig
+
+ # Always use Compressor — it internally detects whether calibration
+ # data is needed and falls back to the zero-shot (RTN) path when it is not.
+ return Compressor
+
+
+_ENTRY_KWARG_OWNERS = {
+ "model_free": "route",
+ "disable_model_free": "route",
+ "scale_dtype": "compressor",
+ "ignore_layers": "compressor",
+ "quant_lm_head": "compressor",
+ "to_quant_block_names": "compressor",
+ "format": "base",
+ "dataset": "base",
+ "batch_size": "base",
+ "model_dtype": "base",
+ "trust_remote_code": "base",
+ "amp": "base",
+ "disable_deterministic_algorithms": "base",
+ "enable_deterministic_algorithms": "base",
+ "static_kv_dtype": "base",
+ "static_attention_dtype": "base",
+ "processor": "mllm",
+ "image_processor": "mllm",
+ "template": "mllm",
+ "extra_data_dir": "mllm",
+ "quant_nontext_module": "mllm",
+ "guidance_scale": "diffusion",
+ "num_inference_steps": "diffusion",
+ "generator_seed": "diffusion",
+}
+
+_SCHEME_FIELDS = set(QuantizationScheme.get_attributes())
+_SIGNROUND_FIELDS = {
+ "iters",
"lr",
"minmax_lr",
+ "lr_scheduler",
+ "momentum",
+ "nblocks",
"enable_minmax_tuning",
"enable_norm_bias_tuning",
+ "gradient_accumulate_steps",
+ "enable_alg_ext",
+ "not_use_best_mse",
+ "dynamic_max_gap",
"enable_quanted_input",
- "enable_opt_rtn",
- "disable_deterministic_algorithms",
- "enable_deterministic_algorithms",
- "static_kv_dtype",
- "static_attention_dtype",
- "rotation_config",
- "processor",
- "image_processor",
- "template",
- "extra_data_dir",
- "quant_nontext_module",
- "guidance_scale",
- "num_inference_steps",
- "generator_seed",
+ "optimizer",
+ "enable_adam",
+ "enable_lfq",
+}
+_RTN_FIELDS = {"disable_opt_rtn", "enable_opt_rtn"}
+_AWQ_FIELDS = {
"duo_scaling",
"n_grid",
+ "seqlen",
+ "nsamples",
+ "batch_size",
+ "apply_smooth",
+ "smooth_iters",
+ "apply_clip",
+ "clip_as_init",
+ "clip_n_grid",
+ "clip_max_shrink",
+ "clip_n_sample_token",
"mappings",
- "algorithm",
- "optimizer",
- "lr_scheduler",
- "not_use_best_mse",
- "dynamic_max_gap",
- "momentum",
- "device",
+}
+_ROTATION_FIELDS = {
+ "hadamard_type",
+ "block_size",
+ "fuse_online_to_weight",
+ "allow_online_rotation",
}
-def _filter_supported_compat_kwargs(kwargs: dict) -> dict:
- supported = {}
- unknown = []
- for key, value in kwargs.items():
- if key in _COMPAT_KWARGS:
- supported[key] = value
- else:
- unknown.append(key)
+def _filter_supported_entry_kwargs(kwargs, *, context="AutoRound"):
+ supported = {key: value for key, value in kwargs.items() if key in _ENTRY_KWARG_OWNERS}
+ unknown = sorted(set(kwargs) - set(_ENTRY_KWARG_OWNERS))
if unknown:
logger.warning_once(
- "AutoRound compatibility path received unsupported kwargs %s. They will be ignored.",
- ", ".join(sorted(unknown)),
+ "%s received unsupported runtime kwargs %s. They will be ignored.",
+ context,
+ ", ".join(unknown),
)
return supported
-class AutoRound:
- """Automatic weight rounding (Signed Gradient Descent) for LLM quantization
-
- Reference:
- Cheng, Wenhua, et al., "Optimize weight rounding via signed gradient descent for
- the quantization of LLMs." arXiv:2309.05516 (2023).
-
- Attributes:
- model (torch.nn.Module | str): The loaded PyTorch model in eval mode.
- tokenizer: Tokenizer used to prepare input text for calibration/tuning.
- platform (str): The platform to load pretrained moded, options: ["hf", "model_scope"]
- bits (int): Weight quantization bits.
- group_size (int or tuple): Per-group size for weight quantization.
- sym (bool): Whether to use symmetric weight quantization.
- layer_config (dict): Per-layer quantization configuration.
- nsamples (int): Number of calibration samples.
- enable_torch_compile (bool): Whether to enable torch.compile for quant blocks/layers. Defaults to True except
- on Windows, where it defaults to False.
+def _split_entry_kwargs(kwargs, *, context="AutoRound"):
+ buckets = {"route": {}, "compressor": {}, "base": {}, "mllm": {}, "diffusion": {}}
+ for key, value in _filter_supported_entry_kwargs(kwargs, context=context).items():
+ buckets[_ENTRY_KWARG_OWNERS[key]][key] = value
+ return buckets
+
+
+def _config_fields(config):
+ fields = set(_SCHEME_FIELDS)
+ name = type(config).__name__.lower()
+ if "awq" in name:
+ fields.update(_AWQ_FIELDS)
+ elif "rtn" in name:
+ fields.update(_RTN_FIELDS)
+ elif "signround" in name or "adamround" in name:
+ fields.update(_SIGNROUND_FIELDS)
+ elif "rotation" in name:
+ fields.update(_ROTATION_FIELDS)
+ return fields
+
+
+def _normalize_alg_configs(alg_configs, direct_kwargs=None):
+ from auto_round.algorithms.config_resolver import split_quantization_configs
+ from auto_round.algorithms.quantization.config import QuantizationConfig
+ from auto_round.algorithms.quantization.rtn.config import RTNConfig
+ from auto_round.algorithms.registry import normalize_algorithm_config, resolve_alg_config, resolve_algorithm_names
+ from auto_round.algorithms.transforms import normalize_rotation_config
+ from auto_round.algorithms.transforms.base import BaseRotationConfig
+
+ direct_kwargs = dict(direct_kwargs or {})
+ legacy_algorithm = direct_kwargs.pop("algorithm", None)
+ if legacy_algorithm is not None:
+ if alg_configs is not None:
+ raise ValueError("`algorithm` and `alg_configs` cannot be used together.")
+ alg_configs = legacy_algorithm
+ logger.warning_once("`algorithm` is deprecated; use `alg_configs` instead.")
+ if "backend" in direct_kwargs:
+ raise ValueError(
+ "Rotation backend selection must be nested in `rotation_config`; "
+ "do not pass it as AutoRound(..., backend=...)."
+ )
+ rotation_config = direct_kwargs.pop("rotation_config", None)
+ config_kwargs = {key: value for key, value in direct_kwargs.items() if key not in _ENTRY_KWARG_OWNERS}
+ if alg_configs is None:
+ # Preserve the legacy entry semantics: zero iterations are RTN, while
+ # positive iterations use SignRound. RTN-only kwargs also select RTN
+ # so they are not silently ignored by the default SignRound config.
+ if direct_kwargs.get("iters") == 0:
+ raw_configs = ["rtn"]
+ else:
+ raw_configs = ["signround"]
+ elif isinstance(alg_configs, str):
+ raw_configs = resolve_algorithm_names(alg_configs)
+ if not raw_configs:
+ raise ValueError("`algorithm`/`alg_configs` must contain at least one algorithm name.")
+ elif isinstance(alg_configs, (list, tuple)):
+ raw_configs = []
+ seen_names = set()
+ for raw_config in alg_configs:
+ if not isinstance(raw_config, str):
+ raw_configs.append(raw_config)
+ continue
+ for canonical_name in resolve_algorithm_names(raw_config):
+ if canonical_name not in seen_names:
+ raw_configs.append(canonical_name)
+ seen_names.add(canonical_name)
+ else:
+ raw_configs = [alg_configs]
+
+ configs = []
+ for raw_config in raw_configs:
+ config = resolve_alg_config(raw_config) if isinstance(raw_config, str) else raw_config
+ if not isinstance(config, (QuantizationConfig, BaseRotationConfig)):
+ raise TypeError(
+ f"alg_configs entries must be algorithm or QuantizationConfig instances, "
+ f"got {type(config).__name__}."
+ )
+ configs.append(normalize_algorithm_config(config))
+
+ # ``iters=0`` has always selected RTN in the public entry and CLI. Apply
+ # that rule after every input form has become a config so aliases, config
+ # objects, and the deprecated ``algorithm`` argument cannot choose
+ # different quantizer implementations.
+ from auto_round.algorithms.quantization.sign_round.config import SignRoundConfig
+
+ direct_iters = config_kwargs.get("iters")
+ for index, config in enumerate(configs):
+ effective_iters = direct_iters if direct_iters is not None else getattr(config, "iters", None)
+ if isinstance(config, SignRoundConfig) and effective_iters == 0:
+ rtn_config = RTNConfig(scheme=config.scheme.copy())
+ rtn_config._user_set_scheme_fields = set(getattr(config, "_user_set_scheme_fields", set()))
+ configs[index] = normalize_algorithm_config(rtn_config)
+
+ if rotation_config is not None:
+ normalized_rotation = normalize_rotation_config(rotation_config)
+ if normalized_rotation is not None:
+ configs.append(normalized_rotation)
+
+ if not any(isinstance(config, QuantizationConfig) for config in configs):
+ raise TypeError(
+ "alg_configs entries must be algorithm aliases or QuantizationConfig instances, "
+ "and must include at least one quantization algorithm config."
+ )
+
+ preprocessors, block_configs = split_quantization_configs(configs)
+ if preprocessors and not block_configs:
+ configs.append(normalize_algorithm_config(RTNConfig()))
+
+ _, block_configs = split_quantization_configs(configs)
+ for key, value in config_kwargs.items():
+ if value is None:
+ continue
+ if key in _SCHEME_FIELDS:
+ targets = block_configs
+ else:
+ targets = [config for config in configs if key in _config_fields(config)]
+ if not targets:
+ # ``iters`` is a legacy route selector. RTN intentionally has no
+ # iterative parameter, so ``iters=0`` must not be reported as an
+ # ignored algorithm-specific error after selecting RTN.
+ if key == "iters" and any(isinstance(config, RTNConfig) for config in configs):
+ continue
+ owner = "AWQ" if key in _AWQ_FIELDS else "the selected algorithm"
+ logger.error(
+ "%s-specific parameter '%s' was provided, but %s is not enabled by alg_configs. "
+ "The parameter is ignored.",
+ owner,
+ key,
+ owner,
+ )
+ continue
+ if len(targets) > 1:
+ logger.error(
+ "Parameter '%s' matches multiple algorithm configs. Pass it through the matching "
+ "config object in 'alg_configs'; the direct value is ignored.",
+ key,
+ )
+ continue
+ target = targets[0]
+ if key in ("disable_opt_rtn", "enable_opt_rtn"):
+ if key == "disable_opt_rtn" or value:
+ target.disable_opt_rtn = False if key == "enable_opt_rtn" else value
+ target.orig_disable_opt_rtn = target.disable_opt_rtn
+ else:
+ setattr(target, key, value)
+ logger.warning(
+ "Passing '%s' directly to AutoRound is supported, but the recommended usage is "
+ "'alg_configs=%sConfig(...)'.",
+ key,
+ type(targets[0]).__name__.replace("Config", ""),
+ )
+ return [normalize_algorithm_config(config) for config in configs]
+
+
+def _prepare_entry_kwargs(alg_configs, direct_kwargs):
+ configs = _normalize_alg_configs(alg_configs, direct_kwargs)
+ runtime_kwargs = {key: value for key, value in direct_kwargs.items() if key in _ENTRY_KWARG_OWNERS}
+ return configs, runtime_kwargs
+
+
+class _CompressorBuilder(object):
+ """Algorithm-config-driven entry point (``scheme`` + ``alg_configs``).
+
+ This is the internal pipeline entry: it resolves the algorithm config(s),
+ routes to the concrete :class:`BaseCompressor` subclass (ZeroShot / DataDriven
+ / ModelFree / …) wired with the right model-type Mixin, and returns that
+ compressor instance. It is distinct from the public dispatcher
+ :class:`auto_round.AutoRound` (in ``auto_round/autoround.py``).
"""
- SKIP_ARGS = ("local_args", "kwargs", "cls", "model_cls", "dynamic_compressor", "alg_configs")
+ @classmethod
+ def _resolve_config(cls, config: Union[str, object, list]) -> Union[object, list[object]]:
+ """Convert string alias(es) to the corresponding config instance(s) with default parameters."""
+ from auto_round.algorithms.registry import resolve_alg_config
+
+ if isinstance(config, str):
+ return resolve_alg_config(config)
+ if isinstance(config, list):
+ return [cls._resolve_config(c) for c in config]
+ return config
- bits: int | None
- group_size: int | tuple | None
- sym: bool | None
- data_type: str | None
- act_bits: int | None
- act_group_size: int | None
- act_sym: bool | None
- act_data_type: str | None
- act_dynamic: bool | None
- super_bits: int | None
- super_group_size: int | None
-
- # all args in __new__ need be passed to the dynamic created class __init__
def __new__(
cls,
model: Union[torch.nn.Module, str],
+ scheme="W4A16",
+ alg_configs: Union[str, object, list[Union[str, object]]] = None,
tokenizer=None,
- platform: str = "hf",
- scheme: Union[str, dict, QuantizationScheme, "AutoScheme"] = "W4A16",
- layer_config: dict[str, Union[str, dict, QuantizationScheme]] = None,
- dataset: Optional[Union[str, list, tuple, torch.utils.data.DataLoader]] = None,
- iters: int = 200,
- seqlen: int = 2048,
- nsamples: int = 128,
- batch_size: int = 8,
- gradient_accumulate_steps: int = 1,
+ platform="hf",
+ format=None,
+ dataset="NeelNanda/pile-10k",
low_gpu_mem_usage: bool = False,
device_map: Union[str, torch.device, int, dict] = 0,
- enable_torch_compile: Optional[bool] = None,
+ iters: int = None,
+ enable_torch_compile: bool = False,
seed: int = 42,
- enable_adam: bool = False,
- enable_alg_ext: bool = False,
- disable_opt_rtn: bool | None = None,
low_cpu_mem_usage: bool = True,
- alg_configs=None,
+ layer_config=None,
+ nsamples: int = None,
+ seqlen: int = None,
**kwargs,
) -> "BaseCompressor":
- """Initialize AutoRound with quantization and tuning configuration.
-
- Args:
- model (torch.nn.Module | str): Model object or model name to load.
- tokenizer: Tokenizer for text processing. Required if `model` is not a string and `iters > 0`.
- platform: The platform to download pretrained model, options: ["hf", "model_scope"]
- scheme (str| dict | QuantizationScheme ): A preset scheme that defines the quantization configurations
- layer_config (dict, optional): Layer-wise quantization config. Defaults to None.
- dataset (str | list | tuple | DataLoader, optional): Calibration data. Defaults to "NeelNanda/pile-10k".
- iters (int, optional): Optimization iterations. Defaults to 200.
- seqlen (int, optional): Calibration sequence length. Defaults to 2048.
- nsamples (int, optional): Number of calibration samples. Defaults to 128.
- batch_size (int, optional): Calibration batch size. Defaults to 8.
- gradient_accumulate_steps (int, optional): Gradient accumulation steps. Defaults to 1.
- low_gpu_mem_usage (bool, optional): Lower GPU memory mode. Defaults to False.
- device_map (str | dict, optional): Device map for each module. Defaults to 0.
- enable_torch_compile (bool, optional): Enable torch.compile for low cost in quantization. Defaults to True
- except on Windows, where it defaults to False. Pass True to force it.
- seed (int, optional): Random seed. Defaults to 42.
- enable_adam (bool, optional): Enable Adam-based optimizer. Defaults to False.
- enable_alg_ext (bool, optional): Enable algorithm extension (primarily for INT2)
- for better accuracy. Defaults to False.
- disable_opt_rtn (bool, optional): Disable RTN-mode optimization (iters=0) for fast quatnziation
- with lower accuracy. Defaults to None.
- low_cpu_mem_usage (bool, optional): Lower CPU memory mode. Defaults to False.
-
- bits (int, optional): Weight quantization bits. Defaults to 4.
- group_size (int or tuple, optional): Weight quantization group size. Defaults to 128.
- sym (bool, optional): Symmetric weight quantization. Defaults to True.
- data_type (str, optional): Weight data type string, e.g., "int". Defaults to "int".
- act_bits (int, optional): Activation quantization bits. Defaults to 16.
- act_group_size (int, optional): Activation group size. Defaults to None.
- act_sym (bool, optional): Symmetric activation quantization. Defaults to None.
- act_data_type (str, optional): Activation data type; inherits weight dtype if None and act_bits < 16.
- act_dynamic (bool, optional): Dynamic activation quantization. Defaults to True.
- model_dtype (str): model dtype used to load pre-trained model.
- amp (bool, optional): Use AMP for tuning. Defaults to True.
- enable_minmax_tuning (bool, optional): Enable weight min-max tuning. Defaults to True.
- lr (float, optional): Learning rate; if None, set to 1.0 / iters except when iters==0.
- minmax_lr (float, optional): Learning rate for min-max tuning; defaults to `lr`.
-
- **kwargs: Backward compatible options:
- - enable_alg_ext, quant_lm_head, lr, lr_scheduler, sampler, not_use_best_mse, dynamic_max_gap,
- super_group_size, super_bits, scale_dtype ("fp16" etc.),
- nblocks, to_quant_block_names,
- enable_norm_bias_tuning, enable_quanted_input,
- disable_deterministic_algorithms, vlm, static_kv_dtype
- Raises:
- ValueError: If invalid device is provided or tokenizer is missing for non-str model with iters > 0.
- RuntimeError: If model parameters are on meta device.
- Example:
- Layer-wise configuration structure:
-
- >>> layer_config = {
- ... "layer1": {
- ... "bits": 3,
- ... "group_size": 128,
- ... "sym": True,
- ... },
- ... "layer2": {
- ... "W8A16"
- ... }
- ... # ...
- ... }
- """
+ from auto_round.algorithms.quantization.rtn.config import OptimizedRTNConfig, RTNConfig
+ from auto_round.algorithms.quantization.sign_round.config import SignRoundConfig
+ from auto_round.algorithms.registry import normalize_algorithm_config
+ from auto_round.compressors.orchestrator import CompressionOrchestrator as Compressor
+ from auto_round.compressors.utils import check_need_act_calibration
+ from auto_round.utils.model import is_model_free_route
+
+ if alg_configs is None:
+ alg_configs = "signround"
+ # TODO wenhuach if key in kwargs could override scheme and alg_config, we should pop and override,
+ # e.g. gradient_accumulate_step
device_map = normalize_default_device_map(device_map)
-
- # Short-circuit: if alg_configs is provided, bypass AutoRoundCompatible and go directly
- # to the new-arch entry point to avoid duplicate keyword argument errors.
- if alg_configs is not None:
- from auto_round.compressors.entry import AutoRound as _NewAutoRound
- from auto_round.compressors.entry import filter_supported_entry_kwargs
-
- entry_kwargs = filter_supported_entry_kwargs(kwargs, context="AutoRound")
-
- return _NewAutoRound(
+ split_kwargs = _split_entry_kwargs(kwargs)
+ route_kwargs = dict(split_kwargs["route"])
+ compressor_kwargs = dict(split_kwargs["compressor"])
+ base_kwargs = dict(split_kwargs["base"])
+ mllm_kwargs = dict(split_kwargs["mllm"])
+ diffusion_kwargs = dict(split_kwargs["diffusion"])
+
+ # Resolve string alias(es) to config instance(s) before routing.
+ alg_configs = cls._resolve_config(alg_configs)
+ if isinstance(alg_configs, list):
+ alg_configs = [normalize_algorithm_config(cfg) for cfg in alg_configs]
+ else:
+ alg_configs = normalize_algorithm_config(alg_configs)
+ configs_for_routing = alg_configs if isinstance(alg_configs, list) else [alg_configs]
+ preprocessor_configs, _, quant_config = _resolve_quant_config_for_routing(configs_for_routing)
+
+ # Model-free routing is now supported directly by the new entry path.
+ model_free_iters = 0 if isinstance(quant_config, RTNConfig) else getattr(quant_config, "iters", None)
+ model_free_disable_opt_rtn = getattr(quant_config, "disable_opt_rtn", None)
+ # Model-free eligibility also depends on base-level options such as
+ # static KV/attention quantization. Keep those options visible to the
+ # route predicate; otherwise the fast path silently drops them and
+ # cannot emit the required export metadata.
+ route_decision_kwargs = dict(base_kwargs, **route_kwargs, format=format)
+ route_scheme = (
+ scheme
+ if hasattr(scheme, "options") and hasattr(scheme, "avg_bits")
+ else QuantizationScheme.from_dict(_preview_resolved_attrs(quant_config, scheme))
+ )
+ if is_model_free_route(
+ model, route_scheme, model_free_iters, model_free_disable_opt_rtn, route_decision_kwargs
+ ):
+ # Direct scheme fields are consumed into ``quant_config`` during
+ # entry normalization. Pass the fully resolved scheme onward so
+ # model-free export does not silently fall back to preset defaults.
+ model_free_scheme = scheme if hasattr(scheme, "options") and hasattr(scheme, "avg_bits") else route_scheme
+ return _build_model_free_compressor(
model,
- scheme,
- alg_configs,
- tokenizer=tokenizer,
- platform=platform,
- format=entry_kwargs.pop("format", None),
- low_gpu_mem_usage=low_gpu_mem_usage,
- device_map=device_map,
- iters=iters,
- gradient_accumulate_steps=gradient_accumulate_steps,
+ model_free_scheme,
+ layer_config,
+ tokenizer,
+ device_map,
+ announced_via_flag=bool(route_kwargs.get("model_free", False)),
enable_torch_compile=enable_torch_compile,
- seed=seed,
- low_cpu_mem_usage=low_cpu_mem_usage,
- layer_config=layer_config,
- dataset=dataset,
- nsamples=nsamples,
- seqlen=seqlen,
- batch_size=batch_size,
- **entry_kwargs,
+ **compressor_kwargs,
+ **base_kwargs,
+ **mllm_kwargs,
+ **diffusion_kwargs,
+ **route_kwargs,
)
- compat_kwargs = _filter_supported_compat_kwargs(kwargs)
- compat_kwargs.update(
- enable_adam=enable_adam,
- enable_alg_ext=enable_alg_ext,
- disable_opt_rtn=disable_opt_rtn,
- )
-
- from auto_round.compressors.entry import AutoRoundCompatible
+ # Eagerly validate scheme constraints that do not require model info.
+ # This mirrors old-arch _check_configs() called at __init__ time so that
+ # callers get ValueError/NotImplementedError on construction, not deferred.
+ _eager_validate_scheme(quant_config, scheme)
- return AutoRoundCompatible(
+ local_args = dict(
model=model,
tokenizer=tokenizer,
platform=platform,
+ format=format,
scheme=scheme,
- layer_config=layer_config,
dataset=dataset,
- iters=iters,
- seqlen=seqlen,
- nsamples=nsamples,
- batch_size=batch_size,
- gradient_accumulate_steps=gradient_accumulate_steps,
low_gpu_mem_usage=low_gpu_mem_usage,
device_map=device_map,
+ iters=iters,
enable_torch_compile=enable_torch_compile,
seed=seed,
low_cpu_mem_usage=low_cpu_mem_usage,
- **compat_kwargs,
+ layer_config=layer_config,
+ nsamples=nsamples,
+ seqlen=seqlen,
+ **compressor_kwargs,
)
+ model_type, ctor_kwargs = _build_model_type_ctor_kwargs(model, base_kwargs, mllm_kwargs, diffusion_kwargs)
- @classmethod
- @torch.no_grad()
- def _sampling_inputs(
+ # Preprocessor algorithms (AWQ, …) require a data-driven host so that
+ # the per-block preprocessor lifecycle (prepare_block_group ->
+ # block_forward_hooks -> pre_quantize_block -> post_quantize_block)
+ # actually runs; the pipeline auto-appends RTN when no block_quantizer
+ # is supplied. SignRound is itself data-driven and shares the same host.
+ if preprocessor_configs or isinstance(quant_config, SignRoundConfig):
+ return _get_compressor_class(model_type, Compressor)(alg_configs, **local_args, **ctor_kwargs)
+ elif isinstance(quant_config, RTNConfig):
+ base_cls = _select_rtn_compressor_base_cls(quant_config, scheme, format, base_kwargs)
+ return _get_compressor_class(model_type, base_cls)(alg_configs, **local_args, **ctor_kwargs)
+
+
+class AutoRound:
+ """Unified AutoRound entry point.
+
+ alg_configs accepts an algorithm alias, one QuantizationConfig, or a
+ sequence of either. When omitted, SignRound is selected. AWQ-only
+ pipelines receive an RTN block quantizer by default.
+ """
+
+ SKIP_ARGS = ("local_args", "kwargs", "cls", "model_cls", "dynamic_compressor", "alg_configs")
+
+ def __new__(
cls,
- input_ids: list[torch.Tensor],
- input_others: dict,
- indices: list[int],
- seqlen: int,
- batch_dim: int = 0,
- share_cache_keys: tuple = (),
- ):
- """Samples inputs based on the given indices and sequence length.
-
- Args:
- input_ids: The list of input tensor containing input_ids.
- input_others: A dictionary containing other input data.
- indices: The indices to sample from the input.
- seqlen: The sequence length.
-
- Returns:
- current_input_ids: The sampled input IDs.
- current_input_others: The sampled other input data.
- """
- current_input_ids = [input_ids[i] for i in indices]
-
- current_input_ids = torch.cat(current_input_ids, dim=batch_dim)
-
- current_input_others = {"positional_inputs": input_others["positional_inputs"]}
- for key in input_others.keys():
- if "positional_inputs" in key:
- continue
- # Shared cache keys (e.g. position_embeddings, position_ids, cache_position) are stored
- # directly as-is (not wrapped in a per-sample list) when batch_size > 1. Indexing such
- # values by sample index would incorrectly decompose them (e.g. (cos, sin)[0] == cos).
- # Always pass them through unchanged.
- if key in share_cache_keys or isinstance(input_others[key], (str, bool, type(None))):
- current_input_others[key] = input_others[key]
- elif input_others[key] is not None:
- current_input_others[key] = [input_others[key][i] for i in indices]
- if len(indices) == 1:
- current_input_others[key] = current_input_others[key][0]
- else:
- try:
- current_input_others[key] = torch.cat(current_input_others[key], dim=0)
- except TypeError as err:
- logger.warning_once("Please check the model cache inputs or try setting batch_size to 1.")
- else:
- current_input_others[key] = None
-
- return current_input_ids, current_input_others
+ model: Union[torch.nn.Module, str],
+ tokenizer=None,
+ platform: str = "hf",
+ scheme: Union[str, dict, QuantizationScheme, "AutoScheme"] = "W4A16",
+ layer_config: dict[str, Union[str, dict, QuantizationScheme]] = None,
+ dataset: Optional[Union[str, list, tuple, torch.utils.data.DataLoader]] = None,
+ iters: int | None = None,
+ seqlen: int = 2048,
+ nsamples: int = 128,
+ batch_size: int = 8,
+ gradient_accumulate_steps: int | None = None,
+ low_gpu_mem_usage: bool = False,
+ device_map: Union[str, torch.device, int, dict] = 0,
+ enable_torch_compile: Optional[bool] = None,
+ seed: int = 42,
+ low_cpu_mem_usage: bool = True,
+ alg_configs=None,
+ algorithm: str | None = None,
+ **kwargs,
+ ) -> "BaseCompressor":
+ direct_kwargs = dict(kwargs)
+ legacy_device = direct_kwargs.pop("device", None)
+ if legacy_device is not None:
+ logger.warning_once("`device` is deprecated, please use `device_map` instead")
+ if device_map in (None, 0):
+ device_map = legacy_device
+ if iters is not None:
+ direct_kwargs["iters"] = iters
+ if gradient_accumulate_steps is not None:
+ direct_kwargs["gradient_accumulate_steps"] = gradient_accumulate_steps
+ if algorithm is not None:
+ direct_kwargs["algorithm"] = algorithm
+
+ configs, runtime_kwargs = _prepare_entry_kwargs(alg_configs, direct_kwargs)
+ runtime_kwargs["batch_size"] = batch_size
+
+ return _CompressorBuilder(
+ model,
+ scheme,
+ configs,
+ tokenizer=tokenizer,
+ platform=platform,
+ format=runtime_kwargs.pop("format", None),
+ dataset=dataset,
+ low_gpu_mem_usage=low_gpu_mem_usage,
+ device_map=normalize_default_device_map(device_map),
+ iters=None,
+ enable_torch_compile=enable_torch_compile,
+ seed=seed,
+ low_cpu_mem_usage=low_cpu_mem_usage,
+ layer_config=layer_config,
+ nsamples=nsamples,
+ seqlen=seqlen,
+ **runtime_kwargs,
+ )
+# Keep legacy entry points available for downstream integrations such as
+# Neural Compressor. They delegate to the unified entry point.
@deprecated("AutoRound")
class AutoRoundLLM:
-
def __new__(cls, *args, **kwargs):
return AutoRound(*args, **kwargs)
@deprecated("AutoRound")
class AutoRoundAdam:
-
def __new__(cls, *args, **kwargs):
kwargs.setdefault("enable_adam", True)
return AutoRound(*args, **kwargs)
@@ -351,13 +788,11 @@ class AutoRoundAdam:
@deprecated("AutoRound")
class AutoRoundMLLM:
-
def __new__(cls, *args, **kwargs):
return AutoRound(*args, **kwargs)
@deprecated("AutoRound")
class AutoRoundDiffusion:
-
def __new__(cls, *args, **kwargs):
return AutoRound(*args, **kwargs)
diff --git a/auto_round/cli/algorithms.py b/auto_round/cli/algorithms.py
index 8291d1d82..193d70583 100644
--- a/auto_round/cli/algorithms.py
+++ b/auto_round/cli/algorithms.py
@@ -22,6 +22,7 @@ from auto_round.algorithms.registry import (
iter_algorithm_entries,
register_algorithm,
resolve_algorithm_alias,
+ resolve_algorithm_names,
)
# ============================================================================
@@ -108,19 +109,21 @@ class AlgorithmHandler(ABC):
if getattr(args, "rotation_hadamard_type", None) and "hadamard" not in names:
names.append("hadamard")
- # Resolve aliases, drop unknowns, deduplicate preserving order
- seen: set[str] = set()
- canonical: list[str] = []
- for n in names:
- c = cls.resolve_alias(n)
- if c and c not in seen:
- canonical.append(c)
- seen.add(c)
+ # Resolve aliases, drop unknowns, deduplicate preserving order.
+ # Python API strings use this same resolver so their composition
+ # semantics cannot drift apart again.
+ canonical = resolve_algorithm_names(names, ignore_unknown=True)
+ seen = set(canonical)
# Default quantization algorithm if none was specified
if not ({"awq", "rtn", "auto_round"} & seen):
canonical.append("rtn" if getattr(args, "iters", 0) == 0 else "auto_round")
+ # Keep the legacy API rule even when the user explicitly spells out
+ # ``--algorithm auto_round``: zero iterations select RTN.
+ if getattr(args, "iters", None) == 0:
+ canonical = ["rtn" if name == "auto_round" else name for name in canonical]
+
return [cls.get(name).build(args, common_kwargs) for name in canonical]
@classmethod
diff --git a/auto_round/cli/main.py b/auto_round/cli/main.py
index 50871dfb7..e99595119 100644
--- a/auto_round/cli/main.py
+++ b/auto_round/cli/main.py
@@ -24,6 +24,8 @@ from auto_round.cli.parser import (
build_quantize_parser,
build_root_parser,
)
+from auto_round.compressors.config_resolution import ConfigResolutionError
+from auto_round.logger import logger
def _extract_common_quantization_kwargs(args) -> dict:
@@ -142,7 +144,7 @@ RECIPES = {
def list_item(argv=None):
args = build_list_parser().parse_args(argv)
if args.item in {"format", "formats"}:
- from auto_round.formats import OutputFormat
+ from auto_round.export.formats import OutputFormat
print("AutoRound supported output formats and quantization scheme:")
print(OutputFormat.get_support_matrix())
@@ -288,8 +290,8 @@ def tune(args):
model_name = model_name[:-1]
logger.info(f"start to quantize {model_name}")
+ from auto_round import AutoRound
from auto_round.compressors.base import BaseOrchestrator as BaseCompressor
- from auto_round.compressors.entry import AutoRound as PipelineAutoRound
if "bloom" in model_name:
args.low_gpu_mem_usage = False
@@ -383,10 +385,10 @@ def tune(args):
from auto_round.utils import clear_memory
- autoround: BaseCompressor = PipelineAutoRound(
+ autoround: BaseCompressor = AutoRound(
model_name,
- scheme,
- alg_configs if len(alg_configs) > 1 else alg_configs[0],
+ scheme=scheme,
+ alg_configs=alg_configs if len(alg_configs) > 1 else alg_configs[0],
**_to_autoround_kwargs(
args,
low_cpu_mem_usage=low_cpu_mem_usage,
@@ -481,20 +483,24 @@ def _print_help(topic=None):
def run():
- argv = list(sys.argv[1:])
- command, command_argv = _normalize_cli_invocation(argv)
-
- if command == "help":
- root_args = build_root_parser().parse_args(argv)
- _print_help(root_args.topic)
- return
- if command == "list":
- list_item(command_argv)
- return
- if command == "eval":
- run_eval(command_argv)
- return
- start(argv=command_argv)
+ try:
+ argv = list(sys.argv[1:])
+ command, command_argv = _normalize_cli_invocation(argv)
+
+ if command == "help":
+ root_args = build_root_parser().parse_args(argv)
+ _print_help(root_args.topic)
+ return
+ if command == "list":
+ list_item(command_argv)
+ return
+ if command == "eval":
+ run_eval(command_argv)
+ return
+ start(argv=command_argv)
+ except ConfigResolutionError as error:
+ logger.error(str(error))
+ raise SystemExit(2) from error
def run_best():
diff --git a/auto_round/compressors/__init__.py b/auto_round/compressors/__init__.py
index 833e764ec..7965acf4b 100644
--- a/auto_round/compressors/__init__.py
+++ b/auto_round/compressors/__init__.py
@@ -18,29 +18,20 @@ from typing import TYPE_CHECKING
if TYPE_CHECKING:
from auto_round.compressors.base import BaseOrchestrator, BaseCompressor
- from auto_round.compressors.orchestrator import CompressionOrchestrator
- from auto_round.compressors.entry import AutoRoundCompatible, AutoRound
from auto_round.compressors.model_free import ModelFreeCompressor
+ from auto_round.compressors.orchestrator import CompressionOrchestrator
__all__ = [
- "AutoRound",
"BaseOrchestrator",
"BaseCompressor", # backward-compat alias
"CompressionOrchestrator",
- "AutoRoundCompatible",
"ModelFreeCompressor",
]
def __getattr__(name):
"""Lazy import to avoid circular dependencies."""
- if name in ("AutoRound", "AutoRoundCompatible"):
- from auto_round.compressors.entry import AutoRound, AutoRoundCompatible
-
- if name == "AutoRound":
- return AutoRound
- return AutoRoundCompatible
- elif name in ("BaseOrchestrator", "BaseCompressor"):
+ if name in ("BaseOrchestrator", "BaseCompressor"):
from auto_round.compressors.base import BaseOrchestrator
return BaseOrchestrator
@@ -48,8 +39,9 @@ def __getattr__(name):
from auto_round.compressors.orchestrator import CompressionOrchestrator
return CompressionOrchestrator
- elif name in ("Compressor", "ZeroShotCompressor"):
- # Backward-compat aliases
+ elif name in ("Compressor", "ZeroShotCompressor", "DataDrivenCompressor", "CalibratedRTNCompressor"):
+ # Backward-compat aliases: these classes were consolidated into
+ # ``CompressionOrchestrator`` (main's #2039 compressor/quantizer refactor).
from auto_round.compressors.orchestrator import CompressionOrchestrator
return CompressionOrchestrator
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py
index f8d6cfd9a..1da140d04 100644
--- a/auto_round/compressors/base.py
+++ b/auto_round/compressors/base.py
@@ -15,7 +15,7 @@ import copy
import gc
import os
import sys
-from dataclasses import asdict, dataclass, fields
+from dataclasses import asdict, dataclass, fields, replace
from typing import Any, Optional, Union
import torch
@@ -27,11 +27,23 @@ from auto_round.algorithms.transforms import (
apply_rotation,
)
from auto_round.auto_scheme.gen_auto_scheme import AutoScheme
+from auto_round.compressors.config_resolution import (
+ FormatResolution,
+ ResolvedScheme,
+ resolve_quantization_config,
+ thaw_mapping,
+)
+from auto_round.compressors.layer_config import (
+ apply_plan_to_model,
+ extract_regex_config,
+ has_quantized_layer_outside_blocks,
+ resolve_layer_config,
+)
from auto_round.compressors.shard_writer import ShardWriter
-from auto_round.compressors.utils import _get_save_folder_name, is_act_static, is_mx_fp, is_nv_fp, set_layer_config
+from auto_round.compressors.utils import _get_save_folder_name, is_mx_fp, is_nv_fp
from auto_round.context.compress import CompressContext
from auto_round.context.model import ModelContext
-from auto_round.formats import OutputFormat, get_formats
+from auto_round.export.formats import OutputFormat, resolve_formats
from auto_round.logger import logger
from auto_round.schemes import (
QuantizationScheme,
@@ -160,11 +172,6 @@ class BaseOrchestrator(object):
is_auto_scheme: bool = False
orig_scheme = None
scheme = None
- scale_dtype = None
- layer_config = None
- has_qlayer_outside_block: bool = False
- regex_config: dict = None
- quant_block_list: list = None
to_quant_block_names = None
ignore_layers: str = ""
quant_lm_head: bool = False
@@ -523,6 +530,100 @@ class BaseOrchestrator(object):
"""Convenience accessor for the tokenizer stored in ``model_context``."""
return self.model_context.tokenizer
+ def _replace_compression_plan(self, **changes) -> None:
+ """Atomically update compatibility state once the immutable plan exists."""
+ plan = self.__dict__.get("compression_plan")
+ if plan is not None:
+ self.__dict__["compression_plan"] = replace(plan, **changes)
+
+ @property
+ def scheme_context(self) -> Optional[QuantizationScheme]:
+ plan = self.__dict__.get("compression_plan")
+ return plan.scheme.value if plan is not None else self.__dict__.get("_scheme_context")
+
+ @scheme_context.setter
+ def scheme_context(self, value: Optional[QuantizationScheme]) -> None:
+ self.__dict__["_scheme_context"] = value
+ plan = self.__dict__.get("compression_plan")
+ if plan is not None and value is not None:
+ self._replace_compression_plan(
+ scheme=ResolvedScheme.from_scheme(value, preset_name=plan.scheme.preset_name)
+ )
+
+ @property
+ def formats(self):
+ raw_value = self.__dict__.get("_formats")
+ plan = self.__dict__.get("compression_plan")
+ if plan is None or isinstance(raw_value, str) or (not plan.formats and raw_value is None):
+ return raw_value
+ return list(plan.formats)
+
+ @formats.setter
+ def formats(self, value) -> None:
+ self.__dict__["_formats"] = value
+ if value is not None and not isinstance(value, str):
+ self._replace_compression_plan(formats=tuple(value))
+
+ @property
+ def layer_config(self) -> Optional[dict]:
+ plan = self.__dict__.get("compression_plan")
+ if plan is None:
+ return self.__dict__.get("_layer_config")
+ return {name: dict(config) for name, config in plan.layer_config.items()}
+
+ @layer_config.setter
+ def layer_config(self, value) -> None:
+ self.__dict__["_layer_config"] = value
+ if value is not None:
+ self._replace_compression_plan(layer_config=value)
+
+ @property
+ def regex_config(self) -> Optional[dict]:
+ plan = self.__dict__.get("compression_plan")
+ if plan is None:
+ return self.__dict__.get("_regex_config")
+ return {name: dict(config) for name, config in plan.regex_config.items()}
+
+ @regex_config.setter
+ def regex_config(self, value) -> None:
+ self.__dict__["_regex_config"] = value
+ if value is not None:
+ self._replace_compression_plan(regex_config=value)
+
+ @property
+ def has_qlayer_outside_block(self) -> bool:
+ plan = self.__dict__.get("compression_plan")
+ if plan is None:
+ return self.__dict__.get("_has_qlayer_outside_block", False)
+ return plan.has_qlayer_outside_block
+
+ @has_qlayer_outside_block.setter
+ def has_qlayer_outside_block(self, value: bool) -> None:
+ self.__dict__["_has_qlayer_outside_block"] = value
+ self._replace_compression_plan(has_qlayer_outside_block=value)
+
+ @property
+ def scale_dtype(self):
+ plan = self.__dict__.get("compression_plan")
+ return plan.scale_dtype if plan is not None else self.__dict__.get("_scale_dtype")
+
+ @scale_dtype.setter
+ def scale_dtype(self, value) -> None:
+ self.__dict__["_scale_dtype"] = value
+ self._replace_compression_plan(scale_dtype=value)
+
+ @property
+ def quant_block_list(self):
+ plan = self.__dict__.get("compression_plan")
+ if plan is None:
+ return self.__dict__.get("_quant_block_list")
+ return [list(group) for group in plan.quant_block_list] if plan.quant_block_list is not None else None
+
+ @quant_block_list.setter
+ def quant_block_list(self, value) -> None:
+ self.__dict__["_quant_block_list"] = value
+ self._replace_compression_plan(quant_block_list=value)
+
# ── Scheme resolution ─────────────────────────────────────────────────────
def resolve_scheme(
@@ -532,7 +633,7 @@ class BaseOrchestrator(object):
) -> None:
"""Phase-1 init: resolve scheme and bind config attrs (no model structure needed).
- Must be called BEFORE ``get_formats()`` and BEFORE ``_scheme_post_init()``.
+ Must be called BEFORE ``_resolve_formats()`` and BEFORE ``_scheme_post_init()``.
Idempotent: safe to call multiple times.
"""
if self._scheme_resolved:
@@ -632,19 +733,53 @@ class BaseOrchestrator(object):
"Please save the model using the `fake` format for now."
)
- layer_config, self.has_qlayer_outside_block, self.regex_config = set_layer_config(
- self.model_context.model,
- self.layer_config,
- self.scheme,
- self.scale_dtype,
- self.supported_types,
- self.inner_supported_types,
- self.quant_block_list,
- self.ignore_layers,
- self.quant_lm_head,
+ preset_name = self.scheme if isinstance(self.scheme, str) else None
+ format_resolution = getattr(
+ self,
+ "_format_resolution",
+ FormatResolution(
+ formats=tuple(self.formats or ()),
+ scheme=ResolvedScheme.from_scheme(self.scheme_context, preset_name=preset_name),
+ scale_dtype=self.scale_dtype,
+ quant_block_list=self.quant_block_list,
+ ),
+ )
+ resolved_layer_config = resolve_layer_config(
+ model=self.model_context.model,
+ scheme=format_resolution.scheme,
+ layer_config=self.layer_config,
+ scale_dtype=self.scale_dtype,
+ supported_types=self.supported_types,
+ inner_supported_types=self.inner_supported_types,
+ quant_block_list=self.quant_block_list,
+ ignore_layers=self.ignore_layers,
+ quant_lm_head=self.quant_lm_head,
enable_gguf_official_mixed=False,
is_mllm=self.model_context.is_mllm,
)
+ regex_config = extract_regex_config(
+ model=self.model_context.model,
+ scheme=format_resolution.scheme,
+ layer_config=self.layer_config,
+ scale_dtype=self.scale_dtype,
+ supported_types=self.supported_types,
+ inner_supported_types=self.inner_supported_types,
+ ignore_layers=self.ignore_layers,
+ )
+ discovery_plan = resolve_quantization_config(
+ (
+ replace(format_resolution, layer_config_patch={})
+ if format_resolution.layer_config_patch
+ else format_resolution
+ ),
+ resolved_layer_config,
+ regex_config=regex_config,
+ has_qlayer_outside_block=has_quantized_layer_outside_blocks(resolved_layer_config),
+ )
+ apply_plan_to_model(self.model_context.model, discovery_plan)
+ layer_config = {name: dict(config) for name, config in discovery_plan.layer_config.items()}
+ self.has_qlayer_outside_block = discovery_plan.has_qlayer_outside_block
+ self.regex_config = {name: dict(config) for name, config in discovery_plan.regex_config.items()}
quant_layer_names = layer_config.keys()
# ---- VLM: peel non-text sub-trees AutoScheme should not score ---- #
@@ -731,6 +866,13 @@ class BaseOrchestrator(object):
def configure_layer_config(self, enable_gguf_official_mixed: bool | None = True) -> None:
"""Build ``self.layer_config`` from the resolved scheme on the patched model."""
+ # External callers (e.g. llm-compressor's AutoRoundModifier) may invoke this
+ # method directly without going through the normal post_init()/_scheme_post_init()
+ # sequence. Make sure the scheme is resolved first so `self.scheme_context` (and
+ # everything derived from it below) is never None -- resolve_scheme() is a no-op
+ # if it already ran.
+ if not self._scheme_resolved and hasattr(self, "_alg_configs"):
+ self.resolve_scheme()
_formats = getattr(self.compress_context, "formats", None)
is_gguf_format = _formats is not None and any(
"gguf" in str(getattr(fmt, "output_format", "")) for fmt in _formats
@@ -773,34 +915,64 @@ class BaseOrchestrator(object):
quant_lm_head=self.quant_lm_head,
mllm=self.model_context.is_mllm,
)
- _gguf_orig_fmt = getattr(self, "_gguf_original_format_name", None)
- if _gguf_orig_fmt and "_MIXED" in _gguf_orig_fmt.upper():
- self.layer_config = _handle_special_schemes(
- _gguf_orig_fmt.lower(),
- self.layer_config,
- self.model_context.model,
- supported_types=SUPPORTED_LAYER_TYPES,
- inner_supported_types=INNER_SUPPORTED_LAYER_TYPES,
- quant_lm_head=self.quant_lm_head,
- mllm=self.model_context.is_mllm,
- )
fill_default_value = not self.is_auto_scheme
- self.layer_config, self.has_qlayer_outside_block, self.regex_config = set_layer_config(
- self.model_context.model,
- self.layer_config,
- self.scheme,
- self.scale_dtype,
- SUPPORTED_LAYER_TYPES,
- INNER_SUPPORTED_LAYER_TYPES,
- self.quant_block_list,
- self.ignore_layers,
- self.quant_lm_head,
+ source_layer_config = self.layer_config
+ format_resolution = getattr(
+ self,
+ "_format_resolution",
+ FormatResolution(
+ formats=tuple(self.formats or ()),
+ scheme=ResolvedScheme.from_scheme(
+ self.scheme_context,
+ preset_name=self._resolve_gguf_preset_string(self.formats or []),
+ ),
+ scale_dtype=self.scale_dtype,
+ quant_block_list=self.quant_block_list,
+ ),
+ )
+ resolved_layer_config = resolve_layer_config(
+ model=self.model_context.model,
+ scheme=format_resolution.scheme,
+ layer_config=source_layer_config,
+ scale_dtype=self.scale_dtype,
+ supported_types=SUPPORTED_LAYER_TYPES,
+ inner_supported_types=INNER_SUPPORTED_LAYER_TYPES,
+ quant_block_list=self.quant_block_list,
+ ignore_layers=self.ignore_layers,
+ quant_lm_head=self.quant_lm_head,
enable_gguf_official_mixed=enable_gguf_official_mixed,
is_mllm=self.model_context.is_mllm,
fill_default_value=fill_default_value,
- gguf_format_name=getattr(self, "_gguf_format_name", None),
)
+ regex_config = extract_regex_config(
+ model=self.model_context.model,
+ scheme=format_resolution.scheme,
+ layer_config=source_layer_config,
+ scale_dtype=self.scale_dtype,
+ supported_types=SUPPORTED_LAYER_TYPES,
+ inner_supported_types=INNER_SUPPORTED_LAYER_TYPES,
+ ignore_layers=self.ignore_layers,
+ fill_default_value=fill_default_value,
+ )
+ # ``resolved_layer_config`` already descends from (and fully subsumes)
+ # ``format_resolution.layer_config_patch`` -- ``source_layer_config`` above was
+ # seeded from that same patch before ``resolve_layer_config()`` expanded any
+ # regex/partial keys (e.g. "self_attn") into concrete layer names. Re-merging the
+ # patch here would reintroduce those now-stale, unexpanded keys into the final
+ # plan, so drop it and rely solely on the fully-resolved layer configuration.
+ if format_resolution.layer_config_patch:
+ format_resolution = replace(format_resolution, layer_config_patch={})
+ self.compression_plan = resolve_quantization_config(
+ format_resolution,
+ resolved_layer_config,
+ regex_config=regex_config,
+ has_qlayer_outside_block=has_quantized_layer_outside_blocks(resolved_layer_config),
+ )
+ self.layer_config = {name: dict(config) for name, config in self.compression_plan.layer_config.items()}
+ self.regex_config = {name: dict(config) for name, config in self.compression_plan.regex_config.items()}
+ self.has_qlayer_outside_block = self.compression_plan.has_qlayer_outside_block
+ apply_plan_to_model(self.model_context.model, self.compression_plan)
if self.is_auto_scheme:
from auto_round.auto_scheme.utils import compute_avg_bits_for_model
@@ -992,149 +1164,165 @@ class BaseOrchestrator(object):
``model_context`` / ``compress_context`` / ``scale_dtype`` /
``CalibrationContext`` from this compressor. ``quantizer.model``
is a property that reads ``model_context.model``.
- - Exposes ``self.quantizer`` as a ``@property`` (see below) that
- transparently delegates to ``self.pipeline.block_quantizer`` so all
- existing call-sites continue to work without modification.
+ - Exposes ``self.alg_composer.block_quantizer`` so all quantization
+ call-sites can reach the block quantizer directly.
Postconditions:
- - ``self.pipeline`` is an ``AlgorithmComposer`` wrapping the block quantizer.
- - ``self.quantizer`` (via property) is ready and shares ``CalibrationContext``
+ - ``self.alg_composer`` is an ``AlgorithmComposer`` wrapping the block quantizer.
+ - ``self.alg_composer.block_quantizer`` is ready and shares ``CalibrationContext``
with the compressor.
"""
from auto_round.algorithms.composer import AlgorithmComposer
self._alg_composer = AlgorithmComposer(self._alg_configs, orchestrator=self)
+ # Sync the fully-resolved scheme state (built by _build_layer_config(), which
+ # always runs before _build_composer() in post_init()) onto the block quantizer
+ # so quantization methods (quantize_block, quantize_layer, etc.) have access to
+ # layer_config, quant_block_list, etc. ``scale_dtype``/``scheme`` are NOT set here:
+ # they are read-only properties backed by ``BaseAlgorithm.bind()``, which already
+ # picked up the final ``self.scale_dtype``/``self.scheme_context`` above.
+ plan = getattr(self, "compression_plan", None)
+ if plan is not None:
+ block_quantizer = self._alg_composer.block_quantizer
+ block_quantizer.layer_config = {name: dict(config) for name, config in plan.layer_config.items()}
+ block_quantizer.has_qlayer_outside_block = plan.has_qlayer_outside_block
+ block_quantizer.regex_config = {name: dict(config) for name, config in plan.regex_config.items()}
+ block_quantizer.quant_block_list = (
+ [list(group) for group in plan.quant_block_list] if plan.quant_block_list is not None else None
+ )
+ block_quantizer.to_quant_block_names = self.to_quant_block_names
+ block_quantizer.ignore_layers = self.ignore_layers
+
+ from auto_round.algorithms.config_resolver import sync_shared_config_from
+
+ sync_shared_config_from(block_quantizer.config, [pre.config for pre in self._alg_composer.preprocessors])
+
+ # Also sync runtime-only state to all preprocessors so they have access to
+ # per-layer quant config during pre-processing (e.g. AWQ grid search uses
+ # layer_config to look up bits/group_size for each layer).
+ for pre in self._alg_composer.preprocessors:
+ pre.layer_config = block_quantizer.layer_config
+
@property
def alg_composer(self) -> Any:
"""The active :class:`~auto_round.algorithms.pipeline.AlgorithmComposer`."""
return self._alg_composer
+ @staticmethod
+ def _resolve_gguf_preset_string(formats: list["OutputFormat"]) -> Optional[str]:
+ """Return the precise GGUF preset string (e.g. ``"gguf:q4_k_m"``) for the
+ single resolved GGUF format, or ``None`` if no GGUF format is present.
+
+ Used by :meth:`_resolve_format_string` so ``self.scheme`` stays a string
+ that :func:`auto_round.schemes.get_gguf_scheme` can short-circuit on,
+ preserving alias disambiguation (Q4_K_S vs Q4_K_M, Q*_0/Q*_1, ...).
+ ``_check_compatibility`` guarantees at most one GGUF format.
+ """
+ for fmt in formats or []:
+ if not fmt.is_gguf():
+ continue
+ # The outer GGUFFormat reports output_format == "gguf"; the precise,
+ # alias-correct, post-rewrite preset lives on backend.output_format
+ # (e.g. "gguf:q4_k_m"). A standalone/inner GGUFFormat already carries
+ # the precise string on output_format itself.
+ backend = getattr(fmt, "backend", None)
+ backend_fmt = getattr(backend, "output_format", None) if backend is not None else None
+ precise = backend_fmt if backend_fmt and backend_fmt != "gguf" else fmt.output_format
+ if precise and precise != "gguf":
+ return precise
+ return None
+
+ def _resolve_format_string(self, format_str: str) -> list["OutputFormat"]:
+ """Resolve one format string via resolve_formats(), then propagate any
+ scheme/layer_config/scale_dtype/quant_block_list correction it makes
+ (e.g. GGUF's gguf_args_check) back onto self.
+
+ `scheme` may or may not be the object passed in — GGUF's "_mixed -> _s"
+ rewrite can rebuild it wholesale — so this always re-pins self.scheme_context
+ explicitly rather than relying on in-place mutation.
+ """
+ preset_name = self.scheme if isinstance(self.scheme, str) else None
+ self._format_resolution = resolve_formats(
+ ResolvedScheme.from_scheme(self.scheme_context, preset_name=preset_name),
+ format=format_str,
+ layer_config=self.layer_config or {},
+ scale_dtype=self.scale_dtype,
+ mllm=self.model_context.is_mllm,
+ iters=getattr(self, "iters", 0),
+ enable_alg_ext=getattr(self, "enable_alg_ext", False),
+ quant_nontext_module=self.quant_nontext_module,
+ quant_block_list=self.quant_block_list,
+ platform=self.platform,
+ is_auto_scheme=self.is_auto_scheme,
+ model=self.model_context.model,
+ )
+ formats = list(self._format_resolution.formats)
+ scheme = self._format_resolution.scheme.value
+ self.layer_config = thaw_mapping(self._format_resolution.layer_config_patch)
+ self.scale_dtype = self._format_resolution.scale_dtype
+ self.quant_block_list = (
+ [list(group) for group in self._format_resolution.quant_block_list]
+ if self._format_resolution.quant_block_list is not None
+ else None
+ )
+ self.scheme_context = scheme
+ for config in self._alg_configs:
+ if hasattr(config, "scheme"):
+ config.scheme = self.scheme_context
+ # self.scheme must stay an independent object from self.scheme_context
+ # (the existing invariant from resolve_scheme(), Phase 1) — never assign
+ # the same reference here even though it would often "work".
+ #
+ # Special-case GGUF: get_gguf_scheme() can only disambiguate presets that
+ # share identical QuantizationScheme fields (e.g. GGUF:Q4_K_S vs GGUF:Q4_K_M,
+ # or any GGUF:Q*_0 / GGUF:Q*_1 that its field-matching loop deliberately
+ # skips) via its string short-circuit. If we hand it the resolved object
+ # form, downstream resolve_layer_config() derives the WRONG (or empty) gguf
+ # preset name, corrupting per-tensor qtype/embedding selection. So when the
+ # resolved format is GGUF, pin self.scheme to the precise resolved preset
+ # string (the outer GGUFFormat keeps it on .backend.output_format, already
+ # past any "_mixed"->"_s" rewrite); _validate_format_combination guarantees at most
+ # one GGUF format here.
+ gguf_preset = self._format_resolution.scheme.preset_name
+ if gguf_preset is not None and not gguf_preset.startswith("gguf:"):
+ gguf_preset = None
+ if gguf_preset is not None:
+ self.scheme = gguf_preset
+ else:
+ self.scheme = copy.deepcopy(scheme)
+ return formats
+
def _resolve_formats(self) -> None:
- """Phase 2 – Format resolution and config attr sync.
+ """Phase 2 - Format resolution and scheme/config sync.
Preconditions:
- - Phase 1 complete: the scheme is resolved (``data_type``, ``bits``,
- ``sym`` etc. are set on both ``self`` and ``self.quantize_config``).
+ - Phase 1 complete: ``self.scheme`` / ``self.scheme_context`` are resolved.
Work performed:
- Converts a string ``self.formats`` to a list of
- :class:`~auto_round.formats.OutputFormat` objects via
- :func:`~auto_round.formats.get_formats`.
+ :class:`~auto_round.export.formats.OutputFormat` objects via
+ :meth:`_resolve_format_string`, which also propagates any scheme
+ correction (e.g. GGUF's ``gguf_args_check``) onto ``self.scheme``,
+ ``self.scheme_context``, the algorithm configs that share it, and
+ ``self.quantize_config``.
- Initialises :class:`~auto_round.compressors.shard_writer.ShardWriter`
when formats are present.
- - **(2b)** Detects format-driven attribute mutations (``bits``, ``sym``,
- ``data_type``, ``group_size``, etc.) that ``gguf_args_check`` may
- have written onto ``self`` inside ``get_formats``, syncs them back
- to ``self.quantize_config``, and rebuilds ``self.scheme`` accordingly.
- - Merges any format-injected entries into ``self.layer_config``.
Postconditions:
- ``self.formats`` is a list (or ``None``).
- ``self.compress_context.formats`` mirrors ``self.formats``.
- - ``self.quantize_config`` and ``self.scheme`` reflect the final attrs.
+ - ``self.scheme``, ``self.scheme_context`` and ``self.quantize_config``
+ all reflect any format-driven corrections (e.g. GGUF).
"""
- # get_formats() inspects data_type / bits etc. that were just resolved.
if isinstance(self.formats, str):
- self.formats = get_formats(self.formats, self)
+ self.formats = self._resolve_format_string(self.formats)
if self.formats is not None:
self.compress_context.formats = self.formats
ShardWriter.reset()
# Defer ShardWriter construction to _ensure_shard_writer() to avoid
# heap fragmentation during post_init (parameter iteration).
- # Snapshot the user-specified layer_config before format processing may
- # inject extra per-layer entries (e.g. GGUF embedding / lm_head).
- _pre_gguf_layer_config = copy.copy(self.layer_config) or {}
-
- # ── 2b: propagate format-adjusted attrs back to quantize_config ─────
- # gguf_args_check (called inside get_formats) may have overridden
- # bits / sym / data_type / super_bits / super_group_size / group_size
- # on *this* BaseCompressor object via setattr(self, ...). Sync those
- # changes to self.quantize_config before creating the quantizer so it is
- # constructed with the definitive final values.
- _gguf_forwarded_attrs = (
- "bits",
- "sym",
- "data_type",
- "super_bits",
- "super_group_size",
- "group_size",
- "act_bits",
- "scale_dtype",
- )
- # Skip this for AutoScheme — the format is for export only, and the
- # per-layer quantization is already determined by orig_scheme.options.
- # Restore scheme attrs from quantize_config for AutoScheme —
- # gguf_args_check may have set __dict__ entries that shadow the proxy.
- if self.is_auto_scheme:
- for _attr in _gguf_forwarded_attrs:
- if _attr in self.__dict__:
- del self.__dict__[_attr]
-
- if not self.is_auto_scheme:
- _any_gguf_attr_changed = False
- for _attr in _gguf_forwarded_attrs:
- if _attr not in self.__dict__:
- continue
- config_val = getattr(self.quantize_config, _attr, None)
- self_val = self.__dict__[_attr]
- if _attr not in ("scale_dtype", "act_bits") and config_val != self_val:
- _any_gguf_attr_changed = True
- if config_val != self_val:
- setattr(self.quantize_config, _attr, self_val)
- # If format resolution changed scheme attrs, rebuild self.scheme so that
- # configure_layer_config() / set_layer_config() see the correct values.
- if _any_gguf_attr_changed:
- from auto_round.schemes import PRESET_SCHEMES
- from auto_round.schemes import QuantizationScheme as _QS
-
- # Prefer to derive the scheme directly from the gguf format name to
- # avoid ambiguity (e.g. Q4_K_S and Q4_K_M share identical weight attrs).
- _gguf_preset_scheme = None
- _gguf_fmt_name = None
- _gguf_original_fmt_name = None
- for _fmt in self.formats or []:
- # GGUFFormat (outer) has output_format="gguf" but backend.output_format="gguf:q4_k_m"
- # GGUFFormat (inner/standalone) has output_format="gguf:q4_k_m"
- _of = getattr(_fmt, "output_format", "")
- if "gguf" in str(_of):
- if str(_of) == "gguf":
- # outer GGUFFormat: full format in _original_format (e.g. "gguf:q2_k_mixed")
- # or backend.output_format (e.g. "gguf:q2_k_s" after _mixed → _s conversion)
- _orig = getattr(_fmt, "_original_format", None)
- if _orig:
- _gguf_original_fmt_name = str(_orig).upper()
- _backend = getattr(_fmt, "backend", None)
- _of = getattr(_backend, "output_format", _of) if _backend is not None else _of
- _preset_key = str(_of).upper()
- if _preset_key in PRESET_SCHEMES:
- _gguf_preset_scheme = PRESET_SCHEMES[_preset_key]
- _gguf_fmt_name = _preset_key
- break
- if _gguf_preset_scheme is not None:
- # Update scheme on both compressor and quantizer.
- self.scheme = _gguf_preset_scheme
- # Store the exact gguf format name so configure_layer_config /
- # set_layer_config can use it directly, avoiding Q4_K_S / Q4_K_M ambiguity.
- self._gguf_format_name = _gguf_fmt_name
- # Store original format name (may include _mixed) for _handle_special_schemes
- if _gguf_original_fmt_name:
- self._gguf_original_format_name = _gguf_original_fmt_name
- else:
- _new_scheme_dict = {f.name: getattr(self, f.name, None) for f in fields(_QS)}
- _new_scheme = _QS.from_dict({k: v for k, v in _new_scheme_dict.items() if v is not None})
- self.scheme = _new_scheme
-
- _gguf_layer_cfg = {
- k: v for k, v in (self.__dict__.get("layer_config") or {}).items() if k not in (_pre_gguf_layer_config)
- }
- if _gguf_layer_cfg:
- if self.layer_config is None:
- self.layer_config = {}
- for _lname, _lval in _gguf_layer_cfg.items():
- self.layer_config.setdefault(_lname, _lval)
-
def _apply_rotations(self) -> None:
"""Phase 4.5 – Apply Hadamard / rotation transforms to the model.
@@ -1189,7 +1377,7 @@ class BaseOrchestrator(object):
self.model_context.apply_patches(self.formats)
def _build_layer_config(self) -> None:
- """Phase 4 – Layer-config construction and quantizer sync.
+ """Phase 4 – Layer-config construction.
Preconditions:
- Phase 3 complete: model topology is final.
@@ -1198,17 +1386,12 @@ class BaseOrchestrator(object):
Work performed:
- Calls :meth:`_scheme_post_init` which walks the patched model to
- build ``self.layer_config``, ``self.quant_block_list``, etc.
- On the AutoScheme path this also runs delta-loss forward/backward
- passes to select per-layer schemes.
- - Syncs the fully-resolved ``layer_config`` and related attrs to
- ``self.quantizer`` so quantization methods have the complete view.
+ build ``self.layer_config``, ``self.quant_block_list``,
+ ``self.compression_plan``, etc. On the AutoScheme path this also
+ runs delta-loss forward/backward passes to select per-layer schemes.
Postconditions:
- - ``self.layer_config`` is fully populated.
- - ``self.quantizer`` mirrors ``layer_config``, ``has_qlayer_outside_block``,
- ``regex_config``, ``quant_block_list``, ``to_quant_block_names``,
- ``scale_dtype``, and ``ignore_layers``.
+ - ``self.layer_config`` and ``self.compression_plan`` are fully populated.
"""
# configure_layer_config() walks the patched model; _gen_auto_scheme()
# (AutoScheme path) runs delta-loss forward+backward passes.
@@ -1273,14 +1456,14 @@ class BaseOrchestrator(object):
if name.startswith("_"):
raise AttributeError(f"'{type(self).__name__}' object has no attribute '{name}'")
- # Delegate to block_quantizer: access _pipeline directly from __dict__ to
- # avoid recursion (quantizer is now a @property backed by _pipeline; going
- # through the property inside __getattr__ would re-trigger __getattr__
- # if _pipeline itself isn't ready yet).
- _pipeline = self.__dict__.get("_pipeline")
- if _pipeline is not None:
+ # Delegate to block_quantizer: access _alg_composer directly from __dict__
+ # to avoid recursion (quantizer attribute access forwards through
+ # _alg_composer.block_quantizer; going through a property inside
+ # __getattr__ would re-trigger __getattr__ if _alg_composer isn't ready yet).
+ _alg_composer = self.__dict__.get("_alg_composer")
+ if _alg_composer is not None:
try:
- return object.__getattribute__(_pipeline.block_quantizer, name)
+ return object.__getattribute__(_alg_composer.block_quantizer, name)
except AttributeError:
pass
@@ -1449,7 +1632,7 @@ class BaseOrchestrator(object):
self.compress_context.output_dir = output_dir
if format is not None:
if isinstance(format, str) and getattr(self, "formats", None) is None:
- self.formats = get_formats(format, self)
+ self.formats = self._resolve_format_string(format)
self.compress_context.formats = self.formats
if not self.model_context.quantized:
@@ -1460,7 +1643,7 @@ class BaseOrchestrator(object):
logger.info("format is not set, using default auto_round format.")
self.formats = "auto_round"
if isinstance(self.formats, str):
- self.formats = get_formats(self.formats, self)
+ self.formats = self._resolve_format_string(self.formats)
self.compress_context.formats = self.formats
for format in self.formats:
save_folder = _get_save_folder_name(format)
@@ -1633,7 +1816,7 @@ class BaseOrchestrator(object):
# self.formats to a default string above, resolve it into OutputFormat objects so that
# quantize() and save_quantized() receive proper objects, not a raw string.
if isinstance(self.formats, str):
- self.formats = get_formats(self.formats, self)
+ self.formats = self._resolve_format_string(self.formats)
self.compress_context.formats = self.formats
# Derive descriptive export dir after post_init so scheme-resolved attrs are available.
_fmt_str = format or (self.formats if isinstance(self.formats, str) else "")
diff --git a/auto_round/compressors/config_resolution/__init__.py b/auto_round/compressors/config_resolution/__init__.py
new file mode 100644
index 000000000..dcdbb3b74
--- /dev/null
+++ b/auto_round/compressors/config_resolution/__init__.py
@@ -0,0 +1,40 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from auto_round.compressors.config_resolution.contracts import (
+ ResolvedQuantizationConfig,
+ FormatResolution,
+ ResolvedScheme,
+ thaw_mapping,
+)
+from auto_round.compressors.config_resolution.errors import (
+ ConfigResolutionError,
+ FormatCompatibilityError,
+ LayerConfigResolutionError,
+ SchemeResolutionError,
+)
+from auto_round.compressors.config_resolution.resolve import resolve_quantization_config, resolve_scheme_value
+
+__all__ = [
+ "ResolvedQuantizationConfig",
+ "FormatCompatibilityError",
+ "FormatResolution",
+ "LayerConfigResolutionError",
+ "ConfigResolutionError",
+ "ResolvedScheme",
+ "SchemeResolutionError",
+ "resolve_quantization_config",
+ "resolve_scheme_value",
+ "thaw_mapping",
+]
diff --git a/auto_round/compressors/config_resolution/contracts.py b/auto_round/compressors/config_resolution/contracts.py
new file mode 100644
index 000000000..f095dc936
--- /dev/null
+++ b/auto_round/compressors/config_resolution/contracts.py
@@ -0,0 +1,127 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+import copy
+from dataclasses import dataclass, field
+from types import MappingProxyType
+from typing import Any, Mapping, Optional, Tuple
+
+from auto_round.schemes import QuantizationScheme
+
+LayerConfig = Mapping[str, Mapping[str, Any]]
+BlockGroups = Tuple[Tuple[str, ...], ...]
+
+
+def _deepcopy_mapping_proxy(value: MappingProxyType, memo: dict) -> MappingProxyType:
+ """``copy.deepcopy`` has no built-in support for ``types.MappingProxyType`` (it isn't
+ picklable), so any object graph that happens to contain one -- e.g. a
+ ``ResolvedQuantizationConfig``/``FormatResolution`` nested inside something a caller deep-copies
+ wholesale -- would otherwise raise ``TypeError: cannot pickle 'mappingproxy' object``.
+ Registering a dispatch entry here teaches ``copy.deepcopy`` to handle it everywhere,
+ instead of requiring every call site that might transitively touch one of our frozen
+ mappings to know about this quirk.
+ """
+ return MappingProxyType(copy.deepcopy(dict(value), memo))
+
+
+copy._deepcopy_dispatch[MappingProxyType] = _deepcopy_mapping_proxy
+
+
+def freeze_mapping(value: Optional[LayerConfig]) -> LayerConfig:
+ """Return an isolated, read-only snapshot of a layer configuration mapping.
+
+ Per-layer configuration values are usually dicts (e.g. ``{"bits": 4}``), but some
+ callers store non-mapping values (e.g. a GGUF preset name string). Only dict-like
+ values are deep-copied and wrapped in a read-only proxy; other values are copied
+ as-is since they are already immutable/opaque to us.
+ """
+ frozen = {}
+ for name, config in dict(value or {}).items():
+ if isinstance(config, Mapping):
+ frozen[name] = MappingProxyType(copy.deepcopy(dict(config)))
+ else:
+ frozen[name] = copy.deepcopy(config)
+ return MappingProxyType(frozen)
+
+
+def thaw_mapping(value: Optional[LayerConfig]) -> dict:
+ """Return a fully mutable, deep-copyable plain-dict snapshot of a frozen mapping.
+
+ This is the inverse of :func:`freeze_mapping` and should be used instead of
+ ``copy.deepcopy`` whenever a caller needs to mutate a previously frozen layer
+ configuration mapping, since ``MappingProxyType`` instances cannot be deep-copied
+ directly.
+ """
+ result = {}
+ for name, config in dict(value or {}).items():
+ result[name] = copy.deepcopy(dict(config)) if isinstance(config, Mapping) else copy.deepcopy(config)
+ return result
+
+
+def freeze_block_groups(value: Optional[Tuple[Tuple[str, ...], ...]]) -> Optional[BlockGroups]:
+ """Freeze the two-dimensional block grouping used by model traversal."""
+ if value is None:
+ return None
+ return tuple(tuple(group) for group in value)
+
+
+@dataclass(frozen=True)
+class ResolvedScheme:
+ _value: QuantizationScheme
+ preset_name: Optional[str] = None
+
+ def __post_init__(self) -> None:
+ object.__setattr__(self, "_value", copy.deepcopy(self._value))
+
+ @property
+ def value(self) -> QuantizationScheme:
+ """Return an isolated scheme value so callers cannot mutate this DTO."""
+ return copy.deepcopy(self._value)
+
+ @classmethod
+ def from_scheme(cls, value: QuantizationScheme, preset_name: Optional[str] = None) -> "ResolvedScheme":
+ return cls(_value=value, preset_name=preset_name)
+
+
+@dataclass(frozen=True)
+class FormatResolution:
+ formats: Tuple[Any, ...]
+ scheme: ResolvedScheme
+ layer_config_patch: LayerConfig = field(default_factory=lambda: MappingProxyType({}))
+ scale_dtype: Any = None
+ quant_block_list: Optional[BlockGroups] = None
+
+ def __post_init__(self) -> None:
+ object.__setattr__(self, "formats", tuple(self.formats))
+ object.__setattr__(self, "layer_config_patch", freeze_mapping(self.layer_config_patch))
+ object.__setattr__(self, "quant_block_list", freeze_block_groups(self.quant_block_list))
+
+
+@dataclass(frozen=True)
+class ResolvedQuantizationConfig:
+ scheme: ResolvedScheme
+ formats: Tuple[Any, ...]
+ layer_config: LayerConfig
+ regex_config: LayerConfig = field(default_factory=lambda: MappingProxyType({}))
+ has_qlayer_outside_block: bool = False
+ scale_dtype: Any = None
+ quant_block_list: Optional[BlockGroups] = None
+
+ def __post_init__(self) -> None:
+ object.__setattr__(self, "formats", tuple(self.formats))
+ object.__setattr__(self, "layer_config", freeze_mapping(self.layer_config))
+ object.__setattr__(self, "regex_config", freeze_mapping(self.regex_config))
+ object.__setattr__(self, "quant_block_list", freeze_block_groups(self.quant_block_list))
diff --git a/auto_round/compressors/config_resolution/errors.py b/auto_round/compressors/config_resolution/errors.py
new file mode 100644
index 000000000..438e2e582
--- /dev/null
+++ b/auto_round/compressors/config_resolution/errors.py
@@ -0,0 +1,33 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+class ConfigResolutionError(ValueError):
+ """Base class for user-facing compression configuration resolution errors.
+
+ Callers that want to surface a clean, no-traceback message (e.g. the CLI) should
+ catch this common base instead of enumerating every specific subclass.
+ """
+
+
+class SchemeResolutionError(ConfigResolutionError):
+ """Raised when a quantization scheme cannot be resolved."""
+
+
+class FormatCompatibilityError(ConfigResolutionError):
+ """Raised when requested export formats are mutually incompatible."""
+
+
+class LayerConfigResolutionError(ConfigResolutionError):
+ """Raised when per-layer quantization configuration cannot be resolved."""
diff --git a/auto_round/compressors/config_resolution/resolve.py b/auto_round/compressors/config_resolution/resolve.py
new file mode 100644
index 000000000..4d501ae0c
--- /dev/null
+++ b/auto_round/compressors/config_resolution/resolve.py
@@ -0,0 +1,69 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+import copy
+from typing import Any, Mapping
+
+from auto_round.compressors.config_resolution.contracts import (
+ FormatResolution,
+ LayerConfig,
+ ResolvedQuantizationConfig,
+ ResolvedScheme,
+ thaw_mapping,
+)
+from auto_round.compressors.config_resolution.errors import SchemeResolutionError
+from auto_round.schemes import QuantizationScheme, parse_scheme
+
+
+def resolve_scheme_value(scheme: Any, overrides: Mapping[str, Any]) -> ResolvedScheme:
+ """Resolve a model-independent scheme without modifying caller-owned inputs."""
+ from auto_round.auto_scheme.gen_auto_scheme import AutoScheme
+
+ if isinstance(scheme, AutoScheme):
+ raise SchemeResolutionError("AutoScheme requires model-aware resolution")
+
+ try:
+ _, _, final_attrs = parse_scheme(copy.deepcopy(scheme), copy.deepcopy(dict(overrides)))
+ except (KeyError, NotImplementedError, TypeError, ValueError) as error:
+ raise SchemeResolutionError(str(error)) from error
+
+ preset_name = None
+ if isinstance(scheme, str):
+ normalized = scheme.strip("'\" ")
+ preset_name = normalized.lower() if normalized.lower().startswith("gguf:") else normalized.upper()
+ return ResolvedScheme.from_scheme(QuantizationScheme.from_dict(final_attrs), preset_name=preset_name)
+
+
+def resolve_quantization_config(
+ format_resolution: FormatResolution,
+ layer_config: LayerConfig,
+ *,
+ regex_config: LayerConfig | None = None,
+ has_qlayer_outside_block: bool = False,
+) -> ResolvedQuantizationConfig:
+ """Combine immutable resolution results into the authoritative quantization config."""
+ merged = thaw_mapping(format_resolution.layer_config_patch)
+ for name, config in layer_config.items():
+ merged.setdefault(name, {}).update(dict(config))
+ return ResolvedQuantizationConfig(
+ scheme=format_resolution.scheme,
+ formats=format_resolution.formats,
+ layer_config=merged,
+ regex_config=regex_config or {},
+ has_qlayer_outside_block=has_qlayer_outside_block,
+ scale_dtype=format_resolution.scale_dtype,
+ quant_block_list=format_resolution.quant_block_list,
+ )
diff --git a/auto_round/compressors/diffusion_mixin.py b/auto_round/compressors/diffusion_mixin.py
index 1b9e53c66..7b22117e7 100644
--- a/auto_round/compressors/diffusion_mixin.py
+++ b/auto_round/compressors/diffusion_mixin.py
@@ -519,9 +519,7 @@ class DiffusionMixin:
# Handle multi-format (convert string to list if needed)
_format = format
if isinstance(_format, str):
- from auto_round.formats import get_formats
-
- _format = get_formats(_format, self)
+ _format = self._resolve_format_string(_format)
for name in pipe.components.keys():
val = getattr(pipe, name)
diff --git a/auto_round/compressors/entry.py b/auto_round/compressors/entry.py
deleted file mode 100644
index 5c838be06..000000000
--- a/auto_round/compressors/entry.py
+++ /dev/null
@@ -1,789 +0,0 @@
-# # Copyright (C) 2026 Intel Corporation
-# # SPDX-License-Identifier: Apache-2.0
-
-import os
-from typing import Any, Callable, Optional, Union
-
-import torch
-
-from auto_round.algorithms.config_resolver import split_quantization_configs
-from auto_round.algorithms.quantization.config import QuantizationConfig
-from auto_round.algorithms.quantization.rtn.config import OptimizedRTNConfig, RTNConfig
-from auto_round.algorithms.quantization.sign_round.config import SignRoundConfig
-from auto_round.algorithms.registry import normalize_algorithm_config, resolve_alg_config
-from auto_round.algorithms.transforms import normalize_rotation_config as _normalize_rotation_alg_config
-from auto_round.algorithms.transforms.awq.config import AWQConfig
-from auto_round.algorithms.transforms.hadamard.config import RotationConfig as _NewArchRotationConfig
-from auto_round.auto_scheme.gen_auto_scheme import AutoScheme
-from auto_round.compressors.base import BaseOrchestrator as BaseCompressor
-from auto_round.compressors.orchestrator import CompressionOrchestrator as Compressor
-from auto_round.compressors.utils import check_need_act_calibration
-from auto_round.logger import logger
-from auto_round.schemes import QuantizationScheme, parse_scheme
-from auto_round.utils.device_manager import normalize_default_device_map
-
-_ENTRY_ROUTE_KWARGS = {"model_free", "disable_model_free", "disable_opt_rtn"}
-_ENTRY_COMPRESSOR_KWARGS = {"scale_dtype", "ignore_layers", "quant_lm_head", "to_quant_block_names"}
-_ENTRY_BASE_KWARGS = {
- "format",
- "dataset",
- "batch_size",
- "model_dtype",
- "trust_remote_code",
- "amp",
- "nblocks",
- "disable_deterministic_algorithms",
- "enable_deterministic_algorithms",
- "static_kv_dtype",
- "static_attention_dtype",
-}
-_ENTRY_MLLM_KWARGS = {"processor", "image_processor", "template", "extra_data_dir", "quant_nontext_module"}
-_ENTRY_DIFFUSION_KWARGS = {"guidance_scale", "num_inference_steps", "generator_seed"}
-_ENTRY_ALLOWED_KWARGS = (
- _ENTRY_ROUTE_KWARGS | _ENTRY_COMPRESSOR_KWARGS | _ENTRY_BASE_KWARGS | _ENTRY_MLLM_KWARGS | _ENTRY_DIFFUSION_KWARGS
-)
-
-
-def filter_supported_entry_kwargs(kwargs: dict[str, Any], *, context: str) -> dict[str, Any]:
- """Return only kwargs supported by the new entry API.
-
- Unsupported kwargs are ignored with a warning so callers can cleanly migrate
- without leaking old-API parameters into compressor constructors.
- """
-
- supported = {}
- unknown = []
- for key, value in kwargs.items():
- if key in _ENTRY_ALLOWED_KWARGS:
- supported[key] = value
- else:
- unknown.append(key)
- if unknown:
- logger.warning_once(
- "%s received unsupported kwargs %s. They will be ignored.",
- context,
- ", ".join(sorted(unknown)),
- )
- return supported
-
-
-def _split_entry_kwargs(kwargs: dict[str, Any]) -> dict[str, dict[str, Any]]:
- """Partition new-entry kwargs by ownership."""
-
- kwargs = filter_supported_entry_kwargs(kwargs, context="AutoRound entry")
- buckets = {
- "route": {},
- "compressor": {},
- "base": {},
- "mllm": {},
- "diffusion": {},
- }
- for key, value in kwargs.items():
- if key in _ENTRY_ROUTE_KWARGS:
- buckets["route"][key] = value
- elif key in _ENTRY_COMPRESSOR_KWARGS:
- buckets["compressor"][key] = value
- elif key in _ENTRY_BASE_KWARGS:
- buckets["base"][key] = value
- elif key in _ENTRY_MLLM_KWARGS:
- buckets["mllm"][key] = value
- elif key in _ENTRY_DIFFUSION_KWARGS:
- buckets["diffusion"][key] = value
- return buckets
-
-
-def _preview_resolved_attrs(config, scheme=None) -> dict:
- """Resolve scheme attributes without mutating config, for routing decisions.
-
- Called in ``AutoRound.__new__`` before the concrete compressor class is
- chosen. ``SchemeMixin.resolve_scheme()`` will do the authoritative
- resolution later; this is just a lightweight preview so routing logic
- (``enable_imatrix``, ``needs_act_calib``, etc.) can use the correct values
- even when the user specified only ``scheme=`` without explicit bit/dtype args.
-
- Returns:
- dict: resolved attributes (may be empty if scheme cannot be previewed).
- """
- if isinstance(scheme, AutoScheme):
- # AutoScheme needs model info — cannot preview, rely on raw config attrs
- return {}
- scheme_attr_names = tuple(config._scheme_fields)
- user_overrides = {k: getattr(config, k) for k in scheme_attr_names if getattr(config, k, None) is not None}
- try:
- _, _, final_attrs = parse_scheme(scheme, user_overrides)
- return final_attrs
- except Exception:
- return {}
-
-
-def _eager_validate_scheme(config, scheme=None) -> None:
- """Eagerly validate scheme/config constraints at construction time.
-
- Mirrors the old-arch ``_check_configs()`` call in ``BaseCompressor.__init__``.
- Raises ``ValueError`` or ``NotImplementedError`` immediately if the scheme
- contains config-only invalid combinations (e.g. tuple group_size with non-fp8
- weight dtype) so that callers get a fast failure rather than a deferred error
- buried inside ``post_init()``.
-
- ``AutoScheme`` is skipped because it requires model information.
- """
- if isinstance(scheme, AutoScheme):
- return
-
- scheme_attr_names = tuple(config._scheme_fields)
- user_overrides = {k: getattr(config, k) for k in scheme_attr_names if getattr(config, k, None) is not None}
- try:
- _, _, final_attrs = parse_scheme(scheme, user_overrides)
- except (ValueError, NotImplementedError):
- raise
- except Exception:
- return # Other parse errors are deferred to post_init
-
- import copy
-
- temp_config = copy.copy(config)
- if hasattr(config, "scheme"):
- temp_config.scheme = config.scheme.copy()
- temp_config._user_set_scheme_fields = set(getattr(config, "_user_set_scheme_fields", set()))
- for key, value in final_attrs.items():
- setattr(temp_config, key, value)
- temp_config.check_config() # raises ValueError / NotImplementedError if invalid
-
-
-# ---------------------------------------------------------------------------
-# Compressor-class registry
-# ---------------------------------------------------------------------------
-# Maps (model_type, base_class_name) → combined class, created lazily.
-_COMPRESSOR_REGISTRY: dict[tuple[str, str], type] = {}
-
-
-def _get_compressor_class(model_type: str, base_cls: type) -> type:
- """Return the compressor class for *base_cls* wired with the right model-type Mixin.
-
- For ``model_type == "llm"`` the bare *base_cls* is returned unchanged.
- For ``"mllm"`` and ``"diffusion"`` the corresponding Mixin is prepended via
- :func:`type` and the result is cached in ``_COMPRESSOR_REGISTRY`` so that
- each ``(model_type, base_cls)`` pair is created at most once per process.
- """
- if model_type == "llm":
- return base_cls
- key = (model_type, base_cls.__name__)
- if key in _COMPRESSOR_REGISTRY:
- return _COMPRESSOR_REGISTRY[key]
- if model_type == "mllm":
- from auto_round.compressors.mllm_mixin import MLLMMixin
-
- mixin = MLLMMixin
- elif model_type == "diffusion":
- from auto_round.compressors.diffusion_mixin import DiffusionMixin
-
- mixin = DiffusionMixin
- else:
- return base_cls
- combined = type(f"{model_type.capitalize()}{base_cls.__name__}", (mixin, base_cls), {})
- _COMPRESSOR_REGISTRY[key] = combined
- return combined
-
-
-def is_weight_scheme(scheme: Union[str, dict, AutoScheme, object]) -> bool:
- if isinstance(scheme, str):
- return scheme.upper().startswith("W")
- if isinstance(scheme, dict):
- return all(isinstance(s, str) and s.upper().startswith("W") for s in scheme.values())
- if isinstance(scheme, AutoScheme):
- opts = scheme.options
- if isinstance(opts, (list, tuple)):
- return all(isinstance(s, str) and s.upper().startswith("W") for s in opts)
- if isinstance(opts, str):
- return opts.upper().startswith("W")
- return False
-
-
-def is_gguf_k_target(value: Union[str, AutoScheme, object]) -> bool:
- if isinstance(value, str):
- normalized = value.strip().lower()
- return normalized.startswith("gguf:") and "_k" in normalized
- if isinstance(value, AutoScheme):
- opts = value.options
- if isinstance(opts, str):
- opts = [opts]
- if isinstance(opts, (list, tuple)):
- return any(isinstance(opt, str) and is_gguf_k_target(opt) for opt in opts)
- return False
-
-
-def _resolve_quant_config_for_routing(alg_configs) -> tuple[list, list, QuantizationConfig]:
- preprocessor_configs, block_quant_configs = split_quantization_configs(alg_configs)
- if len(block_quant_configs) == 0 and preprocessor_configs:
- from auto_round.algorithms.quantization.rtn.config import RTNConfig as _RTNConfig
-
- return preprocessor_configs, block_quant_configs, _RTNConfig()
- if len(block_quant_configs) > 1:
- raise ValueError(
- f"Only one block-quantization config is allowed, but got {len(block_quant_configs)}: "
- f"{[type(c).__name__ for c in block_quant_configs]}"
- )
- if len(block_quant_configs) == 1:
- return preprocessor_configs, block_quant_configs, block_quant_configs[0]
- raise ValueError(
- "At least one quantization algorithm config is required. "
- "Pass a block quantizer such as RTNConfig or SignRoundConfig, "
- "or a quantization preprocessor such as AWQConfig."
- )
-
-
-def _build_model_type_ctor_kwargs(model, base_kwargs, mllm_kwargs, diffusion_kwargs) -> tuple[str, dict[str, Any]]:
- from auto_round.utils.model import detect_model_type
-
- model_type = detect_model_type(model)
- has_multimodal_assets = mllm_kwargs.get("processor") is not None or mllm_kwargs.get("image_processor") is not None
- if has_multimodal_assets and model_type != "mllm":
- model_type = "mllm"
-
- ctor_kwargs = dict(base_kwargs)
- if model_type == "mllm":
- ctor_kwargs.update(mllm_kwargs)
- if model_type == "diffusion":
- ctor_kwargs.update(diffusion_kwargs)
- return model_type, ctor_kwargs
-
-
-def _select_rtn_compressor_base_cls(quant_config: RTNConfig, scheme, format, base_kwargs) -> type:
- enable_imatrix = False
- disable_opt_rtn = getattr(quant_config, "disable_opt_rtn", False)
-
- # Preview resolved scheme attrs once (authoritative resolution happens later).
- resolved_attrs = _preview_resolved_attrs(quant_config, scheme)
-
- # Auto-disable rtn optimization for W8A16/W8A8-equivalent resolved schemes,
- # unless the user already set disable_opt_rtn explicitly.
- if getattr(quant_config, "orig_disable_opt_rtn", None) is None:
- bits = resolved_attrs.get("bits", getattr(quant_config, "bits", None))
- act_bits = resolved_attrs.get("act_bits", getattr(quant_config, "act_bits", None))
- data_type = resolved_attrs.get("data_type", getattr(quant_config, "data_type", None))
- if bits is not None and bits >= 8 and act_bits is not None and act_bits >= 8 and data_type == "int":
- logger.warning("`disable_opt_rtn` is turned on for W8A16/W8A8 quantization to improve efficiency.")
- disable_opt_rtn = True
- quant_config.disable_opt_rtn = True
-
- if not disable_opt_rtn:
- has_gguf_k = is_gguf_k_target(format) or is_gguf_k_target(scheme)
- if has_gguf_k:
- enable_imatrix = True
- else:
- sym = resolved_attrs.get("sym", getattr(quant_config, "sym", None))
- data_type = resolved_attrs.get("data_type", getattr(quant_config, "data_type", "") or "")
- bits = resolved_attrs.get("bits", getattr(quant_config, "bits", None))
- if sym is not None and sym is False:
- enable_imatrix = False
- elif data_type == "int" and (bits is None or bits < 8):
- enable_imatrix = True
- elif is_weight_scheme(scheme):
- enable_imatrix = True
-
- act_bits = resolved_attrs.get("act_bits", getattr(quant_config, "act_bits", None))
- act_data_type = resolved_attrs.get("act_data_type", getattr(quant_config, "act_data_type", None))
- act_dynamic = resolved_attrs.get("act_dynamic", getattr(quant_config, "act_dynamic", None))
- is_act_quantize = act_bits is not None and act_bits <= 8
- needs_act_calib = is_act_quantize and check_need_act_calibration(
- act_dynamic,
- act_data_type,
- act_bits if act_bits is not None else 16,
- static_kv_dtype=base_kwargs.get("static_kv_dtype"),
- static_attention_dtype=base_kwargs.get("static_attention_dtype"),
- )
-
- # AutoScheme always requires calibration data for delta-loss based scheme
- # selection, regardless of whether imatrix is needed.
- quant_config.enable_imatrix = enable_imatrix
- if enable_imatrix or needs_act_calib or isinstance(scheme, AutoScheme):
- if not isinstance(quant_config, OptimizedRTNConfig):
- quant_config.__class__ = OptimizedRTNConfig
-
- if isinstance(quant_config, OptimizedRTNConfig):
- pass # keep OptimizedRTNConfig as-is
- elif not (enable_imatrix or needs_act_calib or isinstance(scheme, AutoScheme)):
- # Pure zero-shot RTN: downgrade to basic RTNConfig
- if isinstance(quant_config, OptimizedRTNConfig):
- quant_config.__class__ = RTNConfig
-
- # Always use Compressor — it internally detects whether calibration
- # data is needed and falls back to the zero-shot (RTN) path when it is not.
- return Compressor
-
-
-class AutoRound(object):
-
- @classmethod
- def _resolve_config(cls, config: Union[str, object, list]) -> Union[object, list[object]]:
- """Convert string alias(es) to the corresponding config instance(s) with default parameters."""
- if isinstance(config, str):
- return resolve_alg_config(config)
- if isinstance(config, list):
- return [cls._resolve_config(c) for c in config]
- return config
-
- def __new__(
- cls,
- model: Union[torch.nn.Module, str],
- scheme="W4A16",
- alg_configs: Union[str, object, list[Union[str, object]]] = None,
- tokenizer=None,
- platform="hf",
- format=None,
- dataset=None,
- low_gpu_mem_usage: bool = False,
- device_map: Union[str, torch.device, int, dict] = 0,
- iters: int = None,
- enable_torch_compile: Optional[bool] = None,
- seed: int = 42,
- low_cpu_mem_usage: bool = True,
- layer_config=None,
- nsamples: int = None,
- seqlen: int = None,
- **kwargs,
- ) -> "BaseCompressor":
- from auto_round.utils.model import is_model_free_route
-
- if alg_configs is None:
- alg_configs = "auto_round"
- # TODO wenhuach if key in kwargs could override scheme and alg_config, we should pop and override,
- # e.g. gradient_accumulate_step
- device_map = normalize_default_device_map(device_map)
- split_kwargs = _split_entry_kwargs(kwargs)
- route_kwargs = dict(split_kwargs["route"])
- compressor_kwargs = dict(split_kwargs["compressor"])
- base_kwargs = dict(split_kwargs["base"])
- mllm_kwargs = dict(split_kwargs["mllm"])
- diffusion_kwargs = dict(split_kwargs["diffusion"])
-
- # Resolve string alias(es) to config instance(s) before routing.
- alg_configs = cls._resolve_config(alg_configs)
- if isinstance(alg_configs, list):
- alg_configs = [normalize_algorithm_config(cfg) for cfg in alg_configs]
- else:
- alg_configs = normalize_algorithm_config(alg_configs)
- configs_for_routing = alg_configs if isinstance(alg_configs, list) else [alg_configs]
- preprocessor_configs, _, quant_config = _resolve_quant_config_for_routing(configs_for_routing)
-
- # Model-free routing is now supported directly by the new entry path.
- model_free_iters = 0 if isinstance(quant_config, RTNConfig) else getattr(quant_config, "iters", None)
- model_free_disable_opt_rtn = getattr(quant_config, "disable_opt_rtn", None)
- route_decision_kwargs = dict(
- route_kwargs,
- format=format,
- static_kv_dtype=base_kwargs.get("static_kv_dtype"),
- static_attention_dtype=base_kwargs.get("static_attention_dtype"),
- )
- if is_model_free_route(model, scheme, model_free_iters, model_free_disable_opt_rtn, route_decision_kwargs):
- from auto_round.compressors.model_free import ModelFreeCompressor
-
- if not isinstance(model, str):
- raise ValueError("model_free=True requires `model` to be a HuggingFace ID or local path string.")
- if not bool(route_kwargs.get("model_free", False)):
- logger.info(
- "Auto-routing to model-free quantization "
- "(iters=0, disable_opt_rtn=True, supported scheme). "
- "Pass disable_model_free=True to use the regular flow."
- )
- return ModelFreeCompressor(
- model_name_or_path=model,
- scheme=scheme,
- layer_config=layer_config,
- tokenizer=tokenizer,
- device_map=device_map,
- enable_torch_compile=enable_torch_compile,
- **compressor_kwargs,
- **base_kwargs,
- **mllm_kwargs,
- **diffusion_kwargs,
- **route_kwargs,
- )
-
- # Eagerly validate scheme constraints that do not require model info.
- # This mirrors old-arch _check_configs() called at __init__ time so that
- # callers get ValueError/NotImplementedError on construction, not deferred.
- _eager_validate_scheme(quant_config, scheme)
-
- local_args = dict(
- model=model,
- tokenizer=tokenizer,
- platform=platform,
- format=format,
- scheme=scheme,
- dataset=dataset,
- low_gpu_mem_usage=low_gpu_mem_usage,
- device_map=device_map,
- iters=iters,
- enable_torch_compile=enable_torch_compile,
- seed=seed,
- low_cpu_mem_usage=low_cpu_mem_usage,
- layer_config=layer_config,
- nsamples=nsamples,
- seqlen=seqlen,
- **compressor_kwargs,
- )
- model_type, ctor_kwargs = _build_model_type_ctor_kwargs(model, base_kwargs, mllm_kwargs, diffusion_kwargs)
-
- # Preprocessor algorithms (AWQ, …) require a data-driven host so that
- # the per-block preprocessor lifecycle (prepare_block_group ->
- # block_forward_hooks -> pre_quantize_block -> pre_quantize_block ->
- # post_quantize_block) actually runs.
- # Preprocessor algorithms require Compressor for per-block lifecycle hooks.
- # The pipeline auto-appends RTN when no block_quantizer is supplied.
- if preprocessor_configs:
- return _get_compressor_class(model_type, Compressor)(alg_configs, **local_args, **ctor_kwargs)
-
- if isinstance(quant_config, SignRoundConfig):
- return _get_compressor_class(model_type, Compressor)(alg_configs, **local_args, **ctor_kwargs)
-
- elif isinstance(quant_config, RTNConfig):
- base_cls = _select_rtn_compressor_base_cls(quant_config, scheme, format, base_kwargs)
- return _get_compressor_class(model_type, base_cls)(alg_configs, **local_args, **ctor_kwargs)
-
-
-class AutoRoundCompatible:
- """AutoRoundCompatible wrapper class for backward compatibility.
-
- This class provides the same API as the old AutoRoundCompatible class but internally
- uses the new AutoRound architecture with Mixin pattern.
-
- Args:
- model: Model object or model name to load
- tokenizer: Tokenizer for text processing
- platform: Platform to download model ("hf" or "model_scope")
- scheme: Quantization scheme (str, dict, or QuantizationScheme)
- layer_config: Layer-wise quantization config
- dataset: Calibration data
- iters: Optimization iterations
- seqlen: Calibration sequence length
- nsamples: Number of calibration samples
- batch_size: Calibration batch size
- gradient_accumulate_steps: Gradient accumulation steps
- low_gpu_mem_usage: Lower GPU memory mode
- device_map: Device map for each module
- enable_torch_compile: Enable torch.compile
- seed: Random seed
- low_cpu_mem_usage: Lower CPU memory mode
- **kwargs: Additional arguments (bits, group_size, sym, etc.)
-
- Example:
- >>> # Old API - still works
- >>> from auto_round.compressors.entry import AutoRoundCompatible
- >>> autoround = AutoRoundCompatible(
- ... model="/models/opt-125m",
- ... bits=4,
- ... group_size=128,
- ... iters=200,
- ... )
- >>> quantized_model, layer_config = autoround.quantize()
- """
-
- SKIP_ARGS = ("local_args", "kwargs", "cls", "config")
-
- bits: int | None
- group_size: int | None
- sym: bool | None
- data_type: str | None
- act_bits: int | None
- act_group_size: int | None
- act_sym: bool | None
- act_data_type: str | None
- act_dynamic: bool | None
- super_bits: int | None
- super_group_size: int | None
-
- @staticmethod
- def _pop_config_kwargs(kwargs: dict[str, Any]) -> tuple[dict[str, Any], dict[str, Any]]:
- """Extract old-API config kwargs and split them by config type."""
- common_keys = ("super_bits", "super_group_size")
- auto_round_only_keys = (
- "nblocks",
- "enable_alg_ext",
- "lr_scheduler",
- "not_use_best_mse",
- "dynamic_max_gap",
- "optimizer",
- "enable_adam",
- "momentum",
- )
- common_kwargs = {}
- auto_round_kwargs = {}
- for key in common_keys:
- if key in kwargs:
- common_kwargs[key] = kwargs.pop(key)
- for key in auto_round_only_keys:
- if key in kwargs:
- auto_round_kwargs[key] = kwargs.pop(key)
- return common_kwargs, auto_round_kwargs
-
- @staticmethod
- def _pop_compressor_only_kwargs(kwargs: dict[str, Any]) -> dict[str, Any]:
- return {
- "scale_dtype": kwargs.pop("scale_dtype", None),
- "ignore_layers": kwargs.pop("ignore_layers", ""),
- "quant_lm_head": kwargs.pop("quant_lm_head", False),
- "to_quant_block_names": kwargs.pop("to_quant_block_names", None),
- }
-
- @staticmethod
- def _resolve_compat_algorithm(algorithm, iters) -> str:
- if algorithm and algorithm.lower() == "awq":
- return "awq"
- if (algorithm and algorithm.lower() == "rtn") or iters == 0:
- return "rtn"
- return "signround"
-
- @staticmethod
- def _pop_shared_quant_kwargs(kwargs: dict[str, Any]) -> dict[str, Any]:
- return {
- "bits": kwargs.pop("bits", None),
- "group_size": kwargs.pop("group_size", None),
- "sym": kwargs.pop("sym", None),
- "data_type": kwargs.pop("data_type", None),
- "act_bits": kwargs.pop("act_bits", None),
- "act_group_size": kwargs.pop("act_group_size", None),
- "act_sym": kwargs.pop("act_sym", None),
- "act_data_type": kwargs.pop("act_data_type", None),
- "act_dynamic": kwargs.pop("act_dynamic", None),
- }
-
- @staticmethod
- def _build_awq_config(
- shared_quant_kwargs: dict[str, Any],
- *,
- seqlen,
- nsamples,
- batch_size,
- kwargs,
- common_config_kwargs,
- ):
- return AWQConfig(
- **shared_quant_kwargs,
- duo_scaling=kwargs.pop("duo_scaling", True),
- n_grid=kwargs.pop("n_grid", 20),
- seqlen=seqlen,
- nsamples=nsamples,
- batch_size=batch_size,
- mappings=kwargs.pop("mappings", None),
- **common_config_kwargs,
- )
-
- @staticmethod
- def _build_rtn_config(shared_quant_kwargs: dict[str, Any], *, kwargs, common_config_kwargs):
- cfg = RTNConfig(
- **shared_quant_kwargs,
- disable_opt_rtn=kwargs.pop("disable_opt_rtn", None),
- enable_opt_rtn=kwargs.pop("enable_opt_rtn", None),
- **common_config_kwargs,
- )
- return normalize_algorithm_config(cfg)
-
- @staticmethod
- def _build_signround_config(
- shared_quant_kwargs: dict[str, Any],
- *,
- iters,
- gradient_accumulate_steps,
- kwargs,
- common_config_kwargs,
- auto_round_config_kwargs,
- ):
- cfg = SignRoundConfig(
- **shared_quant_kwargs,
- iters=iters,
- gradient_accumulate_steps=gradient_accumulate_steps,
- lr=kwargs.pop("lr", None),
- minmax_lr=kwargs.pop("minmax_lr", None),
- enable_minmax_tuning=kwargs.pop("enable_minmax_tuning", True),
- enable_norm_bias_tuning=kwargs.pop("enable_norm_bias_tuning", False),
- enable_quanted_input=kwargs.pop("enable_quanted_input", True),
- **common_config_kwargs,
- **auto_round_config_kwargs,
- )
- return normalize_algorithm_config(cfg)
-
- @classmethod
- def _build_alg_config(
- cls,
- *,
- algorithm,
- iters,
- gradient_accumulate_steps,
- seqlen,
- nsamples,
- batch_size,
- kwargs,
- common_config_kwargs,
- auto_round_config_kwargs,
- ):
- alg_name = cls._resolve_compat_algorithm(algorithm, iters)
- shared_quant_kwargs = cls._pop_shared_quant_kwargs(kwargs)
-
- if alg_name == "awq":
- return cls._build_awq_config(
- shared_quant_kwargs,
- seqlen=seqlen,
- nsamples=nsamples,
- batch_size=batch_size,
- kwargs=kwargs,
- common_config_kwargs=common_config_kwargs,
- )
- if alg_name == "rtn":
- return cls._build_rtn_config(
- shared_quant_kwargs,
- kwargs=kwargs,
- common_config_kwargs=common_config_kwargs,
- )
- return cls._build_signround_config(
- shared_quant_kwargs,
- iters=iters,
- gradient_accumulate_steps=gradient_accumulate_steps,
- kwargs=kwargs,
- common_config_kwargs=common_config_kwargs,
- auto_round_config_kwargs=auto_round_config_kwargs,
- )
-
- @staticmethod
- def _build_entry_forward_kwargs(kwargs: dict[str, Any]) -> dict[str, Any]:
- format_name = kwargs.pop("format", None)
- rotation_config = kwargs.pop("rotation_config", None)
- mllm_kwargs = {
- "processor": kwargs.pop("processor", None),
- "image_processor": kwargs.pop("image_processor", None),
- "template": kwargs.pop("template", None),
- "extra_data_dir": kwargs.pop("extra_data_dir", None),
- "quant_nontext_module": kwargs.pop("quant_nontext_module", False),
- }
- diffusion_kwargs = {
- "guidance_scale": kwargs.pop("guidance_scale", 7.5),
- "num_inference_steps": kwargs.pop("num_inference_steps", 50),
- "generator_seed": kwargs.pop("generator_seed", None),
- }
- return {
- "format": format_name,
- "rotation_config": rotation_config,
- **mllm_kwargs,
- **diffusion_kwargs,
- **kwargs,
- }
-
- def __new__(
- cls,
- model: Union[torch.nn.Module, str],
- tokenizer=None,
- platform: str = "hf",
- scheme: Union[str, dict, QuantizationScheme, AutoScheme] = "W4A16",
- layer_config: dict[str, Union[str, dict, QuantizationScheme]] = None,
- dataset: Optional[Union[str, list, tuple, torch.utils.data.DataLoader]] = None,
- iters: int = 200,
- seqlen: int = 2048,
- nsamples: int = 128,
- batch_size: int = 8,
- gradient_accumulate_steps: int = 1,
- low_gpu_mem_usage: bool = False,
- device_map: Union[str, torch.device, int, dict] = 0,
- enable_torch_compile: Optional[bool] = None,
- seed: int = 42,
- low_cpu_mem_usage: bool = True,
- algorithm: str = None,
- **kwargs,
- ) -> "BaseCompressor":
- """Create AutoRoundCompatible instance using new AutoRound architecture.
-
- This method translates old AutoRoundCompatible API to new AutoRound API.
- """
- from auto_round.utils import is_diffusion_model, is_mllm_model
- from auto_round.utils.model import is_model_free_route
-
- device = kwargs.pop("device", None)
- if device is not None:
- logger.warning_once("`device` is deprecated, please use `device_map` instead")
- if device_map in (None, 0):
- device_map = device
-
- # ---- Model-free fast-path detection --------------------------------
- if is_model_free_route(model, scheme, iters, kwargs.get("disable_opt_rtn"), kwargs):
- from auto_round.compressors.model_free import ModelFreeCompressor
-
- compressor_only_kwargs = cls._pop_compressor_only_kwargs(kwargs)
-
- if not isinstance(model, str):
- raise ValueError("model_free=True requires `model` to be a HuggingFace ID or local path string.")
- if not bool(kwargs.get("model_free", False)):
- logger.info(
- "Auto-routing to model-free quantization "
- "(iters=0, disable_opt_rtn=True, supported scheme). "
- "Pass disable_model_free=True to use the regular flow."
- )
- return ModelFreeCompressor(
- model_name_or_path=model,
- scheme=scheme,
- layer_config=layer_config,
- tokenizer=tokenizer,
- device_map=device_map,
- low_cpu_mem_usage=low_cpu_mem_usage,
- enable_torch_compile=enable_torch_compile,
- **compressor_only_kwargs,
- **kwargs,
- )
- # --------------------------------------------------------------------
-
- compressor_only_kwargs = cls._pop_compressor_only_kwargs(kwargs)
- common_config_kwargs, auto_round_config_kwargs = cls._pop_config_kwargs(kwargs)
-
- config = cls._build_alg_config(
- algorithm=algorithm,
- iters=iters,
- gradient_accumulate_steps=gradient_accumulate_steps,
- seqlen=seqlen,
- nsamples=nsamples,
- batch_size=batch_size,
- kwargs=kwargs,
- common_config_kwargs=common_config_kwargs,
- auto_round_config_kwargs=auto_round_config_kwargs,
- )
-
- forward_kwargs = cls._build_entry_forward_kwargs(kwargs)
- format_name = forward_kwargs.pop("format", None)
- _rotation_config_raw = forward_kwargs.pop("rotation_config", None)
- if _rotation_config_raw is not None:
- _rc = _normalize_rotation_alg_config(_rotation_config_raw)
- if _rc is None:
- _rc = _NewArchRotationConfig()
- config = [config, _rc]
-
- # Check model type for logging (use warning_once to avoid repeating for every block
- # when called from LLM-Compressor which instantiates AutoRound per block)
- if is_mllm_model(model, platform=platform):
- logger.info("Using MLLM mode for multimodal model.")
- elif is_diffusion_model(model):
- logger.info("Using Diffusion mode for diffusion model.")
- else:
- logger.info("Using LLM mode.")
-
- # Create AutoRound instance using new architecture
- compressor = AutoRound(
- model,
- scheme,
- config,
- tokenizer=tokenizer,
- platform=platform,
- format=format_name,
- dataset=dataset,
- iters=iters,
- gradient_accumulate_steps=gradient_accumulate_steps,
- low_gpu_mem_usage=low_gpu_mem_usage,
- device_map=device_map,
- enable_torch_compile=enable_torch_compile,
- seed=seed,
- low_cpu_mem_usage=low_cpu_mem_usage,
- layer_config=layer_config,
- nsamples=nsamples,
- seqlen=seqlen,
- batch_size=batch_size,
- **compressor_only_kwargs,
- **forward_kwargs,
- )
-
- return compressor
diff --git a/auto_round/compressors/layer_config/__init__.py b/auto_round/compressors/layer_config/__init__.py
new file mode 100644
index 000000000..5b0adaf4f
--- /dev/null
+++ b/auto_round/compressors/layer_config/__init__.py
@@ -0,0 +1,27 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from auto_round.compressors.layer_config.resolver import (
+ apply_plan_to_model,
+ extract_regex_config,
+ has_quantized_layer_outside_blocks,
+ resolve_layer_config,
+)
+
+__all__ = [
+ "apply_plan_to_model",
+ "extract_regex_config",
+ "has_quantized_layer_outside_blocks",
+ "resolve_layer_config",
+]
diff --git a/auto_round/compressors/layer_config/resolver.py b/auto_round/compressors/layer_config/resolver.py
new file mode 100644
index 000000000..60f347d2e
--- /dev/null
+++ b/auto_round/compressors/layer_config/resolver.py
@@ -0,0 +1,497 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import copy
+import json
+import os
+import re
+import sys
+from dataclasses import asdict, fields
+from typing import Union
+
+import torch
+
+from auto_round.compressors.config_resolution import ResolvedQuantizationConfig, ResolvedScheme
+from auto_round.compressors.config_resolution.contracts import LayerConfig, freeze_mapping
+from auto_round.logger import logger
+from auto_round.schemes import QuantizationScheme, get_gguf_scheme, is_mx_fp, is_nv_fp
+from auto_round.utils import (
+ INNER_SUPPORTED_LAYER_TYPES,
+ SUPPORTED_LAYER_TYPES,
+ check_to_quantized,
+ compress_layer_names,
+ get_layer_names_in_block,
+ infer_bits_by_data_type,
+ to_standard_regex,
+)
+from auto_round.utils.model import get_module
+
+
+def apply_layer_config_special_cases(
+ layer_config,
+ model,
+ default_dict,
+ supported_types,
+ inner_supported_types,
+ quant_block_list,
+ quant_lm_head,
+ gguf_name,
+) -> tuple[dict, bool, str | None, bool]:
+ """Apply lm-head, shape-divisibility, and block-membership layer rules."""
+ from auto_round.utils.model import get_lm_head_name
+
+ lm_head_name = get_lm_head_name(model)
+ tie_word_embeddings = bool(getattr(getattr(model, "config", None), "tie_word_embeddings", False))
+ if lm_head_name in layer_config:
+ quant_lm_head = True
+ if quant_lm_head and tie_word_embeddings and not gguf_name:
+ quant_lm_head = False
+ logger.warning(
+ "reset `quant_lm_head` to false as quantizing lm_head with tied weights has not been supported currently"
+ )
+ if lm_head_name not in layer_config and quant_lm_head:
+ layer_config[lm_head_name] = copy.deepcopy(default_dict)
+ if not quant_lm_head and not gguf_name:
+ layer_config.pop(lm_head_name, None)
+
+ if default_dict["data_type"] == "int" and default_dict["act_bits"] >= 16 and not gguf_name:
+ for n, m in model.named_modules():
+ if type(m) in supported_types or m.__class__.__name__ in inner_supported_types:
+ if m.weight.shape[0] % 32 or m.weight.shape[1] % 32:
+ layer_config.setdefault(n, copy.deepcopy(default_dict))
+ layer_config[n].update({"bits": 16, "data_type": "fp", "fixed_by_user": True})
+
+ if (is_nv_fp(default_dict["data_type"]) or is_mx_fp(default_dict["data_type"])) and not gguf_name:
+ skipped_layers = []
+ for n, m in model.named_modules():
+ if type(m) in supported_types or m.__class__.__name__ in inner_supported_types:
+ if m.weight.shape[1] % default_dict["group_size"]:
+ layer_config.setdefault(n, copy.deepcopy(default_dict))
+ layer_config[n].update(
+ {"bits": 16, "data_type": "fp", "act_bits": 16, "act_data_type": "fp", "fixed_by_user": True}
+ )
+ skipped_layers.append(n)
+ compressed_skipped_layers = compress_layer_names(skipped_layers)
+ if compressed_skipped_layers:
+ logger.warning_once(
+ f"some layers are skipped quantization (shape not divisible by {default_dict['group_size']}): "
+ f"{compressed_skipped_layers}"
+ )
+
+ for name in get_layer_names_in_block(model, supported_types, quant_block_list, inner_supported_types):
+ if name not in layer_config:
+ layer_config[name] = copy.deepcopy(default_dict)
+ layer_config[name]["fixed_by_user"] = False
+ layer_config[name]["in_blocks"] = True
+
+ has_qlayer_outside_block = False
+ for cfg in layer_config.values():
+ if "in_blocks" not in cfg:
+ cfg["in_blocks"] = False
+ if not cfg["in_blocks"] and check_to_quantized(cfg):
+ has_qlayer_outside_block = True
+ return layer_config, has_qlayer_outside_block, lm_head_name, tie_word_embeddings
+
+
+def _get_safetensor_layer_names_not_in_model(model, all_module_names: list) -> list:
+ """Collect layer names from safetensor files that are not loaded into the model.
+
+ Some tensors (e.g. MTP layers) exist in the original checkpoint but are not
+ instantiated by ``transformers``. This function discovers them so that regex
+ patterns in ``layer_config`` can still match them.
+
+ Returns:
+ List of layer names (the path without the ``.weight`` suffix) for weight
+ tensors present in the safetensor files but absent from *all_module_names*.
+ """
+ name_or_path = None
+ if hasattr(model, "config") and hasattr(model.config, "name_or_path"):
+ name_or_path = model.config.name_or_path
+ if not name_or_path:
+ return []
+
+ if not os.path.isdir(name_or_path):
+ # Resolution is intentionally local-only: model downloads belong to model loading,
+ # not to the pure layer-config planning phase.
+ return []
+
+ try:
+ from safetensors import safe_open
+ except ImportError:
+ return []
+
+ # Build tensor-name list from the safetensors index or single file
+ source_index_file = os.path.join(name_or_path, "model.safetensors.index.json")
+ source_single_file = os.path.join(name_or_path, "model.safetensors")
+
+ tensor_names: list = []
+ if os.path.exists(source_index_file):
+ with open(source_index_file) as f:
+ src_index = json.load(f)
+ tensor_names = list(src_index["weight_map"].keys())
+ elif os.path.exists(source_single_file):
+ with safe_open(source_single_file, framework="pt", device="cpu") as f:
+ tensor_names = list(f.keys())
+ else:
+ return []
+
+ module_name_set = set(all_module_names)
+ extra_layer_names = []
+ for tensor_name in tensor_names:
+ if not tensor_name.endswith(".weight"):
+ continue
+ layer_name = tensor_name[: -len(".weight")]
+ if layer_name not in module_name_set:
+ extra_layer_names.append(layer_name)
+ return extra_layer_names
+
+
+def _resolve_layer_config_presets(
+ layer_config, model, ignore_layers, default_scheme, default_scale_dtype, fill_default_value
+) -> tuple[dict, dict, tuple[str, ...], set[str]]:
+ """Steps 1-4 of layer-config resolution: ignore-layer parsing, normalization,
+ bits-inference, and default-filling."""
+ from auto_round.schemes import QuantizationScheme, preset_name_to_scheme
+
+ def normalize_item(item: Union[str, dict, "QuantizationScheme"], layer_name: str) -> dict:
+ """Convert config entry into dict and validate keys."""
+ if isinstance(item, str):
+ config = asdict(preset_name_to_scheme(item.upper()))
+ elif isinstance(item, QuantizationScheme):
+ config = asdict(item)
+ elif isinstance(item, dict):
+ # "in_blocks" is an internal bookkeeping key injected by LLM-Compressor;
+ # silently drop it before validation.
+ item = {k: v for k, v in item.items() if k != "in_blocks"}
+ scheme_name = item.pop("scheme", None)
+ config = asdict(preset_name_to_scheme(scheme_name.upper())) if scheme_name is not None else {}
+ invalid = set(item) - set(scheme_keys + ("fixed_by_user", "scale_dtype"))
+ if invalid:
+ raise ValueError(
+ f"Invalid keys {invalid} in layer_config for '{layer_name}'. " f"Allowed keys: {scheme_keys}"
+ )
+ config.update(item)
+ else:
+ raise TypeError(
+ f"Unsupported type for layer_config[{layer_name}]: {type(item)}. "
+ f"Expected str, dict, or QuantizationScheme."
+ )
+ # Clean up
+ config = {k: v for k, v in config.items() if v is not None}
+ config["fixed_by_user"] = True
+ return config
+
+ extra_scheme_keys = ("scale_dtype",)
+ scheme_keys = tuple(f.name for f in fields(QuantizationScheme)) + ("scale_dtype",)
+ layer_config = copy.deepcopy(layer_config) or {}
+ ignore_layer_patterns = set()
+ if ignore_layers:
+ ignore_layers = ignore_layers.replace(" ", "").split(",")
+ ignore_layers = [name + "." if name[-1].isdigit() else name for name in ignore_layers]
+ ignore_layer_patterns = set(ignore_layers)
+
+ # 1. ignore_layers -> force 16
+ for name in get_fp_layer_names(model, ignore_layers):
+ layer_config[name] = {
+ "bits": 16,
+ "act_bits": 16,
+ "data_type": "float",
+ "act_data_type": "float",
+ "fixed_by_user": True,
+ }
+
+ # 2. normalize
+ layer_config = {k: normalize_item(v, k) for k, v in layer_config.items()}
+
+ # 3. infer missing bits
+ for cfg in layer_config.values():
+ if "data_type" in cfg and "bits" not in cfg:
+ if (b := infer_bits_by_data_type(cfg["data_type"])) is not None:
+ cfg["bits"] = b
+ if "act_data_type" in cfg and "act_bits" not in cfg:
+ if (b := infer_bits_by_data_type(cfg["act_data_type"])) is not None:
+ cfg["act_bits"] = b
+
+ # 4. fill defaults
+ if isinstance(default_scheme, str):
+ default_dict = asdict(preset_name_to_scheme(default_scheme.upper()))
+ elif default_scheme is None:
+ # Defensive fallback: callers that build/patch layer configs without a resolved
+ # scheme (e.g. external integrations invoking configure_layer_config() out of the
+ # normal init sequence) would otherwise crash `asdict(None)` below.
+ default_dict = asdict(QuantizationScheme())
+ else:
+ default_dict = asdict(default_scheme)
+ default_dict["scale_dtype"] = default_scale_dtype
+
+ # In AutoScheme with mixed gguf:q4_k_m, the super_group_size of gguf:q8_0 layer is None,
+ # which should not be filled by default q4km again
+ for cfg in layer_config.values():
+ for key in scheme_keys:
+ if fill_default_value:
+ cfg.setdefault(key, copy.deepcopy(default_dict.get(key)))
+ else:
+ if key in extra_scheme_keys:
+ cfg.setdefault(key, copy.deepcopy(default_dict.get(key)))
+ else:
+ cfg.setdefault(key, None)
+
+ return layer_config, default_dict, scheme_keys, ignore_layer_patterns
+
+
+def _traverse_and_expand_layer_config(
+ layer_config, model, supported_types, inner_supported_types, ignore_layer_patterns, scheme_keys, gguf_name
+) -> tuple[dict, dict, list[str], tuple]:
+ """Steps 5-6 of layer-config resolution: supported-module collection (incl. the
+ GGUF embedding-type-detection branch) and regex-config expansion."""
+ from auto_round.utils.model import get_module
+
+ # 5. collect supported modules
+ embedding_types = (torch.nn.Embedding,)
+ if gguf_name:
+ if torch.nn.Embedding not in supported_types:
+ supported_types = (*supported_types, torch.nn.Embedding)
+
+ # for some Embedding which type() is not torch.nn.Embedding
+ # for example: transformers.models.gemma3.modeling_gemma3.Gemma3TextScaledWordEmbedding
+ model_module_name = model.__class__.__module__
+ module_cls = sys.modules[model_module_name]
+ for name in module_cls.__dict__:
+ if name.endswith("Embedding") and not name.endswith("RotaryEmbedding"):
+ embedding_types = (*embedding_types, getattr(module_cls, name))
+ supported_types = (*supported_types, *embedding_types)
+
+ all_supported_layer_names, embedding_layer_names = [], []
+ all_module_names = []
+ for n, m in model.named_modules():
+ all_module_names.append(n)
+ if type(m) not in supported_types and m.__class__.__name__ not in inner_supported_types:
+ continue
+ all_supported_layer_names.append(n)
+ if isinstance(m, embedding_types) or m.__class__.__name__.endswith("Embedding"):
+ embedding_layer_names.append(n)
+
+ # Also include layer names from safetensor files not loaded into the model
+ # (e.g. MTP layers that transformers does not instantiate).
+ safetensor_only_names = _get_safetensor_layer_names_not_in_model(model, all_module_names)
+
+ # 6. expand regex configs
+ regex_config = {}
+ for name in list(layer_config.keys()):
+ if name in all_supported_layer_names:
+ continue
+ if name in all_module_names:
+ m = get_module(model, name)
+ if len(list(m.children())) == 0 and type(m) not in supported_types:
+ val = layer_config.pop(name)
+ if name in ignore_layer_patterns:
+ # Keep unsupported ignore_layers entries so export can serialize
+ # them into regex-based extra_config for loaders like vLLM INC.
+ regex_config[name] = val
+ else:
+ logger.warning(
+ f"'{name}' exists in the model but is not a supported quantization target "
+ f"in the current scheme, ignoring its setting in `layer_config`"
+ )
+ continue
+
+ regex = re.compile(to_standard_regex(name))
+ matched = [ln for ln in all_supported_layer_names if regex.search(ln)]
+ safetensor_only_matched = [ln for ln in safetensor_only_names if regex.search(ln)]
+ # skip it for mtp layers not loaded in transformers
+ if not matched and not safetensor_only_matched:
+ # type(mlp.gate) is Qwen3VLMoeTextTopKRouter instead of Linear
+ logger.warning_once(
+ f"Layer name or regex '{name}' in layer_config does not match any supported layers. "
+ + "Please check for typos or update the regex pattern, ignore it for now"
+ )
+ val = layer_config.pop(name)
+ regex_config[name] = val # keep regex config
+ for match in matched:
+ layer_config[match] = val
+
+ return layer_config, regex_config, embedding_layer_names, supported_types
+
+
+def get_fp_layer_names(model: torch.nn.Module, ignore_layers: str):
+ """Identifies and returns layers in the model to exclude from quantization.
+
+ This function processes a comma-separated list of fully precision (FP) layers,
+ matches them to the names of layers in the model, and returns a list of such
+ layers to exclude from quantization.
+
+ Args:
+ model (torch.nn.Module): The model whose layers will be inspected.
+ ignore_layers (str): A comma-separated string of layer names to be excluded
+ from quantization. Whitespace is ignored in this string.
+
+ Returns:
+ list: A list of layer names that match the specified FP layers or are
+ subcomponents of those layers.
+ """
+ from auto_round.utils import SUPPORTED_LAYER_TYPES
+
+ if not ignore_layers:
+ return []
+
+ all_layer_names = []
+ for n, m in model.named_modules():
+ if type(m) in SUPPORTED_LAYER_TYPES:
+ all_layer_names.append(n)
+ not_to_quantized_layers = []
+
+ for fp_layer in ignore_layers:
+ if fp_layer == "":
+ continue
+ if fp_layer in all_layer_names:
+ not_to_quantized_layers.append(fp_layer)
+ continue
+ for name in all_layer_names:
+ if fp_layer in name:
+ not_to_quantized_layers.append(name)
+ not_to_quantized_layers.extend(ignore_layers) # keep regex name for later use
+ if not_to_quantized_layers:
+ logger.info(f"Ignored layers: {compress_layer_names(not_to_quantized_layers)}")
+ return not_to_quantized_layers
+
+
+def resolve_layer_config(
+ *,
+ model,
+ scheme: ResolvedScheme,
+ layer_config,
+ scale_dtype=None,
+ supported_types=None,
+ inner_supported_types=None,
+ quant_block_list=None,
+ ignore_layers: str = "",
+ quant_lm_head: bool = False,
+ enable_gguf_official_mixed: bool = True,
+ is_mllm: bool = False,
+ fill_default_value: bool = True,
+) -> LayerConfig:
+ """Resolve final per-layer configuration without writing model attributes."""
+ supported_types = tuple(SUPPORTED_LAYER_TYPES if supported_types is None else supported_types)
+ inner_supported_types = tuple(
+ INNER_SUPPORTED_LAYER_TYPES if inner_supported_types is None else inner_supported_types
+ )
+ scheme_value = scheme.value
+ gguf_name = scheme.preset_name if (scheme.preset_name or "").startswith("gguf:") else get_gguf_scheme(scheme_value)
+
+ resolved, default_dict, scheme_keys, ignore_patterns = _resolve_layer_config_presets(
+ layer_config,
+ model,
+ ignore_layers,
+ scheme_value,
+ scale_dtype,
+ fill_default_value,
+ )
+ resolved, _, embedding_names, supported_types = _traverse_and_expand_layer_config(
+ resolved,
+ model,
+ supported_types,
+ inner_supported_types,
+ ignore_patterns,
+ scheme_keys,
+ gguf_name,
+ )
+ resolved, has_outside, lm_head_name, tied = apply_layer_config_special_cases(
+ resolved,
+ model,
+ default_dict,
+ supported_types,
+ inner_supported_types,
+ quant_block_list,
+ quant_lm_head,
+ gguf_name,
+ )
+ if gguf_name:
+ from auto_round.export.formats.backends.gguf import apply_gguf_layer_defaults
+
+ resolved, _ = apply_gguf_layer_defaults(
+ dict(resolved),
+ model,
+ gguf_name,
+ lm_head_name,
+ tied,
+ embedding_names,
+ scale_dtype,
+ enable_gguf_official_mixed,
+ is_mllm,
+ has_outside,
+ )
+ return freeze_mapping(resolved)
+
+
+def extract_regex_config(
+ *,
+ model,
+ scheme: ResolvedScheme,
+ layer_config,
+ scale_dtype=None,
+ supported_types=None,
+ inner_supported_types=None,
+ ignore_layers: str = "",
+ fill_default_value: bool = True,
+) -> LayerConfig:
+ """Resolve only the regex entries retained for export metadata."""
+ supported_types = tuple(SUPPORTED_LAYER_TYPES if supported_types is None else supported_types)
+ inner_supported_types = tuple(
+ INNER_SUPPORTED_LAYER_TYPES if inner_supported_types is None else inner_supported_types
+ )
+ scheme_value = scheme.value
+ gguf_name = scheme.preset_name if (scheme.preset_name or "").startswith("gguf:") else get_gguf_scheme(scheme_value)
+ normalized, _, scheme_keys, ignore_patterns = _resolve_layer_config_presets(
+ layer_config,
+ model,
+ ignore_layers,
+ scheme_value,
+ scale_dtype,
+ fill_default_value,
+ )
+ _, regex_config, _, _ = _traverse_and_expand_layer_config(
+ normalized,
+ model,
+ supported_types,
+ inner_supported_types,
+ ignore_patterns,
+ scheme_keys,
+ gguf_name,
+ )
+ return freeze_mapping(regex_config)
+
+
+def has_quantized_layer_outside_blocks(layer_config: LayerConfig) -> bool:
+ """Derive the block-outside flag from a final layer mapping."""
+ return any(
+ not config.get("in_blocks", False) and check_to_quantized(dict(config)) for config in layer_config.values()
+ )
+
+
+def apply_plan_to_model(model, plan: ResolvedQuantizationConfig) -> None:
+ """Apply a resolved plan at the single explicit module-attribute write boundary."""
+ scheme_keys = tuple(field.name for field in fields(QuantizationScheme)) + ("scale_dtype",)
+ for module_name, module in model.named_modules():
+ for key in scheme_keys:
+ if module_name == "" and key == "rotation_config":
+ continue
+ if hasattr(module, key):
+ delattr(module, key)
+
+ for layer_name, config in plan.layer_config.items():
+ module = get_module(model, layer_name)
+ if module is None:
+ continue
+ for name, value in config.items():
+ setattr(module, name, value)
diff --git a/auto_round/compressors/model_free.py b/auto_round/compressors/model_free.py
index be94664a8..3c59ba15b 100644
--- a/auto_round/compressors/model_free.py
+++ b/auto_round/compressors/model_free.py
@@ -104,6 +104,7 @@ from typing import Any, Callable, Optional, Union
import torch
from auto_round import envs
+from auto_round.compressors.config_resolution import thaw_mapping
from auto_round.compressors.utils import is_mx_fp
from auto_round.logger import logger
from auto_round.schemes import PRESET_SCHEMES, QuantizationScheme, preset_name_to_scheme
@@ -1690,6 +1691,10 @@ def _validate_supported_scheme(
# Activation quantization for MXFP is dynamic at inference time, so the
# weight-only RTN path here is independent of act_bits.
if is_mx_fp(data_type):
+ if scheme_obj.act_data_type not in (None, "mx_fp"):
+ raise ValueError(
+ "Model-free MXFP supports only act_data_type='mx_fp', " f"but got '{scheme_obj.act_data_type}'."
+ )
# Restrict to the two explicitly supported MXFP presets when a string
# name is provided. Variants such as MXFP4_RCEIL / MXFP8_RCEIL use a
# different activation format; silently mapping them to "MXFP4" /
@@ -2020,6 +2025,9 @@ class _ModelFreeCompressorCore:
def _parse_scheme(self) -> None:
scheme_in = self.scheme_input
+ scheme_overrides = getattr(self, "user_scheme_overrides", None)
+ if scheme_overrides:
+ scheme_in = _apply_scheme_overrides(scheme_in, scheme_overrides)
if isinstance(scheme_in, str) and scheme_in.upper() == "W4A16_MIXED":
# Match regular-flow mixed recipe behavior in model-free mode:
# default non-expert linear layers use 8-bit; expert overrides are
@@ -2619,6 +2627,9 @@ class ModelFreeCompressor(_ModelFreeCompressorCore):
quant_lm_head=quant_lm_head,
low_cpu_mem_usage=low_cpu_mem_usage,
)
+ # Scheme fields are consumed above from ``kwargs``. Preserve them when
+ # a later format check falls back to the regular AutoRound flow.
+ fallback_init.update(self.user_scheme_overrides)
self._fallback_init_kwargs = fallback_init
if quant_nontext_module:
@@ -2634,7 +2645,7 @@ class ModelFreeCompressor(_ModelFreeCompressorCore):
logger.info(
"Format '%s' is not supported by model-free mode; falling back to the regular AutoRound flow.",
- format,
+ self.format,
)
logger.info(
"fallbacked_init_kwargs: %s",
@@ -2718,7 +2729,7 @@ class ModelFreeCompressor(_ModelFreeCompressorCore):
if not callable(post_init):
raise RuntimeError("AutoScheme fallback compressor has no callable post_init().")
post_init() # pylint: disable=E1102
- layer_config = copy.deepcopy(getattr(compressor, "layer_config", {}) or {})
+ layer_config = thaw_mapping(getattr(compressor, "layer_config", {}) or {})
finally:
# Release the model that was loaded only for scoring so the
# packing phase keeps model-free's low memory footprint.
diff --git a/auto_round/compressors/utils.py b/auto_round/compressors/utils.py
index d7349166f..d247e8959 100644
--- a/auto_round/compressors/utils.py
+++ b/auto_round/compressors/utils.py
@@ -11,139 +11,99 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-import copy
-import json
import os
import random
import re
-import sys
-from dataclasses import asdict, fields
-from enum import Enum
-from typing import TYPE_CHECKING, Callable, Union
+from typing import Union
import torch
-import transformers
from torch.amp import autocast
-from auto_round.export.export_to_gguf.config import GGML_QUANT_SIZES, GGUF_CONFIG, GGUF_INNER_CONFIG, QK_K, ModelType
-from auto_round.export.export_to_gguf.gguf_dtype import GGUFDTypeSelector, gguf_format_to_ftype
+from auto_round.compressors.config_resolution import LayerConfigResolutionError
from auto_round.logger import logger
+from auto_round.schemes import BackendDataType # re-exported: qlinear_fp/qlinear_int import it from here
+from auto_round.schemes import (
+ QuantizationScheme,
+ is_mx_fp,
+ is_mx_int,
+ is_nv_fp,
+ is_standard_fp,
+)
from auto_round.utils import (
check_to_quantized,
- compress_layer_names,
get_layer_names_in_block,
get_module,
- to_standard_regex,
)
from auto_round.utils.device_manager import device_manager
-if TYPE_CHECKING:
- from auto_round.schemes import QuantizationScheme
-
-
-class BackendDataType(str, Enum):
- STANDARD_FP = "fp"
- MX_FP = "mx_fp"
- NV_FP = "nv_fp"
- MX_INT = "mx_int"
-
-
-def is_standard_fp(backend):
- backend = backend.lower()
- return BackendDataType.STANDARD_FP in backend and not is_mx_fp(backend) and not is_nv_fp(backend)
-
-
-def is_mx_fp(backend):
- backend = backend.lower()
- return BackendDataType.MX_FP in backend
+def _as_scheme(ar_or_scheme) -> "QuantizationScheme":
+ """Resolve a compressor-like object or QuantizationScheme to a QuantizationScheme.
-def is_mx_int(backend):
- backend = backend.lower()
- return BackendDataType.MX_INT in backend
+ `ar` (the compressor) exposes the same flat attribute names as QuantizationScheme
+ (bits, data_type, act_bits, ...), so QuantizationScheme.from_dict can read them
+ directly without needing ar.scheme to already be resolved.
+ """
+ if isinstance(ar_or_scheme, QuantizationScheme):
+ return ar_or_scheme
+ return QuantizationScheme(
+ bits=ar_or_scheme.bits,
+ group_size=ar_or_scheme.group_size,
+ sym=getattr(ar_or_scheme, "sym", True),
+ data_type=ar_or_scheme.data_type,
+ act_bits=ar_or_scheme.act_bits,
+ act_group_size=getattr(ar_or_scheme, "act_group_size", None),
+ act_sym=getattr(ar_or_scheme, "act_sym", None),
+ act_data_type=ar_or_scheme.act_data_type,
+ act_dynamic=getattr(ar_or_scheme, "act_dynamic", None),
+ super_bits=getattr(ar_or_scheme, "super_bits", None),
+ super_group_size=getattr(ar_or_scheme, "super_group_size", None),
+ )
-def is_nv_fp(backend):
- backend = backend.lower()
- return BackendDataType.NV_FP in backend
+# ``is_standard_fp`` / ``is_mx_fp`` / ``is_nv_fp`` / ``is_mx_int`` (data_type-string
+# classifiers) now live in ``auto_round.schemes`` as the single authority and are
+# re-exported above so existing ``from auto_round.compressors.utils import is_mx_fp``
+# call sites keep working.
def is_wint_woq(ar):
"""Returns True for integer weight-only quantization with non-quantized activations (`act_bits >= 16`)."""
- return "int" in ar.data_type and ar.act_bits >= 16 and ar.super_group_size is None
-
-
-def is_wint_a16(ar):
- """Backward-compatible alias for `is_wint_woq()`."""
- return is_wint_woq(ar)
-
-
-def _is_weight_fp8_activation_static_fp8(
- bit: int, group_size: int, sym: bool, data_type: str, act_dynamic: bool
-) -> bool:
- return bit == 8 and group_size == -1 and sym and data_type == "fp" and not act_dynamic
+ return _as_scheme(ar).is_wint_woq()
def is_wfp8afp8(ar):
- if (
- ("fp8" in ar.act_data_type or ("fp" in ar.act_data_type and ar.act_bits == 8))
- and ("fp8" in ar.data_type or ("fp" in ar.data_type and ar.bits == 8))
- and is_standard_fp(ar.act_data_type)
- and is_standard_fp(ar.data_type)
- ):
- return True
- else:
- return False
+ return _as_scheme(ar).is_wfp8afp8()
def is_wint8aint8(ar):
- if ("int8" in ar.act_data_type or ("int" in ar.act_data_type and ar.act_bits == 8)) and (
- "int8" in ar.data_type or ("int" in ar.data_type and ar.bits == 8)
- ):
- return True
- else:
- return False
+ return _as_scheme(ar).is_wint8aint8()
-def is_act_static(ar_or_format: Union[str, Callable]) -> bool:
- if isinstance(ar_or_format, str): # TODO this is not robust
+def is_act_static(ar_or_format):
+ if isinstance(ar_or_format, str):
return "fp8_static" in ar_or_format.lower()
- if not ar_or_format.act_dynamic:
- return True
- return False
+ return _as_scheme(ar_or_format).is_act_static()
-def is_dynamic_wint8aint8(ar_or_format: Union[str, Callable]) -> bool:
+def is_dynamic_wint8aint8(ar_or_format):
if isinstance(ar_or_format, str):
return "int8_w8a8" in ar_or_format.lower()
- if not ar_or_format.act_dynamic:
- return False
- if is_wint8aint8(ar_or_format):
- return True
- return False
+ return _as_scheme(ar_or_format).is_dynamic_wint8aint8()
-def is_wint4aint4(ar_or_scheme: Union[str, Callable]):
+def is_wint4aint4(ar_or_scheme):
if isinstance(ar_or_scheme, str):
return "int4" in ar_or_scheme.lower()
- elif (
- "int4" in ar_or_scheme.act_data_type or ("int" in ar_or_scheme.act_data_type and ar_or_scheme.act_bits == 4)
- ) and ("int4" in ar_or_scheme.data_type or ("int" in ar_or_scheme.data_type and ar_or_scheme.bits == 4)):
- return True
- return False
+ return _as_scheme(ar_or_scheme).is_wint4aint4()
-def is_dynamic_afp8(ar_or_format: Callable) -> bool:
- return ar_or_format.act_dynamic and ar_or_format.act_data_type.startswith("fp") and ar_or_format.act_bits == 8
+def is_dynamic_afp8(ar_or_format):
+ return _as_scheme(ar_or_format).is_dynamic_afp8()
-def is_block_wfp8(ar_or_format: Callable) -> bool:
- return (
- isinstance(ar_or_format.group_size, tuple)
- and len(ar_or_format.group_size) == 2
- and ar_or_format.data_type.startswith("fp")
- and ar_or_format.bits == 8
- )
+def is_block_wfp8(ar_or_format):
+ return _as_scheme(ar_or_format).is_block_wfp8()
def block_forward(
@@ -281,63 +241,6 @@ def infer_bits_by_data_type(data_type: str):
return None
-def _get_safetensor_layer_names_not_in_model(model, all_module_names: list) -> list:
- """Collect layer names from safetensor files that are not loaded into the model.
-
- Some tensors (e.g. MTP layers) exist in the original checkpoint but are not
- instantiated by ``transformers``. This function discovers them so that regex
- patterns in ``layer_config`` can still match them.
-
- Returns:
- List of layer names (the path without the ``.weight`` suffix) for weight
- tensors present in the safetensor files but absent from *all_module_names*.
- """
- name_or_path = None
- if hasattr(model, "config") and hasattr(model.config, "name_or_path"):
- name_or_path = model.config.name_or_path
- if not name_or_path:
- return []
-
- if not os.path.isdir(name_or_path):
- try:
- from auto_round.utils.model import download_hf_model
-
- name_or_path = download_hf_model(name_or_path)
- except Exception as e:
- logger.debug(f"Could not resolve source model path to check for missing tensors: {e}")
- return []
-
- try:
- from safetensors import safe_open
- except ImportError:
- return []
-
- # Build tensor-name list from the safetensors index or single file
- source_index_file = os.path.join(name_or_path, "model.safetensors.index.json")
- source_single_file = os.path.join(name_or_path, "model.safetensors")
-
- tensor_names: list = []
- if os.path.exists(source_index_file):
- with open(source_index_file) as f:
- src_index = json.load(f)
- tensor_names = list(src_index["weight_map"].keys())
- elif os.path.exists(source_single_file):
- with safe_open(source_single_file, framework="pt", device="cpu") as f:
- tensor_names = list(f.keys())
- else:
- return []
-
- module_name_set = set(all_module_names)
- extra_layer_names = []
- for tensor_name in tensor_names:
- if not tensor_name.endswith(".weight"):
- continue
- layer_name = tensor_name[: -len(".weight")]
- if layer_name not in module_name_set:
- extra_layer_names.append(layer_name)
- return extra_layer_names
-
-
def set_layer_config(
model: torch.nn.Module,
layer_config: dict[str, Union[str, dict, "QuantizationScheme"]],
@@ -351,722 +254,84 @@ def set_layer_config(
enable_gguf_official_mixed: bool = True,
is_mllm: bool = False,
fill_default_value=True,
- gguf_format_name: str = None,
) -> tuple[dict, bool, dict]:
- """
- Normalize, validate, and expand layer-specific quantization configs.
- Returns (final_layer_config, has_quant_layer_outside_block)
- """
-
- from auto_round.schemes import QuantizationScheme, get_gguf_scheme, preset_name_to_scheme
- from auto_round.utils.model import get_layer_names_in_block, get_lm_head_name, get_module, is_separate_lm_head
-
- # ---- helpers -------------------------------------------------
- def dispatch_layer_config(layer_config: dict[str, dict]) -> None:
- """Assign scheme values as attributes to matched modules."""
- for layer_name, scheme in layer_config.items():
- module = get_module(model, layer_name)
- if module is None:
- # Layer exists in safetensor files but is not loaded into the model
- # (e.g. MTP layers that transformers does not instantiate). Skip.
- continue
- for attr, value in scheme.items():
- setattr(module, attr, value)
-
- def normalize_item(item: Union[str, dict, "QuantizationScheme"], layer_name: str) -> dict:
- """Convert config entry into dict and validate keys."""
- if isinstance(item, str):
- config = asdict(preset_name_to_scheme(item.upper()))
- elif isinstance(item, QuantizationScheme):
- config = asdict(item)
- elif isinstance(item, dict):
- # "in_blocks" is an internal bookkeeping key injected by LLM-Compressor;
- # silently drop it before validation.
- item = {k: v for k, v in item.items() if k != "in_blocks"}
- scheme_name = item.pop("scheme", None)
- config = asdict(preset_name_to_scheme(scheme_name.upper())) if scheme_name is not None else {}
- invalid = set(item) - set(scheme_keys + ("fixed_by_user", "scale_dtype"))
- if invalid:
- raise ValueError(
- f"Invalid keys {invalid} in layer_config for '{layer_name}'. " f"Allowed keys: {scheme_keys}"
- )
- config.update(item)
- else:
- raise TypeError(
- f"Unsupported type for layer_config[{layer_name}]: {type(item)}. "
- f"Expected str, dict, or QuantizationScheme."
- )
- # Clean up
- config = {k: v for k, v in config.items() if v is not None}
- config["fixed_by_user"] = True
- return config
-
- # ---- main logic ----------------------------------------------
- extra_scheme_keys = ("scale_dtype",)
- scheme_keys = tuple(f.name for f in fields(QuantizationScheme)) + ("scale_dtype",)
- layer_config = copy.deepcopy(layer_config) or {}
- ignore_layer_patterns = set()
- if ignore_layers:
- ignore_layers = ignore_layers.replace(" ", "").split(",")
- ignore_layers = [name + "." if name[-1].isdigit() else name for name in ignore_layers]
- ignore_layer_patterns = set(ignore_layers)
-
- # 1. ignore_layers -> force 16
- for name in get_fp_layer_names(model, ignore_layers):
- layer_config[name] = {
- "bits": 16,
- "act_bits": 16,
- "data_type": "float",
- "act_data_type": "float",
- "fixed_by_user": True,
- }
-
- # 2. normalize
- layer_config = {k: normalize_item(v, k) for k, v in layer_config.items()}
-
- # 3. infer missing bits
- for cfg in layer_config.values():
- if "data_type" in cfg and "bits" not in cfg:
- if (b := infer_bits_by_data_type(cfg["data_type"])) is not None:
- cfg["bits"] = b
- if "act_data_type" in cfg and "act_bits" not in cfg:
- if (b := infer_bits_by_data_type(cfg["act_data_type"])) is not None:
- cfg["act_bits"] = b
-
- # 4. fill defaults
- if isinstance(default_scheme, str):
- default_dict = asdict(preset_name_to_scheme(default_scheme.upper()))
- else:
- default_dict = asdict(default_scheme)
- default_dict["scale_dtype"] = default_scale_dtype
-
- # In AutoScheme with mixed gguf:q4_k_m, the super_group_size of gguf:q8_0 layer is None,
- # which should not be filled by default q4km again
- for cfg in layer_config.values():
- for key in scheme_keys:
- if fill_default_value:
- cfg.setdefault(key, copy.deepcopy(default_dict.get(key)))
- else:
- if key in extra_scheme_keys:
- cfg.setdefault(key, copy.deepcopy(default_dict.get(key)))
- else:
- cfg.setdefault(key, None)
-
- # 5. collect supported modules
- embedding_types = (torch.nn.Embedding,)
- gguf_name = gguf_format_name or get_gguf_scheme(default_scheme)
- if gguf_name:
- if torch.nn.Embedding not in supported_types:
- supported_types = (*supported_types, torch.nn.Embedding)
-
- # for some Embedding which type() is not torch.nn.Embedding
- # for example: transformers.models.gemma3.modeling_gemma3.Gemma3TextScaledWordEmbedding
- model_module_name = model.__class__.__module__
- module_cls = sys.modules[model_module_name]
- for name in module_cls.__dict__:
- if name.endswith("Embedding") and not name.endswith("RotaryEmbedding"):
- embedding_types = (*embedding_types, getattr(module_cls, name))
- supported_types = (*supported_types, *embedding_types)
-
- all_supported_layer_names, embedding_layer_names = [], []
- all_module_names = []
- for n, m in model.named_modules():
- all_module_names.append(n)
- # cleanup stale attributes
- for key in scheme_keys:
- # `rotation_config` on the root model carries the active
- # Hadamard rotation state (weights + hooks)
- if n == "" and key == "rotation_config":
- continue
- if hasattr(m, key):
- delattr(m, key)
- if type(m) not in supported_types and m.__class__.__name__ not in inner_supported_types:
- continue
- all_supported_layer_names.append(n)
- if isinstance(m, embedding_types) or m.__class__.__name__.endswith("Embedding"):
- embedding_layer_names.append(n)
-
- # Also include layer names from safetensor files not loaded into the model
- # (e.g. MTP layers that transformers does not instantiate).
- safetensor_only_names = _get_safetensor_layer_names_not_in_model(model, all_module_names)
-
- # 6. expand regex configs
- regex_config = {}
- for name in list(layer_config.keys()):
- if name in all_supported_layer_names:
- continue
- if name in all_module_names:
- m = get_module(model, name)
- if len(list(m.children())) == 0 and type(m) not in supported_types:
- val = layer_config.pop(name)
- if name in ignore_layer_patterns:
- # Keep unsupported ignore_layers entries so export can serialize
- # them into regex-based extra_config for loaders like vLLM INC.
- regex_config[name] = val
- else:
- logger.warning(
- f"'{name}' exists in the model but is not a supported quantization target "
- f"in the current scheme, ignoring its setting in `layer_config`"
- )
- continue
-
- regex = re.compile(to_standard_regex(name))
- matched = [ln for ln in all_supported_layer_names if regex.search(ln)]
- safetensor_only_matched = [ln for ln in safetensor_only_names if regex.search(ln)]
- # skip it for mtp layers not loaded in transformers
- if not matched and not safetensor_only_matched:
- # type(mlp.gate) is Qwen3VLMoeTextTopKRouter instead of Linear
- logger.warning_once(
- f"Layer name or regex '{name}' in layer_config does not match any supported layers. "
- + "Please check for typos or update the regex pattern, ignore it for now"
- )
- val = layer_config.pop(name)
- regex_config[name] = val # keep regex config
- for match in matched:
- layer_config[match] = val
-
- # 7. lm_head
- lm_head_name = get_lm_head_name(model)
- tie_word_embeddings = False
- if hasattr(model, "config") and hasattr(model.config, "tie_word_embeddings"):
- tie_word_embeddings = model.config.tie_word_embeddings
-
- if lm_head_name in layer_config:
- quant_lm_head = True
-
- if quant_lm_head and tie_word_embeddings and not gguf_name:
- quant_lm_head = False
- logger.warning(
- "reset `quant_lm_head` to false as quantizing " "lm_head with tied weights has not been supported currently"
- )
-
- if lm_head_name not in layer_config and quant_lm_head:
- layer_config[lm_head_name] = copy.deepcopy(default_dict)
-
- if not quant_lm_head and not gguf_name:
- layer_config.pop(lm_head_name, None)
-
- # 8. enforce shape divisibility for int weight-only
- if default_dict["data_type"] == "int" and default_dict["act_bits"] >= 16 and not gguf_name:
- for n, m in model.named_modules():
- if type(m) in supported_types or m.__class__.__name__ in inner_supported_types:
- if m.weight.shape[0] % 32 or m.weight.shape[1] % 32:
- layer_config.setdefault(n, copy.deepcopy(default_dict))
- layer_config[n].update({"bits": 16, "data_type": "fp", "fixed_by_user": True})
- # logger.warning_once(f"{n} skipped quantization (shape not divisible by 32).")
- # enforce shape divisibility for mxfp/nvfp
- if (is_nv_fp(default_dict["data_type"]) or is_mx_fp(default_dict["data_type"])) and not gguf_name:
- skipped_layers = []
- for n, m in model.named_modules():
- if type(m) in supported_types or m.__class__.__name__ in inner_supported_types:
- if m.weight.shape[1] % default_dict["group_size"]:
- layer_config.setdefault(n, copy.deepcopy(default_dict))
- layer_config[n].update(
- {"bits": 16, "data_type": "fp", "act_bits": 16, "act_data_type": "fp", "fixed_by_user": True}
- )
- skipped_layers.append(n)
-
- compressed_skipped_layers = compress_layer_names(skipped_layers)
- if compressed_skipped_layers:
- logger.warning_once(
- f"some layers are skipped quantization (shape not divisible by {default_dict['group_size']}): "
- f"{compressed_skipped_layers}"
- )
-
- # 9. block layers: mark as in_blocks=True
- for name in get_layer_names_in_block(model, supported_types, quant_block_list, inner_supported_types):
- if name not in layer_config:
- layer_config[name] = copy.deepcopy(default_dict)
- layer_config[name]["fixed_by_user"] = False
- layer_config[name]["in_blocks"] = True
-
- # ---- restore: ensure missing in_blocks are set to False and compute flag ----
- has_qlayer_outside_block = False
- for cfg in layer_config.values():
- if "in_blocks" not in cfg:
- cfg["in_blocks"] = False
- # mark layer outside block
- if not cfg["in_blocks"] and check_to_quantized(cfg):
- has_qlayer_outside_block = True
-
- # 10. GGUF handling
- if not gguf_name:
- dispatch_layer_config(layer_config)
- return layer_config, has_qlayer_outside_block, regex_config
-
- # embed + lm_head defaults for gguf
- tie_word_embeddings &= not is_separate_lm_head(model)
- if lm_head_name not in layer_config and not tie_word_embeddings:
- cfg = GGUF_INNER_CONFIG[GGUF_CONFIG[gguf_name.lower()]["lm_head"]]
- cfg = {**cfg, "fixed_by_user": False, "scale_dtype": default_scale_dtype}
- layer_config[lm_head_name] = cfg
- has_qlayer_outside_block = True
- for emd_name in embedding_layer_names:
- if emd_name in layer_config:
- continue
- if not tie_word_embeddings:
- cfg = GGUF_INNER_CONFIG[GGUF_CONFIG[gguf_name.lower()]["embedding"]]
- else:
- cfg = GGUF_INNER_CONFIG[GGUF_CONFIG[gguf_name.lower()]["lm_head"]]
- cfg = {**cfg, "fixed_by_user": False, "scale_dtype": default_scale_dtype}
- layer_config[emd_name] = cfg
-
- if enable_gguf_official_mixed:
- model_type = ModelType.MMPROJ if is_mllm else ModelType.TEXT
- layer_config, _ = get_layer_config_by_gguf_format(layer_config, gguf_name.lower(), model, model_type)
-
- _apply_gguf_shape_fallback(layer_config, model)
- dispatch_layer_config(layer_config)
- return layer_config, has_qlayer_outside_block, regex_config
-
-
-def _use_more_bits(i_layer: int, n_layer: int):
- return (i_layer < n_layer // 8) or (i_layer >= 7 * n_layer // 8) or ((i_layer - n_layer // 8) % 3 == 2)
-
-
-def _search_gguf_type(gguf_type):
- if gguf_type in GGUF_INNER_CONFIG:
- return gguf_type
- pattern = re.compile("gguf:q([0-9]{1,})_[01k]")
- bits = re.search(pattern, gguf_type)
- if not bits:
- raise KeyError(f"{gguf_type} is not a correct gguf type, please check")
-
- for suffix in ["_k", "_0", "_1"]:
- if gguf_type.endswith(suffix):
- continue
- if (tmp_type := re.sub("_[01k]", suffix, gguf_type)) in GGUF_INNER_CONFIG:
- return tmp_type
- return None
-
-
-def gguf_type_fallback(gguf_type: str) -> str:
- gguf_type = gguf_type.lower()
- if gguf_type in ("gguf:q2_k", "gguf:q3_k", "gguf:q4_k"):
- gguf_type = "gguf:q5_0"
- elif gguf_type == "gguf:q5_k":
- gguf_type = "gguf:q5_0"
- elif gguf_type == "gguf:q6_k":
- gguf_type = "gguf:q8_0"
- return gguf_type
-
-
-def get_gguf_qtype_by_layer_config(layer_config):
- import gguf # pylint: disable=E0401
-
- if layer_config["bits"] >= 16:
- return None
- bits = layer_config["bits"]
- super_bits = layer_config.get("super_bits", None)
- sym = layer_config["sym"]
- group_size = layer_config.get("group_size", None)
- super_group_size = layer_config.get("super_group_size", None)
- if bits == 2 and super_bits == 4 and not sym and group_size == 16 and super_group_size == 16:
- return gguf.GGMLQuantizationType.Q2_K
- if bits == 3 and super_bits == 6 and sym and group_size == 16 and super_group_size == 16:
- return gguf.GGMLQuantizationType.Q3_K
- if bits == 4:
- if super_bits is not None and super_bits == 6 and not sym and group_size == 32 and super_group_size == 8:
- return gguf.GGMLQuantizationType.Q4_K
- if super_bits is None and sym and group_size == 32:
- return gguf.GGMLQuantizationType.Q4_0
- if super_bits is None and not sym and group_size == 32:
- return gguf.GGMLQuantizationType.Q4_1
- if bits == 5:
- if super_bits == 6 and not sym and group_size == 32 and super_group_size == 8:
- return gguf.GGMLQuantizationType.Q5_K
- if super_bits is None and sym and group_size == 32:
- return gguf.GGMLQuantizationType.Q5_0
- if super_bits is None and not sym and group_size == 32:
- return gguf.GGMLQuantizationType.Q5_1
- if bits == 6 and super_bits == 8 and group_size == 16 and super_group_size == 16:
- return gguf.GGMLQuantizationType.Q6_K
- if bits == 8 and sym and group_size == 32:
- return gguf.GGMLQuantizationType.Q8_0
- raise ValueError("Unknown layer config")
-
-
-def _apply_gguf_shape_fallback(layer_config, model):
- from auto_round.utils.model import get_module
-
- for layer_name, config in layer_config.items():
- if not check_to_quantized(config):
- continue
- layer = get_module(model, layer_name)
- if layer is None or not hasattr(layer, "weight"):
- continue
- try:
- qtype = get_gguf_qtype_by_layer_config(config)
- except ValueError:
- continue
- if qtype is None:
- continue
-
- gguf_type = f"gguf:{qtype.name.lower()}"
- block_size = GGML_QUANT_SIZES[gguf_type.split(":")[-1]][0]
- input_features = (
- layer.weight.shape[0] if type(layer) == transformers.pytorch_utils.Conv1D else layer.weight.shape[-1]
- )
- if input_features % block_size == 0:
- continue
-
- fallback_type = _gguf_type_fallback(gguf_type)
- fallback_block_size = GGML_QUANT_SIZES[fallback_type.split(":")[-1]][0]
- if input_features % fallback_block_size != 0:
- fallback_type = "gguf:bf16"
-
- preserved = {key: config[key] for key in ("fixed_by_user", "scale_dtype") if key in config}
- config.update(GGUF_INNER_CONFIG[fallback_type])
- config.update(preserved)
- logger.warning(
- "fallback %s to %s before quantization, because input_features(%s) is not divisible by %s block_size(%s)",
- layer_name,
- fallback_type,
- input_features,
- gguf_type,
- block_size,
+ """Compatibility adapter for the pure layer-config resolver and explicit apply phase."""
+ from auto_round.compressors.config_resolution import (
+ ResolvedQuantizationConfig,
+ ResolvedScheme,
+ resolve_scheme_value,
+ )
+ from auto_round.compressors.layer_config import (
+ apply_plan_to_model,
+ extract_regex_config,
+ has_quantized_layer_outside_blocks,
+ resolve_layer_config,
+ )
+ from auto_round.schemes import get_gguf_scheme
+
+ if isinstance(default_scheme, ResolvedScheme):
+ resolved_scheme = default_scheme
+ elif isinstance(default_scheme, QuantizationScheme):
+ resolved_scheme = ResolvedScheme.from_scheme(
+ default_scheme,
+ preset_name=get_gguf_scheme(default_scheme) or None,
)
-
-
-def _get_digital_in_layer_name(layer_name):
- pattern = re.compile(r"([a-zA-Z]+\.){1,}(\d+)")
- res = re.search(pattern, layer_name)
- if res:
- return int(res[2])
else:
- return None
-
-
-def _gguf_type_fallback(gguf_type: str) -> str:
- gguf_type = gguf_type.lower()
- if gguf_type in ("gguf:q2_k", "gguf:q3_k", "gguf:q4_k"):
- gguf_type = "gguf:q5_0"
- elif gguf_type == "gguf:q5_k":
- gguf_type = "gguf:q5_0"
- elif gguf_type == "gguf:q6_k":
- gguf_type = "gguf:q8_0"
- return gguf_type
-
+ resolved_scheme = resolve_scheme_value(default_scheme, {})
-def _infer_gguf_n_layers_from_model(model, model_type=ModelType.TEXT):
- if model is None or not hasattr(model, "named_modules"):
- return None, None
-
- from auto_round.utils.common import MM_MODULE_KEYS
-
- text_layer_ids = set()
- vision_layer_ids = set()
- layer_pattern = re.compile(r"(?:^|\.)layers\.(\d+)(?:\.|$)")
- for module_name, _ in model.named_modules():
- match = layer_pattern.search(module_name)
- if match is None:
- continue
- layer_id = int(match.group(1))
- if model_type != ModelType.TEXT and any(key in module_name.lower() for key in MM_MODULE_KEYS):
- vision_layer_ids.add(layer_id)
- else:
- text_layer_ids.add(layer_id)
-
- n_layer = max(text_layer_ids) + 1 if text_layer_ids else None
- n_layer_vision = max(vision_layer_ids) + 1 if vision_layer_ids else None
- return n_layer, n_layer_vision
-
-
-def _resolve_gguf_n_layers(config, model_type=ModelType.TEXT, model=None):
- """Resolve text (and vision) layer counts from config, then model structure.
-
- Multimodal configs (e.g. Qwen3.5, Qwen3-VL) keep the text hparams only in
- ``config.text_config``, so both it and the top-level config must be checked.
- If config naming changes, fall back to the loaded module hierarchy so GGUF
- official mixed formats still receive a usable block count.
- """
- n_layer = None
- n_layer_vision = None
- text_config = getattr(config, "text_config", None)
- vision_config = None
- audio_config = getattr(config, "audio_config", None)
- for name in ["n_layers", "num_hidden_layers", "n_layer", "num_layers", "depth"]:
- if hasattr(config, name):
- n_layer = getattr(config, name)
- if text_config is not None and hasattr(text_config, name):
- # the text hparams in text_config are authoritative for multimodal models;
- # for text-only usage they still fill in when the top-level config lacks them
- if model_type != ModelType.TEXT or n_layer is None:
- n_layer = getattr(text_config, name)
- if model_type != ModelType.TEXT:
- for config_name in ["vision_config", "vision_encoder"]:
- if hasattr(config, config_name):
- vision_config = getattr(config, config_name)
- if hasattr(vision_config, name):
- n_layer_vision = getattr(vision_config, name)
- break
- if n_layer and n_layer_vision:
- break
-
- inferred_n_layer, inferred_n_layer_vision = _infer_gguf_n_layers_from_model(model, model_type)
- n_layer = n_layer or inferred_n_layer
- n_layer_vision = n_layer_vision or inferred_n_layer_vision
-
- if (
- model_type != ModelType.TEXT
- and n_layer_vision is None
- and vision_config is not None
- and audio_config is not None
- ):
- # llama.cpp's Gemma4 mmproj converter uses a fixed tensor-name map size
- # when vision and audio encoders coexist, because the two encoders do not
- # expose a single transformer block count in config.json.
- n_layer_vision = 128
- return n_layer, n_layer_vision
-
-
-##https://github.com/ggml-org/llama.cpp/blob/9e31bec4fd53634c9e5b04650488a09a055f5dab/src/llama-quant.cpp#L129
-def get_layer_config_by_gguf_format(layer_config, target_gguf_format: str, model, model_type=ModelType.TEXT):
- # # TODO: support for other format later
- # target_gguf_format = next((fmt for fmt in gguf_format if fmt != "fake"), None)
-
- import gguf # pylint: disable=E0401
-
- from auto_round.export.export_to_gguf.llama_cpp_conversion import get_conversion
- from auto_round.schemes import QuantizationScheme, get_gguf_scheme
- from auto_round.utils.common import MM_MODULE_KEYS
- from auto_round.utils.model import get_lm_head_name, get_module, is_separate_lm_head
-
- try:
- hparams = model.config.to_dict()
- conversion = get_conversion(hparams=hparams, model_type=model_type)
- model_architecture = conversion.get_model_architecture(
- hparams=hparams, model_type=conversion.model_type(model_type)
- )
- except AttributeError as e:
- raise ImportError(
- "Please use the latest gguf-py, you can use the following command to install it:\n"
- "git clone https://github.com/ggml-org/llama.cpp.git && cd llama.cpp/gguf-py"
- " && pip install . sentencepiece"
- )
- try:
- if model_type != ModelType.TEXT:
- model_class_vision = conversion.get_model_class(model_architecture, model_type=model_type)
- model_class = conversion.get_model_class(model_architecture, model_type=ModelType.TEXT)
-
- except NotImplementedError:
- return layer_config, {}
-
- n_layer, n_layer_vision = _resolve_gguf_n_layers(model.config, model_type, model=model)
-
- if n_layer is None:
- return layer_config, {}
-
- tensor_map = gguf.get_tensor_name_map(model_class.model_arch, n_layer)
- if model_type != ModelType.TEXT:
- tensor_map_vision = gguf.get_tensor_name_map(model_class_vision.model_arch, n_layer_vision)
-
- def _set_config(config, target_config):
- for k, v in target_config.items():
- if isinstance(config, dict):
- config[k] = v
- else:
- setattr(config, k, v)
- return config
-
- gguf_format_config = {}
- lm_head_name = get_lm_head_name(model)
- inner_gguf_format = GGUF_CONFIG[target_gguf_format]["mostly"]
- # ggml_type = getattr(gguf.GGMLQuantizationType,inner_gguf_format.split(":")[-1].upper())
- block_size = GGML_QUANT_SIZES[inner_gguf_format.split(":")[-1].lower()][0]
- tie_word_embeddings = True
- if hasattr(model, "config") and hasattr(model.config, "tie_word_embeddings"):
- tie_word_embeddings = model.config.tie_word_embeddings
- tie_word_embeddings &= not is_separate_lm_head(model)
-
- dtype_selector = GGUFDTypeSelector(
- hparams, gguf_format_to_ftype(target_gguf_format), model_class.model_arch, n_layer
+ resolved = resolve_layer_config(
+ model=model,
+ scheme=resolved_scheme,
+ layer_config=layer_config,
+ scale_dtype=default_scale_dtype,
+ supported_types=supported_types,
+ inner_supported_types=inner_supported_types,
+ quant_block_list=quant_block_list,
+ ignore_layers=ignore_layers,
+ quant_lm_head=quant_lm_head,
+ enable_gguf_official_mixed=enable_gguf_official_mixed,
+ is_mllm=is_mllm,
+ fill_default_value=fill_default_value,
)
- layer_config_copy = copy.deepcopy(layer_config)
- base_target_bits = None
- if inner_gguf_format.startswith("gguf:q") and len(inner_gguf_format) >= 7 and (inner_gguf_format[6]).isdigit():
- base_target_bits = int(inner_gguf_format[6])
-
- def _resolve_gguf_name(layer_name):
- if model_type != ModelType.TEXT and any([key in layer_name for key in MM_MODULE_KEYS]):
- gguf_layer_name = tensor_map_vision.get_name(layer_name)
- if gguf_layer_name is None:
- for key in MM_MODULE_KEYS:
- gguf_layer_name = tensor_map_vision.get_name(layer_name.replace(f".{key}", ""))
- if gguf_layer_name is not None:
- break
- else:
- gguf_layer_name = tensor_map.get_name(layer_name)
- if gguf_layer_name is None:
- gguf_layer_name = tensor_map.get_name(layer_name.replace(".language_model", ""))
- return gguf_layer_name
-
- dtype_selector.n_attention_wv = sum(
- 1
- for layer_name, config in layer_config_copy.items()
- if check_to_quantized(config)
- and (gguf_name := _resolve_gguf_name(layer_name))
- and any(key in gguf_name for key in ("attn_v", "attn_qkv", "attn_kv_b"))
+ regex_config = extract_regex_config(
+ model=model,
+ scheme=resolved_scheme,
+ layer_config=layer_config,
+ scale_dtype=default_scale_dtype,
+ supported_types=supported_types,
+ inner_supported_types=inner_supported_types,
+ ignore_layers=ignore_layers,
+ fill_default_value=fill_default_value,
+ )
+ has_outside = has_quantized_layer_outside_blocks(resolved)
+ plan = ResolvedQuantizationConfig(
+ scheme=resolved_scheme,
+ formats=(),
+ layer_config=resolved,
+ regex_config=regex_config,
+ has_qlayer_outside_block=has_outside,
+ scale_dtype=default_scale_dtype,
+ quant_block_list=quant_block_list,
+ )
+ apply_plan_to_model(model, plan)
+ return (
+ {name: dict(config) for name, config in plan.layer_config.items()},
+ plan.has_qlayer_outside_block,
+ {name: dict(config) for name, config in plan.regex_config.items()},
)
- for layer_name, config in layer_config_copy.items():
- if not check_to_quantized(config):
- continue
- # Reset target_bits each iteration to prevent lm_head/embedding settings
- # from bleeding into subsequent block layers and bypassing their special logic.
- target_bits = base_target_bits
- new_type = GGUF_CONFIG[target_gguf_format]["mostly"]
- layer = get_module(model, layer_name)
- if type(layer) == transformers.pytorch_utils.Conv1D:
- input_features = layer.weight.shape[0]
- else:
- input_features = layer.weight.shape[-1]
- i_layer = _get_digital_in_layer_name(layer_name)
-
- if lm_head_name is not None and layer_name == lm_head_name:
- target_bits = int(re.search("gguf:q([0-9]{1,})_[01k]", GGUF_CONFIG[target_gguf_format]["lm_head"]).group(1))
- if isinstance(layer, torch.nn.Embedding):
- embedding_format_key = "lm_head" if tie_word_embeddings else "embedding"
- target_bits = int(
- re.search("gguf:q([0-9]{1,})_[01k]", GGUF_CONFIG[target_gguf_format][embedding_format_key]).group(1)
- )
-
- gguf_name = _resolve_gguf_name(layer_name)
- bits_index = 6
- if config.get("fixed_by_user", False):
- if "bits" not in config:
- logger.warning(
- f"Setting layer_config requires providing bits, {layer_name} has not bits,"
- f" using bits={target_bits} instead."
- )
- new_type = new_type[:bits_index] + str(target_bits) + new_type[bits_index + 1 :]
- else:
- config_tmp = config.copy()
- scheme_keys = [f.name for f in fields(QuantizationScheme)]
- for key in config.keys():
- if key not in scheme_keys:
- config_tmp.pop(key, None)
- matched_scheme = get_gguf_scheme(QuantizationScheme.from_dict(config_tmp)) # check matched
- if not matched_scheme:
- if config.get("super_group_size", None) is not None or config.get("super_bits", None) is not None:
- new_type = new_type[:bits_index] + str(config["bits"]) + "_k"
- if new_type not in GGUF_INNER_CONFIG:
- prefix_idx = 0 if config.get("sym", True) else 1
- new_type = new_type[:bits_index] + str(config["bits"]) + f"_{prefix_idx}"
- if new_type not in GGUF_INNER_CONFIG:
- new_type = new_type[:bits_index] + str(config["bits"]) + f"_{1-prefix_idx}"
- if new_type not in GGUF_INNER_CONFIG:
- raise ValueError(
- f"the setting in layer_config {layer_name} "
- f"could not match any supported gguf format, please have a check."
- )
-
- new_type = new_type[:bits_index] + str(config["bits"]) + new_type[bits_index + 1 :]
- new_type = _search_gguf_type(new_type)
- if new_type is None:
- raise ValueError(f"invalid bit setting for {layer_name}")
- elif lm_head_name is not None and layer_name == lm_head_name and not tie_word_embeddings:
- if gguf.MODEL_ARCH.FALCON == model_class.model_arch or input_features % block_size != 0:
- new_type = "gguf:q8_0"
- elif "lm_head" in GGUF_CONFIG[target_gguf_format]:
- new_type = GGUF_CONFIG[target_gguf_format]["lm_head"]
- elif new_type != "gguf:q8_0":
- new_type = "gguf:q6_k"
- elif lm_head_name is not None and layer_name == lm_head_name and tie_word_embeddings:
- continue
- elif isinstance(layer, torch.nn.Embedding):
- embedding_format_key = "lm_head" if tie_word_embeddings else "embedding"
- if embedding_format_key in GGUF_CONFIG[target_gguf_format]:
- new_type = GGUF_CONFIG[target_gguf_format][embedding_format_key]
- elif target_bits is not None and "bits" in config and config["bits"] != target_bits:
- new_type = new_type[:bits_index] + str(config["bits"]) + new_type[bits_index + 1 :]
- new_type = _search_gguf_type(new_type)
- if new_type is None:
- raise ValueError(f"invalid bit setting for {layer_name}")
- elif gguf_name is not None:
- gguf_weight_name = gguf_name if gguf_name.endswith(".weight") else f"{gguf_name}.weight"
- new_type = dtype_selector.select_gguf_type(gguf_weight_name, len(layer.weight.shape), i_layer or 0)
- new_block_size = GGML_QUANT_SIZES[new_type.split(":")[-1].lower()][0]
- if input_features % new_block_size != 0:
- new_type = _gguf_type_fallback(new_type)
- new_block_size = GGML_QUANT_SIZES[new_type.split(":")[-1].lower()][0]
- if input_features % new_block_size != 0:
- new_type = "gguf:bf16"
- logger.warning(
- f"fallback {layer_name} to {new_type}, "
- f"because input_features({input_features}) % block_size({block_size}) != 0"
- )
- # for deepseek v2
- if layer_name.endswith("kv_b_proj") and new_type.endswith("_k") and "Deepseek" in model.config.architectures[0]:
- fallback = False
-
- # calc if need fallback
- qk_nope_head_dim = model.config.qk_nope_head_dim
- kv_b_shape = get_module(model, layer_name).weight.shape
-
- if (
- qk_nope_head_dim < QK_K
- or qk_nope_head_dim % QK_K != 0
- or kv_b_shape[-1] < QK_K
- or kv_b_shape[-1] % QK_K != 0
- ):
- fallback = True
- if fallback:
- tmp_type = _gguf_type_fallback(new_type)
- logger.warning_once(
- f"self_attn.kv_b_proj does not support the use of {new_type}, replace it with {tmp_type}"
- )
- new_type = tmp_type
-
- target_config = GGUF_INNER_CONFIG[new_type]
-
- _set_config(layer_config[layer_name], target_config)
- _set_config(layer, target_config)
- gguf_format_config[layer_name] = new_type
-
- return layer_config, gguf_format_config
-
-
-def get_fp_layer_names(model: torch.nn.Module, ignore_layers: str):
- """Identifies and returns layers in the model to exclude from quantization.
-
- This function processes a comma-separated list of fully precision (FP) layers,
- matches them to the names of layers in the model, and returns a list of such
- layers to exclude from quantization.
-
- Args:
- model (torch.nn.Module): The model whose layers will be inspected.
- ignore_layers (str): A comma-separated string of layer names to be excluded
- from quantization. Whitespace is ignored in this string.
-
- Returns:
- list: A list of layer names that match the specified FP layers or are
- subcomponents of those layers.
- """
- from auto_round.utils import SUPPORTED_LAYER_TYPES
-
- if not ignore_layers:
- return []
- all_layer_names = []
- for n, m in model.named_modules():
- if type(m) in SUPPORTED_LAYER_TYPES:
- all_layer_names.append(n)
- not_to_quantized_layers = []
+from auto_round.compressors.layer_config.resolver import get_fp_layer_names
- for fp_layer in ignore_layers:
- if fp_layer == "":
- continue
- if fp_layer in all_layer_names:
- not_to_quantized_layers.append(fp_layer)
- continue
- for name in all_layer_names:
- if fp_layer in name:
- not_to_quantized_layers.append(name)
- not_to_quantized_layers.extend(ignore_layers) # keep regex name for later use
- if not_to_quantized_layers:
- logger.info(f"Ignored layers: {compress_layer_names(not_to_quantized_layers)}")
- return not_to_quantized_layers
+# Explicit compatibility exports for callers that historically imported GGUF and
+# ignore-layer helpers from compressors.utils.
+from auto_round.export.formats.backends.gguf import (
+ _apply_gguf_shape_fallback,
+ _infer_gguf_n_layers_from_model,
+ _resolve_gguf_n_layers,
+ get_layer_config_by_gguf_format,
+ gguf_type_fallback,
+)
def get_shared_keys(model):
@@ -1190,8 +455,7 @@ def _get_quantized_layer_names_outside_blocks(model, layer_config, supported_typ
continue
layer = get_module(model, key)
if layer is None:
- logger.error(f"could not find layer {key} in the model, exit...")
- exit(-1)
+ raise LayerConfigResolutionError(f"could not find layer '{key}' in the model")
if type(layer) in supported_types and check_to_quantized(layer_config[key]):
layer_names.append(key)
diff --git a/auto_round/data_type/nvfp.py b/auto_round/data_type/nvfp.py
index f9f8a88fe..bab5b5410 100644
--- a/auto_round/data_type/nvfp.py
+++ b/auto_round/data_type/nvfp.py
@@ -393,7 +393,7 @@ def opt_rtn_fast_nvfp4(
**kwargs,
):
tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)
- if imatrix is None:
+ if not isinstance(imatrix, torch.Tensor):
qw = 1.0
else:
imatrix = imatrix.reshape(1, -1)
diff --git a/auto_round/data_type/utils.py b/auto_round/data_type/utils.py
index 4bbc35179..7e4be30db 100644
--- a/auto_round/data_type/utils.py
+++ b/auto_round/data_type/utils.py
@@ -248,8 +248,8 @@ def search_optimized_init_scale(
search_fn, _ = _resolve_optimized_dtype_funcs(data_type, q_scale_thresh)
if search_fn is None:
return None
- if imatrix is None:
- imatrix = 1.0
+ if imatrix is None or not isinstance(imatrix, torch.Tensor):
+ imatrix = torch.ones_like(weight_reshape)
return search_fn(weight_reshape, bits, imatrix)
@@ -270,10 +270,11 @@ def reshape_imatrix_for_weight(imatrix, weight_reshape: torch.Tensor, group_size
Encapsulates the imatrix grouping logic shared by the SignRound optimized
wrapper and AWQ's internal QDQ so callers never handle the low-level reshape.
- Returns the scalar ``1.0`` when no imatrix is available (uniform importance).
+ Returns a tensor of ones when no imatrix is available (uniform importance),
+ keeping every downstream optimized dtype implementation on its tensor path.
"""
- if imatrix is None:
- return 1.0
+ if imatrix is None or not isinstance(imatrix, torch.Tensor):
+ return torch.ones_like(weight_reshape)
imatrix = imatrix.reshape(1, -1)
imatrix = reshape_pad_tensor_by_group_size(imatrix, group_size, val=1e-5)[0].view(1, -1)
imatrix = imatrix.expand(weight_reshape.numel() // imatrix.numel(), -1)
diff --git a/auto_round/export/export_to_autoround/export.py b/auto_round/export/export_to_autoround/export.py
index 4b9c9e30e..fd02d98d9 100644
--- a/auto_round/export/export_to_autoround/export.py
+++ b/auto_round/export/export_to_autoround/export.py
@@ -28,6 +28,7 @@ from tqdm import tqdm
from auto_round.compressors.utils import is_mx_fp, is_nv_fp, is_standard_fp
from auto_round.export.export_to_autoround.utils import check_neq_config
+from auto_round.export.formats import BackendDataType
from auto_round.export.utils import (
filter_quantization_config,
get_autogptq_packing_qlinear,
@@ -37,7 +38,6 @@ from auto_round.export.utils import (
save_model,
save_pretrained_artifact,
)
-from auto_round.formats import AutoRoundExportFormat
from auto_round.logger import logger
from auto_round.schemes import QuantizationScheme
from auto_round.utils import (
@@ -275,7 +275,7 @@ def save_quantized_as_autoround(
if (
(serialization_dict.get("sym") is None or serialization_dict.get("sym"))
and ("gptq" not in backend and "awq" not in backend)
- and (AutoRoundExportFormat.FP8_STATIC.value not in backend)
+ and (BackendDataType.FP8_STATIC.value not in backend)
):
backend = backend.replace("auto_round", "auto_round:auto_gptq")
diff --git a/auto_round/export/export_to_gguf/config.py b/auto_round/export/export_to_gguf/config.py
index 56f2af953..f302949cd 100644
--- a/auto_round/export/export_to_gguf/config.py
+++ b/auto_round/export/export_to_gguf/config.py
@@ -11,189 +11,42 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+import copy
from enum import IntEnum
+from auto_round.schemes import GGUF_PRESET_ALIASES, GGUF_SCHEME_FACTS
+
class ModelType(IntEnum):
TEXT = 1
MMPROJ = 2
-GGUF_INNER_CONFIG = {}
-
-GGUF_INNER_CONFIG["gguf:q4_0"] = {
- "bits": 4,
- "act_bits": 16,
- "group_size": 32,
- "sym": True,
- "data_type": "int",
- "embedding": "gguf:q4_0",
- "lm_head": "gguf:q6_k",
- "super_bits": None,
- "super_group_size": None,
-}
-
-GGUF_INNER_CONFIG["gguf:q4_1"] = {
- "bits": 4,
- "act_bits": 16,
- "group_size": 32,
- "sym": False,
- "data_type": "int_asym_float_zp",
- "embedding": "gguf:q4_1",
- "lm_head": "gguf:q6_k",
- "super_bits": None,
- "super_group_size": None,
-}
-
-GGUF_INNER_CONFIG["gguf:q5_0"] = {
- "bits": 5,
- "act_bits": 16,
- "group_size": 32,
- "sym": True,
- "data_type": "int",
- "embedding": "gguf:q5_0",
- "lm_head": "gguf:q6_k",
- "super_bits": None,
- "super_group_size": None,
-}
-
-GGUF_INNER_CONFIG["gguf:q5_1"] = {
- "bits": 5,
- "act_bits": 16,
- "group_size": 32,
- "sym": False,
- "data_type": "int_asym_float_zp",
- "embedding": "gguf:q5_1",
- "lm_head": "gguf:q6_k",
- "super_bits": None,
- "super_group_size": None,
+_GGUF_FORMAT_FIELDS = {
+ "gguf:q4_0": {"embedding": "gguf:q4_0", "lm_head": "gguf:q6_k"},
+ "gguf:q4_1": {"embedding": "gguf:q4_1", "lm_head": "gguf:q6_k"},
+ "gguf:q5_0": {"embedding": "gguf:q5_0", "lm_head": "gguf:q6_k"},
+ "gguf:q5_1": {"embedding": "gguf:q5_1", "lm_head": "gguf:q6_k"},
+ "gguf:q8_0": {"embedding": "gguf:q8_0", "lm_head": "gguf:q8_0"},
+ "gguf:q2_k": {"embedding": "gguf:q2_k", "lm_head": "gguf:q6_k"},
+ "gguf:q3_k": {"embedding": "gguf:q3_k", "lm_head": "gguf:q6_k"},
+ "gguf:q4_k": {"embedding": "gguf:q4_k", "lm_head": "gguf:q6_k"},
+ "gguf:q5_k": {"embedding": "gguf:q5_k", "lm_head": "gguf:q6_k"},
+ "gguf:q6_k": {"embedding": "gguf:q6_k", "lm_head": "gguf:q6_k"},
+ "gguf:bf16": {"embedding": None, "lm_head": None},
}
+_GGUF_FORMAT_FIELDS["gguf:fp16"] = _GGUF_FORMAT_FIELDS["gguf:bf16"]
-GGUF_INNER_CONFIG["gguf:q8_0"] = {
- "bits": 8,
- "act_bits": 16,
- "group_size": 32,
- "sym": True,
- "data_type": "int",
- "embedding": "gguf:q8_0",
- "lm_head": "gguf:q8_0",
- "super_bits": None,
- "super_group_size": None,
-}
-
-GGUF_INNER_CONFIG["gguf:q2_k"] = {
- "bits": 2,
- "act_bits": 16,
- "super_group_size": 16,
- "super_bits": 4,
- "group_size": 16,
- "sym": False,
- "data_type": "int_asym_dq",
- "embedding": "gguf:q2_k",
- "lm_head": "gguf:q6_k",
-}
-
-GGUF_INNER_CONFIG["gguf:q3_k"] = {
- "bits": 3,
- "act_bits": 16,
- "super_group_size": 16,
- "super_bits": 6,
- "group_size": 16,
- "sym": True,
- "data_type": "int_sym_dq",
- "embedding": "gguf:q3_k",
- "lm_head": "gguf:q6_k",
-}
-
-GGUF_INNER_CONFIG["gguf:q4_k"] = {
- "bits": 4,
- "act_bits": 16,
- "super_group_size": 8,
- "super_bits": 6,
- "group_size": 32,
- "sym": False,
- "data_type": "int_asym_dq",
- "embedding": "gguf:q4_k",
- "lm_head": "gguf:q6_k",
-}
-
-GGUF_INNER_CONFIG["gguf:q5_k"] = {
- "bits": 5,
- "act_bits": 16,
- "super_group_size": 8,
- "super_bits": 6,
- "group_size": 32,
- "sym": False,
- "data_type": "int_asym_dq",
- "embedding": "gguf:q5_k",
- "lm_head": "gguf:q6_k",
-}
-
-GGUF_INNER_CONFIG["gguf:q6_k"] = {
- "bits": 6,
- "act_bits": 16,
- "super_group_size": 16,
- "super_bits": 8,
- "group_size": 16,
- "sym": True,
- "data_type": "int_sym_dq",
- "embedding": "gguf:q6_k",
- "lm_head": "gguf:q6_k",
-}
-
-GGUF_INNER_CONFIG["gguf:bf16"] = GGUF_INNER_CONFIG["gguf:fp16"] = {
- "bits": 16,
- "act_bits": 16,
- "super_group_size": None,
- "super_bits": None,
- "group_size": None,
- "sym": True,
- "data_type": "int_sym_dq",
- "embedding": None,
- "lm_head": None,
-}
+GGUF_INNER_CONFIG = {}
+for key, facts in GGUF_SCHEME_FACTS.items():
+ GGUF_INNER_CONFIG[key] = {**copy.deepcopy(facts), **_GGUF_FORMAT_FIELDS[key]}
GGUF_CONFIG = {}
-GGUF_CONFIG["gguf:q4_0"] = GGUF_INNER_CONFIG["gguf:q4_0"]
-GGUF_CONFIG["gguf:q4_0"]["mostly"] = "gguf:q4_0"
-GGUF_CONFIG["gguf:q4_1"] = GGUF_INNER_CONFIG["gguf:q4_1"]
-GGUF_CONFIG["gguf:q4_1"]["mostly"] = "gguf:q4_1"
-GGUF_CONFIG["gguf:q5_0"] = GGUF_INNER_CONFIG["gguf:q5_0"]
-GGUF_CONFIG["gguf:q5_0"]["mostly"] = "gguf:q5_0"
-GGUF_CONFIG["gguf:q5_1"] = GGUF_INNER_CONFIG["gguf:q5_1"]
-GGUF_CONFIG["gguf:q5_1"]["mostly"] = "gguf:q5_1"
-GGUF_CONFIG["gguf:q2_k_s"] = GGUF_INNER_CONFIG["gguf:q2_k"]
-GGUF_CONFIG["gguf:q2_k_s"]["mostly"] = "gguf:q2_k"
-# GGUF_CONFIG["gguf:q3_k"] = GGUF_INNER_CONFIG["gguf:q3_k"]
-# GGUF_CONFIG["gguf:q3_k"]["mostly"] = "gguf:q3_k"
-GGUF_CONFIG["gguf:q3_k_s"] = GGUF_INNER_CONFIG["gguf:q3_k"]
-GGUF_CONFIG["gguf:q3_k_s"]["mostly"] = "gguf:q3_k"
-GGUF_CONFIG["gguf:q3_k_m"] = GGUF_INNER_CONFIG["gguf:q3_k"]
-GGUF_CONFIG["gguf:q3_k_m"]["mostly"] = "gguf:q3_k"
-GGUF_CONFIG["gguf:q3_k_l"] = GGUF_INNER_CONFIG["gguf:q3_k"]
-GGUF_CONFIG["gguf:q3_k_l"]["mostly"] = "gguf:q3_k"
-# GGUF_CONFIG["gguf:q4_k"] = GGUF_INNER_CONFIG["gguf:q4_k"]
-# GGUF_CONFIG["gguf:q4_k"]["mostly"]= "gguf:q4_k"
-GGUF_CONFIG["gguf:q4_k_s"] = GGUF_INNER_CONFIG["gguf:q4_k"]
-GGUF_CONFIG["gguf:q4_k_s"]["mostly"] = "gguf:q4_k"
-GGUF_CONFIG["gguf:q4_k_m"] = GGUF_INNER_CONFIG["gguf:q4_k"]
-GGUF_CONFIG["gguf:q4_k_m"]["mostly"] = "gguf:q4_k"
-# GGUF_CONFIG["gguf:q5_k"] = GGUF_INNER_CONFIG["gguf:q5_k"]
-# GGUF_CONFIG["gguf:q5_k"]["mostly"]= "gguf:q5_k"
-GGUF_CONFIG["gguf:q5_k_s"] = GGUF_INNER_CONFIG["gguf:q5_k"]
-GGUF_CONFIG["gguf:q5_k_s"]["mostly"] = "gguf:q5_k"
-GGUF_CONFIG["gguf:q5_k_m"] = GGUF_INNER_CONFIG["gguf:q5_k"]
-GGUF_CONFIG["gguf:q5_k_m"]["mostly"] = "gguf:q5_k"
-GGUF_CONFIG["gguf:q6_k"] = GGUF_INNER_CONFIG["gguf:q6_k"]
-GGUF_CONFIG["gguf:q6_k"]["mostly"] = "gguf:q6_k"
-GGUF_CONFIG["gguf:q8_0"] = GGUF_INNER_CONFIG["gguf:q8_0"]
-GGUF_CONFIG["gguf:q8_0"]["mostly"] = "gguf:q8_0"
-# GGUF_CONFIG["gguf:fp16"] = GGUF_INNER_CONFIG["gguf:fp16"]
-# GGUF_CONFIG["gguf:fp16"]["mostly"]= "gguf:fp16"
-# GGUF_CONFIG["gguf:bf16"] = GGUF_INNER_CONFIG["gguf:fp16"]
-# GGUF_CONFIG["gguf:bf16"]["mostly"]= "gguf:bf16"
-GGUF_CONFIG["gguf:q2_k_mixed"] = GGUF_INNER_CONFIG["gguf:q2_k"]
+for alias_key, facts_key in GGUF_PRESET_ALIASES.items():
+ GGUF_CONFIG[alias_key] = GGUF_INNER_CONFIG[facts_key]
+ if alias_key != "gguf:q2_k_mixed":
+ GGUF_CONFIG[alias_key]["mostly"] = facts_key
QK_K = 256
diff --git a/auto_round/export/formats/__init__.py b/auto_round/export/formats/__init__.py
new file mode 100644
index 000000000..d401af8c8
--- /dev/null
+++ b/auto_round/export/formats/__init__.py
@@ -0,0 +1,43 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from auto_round.export.formats.backends import (
+ AutoAWQFormat,
+ AutoGPTQFormat,
+ AutoRoundFormat,
+ FakeFormat,
+ FP8Format,
+ GGUFFormat,
+ LLMCompressorFormat,
+ MLXFormat,
+)
+from auto_round.export.formats.base import (
+ BackendDataType,
+ OutputFormat,
+)
+from auto_round.export.formats.resolver import resolve_formats
+
+__all__ = [
+ "BackendDataType",
+ "AutoRoundFormat",
+ "AutoAWQFormat",
+ "AutoGPTQFormat",
+ "FakeFormat",
+ "FP8Format",
+ "GGUFFormat",
+ "LLMCompressorFormat",
+ "MLXFormat",
+ "OutputFormat",
+ "resolve_formats",
+]
diff --git a/auto_round/export/formats/backends/__init__.py b/auto_round/export/formats/backends/__init__.py
new file mode 100644
index 000000000..263ecbf1d
--- /dev/null
+++ b/auto_round/export/formats/backends/__init__.py
@@ -0,0 +1,33 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from auto_round.export.formats.backends.autoround import AutoRoundFormat
+from auto_round.export.formats.backends.fake import FakeFormat
+from auto_round.export.formats.backends.fp8 import FP8Format
+from auto_round.export.formats.backends.gguf import GGUFFormat
+from auto_round.export.formats.backends.auto_awq import AutoAWQFormat
+from auto_round.export.formats.backends.auto_gptq import AutoGPTQFormat
+from auto_round.export.formats.backends.llm_compressor import LLMCompressorFormat
+from auto_round.export.formats.backends.mlx import MLXFormat
+
+__all__ = [
+ "AutoAWQFormat",
+ "AutoGPTQFormat",
+ "AutoRoundFormat",
+ "FakeFormat",
+ "FP8Format",
+ "GGUFFormat",
+ "LLMCompressorFormat",
+ "MLXFormat",
+]
diff --git a/auto_round/export/formats/backends/auto_awq.py b/auto_round/export/formats/backends/auto_awq.py
new file mode 100644
index 000000000..1ab8c2831
--- /dev/null
+++ b/auto_round/export/formats/backends/auto_awq.py
@@ -0,0 +1,127 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import re
+from typing import Callable, Union
+
+import torch
+import transformers
+
+from auto_round.export.formats.base import OutputFormat, _check_divisible_by_32
+from auto_round.logger import logger
+from auto_round.schemes import QuantizationScheme
+
+
[email protected]("auto_awq")
+class AutoAWQFormat(OutputFormat):
+ support_schemes = ["W4A16"]
+ format_name = "auto_awq"
+
+ @classmethod
+ def check_scheme_args(cls, scheme: QuantizationScheme) -> bool:
+ error_logs = []
+ if scheme.bits != 4:
+ error_logs.append(f"bits={scheme.bits}")
+ if not re.search("int", scheme.data_type):
+ error_logs.append(f"data_type={scheme.data_type}")
+ if scheme.super_bits:
+ error_logs.append(f"super_bits={scheme.super_bits}")
+ if scheme.super_group_size:
+ error_logs.append(f"super_group_size={scheme.super_group_size}")
+ if error_logs:
+ raise ValueError(
+ f"{cls.format_name} format support quantization scheme with {','.join(cls.support_schemes)} "
+ f"but got {', '.join(error_logs)}, please have a check."
+ )
+ return True
+
+ @staticmethod
+ def check_awq_gemm_compatibility(model, bits, group_size, sym, layer_configs=None):
+ """Check whether a model is compatible with the AutoAWQ GEMM kernel."""
+ from auto_round.utils.model import get_layer_names_in_block, get_module
+
+ if bits != 4:
+ return False, "AutoAWQ GEMM kernel only supports 4 bits"
+ for _, module in model.named_modules():
+ if type(module) == transformers.pytorch_utils.Conv1D:
+ return False, "AutoAWQ GEMM kernel does not support conv1d"
+
+ layer_names = get_layer_names_in_block(model)
+ for layer_name in layer_names:
+ if (
+ layer_configs is not None
+ and layer_name in layer_configs.keys()
+ and layer_configs[layer_name].get("bits", bits) > 8
+ ):
+ continue
+
+ layer = get_module(model, layer_name)
+ if layer.in_features % group_size != 0:
+ return False, f"Layer {layer_name} in_features is not multiple of group_size {group_size}"
+ if layer.out_features % (32 // bits) != 0:
+ return False, f"Layer {layer_name} out_features is not multiple of 32 // bits"
+
+ return True, ""
+
+ def check_and_reset_format(self, scheme: QuantizationScheme, ctx):
+ awq_supported, info = self.check_awq_gemm_compatibility(
+ ctx.model, scheme.bits, scheme.group_size, scheme.sym, ctx.layer_config
+ )
+ if not awq_supported:
+ logger.warning(f"The AutoAWQ format may not be supported due to {info}")
+ if scheme.bits != 4:
+ raise ValueError(f"auto_awq format support quantization scheme with W4A16 but got bits={scheme.bits}")
+
+ if self.backend is None:
+ ctx.layer_config = _check_divisible_by_32(scheme, ctx.model, ctx.layer_config)
+
+ return super().check_and_reset_format(scheme, ctx)
+
+ def pack_layer(self, layer_name, model, device=None, **kwargs):
+ from auto_round.export.export_to_awq.export import pack_layer
+
+ pack_layer(layer_name, model, backend=self.output_format, device=device)
+
+ def save_quantized(
+ self,
+ output_dir: str,
+ model: torch.nn.Module = None,
+ tokenizer: Callable = None,
+ layer_config: dict = None,
+ inplace: bool = True,
+ device: Union[str, torch.device] = "cpu",
+ serialization_dict: dict = None,
+ **kwargs,
+ ) -> torch.nn.Module:
+ backend = self.get_backend_name()
+ if backend == "auto_round:auto_awq":
+ from auto_round.export.export_to_autoround.export import save_quantized_as_autoround
+
+ export_func = save_quantized_as_autoround
+ else:
+ from auto_round.export.export_to_awq.export import save_quantized_as_autoawq
+
+ export_func = save_quantized_as_autoawq
+
+ return export_func(
+ output_dir=output_dir,
+ model=model,
+ tokenizer=tokenizer,
+ layer_config=layer_config,
+ inplace=inplace,
+ backend=backend,
+ device=device,
+ serialization_dict=serialization_dict,
+ **kwargs,
+ )
diff --git a/auto_round/export/formats/backends/auto_gptq.py b/auto_round/export/formats/backends/auto_gptq.py
new file mode 100644
index 000000000..4fcaaca61
--- /dev/null
+++ b/auto_round/export/formats/backends/auto_gptq.py
@@ -0,0 +1,100 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import re
+from typing import Callable, Union
+
+import torch
+
+from auto_round.export.formats.base import OutputFormat, _check_divisible_by_32
+from auto_round.logger import logger
+from auto_round.schemes import QuantizationScheme
+
+
[email protected]("auto_gptq", "gptqmodel")
+class AutoGPTQFormat(OutputFormat):
+ support_schemes = ["W4A16", "W2A16", "W3A16", "W8A16", "BF16", "W2A16G64", "W2A16G32", "W4A16_MIXED"]
+ format_name = "auto_gptq"
+
+ def check_and_reset_format(self, scheme: QuantizationScheme, ctx):
+ if not scheme.sym:
+ logger.warning(
+ "the asymmetrical kernel of the GPTQ format may result in a noticeable accuracy drop,"
+ " particularly for 2-bit quantization and smaller models."
+ " We recommend exporting to either the AutoAWQ format ( only 4 bits) or "
+ "the AutoRound format(2/3/4/8 bits)."
+ )
+ if self.backend is None:
+ ctx.layer_config = _check_divisible_by_32(scheme, ctx.model, ctx.layer_config)
+ return super().check_and_reset_format(scheme, ctx)
+
+ @classmethod
+ def check_scheme_args(cls, scheme: QuantizationScheme) -> bool:
+ error_logs = []
+ if scheme.bits not in [2, 3, 4, 8, 16]:
+ error_logs.append(f"bits={scheme.bits}")
+ if not re.search("int", scheme.data_type):
+ error_logs.append(f"data_type={scheme.data_type}")
+ if scheme.super_bits:
+ error_logs.append(f"super_bits={scheme.super_bits}")
+ if scheme.super_group_size:
+ error_logs.append(f"super_group_size={scheme.super_group_size}")
+ if error_logs:
+ raise ValueError(
+ f"{cls.format_name} format support quantization scheme with {','.join(cls.support_schemes)} "
+ f"but got {', '.join(error_logs)}, please have a check."
+ )
+ return True
+
+ def pack_layer(self, layer_name, model, device=None, **kwargs):
+ if self.output_format.startswith("auto_round"):
+ from auto_round.export.export_to_autoround.export import pack_layer
+
+ pack_layer(layer_name, model, backend=self.output_format, device=device)
+ else:
+ from auto_round.export.export_to_autogptq.export import pack_layer
+
+ pack_layer(layer_name, model, backend=self.output_format, device=device)
+
+ def save_quantized(
+ self,
+ output_dir: str,
+ model: torch.nn.Module = None,
+ tokenizer: Callable = None,
+ layer_config: dict = None,
+ inplace: bool = True,
+ device: Union[str, torch.device] = "cpu",
+ serialization_dict: dict = None,
+ **kwargs,
+ ) -> torch.nn.Module:
+ backend = self.get_backend_name()
+ if backend == "auto_round:auto_gptq" or backend == "auto_round:gptqmodel":
+ from auto_round.export.export_to_autoround.export import save_quantized_as_autoround
+
+ export_func = save_quantized_as_autoround
+ else:
+ from auto_round.export.export_to_autogptq.export import save_quantized_as_autogptq
+
+ export_func = save_quantized_as_autogptq
+ return export_func(
+ output_dir=output_dir,
+ model=model,
+ tokenizer=tokenizer,
+ layer_config=layer_config,
+ inplace=inplace,
+ device=device,
+ backend=backend,
+ serialization_dict=serialization_dict,
+ **kwargs,
+ )
diff --git a/auto_round/export/formats/backends/autoround.py b/auto_round/export/formats/backends/autoround.py
new file mode 100644
index 000000000..21ab8daaa
--- /dev/null
+++ b/auto_round/export/formats/backends/autoround.py
@@ -0,0 +1,217 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import re
+from typing import Any, Callable, Union
+
+import torch
+
+from auto_round.export.formats.backends.auto_awq import AutoAWQFormat
+from auto_round.export.formats.backends.auto_gptq import AutoGPTQFormat
+from auto_round.export.formats.backends.mlx import MLXFormat
+from auto_round.export.formats.base import BackendDataType, OutputFormat, _check_divisible_by_32
+from auto_round.logger import logger
+from auto_round.schemes import QuantizationScheme
+
+
[email protected]("auto_round")
[email protected]("auto_round:auto_awq")
[email protected]("auto_round:llm_compressor")
[email protected]("auto_round:gptqmodel", "auto_round:auto_gptq")
[email protected]("auto_round:fp8")
[email protected]("auto_round:mlx")
+class AutoRoundFormat(OutputFormat):
+ support_schemes = [
+ "W4A16",
+ "W4A16_MIXED",
+ "W2A16",
+ "W3A16",
+ "W8A16",
+ "MXFP4",
+ "MXFP8",
+ "NVFP4",
+ "FPW8A16",
+ "W2A16G64",
+ "W2A16G32",
+ "FP8_STATIC",
+ "BF16",
+ "FP8_BLOCK",
+ "MXINT4",
+ ]
+ format_name = "auto_round"
+
+ def __init__(self, format: str, scheme: QuantizationScheme, ctx: Any):
+ self.output_format = "auto_round"
+ self.backend = None
+
+ if format == "auto_round":
+ if scheme.sym and "int" in scheme.data_type and "mx" not in scheme.data_type:
+ self.backend = AutoGPTQFormat("auto_round:auto_gptq", scheme, ctx)
+ elif scheme.bits == 4 and not scheme.sym and "int" in scheme.data_type:
+ if ctx.layer_config is None:
+ enable_awq = True
+ else:
+ enable_awq = all(
+ config["bits"] == scheme.bits or config["bits"] >= 16 for config in ctx.layer_config.values()
+ )
+ if enable_awq:
+ self.backend = AutoAWQFormat("auto_round:auto_awq", scheme, ctx)
+ elif scheme.is_nv_fp() or scheme.is_mx_fp():
+ self.backend = AutoRoundFormat(scheme.data_type, scheme, ctx)
+ elif scheme.is_mx_int() and scheme.bits == 4: # only add mx_int4 now
+ self.backend = AutoRoundFormat(scheme.data_type, scheme, ctx)
+ elif scheme.is_act_static(): # static wfp8afp8
+ self.backend = AutoRoundFormat(BackendDataType.FP8_STATIC.value, scheme, ctx)
+ elif scheme.data_type.startswith("fp") and scheme.bits == 8 and scheme.act_bits >= 16: # woq fp8
+ self.backend = AutoRoundFormat(BackendDataType.FP8.value, scheme, ctx)
+ elif scheme.data_type.startswith("fp") and scheme.bits == 8 and isinstance(scheme.group_size, tuple):
+ self.backend = AutoRoundFormat("auto_round:fp8", scheme, ctx)
+ elif scheme.act_bits < 16:
+ raise ValueError(
+ "AutoRound format does not support exporting "
+ "for the current quantization configuration, "
+ "please change to `fake` format for research purpose"
+ )
+ # for auto_round:fp8_static, auto_round:nv_fp etc.
+ elif not format.startswith("auto_round"):
+ if format == "mlx":
+ self.backend = MLXFormat("mlx", scheme, ctx)
+ elif format.upper() not in list(BackendDataType.__members__.keys()):
+ raise KeyError(f"Unsupported backend format auto_round:{format}, please check")
+ else:
+ self.output_format = f"auto_round:{format}"
+ self.backend = None
+ elif format == "auto_round:mlx":
+ self.backend = MLXFormat("mlx", scheme, ctx)
+ else:
+ backend = format.split(":")[1] if ":" in format else None
+ self.backend = self._format_list.get(backend)(format, scheme, ctx) if backend else None
+
+ if self.backend is not None:
+ self.support_schemes = self.backend.support_schemes
+
+ def check_and_reset_format(self, scheme: QuantizationScheme, ctx: Any):
+ layer_config, quant_block_list = ctx.layer_config, ctx.quant_block_list
+ if self.backend is not None:
+ new_format, scheme, layer_config, quant_block_list = self.backend.check_and_reset_format(scheme, ctx)
+ ctx.layer_config, ctx.quant_block_list = layer_config, quant_block_list
+ self.backend = (
+ OutputFormat._format_list[new_format](new_format, scheme, ctx) if new_format else self.backend
+ )
+
+ if scheme.act_bits <= 8:
+ if scheme.is_act_standard_fp() and not scheme.act_dynamic:
+ if (
+ scheme.act_group_size != 0
+ and not scheme.act_dynamic
+ and self.get_backend_name() == f"auto_round:{BackendDataType.FP8.value}"
+ ):
+ logger.warning(
+ f"Please note that quantize activation with act_group_size={scheme.act_group_size}"
+ " may result in failure to export or import normally."
+ )
+ if self.backend is None:
+ layer_config = _check_divisible_by_32(scheme, ctx.model, layer_config)
+ return None, scheme, layer_config, quant_block_list
+
+ def pack_layer(self, layer_name, model, device=None, **kwargs):
+ if self.backend is not None:
+ return self.backend.pack_layer(layer_name, model, device=device, **kwargs)
+
+ backend = self.get_backend_name()
+
+ if self.output_format in [
+ f"auto_round:{BackendDataType.NV_FP.value}",
+ f"auto_round:{BackendDataType.MX_FP.value}",
+ f"auto_round:{BackendDataType.MX_FP_RCEIL.value}",
+ f"auto_round:{BackendDataType.NV_FP4_WITH_STATIC_GS.value}",
+ ]:
+ from auto_round.export.export_to_autoround.export_to_nvfp_mx import pack_layer
+
+ pack_func = pack_layer
+ elif self.output_format in [f"auto_round:{BackendDataType.MX_INT.value}"]:
+ from auto_round.export.export_to_autoround.export_to_nvfp_mx import pack_layer
+
+ pack_func = pack_layer
+ elif self.output_format in [
+ f"auto_round:{BackendDataType.FP8.value}",
+ f"auto_round:{BackendDataType.FP8_STATIC.value}",
+ f"auto_round:{BackendDataType.FP8_STATIC.value}",
+ ]:
+ from auto_round.export.export_to_autoround.export_to_fp8 import pack_layer
+
+ pack_func = pack_layer
+ else:
+ from auto_round.export.export_to_autoround.export import pack_layer
+
+ pack_func = pack_layer
+ return pack_func(layer_name, model, backend, device)
+
+ def save_quantized(
+ self,
+ output_dir: str,
+ model: torch.nn.Module = None,
+ tokenizer: Callable = None,
+ layer_config: dict = None,
+ inplace: bool = True,
+ device: Union[str, torch.device] = "cpu",
+ serialization_dict: dict = None,
+ **kwargs,
+ ) -> torch.nn.Module:
+ if self.backend is not None:
+ extra_kwargs = {}
+ if isinstance(self.backend, MLXFormat):
+ extra_kwargs["autoround_format"] = True
+ return self.backend.save_quantized(
+ output_dir=output_dir,
+ model=model,
+ tokenizer=tokenizer,
+ layer_config=layer_config,
+ inplace=inplace,
+ device=device,
+ serialization_dict=serialization_dict,
+ **extra_kwargs,
+ **kwargs,
+ )
+ backend = self.get_backend_name()
+ if re.search(f"{BackendDataType.MX_FP.value}|{BackendDataType.NV_FP.value}", backend):
+ from auto_round.export.export_to_autoround.export_to_nvfp_mx import save_quantized_as_fp
+
+ backend = "auto_round:llm_compressor"
+ export_func = save_quantized_as_fp
+ elif serialization_dict.get("data_type", "int") == "fp" and serialization_dict.get("bits", 16) == 8:
+ from auto_round.export.export_to_autoround.export_to_fp8 import save_quantized_as_autoround
+
+ backend = "auto_round:fp8_static" if serialization_dict.get("act_bits", 16) == 8 else None
+ export_func = save_quantized_as_autoround
+ elif re.search(f"{BackendDataType.MX_INT.value}", backend):
+ from auto_round.export.export_to_autoround.export_to_nvfp_mx import save_quantized_as_fp
+
+ backend = "auto_round"
+ export_func = save_quantized_as_fp
+ else:
+ from auto_round.export.export_to_autoround.export import save_quantized_as_autoround
+
+ export_func = save_quantized_as_autoround
+ return export_func(
+ output_dir=output_dir,
+ model=model,
+ tokenizer=tokenizer,
+ layer_config=layer_config,
+ inplace=inplace,
+ device=device,
+ backend=backend,
+ serialization_dict=serialization_dict,
+ **kwargs,
+ )
diff --git a/auto_round/export/formats/backends/fake.py b/auto_round/export/formats/backends/fake.py
new file mode 100644
index 000000000..6c1e9a703
--- /dev/null
+++ b/auto_round/export/formats/backends/fake.py
@@ -0,0 +1,65 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Any, Callable, Union
+
+import torch
+
+from auto_round.export.formats.base import OutputFormat
+from auto_round.logger import logger
+from auto_round.schemes import QuantizationScheme
+from auto_round.utils import copy_python_files_from_model_cache, unsupported_meta_device
+
+
+class FakeFormat(OutputFormat):
+ support_schemes = None
+ format_name = "fake"
+
+ def check_and_reset_format(
+ self, scheme: QuantizationScheme, ctx: Any
+ ) -> tuple[None, QuantizationScheme, dict, list]:
+ return None, scheme, ctx.layer_config, ctx.quant_block_list
+
+ # fake format will not execute pack_layer.
+ def pack_layer(self, *args, **kwargs):
+ pass
+
+ def save_quantized(
+ self,
+ output_dir: str,
+ model: torch.nn.Module = None,
+ tokenizer: Callable = None,
+ layer_config: dict = None,
+ inplace: bool = True,
+ device: Union[str, torch.device] = "cpu",
+ serialization_dict: dict = None,
+ **kwargs,
+ ):
+ if not unsupported_meta_device(model):
+ model = model.to("cpu")
+ model.save_pretrained(output_dir)
+ elif hasattr(model, "config") and model.config is not None:
+ model.config.save_pretrained(output_dir)
+
+ if tokenizer is not None and hasattr(tokenizer, "save_pretrained"):
+ tokenizer.save_pretrained(output_dir)
+ processor = kwargs.get("processor", None)
+ if processor is not None:
+ processor.save_pretrained(output_dir)
+ try:
+ copy_python_files_from_model_cache(model, output_dir)
+ except Exception as e:
+ logger.warning("Skipping source model Python file copy due to error: %s", e)
+ return model
diff --git a/auto_round/export/formats/backends/fp8.py b/auto_round/export/formats/backends/fp8.py
new file mode 100644
index 000000000..1ceea2089
--- /dev/null
+++ b/auto_round/export/formats/backends/fp8.py
@@ -0,0 +1,92 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Callable, Union
+
+import torch
+
+from auto_round.export.formats.base import OutputFormat
+from auto_round.schemes import QuantizationScheme
+
+
+class FP8Format(OutputFormat):
+ support_schemes = ["FP8_BLOCK"]
+ format_name = "fp8"
+
+ @classmethod
+ def check_scheme_args(cls: OutputFormat, scheme: QuantizationScheme) -> bool:
+ error_logs = []
+ if scheme.bits != 8:
+ error_logs.append(f"bits={scheme.bits}")
+ if scheme.data_type != "fp":
+ error_logs.append(f"data_type={scheme.data_type}")
+ if not isinstance(scheme.group_size, tuple):
+ error_logs.append(f"group_size={scheme.group_size}")
+ if not scheme.act_dynamic:
+ error_logs.append(f"act_dynamic={scheme.act_dynamic}")
+ if not isinstance(scheme.act_group_size, int):
+ error_logs.append(f"act_group_size={scheme.act_group_size}")
+ if scheme.act_bits != 8:
+ error_logs.append(f"act_bits={scheme.act_bits}")
+ if scheme.act_data_type != "fp":
+ error_logs.append(f"act_data_type={scheme.act_data_type}")
+ if error_logs:
+ raise ValueError(
+ f"{cls.format_name} format support quantization scheme with {','.join(cls.support_schemes)} "
+ f"but got {', '.join(error_logs)}, please have a check."
+ )
+ return True
+
+ def pack_layer(self, layer_name, model, device=None, **kwargs):
+ from auto_round.export.export_to_autoround.export_to_fp8 import pack_layer
+
+ pack_layer(layer_name, model, self.output_format, device=device)
+
+ def save_quantized(
+ self,
+ output_dir: str,
+ model: torch.nn.Module = None,
+ tokenizer: Callable = None,
+ layer_config: dict = None,
+ inplace: bool = True,
+ device: Union[str, torch.device] = "cpu",
+ serialization_dict: dict = None,
+ **kwargs,
+ ) -> torch.nn.Module:
+ from auto_round.export.export_to_autoround.export_to_fp8 import save_quantized_as_autoround
+
+ backend = self.get_backend_name()
+
+ if isinstance(serialization_dict["group_size"], tuple):
+ serialization_dict["weight_block_size"] = serialization_dict["group_size"]
+
+ ignored_layers = []
+ for layer_name, cfg in layer_config.items():
+ if cfg["bits"] >= 16 and cfg["act_bits"] >= 16:
+ ignored_layers.append(layer_name)
+ if len(ignored_layers) > 0:
+ serialization_dict["ignored_layers"] = ignored_layers
+
+ return save_quantized_as_autoround(
+ output_dir=output_dir,
+ model=model,
+ tokenizer=tokenizer,
+ layer_config=layer_config,
+ inplace=inplace,
+ device=device,
+ serialization_dict=serialization_dict,
+ quant_method=backend,
+ **kwargs,
+ )
diff --git a/auto_round/export/formats/backends/gguf.py b/auto_round/export/formats/backends/gguf.py
new file mode 100644
index 000000000..cccb86719
--- /dev/null
+++ b/auto_round/export/formats/backends/gguf.py
@@ -0,0 +1,801 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import copy
+import os
+import re
+from dataclasses import fields
+from typing import Any, Callable, Union
+
+import torch
+import transformers
+
+from auto_round.compressors.config_resolution.contracts import thaw_mapping
+from auto_round.compressors.config_resolution.errors import FormatCompatibilityError
+from auto_round.export.export_to_gguf.config import GGML_QUANT_SIZES, GGUF_CONFIG, GGUF_INNER_CONFIG, QK_K, ModelType
+from auto_round.export.export_to_gguf.gguf_dtype import GGUFDTypeSelector, gguf_format_to_ftype
+from auto_round.export.formats.base import OutputFormat
+from auto_round.logger import logger
+from auto_round.schemes import QuantizationScheme
+from auto_round.utils import check_to_quantized, find_matching_blocks, get_block_names, get_module
+
+
+class GGUFFormat(OutputFormat):
+ support_schemes = [
+ "GGUF:Q4_0",
+ "GGUF:Q4_1",
+ "GGUF:Q5_0",
+ "GGUF:Q5_1",
+ "GGUF:Q2_K_S",
+ "GGUF:Q3_K_S",
+ "GGUF:Q3_K_M",
+ "GGUF:Q3_K_L",
+ "GGUF:Q4_K_S",
+ "GGUF:Q4_K_M",
+ "GGUF:Q5_K_S",
+ "GGUF:Q5_K_M",
+ "GGUF:Q6_K",
+ "GGUF:Q8_0",
+ "GGUF:Q2_K_MIXED",
+ ]
+ format_name = "gguf"
+
+ def __init__(self, format: str, scheme: QuantizationScheme, ctx: Any):
+ self.is_auto_scheme = ctx.is_auto_scheme
+ layer_config = ctx.layer_config
+ if format.startswith("gguf:"):
+ self._original_format = format # preserve "gguf:q2_k_mixed" etc. for Phase 2b
+ self.output_format = "gguf"
+ self.backend_cls = GGUFFormat
+ self.backend, scheme, layer_config = GGUFFormat.build(format.split(":")[-1], scheme, ctx)
+ else:
+ gguf_format = f"gguf:{format.lower()}"
+ if format.lower().endswith("_mixed"):
+ from auto_round.schemes import _handle_special_schemes
+ from auto_round.utils.model import is_moe_model
+
+ if format.lower() == "q2_k_mixed" and (ctx.iters or 0) > 0 and not is_moe_model(ctx.model):
+ logger.warning(
+ "gguf:q2_k_mixed only supports MoE models with iters>0. "
+ "It is not an MoE model, falling back to gguf:q2_k_s."
+ )
+ gguf_format = "gguf:q2_k_s"
+ else:
+ layer_config = _handle_special_schemes(
+ gguf_format, layer_config, ctx.model, quant_nontext_module=ctx.quant_nontext_module
+ )
+ gguf_format = gguf_format.lower().replace("_mixed", "_s")
+ if isinstance(scheme, str) and scheme.lower() != gguf_format:
+ # Defensive: no live call site passes a raw string scheme here today, but this
+ # is kept for external callers that construct OutputFormat before scheme resolution.
+ logger.warning(f"reset scheme {scheme.lower()} to {gguf_format} for gguf format export")
+ scheme = gguf_format
+ self.output_format = gguf_format
+ self.backend = None
+ from auto_round.schemes import GGUF_PRESET_ALIASES, GGUF_SCHEME_FACTS
+
+ facts_name = GGUF_PRESET_ALIASES.get(gguf_format)
+ if facts_name is not None:
+ scheme = copy.deepcopy(scheme)
+ for key, value in GGUF_SCHEME_FACTS[facts_name].items():
+ setattr(scheme, key, value)
+ self.mllm = ctx.mllm
+ self._resolved_layer_config = layer_config
+ self._resolved_scheme = scheme
+
+ @classmethod
+ def build(
+ cls,
+ format: str,
+ scheme: QuantizationScheme,
+ ctx: Any,
+ ) -> tuple["GGUFFormat", QuantizationScheme, dict]:
+ """Construct a GGUFFormat and surface the (possibly corrected) scheme/layer_config.
+
+ Plain ``__init__`` cannot report a corrected ``scheme``/``layer_config`` back to its
+ caller, but GGUF resolution can replace either (the ``_mixed`` -> ``_s`` rewrite, and the
+ legacy string-scheme correction above). ``resolve_formats`` calls this instead of the bare
+ constructor for every ``gguf:``-prefixed dispatch.
+ """
+ instance = cls(format, scheme, ctx)
+ return instance, instance._resolved_scheme, instance._resolved_layer_config
+
+ @classmethod
+ def check_scheme_args(cls: OutputFormat, scheme: QuantizationScheme) -> bool:
+ error_logs = []
+ if not re.search("int", scheme.data_type):
+ error_logs.append(f"data_type={scheme.data_type}")
+ if error_logs:
+ raise ValueError(
+ f"{cls.format_name} format support quantization scheme with {','.join(cls.support_schemes)} "
+ f"but got {', '.join(error_logs)}, please have a check."
+ )
+ return True
+
+ def check_and_reset_format(self, scheme: QuantizationScheme, ctx: Any):
+ if ctx.iters != 0 and scheme.bits != 3 and not ctx.enable_alg_ext:
+ logger.warning_once(
+ "`iters=0` is recommended when exporting to current GGUF format"
+ " or add `enable_alg_ext` for better accuracy with much more tuning cost."
+ " Please refer to https://github.com/intel/auto-round/tree/main/docs/gguf_alg_ext_acc.md"
+ " for the accuracy results."
+ )
+ elif scheme.bits >= 8 and ctx.iters != 0:
+ logger.warning_once("`iters=0` is recommended for bits>=8")
+
+ if ctx.quant_nontext_module:
+ # for gguf export, leave vl model for gguf itself
+ all_blocks = get_block_names(ctx.model, False)
+ ctx.quant_block_list = find_matching_blocks(ctx.model, all_blocks, None)
+ return super().check_and_reset_format(scheme, ctx)
+
+ def pack_layer(
+ self,
+ name,
+ model,
+ backend,
+ output_dir,
+ layer_config,
+ tokenizer,
+ processor=None,
+ image_processor=None,
+ model_type=ModelType.TEXT,
+ device="cpu",
+ quant_nontext_module=False,
+ ):
+ from auto_round.export.export_to_gguf.export import pack_gguf_layer
+
+ pack_gguf_layer(
+ name,
+ model,
+ backend,
+ output_dir,
+ layer_config,
+ tokenizer,
+ processor,
+ image_processor,
+ model_type,
+ device,
+ quant_nontext_module,
+ is_auto_scheme=self.is_auto_scheme,
+ )
+
+ def save_quantized(
+ self,
+ output_dir: str,
+ model: torch.nn.Module = None,
+ tokenizer: Callable = None,
+ layer_config: dict = None,
+ inplace: bool = True,
+ device: Union[str, torch.device] = "cpu",
+ serialization_dict: dict = None,
+ **kwargs,
+ ) -> torch.nn.Module:
+ from auto_round.export.export_to_gguf.export import save_quantized_as_gguf
+
+ backend = self.get_backend_name()
+ return save_quantized_as_gguf(
+ output_dir=output_dir,
+ model=model,
+ backend=backend,
+ layer_config=layer_config,
+ mllm=self.mllm,
+ device=device,
+ serialization_dict=serialization_dict,
+ is_auto_scheme=self.is_auto_scheme,
+ **kwargs,
+ )
+
+ @staticmethod
+ def gguf_args_check(
+ scheme: QuantizationScheme,
+ model,
+ platform: str,
+ formats: Union[str, list[str]] = None,
+ model_type=ModelType.TEXT,
+ ) -> QuantizationScheme:
+ import argparse
+
+ from auto_round.export.export_to_gguf.config import GGUF_CONFIG
+ from auto_round.export.export_to_gguf.llama_cpp_conversion import get_conversion
+ from auto_round.logger import logger
+ from auto_round.utils.model import download_or_get_path, get_gguf_architecture
+
+ formats = [formats] if isinstance(formats, str) else formats
+ formats = sorted(formats, key=lambda x: len(x))
+ export_gguf = False
+ for f in formats:
+ if f.startswith("gguf"):
+ export_gguf = True
+
+ if f.startswith("gguf") and f not in GGUF_CONFIG:
+ logger.error(f"{f} is not supported, please check.")
+
+ if export_gguf:
+ if isinstance(model, str):
+ model_path = model
+ else:
+ model_path = model.name_or_path
+ if not os.path.isdir(model_path):
+ model_path = download_or_get_path(model_path, platform)
+ try:
+ conversion = get_conversion(model_path, model_type=ModelType.TEXT)
+ except AttributeError as e:
+ print(f"[gguf conversion] error: {e}")
+ raise ImportError(
+ "Please use the latest gguf-py, you can use the following command to install it:\n"
+ "git clone https://github.com/ggml-org/llama.cpp.git && cd llama.cpp/gguf-py"
+ " && pip install . sentencepiece"
+ )
+ model_architecture = get_gguf_architecture(model_path, model_type=ModelType.TEXT)
+ if not conversion.is_supported(model_architecture, ModelType.TEXT):
+ raise FormatCompatibilityError(f"Model {model_architecture} is not supported to export gguf format.")
+
+ pattern = re.compile(r"q\d_k")
+ pre_dq_format = ""
+ unsupported_list, reset_list = [], []
+ for format in GGUF_CONFIG:
+ if format in formats:
+ if format == "q6_k_s":
+ logger.warning("Please note that q6_k_s is q6_k.")
+
+ if re.search(pattern, format):
+ if pre_dq_format and re.search(pattern, format).group() not in pre_dq_format:
+ raise FormatCompatibilityError(f"Cannot export {pre_dq_format} and {format} at the same time.")
+ else:
+ pre_dq_format = format
+
+ unsupported_list, reset_list = [], []
+ gguf_config = GGUF_CONFIG[format]
+ for k, v in gguf_config.items():
+ if not hasattr(scheme, k):
+ continue
+ if k == "data_type":
+ if re.search(r"q\d_1", format) and len(formats) > 1:
+ v = "int"
+ # No caller passes an argparse.Namespace here; kept for dual-mode parity.
+ if k == "sym" and isinstance(scheme, argparse.Namespace):
+ k = "asym"
+ v = not v
+ if getattr(scheme, k) != v:
+ unsupported_list.append(f"{k}={getattr(scheme, k)}")
+ reset_list.append(f"{k}={v}")
+ setattr(scheme, k, v)
+ if len(unsupported_list) > 0:
+ logger.info(
+ f"format {format} does not support for {', '.join(unsupported_list)},"
+ f" reset to {', '.join(reset_list)}."
+ )
+ return scheme
+
+ def immediate_pack(
+ self,
+ name: str,
+ model: torch.nn.Module,
+ device: torch.device,
+ output_dir: str = None,
+ mllm: bool = False,
+ layer_config: dict = None,
+ tokenizer=None,
+ processor=None,
+ image_processor=None,
+ quant_nontext_module: bool = False,
+ **kwargs,
+ ):
+ m = get_module(model, name)
+ if not check_to_quantized(m):
+ return
+ model_type = ModelType.MMPROJ if mllm else ModelType.TEXT
+ self.pack_layer(
+ name,
+ model,
+ self.get_backend_name(),
+ output_dir,
+ layer_config=layer_config,
+ tokenizer=tokenizer,
+ processor=processor,
+ image_processor=image_processor,
+ model_type=model_type,
+ device=device,
+ quant_nontext_module=quant_nontext_module,
+ )
+
+
+def apply_gguf_layer_defaults(
+ layer_config,
+ model,
+ gguf_name,
+ lm_head_name,
+ tie_word_embeddings,
+ embedding_layer_names,
+ default_scale_dtype,
+ enable_gguf_official_mixed,
+ is_mllm,
+ has_qlayer_outside_block,
+) -> tuple[dict, bool]:
+ """Step 10 of layer-config resolution: embed/lm_head GGUF defaults plus the
+ GGUF-format-specific per-layer mapping. Only called when `gguf_name` is truthy."""
+ from auto_round.utils.model import is_separate_lm_head
+
+ # embed + lm_head defaults for gguf
+ tie_word_embeddings &= not is_separate_lm_head(model)
+ if lm_head_name not in layer_config and not tie_word_embeddings:
+ cfg = GGUF_INNER_CONFIG[GGUF_CONFIG[gguf_name.lower()]["lm_head"]]
+ cfg = {**cfg, "fixed_by_user": False, "scale_dtype": default_scale_dtype}
+ layer_config[lm_head_name] = cfg
+ has_qlayer_outside_block = True
+ for emd_name in embedding_layer_names:
+ if emd_name in layer_config:
+ continue
+ if not tie_word_embeddings:
+ cfg = GGUF_INNER_CONFIG[GGUF_CONFIG[gguf_name.lower()]["embedding"]]
+ else:
+ cfg = GGUF_INNER_CONFIG[GGUF_CONFIG[gguf_name.lower()]["lm_head"]]
+ cfg = {**cfg, "fixed_by_user": False, "scale_dtype": default_scale_dtype}
+ layer_config[emd_name] = cfg
+
+ if enable_gguf_official_mixed:
+ model_type = ModelType.MMPROJ if is_mllm else ModelType.TEXT
+ layer_config, _ = get_layer_config_by_gguf_format(layer_config, gguf_name.lower(), model, model_type)
+
+ _apply_gguf_shape_fallback(layer_config, model)
+ return layer_config, has_qlayer_outside_block
+
+
+def _use_more_bits(i_layer: int, n_layer: int):
+ return (i_layer < n_layer // 8) or (i_layer >= 7 * n_layer // 8) or ((i_layer - n_layer // 8) % 3 == 2)
+
+
+def _search_gguf_type(gguf_type):
+ if gguf_type in GGUF_INNER_CONFIG:
+ return gguf_type
+ pattern = re.compile("gguf:q([0-9]{1,})_[01k]")
+ bits = re.search(pattern, gguf_type)
+ if not bits:
+ raise KeyError(f"{gguf_type} is not a correct gguf type, please check")
+
+ for suffix in ["_k", "_0", "_1"]:
+ if gguf_type.endswith(suffix):
+ continue
+ if (tmp_type := re.sub("_[01k]", suffix, gguf_type)) in GGUF_INNER_CONFIG:
+ return tmp_type
+ return None
+
+
+def gguf_type_fallback(gguf_type: str) -> str:
+ gguf_type = gguf_type.lower()
+ if gguf_type in ("gguf:q2_k", "gguf:q3_k", "gguf:q4_k"):
+ gguf_type = "gguf:q5_0"
+ elif gguf_type == "gguf:q5_k":
+ gguf_type = "gguf:q5_0"
+ elif gguf_type == "gguf:q6_k":
+ gguf_type = "gguf:q8_0"
+ return gguf_type
+
+
+def get_gguf_qtype_by_layer_config(layer_config):
+ import gguf # pylint: disable=E0401
+
+ if layer_config["bits"] >= 16:
+ return None
+ bits = layer_config["bits"]
+ super_bits = layer_config.get("super_bits", None)
+ sym = layer_config["sym"]
+ group_size = layer_config.get("group_size", None)
+ super_group_size = layer_config.get("super_group_size", None)
+ if bits == 2 and super_bits == 4 and not sym and group_size == 16 and super_group_size == 16:
+ return gguf.GGMLQuantizationType.Q2_K
+ if bits == 3 and super_bits == 6 and sym and group_size == 16 and super_group_size == 16:
+ return gguf.GGMLQuantizationType.Q3_K
+ if bits == 4:
+ if super_bits is not None and super_bits == 6 and not sym and group_size == 32 and super_group_size == 8:
+ return gguf.GGMLQuantizationType.Q4_K
+ if super_bits is None and sym and group_size == 32:
+ return gguf.GGMLQuantizationType.Q4_0
+ if super_bits is None and not sym and group_size == 32:
+ return gguf.GGMLQuantizationType.Q4_1
+ if bits == 5:
+ if super_bits == 6 and not sym and group_size == 32 and super_group_size == 8:
+ return gguf.GGMLQuantizationType.Q5_K
+ if super_bits is None and sym and group_size == 32:
+ return gguf.GGMLQuantizationType.Q5_0
+ if super_bits is None and not sym and group_size == 32:
+ return gguf.GGMLQuantizationType.Q5_1
+ if bits == 6 and super_bits == 8 and group_size == 16 and super_group_size == 16:
+ return gguf.GGMLQuantizationType.Q6_K
+ if bits == 8 and sym and group_size == 32:
+ return gguf.GGMLQuantizationType.Q8_0
+ raise ValueError("Unknown layer config")
+
+
+def _apply_gguf_shape_fallback(layer_config, model):
+ from auto_round.utils.model import get_module
+
+ for layer_name, config in layer_config.items():
+ if not check_to_quantized(config):
+ continue
+ layer = get_module(model, layer_name)
+ if layer is None or not hasattr(layer, "weight"):
+ continue
+ try:
+ qtype = get_gguf_qtype_by_layer_config(config)
+ except ValueError:
+ continue
+ if qtype is None:
+ continue
+
+ gguf_type = f"gguf:{qtype.name.lower()}"
+ block_size = GGML_QUANT_SIZES[gguf_type.split(":")[-1]][0]
+ input_features = (
+ layer.weight.shape[0] if type(layer) == transformers.pytorch_utils.Conv1D else layer.weight.shape[-1]
+ )
+ if input_features % block_size == 0:
+ continue
+
+ fallback_type = _gguf_type_fallback(gguf_type)
+ fallback_block_size = GGML_QUANT_SIZES[fallback_type.split(":")[-1]][0]
+ if input_features % fallback_block_size != 0:
+ fallback_type = "gguf:bf16"
+
+ preserved = {key: config[key] for key in ("fixed_by_user", "scale_dtype") if key in config}
+ config.update(GGUF_INNER_CONFIG[fallback_type])
+ config.update(preserved)
+ logger.warning(
+ "fallback %s to %s before quantization, because input_features(%s) is not divisible by %s block_size(%s)",
+ layer_name,
+ fallback_type,
+ input_features,
+ gguf_type,
+ block_size,
+ )
+
+
+def _get_digital_in_layer_name(layer_name):
+ pattern = re.compile(r"([a-zA-Z]+\.){1,}(\d+)")
+ res = re.search(pattern, layer_name)
+ if res:
+ return int(res[2])
+ else:
+ return None
+
+
+def _gguf_type_fallback(gguf_type: str) -> str:
+ gguf_type = gguf_type.lower()
+ if gguf_type in ("gguf:q2_k", "gguf:q3_k", "gguf:q4_k"):
+ gguf_type = "gguf:q5_0"
+ elif gguf_type == "gguf:q5_k":
+ gguf_type = "gguf:q5_0"
+ elif gguf_type == "gguf:q6_k":
+ gguf_type = "gguf:q8_0"
+ return gguf_type
+
+
+def _select_gguf_layer_type(
+ layer_name,
+ config,
+ layer,
+ model,
+ model_class,
+ *,
+ target_gguf_format,
+ lm_head_name,
+ tie_word_embeddings,
+ block_size,
+ base_target_bits,
+ dtype_selector,
+ gguf_name,
+ i_layer,
+) -> str | None:
+ """Per-layer GGUF-type-selection cascade extracted from
+ `get_layer_config_by_gguf_format`'s loop body.
+
+ Returns the selected gguf type string, or ``None`` as the sentinel for the
+ `lm_head_name == layer_name and tie_word_embeddings` skip case, which the
+ caller must handle with `continue`.
+ """
+ import gguf # pylint: disable=E0401
+
+ from auto_round.schemes import QuantizationScheme, get_gguf_scheme
+
+ if type(layer) == transformers.pytorch_utils.Conv1D:
+ input_features = layer.weight.shape[0]
+ else:
+ input_features = layer.weight.shape[-1]
+
+ # Reset target_bits each iteration to prevent lm_head/embedding settings
+ # from bleeding into subsequent block layers and bypassing their special logic.
+ target_bits = base_target_bits
+ new_type = GGUF_CONFIG[target_gguf_format]["mostly"]
+
+ if lm_head_name is not None and layer_name == lm_head_name:
+ target_bits = int(re.search("gguf:q([0-9]{1,})_[01k]", GGUF_CONFIG[target_gguf_format]["lm_head"]).group(1))
+ if isinstance(layer, torch.nn.Embedding):
+ embedding_format_key = "lm_head" if tie_word_embeddings else "embedding"
+ target_bits = int(
+ re.search("gguf:q([0-9]{1,})_[01k]", GGUF_CONFIG[target_gguf_format][embedding_format_key]).group(1)
+ )
+
+ bits_index = 6
+ if config.get("fixed_by_user", False):
+ if "bits" not in config:
+ logger.warning(
+ f"Setting layer_config requires providing bits, {layer_name} has not bits,"
+ f" using bits={target_bits} instead."
+ )
+ new_type = new_type[:bits_index] + str(target_bits) + new_type[bits_index + 1 :]
+ else:
+ config_tmp = config.copy()
+ scheme_keys = [f.name for f in fields(QuantizationScheme)]
+ for key in config.keys():
+ if key not in scheme_keys:
+ config_tmp.pop(key, None)
+ matched_scheme = get_gguf_scheme(QuantizationScheme.from_dict(config_tmp)) # check matched
+ if not matched_scheme:
+ if config.get("super_group_size", None) is not None or config.get("super_bits", None) is not None:
+ new_type = new_type[:bits_index] + str(config["bits"]) + "_k"
+ if new_type not in GGUF_INNER_CONFIG:
+ prefix_idx = 0 if config.get("sym", True) else 1
+ new_type = new_type[:bits_index] + str(config["bits"]) + f"_{prefix_idx}"
+ if new_type not in GGUF_INNER_CONFIG:
+ new_type = new_type[:bits_index] + str(config["bits"]) + f"_{1-prefix_idx}"
+ if new_type not in GGUF_INNER_CONFIG:
+ raise ValueError(
+ f"the setting in layer_config {layer_name} "
+ f"could not match any supported gguf format, please have a check."
+ )
+
+ new_type = new_type[:bits_index] + str(config["bits"]) + new_type[bits_index + 1 :]
+ new_type = _search_gguf_type(new_type)
+ if new_type is None:
+ raise ValueError(f"invalid bit setting for {layer_name}")
+ elif lm_head_name is not None and layer_name == lm_head_name and not tie_word_embeddings:
+ if gguf.MODEL_ARCH.FALCON == model_class.model_arch or input_features % block_size != 0:
+ new_type = "gguf:q8_0"
+ elif "lm_head" in GGUF_CONFIG[target_gguf_format]:
+ new_type = GGUF_CONFIG[target_gguf_format]["lm_head"]
+ elif new_type != "gguf:q8_0":
+ new_type = "gguf:q6_k"
+ elif lm_head_name is not None and layer_name == lm_head_name and tie_word_embeddings:
+ return None
+ elif isinstance(layer, torch.nn.Embedding):
+ embedding_format_key = "lm_head" if tie_word_embeddings else "embedding"
+ if embedding_format_key in GGUF_CONFIG[target_gguf_format]:
+ new_type = GGUF_CONFIG[target_gguf_format][embedding_format_key]
+ elif target_bits is not None and "bits" in config and config["bits"] != target_bits:
+ new_type = new_type[:bits_index] + str(config["bits"]) + new_type[bits_index + 1 :]
+ new_type = _search_gguf_type(new_type)
+ if new_type is None:
+ raise ValueError(f"invalid bit setting for {layer_name}")
+ elif gguf_name is not None:
+ gguf_weight_name = gguf_name if gguf_name.endswith(".weight") else f"{gguf_name}.weight"
+ new_type = dtype_selector.select_gguf_type(gguf_weight_name, len(layer.weight.shape), i_layer or 0)
+ new_block_size = GGML_QUANT_SIZES[new_type.split(":")[-1].lower()][0]
+ if input_features % new_block_size != 0:
+ new_type = _gguf_type_fallback(new_type)
+ new_block_size = GGML_QUANT_SIZES[new_type.split(":")[-1].lower()][0]
+ if input_features % new_block_size != 0:
+ new_type = "gguf:bf16"
+ logger.warning(
+ f"fallback {layer_name} to {new_type}, "
+ f"because input_features({input_features}) % block_size({block_size}) != 0"
+ )
+ # for deepseek v2
+ if layer_name.endswith("kv_b_proj") and new_type.endswith("_k") and "Deepseek" in model.config.architectures[0]:
+ fallback = False
+
+ # calc if need fallback
+ qk_nope_head_dim = model.config.qk_nope_head_dim
+ kv_b_shape = get_module(model, layer_name).weight.shape
+
+ if (
+ qk_nope_head_dim < QK_K
+ or qk_nope_head_dim % QK_K != 0
+ or kv_b_shape[-1] < QK_K
+ or kv_b_shape[-1] % QK_K != 0
+ ):
+ fallback = True
+ if fallback:
+ tmp_type = _gguf_type_fallback(new_type)
+ logger.warning_once(
+ f"self_attn.kv_b_proj does not support the use of {new_type}, replace it with {tmp_type}"
+ )
+ new_type = tmp_type
+
+ return new_type
+
+
+def _infer_gguf_n_layers_from_model(model, model_type=ModelType.TEXT):
+ if model is None or not hasattr(model, "named_modules"):
+ return None, None
+
+ from auto_round.utils.common import MM_MODULE_KEYS
+
+ text_layer_ids = set()
+ vision_layer_ids = set()
+ layer_pattern = re.compile(r"(?:^|\.)layers\.(\d+)(?:\.|$)")
+ for module_name, _ in model.named_modules():
+ match = layer_pattern.search(module_name)
+ if match is None:
+ continue
+ layer_id = int(match.group(1))
+ if model_type != ModelType.TEXT and any(key in module_name.lower() for key in MM_MODULE_KEYS):
+ vision_layer_ids.add(layer_id)
+ else:
+ text_layer_ids.add(layer_id)
+
+ n_layer = max(text_layer_ids) + 1 if text_layer_ids else None
+ n_layer_vision = max(vision_layer_ids) + 1 if vision_layer_ids else None
+ return n_layer, n_layer_vision
+
+
+def _resolve_gguf_n_layers(config, model_type=ModelType.TEXT, model=None):
+ """Resolve text (and vision) layer counts from config, then model structure.
+
+ Multimodal configs (e.g. Qwen3.5, Qwen3-VL) keep the text hparams only in
+ ``config.text_config``, so both it and the top-level config must be checked.
+ If config naming changes, fall back to the loaded module hierarchy so GGUF
+ official mixed formats still receive a usable block count.
+ """
+ n_layer = None
+ n_layer_vision = None
+ text_config = getattr(config, "text_config", None)
+ vision_config = None
+ audio_config = getattr(config, "audio_config", None)
+ for name in ["n_layers", "num_hidden_layers", "n_layer", "num_layers", "depth"]:
+ if hasattr(config, name):
+ n_layer = getattr(config, name)
+ if text_config is not None and hasattr(text_config, name):
+ # the text hparams in text_config are authoritative for multimodal models;
+ # for text-only usage they still fill in when the top-level config lacks them
+ if model_type != ModelType.TEXT or n_layer is None:
+ n_layer = getattr(text_config, name)
+ if model_type != ModelType.TEXT:
+ for config_name in ["vision_config", "vision_encoder"]:
+ if hasattr(config, config_name):
+ vision_config = getattr(config, config_name)
+ if hasattr(vision_config, name):
+ n_layer_vision = getattr(vision_config, name)
+ break
+ if n_layer and n_layer_vision:
+ break
+
+ inferred_n_layer, inferred_n_layer_vision = _infer_gguf_n_layers_from_model(model, model_type)
+ n_layer = n_layer or inferred_n_layer
+ n_layer_vision = n_layer_vision or inferred_n_layer_vision
+
+ if (
+ model_type != ModelType.TEXT
+ and n_layer_vision is None
+ and vision_config is not None
+ and audio_config is not None
+ ):
+ # llama.cpp's Gemma4 mmproj converter uses a fixed tensor-name map size
+ # when vision and audio encoders coexist, because the two encoders do not
+ # expose a single transformer block count in config.json.
+ n_layer_vision = 128
+ return n_layer, n_layer_vision
+
+
+##https://github.com/ggml-org/llama.cpp/blob/9e31bec4fd53634c9e5b04650488a09a055f5dab/src/llama-quant.cpp#L129
+def get_layer_config_by_gguf_format(layer_config, target_gguf_format: str, model, model_type=ModelType.TEXT):
+ import gguf # pylint: disable=E0401
+
+ from auto_round.export.export_to_gguf.llama_cpp_conversion import get_conversion
+ from auto_round.utils.common import MM_MODULE_KEYS
+ from auto_round.utils.model import get_lm_head_name, get_module, is_separate_lm_head
+
+ try:
+ hparams = model.config.to_dict()
+ conversion = get_conversion(hparams=hparams, model_type=model_type)
+ model_architecture = conversion.get_model_architecture(
+ hparams=hparams, model_type=conversion.model_type(model_type)
+ )
+ except AttributeError as e:
+ raise ImportError(
+ "Please use the latest gguf-py, you can use the following command to install it:\n"
+ "git clone https://github.com/ggml-org/llama.cpp.git && cd llama.cpp/gguf-py"
+ " && pip install . sentencepiece"
+ )
+ try:
+ if model_type != ModelType.TEXT:
+ model_class_vision = conversion.get_model_class(model_architecture, model_type=model_type)
+ model_class = conversion.get_model_class(model_architecture, model_type=ModelType.TEXT)
+
+ except NotImplementedError:
+ return layer_config, {}
+
+ n_layer, n_layer_vision = _resolve_gguf_n_layers(model.config, model_type, model=model)
+
+ if n_layer is None:
+ return layer_config, {}
+
+ tensor_map = gguf.get_tensor_name_map(model_class.model_arch, n_layer)
+ if model_type != ModelType.TEXT:
+ tensor_map_vision = gguf.get_tensor_name_map(model_class_vision.model_arch, n_layer_vision)
+
+ def _set_config(config, target_config):
+ for k, v in target_config.items():
+ config[k] = v
+ return config
+
+ gguf_format_config = {}
+ lm_head_name = get_lm_head_name(model)
+ inner_gguf_format = GGUF_CONFIG[target_gguf_format]["mostly"]
+ # ggml_type = getattr(gguf.GGMLQuantizationType,inner_gguf_format.split(":")[-1].upper())
+ block_size = GGML_QUANT_SIZES[inner_gguf_format.split(":")[-1].lower()][0]
+ tie_word_embeddings = True
+ if hasattr(model, "config") and hasattr(model.config, "tie_word_embeddings"):
+ tie_word_embeddings = model.config.tie_word_embeddings
+ tie_word_embeddings &= not is_separate_lm_head(model)
+
+ dtype_selector = GGUFDTypeSelector(
+ hparams, gguf_format_to_ftype(target_gguf_format), model_class.model_arch, n_layer
+ )
+ layer_config_copy = thaw_mapping(layer_config)
+ base_target_bits = None
+ if inner_gguf_format.startswith("gguf:q") and len(inner_gguf_format) >= 7 and (inner_gguf_format[6]).isdigit():
+ base_target_bits = int(inner_gguf_format[6])
+
+ def _resolve_gguf_name(layer_name):
+ if model_type != ModelType.TEXT and any([key in layer_name for key in MM_MODULE_KEYS]):
+ gguf_layer_name = tensor_map_vision.get_name(layer_name)
+ if gguf_layer_name is None:
+ for key in MM_MODULE_KEYS:
+ gguf_layer_name = tensor_map_vision.get_name(layer_name.replace(f".{key}", ""))
+ if gguf_layer_name is not None:
+ break
+ else:
+ gguf_layer_name = tensor_map.get_name(layer_name)
+ if gguf_layer_name is None:
+ gguf_layer_name = tensor_map.get_name(layer_name.replace(".language_model", ""))
+ return gguf_layer_name
+
+ dtype_selector.n_attention_wv = sum(
+ 1
+ for layer_name, config in layer_config_copy.items()
+ if check_to_quantized(config)
+ and (gguf_name := _resolve_gguf_name(layer_name))
+ and any(key in gguf_name for key in ("attn_v", "attn_qkv", "attn_kv_b"))
+ )
+
+ for layer_name, config in layer_config_copy.items():
+ if not check_to_quantized(config):
+ continue
+ layer = get_module(model, layer_name)
+ i_layer = _get_digital_in_layer_name(layer_name)
+ new_type = _select_gguf_layer_type(
+ layer_name,
+ config,
+ layer,
+ model,
+ model_class,
+ target_gguf_format=target_gguf_format,
+ lm_head_name=lm_head_name,
+ tie_word_embeddings=tie_word_embeddings,
+ block_size=block_size,
+ base_target_bits=base_target_bits,
+ dtype_selector=dtype_selector,
+ gguf_name=_resolve_gguf_name(layer_name),
+ i_layer=i_layer,
+ )
+ if new_type is None:
+ continue
+
+ target_config = GGUF_INNER_CONFIG[new_type]
+
+ _set_config(layer_config[layer_name], target_config)
+ gguf_format_config[layer_name] = new_type
+
+ return layer_config, gguf_format_config
diff --git a/auto_round/export/formats/backends/llm_compressor.py b/auto_round/export/formats/backends/llm_compressor.py
new file mode 100644
index 000000000..52d18037b
--- /dev/null
+++ b/auto_round/export/formats/backends/llm_compressor.py
@@ -0,0 +1,220 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import re
+from typing import Any, Callable, Union
+
+import torch
+
+from auto_round.compressors.config_resolution.errors import FormatCompatibilityError
+from auto_round.export.formats.base import BackendDataType, OutputFormat
+from auto_round.logger import logger
+from auto_round.schemes import QuantizationScheme
+
+
[email protected]("llm_compressor")
+class LLMCompressorFormat(OutputFormat):
+ support_schemes = [
+ "MXFP4",
+ "MXFP8",
+ "NVFP4",
+ "FPW8A16",
+ "FP8_STATIC",
+ "INT8",
+ "INT8_W8A8",
+ "FP8_BLOCK",
+ "W4A16",
+ "W8A16",
+ ]
+ format_name = "llm_compressor"
+
+ def __init__(self, format: str, scheme: QuantizationScheme, ctx: Any):
+ if not self.is_support_scheme(scheme):
+ raise FormatCompatibilityError(
+ f"Currently, the llm_compressor format only supports {self.support_schemes}, "
+ f"but got scheme {scheme}, please change to fake or auto_round etc."
+ )
+ # if format.startswith("llm_compressor"):
+ if re.search("^(auto_round:)?llm_compressor", format):
+ self.output_format = format
+ self.backend = None
+ if scheme.is_nv_fp() or scheme.is_mx_fp():
+ from auto_round.export.export_to_llmcompressor import check_compressed_tensors_supported
+
+ check_compressed_tensors_supported(raise_error=True)
+ self.backend = LLMCompressorFormat(scheme.data_type, scheme, ctx)
+ elif scheme.is_dynamic_afp8() and scheme.is_block_wfp8():
+ self.backend = LLMCompressorFormat(BackendDataType.FP8_BLOCK.value, scheme, ctx)
+ elif scheme.is_act_static():
+ self.backend = LLMCompressorFormat(BackendDataType.FP8_STATIC.value, scheme, ctx)
+ if scheme.act_group_size != 0:
+ logger.warning(
+ f"scheme FP8_STATIC export to llm_compressor format only support for act_group_size 0,"
+ f" ,but got act_group_size={scheme.act_group_size}, reset = 0"
+ )
+ scheme.act_group_size = 0
+ if scheme.group_size > 0:
+ logger.warning(
+ f"please note that group_size={scheme.group_size}"
+ " may not be supported for llm_compressor format, and cannot be loaded in llm_compressor"
+ )
+ elif scheme.is_dynamic_wint8aint8():
+ from auto_round.export.export_to_llmcompressor import check_compressed_tensors_supported
+
+ check_compressed_tensors_supported()
+ self.backend = LLMCompressorFormat(BackendDataType.INT8.name, scheme, ctx)
+ self.backend.output_format = f"llm_compressor:{BackendDataType.INT8.value}"
+ elif scheme.is_wint_woq():
+ from auto_round.export.export_to_llmcompressor import check_compressed_tensors_supported
+
+ check_compressed_tensors_supported()
+ self.backend = LLMCompressorFormat(BackendDataType.WINT_A16.value, scheme, ctx)
+ else:
+ if format.upper() not in list(BackendDataType.__members__.keys()):
+ raise KeyError(f"Unsupported backend format llm_compressor:{format}, please check")
+ self.output_format = f"llm_compressor:{format}"
+ self.backend = None
+
+ @classmethod
+ def check_scheme_args(cls: OutputFormat, scheme: QuantizationScheme) -> bool:
+ error_logs = []
+ if scheme.bits not in [4, 8, 16]:
+ error_logs.append(f"bits={scheme.bits}")
+ if not re.search("mxfp|fp|nvfp|int", scheme.data_type):
+ error_logs.append(f"data_type={scheme.data_type}")
+ if scheme.data_type == "fp" and scheme.bits != 8:
+ error_logs.append(f"data_type={scheme.data_type}, bits={scheme.bits}")
+ if scheme.data_type == "int" and scheme.bits not in [4, 8]:
+ error_logs.append(f"data_type={scheme.data_type}, bits={scheme.bits}")
+ if scheme.super_bits:
+ error_logs.append(f"super_bits={scheme.super_bits}")
+ if scheme.super_group_size:
+ error_logs.append(f"super_group_size={scheme.super_group_size}")
+ if isinstance(scheme.group_size, tuple):
+ if scheme.bits != 8:
+ error_logs.append(f"bits={scheme.bits}")
+ if scheme.data_type != "fp":
+ error_logs.append(f"data_type={scheme.data_type}")
+ if len(scheme.group_size) != 2:
+ error_logs.append(f"group_size={scheme.group_size}")
+ if not scheme.act_dynamic:
+ error_logs.append(f"act_dynamic={scheme.act_dynamic}")
+ if not isinstance(scheme.act_group_size, int):
+ error_logs.append(f"act_group_size={scheme.act_group_size}")
+ if scheme.act_bits != 8:
+ error_logs.append(f"act_bits={scheme.act_bits}")
+ if scheme.act_data_type != "fp":
+ error_logs.append(f"act_data_type={scheme.act_data_type}")
+ if error_logs:
+ raise ValueError(
+ f"LLMCompressor format support quantization scheme with {','.join(cls.support_schemes)} "
+ f"but got {', '.join(error_logs)}, please have a check."
+ )
+ return True
+
+ def check_and_reset_format(
+ self, scheme: QuantizationScheme, ctx: Any
+ ) -> tuple[str, QuantizationScheme, dict, list]:
+ layer_config, quant_block_list = ctx.layer_config, ctx.quant_block_list
+ if self.backend is not None:
+ new_format, scheme, layer_config, quant_block_list = self.backend.check_and_reset_format(scheme, ctx)
+ ctx.layer_config, ctx.quant_block_list = layer_config, quant_block_list
+ self.backend = (
+ OutputFormat._format_list[new_format](new_format, scheme, ctx) if new_format else self.backend
+ )
+
+ if scheme.act_bits <= 8 and (not scheme.is_act_standard_fp() or scheme.act_dynamic):
+ if (scheme.is_act_nv_fp() and "static_gs" in scheme.act_data_type) or scheme.is_act_mx_fp():
+ return None, scheme, layer_config, quant_block_list
+ elif scheme.is_dynamic_afp8() and scheme.is_block_wfp8():
+ return None, scheme, layer_config, quant_block_list
+ else:
+ bits, group_size, sym, act_bits = 8, -1, True, 8
+ assert (
+ scheme.bits == bits
+ and scheme.group_size == group_size
+ and scheme.sym == sym
+ and scheme.act_bits == act_bits
+ and scheme.act_dynamic
+ ), (
+ f"Currently only support to export llm_compressor format for sym dynamic quantized"
+ f" W{scheme.bits}A{scheme.act_bits} model with group_size={group_size},"
+ f" but got bits={scheme.bits}, group_size={scheme.group_size}, sym={scheme.sym},"
+ f" act_bits={scheme.act_bits}"
+ )
+ return None, scheme, layer_config, quant_block_list
+ return None, scheme, layer_config, quant_block_list
+
+ def pack_layer(self, layer_name, model, device=None, **kwargs):
+ if self.backend is not None:
+ return self.backend.pack_layer(layer_name, model, device=device, **kwargs)
+ if re.search(f"{BackendDataType.MX_FP.value}|{BackendDataType.NV_FP.value}", self.output_format):
+ from auto_round.export.export_to_llmcompressor.export_to_fp import pack_layer
+
+ return pack_layer(layer_name, model, device=device)
+ elif re.search(f"{BackendDataType.FP8_STATIC.value}", self.output_format):
+ from auto_round.export.export_to_llmcompressor.export_to_static_fp import pack_layer
+
+ return pack_layer(layer_name, model, self.get_backend_name(), device=device)
+ elif re.search(f"{BackendDataType.INT8.value}", self.output_format):
+ from auto_round.export.export_to_llmcompressor.export import pack_layer
+
+ return pack_layer(layer_name, model, device=device)
+ elif re.search(f"{BackendDataType.FP8_BLOCK.value}", self.output_format):
+ from auto_round.export.export_to_llmcompressor.export import pack_layer
+
+ return pack_layer(layer_name, model, device=device)
+ elif re.search(f"{BackendDataType.WINT_A16.value}", self.output_format):
+ from auto_round.export.export_to_llmcompressor.export import pack_layer
+
+ return pack_layer(layer_name, model, device=device)
+ ## passed as no other llm_compressor format is supported yet
+ logger.warning("No other llm_compressor packing format(except NVFP&MXFP) is supported yet, skip packing")
+ return
+
+ def save_quantized(
+ self,
+ output_dir: str,
+ model: torch.nn.Module = None,
+ tokenizer: Callable = None,
+ layer_config: dict = None,
+ inplace: bool = True,
+ device: Union[str, torch.device] = "cpu",
+ serialization_dict: dict = None,
+ **kwargs,
+ ) -> torch.nn.Module:
+ backend = self.get_backend_name()
+ if re.search(f"{BackendDataType.MX_FP.value}|{BackendDataType.NV_FP.value}", backend):
+ from auto_round.export.export_to_llmcompressor.export_to_fp import save_quantized_as_fp
+
+ export_func = save_quantized_as_fp
+ elif re.search(f"{BackendDataType.FP8_STATIC.value}", backend):
+ from auto_round.export.export_to_llmcompressor.export_to_static_fp import save_quantized_as_static_fp
+
+ export_func = save_quantized_as_static_fp
+ else:
+ from auto_round.export.export_to_llmcompressor.export import save_quantized_as_llmcompressor
+
+ export_func = save_quantized_as_llmcompressor
+ return export_func(
+ output_dir=output_dir,
+ model=model,
+ tokenizer=tokenizer,
+ layer_config=layer_config,
+ inplace=inplace,
+ device=device,
+ backend=backend,
+ serialization_dict=serialization_dict,
+ **kwargs,
+ )
diff --git a/auto_round/export/formats/backends/mlx.py b/auto_round/export/formats/backends/mlx.py
new file mode 100644
index 000000000..37717a297
--- /dev/null
+++ b/auto_round/export/formats/backends/mlx.py
@@ -0,0 +1,76 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import re
+from typing import Callable, Union
+
+import torch
+
+from auto_round.export.formats.base import OutputFormat
+from auto_round.schemes import QuantizationScheme
+
+
+class MLXFormat(OutputFormat):
+ support_schemes = ["W2A16", "W2A16G32", "W2A16G64", "W3A16", "W4A16", "W5A16", "W6A16", "W8A16", "BF16"]
+ format_name = "mlx"
+
+ @classmethod
+ def check_scheme_args(cls: OutputFormat, scheme: QuantizationScheme) -> bool:
+ error_logs = []
+ if scheme.bits not in [2, 3, 4, 5, 6, 8, 16]:
+ error_logs.append(f"bits={scheme.bits}")
+ if scheme.act_bits != 16:
+ error_logs.append(f"act_bits={scheme.act_bits}")
+ if not re.search("int", scheme.data_type):
+ error_logs.append(f"data_type={scheme.data_type}")
+ if scheme.super_bits:
+ error_logs.append(f"super_bits={scheme.super_bits}")
+ if scheme.super_group_size:
+ error_logs.append(f"super_group_size={scheme.super_group_size}")
+ if error_logs:
+ raise ValueError(
+ f"MLX format support quantization scheme with {','.join(cls.support_schemes)} "
+ f"but got {', '.join(error_logs)}, please have a check."
+ )
+ return True
+
+ def pack_layer(self, layer_name, model, device=None, **kwargs):
+ from auto_round.export.export_to_mlx.export import pack_layer
+
+ pack_layer(layer_name, model, device=device, **kwargs)
+
+ def save_quantized(
+ self,
+ output_dir: str,
+ model: torch.nn.Module = None,
+ tokenizer: Callable = None,
+ layer_config: dict = None,
+ inplace: bool = True,
+ device: Union[str, torch.device] = "cpu",
+ serialization_dict: dict = None,
+ **kwargs,
+ ) -> torch.nn.Module:
+ from auto_round.export.export_to_mlx.export import save_quantized_as_mlx
+
+ return save_quantized_as_mlx(
+ output_dir=output_dir,
+ model=model,
+ tokenizer=tokenizer,
+ layer_config=layer_config,
+ inplace=inplace,
+ device=device,
+ serialization_dict=serialization_dict,
+ **kwargs,
+ )
diff --git a/auto_round/export/formats/base.py b/auto_round/export/formats/base.py
new file mode 100644
index 000000000..80636f99b
--- /dev/null
+++ b/auto_round/export/formats/base.py
@@ -0,0 +1,237 @@
+# Copyright (c) 2025 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+import copy
+import os
+import re
+from abc import ABC, abstractmethod
+from dataclasses import asdict
+from enum import Enum
+from typing import Any, Callable, Optional, Union
+
+import torch
+import transformers
+
+from auto_round.compressors.config_resolution.errors import FormatCompatibilityError
+from auto_round.export.export_to_gguf.config import ModelType
+from auto_round.schemes import QuantizationScheme
+from auto_round.utils import (
+ INNER_SUPPORTED_LAYER_TYPES,
+ SUPPORTED_LAYER_TYPES,
+ check_to_quantized,
+ compress_layer_names,
+ copy_python_files_from_model_cache,
+ find_matching_blocks,
+ get_block_names,
+ get_module,
+ logger,
+ unsupported_meta_device,
+)
+
+
+class BackendDataType(str, Enum):
+ """Quantized data-type/backend-variant identifiers used as the suffix after a format
+ name (e.g. ``auto_round:fp8_static``, ``llm_compressor:int8_w8a8``). Shared across the
+ ``auto_round`` and ``llm_compressor`` output formats -- not tied to a single format.
+ """
+
+ # Weight: FP8, per-channel, may be extended to per-tensor in future
+ # Activation: FP8, per-tensor
+ FP8_STATIC = "fp8_static"
+ MXFP8 = "mxfp8"
+ MXFP4 = "mxfp4"
+ NVFP4 = "nvfp4"
+ FP8 = "fp8"
+ MX_FP = "mx_fp"
+ NV_FP = "nv_fp"
+ MX_FP_RCEIL = "mx_fp_rceil"
+ NV_FP4_WITH_STATIC_GS = "nv_fp4_with_static_gs"
+ INT8 = "int8_w8a8"
+ FP8_BLOCK = "fp8_block"
+ MXINT4 = "mxint4"
+ MX_INT = "mx_int"
+ WINT_A16 = "wint_a16"
+
+
+def _check_divisible_by_32(scheme: QuantizationScheme, model, layer_config: dict) -> dict:
+ if model is None:
+ return layer_config
+ default_dict = asdict(scheme)
+ skipped_layers = []
+ if default_dict["data_type"] == "int" and default_dict["act_bits"] >= 16:
+ for n, m in model.named_modules():
+ if type(m) in SUPPORTED_LAYER_TYPES or m.__class__.__name__ in INNER_SUPPORTED_LAYER_TYPES:
+ if m.weight.shape[0] % 32 or m.weight.shape[1] % 32:
+ if layer_config is None:
+ layer_config = {}
+ if layer_config.get(n) is not None and layer_config[n]["bits"] >= 16:
+ continue
+ layer_config.setdefault(n, copy.deepcopy(default_dict))
+ layer_config[n].update({"bits": 16, "data_type": "fp", "fixed_by_user": True})
+ skipped_layers.append(n)
+ compressed_skipped_layers = compress_layer_names(skipped_layers)
+ if compressed_skipped_layers:
+ logger.warning_once(
+ f"some layers are skipped quantization (shape not divisible by 32): {compressed_skipped_layers}"
+ )
+ return layer_config
+
+
+class OutputFormat(ABC):
+ """ "Base class for different output formats.
+
+ format: determines which method from export module to use for exporting.
+ For example, auto_round, gguf, llm_compressor etc.
+ backend: determines the specific export process within the format.
+ For example, auto_round:fp8_static, auto_round:auto_awq etc.
+ """
+
+ support_schemes: list = []
+ _format_list: dict[str, OutputFormat] = {}
+ format_name = "base"
+
+ def __init__(self, format: str, scheme: QuantizationScheme, ctx: Any):
+ """Initialize the OutputFormat class."""
+ self.output_format = format
+ self.backend = None
+ self.mllm = ctx.mllm
+
+ if not self.is_fake() and not self.is_support_scheme(scheme):
+ raise FormatCompatibilityError(
+ f"Currently, the {self.format_name} format only supports {self.support_schemes}, "
+ f"but got scheme {scheme}, please change to fake or auto_round etc."
+ )
+
+ @classmethod
+ def register(cls, *names: str) -> Callable[[OutputFormat], OutputFormat]:
+ assert names
+
+ def func(output_format: OutputFormat) -> OutputFormat:
+ for name in names:
+ cls._format_list[name] = output_format
+ return output_format
+
+ return func
+
+ @classmethod
+ def get_support_matrix(cls: OutputFormat) -> str:
+ output_str = ""
+ for k, v in sorted(cls._format_list.items()):
+ if k == "fake":
+ support_schemes = "All schemes"
+ else:
+ if ":" in k and k.split(":")[1] in cls._format_list:
+ support_schemes = cls._format_list[k.split(":")[1]].support_schemes
+ else:
+ support_schemes = v.support_schemes
+ support_schemes = ", ".join(support_schemes).rstrip(",")
+ output_str += f"\x1b[31;1m{k}\x1b[0m support scheme:\n\t{support_schemes}\n"
+ return output_str
+
+ def get_backend_name(self) -> str:
+ if self.backend is None:
+ return self.output_format
+
+ # auto_round:llm_compressor:fp8_static
+ if self.backend.backend is not None:
+ return f"{self.output_format}:{self.backend.get_backend_name()}"
+ # auto_round:fp8_static, llm_compressor:fp8_static, auto_round:auto_awq
+ else:
+ return self.backend.get_backend_name()
+
+ @classmethod
+ def is_support_scheme(cls: OutputFormat, scheme: Union[str, QuantizationScheme]) -> bool:
+ if isinstance(scheme, str) and scheme.upper() in cls.support_schemes:
+ return True
+ if isinstance(scheme, QuantizationScheme):
+ return cls.check_scheme_args(scheme)
+ return False
+
+ @classmethod
+ def check_scheme_args(cls: OutputFormat, scheme: QuantizationScheme) -> bool:
+ return True
+
+ def check_and_reset_format(
+ self,
+ scheme: QuantizationScheme,
+ ctx: Any,
+ ) -> tuple[Optional[str], QuantizationScheme, dict, list]:
+ layer_config, quant_block_list = ctx.layer_config, ctx.quant_block_list
+ if self.backend is not None:
+ new_format, scheme, layer_config, quant_block_list = self.backend.check_and_reset_format(scheme, ctx)
+ ctx.layer_config, ctx.quant_block_list = layer_config, quant_block_list
+ self.backend = (
+ OutputFormat._format_list[new_format](new_format, scheme, ctx) if new_format else self.backend
+ )
+
+ w_fp8 = scheme.data_type.startswith("fp") and scheme.bits == 8
+ act_fp8 = scheme.act_data_type.startswith("fp") and scheme.act_bits == 8
+ is_block_dynamic_fp8 = (
+ self.format_name in ["fp8", "auto_round:fp8"]
+ and isinstance(scheme.group_size, tuple)
+ and scheme.act_dynamic
+ )
+ if (w_fp8 or act_fp8) and not is_block_dynamic_fp8:
+ error_msg = (
+ f"is only supported to export auto_round or llm_compressor format,"
+ f" but got {self.format_name}, please check."
+ )
+ error_msg = ("act_data_type<fp8> " + error_msg) if act_fp8 else error_msg
+ error_msg = ("data_type<fp8> " + error_msg) if w_fp8 else error_msg
+ raise FormatCompatibilityError(error_msg)
+
+ if (
+ scheme.act_bits <= 8
+ and (not scheme.is_act_standard_fp() or scheme.act_dynamic)
+ and not is_block_dynamic_fp8
+ ):
+ logger.warning(
+ f"{self.format_name} format does not support the current activation quantization configuration,"
+ " reset to fake format and save."
+ )
+ return "fake", scheme, layer_config, quant_block_list
+
+ return None, scheme, layer_config, quant_block_list
+
+ @abstractmethod
+ def pack_layer(self, *args, **kwargs):
+ pass
+
+ @abstractmethod
+ def save_quantized(self, *args, **kwargs):
+ pass
+
+ def immediate_pack(self, name: str, model: torch.nn.Module, device: torch.device, **kwargs):
+ m = get_module(model, name)
+ if not check_to_quantized(m):
+ return
+
+ self.pack_layer(name, model, device=device)
+
+ def is_gguf(self) -> bool:
+ return "gguf" in self.output_format
+
+ def is_fake(self) -> bool:
+ return self.output_format == "fake"
+
+ def is_gptq(self) -> bool:
+ return "gptq" in self.output_format or (self.backend is not None and self.backend.is_gptq())
+
+ def is_awq(self) -> bool:
+ return "awq" in self.output_format or (self.backend is not None and self.backend.is_awq())
+
+ def is_llm_compressor(self) -> bool:
+ return "llm_compressor" in self.output_format or (self.backend is not None and self.backend.is_llm_compressor())
diff --git a/auto_round/export/formats/resolver.py b/auto_round/export/formats/resolver.py
new file mode 100644
index 000000000..73c6739c1
--- /dev/null
+++ b/auto_round/export/formats/resolver.py
@@ -0,0 +1,174 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+from types import SimpleNamespace
+from typing import Iterable
+
+import torch
+
+from auto_round.compressors.config_resolution import (
+ FormatCompatibilityError,
+ FormatResolution,
+ ResolvedScheme,
+ thaw_mapping,
+)
+from auto_round.export.formats.backends.gguf import GGUFFormat
+from auto_round.export.formats.base import OutputFormat
+from auto_round.schemes import get_gguf_scheme
+from auto_round.utils import SUPPORTED_FORMATS, logger
+
+
+def _deduplicate(names: Iterable[str]) -> list[str]:
+ return list(dict.fromkeys(names))
+
+
+def _normalize_format_names(value: str, bits: int) -> list[str]:
+ names = value.lower().replace("q*_", f"q{bits}_").replace(" ", "").split(",")
+ return _deduplicate(name for name in names if name)
+
+
+def _validate_format_combination(names: list[str]) -> None:
+ has_gguf = any(name.startswith("gguf") for name in names)
+ real_companions = [name for name in names if name != "fake" and not name.startswith("gguf")]
+ if has_gguf and real_companions:
+ raise FormatCompatibilityError(
+ f"GGUF format is not compatible with other formats, but got {names}, please choose only one of them"
+ )
+
+
+def _apply_scheme_format_constraint(names: list[str], scheme: ResolvedScheme) -> list[str]:
+ preset_name = scheme.preset_name or ""
+ gguf_name = preset_name.lower() if preset_name.lower().startswith("gguf:") else get_gguf_scheme(scheme.value)
+ if not gguf_name:
+ return names
+ if gguf_name.endswith("_mixed"):
+ gguf_name = gguf_name.replace("_mixed", "_s")
+ if all(name in {"fake", gguf_name} for name in names):
+ return names
+ if any(name.startswith("gguf") for name in names if name != "fake"):
+ logger.warning(
+ "scheme %s is GGUF, but format %s specifies a different GGUF type. "
+ "The scheme-driven per-layer quantization may differ from the file-level GGUF format type.",
+ gguf_name,
+ ",".join(names),
+ )
+ return names
+ reset_names = [gguf_name]
+ if "fake" in names:
+ reset_names.append("fake")
+ logger.warning(
+ "reset format %s to %s since scheme %s can only be exported to format %s or fake",
+ ",".join(names),
+ ",".join(reset_names),
+ gguf_name,
+ gguf_name,
+ )
+ return reset_names
+
+
+def _precise_gguf_name(formats: list[OutputFormat]) -> str | None:
+ for output_format in formats:
+ if not output_format.is_gguf():
+ continue
+ backend = getattr(output_format, "backend", None)
+ backend_name = getattr(backend, "output_format", None)
+ precise = backend_name if backend_name and backend_name != "gguf" else output_format.output_format
+ if precise and precise != "gguf":
+ return precise.lower()
+ return None
+
+
+def resolve_formats(
+ scheme: ResolvedScheme,
+ *,
+ format: str = None,
+ layer_config: dict = None,
+ scale_dtype=None,
+ quant_block_list=None,
+ mllm: bool = False,
+ iters: int = 0,
+ enable_alg_ext: bool = False,
+ quant_nontext_module: bool = False,
+ platform: str = None,
+ is_auto_scheme: bool = False,
+ model=None,
+) -> FormatResolution:
+ """Resolve format selection using caller-isolated state and no exporter capability checks."""
+ scheme_value = scheme.value
+ names = _normalize_format_names(format or "auto_round", scheme_value.bits)
+ _validate_format_combination(names)
+ names = _deduplicate(_apply_scheme_format_constraint(names, scheme))
+
+ for name in names:
+ if name not in SUPPORTED_FORMATS:
+ raise ValueError(f"{name} is not supported, we only support {SUPPORTED_FORMATS}")
+
+ layer_config = thaw_mapping(layer_config)
+ quant_block_list = [list(group) for group in quant_block_list] if quant_block_list is not None else None
+ context = SimpleNamespace(
+ model=model,
+ layer_config=layer_config,
+ scale_dtype=scale_dtype,
+ mllm=mllm,
+ iters=iters,
+ enable_alg_ext=enable_alg_ext,
+ quant_nontext_module=quant_nontext_module,
+ quant_block_list=quant_block_list,
+ platform=platform,
+ is_auto_scheme=is_auto_scheme,
+ )
+
+ selected = []
+ requested = format or "auto_round"
+ for name in names:
+ if name.startswith("gguf:"):
+ output_format, scheme_value, context.layer_config = GGUFFormat.build(name, scheme_value, context)
+ elif name in OutputFormat._format_list:
+ output_format = OutputFormat._format_list[name](name, scheme_value, context)
+ else:
+ raise KeyError(f"Unsupported format {name}, please choose from {SUPPORTED_FORMATS}")
+
+ new_name, scheme_value, context.layer_config, context.quant_block_list = output_format.check_and_reset_format(
+ scheme_value, context
+ )
+ if new_name is not None:
+ if new_name in requested:
+ continue
+ output_format = OutputFormat._format_list[new_name](new_name, scheme_value, context)
+ selected.append(output_format)
+
+ if any(output_format.is_gguf() for output_format in selected) and any(
+ output_format.is_fake() for output_format in selected
+ ):
+ selected.sort(key=lambda output_format: 0 if output_format.is_fake() else 1)
+
+ scale_dtype = context.scale_dtype
+ if len(selected) == 1 and selected[0].is_gguf() and scale_dtype != torch.float32:
+ scale_dtype = torch.float32
+ logger.info("change `scale_dtype` to `torch.float32` for gguf format")
+
+ precise_gguf_name = _precise_gguf_name(selected)
+ resolved_scheme = ResolvedScheme.from_scheme(
+ scheme_value,
+ preset_name=precise_gguf_name or scheme.preset_name,
+ )
+ return FormatResolution(
+ formats=tuple(selected),
+ scheme=resolved_scheme,
+ layer_config_patch=context.layer_config,
+ scale_dtype=scale_dtype,
+ quant_block_list=context.quant_block_list,
+ )
diff --git a/auto_round/formats.py b/auto_round/formats.py
index 314ceb9cf..e28ed025c 100644
--- a/auto_round/formats.py
+++ b/auto_round/formats.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2025 Intel Corporation
+# Copyright (c) 2026 Intel Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -12,56 +12,29 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from __future__ import annotations
+"""Backward-compatible imports for the relocated output-format API."""
-import copy
-import os
-import re
-import sys
-from abc import ABC, abstractmethod
-from dataclasses import asdict
from enum import Enum
-from typing import TYPE_CHECKING, Callable, Union
-import torch
-import transformers
-
-from auto_round.compressors.utils import (
- is_act_static,
- is_block_wfp8,
- is_dynamic_afp8,
- is_dynamic_wint8aint8,
- is_mx_fp,
- is_mx_int,
- is_nv_fp,
- is_standard_fp,
- is_wfp8afp8,
- is_wint_woq,
-)
-from auto_round.export.export_to_gguf.config import ModelType
-from auto_round.schemes import (
- PRESET_SCHEMES,
- QuantizationScheme,
- get_gguf_scheme,
-)
-from auto_round.utils import (
- INNER_SUPPORTED_LAYER_TYPES,
- SUPPORTED_FORMATS,
- SUPPORTED_LAYER_TYPES,
- check_to_quantized,
- compress_layer_names,
- copy_python_files_from_model_cache,
- find_matching_blocks,
- get_block_names,
- get_module,
- logger,
- unsupported_meta_device,
+from auto_round.compressors.config_resolution import ResolvedScheme
+from auto_round.export.formats import (
+ AutoAWQFormat,
+ AutoGPTQFormat,
+ AutoRoundFormat,
+ BackendDataType,
+ FakeFormat,
+ FP8Format,
+ GGUFFormat,
+ LLMCompressorFormat,
+ MLXFormat,
+ OutputFormat,
+ resolve_formats,
)
class AutoRoundExportFormat(str, Enum):
- # Weight: FP8, per-channel, may be extended to per-tensor in future
- # Activation: FP8, per-tensor
+ """Legacy names retained for callers importing the old module."""
+
FP8_STATIC = "fp8_static"
MXFP8 = "mxfp8"
MXFP4 = "mxfp4"
@@ -78,1246 +51,37 @@ class AutoRoundExportFormat(str, Enum):
WINT_A16 = "wint_a16"
-if TYPE_CHECKING:
- from auto_round.compressors.base import BaseOrchestrator as BaseCompressor
-
-
-def _check_compatibility(formats: list[str], ar: BaseCompressor):
- if (
- any(["gguf" in f.lower() for f in formats])
- and len([f for f in formats if f.lower() != "fake" and not f.lower().startswith("gguf")]) > 1
- ):
- raise ValueError(
- f"GGUF format is not compatible with other formats, but got {formats}, please choose only one of them"
- )
- gguf_format_name = get_gguf_scheme(ar.scheme)
- if gguf_format_name:
- if gguf_format_name.lower().endswith("mixed"):
- gguf_format_name = gguf_format_name.lower().replace("_mixed", "_s")
- if any([f.lower() not in ["fake", gguf_format_name.lower()] for f in formats]):
- has_gguf_format = any(f.lower().startswith("gguf") for f in formats if f.lower() != "fake")
- if has_gguf_format:
- logger.warning(
- f"scheme {gguf_format_name} is GGUF, but format {','.join(formats)} specifies "
- f"a different GGUF type. The scheme-driven per-layer quantization may differ from the "
- f"file-level GGUF format type."
- )
- else:
- tmp_format_name = (
- gguf_format_name.lower() if "fake" not in formats else (f"{gguf_format_name.lower()},fake")
- )
- logger.warning(
- f"reset format {','.join(formats)} to {tmp_format_name} "
- f"since scheme {gguf_format_name} can only be exported to format "
- f"{gguf_format_name.lower()} or fake"
- )
- formats = tmp_format_name.split(",")
-
- if isinstance(ar.group_size, tuple) and any(["auto_round" in f.lower() for f in formats]):
- logger.warning(
- "auto_round:fp8 format only supports vLLM inference for now. "
- "We recommend using the FP8 format via `--format fp8` instead."
- )
-
- return formats
-
-
-def get_formats(
- format: str,
- ar: BaseCompressor,
-) -> list[OutputFormat]:
- """Get the list of OutputFormat instances based on the provided name."""
-
- def remove_duplicates(lst):
- seen = set()
- return [x for x in lst if not (x in seen or seen.add(x))]
-
- formats = format.lower().replace("q*_", f"q{ar.bits}_").replace(" ", "").split(",")
- formats = remove_duplicates(formats) # need the keep origin order
-
- formats = _check_compatibility(formats, ar)
-
- formats = remove_duplicates(formats)
-
- for fmt in formats:
- if fmt not in SUPPORTED_FORMATS:
- raise ValueError(f"{fmt} is not supported, we only support {SUPPORTED_FORMATS}")
-
- for i in range(len(formats)):
- if formats[i].startswith("gguf:"):
- formats[i] = GGUFFormat(formats[i], ar)
- elif formats[i] not in OutputFormat._format_list:
- raise KeyError(f"Unsupported format {formats[i]}, please choose from {SUPPORTED_FORMATS}")
- else:
- formats[i] = OutputFormat._format_list[formats[i]](formats[i], ar)
-
- new_format = formats[i].check_and_reset_format(ar)
- if new_format is not None:
- if new_format not in format:
- formats[i] = OutputFormat._format_list[new_format](new_format, ar)
- else:
- formats[i] = None
-
- formats = [fmt for fmt in formats if fmt is not None]
-
- # Ensure fake format is processed before GGUF — GGUF export may clear
- # model weights via low_cpu_mem_usage, causing zeroed weights for fake.
- if any(fmt.is_gguf() for fmt in formats) and any(fmt.is_fake() for fmt in formats):
- formats.sort(key=lambda f: 0 if f.is_fake() else 1)
-
- if len(formats) == 1 and formats[0].is_gguf() and ar.scale_dtype != torch.float32:
- ar.scale_dtype = torch.float32
- logger.info("change `scale_dtype` to `torch.float32` for gguf format")
-
- return formats
-
-
-def _check_divisible_by_32(ar):
- from auto_round.schemes import preset_name_to_scheme
-
- if isinstance(ar.scheme, str):
- default_dict = asdict(preset_name_to_scheme(ar.scheme.upper()))
- else:
- default_dict = asdict(ar.scheme)
- skipped_layers = []
- if default_dict["data_type"] == "int" and default_dict["act_bits"] >= 16:
- for n, m in ar.model.named_modules():
- if type(m) in SUPPORTED_LAYER_TYPES or m.__class__.__name__ in INNER_SUPPORTED_LAYER_TYPES:
- if m.weight.shape[0] % 32 or m.weight.shape[1] % 32:
- if ar.layer_config is None:
- ar.layer_config = {}
- if ar.layer_config.get(n) is not None and ar.layer_config[n]["bits"] >= 16:
- continue
- ar.layer_config.setdefault(n, copy.deepcopy(default_dict))
- ar.layer_config[n].update({"bits": 16, "data_type": "fp", "fixed_by_user": True})
- skipped_layers.append(n)
- compressed_skipped_layers = compress_layer_names(skipped_layers)
- if compressed_skipped_layers:
- logger.warning_once(
- f"some layers are skipped quantization (shape not divisible by 32): {compressed_skipped_layers}"
- )
-
-
-class OutputFormat(ABC):
- """ "Base class for different output formats.
-
- format: determines which method from export module to use for exporting.
- For example, auto_round, gguf, llm_compressor etc.
- backend: determines the specific export process within the format.
- For example, auto_round:fp8_static, auto_round:auto_awq etc.
- """
-
- support_schemes: list = []
- _format_list: dict[str, OutputFormat] = {}
- format_name = "base"
-
- def __init__(self, format: str, ar: BaseCompressor):
- """Initialize the OutputFormat class."""
- self.output_format = format
- self.backend = None
-
- if not self.is_fake() and not self.is_support_scheme(ar.scheme):
- logger.error(
- f"Currently, the {self.format_name} format only supports {self.support_schemes}, "
- f"but got scheme {ar.scheme}, please change to fake or auto_round etc."
- )
- exit(-1)
-
- @classmethod
- def register(cls, *names: str) -> Callable[[OutputFormat], OutputFormat]:
- assert names
-
- def func(output_format: OutputFormat) -> OutputFormat:
- for name in names:
- cls._format_list[name] = output_format
- return output_format
-
- return func
-
- @classmethod
- def get_support_matrix(cls: OutputFormat) -> str:
- output_str = ""
- for k, v in sorted(cls._format_list.items()):
- if k == "fake":
- support_schemes = "All schemes"
- else:
- if ":" in k and k.split(":")[1] in cls._format_list:
- support_schemes = cls._format_list[k.split(":")[1]].support_schemes
- else:
- support_schemes = v.support_schemes
- support_schemes = ", ".join(support_schemes).rstrip(",")
- output_str += f"\x1b[31;1m{k}\x1b[0m support scheme:\n\t{support_schemes}\n"
- return output_str
-
- def get_backend_name(self) -> str:
- if self.backend is None:
- return self.output_format
-
- # auto_round:llm_compressor:fp8_static
- if self.backend.backend is not None:
- return f"{self.output_format}:{self.backend.get_backend_name()}"
- # auto_round:fp8_static, llm_compressor:fp8_static, auto_round:auto_awq
- else:
- return self.backend.get_backend_name()
-
- @classmethod
- def is_support_scheme(cls: OutputFormat, scheme: Union[str, QuantizationScheme]) -> bool:
- if isinstance(scheme, str) and scheme.upper() in cls.support_schemes:
- return True
- if isinstance(scheme, QuantizationScheme):
- return cls.check_scheme_args(scheme)
- return False
-
- @classmethod
- def check_scheme_args(cls: OutputFormat, scheme: QuantizationScheme) -> bool:
- return True
-
- def check_and_reset_format(self, ar: BaseCompressor) -> str:
- if self.backend is not None:
- new_format = self.backend.check_and_reset_format(ar)
- self.backend = OutputFormat._format_list[new_format](new_format, ar) if new_format else self.backend
-
- w_fp8 = ar.data_type.startswith("fp") and ar.bits == 8
- act_fp8 = ar.act_data_type.startswith("fp") and ar.act_bits == 8
- is_block_dynamic_fp8 = (
- self.format_name in ["fp8", "auto_round:fp8"] and isinstance(ar.group_size, tuple) and ar.act_dynamic
- )
- if (w_fp8 or act_fp8) and not is_block_dynamic_fp8:
- error_msg = (
- f"is only supported to export auto_round or llm_compressor format,"
- f" but got {self.format_name}, please check."
- )
- error_msg = ("act_data_type<fp8> " + error_msg) if act_fp8 else error_msg
- error_msg = ("data_type<fp8> " + error_msg) if w_fp8 else error_msg
- logger.error(error_msg)
- sys.exit(-1)
-
- if ar.act_bits <= 8 and (not is_standard_fp(ar.act_data_type) or ar.act_dynamic) and not is_block_dynamic_fp8:
- logger.warning(
- f"{self.format_name} format does not support the current activation quantization configuration,"
- " reset to fake format and save."
- )
- return "fake"
-
- return None
-
- @abstractmethod
- def pack_layer(self, *args, **kwargs):
- pass
-
- @abstractmethod
- def save_quantized(self, *args, **kwargs):
- pass
-
- def immediate_pack(self, name: str, model: torch.nn.Module, device: torch.device, **kwargs):
- m = get_module(model, name)
- if not check_to_quantized(m):
- return
-
- self.pack_layer(name, model, device=device)
-
- def is_gguf(self) -> bool:
- return "gguf" in self.output_format
-
- def is_fake(self) -> bool:
- return self.output_format == "fake"
-
- def is_gptq(self) -> bool:
- return "gptq" in self.output_format or (self.backend is not None and self.backend.is_gptq())
-
- def is_awq(self) -> bool:
- return "awq" in self.output_format or (self.backend is not None and self.backend.is_awq())
-
- def is_llm_compressor(self) -> bool:
- return "llm_compressor" in self.output_format or (self.backend is not None and self.backend.is_llm_compressor())
-
-
-class FakeFormat(OutputFormat):
- support_schemes = None
- format_name = "fake"
-
- def check_and_reset_format(self, ar: BaseCompressor) -> str:
- return None
-
- # fake format will not execute pack_layer.
- def pack_layer(self, *args, **kwargs):
- pass
-
- def save_quantized(
- self,
- output_dir: str,
- model: torch.nn.Module = None,
- tokenizer: Callable = None,
- layer_config: dict = None,
- inplace: bool = True,
- device: Union[str, torch.device] = "cpu",
- serialization_dict: dict = None,
- **kwargs,
- ):
- if not unsupported_meta_device(model):
- model = model.to("cpu")
- model.save_pretrained(output_dir)
- elif hasattr(model, "config") and model.config is not None:
- model.config.save_pretrained(output_dir)
-
- if tokenizer is not None and hasattr(tokenizer, "save_pretrained"):
- tokenizer.save_pretrained(output_dir)
- processor = kwargs.get("processor", None)
- if processor is not None:
- processor.save_pretrained(output_dir)
- try:
- copy_python_files_from_model_cache(model, output_dir)
- except Exception as e:
- logger.warning("Skipping source model Python file copy due to error: %s", e)
- return model
-
-
[email protected]("llm_compressor")
-class LLMCompressorFormat(OutputFormat):
- support_schemes = [
- "MXFP4",
- "MXFP8",
- "NVFP4",
- "FPW8A16",
- "FP8_STATIC",
- "INT8",
- "INT8_W8A8",
- "FP8_BLOCK",
- "W4A16",
- "W8A16",
- ]
- format_name = "llm_compressor"
-
- def __init__(self, format, ar):
- if not self.is_support_scheme(ar.scheme):
- logger.error(
- f"Currently, the llm_compressor format only supports {self.support_schemes}, "
- f"but got scheme {ar.scheme}, please change to fake or auto_round etc."
- )
- exit(-1)
- # if format.startswith("llm_compressor"):
- if re.search("^(auto_round:)?llm_compressor", format):
- self.output_format = format
- self.backend = None
- if is_nv_fp(ar.data_type) or is_mx_fp(ar.data_type):
- from auto_round.export.export_to_llmcompressor import check_compressed_tensors_supported
-
- check_compressed_tensors_supported(raise_error=True)
- self.backend = LLMCompressorFormat(ar.data_type, ar)
- elif is_dynamic_afp8(ar) and is_block_wfp8(ar):
- self.backend = LLMCompressorFormat(AutoRoundExportFormat.FP8_BLOCK.value, ar)
- elif is_act_static(ar):
- self.backend = LLMCompressorFormat(AutoRoundExportFormat.FP8_STATIC.value, ar)
- if ar.act_group_size != 0:
- logger.warning(
- f"scheme FP8_STATIC export to llm_compressor format only support for act_group_size 0,"
- f" ,but got act_group_size={ar.act_group_size}, reset = 0"
- )
- ar.act_group_size = 0
- if ar.group_size > 0:
- logger.warning(
- f"please note that group_size={ar.group_size}"
- " may not be supported for llm_compressor format, and cannot be loaded in llm_compressor"
- )
- elif is_dynamic_wint8aint8(ar):
- from auto_round.export.export_to_llmcompressor import check_compressed_tensors_supported
-
- check_compressed_tensors_supported()
- self.backend = LLMCompressorFormat(AutoRoundExportFormat.INT8.name, ar)
- self.backend.output_format = f"llm_compressor:{AutoRoundExportFormat.INT8.value}"
- elif is_wint_woq(ar):
- from auto_round.export.export_to_llmcompressor import check_compressed_tensors_supported
-
- check_compressed_tensors_supported()
- self.backend = LLMCompressorFormat(AutoRoundExportFormat.WINT_A16.value, ar)
- else:
- if format.upper() not in list(AutoRoundExportFormat.__members__.keys()):
- raise KeyError(f"Unsupported backend format llm_compressor:{format}, please check")
- self.output_format = f"llm_compressor:{format}"
- self.backend = None
-
- @classmethod
- def check_scheme_args(cls: OutputFormat, scheme: QuantizationScheme) -> bool:
- error_logs = []
- if scheme.bits not in [4, 8, 16]:
- error_logs.append(f"bits={scheme.bits}")
- if not re.search("mxfp|fp|nvfp|int", scheme.data_type):
- error_logs.append(f"data_type={scheme.data_type}")
- if scheme.data_type == "fp" and scheme.bits != 8:
- error_logs.append(f"data_type={scheme.data_type}, bits={scheme.bits}")
- if scheme.data_type == "int" and scheme.bits not in [4, 8]:
- error_logs.append(f"data_type={scheme.data_type}, bits={scheme.bits}")
- if scheme.super_bits:
- error_logs.append(f"super_bits={scheme.super_bits}")
- if scheme.super_group_size:
- error_logs.append(f"super_group_size={scheme.super_group_size}")
- if isinstance(scheme.group_size, tuple):
- if scheme.bits != 8:
- error_logs.append(f"bits={scheme.bits}")
- if scheme.data_type != "fp":
- error_logs.append(f"data_type={scheme.data_type}")
- if len(scheme.group_size) != 2:
- error_logs.append(f"group_size={scheme.group_size}")
- if not scheme.act_dynamic:
- error_logs.append(f"act_dynamic={scheme.act_dynamic}")
- if not isinstance(scheme.act_group_size, int):
- error_logs.append(f"act_group_size={scheme.act_group_size}")
- if scheme.act_bits != 8:
- error_logs.append(f"act_bits={scheme.act_bits}")
- if scheme.act_data_type != "fp":
- error_logs.append(f"act_data_type={scheme.act_data_type}")
- if error_logs:
- raise ValueError(
- f"LLMCompressor format support quantization scheme with {','.join(cls.support_schemes)} "
- f"but got {', '.join(error_logs)}, please have a check."
- )
- return True
-
- def check_and_reset_format(self, ar: BaseCompressor) -> str | None:
- if self.backend is not None:
- new_format = self.backend.check_and_reset_format(ar)
- self.backend = OutputFormat._format_list[new_format](new_format, ar) if new_format else self.backend
-
- if ar.act_bits <= 8 and (not is_standard_fp(ar.act_data_type) or ar.act_dynamic):
- if (is_nv_fp(ar.act_data_type) and "static_gs" in ar.act_data_type) or (is_mx_fp(ar.act_data_type)):
- return None
- elif is_dynamic_afp8(ar) and is_block_wfp8(ar):
- return None
- else:
- bits, group_size, sym, act_bits = 8, -1, True, 8
- assert (
- ar.bits == bits
- and ar.group_size == group_size
- and ar.sym == sym
- and ar.act_bits == act_bits
- and ar.act_dynamic
- ), (
- f"Currently only support to export llm_compressor format for sym dynamic quantized"
- f" W{ar.bits}A{ar.act_bits} model with group_size={group_size},"
- f" but got bits={ar.bits}, group_size={ar.group_size}, sym={ar.sym},"
- f" act_bits={ar.act_bits}"
- )
- return None
- return None
-
- def pack_layer(self, layer_name, model, device=None, **kwargs):
- if self.backend is not None:
- return self.backend.pack_layer(layer_name, model, device=device, **kwargs)
- if re.search(f"{AutoRoundExportFormat.MX_FP.value}|{AutoRoundExportFormat.NV_FP.value}", self.output_format):
- from auto_round.export.export_to_llmcompressor.export_to_fp import pack_layer
-
- return pack_layer(layer_name, model, device=device)
- elif re.search(f"{AutoRoundExportFormat.FP8_STATIC.value}", self.output_format):
- from auto_round.export.export_to_llmcompressor.export_to_static_fp import pack_layer
-
- return pack_layer(layer_name, model, self.get_backend_name(), device=device)
- elif re.search(f"{AutoRoundExportFormat.INT8.value}", self.output_format):
- from auto_round.export.export_to_llmcompressor.export import pack_layer
-
- return pack_layer(layer_name, model, device=device)
- elif re.search(f"{AutoRoundExportFormat.FP8_BLOCK.value}", self.output_format):
- from auto_round.export.export_to_llmcompressor.export import pack_layer
-
- return pack_layer(layer_name, model, device=device)
- elif re.search(f"{AutoRoundExportFormat.WINT_A16.value}", self.output_format):
- from auto_round.export.export_to_llmcompressor.export import pack_layer
-
- return pack_layer(layer_name, model, device=device)
- ## passed as no other llm_compressor format is supported yet
- logger.warning("No other llm_compressor packing format(except NVFP&MXFP) is supported yet, skip packing")
- return
-
- def save_quantized(
- self,
- output_dir: str,
- model: torch.nn.Module = None,
- tokenizer: Callable = None,
- layer_config: dict = None,
- inplace: bool = True,
- device: Union[str, torch.device] = "cpu",
- serialization_dict: dict = None,
- **kwargs,
- ) -> torch.nn.Module:
- backend = self.get_backend_name()
- if re.search(f"{AutoRoundExportFormat.MX_FP.value}|{AutoRoundExportFormat.NV_FP.value}", backend):
- from auto_round.export.export_to_llmcompressor.export_to_fp import save_quantized_as_fp
-
- export_func = save_quantized_as_fp
- elif re.search(f"{AutoRoundExportFormat.FP8_STATIC.value}", backend):
- from auto_round.export.export_to_llmcompressor.export_to_static_fp import save_quantized_as_static_fp
-
- export_func = save_quantized_as_static_fp
- else:
- from auto_round.export.export_to_llmcompressor.export import save_quantized_as_llmcompressor
-
- export_func = save_quantized_as_llmcompressor
- return export_func(
- output_dir=output_dir,
- model=model,
- tokenizer=tokenizer,
- layer_config=layer_config,
- inplace=inplace,
- device=device,
- backend=backend,
- serialization_dict=serialization_dict,
- **kwargs,
- )
-
-
[email protected]("auto_gptq", "gptqmodel")
-class AutoGPTQFormat(OutputFormat):
- support_schemes = ["W4A16", "W2A16", "W3A16", "W8A16", "BF16", "W2A16G64", "W2A16G32", "W4A16_MIXED"]
- format_name = "auto_gptq"
-
- def check_and_reset_format(self, ar):
- if not ar.sym:
- logger.warning(
- "the asymmetrical kernel of the GPTQ format may result in a noticeable accuracy drop,"
- " particularly for 2-bit quantization and smaller models."
- " We recommend exporting to either the AutoAWQ format ( only 4 bits) or "
- "the AutoRound format(2/3/4/8 bits)."
- )
- if self.backend is None:
- _check_divisible_by_32(ar)
- return super().check_and_reset_format(ar)
-
- @classmethod
- def check_scheme_args(cls: OutputFormat, scheme: QuantizationScheme) -> bool:
- error_logs = []
- if scheme.bits not in [2, 3, 4, 8, 16]:
- error_logs.append(f"bits={scheme.bits}")
- if not re.search("int", scheme.data_type):
- error_logs.append(f"data_type={scheme.data_type}")
- if scheme.super_bits:
- error_logs.append(f"super_bits={scheme.super_bits}")
- if scheme.super_group_size:
- error_logs.append(f"super_group_size={scheme.super_group_size}")
- if error_logs:
- raise ValueError(
- f"{cls.format_name} format support quantization scheme with {','.join(cls.support_schemes)} "
- f"but got {', '.join(error_logs)}, please have a check."
- )
- return True
-
- def pack_layer(self, layer_name, model, device=None, **kwargs):
- if self.output_format.startswith("auto_round"):
- from auto_round.export.export_to_autoround.export import pack_layer
-
- pack_layer(layer_name, model, backend=self.output_format, device=device)
- else:
- from auto_round.export.export_to_autogptq.export import pack_layer
-
- pack_layer(layer_name, model, backend=self.output_format, device=device)
-
- def save_quantized(
- self,
- output_dir: str,
- model: torch.nn.Module = None,
- tokenizer: Callable = None,
- layer_config: dict = None,
- inplace: bool = True,
- device: Union[str, torch.device] = "cpu",
- serialization_dict: dict = None,
- **kwargs,
- ) -> torch.nn.Module:
- backend = self.get_backend_name()
- if backend == "auto_round:auto_gptq" or backend == "auto_round:gptqmodel":
- from auto_round.export.export_to_autoround.export import save_quantized_as_autoround
-
- export_func = save_quantized_as_autoround
- else:
- from auto_round.export.export_to_autogptq.export import save_quantized_as_autogptq
-
- export_func = save_quantized_as_autogptq
- return export_func(
- output_dir=output_dir,
- model=model,
- tokenizer=tokenizer,
- layer_config=layer_config,
- inplace=inplace,
- device=device,
- backend=backend,
- serialization_dict=serialization_dict,
- **kwargs,
- )
-
-
[email protected]("auto_awq")
-class AutoAWQFormat(OutputFormat):
- support_schemes = ["W4A16"]
- format_name = "auto_awq"
-
- @classmethod
- def check_scheme_args(cls: OutputFormat, scheme: QuantizationScheme) -> bool:
- error_logs = []
- if scheme.bits != 4:
- error_logs.append(f"bits={scheme.bits}")
- if not re.search("int", scheme.data_type):
- error_logs.append(f"data_type={scheme.data_type}")
- if scheme.super_bits:
- error_logs.append(f"super_bits={scheme.super_bits}")
- if scheme.super_group_size:
- error_logs.append(f"super_group_size={scheme.super_group_size}")
- if error_logs:
- raise ValueError(
- f"{cls.format_name} format support quantization scheme with {','.join(cls.support_schemes)} "
- f"but got {', '.join(error_logs)}, please have a check."
- )
- return True
-
- @staticmethod
- def check_awq_gemm_compatibility(model, bits, group_size, sym, layer_configs=None):
- """Checks if a model is compatible with the AutoAWQ GEMM kernel.
-
- Args:
- model: The model object to evaluate, typically a PyTorch model.
- bits (int): The number of bits for quantization (must be 4 for compatibility).
- group_size (int): The group size for quantization.
- sym (bool): Whether symmetric quantization is used (not utilized in the current function logic).
- layer_configs (dict, optional): A dictionary mapping layer names to configurations, where each
- configuration can specify a custom number of bits for the layer.
-
- Returns:
- tuple: A tuple containing:
- - bool: `True` if the model is compatible, `False` otherwise.
- - str: An error message describing why the model is incompatible, or an empty string if compatible.
- """
- from auto_round.utils.model import get_layer_names_in_block, get_module
-
- if bits != 4:
- return False, "AutoAWQ GEMM kernel only supports 4 bits"
- for n, m in model.named_modules():
- if type(m) == transformers.pytorch_utils.Conv1D:
- return False, "AutoAWQ GEMM kernel does not support conv1d"
-
- layer_names = get_layer_names_in_block(model)
- for layer_name in layer_names:
- if (
- layer_configs is not None
- and layer_name in layer_configs.keys()
- and layer_configs[layer_name].get("bits", bits) > 8
- ):
- continue
-
- layer = get_module(model, layer_name)
- if layer.in_features % group_size != 0:
- return False, f"Layer {layer_name} in_features is not multiple of group_size {group_size}"
- if layer.out_features % (32 // bits) != 0:
- return False, f"Layer {layer_name} out_features is not multiple of 32 // bits"
-
- return True, ""
-
- def check_and_reset_format(self, ar):
- awq_supported, info = self.check_awq_gemm_compatibility(
- ar.model, ar.bits, ar.group_size, ar.sym, ar.layer_config
- )
- if not awq_supported:
- logger.warning(f"The AutoAWQ format may not be supported due to {info}")
- if ar.bits != 4:
- raise ValueError(f"auto_awq format support quantization scheme with W4A16 but got bits={ar.bits}")
-
- if self.backend is None:
- _check_divisible_by_32(ar)
-
- return super().check_and_reset_format(ar)
-
- def pack_layer(self, layer_name, model, device=None, **kwargs):
- from auto_round.export.export_to_awq.export import pack_layer
-
- pack_layer(layer_name, model, backend=self.output_format, device=device)
-
- def save_quantized(
- self,
- output_dir: str,
- model: torch.nn.Module = None,
- tokenizer: Callable = None,
- layer_config: dict = None,
- inplace: bool = True,
- device: Union[str, torch.device] = "cpu",
- serialization_dict: dict = None,
- **kwargs,
- ) -> torch.nn.Module:
- backend = self.get_backend_name()
- if backend == "auto_round:auto_awq":
- from auto_round.export.export_to_autoround.export import save_quantized_as_autoround
-
- export_func = save_quantized_as_autoround
- else:
- from auto_round.export.export_to_awq.export import save_quantized_as_autoawq
-
- export_func = save_quantized_as_autoawq
-
- return export_func(
- output_dir=output_dir,
- model=model,
- tokenizer=tokenizer,
- layer_config=layer_config,
- inplace=inplace,
- backend=backend,
- device=device,
- serialization_dict=serialization_dict,
- **kwargs,
- )
-
-
-class GGUFFormat(OutputFormat):
- support_schemes = [
- "GGUF:Q4_0",
- "GGUF:Q4_1",
- "GGUF:Q5_0",
- "GGUF:Q5_1",
- "GGUF:Q2_K_S",
- "GGUF:Q3_K_S",
- "GGUF:Q3_K_M",
- "GGUF:Q3_K_L",
- "GGUF:Q4_K_S",
- "GGUF:Q4_K_M",
- "GGUF:Q5_K_S",
- "GGUF:Q5_K_M",
- "GGUF:Q6_K",
- "GGUF:Q8_0",
- "GGUF:Q2_K_MIXED",
- ]
- format_name = "gguf"
-
- def __init__(self, format: str, ar: BaseCompressor):
- self.is_auto_scheme = bool(getattr(ar, "is_auto_scheme", False))
- if format.startswith("gguf:"):
- self._original_format = format # preserve "gguf:q2_k_mixed" etc. for Phase 2b
- self.output_format = "gguf"
- self.backend_cls = GGUFFormat
- self.backend = GGUFFormat(format.split(":")[-1], ar)
-
- resolved_format = self.backend.output_format
- self.gguf_args_check(ar, resolved_format, model_type=ModelType.TEXT)
- if ar.mllm:
- self.gguf_args_check(ar, resolved_format, model_type=ModelType.MMPROJ)
- else:
- scheme = ar.scheme
- gguf_format = f"gguf:{format.lower()}"
- if format.lower().endswith("_mixed"):
- from auto_round.schemes import _handle_special_schemes
- from auto_round.utils.model import is_moe_model
-
- if format.lower() == "q2_k_mixed" and (getattr(ar, "iters", 0) or 0) > 0 and not is_moe_model(ar.model):
- logger.warning(
- "gguf:q2_k_mixed only supports MoE models with iters>0. "
- "It is not an MoE model, falling back to gguf:q2_k_s."
- )
- gguf_format = "gguf:q2_k_s"
- else:
- ar.layer_config = _handle_special_schemes(
- gguf_format, ar.layer_config, ar.model, quant_nontext_module=ar.quant_nontext_module
- )
- gguf_format = gguf_format.lower().replace("_mixed", "_s")
- if isinstance(scheme, str) and scheme.lower() != gguf_format and not getattr(ar, "is_auto_scheme", False):
- logger.warning(f"reset scheme {scheme.lower()} to {gguf_format} for gguf format export")
- ar.scheme = gguf_format
- self.output_format = gguf_format
- self.backend = None
- self.mllm = ar.mllm
-
- @classmethod
- def check_scheme_args(cls: OutputFormat, scheme: QuantizationScheme) -> bool:
- error_logs = []
- if not re.search("int", scheme.data_type):
- error_logs.append(f"data_type={scheme.data_type}")
- if error_logs:
- raise ValueError(
- f"{cls.format_name} format support quantization scheme with {','.join(cls.support_schemes)} "
- f"but got {', '.join(error_logs)}, please have a check."
- )
- return True
-
- def check_and_reset_format(self, ar):
- if getattr(ar, "iters", 0) != 0 and ar.bits != 3 and not ar.enable_alg_ext:
- logger.warning_once(
- "`iters=0` is recommended when exporting to current GGUF format"
- " or add `enable_alg_ext` for better accuracy with much more tuning cost."
- " Please refer to https://github.com/intel/auto-round/tree/main/docs/gguf_alg_ext_acc.md"
- " for the accuracy results."
- )
- elif ar.bits >= 8 and getattr(ar, "iters", 0) != 0:
- logger.warning_once("`iters=0` is recommended for bits>=8")
-
- if getattr(ar, "quant_nontext_module", False):
- # for gguf export, leave vl model for gguf itself
- all_blocks = get_block_names(ar.model, False)
- ar.quant_block_list = find_matching_blocks(ar.model, all_blocks, None)
- return super().check_and_reset_format(ar)
-
- def pack_layer(
- self,
- name,
- model,
- backend,
- output_dir,
- layer_config,
- tokenizer,
- processor=None,
- image_processor=None,
- model_type=ModelType.TEXT,
- device="cpu",
- quant_nontext_module=False,
- ):
- from auto_round.export.export_to_gguf.export import pack_gguf_layer
-
- pack_gguf_layer(
- name,
- model,
- backend,
- output_dir,
- layer_config,
- tokenizer,
- processor,
- image_processor,
- model_type,
- device,
- quant_nontext_module,
- is_auto_scheme=self.is_auto_scheme,
- )
-
- def save_quantized(
- self,
- output_dir: str,
- model: torch.nn.Module = None,
- tokenizer: Callable = None,
- layer_config: dict = None,
- inplace: bool = True,
- device: Union[str, torch.device] = "cpu",
- serialization_dict: dict = None,
- **kwargs,
- ) -> torch.nn.Module:
- from auto_round.export.export_to_gguf.export import save_quantized_as_gguf
-
- backend = self.get_backend_name()
- return save_quantized_as_gguf(
- output_dir=output_dir,
- model=model,
- backend=backend,
- layer_config=layer_config,
- mllm=self.mllm,
- device=device,
- serialization_dict=serialization_dict,
- is_auto_scheme=self.is_auto_scheme,
- **kwargs,
- )
-
- @staticmethod
- def gguf_args_check(args_or_ar, formats: Union[str, list[str]] = None, model_type=ModelType.TEXT):
- import argparse
-
- from auto_round.export.export_to_gguf.config import GGUF_CONFIG
- from auto_round.export.export_to_gguf.llama_cpp_conversion import get_conversion
- from auto_round.logger import logger
- from auto_round.utils.model import download_or_get_path, get_gguf_architecture
-
- formats = [formats] if isinstance(formats, str) else formats
- formats = sorted(formats, key=lambda x: len(x))
- export_gguf = False
- for f in formats:
- if f.startswith("gguf"):
- export_gguf = True
-
- if f.startswith("gguf") and f not in GGUF_CONFIG:
- logger.error(f"{f} is not supported, please check.")
-
- if export_gguf:
- if isinstance(args_or_ar.model, str):
- model_path = args_or_ar.model
- else:
- model_path = args_or_ar.model.name_or_path
- if not os.path.isdir(model_path):
- model_path = download_or_get_path(model_path, args_or_ar.platform)
- conversion = get_conversion(model_path, model_type=ModelType.TEXT)
- model_architecture = get_gguf_architecture(model_path, model_type=ModelType.TEXT)
- if not conversion.is_supported(model_architecture, ModelType.TEXT):
- logger.error(f"Model {model_architecture} is not supported to export gguf format.")
- sys.exit(1)
-
- pattern = re.compile(r"q\d_k")
- pre_dq_format = ""
- unsupported_list, reset_list = [], []
- for format in GGUF_CONFIG:
- if format in formats:
- if format == "q6_k_s":
- logger.warning("Please note that q6_k_s is q6_k.")
-
- if re.search(pattern, format):
- if pre_dq_format and re.search(pattern, format).group() not in pre_dq_format:
- logger.error(f"Cannot export {pre_dq_format} and {format} at the same time.")
- sys.exit(-1)
- else:
- pre_dq_format = format
-
- unsupported_list, reset_list = [], []
- gguf_config = GGUF_CONFIG[format]
- for k, v in gguf_config.items():
- if not hasattr(args_or_ar, k):
- continue
- if k == "data_type":
- if re.search(r"q\d_1", format) and len(formats) > 1:
- v = "int"
- if k == "sym" and isinstance(args_or_ar, argparse.Namespace):
- k = "asym"
- v = not v
- if getattr(args_or_ar, k) != v:
- unsupported_list.append(f"{k}={getattr(args_or_ar, k)}")
- reset_list.append(f"{k}={v}")
- setattr(args_or_ar, k, v)
- if len(unsupported_list) > 0:
- logger.info(
- f"format {format} does not support for {', '.join(unsupported_list)},"
- f" reset to {', '.join(reset_list)}."
- )
- # Removed obsolete commented-out block for improved readability and maintainability.
- return args_or_ar
-
- def immediate_pack(
- self,
- name: str,
- model: torch.nn.Module,
- device: torch.device,
- output_dir: str = None,
- mllm: bool = False,
- layer_config: dict = None,
- tokenizer=None,
- processor=None,
- image_processor=None,
- quant_nontext_module: bool = False,
- **kwargs,
- ):
- m = get_module(model, name)
- if not check_to_quantized(m):
- return
- model_type = ModelType.MMPROJ if mllm else ModelType.TEXT
- self.pack_layer(
- name,
- model,
- self.get_backend_name(),
- output_dir,
- layer_config=layer_config,
- tokenizer=tokenizer,
- processor=processor,
- image_processor=image_processor,
- model_type=model_type,
- device=device,
- quant_nontext_module=quant_nontext_module,
- )
-
-
-class FP8Format(OutputFormat):
- support_schemes = ["FP8_BLOCK"]
- format_name = "fp8"
-
- @classmethod
- def check_scheme_args(cls: OutputFormat, scheme: QuantizationScheme) -> bool:
- error_logs = []
- if scheme.bits != 8:
- error_logs.append(f"bits={scheme.bits}")
- if scheme.data_type != "fp":
- error_logs.append(f"data_type={scheme.data_type}")
- if not isinstance(scheme.group_size, tuple):
- error_logs.append(f"group_size={scheme.group_size}")
- if not scheme.act_dynamic:
- error_logs.append(f"act_dynamic={scheme.act_dynamic}")
- if not isinstance(scheme.act_group_size, int):
- error_logs.append(f"act_group_size={scheme.act_group_size}")
- if scheme.act_bits != 8:
- error_logs.append(f"act_bits={scheme.act_bits}")
- if scheme.act_data_type != "fp":
- error_logs.append(f"act_data_type={scheme.act_data_type}")
- if error_logs:
- raise ValueError(
- f"{cls.format_name} format support quantization scheme with {','.join(cls.support_schemes)} "
- f"but got {', '.join(error_logs)}, please have a check."
- )
- return True
-
- def pack_layer(self, layer_name, model, device=None, **kwargs):
- from auto_round.export.export_to_autoround.export_to_fp8 import pack_layer
-
- pack_layer(layer_name, model, self.output_format, device=device)
-
- def save_quantized(
- self,
- output_dir: str,
- model: torch.nn.Module = None,
- tokenizer: Callable = None,
- layer_config: dict = None,
- inplace: bool = True,
- device: Union[str, torch.device] = "cpu",
- serialization_dict: dict = None,
- **kwargs,
- ) -> torch.nn.Module:
- from auto_round.export.export_to_autoround.export_to_fp8 import save_quantized_as_autoround
-
- backend = self.get_backend_name()
-
- if isinstance(serialization_dict["group_size"], tuple):
- serialization_dict["weight_block_size"] = serialization_dict["group_size"]
-
- ignored_layers = []
- for layer_name, cfg in layer_config.items():
- if cfg["bits"] >= 16 and cfg["act_bits"] >= 16:
- ignored_layers.append(layer_name)
- if len(ignored_layers) > 0:
- serialization_dict["ignored_layers"] = ignored_layers
-
- return save_quantized_as_autoround(
- output_dir=output_dir,
- model=model,
- tokenizer=tokenizer,
- layer_config=layer_config,
- inplace=inplace,
- device=device,
- serialization_dict=serialization_dict,
- quant_method=backend,
- **kwargs,
- )
-
-
-class MLXFormat(OutputFormat):
- support_schemes = ["W2A16", "W2A16G32", "W2A16G64", "W3A16", "W4A16", "W5A16", "W6A16", "W8A16", "BF16"]
- format_name = "mlx"
-
- @classmethod
- def check_scheme_args(cls: OutputFormat, scheme: QuantizationScheme) -> bool:
- error_logs = []
- if scheme.bits not in [2, 3, 4, 5, 6, 8, 16]:
- error_logs.append(f"bits={scheme.bits}")
- if scheme.act_bits != 16:
- error_logs.append(f"act_bits={scheme.act_bits}")
- if not re.search("int", scheme.data_type):
- error_logs.append(f"data_type={scheme.data_type}")
- if scheme.super_bits:
- error_logs.append(f"super_bits={scheme.super_bits}")
- if scheme.super_group_size:
- error_logs.append(f"super_group_size={scheme.super_group_size}")
- if error_logs:
- raise ValueError(
- f"MLX format support quantization scheme with {','.join(cls.support_schemes)} "
- f"but got {', '.join(error_logs)}, please have a check."
- )
- return True
-
- def pack_layer(self, layer_name, model, device=None, **kwargs):
- from auto_round.export.export_to_mlx.export import pack_layer
-
- pack_layer(layer_name, model, device=device, **kwargs)
-
- def save_quantized(
- self,
- output_dir: str,
- model: torch.nn.Module = None,
- tokenizer: Callable = None,
- layer_config: dict = None,
- inplace: bool = True,
- device: Union[str, torch.device] = "cpu",
- serialization_dict: dict = None,
- **kwargs,
- ) -> torch.nn.Module:
- from auto_round.export.export_to_mlx.export import save_quantized_as_mlx
-
- return save_quantized_as_mlx(
- output_dir=output_dir,
- model=model,
- tokenizer=tokenizer,
- layer_config=layer_config,
- inplace=inplace,
- device=device,
- serialization_dict=serialization_dict,
- **kwargs,
- )
-
-
[email protected]("auto_round")
[email protected]("auto_round:auto_awq")
[email protected]("auto_round:llm_compressor")
[email protected]("auto_round:gptqmodel", "auto_round:auto_gptq")
[email protected]("auto_round:fp8")
[email protected]("auto_round:mlx")
-class AutoRoundFormat(OutputFormat):
- support_schemes = [
- "W4A16",
- "W4A16_MIXED",
- "W2A16",
- "W3A16",
- "W8A16",
- "MXFP4",
- "MXFP8",
- "NVFP4",
- "FPW8A16",
- "W2A16G64",
- "W2A16G32",
- "FP8_STATIC",
- "BF16",
- "FP8_BLOCK",
- "MXINT4",
- ]
- format_name = "auto_round"
-
- def __init__(self, format: str, ar: BaseCompressor):
- self.output_format = "auto_round"
- self.backend = None
-
- if format == "auto_round":
- if ar.sym and "int" in ar.data_type and "mx" not in ar.data_type:
- self.backend = AutoGPTQFormat("auto_round:auto_gptq", ar)
- elif ar.bits == 4 and not ar.sym and "int" in ar.data_type:
- if ar.layer_config is None:
- enable_awq = True
- else:
- enable_awq = all(
- config["bits"] == ar.bits or config["bits"] >= 16 for config in ar.layer_config.values()
- )
- if enable_awq:
- self.backend = AutoAWQFormat("auto_round:auto_awq", ar)
- elif is_nv_fp(ar.data_type) or is_mx_fp(ar.data_type):
- self.backend = AutoRoundFormat(ar.data_type, ar)
- elif is_mx_int(ar.data_type) and ar.bits == 4: # only add mx_int4 now
- self.backend = AutoRoundFormat(ar.data_type, ar)
- elif is_act_static(ar): # static wfp8afp8
- self.backend = AutoRoundFormat(AutoRoundExportFormat.FP8_STATIC.value, ar)
- elif ar.data_type.startswith("fp") and ar.bits == 8 and ar.act_bits >= 16: # woq fp8
- self.backend = AutoRoundFormat(AutoRoundExportFormat.FP8.value, ar)
- elif ar.data_type.startswith("fp") and ar.bits == 8 and isinstance(ar.group_size, tuple):
- self.backend = AutoRoundFormat("auto_round:fp8", ar)
- elif ar.act_bits < 16:
- raise ValueError(
- "AutoRound format does not support exporting "
- "for the current quantization configuration, "
- "please change to `fake` format for research purpose"
- )
- # for auto_round:fp8_static, auto_round:nv_fp etc.
- elif not format.startswith("auto_round"):
- if format == "mlx":
- self.backend = MLXFormat("mlx", ar)
- elif format.upper() not in list(AutoRoundExportFormat.__members__.keys()):
- raise KeyError(f"Unsupported backend format auto_round:{format}, please check")
- else:
- self.output_format = f"auto_round:{format}"
- self.backend = None
- elif format == "auto_round:mlx":
- self.backend = MLXFormat("mlx", ar)
- else:
- backend = format.split(":")[1] if ":" in format else None
- self.backend = self._format_list.get(backend)(format, ar) if backend else None
-
- if self.backend is not None:
- self.support_schemes = self.backend.support_schemes
-
- def check_and_reset_format(self, ar):
- if self.backend is not None:
- new_format = self.backend.check_and_reset_format(ar)
- self.backend = OutputFormat._format_list[new_format](new_format, ar) if new_format else self.backend
-
- if ar.act_bits <= 8:
- if is_standard_fp(ar.act_data_type) and not ar.act_dynamic:
- if (
- ar.act_group_size != 0
- and not ar.act_dynamic
- and self.get_backend_name() == f"auto_round:{AutoRoundExportFormat.FP8.value}"
- ):
- logger.warning(
- f"Please note that quantize activation with act_group_size={ar.act_group_size}"
- " may result in failure to export or import normally."
- )
- if self.backend is None:
- _check_divisible_by_32(ar)
- return None
-
- def pack_layer(self, layer_name, model, device=None, **kwargs):
- if self.backend is not None:
- return self.backend.pack_layer(layer_name, model, device=device, **kwargs)
-
- backend = self.get_backend_name()
-
- if self.output_format in [
- f"auto_round:{AutoRoundExportFormat.NV_FP.value}",
- f"auto_round:{AutoRoundExportFormat.MX_FP.value}",
- f"auto_round:{AutoRoundExportFormat.MX_FP_RCEIL.value}",
- f"auto_round:{AutoRoundExportFormat.NV_FP4_WITH_STATIC_GS.value}",
- ]:
- from auto_round.export.export_to_autoround.export_to_nvfp_mx import pack_layer
-
- pack_func = pack_layer
- elif self.output_format in [f"auto_round:{AutoRoundExportFormat.MX_INT.value}"]:
- from auto_round.export.export_to_autoround.export_to_nvfp_mx import pack_layer
-
- pack_func = pack_layer
- elif self.output_format in [
- f"auto_round:{AutoRoundExportFormat.FP8.value}",
- f"auto_round:{AutoRoundExportFormat.FP8_STATIC.value}",
- f"auto_round:{AutoRoundExportFormat.FP8_STATIC.value}",
- ]:
- from auto_round.export.export_to_autoround.export_to_fp8 import pack_layer
-
- pack_func = pack_layer
- else:
- from auto_round.export.export_to_autoround.export import pack_layer
-
- pack_func = pack_layer
- return pack_func(layer_name, model, backend, device)
-
- def save_quantized(
- self,
- output_dir: str,
- model: torch.nn.Module = None,
- tokenizer: Callable = None,
- layer_config: dict = None,
- inplace: bool = True,
- device: Union[str, torch.device] = "cpu",
- serialization_dict: dict = None,
- **kwargs,
- ) -> torch.nn.Module:
- if self.backend is not None:
- extra_kwargs = {}
- if isinstance(self.backend, MLXFormat):
- extra_kwargs["autoround_format"] = True
- return self.backend.save_quantized(
- output_dir=output_dir,
- model=model,
- tokenizer=tokenizer,
- layer_config=layer_config,
- inplace=inplace,
- device=device,
- serialization_dict=serialization_dict,
- **extra_kwargs,
- **kwargs,
- )
- backend = self.get_backend_name()
- if re.search(f"{AutoRoundExportFormat.MX_FP.value}|{AutoRoundExportFormat.NV_FP.value}", backend):
- from auto_round.export.export_to_autoround.export_to_nvfp_mx import save_quantized_as_fp
-
- backend = "auto_round:llm_compressor"
- export_func = save_quantized_as_fp
- elif serialization_dict.get("data_type", "int") == "fp" and serialization_dict.get("bits", 16) == 8:
- from auto_round.export.export_to_autoround.export_to_fp8 import save_quantized_as_autoround
-
- backend = "auto_round:fp8_static" if serialization_dict.get("act_bits", 16) == 8 else None
- export_func = save_quantized_as_autoround
- elif re.search(f"{AutoRoundExportFormat.MX_INT.value}", backend):
- from auto_round.export.export_to_autoround.export_to_nvfp_mx import save_quantized_as_fp
-
- backend = "auto_round"
- export_func = save_quantized_as_fp
- else:
- from auto_round.export.export_to_autoround.export import save_quantized_as_autoround
-
- export_func = save_quantized_as_autoround
- return export_func(
- output_dir=output_dir,
- model=model,
- tokenizer=tokenizer,
- layer_config=layer_config,
- inplace=inplace,
- device=device,
- backend=backend,
- serialization_dict=serialization_dict,
- **kwargs,
- )
+def get_formats(format: str, ar):
+ """Resolve formats using the relocated resolver for the legacy API."""
+ from auto_round.schemes import QuantizationScheme, parse_scheme
+
+ scheme = ar.scheme
+ if not isinstance(scheme, QuantizationScheme):
+ _, _, attrs = parse_scheme(scheme, {})
+ scheme = QuantizationScheme.from_dict(attrs)
+ resolution = resolve_formats(
+ ResolvedScheme.from_scheme(scheme),
+ format=format,
+ layer_config=getattr(ar, "layer_config", None),
+ scale_dtype=getattr(ar, "scale_dtype", None),
+ iters=getattr(ar, "iters", 0) or 0,
+ model=getattr(ar, "model", None),
+ )
+ return list(resolution.formats)
+
+
+__all__ = [
+ "AutoAWQFormat",
+ "AutoRoundExportFormat",
+ "AutoGPTQFormat",
+ "AutoRoundFormat",
+ "BackendDataType",
+ "FakeFormat",
+ "FP8Format",
+ "GGUFFormat",
+ "LLMCompressorFormat",
+ "MLXFormat",
+ "OutputFormat",
+ "resolve_formats",
+ "get_formats",
+]
diff --git a/auto_round/inference/convert_model.py b/auto_round/inference/convert_model.py
index 722cfd318..b04c1bbaa 100644
--- a/auto_round/inference/convert_model.py
+++ b/auto_round/inference/convert_model.py
@@ -21,7 +21,7 @@ from packaging.version import Version
from tqdm import tqdm
from transformers.pytorch_utils import Conv1D
-from auto_round.formats import AutoRoundExportFormat
+from auto_round.export.formats import BackendDataType
from auto_round.inference.backend import (
BackendInfos,
dynamic_import_inference_linear,
@@ -559,11 +559,11 @@ def _create_quant_layer(layer, layer_backend, config, in_features, out_features,
bias=bias,
)
elif (
- AutoRoundExportFormat.FP8_STATIC.value in layer_backend
- or AutoRoundExportFormat.MXFP8.value in layer_backend
- or AutoRoundExportFormat.MXFP4.value in layer_backend
- or AutoRoundExportFormat.NVFP4.value in layer_backend
- or AutoRoundExportFormat.MXINT4.value in layer_backend
+ BackendDataType.FP8_STATIC.value in layer_backend
+ or BackendDataType.MXFP8.value in layer_backend
+ or BackendDataType.MXFP4.value in layer_backend
+ or BackendDataType.NVFP4.value in layer_backend
+ or BackendDataType.MXINT4.value in layer_backend
):
return QuantLinear.from_original(config, layer)
diff --git a/auto_round/schemes.py b/auto_round/schemes.py
index 8fa21d018..31da6d95c 100644
--- a/auto_round/schemes.py
+++ b/auto_round/schemes.py
@@ -14,19 +14,186 @@
import copy
from copy import deepcopy
from dataclasses import asdict, dataclass, fields
+from enum import Enum
from typing import TYPE_CHECKING, Any, Optional, Union
import torch
+GGUF_SCHEME_FACTS: dict[str, dict] = {
+ "gguf:q4_0": {
+ "bits": 4,
+ "act_bits": 16,
+ "group_size": 32,
+ "sym": True,
+ "data_type": "int",
+ "super_bits": None,
+ "super_group_size": None,
+ },
+ "gguf:q4_1": {
+ "bits": 4,
+ "act_bits": 16,
+ "group_size": 32,
+ "sym": False,
+ "data_type": "int_asym_float_zp",
+ "super_bits": None,
+ "super_group_size": None,
+ },
+ "gguf:q5_0": {
+ "bits": 5,
+ "act_bits": 16,
+ "group_size": 32,
+ "sym": True,
+ "data_type": "int",
+ "super_bits": None,
+ "super_group_size": None,
+ },
+ "gguf:q5_1": {
+ "bits": 5,
+ "act_bits": 16,
+ "group_size": 32,
+ "sym": False,
+ "data_type": "int_asym_float_zp",
+ "super_bits": None,
+ "super_group_size": None,
+ },
+ "gguf:q8_0": {
+ "bits": 8,
+ "act_bits": 16,
+ "group_size": 32,
+ "sym": True,
+ "data_type": "int",
+ "super_bits": None,
+ "super_group_size": None,
+ },
+ "gguf:q2_k": {
+ "bits": 2,
+ "act_bits": 16,
+ "group_size": 16,
+ "sym": False,
+ "data_type": "int_asym_dq",
+ "super_bits": 4,
+ "super_group_size": 16,
+ },
+ "gguf:q3_k": {
+ "bits": 3,
+ "act_bits": 16,
+ "group_size": 16,
+ "sym": True,
+ "data_type": "int_sym_dq",
+ "super_bits": 6,
+ "super_group_size": 16,
+ },
+ "gguf:q4_k": {
+ "bits": 4,
+ "act_bits": 16,
+ "group_size": 32,
+ "sym": False,
+ "data_type": "int_asym_dq",
+ "super_bits": 6,
+ "super_group_size": 8,
+ },
+ "gguf:q5_k": {
+ "bits": 5,
+ "act_bits": 16,
+ "group_size": 32,
+ "sym": False,
+ "data_type": "int_asym_dq",
+ "super_bits": 6,
+ "super_group_size": 8,
+ },
+ "gguf:q6_k": {
+ "bits": 6,
+ "act_bits": 16,
+ "group_size": 16,
+ "sym": True,
+ "data_type": "int_sym_dq",
+ "super_bits": 8,
+ "super_group_size": 16,
+ },
+ "gguf:bf16": {
+ "bits": 16,
+ "act_bits": 16,
+ "group_size": None,
+ "sym": True,
+ "data_type": "int_sym_dq",
+ "super_bits": None,
+ "super_group_size": None,
+ },
+}
+GGUF_SCHEME_FACTS["gguf:fp16"] = GGUF_SCHEME_FACTS["gguf:bf16"]
+
+GGUF_PRESET_ALIASES: dict[str, str] = {
+ "gguf:q4_0": "gguf:q4_0",
+ "gguf:q4_1": "gguf:q4_1",
+ "gguf:q5_0": "gguf:q5_0",
+ "gguf:q5_1": "gguf:q5_1",
+ "gguf:q2_k_s": "gguf:q2_k",
+ "gguf:q2_k_mixed": "gguf:q2_k",
+ "gguf:q3_k_s": "gguf:q3_k",
+ "gguf:q3_k_m": "gguf:q3_k",
+ "gguf:q3_k_l": "gguf:q3_k",
+ "gguf:q4_k_s": "gguf:q4_k",
+ "gguf:q4_k_m": "gguf:q4_k",
+ "gguf:q5_k_s": "gguf:q5_k",
+ "gguf:q5_k_m": "gguf:q5_k",
+ "gguf:q6_k": "gguf:q6_k",
+ "gguf:q8_0": "gguf:q8_0",
+}
+
from auto_round.logger import logger
from auto_round.utils import SUPPORTED_DTYPES, contain_any_mm_keys, infer_bits_by_data_type
-__all__ = ["QuantizationScheme", "get_gguf_scheme", "preset_name_to_scheme"]
+__all__ = [
+ "QuantizationScheme",
+ "BackendDataType",
+ "GGUF_SCHEME_FACTS",
+ "GGUF_PRESET_ALIASES",
+ "is_standard_fp",
+ "is_mx_fp",
+ "is_nv_fp",
+ "is_mx_int",
+ "get_gguf_scheme",
+ "preset_name_to_scheme",
+]
if TYPE_CHECKING:
from auto_round.auto_scheme.gen_auto_scheme import AutoScheme
+class BackendDataType(str, Enum):
+ STANDARD_FP = "fp"
+ MX_FP = "mx_fp"
+ NV_FP = "nv_fp"
+ MX_INT = "mx_int"
+ FP8_STATIC = "fp8_static"
+ FP8 = "fp8"
+
+
+# --- data_type string classifiers (single authority) ------------------------
+# These classify a raw data_type / backend string (e.g. "mx_fp4", "nv_fp",
+# "fp8_static"). They are the one place the substring rules live: both the
+# ``QuantizationScheme.is_*`` methods and ``compressors.utils`` (which re-exports
+# them for its many string call sites) delegate here, so there is no duplicate
+# copy of the "what makes a dtype mx_fp/nv_fp/..." logic.
+
+
+def is_mx_fp(backend: str) -> bool:
+ return BackendDataType.MX_FP in backend.lower()
+
+
+def is_nv_fp(backend: str) -> bool:
+ return BackendDataType.NV_FP in backend.lower()
+
+
+def is_mx_int(backend: str) -> bool:
+ return BackendDataType.MX_INT in backend.lower()
+
+
+def is_standard_fp(backend: str) -> bool:
+ backend = backend.lower()
+ return BackendDataType.STANDARD_FP in backend and not is_mx_fp(backend) and not is_nv_fp(backend)
+
+
@dataclass
class QuantizationScheme:
bits: int = 4
@@ -118,6 +285,78 @@ class QuantizationScheme:
return False
return True
+ # --- classification predicates (single authority, see D1) ---
+
+ def is_standard_fp(self) -> bool:
+ return self.data_type is not None and is_standard_fp(self.data_type)
+
+ def is_mx_fp(self) -> bool:
+ return self.data_type is not None and is_mx_fp(self.data_type)
+
+ def is_nv_fp(self) -> bool:
+ return self.data_type is not None and is_nv_fp(self.data_type)
+
+ def is_mx_int(self) -> bool:
+ return self.data_type is not None and is_mx_int(self.data_type)
+
+ def is_act_standard_fp(self) -> bool:
+ return self.act_data_type is not None and is_standard_fp(self.act_data_type)
+
+ def is_act_mx_fp(self) -> bool:
+ return self.act_data_type is not None and is_mx_fp(self.act_data_type)
+
+ def is_act_nv_fp(self) -> bool:
+ return self.act_data_type is not None and is_nv_fp(self.act_data_type)
+
+ def is_act_quantize(self) -> bool:
+ return self.act_bits is not None and self.act_bits <= 8
+
+ def is_wint_woq(self) -> bool:
+ """Integer weight-only quantization with non-quantized activations (`act_bits >= 16`)."""
+ return "int" in self.data_type and self.act_bits >= 16 and self.super_group_size is None
+
+ def is_wfp8afp8(self) -> bool:
+ if self.act_data_type is None or self.data_type is None:
+ return False
+ return (
+ ("fp8" in self.act_data_type or ("fp" in self.act_data_type and self.act_bits == 8))
+ and ("fp8" in self.data_type or ("fp" in self.data_type and self.bits == 8))
+ and self.is_act_standard_fp()
+ and self.is_standard_fp()
+ )
+
+ def is_wint8aint8(self) -> bool:
+ return ("int8" in self.act_data_type or ("int" in self.act_data_type and self.act_bits == 8)) and (
+ "int8" in self.data_type or ("int" in self.data_type and self.bits == 8)
+ )
+
+ def is_wint4aint4(self) -> bool:
+ return ("int4" in self.act_data_type or ("int" in self.act_data_type and self.act_bits == 4)) and (
+ "int4" in self.data_type or ("int" in self.data_type and self.bits == 4)
+ )
+
+ def is_dynamic_afp8(self) -> bool:
+ return self.act_dynamic and self.act_data_type.startswith("fp") and self.act_bits == 8
+
+ def is_block_wfp8(self) -> bool:
+ return (
+ isinstance(self.group_size, tuple)
+ and len(self.group_size) == 2
+ and self.data_type.startswith("fp")
+ and self.bits == 8
+ )
+
+ def is_static_afp8(self) -> bool:
+ return self.act_data_type is not None and BackendDataType.FP8_STATIC in self.act_data_type
+
+ def is_act_static(self) -> bool:
+ return not self.act_dynamic
+
+ def is_dynamic_wint8aint8(self) -> bool:
+ if not self.act_dynamic:
+ return False
+ return self.is_wint8aint8()
+
def preset_name_to_scheme(name: str) -> QuantizationScheme:
"""Get a QuantizationScheme instance from a preset scheme name."""
@@ -575,14 +814,9 @@ PRESET_SCHEMES = {
"FP8_BLOCK": FP8_BLOCK,
"MXINT4": MXINT4,
}
-from auto_round.export.export_to_gguf.config import GGUF_CONFIG
-for key, val in GGUF_CONFIG.items():
- value = copy.deepcopy(val)
- value.pop("mostly", None)
- value.pop("embedding", None)
- value.pop("lm_head", None)
- PRESET_SCHEMES[key.upper()] = QuantizationScheme.from_dict(value)
+for alias_key, facts_key in GGUF_PRESET_ALIASES.items():
+ PRESET_SCHEMES[alias_key.upper()] = QuantizationScheme.from_dict(GGUF_SCHEME_FACTS[facts_key])
def _handle_special_schemes(
@@ -653,6 +887,8 @@ def _handle_special_schemes(
def get_gguf_scheme(scheme: Union[str, QuantizationScheme]) -> str:
+ if scheme is None:
+ return ""
if isinstance(scheme, str) and scheme.upper().startswith("GGUF"):
return scheme
if isinstance(scheme, str):
diff --git a/auto_round/special_model_handler.py b/auto_round/special_model_handler.py
index bcddba7fd..c73e0e5c4 100644
--- a/auto_round/special_model_handler.py
+++ b/auto_round/special_model_handler.py
@@ -17,7 +17,7 @@ from typing import Any, Callable
import torch
-from auto_round.formats import OutputFormat
+from auto_round.export.formats import OutputFormat
from auto_round.modeling.fused_moe.replace_modules import apply_replacements, release_original_module_
from auto_round.utils import is_moe_model_via_config, logger
diff --git a/docs/step_by_step.md b/docs/step_by_step.md
index 74a178834..f10466382 100644
--- a/docs/step_by_step.md
+++ b/docs/step_by_step.md
@@ -363,12 +363,12 @@ AWQ-specific options:
#### API Usage
```python
-from auto_round import AutoRound
+from auto_round import AWQConfig, AutoRound
ar = AutoRound(
"Qwen/Qwen3-0.6B",
scheme="INT8",
- algorithm="awq",
+ alg_configs=AWQConfig(),
)
output_dir = "./tmp_awq"
@@ -489,7 +489,7 @@ When using AutoScheme with `model_free=True`, only INT (`W2A16`/`W4A16`/`W8A16`)
### AWQ Quantization Algorithm
-AWQ (`algorithm="awq"`) is a pre-processing quantization algorithm that analyzes activation patterns and applies channel-wise scaling to protect salient weights. It runs BEFORE the actual quantization (RTN by default, or auto_round/SignRound).
+AWQ (`alg_configs="awq"` or `alg_configs=AWQConfig()`) is a pre-processing quantization algorithm that analyzes activation patterns and applies channel-wise scaling to protect salient weights. It runs BEFORE the actual quantization (RTN by default, or auto_round/SignRound).
#### CLI Usage
```bash
@@ -506,21 +506,22 @@ auto-round --model Qwen/Qwen3-0.6B --algorithm awq,auto_round --scheme W4A16
#### API Usage
```python
-from auto_round import AutoRound
-from auto_round.algorithms.quantization.awq.config import AWQConfig
-from auto_round.algorithms.quantization.sign_round.config import SignRoundConfig
+from auto_round import AWQConfig, AutoRound, SignRoundConfig
+
+# String alias (AWQ defaults, with RTN appended automatically)
+ar = AutoRound(model, tokenizer, alg_configs="awq", scheme="W4A16")
# AWQ + default RTN (simplest)
-ar = AutoRound(model, tokenizer, algorithm="awq", scheme="W4A16")
+ar = AutoRound(model, tokenizer, alg_configs=AWQConfig(), scheme="W4A16")
# AWQ + AutoRound via alg_configs (explicit pipeline)
ar = AutoRound(model, tokenizer, alg_configs=[AWQConfig(), SignRoundConfig(iters=200)], scheme="W4A16")
ar.quantize_and_save(output_dir="./qmodel")
```
-**Important Note**: `algorithm="awq"` (quantization algorithm) and `format="auto_awq"` (export format) are independent. You can use:
-- `algorithm="awq"` + `format="auto_round"`: AWQ smoothing + AutoRound packing
-- `algorithm="auto_round"` + `format="auto_awq"`: No AWQ smoothing + AutoAWQ packing
+**Important Note**: `alg_configs="awq"` or `alg_configs=AWQConfig()` (quantization algorithm) and `format="auto_awq"` (export format) are independent. You can use:
+- `alg_configs="awq"` + `format="auto_round"`: AWQ smoothing + AutoRound packing
+- `alg_configs="signround"` + `format="auto_awq"`: No AWQ smoothing + AutoAWQ packing
### OPT RTN Mode
AutoRound also supports Optimized RTN (Round-To-Nearest) mode for fast, calibration-free baseline quantization. Setting `iters=0` tp enable it and we recommend using `group_size=32` for better results. Check [accuracy comparison](./opt_rtn.md) between RTN and OPT RTN mode
diff --git a/docs/step_by_step_CN.md b/docs/step_by_step_CN.md
index 9163b93d8..77b5eedfa 100644
--- a/docs/step_by_step_CN.md
+++ b/docs/step_by_step_CN.md
@@ -360,12 +360,12 @@ AWQ 专用选项:
W8A8 搭配 AWQ 平滑化:
```python
-from auto_round import AutoRound
+from auto_round import AWQConfig, AutoRound
ar = AutoRound(
"Qwen/Qwen3-0.6B",
scheme="INT8",
- algorithm="awq",
+ alg_configs=AWQConfig(),
)
output_dir = "./tmp_awq"
@@ -487,7 +487,7 @@ AutoScheme 目前还**不支持对嵌入层(Embedding layer)进行自动量
### AWQ 量化算法
-AWQ(`algorithm="awq"`)是一种预处理量化算法,通过分析激活分布并应用通道缩放(channel-wise scaling)来保护重要的权重。它在实际量化(默认为 RTN,或使用 auto_round/SignRound)之前运行。
+AWQ(`alg_configs="awq"` 或 `alg_configs=AWQConfig()`)是一种预处理量化算法,通过分析激活分布并应用通道缩放(channel-wise scaling)来保护重要的权重。它在实际量化(默认为 RTN,或使用 auto_round/SignRound)之前运行。
#### 命令行用法
```bash
@@ -504,21 +504,22 @@ auto-round --model Qwen/Qwen3-0.6B --algorithm awq,auto_round --scheme W4A16
#### API 用法
```python
-from auto_round import AutoRound
-from auto_round.algorithms.quantization.awq.config import AWQConfig
-from auto_round.algorithms.quantization.sign_round.config import SignRoundConfig
+from auto_round import AWQConfig, AutoRound, SignRoundConfig
+
+# 字符串别名(使用 AWQ 默认参数,并自动追加 RTN)
+ar = AutoRound(model, tokenizer, alg_configs="awq", scheme="W4A16")
# AWQ + 默认 RTN (最简用法)
-ar = AutoRound(model, tokenizer, algorithm="awq", scheme="W4A16")
+ar = AutoRound(model, tokenizer, alg_configs=AWQConfig(), scheme="W4A16")
# 通过 alg_configs 指定 AWQ + AutoRound (显式流水线)
ar = AutoRound(model, tokenizer, alg_configs=[AWQConfig(), SignRoundConfig(iters=200)], scheme="W4A16")
ar.quantize_and_save(output_dir="./qmodel")
```
-**重要提示**:`algorithm="awq"`(量化算法)与 `format="auto_awq"`(导出格式)是相互独立的。你可以使用:
-- `algorithm="awq"` + `format="auto_round"`:AWQ 平滑 + AutoRound 打包
-- `algorithm="auto_round"` + `format="auto_awq"`:不使用 AWQ 平滑 + AutoAWQ 打包
+**重要提示**:`alg_configs="awq"` 或 `alg_configs=AWQConfig()`(量化算法)与 `format="auto_awq"`(导出格式)是相互独立的。你可以使用:
+- `alg_configs="awq"` + `format="auto_round"`:AWQ 平滑 + AutoRound 打包
+- `alg_configs="signround"` + `format="auto_awq"`:不使用 AWQ 平滑 + AutoAWQ 打包
### OPT-RTN 模式
AutoRound 还提供优化版 RTN(Round-To-Nearest,就近舍入)模式,无需标定数据即可实现快速基线量化。**启用方式为 `iters=0`**。同时为获得更好的效果,推荐搭配 `group_size=32` 。RTN 与 OPT RTN 模式的精度对比详见[《精度对比报告》](./opt_rtn.md)。
diff --git a/test/test_cpu/algorithms/test_awq.py b/test/test_cpu/algorithms/test_awq.py
index 32966bd07..1bb27dbd7 100644
--- a/test/test_cpu/algorithms/test_awq.py
+++ b/test/test_cpu/algorithms/test_awq.py
@@ -28,7 +28,7 @@ import pytest
import torch
from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer
-from auto_round import AutoRound
+from auto_round import AutoRound, AWQConfig, SignRoundConfig
from ...helpers import generate_prompt, get_model_path, opt_name_or_path, save_tiny_model
@@ -50,7 +50,7 @@ class TestAWQNormalLLM:
ar = AutoRound(
tiny_opt_model_path,
scheme="W4A16",
- algorithm="awq",
+ alg_configs=AWQConfig(n_grid=1),
n_grid=1,
nsamples=2,
seqlen=32,
@@ -74,7 +74,7 @@ class TestAWQNormalLLM:
ar = AutoRound(
tiny_opt_model_path,
scheme="W4A16",
- algorithm="awq",
+ alg_configs=AWQConfig(n_grid=1),
n_grid=1,
nsamples=2,
seqlen=8,
@@ -98,7 +98,7 @@ class TestAWQNormalLLM:
bits=bits,
group_size=group_size,
sym=sym,
- algorithm="awq",
+ alg_configs=AWQConfig(n_grid=1),
n_grid=1,
nsamples=2,
seqlen=8,
@@ -117,7 +117,7 @@ class TestAWQNonIntegerSchemes:
"""Regression: AWQ smoothing must run under non-integer schemes (MX/NV-FP).
AWQ's grid-search / clip loss reproduces the block quantizer's weight QDQ.
- The reported failure mode was an end-to-end ``algorithm='awq'`` run raising
+ The reported failure mode was an end-to-end AWQ run raising
under an MXFP/NVFP scheme.
"""
@@ -126,11 +126,12 @@ class TestAWQNonIntegerSchemes:
ar = AutoRound(
tiny_opt_model_path,
scheme=scheme,
- algorithm="awq,signround",
+ alg_configs=[AWQConfig(n_grid=1), SignRoundConfig()],
n_grid=1,
nsamples=2,
seqlen=8,
batch_size=2,
+ dataset=["local AWQ calibration sample with enough tokens for quantization"] * 2,
)
model, layer_config = ar.quantize()
@@ -158,7 +159,7 @@ class TestAWQW8A8LLMCompressor:
ar = AutoRound(
tiny_opt_model_path,
scheme="INT8",
- algorithm="awq",
+ alg_configs=AWQConfig(n_grid=1),
nsamples=2,
seqlen=8,
n_grid=1,
@@ -246,7 +247,7 @@ class TestAWQMoE:
ar = AutoRound(
tiny_qwen_moe_model_path,
scheme="W4A16",
- algorithm="awq",
+ alg_configs=AWQConfig(n_grid=1),
n_grid=1,
nsamples=2,
seqlen=8,
@@ -280,7 +281,7 @@ class TestAWQMoE:
ar = AutoRound(
tiny_qwen_moe_model_path,
scheme="W4A16",
- algorithm="awq",
+ alg_configs=AWQConfig(n_grid=1),
n_grid=1,
nsamples=2,
seqlen=32,
@@ -456,6 +457,24 @@ class TestAWQUseV2ScaleSearch:
assert init_scale is not None, dt
assert init_scale.shape[0] == weight_reshape.shape[0]
+ # Scalar imatrix sentinels mean uniform importance and must not enter
+ # the tensor reshape path.
+ scalar_init_scale = search_optimized_init_scale(weight_reshape, "mx_fp4", 4, 1.0)
+ assert scalar_init_scale is not None
+ assert scalar_init_scale.shape == init_scale.shape
+
# asym int and *_dq are not part of the optimized init-scale path.
assert search_optimized_init_scale(torch.randn(4, 128), "int_asym", 4, None) is None
assert search_optimized_init_scale(torch.randn(4, 128), "int_sym_dq", 4, None) is None
+
+ def test_nvfp4_opt_rtn_accepts_uniform_imatrix_sentinel(self):
+ """AWQ's SignRound-V1 QDQ may call optimized RTN without a collected imatrix."""
+ import torch
+
+ from auto_round.data_type.nvfp import opt_rtn_fast_nvfp4
+
+ weight = torch.randn(8, 32)
+ qdq_weight, _, _ = opt_rtn_fast_nvfp4(weight, bits=4, group_size=16, imatrix=1.0)
+
+ assert qdq_weight.shape == weight.shape
+ assert torch.isfinite(qdq_weight).all()
diff --git a/test/test_cpu/config_resolution/test_config_snapshots.py b/test/test_cpu/config_resolution/test_config_snapshots.py
new file mode 100644
index 000000000..fe36b1005
--- /dev/null
+++ b/test/test_cpu/config_resolution/test_config_snapshots.py
@@ -0,0 +1,110 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pytest
+import torch
+import torch.nn as nn
+
+from auto_round.compressors.config_resolution import resolve_quantization_config, resolve_scheme_value
+from auto_round.export.formats import resolve_formats
+
+EXPECTED_PLAN_SNAPSHOTS = {
+ "w4_autoround": {
+ "scheme_name": "W4A16",
+ "format_name": "auto_round",
+ "backend": "auto_round:auto_gptq",
+ "bits": 4,
+ "group_size": 128,
+ "data_type": "int",
+ "act_bits": 16,
+ "act_data_type": "float",
+ "scale_dtype": "torch.float16",
+ },
+ "fp8_static_llmc": {
+ "scheme_name": "FP8_STATIC",
+ "format_name": "llm_compressor",
+ "backend": "llm_compressor:fp8_static",
+ "bits": 8,
+ "group_size": -1,
+ "data_type": "fp",
+ "act_bits": 8,
+ "act_data_type": "fp",
+ "scale_dtype": "torch.float16",
+ },
+ "gguf_q4_k_m": {
+ "scheme_name": "GGUF:Q4_K_M",
+ "format_name": "gguf:q4_k_m",
+ "backend": "gguf:q4_k_m",
+ "bits": 4,
+ "group_size": 32,
+ "data_type": "int_asym_dq",
+ "act_bits": 16,
+ "act_data_type": "float",
+ "scale_dtype": "torch.float32",
+ },
+ "mxfp4_autoround": {
+ "scheme_name": "MXFP4",
+ "format_name": "auto_round",
+ "backend": "auto_round:mx_fp",
+ "bits": 4,
+ "group_size": 32,
+ "data_type": "mx_fp",
+ "act_bits": 4,
+ "act_data_type": "mx_fp",
+ "scale_dtype": "torch.float16",
+ },
+ "nvfp4_autoround": {
+ "scheme_name": "NVFP4",
+ "format_name": "auto_round",
+ "backend": "auto_round:nv_fp",
+ "bits": 4,
+ "group_size": 16,
+ "data_type": "nv_fp",
+ "act_bits": 4,
+ "act_data_type": "nv_fp4_with_static_gs",
+ "scale_dtype": "torch.float16",
+ },
+}
+
+
[email protected]("case_name", EXPECTED_PLAN_SNAPSHOTS)
+def test_scheme_format_plan_snapshot(case_name):
+ expected = EXPECTED_PLAN_SNAPSHOTS[case_name]
+ scheme = resolve_scheme_value(expected["scheme_name"], {})
+ resolution = resolve_formats(
+ scheme,
+ format=expected["format_name"],
+ layer_config={"layer": {"bits": scheme.value.bits}},
+ scale_dtype=torch.float16,
+ model=nn.Sequential(nn.Linear(32, 32)),
+ )
+ plan = resolve_quantization_config(resolution, {"layer": {"bits": resolution.scheme.value.bits}})
+ scheme_value = plan.scheme.value
+ snapshot = {
+ "scheme_name": expected["scheme_name"],
+ "format_name": expected["format_name"],
+ "backend": plan.formats[0].get_backend_name(),
+ "bits": scheme_value.bits,
+ "group_size": scheme_value.group_size,
+ "data_type": scheme_value.data_type,
+ "act_bits": scheme_value.act_bits,
+ "act_data_type": scheme_value.act_data_type,
+ "scale_dtype": str(plan.scale_dtype),
+ }
+
+ assert snapshot == expected
+ assert {name: dict(config) for name, config in plan.layer_config.items()} == {"layer": {"bits": expected["bits"]}}
+ assert dict(plan.regex_config) == {}
+ assert plan.has_qlayer_outside_block is False
+ assert plan.quant_block_list is None
diff --git a/test/test_cpu/config_resolution/test_resolution.py b/test/test_cpu/config_resolution/test_resolution.py
new file mode 100644
index 000000000..8fdc8781c
--- /dev/null
+++ b/test/test_cpu/config_resolution/test_resolution.py
@@ -0,0 +1,85 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from dataclasses import FrozenInstanceError
+from types import SimpleNamespace
+
+import pytest
+
+from auto_round.auto_scheme.gen_auto_scheme import AutoScheme
+from auto_round.cli import main as cli_main
+from auto_round.compressors.config_resolution import (
+ FormatCompatibilityError,
+ ResolvedScheme,
+ SchemeResolutionError,
+ resolve_scheme_value,
+)
+from auto_round.schemes import QuantizationScheme
+
+
+def test_resolve_scheme_value_preserves_precise_gguf_identity():
+ resolved = resolve_scheme_value("GGUF:Q4_K_M", {})
+
+ assert resolved.preset_name == "gguf:q4_k_m"
+ assert resolved.value.bits == 4
+ assert resolved.value.data_type == "int_asym_dq"
+ assert not isinstance(resolved.value, str)
+
+
+def test_resolve_scheme_value_applies_overrides_without_mutating_input():
+ overrides = {"bits": 8}
+
+ resolved = resolve_scheme_value("W4A16", overrides)
+
+ assert resolved.preset_name == "W4A16"
+ assert resolved.value.bits == 8
+ assert overrides == {"bits": 8}
+
+
+def test_resolve_scheme_value_defers_model_aware_auto_scheme():
+ scheme = AutoScheme(avg_bits=4.0, options=["W4A16", "W8A16"])
+
+ with pytest.raises(SchemeResolutionError, match="model-aware"):
+ resolve_scheme_value(scheme, {})
+
+ assert scheme.options == ["W4A16", "W8A16"]
+
+
+def test_resolved_scheme_is_deeply_isolated_and_frozen():
+ source = QuantizationScheme(bits=4, data_type="int", rotation_config={"mode": "hadamard"})
+ resolved = ResolvedScheme.from_scheme(source, preset_name="W4A16")
+
+ source.bits = 8
+ source.rotation_config["mode"] = "random"
+ leaked_copy = resolved.value
+ leaked_copy.bits = 2
+ leaked_copy.rotation_config["mode"] = "identity"
+
+ assert resolved.value.bits == 4
+ assert resolved.value.rotation_config == {"mode": "hadamard"}
+ with pytest.raises(FrozenInstanceError):
+ resolved.preset_name = "W8A16"
+
+
+def test_cli_converts_domain_error_to_nonzero_exit(monkeypatch):
+ messages = []
+ monkeypatch.setattr(cli_main.sys, "argv", ["auto-round"])
+ monkeypatch.setattr(cli_main, "start", lambda argv=None: (_ for _ in ()).throw(FormatCompatibilityError("bad")))
+ monkeypatch.setattr(cli_main, "logger", SimpleNamespace(error=messages.append))
+
+ with pytest.raises(SystemExit) as error:
+ cli_main.run()
+
+ assert error.value.code != 0
+ assert messages == ["bad"]
diff --git a/test/test_cpu/core/test_autoopt.py b/test/test_cpu/core/test_autoopt.py
index e65ff6436..7dfffea94 100644
--- a/test/test_cpu/core/test_autoopt.py
+++ b/test/test_cpu/core/test_autoopt.py
@@ -6,7 +6,7 @@ import torch
import transformers
from transformers import AutoModelForCausalLM, AutoTokenizer
-from auto_round import AutoRoundAdam
+from auto_round import AdamRoundConfig, AutoRound
class TestAutoRound:
@@ -29,13 +29,15 @@ class TestAutoRound:
llm_block_names = get_block_names(tiny_opt_model, quant_vision=True)
bits, group_size, sym, batch_size = 4, 128, False, 20
- adamround = AutoRoundAdam(
+ adamround = AutoRound(
tiny_opt_model,
opt_tokenizer,
- bits=bits,
- group_size=group_size,
- sym=sym,
- iters=2,
+ alg_configs=AdamRoundConfig(
+ bits=bits,
+ group_size=group_size,
+ sym=sym,
+ iters=2,
+ ),
seqlen=2,
batch_size=batch_size,
dataset=dataloader,
diff --git a/test/test_cpu/core/test_autoround_entry.py b/test/test_cpu/core/test_autoround_entry.py
new file mode 100644
index 000000000..0b4d8c0ea
--- /dev/null
+++ b/test/test_cpu/core/test_autoround_entry.py
@@ -0,0 +1,212 @@
+import inspect
+
+import pytest
+
+from auto_round.algorithms.quantization.rtn.config import RTNConfig
+from auto_round.algorithms.quantization.sign_round.config import SignRoundConfig, SignRoundV2Config
+from auto_round.algorithms.transforms.awq.config import AWQConfig
+from auto_round.algorithms.transforms.hadamard.config import RotationConfig
+from auto_round.autoround import _normalize_alg_configs
+
+
+def _types(configs):
+ return [type(config).__name__ for config in configs]
+
+
+def test_default_alg_configs_is_signround():
+ configs = _normalize_alg_configs(None)
+
+ assert _types(configs) == ["SignRoundConfig"]
+
+
+def test_public_entry_keeps_legacy_algorithm_parameter():
+ from auto_round import AutoRound
+
+ assert "algorithm" in inspect.signature(AutoRound.__new__).parameters
+
+
+ "value",
+ [
+ "signround",
+ SignRoundConfig(iters=3),
+ [SignRoundConfig(iters=3)],
+ ("signround",),
+ ],
+)
+def test_alg_configs_accepts_alias_config_and_sequence(value):
+ configs = _normalize_alg_configs(value)
+
+ assert len(configs) == 1
+ assert isinstance(configs[0], SignRoundConfig)
+
+
+def test_awq_only_appends_rtn():
+ configs = _normalize_alg_configs(AWQConfig())
+
+ assert isinstance(configs[0], AWQConfig)
+ assert isinstance(configs[1], RTNConfig)
+
+
+def test_awq_and_signround_does_not_append_rtn():
+ configs = _normalize_alg_configs([AWQConfig(), SignRoundConfig(iters=3)])
+
+ assert _types(configs) == ["AWQConfig", "SignRoundConfig"]
+
+
+def test_direct_signround_kwargs_warn_and_apply(monkeypatch):
+ warnings = []
+ monkeypatch.setattr("auto_round.autoround.logger.warning", lambda message, *args: warnings.append(message % args))
+
+ configs = _normalize_alg_configs(None, direct_kwargs={"iters": 7, "lr": 0.2})
+
+ assert isinstance(configs[0], SignRoundConfig)
+ assert configs[0].iters == 7
+ assert configs[0].lr == 0.2
+ assert any("alg_configs" in message for message in warnings)
+
+
+def test_zero_iters_selects_rtn_and_forwards_disable_opt_rtn():
+ configs = _normalize_alg_configs(None, direct_kwargs={"iters": 0, "disable_opt_rtn": True})
+
+ assert len(configs) == 1
+ assert isinstance(configs[0], RTNConfig)
+ assert configs[0].disable_opt_rtn is True
+
+
[email protected]("algorithm", ["signround", "auto_round", "autoround"])
+def test_legacy_signround_alias_with_zero_iters_selects_rtn(algorithm):
+ legacy_configs = _normalize_alg_configs(None, direct_kwargs={"algorithm": algorithm, "iters": 0})
+ unified_configs = _normalize_alg_configs(algorithm, direct_kwargs={"iters": 0})
+
+ assert len(legacy_configs) == len(unified_configs) == 1
+ assert isinstance(legacy_configs[0], RTNConfig)
+ assert isinstance(unified_configs[0], RTNConfig)
+
+
+def test_explicit_zero_iter_signround_config_selects_rtn():
+ configs = _normalize_alg_configs(SignRoundConfig(iters=0))
+
+ assert len(configs) == 1
+ assert isinstance(configs[0], RTNConfig)
+
+
+def test_combined_awq_zero_iters_keeps_awq_and_selects_rtn():
+ configs = _normalize_alg_configs(None, direct_kwargs={"algorithm": "awq,signround", "iters": 0})
+
+ assert _types(configs) == ["AWQConfig", "OptimizedRTNConfig"]
+
+
+ ("algorithm", "expected_types"),
+ [
+ ("rtn", ["OptimizedRTNConfig"]),
+ ("awq", ["AWQConfig", "OptimizedRTNConfig"]),
+ ("signround", ["SignRoundConfig"]),
+ ("awq,signround", ["AWQConfig", "SignRoundConfig"]),
+ ],
+)
+def test_legacy_algorithm_matches_alg_configs_aliases(algorithm, expected_types):
+ legacy_configs = _normalize_alg_configs(None, direct_kwargs={"algorithm": algorithm})
+ unified_configs = _normalize_alg_configs(algorithm)
+
+ assert _types(legacy_configs) == expected_types
+ assert _types(unified_configs) == expected_types
+
+
+def test_algorithm_and_alg_configs_are_mutually_exclusive():
+ with pytest.raises(ValueError, match="cannot be used together"):
+ _normalize_alg_configs("signround", direct_kwargs={"algorithm": "rtn"})
+
+
+def test_string_and_string_sequence_use_same_alias_deduplication():
+ comma_configs = _normalize_alg_configs("awq,awq,signround")
+ sequence_configs = _normalize_alg_configs(["awq", "awq", "signround"])
+
+ assert _types(comma_configs) == ["AWQConfig", "SignRoundConfig"]
+ assert _types(sequence_configs) == _types(comma_configs)
+
+
+def test_legacy_combined_algorithm_forwards_each_algorithm_option():
+ configs = _normalize_alg_configs(
+ None,
+ direct_kwargs={"algorithm": "awq,signround", "n_grid": 7, "iters": 3, "enable_alg_ext": False},
+ )
+
+ assert _types(configs) == ["AWQConfig", "SignRoundConfig"]
+ assert configs[0].n_grid == 7
+ assert configs[1].iters == 3
+ assert configs[1].enable_alg_ext is False
+
+
+ "configs",
+ [
+ lambda: _normalize_alg_configs(None, direct_kwargs={"algorithm": "awq,signround", "iters": 3}),
+ lambda: _normalize_alg_configs([AWQConfig(), SignRoundConfig(iters=3)]),
+ ],
+)
+def test_legacy_and_unified_composition_select_same_v1_quantizer(configs):
+ from auto_round.algorithms.composer import AlgorithmComposer
+
+ composer = AlgorithmComposer(configs())
+
+ assert [type(pre).__name__ for pre in composer.preprocessors] == ["AWQTransform"]
+ assert type(composer.block_quantizer).__name__ == "SignRoundQuantizer"
+
+
+def test_direct_enable_opt_rtn_forces_optimized_rtn():
+ configs = _normalize_alg_configs(None, direct_kwargs={"iters": 0, "enable_opt_rtn": True})
+
+ assert len(configs) == 1
+ assert isinstance(configs[0], RTNConfig)
+ assert configs[0].disable_opt_rtn is False
+ assert configs[0].orig_disable_opt_rtn is False
+
+
+def test_rtn_switches_only_when_iters_is_zero():
+ configs = _normalize_alg_configs(None, direct_kwargs={"disable_opt_rtn": True})
+
+ assert isinstance(configs[0], SignRoundConfig)
+
+
+def test_direct_enable_alg_ext_normalizes_signround_variant():
+ configs = _normalize_alg_configs(None, direct_kwargs={"iters": 1, "enable_alg_ext": True})
+
+ assert len(configs) == 1
+ assert isinstance(configs[0], SignRoundV2Config)
+
+
+def test_direct_rotation_config_is_added_to_pipeline():
+ configs = _normalize_alg_configs(None, direct_kwargs={"rotation_config": "default"})
+
+ assert any(isinstance(config, RotationConfig) for config in configs)
+
+
+def test_direct_enable_lfq_is_forwarded():
+ configs = _normalize_alg_configs(None, direct_kwargs={"enable_lfq": True})
+
+ assert isinstance(configs[0], SignRoundConfig)
+ assert configs[0].enable_lfq is True
+
+
+def test_rotation_config_accepts_spinquant_shorthand():
+ configs = _normalize_alg_configs(None, direct_kwargs={"rotation_config": "quarot"})
+
+ assert any(type(config).__name__ == "SpinQuantConfig" for config in configs)
+
+
+def test_rotation_backend_must_be_nested_in_rotation_config():
+ with pytest.raises(ValueError, match="rotation_config"):
+ _normalize_alg_configs(None, direct_kwargs={"backend": "transform"})
+
+
+def test_awq_kwargs_without_awq_error_and_are_ignored(monkeypatch):
+ errors = []
+ monkeypatch.setattr("auto_round.autoround.logger.error", lambda message, *args: errors.append(message % args))
+
+ configs = _normalize_alg_configs(None, direct_kwargs={"n_grid": 7, "apply_clip": True})
+
+ assert isinstance(configs[0], SignRoundConfig)
+ assert not hasattr(configs[0], "n_grid")
+ assert any("AWQ" in message for message in errors)
diff --git a/test/test_cpu/core/test_awq_autoround_smoke.py b/test/test_cpu/core/test_awq_autoround_smoke.py
index d529c9c6e..0f7d53faf 100644
--- a/test/test_cpu/core/test_awq_autoround_smoke.py
+++ b/test/test_cpu/core/test_awq_autoround_smoke.py
@@ -1,21 +1,24 @@
"""Minimal runtime smoke for AWQ + AutoRound fusion."""
+from auto_round import AutoRound
from auto_round.algorithms.quantization.sign_round.config import SignRoundConfig
from auto_round.algorithms.transforms.awq.config import AWQConfig
-from auto_round.compressors.entry import AutoRound
-def test_awq_plus_autoround_quantize_smoke(tiny_opt_model_path):
+def test_awq_plus_autoround_quantize_smoke(tiny_opt_model_path, dataloader):
ar = AutoRound(
tiny_opt_model_path,
scheme="W4A16",
alg_configs=[AWQConfig(n_grid=2), SignRoundConfig(iters=1)],
nsamples=1,
seqlen=8,
+ dataset=dataloader,
low_cpu_mem_usage=False,
)
model, layer_config = ar.quantize()
+ assert [type(pre).__name__ for pre in ar.alg_composer.preprocessors] == ["AWQTransform"]
+ assert type(ar.alg_composer.block_quantizer).__name__ == "SignRoundQuantizer"
assert model is not None
assert layer_config
diff --git a/test/test_cpu/core/test_compression_plan_state.py b/test/test_cpu/core/test_compression_plan_state.py
new file mode 100644
index 000000000..38f0f3be2
--- /dev/null
+++ b/test/test_cpu/core/test_compression_plan_state.py
@@ -0,0 +1,95 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import inspect
+
+from auto_round import AutoRound, RTNConfig
+from auto_round.compressors.base import BaseCompressor
+
+
+def test_post_init_builds_authoritative_compression_plan(tiny_opt_model_path):
+ compressor = AutoRound(
+ tiny_opt_model_path,
+ scheme="W4A16",
+ alg_configs=RTNConfig(),
+ format="auto_round",
+ nsamples=1,
+ seqlen=8,
+ dataset=["local calibration sample"],
+ low_cpu_mem_usage=False,
+ )
+
+ compressor.post_init()
+
+ assert compressor._format_resolution.scheme.preset_name == compressor.compression_plan.scheme.preset_name
+ assert compressor._format_resolution.formats == compressor.compression_plan.formats
+ assert compressor.compression_plan.scheme.value == compressor.scheme_context
+ assert compressor.compression_plan.formats == tuple(compressor.formats)
+ assert dict(compressor.compression_plan.layer_config) == compressor.layer_config
+ assert dict(compressor.compression_plan.regex_config) == compressor.regex_config
+ assert compressor.compression_plan.has_qlayer_outside_block == compressor.has_qlayer_outside_block
+
+
+def test_legacy_state_views_cannot_mutate_authoritative_plan(tiny_opt_model_path):
+ compressor = AutoRound(
+ tiny_opt_model_path,
+ scheme="W4A16",
+ alg_configs=RTNConfig(),
+ format="auto_round",
+ nsamples=1,
+ seqlen=8,
+ dataset=["local calibration sample"],
+ low_cpu_mem_usage=False,
+ )
+ compressor.post_init()
+
+ compressor.scheme_context.bits = 2
+ compressor.formats.clear()
+ compressor.layer_config.clear()
+ compressor.regex_config.clear()
+
+ assert compressor.scheme_context.bits == compressor.compression_plan.scheme.value.bits == 4
+ assert compressor.formats == list(compressor.compression_plan.formats)
+ assert compressor.layer_config == {
+ name: dict(config) for name, config in compressor.compression_plan.layer_config.items()
+ }
+ assert compressor.regex_config == {
+ name: dict(config) for name, config in compressor.compression_plan.regex_config.items()
+ }
+
+
+def test_quantizer_sync_reads_authoritative_plan_instead_of_parallel_state():
+ source = inspect.getsource(BaseCompressor._build_layer_config)
+
+ assert "self.quantizer.layer_config = self.layer_config" not in source
+ assert "self.quantizer.regex_config = self.regex_config" not in source
+ assert "self.quantizer.scale_dtype = self.scale_dtype" not in source
+
+
+def test_format_resolution_does_not_copy_scheme_fields_one_by_one():
+ source = inspect.getsource(BaseCompressor._resolve_format_string)
+
+ assert "fields(QuantizationScheme)" not in source
+
+
+def test_final_layer_config_path_does_not_call_legacy_mutating_adapter():
+ source = inspect.getsource(BaseCompressor.configure_layer_config)
+
+ assert "set_layer_config(" not in source
+
+
+def test_auto_scheme_layer_discovery_uses_pure_resolver():
+ source = inspect.getsource(BaseCompressor._gen_auto_scheme)
+
+ assert "set_layer_config(" not in source
diff --git a/test/test_cpu/core/test_entry_contract.py b/test/test_cpu/core/test_entry_contract.py
new file mode 100644
index 000000000..0ebda2063
--- /dev/null
+++ b/test/test_cpu/core/test_entry_contract.py
@@ -0,0 +1,50 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from auto_round import autoround
+from auto_round.autoround import _split_entry_kwargs
+
+
+def test_split_entry_kwargs_partitions_owned_fields():
+ processor = object()
+
+ grouped = _split_entry_kwargs(
+ {"dataset": ["sample"], "scale_dtype": "fp32", "processor": processor, "model_free": False}
+ )
+
+ assert grouped["base"]["dataset"] == ["sample"]
+ assert grouped["compressor"]["scale_dtype"] == "fp32"
+ assert grouped["mllm"]["processor"] is processor
+ assert grouped["route"]["model_free"] is False
+
+
+def test_split_entry_kwargs_ignores_unknown_fields(monkeypatch):
+ warnings = []
+ monkeypatch.setattr(autoround.logger, "warning_once", lambda message, *args: warnings.append(message % args))
+
+ grouped = _split_entry_kwargs({"unknown_option": 1}, context="test entry")
+
+ assert all(not values for values in grouped.values())
+ assert "unknown_option" in warnings[0]
+
+
+def test_cli_explicit_auto_round_with_zero_iters_selects_rtn():
+ from argparse import Namespace
+
+ from auto_round.cli.algorithms import AlgorithmHandler
+
+ args = Namespace(algorithm="auto_round", iters=0)
+ configs = AlgorithmHandler.build_configs(args, {})
+
+ assert type(configs[0]).__name__ == "RTNConfig"
diff --git a/test/test_cpu/core/test_entry_scheme_unification.py b/test/test_cpu/core/test_entry_scheme_unification.py
new file mode 100644
index 000000000..b6362f383
--- /dev/null
+++ b/test/test_cpu/core/test_entry_scheme_unification.py
@@ -0,0 +1,104 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Entry-layer scheme unification: routing decisions in ``entry.py`` read scheme
+fields from a single resolved-scheme source (``_preview_resolved_attrs``) rather
+than double-reading raw config attrs. These lock that the resolved values — and
+the resulting compressor-class routing — are identical whether the user passes
+``scheme=`` alone or the equivalent bit/dtype overrides on the alg config.
+"""
+
+import ast
+import inspect
+
+from auto_round.algorithms.quantization.rtn.config import OptimizedRTNConfig, RTNConfig
+from auto_round.autoround import (
+ _collect_config_scheme_overrides,
+ _preview_resolved_attrs,
+ _select_rtn_compressor_base_cls,
+)
+from auto_round.compressors.orchestrator import CompressionOrchestrator
+
+
+def test_rtn_routing_does_not_fallback_to_raw_config_scheme_fields():
+ tree = ast.parse(inspect.getsource(_select_rtn_compressor_base_cls))
+ forbidden_fields = {"bits", "act_bits", "data_type", "sym", "act_data_type", "act_dynamic"}
+ raw_reads = []
+ for node in ast.walk(tree):
+ if not isinstance(node, ast.Call) or not isinstance(node.func, ast.Name) or node.func.id != "getattr":
+ continue
+ if len(node.args) < 2 or not isinstance(node.args[0], ast.Name) or node.args[0].id != "quant_config":
+ continue
+ if isinstance(node.args[1], ast.Constant) and node.args[1].value in forbidden_fields:
+ raw_reads.append(node.args[1].value)
+
+ assert raw_reads == []
+
+
+def test_collect_config_scheme_overrides_omits_unset_fields():
+ cfg = RTNConfig(bits=8, data_type="int")
+ overrides = _collect_config_scheme_overrides(cfg)
+ assert overrides["bits"] == 8
+ assert overrides["data_type"] == "int"
+ # Unset scheme fields must not appear (scheme's own value should win).
+ assert "act_group_size" not in overrides
+
+
+def test_preview_resolves_scheme_only_and_override_to_same_attrs():
+ # scheme="W8A16" vs bits=8 override on top of a W-generic scheme must resolve
+ # to the same bits/data_type for routing.
+ from_scheme = _preview_resolved_attrs(RTNConfig(), "W8A16")
+ from_override = _preview_resolved_attrs(RTNConfig(bits=8), "W8A16")
+ assert from_scheme.get("bits") == from_override.get("bits") == 8
+
+
+def test_preview_falls_back_to_config_overrides_when_preview_skipped():
+ # An unknown scheme string makes parse_scheme raise; the resolver must then
+ # surface the config's explicit overrides (not an empty dict) so routing still
+ # sees the user's bits.
+ cfg = RTNConfig(bits=4, data_type="int")
+ resolved = _preview_resolved_attrs(cfg, "definitely-not-a-real-scheme-xyz")
+ assert resolved.get("bits") == 4
+ assert resolved.get("data_type") == "int"
+
+
+def test_routing_matches_between_scheme_only_and_equivalent_override():
+ base_kwargs = {}
+ # sym W4A16 (int4, sym) -> imatrix enabled, identical whether the bits/dtype
+ # come from the scheme or from explicit config overrides. The compressor class
+ # returned is always CompressionOrchestrator (it internally detects whether
+ # calibration data is needed); the imatrix/zero-shot decision now surfaces via
+ # ``enable_imatrix`` and the resolved config class.
+ cfg_scheme = RTNConfig()
+ cfg_override = RTNConfig(bits=4, data_type="int", sym=True)
+ via_scheme = _select_rtn_compressor_base_cls(cfg_scheme, "W4A16", None, base_kwargs)
+ via_override = _select_rtn_compressor_base_cls(cfg_override, "W4A16", None, base_kwargs)
+ assert via_scheme is via_override is CompressionOrchestrator
+ assert cfg_scheme.enable_imatrix is cfg_override.enable_imatrix is True
+ assert isinstance(cfg_scheme, OptimizedRTNConfig) and isinstance(cfg_override, OptimizedRTNConfig)
+
+ # asym W4A16 disables imatrix, again identical across paths.
+ cfg_scheme_asym = RTNConfig(sym=False)
+ cfg_override_asym = RTNConfig(bits=4, data_type="int", sym=False)
+ via_scheme_asym = _select_rtn_compressor_base_cls(cfg_scheme_asym, "W4A16", None, base_kwargs)
+ via_override_asym = _select_rtn_compressor_base_cls(cfg_override_asym, "W4A16", None, base_kwargs)
+ assert via_scheme_asym is via_override_asym is CompressionOrchestrator
+ assert cfg_scheme_asym.enable_imatrix is cfg_override_asym.enable_imatrix is False
+
+
+def test_w8a16_symmetric_routes_to_zero_shot():
+ # sym W8A16 (bits>=8, sym True) -> no imatrix, no act calib -> zero-shot RTN path.
+ cfg = RTNConfig()
+ cls = _select_rtn_compressor_base_cls(cfg, "W8A16", None, {})
+ assert cls is CompressionOrchestrator
+ assert cfg.enable_imatrix is False
diff --git a/test/test_cpu/core/test_format_decoupling.py b/test/test_cpu/core/test_format_decoupling.py
new file mode 100644
index 000000000..b9dac1280
--- /dev/null
+++ b/test/test_cpu/core/test_format_decoupling.py
@@ -0,0 +1,129 @@
+# Copyright (c) 2025 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Decoupling regression for the `export-format` capability.
+
+Two behaviors that must survive the scheme/format/compressor decoupling:
+
+1. Format-selection: ``resolve_formats(scheme, ...)`` (now scheme-centric, no ``ar``)
+ resolves the same backend as pre-refactor for representative combos.
+2. GGUF scheme correction propagates back onto ``self.scheme`` /
+ ``self.scheme_context`` / ``self.quantize_config`` (the staleness fix), with
+ ``self.scheme`` pinned to the precise preset *string* for alias
+ disambiguation.
+"""
+
+from types import SimpleNamespace
+
+import pytest
+import torch
+import torch.nn as nn
+
+from auto_round.algorithms.quantization.config import QuantizationConfig
+from auto_round.compressors.base import BaseCompressor
+from auto_round.compressors.config_resolution import FormatResolution, ResolvedScheme
+from auto_round.export.formats import resolve_formats
+from auto_round.schemes import QuantizationScheme, parse_scheme
+
+
+def _resolved_scheme(scheme_name):
+ # Real call sites always fully resolve the scheme (parse_scheme's third return,
+ # final_attrs) before calling resolve_formats — e.g. act_data_type defaults to "float"
+ # rather than staying None. Mirror that precondition here.
+ _, _, final_attrs = parse_scheme(scheme_name, {})
+ return QuantizationScheme.from_dict(final_attrs)
+
+
+# Harvested verbatim from a live run against pre-refactor code (HEAD=33b7df0e).
+EXPECTED_FORMAT_SELECTION_BASELINE = {
+ ("W4A16", "auto_round"): ["auto_round:auto_gptq"],
+ ("FP8_STATIC", "llm_compressor"): ["llm_compressor:fp8_static"],
+ ("MXFP4", "auto_round"): ["auto_round:mx_fp"],
+ ("NVFP4", "auto_round"): ["auto_round:nv_fp"],
+}
+
+
+def test_format_selection_baseline():
+ results = {}
+ for scheme_name, fmt in EXPECTED_FORMAT_SELECTION_BASELINE:
+ scheme = _resolved_scheme(scheme_name)
+ # _check_divisible_by_32 iterates model.named_modules() for int/act_bits>=16,
+ # so a real nn.Module is required.
+ model = nn.Sequential(nn.Linear(32, 32))
+ resolution = resolve_formats(
+ ResolvedScheme.from_scheme(scheme), format=fmt, model=model, scale_dtype=torch.float16
+ )
+ results[(scheme_name, fmt)] = [f.get_backend_name() for f in resolution.formats]
+ assert results == EXPECTED_FORMAT_SELECTION_BASELINE
+
+
+def test_gguf_correction_propagates_to_scheme_and_quantize_config(monkeypatch):
+ # W4A16's plain int-woq fields don't match every field gguf_args_check enforces
+ # for a GGUF target; after resolving "gguf:q4_k_m" the scheme-bearing objects
+ # must all agree, and self.scheme must be pinned to the precise preset string so
+ # get_gguf_scheme()'s string short-circuit can disambiguate Q4_K_S vs Q4_K_M.
+ from auto_round.schemes import get_gguf_scheme
+
+ original_scheme = _resolved_scheme("W4A16")
+ corrected_scheme = _resolved_scheme("GGUF:Q4_K_M")
+ quantize_config = QuantizationConfig(scheme=original_scheme.copy())
+ alg_config = QuantizationConfig(scheme=original_scheme.copy())
+ resolved_format = SimpleNamespace(
+ is_gguf=lambda: True,
+ output_format="gguf",
+ backend=SimpleNamespace(output_format="gguf:q4_k_m"),
+ )
+
+ def fake_resolve_formats(*args, **kwargs):
+ return FormatResolution(
+ formats=(resolved_format,),
+ scheme=ResolvedScheme.from_scheme(corrected_scheme, preset_name="gguf:q4_k_m"),
+ layer_config_patch={"layer": {"bits": 4}},
+ scale_dtype=torch.float32,
+ quant_block_list=(("model.layers.0",),),
+ )
+
+ monkeypatch.setattr("auto_round.compressors.base.resolve_formats", fake_resolve_formats)
+
+ class FormatResolutionHost:
+ _resolve_gguf_preset_string = staticmethod(BaseCompressor._resolve_gguf_preset_string)
+ _resolve_format_string = BaseCompressor._resolve_format_string
+
+ ar = FormatResolutionHost()
+ ar.scheme_context = original_scheme
+ ar.scheme = original_scheme.copy()
+ ar.quantize_config = quantize_config
+ ar._alg_configs = [alg_config, quantize_config]
+ ar.model_context = SimpleNamespace(model=nn.Linear(1, 1), is_mllm=False)
+ ar.layer_config = None
+ ar.scale_dtype = torch.float16
+ ar.quant_nontext_module = False
+ ar.quant_block_list = None
+ ar.platform = "hf"
+ ar.is_auto_scheme = False
+
+ formats = ar._resolve_format_string("gguf:q4_k_m")
+
+ assert formats == [resolved_format]
+ assert isinstance(ar.scheme, str)
+ assert ar.scheme.lower() == "gguf:q4_k_m"
+ assert get_gguf_scheme(ar.scheme) == ar.scheme # short-circuit preserved
+ assert ar.scheme_context == corrected_scheme
+ assert alg_config.scheme is ar.scheme_context
+ assert quantize_config.scheme is ar.scheme_context
+ assert ar.scheme_context.bits == ar.quantize_config.bits
+ assert ar.scheme_context.data_type == ar.quantize_config.data_type
+ assert ar.scheme_context.sym == ar.quantize_config.sym
+ assert ar.layer_config == {"layer": {"bits": 4}}
+ assert ar.scale_dtype is torch.float32
+ assert ar.quant_block_list == [["model.layers.0"]]
diff --git a/test/test_cpu/core/test_init.py b/test/test_cpu/core/test_init.py
index 2741f9745..e9f0ccccc 100644
--- a/test/test_cpu/core/test_init.py
+++ b/test/test_cpu/core/test_init.py
@@ -5,21 +5,15 @@ from auto_round import AutoRound
from auto_round.auto_scheme import AutoScheme
from auto_round.cli.parser import build_quantize_parser
from auto_round.compressors.base import BaseOrchestrator
-from auto_round.compressors.entry import AutoRound as NewAutoRound
-from auto_round.compressors.entry import AutoRoundCompatible
def test_torch_compile_platform_default_is_deferred():
assert inspect.signature(AutoRound.__new__).parameters["enable_torch_compile"].default is None
- assert inspect.signature(NewAutoRound.__new__).parameters["enable_torch_compile"].default is None
- assert inspect.signature(AutoRoundCompatible.__new__).parameters["enable_torch_compile"].default is None
assert inspect.signature(BaseOrchestrator.__init__).parameters["enable_torch_compile"].default is None
def test_dataset_default_is_deferred():
assert inspect.signature(AutoRound.__new__).parameters["dataset"].default is None
- assert inspect.signature(NewAutoRound.__new__).parameters["dataset"].default is None
- assert inspect.signature(AutoRoundCompatible.__new__).parameters["dataset"].default is None
assert inspect.signature(BaseOrchestrator.__init__).parameters["dataset"].default is None
@@ -103,7 +97,7 @@ def test_argparse_check(tiny_opt_model_path):
# so CalibrationState defaulted to 1 regardless of the user's value.
steps = 8
- ar = NewAutoRound(
+ ar = AutoRound(
tiny_opt_model_path,
scheme="W4A16",
gradient_accumulate_steps=steps,
diff --git a/test/test_cpu/core/test_legacy_plan_parity.py b/test/test_cpu/core/test_legacy_plan_parity.py
new file mode 100644
index 000000000..1ecd6b2a7
--- /dev/null
+++ b/test/test_cpu/core/test_legacy_plan_parity.py
@@ -0,0 +1,35 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from auto_round import AutoRound, SignRoundConfig
+
+
+def test_direct_and_algorithm_config_entries_build_equivalent_plans(tiny_opt_model_path):
+ common = {
+ "model": tiny_opt_model_path,
+ "scheme": "W4A16",
+ "iters": 0,
+ "nsamples": 1,
+ "seqlen": 8,
+ "dataset": ["local calibration sample"],
+ "low_cpu_mem_usage": False,
+ }
+ direct = AutoRound(bits=4, group_size=128, **common)
+ configured = AutoRound(alg_configs=SignRoundConfig(bits=4, group_size=128, iters=0), **common)
+
+ direct.post_init()
+ configured.post_init()
+
+ assert direct.compression_plan.scheme.value == configured.compression_plan.scheme.value
+ assert direct.compression_plan.layer_config == configured.compression_plan.layer_config
diff --git a/test/test_cpu/core/test_pipeline_fail_fast.py b/test/test_cpu/core/test_pipeline_fail_fast.py
index 59972e557..ceb1f8ce3 100644
--- a/test/test_cpu/core/test_pipeline_fail_fast.py
+++ b/test/test_cpu/core/test_pipeline_fail_fast.py
@@ -2,6 +2,7 @@
import pytest
+from auto_round import AutoRound as NewAutoRound
from auto_round import AWQConfig, OptimizedRTNConfig, RotationConfig, RTNConfig, SignRoundConfig, SpinQuantConfig
from auto_round.algorithms.composer import AlgorithmComposer, _can_compile_block_forward
from auto_round.algorithms.config_resolver import (
@@ -11,9 +12,8 @@ from auto_round.algorithms.config_resolver import (
)
from auto_round.algorithms.quantization import registry as _r
from auto_round.algorithms.quantization.rtn.quantizer import RTNQuantizer
+from auto_round.autoround import _select_rtn_compressor_base_cls
from auto_round.compressors.base import collect_user_scheme_overrides
-from auto_round.compressors.entry import AutoRound as NewAutoRound
-from auto_round.compressors.entry import _select_rtn_compressor_base_cls
from auto_round.compressors.orchestrator import CompressionOrchestrator as Compressor
from auto_round.logger import logger
from auto_round.schemes import QuantizationScheme
@@ -108,78 +108,49 @@ def test_new_entry_defaults_to_autoround_config(monkeypatch):
monkeypatch.setattr(Compressor, "__init__", _fake_init)
monkeypatch.setattr("auto_round.utils.model.detect_model_type", lambda *args, **kwargs: "llm")
- NewAutoRound("dummy-model", "W4A16", iters=1, seqlen=8, nsamples=1)
+ NewAutoRound("dummy-model", scheme="W4A16", iters=1, seqlen=8, nsamples=1)
- assert isinstance(captured["config"], SignRoundConfig)
+ assert isinstance(captured["config"], list)
+ assert isinstance(captured["config"][0], SignRoundConfig)
-def test_entry_rejects_configs_without_quantization_members():
- with pytest.raises(ValueError, match="At least one quantization algorithm config"):
- NewAutoRound("dummy-model", "W4A16", [RotationConfig()])
-
-
-def test_compat_entry_preserves_spinquant_dict_config(monkeypatch):
+def test_new_entry_zero_iters_defaults_to_rtn_config(monkeypatch):
captured = {}
- rotation_config = {
- "algorithm": "spinquant",
- "r1": True,
- "r2": True,
- "r3": False,
- "r4": False,
- "rotation_size": 128,
- "trainable_rotation": False,
- "trainable_smooth": False,
- }
def _fake_init(self, config, **kwargs):
captured["config"] = config
monkeypatch.setattr(Compressor, "__init__", _fake_init)
- monkeypatch.setattr("auto_round.utils.is_mllm_model", lambda *args, **kwargs: False)
- monkeypatch.setattr("auto_round.utils.is_diffusion_model", lambda *args, **kwargs: False)
monkeypatch.setattr("auto_round.utils.model.detect_model_type", lambda *args, **kwargs: "llm")
- from auto_round.autoround import AutoRound as CompatAutoRound
-
- CompatAutoRound(
- "dummy-model",
- scheme="W4A16",
- iters=1,
- seqlen=8,
- nsamples=1,
- rotation_config=rotation_config,
- )
+ NewAutoRound("dummy-model", scheme="W4A16", iters=0, disable_opt_rtn=True, disable_model_free=True)
- configs = captured["config"] if isinstance(captured["config"], list) else [captured["config"]]
- spinquant_cfg = next(cfg for cfg in configs if isinstance(cfg, SpinQuantConfig))
- assert spinquant_cfg.rotation_size == rotation_config["rotation_size"]
- assert spinquant_cfg.r1 is rotation_config["r1"]
- assert spinquant_cfg.r2 is rotation_config["r2"]
- assert spinquant_cfg.r3 is rotation_config["r3"]
- assert spinquant_cfg.r4 is rotation_config["r4"]
- assert spinquant_cfg.trainable_rotation is rotation_config["trainable_rotation"]
- assert spinquant_cfg.trainable_smooth is rotation_config["trainable_smooth"]
+ assert isinstance(captured["config"], list)
+ assert isinstance(captured["config"][0], RTNConfig)
+ assert captured["config"][0].disable_opt_rtn is True
-def test_entry_warns_and_drops_unsupported_kwargs(monkeypatch, tiny_opt_model_path):
- calls = []
+def test_new_entry_accepts_rotation_config_in_algorithm_list(monkeypatch):
+ captured = {}
- def _record_warning(message, *args):
- calls.append(message % args)
+ def _fake_init(self, config, **kwargs):
+ captured["config"] = config
- monkeypatch.setattr(logger, "warning_once", _record_warning)
+ monkeypatch.setattr(Compressor, "__init__", _fake_init)
+ monkeypatch.setattr("auto_round.utils.model.detect_model_type", lambda *args, **kwargs: "llm")
NewAutoRound(
- tiny_opt_model_path,
- "W4A16",
- RTNConfig(disable_opt_rtn=True),
- nsamples=1,
- seqlen=8,
- low_cpu_mem_usage=False,
- nonsense_kwarg=123,
+ "dummy-model",
+ scheme="MXFP4",
+ alg_configs=[SignRoundConfig(iters=1), RotationConfig()],
)
- assert any("unsupported kwargs nonsense_kwarg" in msg for msg in calls)
+ assert any(isinstance(config, RotationConfig) for config in captured["config"])
+
+
+def test_entry_rejects_configs_without_quantization_members():
+ with pytest.raises(TypeError, match="alg_configs entries must be algorithm aliases"):
+ NewAutoRound("dummy-model", scheme="W4A16", alg_configs=[RotationConfig()])
def test_shared_config_values_inherit_across_matching_attrs_only():
diff --git a/test/test_cpu/export/test_export.py b/test/test_cpu/export/test_export.py
index e23d2afbd..f8bb7cf21 100644
--- a/test/test_cpu/export/test_export.py
+++ b/test/test_cpu/export/test_export.py
@@ -8,10 +8,12 @@ from packaging import version
from transformers import AutoModelForCausalLM, AutoRoundConfig, AutoTokenizer
from auto_round import AutoRound
+from auto_round.compressors.config_resolution import ResolvedScheme
from auto_round.export.export_to_autogptq import export as autogptq_export
from auto_round.export.export_to_autoround import export as autoround_export
from auto_round.export.export_to_autoround import export_to_fp8 as autoround_fp8_export
from auto_round.export.export_to_awq import export as awq_export
+from auto_round.export.formats import resolve_formats
from ...helpers import forbid_threaded_packing, get_model_path
@@ -381,14 +383,18 @@ class TestAutoRound:
print(res)
def test_export_format(self):
- from auto_round.formats import get_formats
-
autoround = AutoRound(
self.model_name,
scheme="FP8_STATIC",
)
autoround.post_init()
- format_list = get_formats("auto_round, llm_compressor, auto_round:llm_compressor", autoround)
+ resolution = resolve_formats(
+ ResolvedScheme.from_scheme(autoround.scheme_context),
+ format="auto_round, llm_compressor, auto_round:llm_compressor",
+ model=autoround.model_context.model,
+ scale_dtype=autoround.scale_dtype,
+ )
+ format_list = list(resolution.formats)
assert len(format_list) == 3
assert format_list[0].output_format == "auto_round"
assert format_list[0].get_backend_name() == "auto_round:fp8_static"
@@ -402,7 +408,13 @@ class TestAutoRound:
scheme="W4A16",
)
autoround.post_init()
- format_list = get_formats("auto_round:auto_awq, auto_gptq", autoround)
+ resolution = resolve_formats(
+ ResolvedScheme.from_scheme(autoround.scheme_context),
+ format="auto_round:auto_awq, auto_gptq",
+ model=autoround.model_context.model,
+ scale_dtype=autoround.scale_dtype,
+ )
+ format_list = list(resolution.formats)
assert format_list[0].output_format == "auto_round"
assert format_list[0].get_backend_name() == "auto_round:auto_awq"
assert format_list[1].output_format == "auto_gptq"
@@ -413,7 +425,13 @@ class TestAutoRound:
scheme="INT8",
)
autoround.post_init()
- format_list = get_formats("llm_compressor, auto_round:llm_compressor", autoround)
+ resolution = resolve_formats(
+ ResolvedScheme.from_scheme(autoround.scheme_context),
+ format="llm_compressor, auto_round:llm_compressor",
+ model=autoround.model_context.model,
+ scale_dtype=autoround.scale_dtype,
+ )
+ format_list = list(resolution.formats)
assert format_list[0].output_format == "llm_compressor"
assert format_list[0].get_backend_name() == "llm_compressor:int8_w8a8"
assert format_list[1].output_format == "auto_round"
@@ -425,15 +443,19 @@ class TestAutoRound:
scheme="INT8_W8A8",
)
autoround_old.post_init()
- format_list_old = get_formats("llm_compressor, auto_round:llm_compressor", autoround_old)
+ resolution_old = resolve_formats(
+ ResolvedScheme.from_scheme(autoround_old.scheme_context),
+ format="llm_compressor, auto_round:llm_compressor",
+ model=autoround_old.model_context.model,
+ scale_dtype=autoround_old.scale_dtype,
+ )
+ format_list_old = list(resolution_old.formats)
assert format_list_old[0].output_format == "llm_compressor"
assert format_list_old[0].get_backend_name() == "llm_compressor:int8_w8a8"
assert format_list_old[1].output_format == "auto_round"
assert format_list_old[1].get_backend_name() == "auto_round:llm_compressor:int8_w8a8"
def test_export_format_with_scheme(self, tiny_qwen_model_path):
- from auto_round.formats import get_formats
-
ar = AutoRound(
model=tiny_qwen_model_path,
scheme="W4A16",
@@ -443,10 +465,20 @@ class TestAutoRound:
)
ar.post_init()
with pytest.raises(ValueError, match="auto_awq format support quantization scheme with W4A16 but got bits=2"):
- get_formats("auto_round:auto_awq", ar)
+ resolve_formats(
+ ResolvedScheme.from_scheme(ar.scheme_context),
+ format="auto_round:auto_awq",
+ model=ar.model_context.model,
+ scale_dtype=ar.scale_dtype,
+ )
with pytest.raises(ValueError, match="but got bits=2, data_type=int"):
- get_formats("auto_round:llm_compressor", ar)
+ resolve_formats(
+ ResolvedScheme.from_scheme(ar.scheme_context),
+ format="auto_round:llm_compressor",
+ model=ar.model_context.model,
+ scale_dtype=ar.scale_dtype,
+ )
ar = AutoRound(
model=tiny_qwen_model_path,
@@ -457,7 +489,12 @@ class TestAutoRound:
)
ar.post_init()
with pytest.raises(ValueError, match="but got data_type=fp, bits=4"):
- get_formats("auto_round:llm_compressor", ar)
+ resolve_formats(
+ ResolvedScheme.from_scheme(ar.scheme_context),
+ format="auto_round:llm_compressor",
+ model=ar.model_context.model,
+ scale_dtype=ar.scale_dtype,
+ )
ar = AutoRound(
model=tiny_qwen_model_path,
@@ -467,7 +504,12 @@ class TestAutoRound:
sym=True,
)
ar.post_init()
- get_formats("auto_round:auto_awq", ar)
+ resolve_formats(
+ ResolvedScheme.from_scheme(ar.scheme_context),
+ format="auto_round:auto_awq",
+ model=ar.model_context.model,
+ scale_dtype=ar.scale_dtype,
+ )
def test_autoawq_qwen3_vl_infer(self, dataloader):
model_path = get_model_path("Qwen/Qwen3-VL-2B-Instruct")
diff --git a/test/test_cpu/formats/test_format_resolver.py b/test/test_cpu/formats/test_format_resolver.py
new file mode 100644
index 000000000..f2b922576
--- /dev/null
+++ b/test/test_cpu/formats/test_format_resolver.py
@@ -0,0 +1,47 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pytest
+
+from auto_round.compressors.config_resolution import FormatCompatibilityError, resolve_scheme_value
+from auto_round.export.formats import resolve_formats
+
+
+def test_resolve_formats_does_not_mutate_inputs():
+ layer_config = {"layer": {"bits": 4}}
+ scheme = resolve_scheme_value("W4A16", {})
+
+ result = resolve_formats(scheme, format="auto_round", layer_config=layer_config, model=None)
+
+ layer_config["layer"]["bits"] = 8
+ assert scheme.value.bits == 4
+ assert result.scheme.value.bits == 4
+ assert result.layer_config_patch["layer"]["bits"] == 4
+
+
+def test_resolve_formats_rejects_gguf_with_real_companion():
+ scheme = resolve_scheme_value("W4A16", {})
+
+ with pytest.raises(FormatCompatibilityError):
+ resolve_formats(scheme, format="gguf:q4_k_m,auto_round", model=None)
+
+
+def test_resolve_formats_allows_gguf_with_fake_companion():
+ scheme = resolve_scheme_value("W4A16", {})
+
+ result = resolve_formats(scheme, format="gguf:q4_k_m,fake", model=None)
+
+ assert sorted(f.output_format for f in result.formats) == ["fake", "gguf"]
+ assert any(f.is_gguf() for f in result.formats)
+ assert any(f.is_fake() for f in result.formats)
diff --git a/test/test_cpu/integrations/test_llmc_integration.py b/test/test_cpu/integrations/test_llmc_integration.py
index 9a3082be6..c297dc95a 100644
--- a/test/test_cpu/integrations/test_llmc_integration.py
+++ b/test/test_cpu/integrations/test_llmc_integration.py
@@ -7,7 +7,7 @@ from llmcompressor import oneshot
from llmcompressor.modifiers.autoround import AutoRoundModifier
from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer
-from auto_round import AutoRound
+from auto_round import AutoRound, AWQConfig
from auto_round.calib_dataset import get_dataset
from ...envs import multi_card
@@ -253,7 +253,7 @@ def test_llmc_awq_w8a8_export_config_args(tiny_opt_model_path, tmp_path):
ar = AutoRound(
tiny_opt_model_path,
scheme="INT8",
- algorithm="awq",
+ alg_configs=AWQConfig(),
nsamples=2,
seqlen=32,
batch_size=2,
diff --git a/test/test_cpu/layer_config/test_apply.py b/test/test_cpu/layer_config/test_apply.py
new file mode 100644
index 000000000..976cabdbc
--- /dev/null
+++ b/test/test_cpu/layer_config/test_apply.py
@@ -0,0 +1,33 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import torch.nn as nn
+
+from auto_round.compressors.config_resolution import ResolvedQuantizationConfig, ResolvedScheme
+from auto_round.compressors.layer_config import apply_plan_to_model
+from auto_round.schemes import QuantizationScheme
+
+
+def test_apply_plan_is_the_explicit_model_write_boundary():
+ model = nn.Sequential(nn.Linear(32, 32))
+ plan = ResolvedQuantizationConfig(
+ scheme=ResolvedScheme.from_scheme(QuantizationScheme()),
+ formats=(),
+ layer_config={"0": {"bits": 4, "data_type": "int"}},
+ )
+
+ apply_plan_to_model(model, plan)
+
+ assert model[0].bits == 4
+ assert model[0].data_type == "int"
diff --git a/test/test_cpu/layer_config/test_layer_config_resolver.py b/test/test_cpu/layer_config/test_layer_config_resolver.py
new file mode 100644
index 000000000..ad369cd48
--- /dev/null
+++ b/test/test_cpu/layer_config/test_layer_config_resolver.py
@@ -0,0 +1,60 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import torch.nn as nn
+
+from auto_round.compressors.config_resolution import ResolvedScheme
+from auto_round.compressors.layer_config import (
+ extract_regex_config,
+ has_quantized_layer_outside_blocks,
+ resolve_layer_config,
+)
+from auto_round.schemes import QuantizationScheme
+
+
+def test_resolver_does_not_write_quantization_attributes_to_modules():
+ model = nn.Sequential(nn.Linear(32, 32))
+ scheme = ResolvedScheme.from_scheme(QuantizationScheme(act_bits=16, act_data_type="float"))
+
+ resolved = resolve_layer_config(
+ model=model,
+ scheme=scheme,
+ layer_config={"0": {}},
+ supported_types=(nn.Linear,),
+ inner_supported_types=(),
+ )
+
+ assert "0" in resolved
+ assert not hasattr(model[0], "bits")
+
+
+def test_extract_regex_config_keeps_pattern_separate_from_expanded_layers():
+ model = nn.Sequential(nn.Linear(32, 32))
+ scheme = ResolvedScheme.from_scheme(QuantizationScheme(act_bits=16, act_data_type="float"))
+
+ regex_config = extract_regex_config(
+ model=model,
+ scheme=scheme,
+ layer_config={"0": {"bits": 4}, "missing.*": {"bits": 16}},
+ supported_types=(nn.Linear,),
+ inner_supported_types=(),
+ )
+
+ assert set(regex_config) == {"missing.*"}
+ assert regex_config["missing.*"]["bits"] == 16
+
+
+def test_has_quantized_layer_outside_blocks_is_derived_from_final_mapping():
+ assert has_quantized_layer_outside_blocks({"layer": {"bits": 4, "in_blocks": False}})
+ assert not has_quantized_layer_outside_blocks({"layer": {"bits": 16, "in_blocks": False}})
diff --git a/test/test_cpu/models/test_moe_model.py b/test/test_cpu/models/test_moe_model.py
index cb21213f7..d97c28663 100644
--- a/test/test_cpu/models/test_moe_model.py
+++ b/test/test_cpu/models/test_moe_model.py
@@ -61,9 +61,10 @@ def test_gptoss(scheme, tiny_gpt_oss_model_path, tmp_path):
assert (
single_expert_cnt == config.num_local_experts
), f"Expected {config.num_local_experts} GPTOssSingleExpert modules, found {single_expert_cnt}."
+ expected_quant_linear_cnt = config.num_hidden_layers * 3 * config.num_local_experts
assert (
- quant_linear_cnt == config.num_hidden_layers * 3 * config.num_local_experts
- ), f"Expected {config.num_hidden_layers * 3 * config.num_local_experts} QuantLinear modules, found {quant_linear_cnt}."
+ quant_linear_cnt == expected_quant_linear_cnt
+ ), f"Expected {expected_quant_linear_cnt} QuantLinear modules, found {quant_linear_cnt}."
# verify the quantized model can be loaded and run inference
loaded_model = GptOssForCausalLM.from_pretrained(save_folder)
diff --git a/test/test_cpu/quantization/test_model_free.py b/test/test_cpu/quantization/test_model_free.py
index 8bba4b9fd..46ea0d35e 100644
--- a/test/test_cpu/quantization/test_model_free.py
+++ b/test/test_cpu/quantization/test_model_free.py
@@ -74,6 +74,33 @@ from auto_round.compressors.model_free import (
from auto_round.schemes import QuantizationScheme
from auto_round.utils.model import is_model_free_route
+
+def test_model_free_preserves_explicit_scheme_overrides():
+ from auto_round.compressors.model_free import ModelFreeCompressor
+
+ compressor = ModelFreeCompressor("unused-model-path", scheme="W8A16", sym=False)
+ compressor._parse_scheme()
+
+ assert compressor.default_scheme["sym"] is False
+
+
+def test_model_free_entry_passes_resolved_scheme_overrides(tiny_opt_model_path):
+ from auto_round import AutoRound
+
+ compressor = AutoRound(
+ tiny_opt_model_path,
+ scheme="W4A16",
+ sym=False,
+ iters=0,
+ disable_opt_rtn=True,
+ device_map="cpu",
+ enable_torch_compile=False,
+ )
+
+ assert type(compressor).__name__ == "ModelFreeCompressor"
+ assert compressor.scheme_input.sym is False
+
+
from ...envs import require_compressed_tensors
# ---------------------------------------------------------------------------
diff --git a/test/test_cpu/quantization/test_mx_quant_linear.py b/test/test_cpu/quantization/test_mx_quant_linear.py
index 1ec5cb729..63733946d 100644
--- a/test/test_cpu/quantization/test_mx_quant_linear.py
+++ b/test/test_cpu/quantization/test_mx_quant_linear.py
@@ -5,15 +5,15 @@ from auto_round.data_type.utils import get_quant_func
from auto_round.experimental import qmodules as ar_qmodules
from auto_round.export.export_to_autoround.qlinear_fp import QuantLinear as _MXFPLinear
from auto_round.export.export_to_autoround.qlinear_int import QuantLinear as _MXINTLinear
-from auto_round.formats import AutoRoundExportFormat
+from auto_round.export.formats import BackendDataType
from auto_round.schemes import PRESET_SCHEMES
-mx_schemes = [AutoRoundExportFormat.MXFP8.value, AutoRoundExportFormat.MXFP4.value]
-mx_int_schemes = [AutoRoundExportFormat.MXINT4.value]
+mx_schemes = [BackendDataType.MXFP8.value, BackendDataType.MXFP4.value]
+mx_int_schemes = [BackendDataType.MXINT4.value]
QMODULE_MAPPING = {
- AutoRoundExportFormat.MXFP8.value: ar_qmodules.MXFP8QuantLinear,
- AutoRoundExportFormat.MXFP4.value: ar_qmodules.MXFP4QuantLinear,
- AutoRoundExportFormat.MXINT4.value: ar_qmodules.MXINT4QuantLinear,
+ BackendDataType.MXFP8.value: ar_qmodules.MXFP8QuantLinear,
+ BackendDataType.MXFP4.value: ar_qmodules.MXFP4QuantLinear,
+ BackendDataType.MXINT4.value: ar_qmodules.MXINT4QuantLinear,
}
diff --git a/test/test_cpu/quantization/test_mxfp_save_load.py b/test/test_cpu/quantization/test_mxfp_save_load.py
index ea7e4478d..d2b0a69e2 100644
--- a/test/test_cpu/quantization/test_mxfp_save_load.py
+++ b/test/test_cpu/quantization/test_mxfp_save_load.py
@@ -10,23 +10,23 @@ from auto_round import AutoRound
from auto_round import schemes as ar_schemes
from auto_round.experimental import qmodules as ar_qmodules
from auto_round.export.export_to_autoround import qlinear_fp as ar_qlinear_fp
-from auto_round.formats import AutoRoundExportFormat
+from auto_round.export.formats import BackendDataType
from auto_round.inference.backend import MX_TENSOR_DATA_TYPES
from ...envs import has_module
from ...helpers import get_model_path
testing_scheme_name_lst = [
- AutoRoundExportFormat.MXFP8.value,
- AutoRoundExportFormat.MXFP4.value,
+ BackendDataType.MXFP8.value,
+ BackendDataType.MXFP4.value,
]
QMODULE_MAPPING = {
- AutoRoundExportFormat.MXFP8.value: ar_qmodules.MXFP8QuantLinear,
- AutoRoundExportFormat.MXFP4.value: ar_qmodules.MXFP4QuantLinear,
+ BackendDataType.MXFP8.value: ar_qmodules.MXFP8QuantLinear,
+ BackendDataType.MXFP4.value: ar_qmodules.MXFP4QuantLinear,
}
SCHEMES_MAPPING = {
- AutoRoundExportFormat.MXFP8.value: ar_schemes.MXFP8,
- AutoRoundExportFormat.MXFP4.value: ar_schemes.MXFP4,
+ BackendDataType.MXFP8.value: ar_schemes.MXFP8,
+ BackendDataType.MXFP4.value: ar_schemes.MXFP4,
}
MX_TENSOR_DATA_TYPES_FP = [i for i in MX_TENSOR_DATA_TYPES if "int" not in i]
diff --git a/test/test_cpu/quantization/test_nvfp4_quant_linear.py b/test/test_cpu/quantization/test_nvfp4_quant_linear.py
index 9dc84fe15..860ef0ef0 100644
--- a/test/test_cpu/quantization/test_nvfp4_quant_linear.py
+++ b/test/test_cpu/quantization/test_nvfp4_quant_linear.py
@@ -5,11 +5,11 @@ from auto_round.data_type.nvfp import calculate_gparam
from auto_round.data_type.utils import get_quant_func
from auto_round.experimental import qmodules as ar_qmodules
from auto_round.export.export_to_autoround.qlinear_fp import QuantLinear as _FPLinear
-from auto_round.formats import AutoRoundExportFormat
+from auto_round.export.formats import BackendDataType
from auto_round.schemes import PRESET_SCHEMES
QMODULE_MAPPING = {
- AutoRoundExportFormat.NVFP4.value: ar_qmodules.NVFP4QuantLinear,
+ BackendDataType.NVFP4.value: ar_qmodules.NVFP4QuantLinear,
}
@@ -35,7 +35,7 @@ def test_calculate_gparam_with_float8_input():
assert torch.isfinite(global_scale)
[email protected]("scheme", [AutoRoundExportFormat.NVFP4.value])
[email protected]("scheme", [BackendDataType.NVFP4.value])
@torch.inference_mode()
def test_nvfp4_quantlinear_from_original_and_forward(scheme):
"""
diff --git a/test/test_cpu/schemes/test_scheme_decoupling.py b/test/test_cpu/schemes/test_scheme_decoupling.py
new file mode 100644
index 000000000..136eece0b
--- /dev/null
+++ b/test/test_cpu/schemes/test_scheme_decoupling.py
@@ -0,0 +1,133 @@
+# Copyright (c) 2025 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Decoupling regression for the `quantization-scheme` capability.
+
+Covers the single-authority classification predicates on ``QuantizationScheme``
+(D1), the two user-confirmed drift-point semantics (D2, Option A), and the GGUF
+scheme-facts source that was pushed down into ``schemes.py`` (D4).
+"""
+
+import copy
+
+from auto_round.algorithms.quantization.config import QuantizationConfig
+from auto_round.schemes import GGUF_PRESET_ALIASES, GGUF_SCHEME_FACTS, BackendDataType, QuantizationScheme
+
+
+def _scheme(**overrides):
+ base = dict(
+ bits=4,
+ group_size=128,
+ sym=True,
+ data_type="int",
+ act_bits=16,
+ act_group_size=None,
+ act_sym=None,
+ act_data_type=None,
+ act_dynamic=None,
+ super_bits=None,
+ super_group_size=None,
+ )
+ base.update(overrides)
+ return QuantizationScheme(**base)
+
+
+# --- D1: classification predicates are single-authority on QuantizationScheme ---
+
+
+def test_backend_data_type_members():
+ assert BackendDataType.STANDARD_FP == "fp"
+ assert BackendDataType.MX_FP == "mx_fp"
+ assert BackendDataType.NV_FP == "nv_fp"
+ assert BackendDataType.MX_INT == "mx_int"
+ assert BackendDataType.FP8_STATIC == "fp8_static"
+ assert BackendDataType.FP8 == "fp8"
+
+
+def test_standard_and_mx_and_nv_predicates():
+ assert _scheme(data_type="fp").is_standard_fp() is True
+ assert _scheme(data_type="mx_fp").is_standard_fp() is False
+ assert _scheme(data_type="nv_fp").is_standard_fp() is False
+ assert _scheme(data_type="mx_fp").is_mx_fp() is True
+ assert _scheme(data_type="nv_fp").is_nv_fp() is True
+ assert _scheme(data_type="mx_int").is_mx_int() is True
+ assert _scheme(act_data_type="fp").is_act_standard_fp() is True
+ assert _scheme(act_data_type="mx_fp").is_act_mx_fp() is True
+ assert _scheme(act_data_type="nv_fp").is_act_nv_fp() is True
+
+
+def test_int_and_fp8_shape_predicates():
+ assert _scheme(data_type="int", act_bits=16, super_group_size=None).is_wint_woq() is True
+ assert _scheme(data_type="int", act_bits=8).is_wint_woq() is False
+ assert _scheme(data_type="int", act_bits=16, super_group_size=8).is_wint_woq() is False
+ assert _scheme(data_type="fp", bits=8, act_data_type="fp", act_bits=8).is_wfp8afp8() is True
+ assert _scheme(data_type="int8", act_data_type="int8").is_wint8aint8() is True
+ assert _scheme(data_type="int4", bits=4, act_data_type="int4", act_bits=4).is_wint4aint4() is True
+ assert _scheme(act_dynamic=True, act_data_type="fp", act_bits=8).is_dynamic_afp8() is True
+ assert _scheme(group_size=(128, 128), data_type="fp", bits=8).is_block_wfp8() is True
+ assert _scheme(act_data_type="fp8_static").is_static_afp8() is True
+ assert _scheme(act_bits=8).is_act_quantize() is True
+ assert _scheme(act_bits=16).is_act_quantize() is False
+
+
+# --- D2: the two confirmed drift points use Option A (utils.py) semantics ---
+
+
+def test_is_act_static_matches_fp8_static_preset_option_a():
+ # FP8_STATIC preset itself has data_type="fp"/act_data_type="fp" (no literal
+ # "fp8_static"); Option A must still classify it as static.
+ assert _scheme(data_type="fp", bits=8, act_data_type="fp", act_bits=8, act_dynamic=False).is_act_static()
+ assert not _scheme(data_type="fp", bits=8, act_data_type="fp", act_bits=8, act_dynamic=True).is_act_static()
+
+
+def test_is_dynamic_wint8aint8_requires_int8_shape_option_a():
+ # Option A: act_dynamic alone is not sufficient (AND, not OR).
+ assert _scheme(data_type="int8", act_data_type="int8", act_dynamic=True).is_dynamic_wint8aint8()
+ assert not _scheme(data_type="fp", act_data_type="fp", act_dynamic=True, bits=8, act_bits=8).is_dynamic_wint8aint8()
+
+
+def test_quantization_config_predicates_delegate_to_scheme_option_a():
+ # config.py properties must delegate to the scheme's single-authority predicates.
+ assert (
+ QuantizationConfig(
+ bits=8, data_type="fp", act_bits=8, act_data_type="fp", act_dynamic=True
+ ).is_dynamic_wint8aint8
+ is False
+ )
+ assert (
+ QuantizationConfig(bits=8, data_type="fp", act_bits=8, act_data_type="fp", act_dynamic=False).is_act_static
+ is True
+ )
+
+
+# --- D4: GGUF scheme facts pushed down into schemes.py ---
+
+
+def test_gguf_scheme_facts_only_hold_scheme_fields():
+ scheme_field_names = {"bits", "act_bits", "group_size", "sym", "data_type", "super_bits", "super_group_size"}
+ for key, facts in GGUF_SCHEME_FACTS.items():
+ assert set(facts.keys()) <= scheme_field_names, f"{key} leaks format fields: {facts.keys()}"
+ assert not ({"embedding", "lm_head", "mostly"} & set(facts.keys()))
+
+
+def test_gguf_preset_aliases_and_facts_are_independent_objects():
+ # Aliases resolve to a base facts key...
+ assert GGUF_PRESET_ALIASES["gguf:q4_k_m"] == "gguf:q4_k"
+ assert GGUF_PRESET_ALIASES["gguf:q2_k_mixed"] == "gguf:q2_k"
+ # ...and mutating one facts entry must not bleed into a sibling (no shared refs).
+ facts_copy = copy.deepcopy(GGUF_SCHEME_FACTS)
+ GGUF_SCHEME_FACTS["gguf:q4_k"]["bits"] = 999
+ try:
+ assert GGUF_SCHEME_FACTS["gguf:q3_k"]["bits"] == facts_copy["gguf:q3_k"]["bits"]
+ finally:
+ GGUF_SCHEME_FACTS["gguf:q4_k"]["bits"] = facts_copy["gguf:q4_k"]["bits"]
diff --git a/test/test_cpu/utils/test_layer_config_resolution.py b/test/test_cpu/utils/test_layer_config_resolution.py
new file mode 100644
index 000000000..60d77ff41
--- /dev/null
+++ b/test/test_cpu/utils/test_layer_config_resolution.py
@@ -0,0 +1,239 @@
+# Copyright (c) 2025 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Characterization for the `layer-config-resolution` capability.
+
+Locks the per-layer output of ``set_layer_config`` (the public entry point that
+the ``_resolve_*`` / ``_apply_*`` helpers were split out of) against verbatim
+pre-refactor baselines for the W4A16, mixed-precision, and GGUF paths, plus the
+ignore-layer regex-export behavior. Baselines were harvested from HEAD=33b7df0e.
+"""
+
+import shutil
+import tempfile
+from types import SimpleNamespace
+
+import torch
+import torch.nn as nn
+from safetensors.torch import save_file
+
+from auto_round.compressors.utils import set_layer_config
+from auto_round.schemes import PRESET_SCHEMES
+
+
+class _TinyModel(nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.model = nn.Module()
+ self.model.layers = nn.ModuleList([nn.Module() for _ in range(2)])
+ for i, layer in enumerate(self.model.layers):
+ layer.self_attn = nn.Module()
+ layer.self_attn.q_proj = nn.Linear(32, 32)
+ layer.mlp = nn.Module()
+ layer.mlp.up_proj = nn.Linear(32, 64)
+ self.lm_head = nn.Linear(32, 100)
+
+
+class _Router(nn.Module):
+ def __init__(self, size: int = 32):
+ super().__init__()
+ self.weight = nn.Parameter(torch.randn(size, size))
+
+
+class _UnsupportedGateModel(nn.Module):
+ def __init__(self, size: int = 32):
+ super().__init__()
+ self.mlp = nn.Module()
+ self.mlp.gate = _Router(size)
+ self.mlp.up_proj = nn.Linear(size, size)
+
+
+def _layer_config_for(scheme_name, layer_config=None, name_or_path=None):
+ model = _TinyModel()
+ if name_or_path is not None:
+ # GGUF path calls is_separate_lm_head(model), which reads model.name_or_path
+ # and inspects a real on-disk safetensors file — a local dir is required to
+ # avoid a network download.
+ model.name_or_path = name_or_path
+ scheme = PRESET_SCHEMES[scheme_name]
+ # set_layer_config returns (layer_config, has_qlayer_outside_block, regex_config).
+ return set_layer_config(
+ model=model,
+ layer_config=layer_config,
+ default_scheme=scheme,
+ default_scale_dtype=torch.float16,
+ supported_types=(nn.Linear,),
+ inner_supported_types=(),
+ quant_block_list=None,
+ quant_lm_head=False,
+ enable_gguf_official_mixed=False,
+ is_mllm=False,
+ )
+
+
+_COMMON_NON_LM_HEAD = {
+ "act_bits": 16,
+ "act_data_type": None,
+ "act_dynamic": None,
+ "act_group_size": None,
+ "act_sym": None,
+ "bits": 4,
+ "data_type": "int",
+ "group_size": 128,
+ "in_blocks": True,
+ "rotation_config": None,
+ "scale_dtype": torch.float16,
+ "super_bits": None,
+ "super_group_size": None,
+ "sym": True,
+}
+
+_LM_HEAD_FP16 = {
+ "act_bits": 16,
+ "act_data_type": None,
+ "act_dynamic": None,
+ "act_group_size": None,
+ "act_sym": None,
+ "bits": 16,
+ "data_type": "fp",
+ "fixed_by_user": True,
+ "group_size": 128,
+ "in_blocks": False,
+ "rotation_config": None,
+ "scale_dtype": torch.float16,
+ "super_bits": None,
+ "super_group_size": None,
+ "sym": True,
+}
+
+EXPECTED_W4A16_BASELINE = (
+ {
+ "lm_head": dict(_LM_HEAD_FP16),
+ "model.layers.0.self_attn.q_proj": {**_COMMON_NON_LM_HEAD, "fixed_by_user": False},
+ "model.layers.0.mlp.up_proj": {**_COMMON_NON_LM_HEAD, "fixed_by_user": False},
+ "model.layers.1.self_attn.q_proj": {**_COMMON_NON_LM_HEAD, "fixed_by_user": False},
+ "model.layers.1.mlp.up_proj": {**_COMMON_NON_LM_HEAD, "fixed_by_user": False},
+ },
+ False,
+ {},
+)
+
+EXPECTED_MIXED_BASELINE = (
+ {
+ "lm_head": dict(_LM_HEAD_FP16),
+ "model.layers.0.self_attn.q_proj": {**_COMMON_NON_LM_HEAD, "bits": 8, "fixed_by_user": True},
+ "model.layers.0.mlp.up_proj": {**_COMMON_NON_LM_HEAD, "fixed_by_user": False},
+ "model.layers.1.self_attn.q_proj": {**_COMMON_NON_LM_HEAD, "fixed_by_user": False},
+ "model.layers.1.mlp.up_proj": {**_COMMON_NON_LM_HEAD, "fixed_by_user": False},
+ },
+ False,
+ {},
+)
+
+_GGUF_NON_LM_HEAD = {
+ "act_bits": 16,
+ "act_data_type": None,
+ "act_dynamic": None,
+ "act_group_size": None,
+ "act_sym": None,
+ "bits": 5,
+ "data_type": "int",
+ "embedding": "gguf:q5_0",
+ "fixed_by_user": False,
+ "group_size": 32,
+ "in_blocks": True,
+ "lm_head": "gguf:q6_k",
+ "mostly": "gguf:q5_0",
+ "rotation_config": None,
+ "scale_dtype": torch.float16,
+ "super_bits": None,
+ "super_group_size": None,
+ "sym": True,
+}
+
+# NOTE: q_proj/up_proj (in_features=32) and lm_head (in_features=32) no longer keep
+# their plain gguf:q4_k_m / gguf:q6_k config: `_apply_gguf_shape_fallback` (merged from
+# main's #1960, "fix few bugs cause less accuracy of gguf format") falls each layer back
+# to the largest gguf type whose block_size (32 / 32) evenly divides in_features=32 —
+# gguf:q5_0 for the K-quant body layers, gguf:q8_0 for the K-quant lm_head — since
+# gguf:q4_k_m / gguf:q6_k both require block_size=256. Rebaselined post-merge
+# (2026-07-08); see docs/behavior-changes/decouple-scheme-format-layer-config.md.
+EXPECTED_GGUF_BASELINE = (
+ {
+ "model.layers.0.self_attn.q_proj": dict(_GGUF_NON_LM_HEAD),
+ "model.layers.0.mlp.up_proj": dict(_GGUF_NON_LM_HEAD),
+ "model.layers.1.self_attn.q_proj": dict(_GGUF_NON_LM_HEAD),
+ "model.layers.1.mlp.up_proj": dict(_GGUF_NON_LM_HEAD),
+ "lm_head": {
+ "act_bits": 16,
+ "bits": 8,
+ "data_type": "int",
+ "embedding": "gguf:q8_0",
+ "fixed_by_user": False,
+ "group_size": 32,
+ "lm_head": "gguf:q8_0",
+ "mostly": "gguf:q8_0",
+ "scale_dtype": torch.float16,
+ "super_bits": None,
+ "super_group_size": None,
+ "sym": True,
+ },
+ },
+ True,
+ {},
+)
+
+
+def test_w4a16_plain_baseline():
+ assert _layer_config_for("W4A16") == EXPECTED_W4A16_BASELINE
+
+
+def test_mixed_precision_override_baseline():
+ override = {"model.layers.0.self_attn.q_proj": {"bits": 8, "data_type": "int"}}
+ assert _layer_config_for("W4A16", layer_config=override) == EXPECTED_MIXED_BASELINE
+
+
+def test_gguf_q4_k_m_baseline():
+ tmp_dir = tempfile.mkdtemp(prefix="ar_characterization_gguf_")
+ try:
+ # Empty safetensors file so is_separate_lm_head() resolves locally (tied head).
+ save_file({}, f"{tmp_dir}/model.safetensors")
+ result = _layer_config_for("GGUF:Q4_K_M", name_or_path=tmp_dir)
+ finally:
+ shutil.rmtree(tmp_dir, ignore_errors=True)
+ assert result == EXPECTED_GGUF_BASELINE
+
+
+def test_ignore_layer_unsupported_module_kept_as_regex_export_entry():
+ # An ignore target on an unsupported module type must not land in layer_config
+ # but must be preserved verbatim in regex_config for export.
+ layer_config, _, regex_config = _layer_config_for("W4A16") # warm-up sanity
+ model = _UnsupportedGateModel()
+ layer_config, _, regex_config = set_layer_config(
+ model=model,
+ layer_config={},
+ default_scheme="W4A16",
+ default_scale_dtype=torch.float16,
+ supported_types=(nn.Linear,),
+ inner_supported_types=(),
+ ignore_layers="mlp.gate",
+ quant_lm_head=False,
+ enable_gguf_official_mixed=False,
+ is_mllm=False,
+ )
+ assert "mlp.gate" not in layer_config
+ assert regex_config["mlp.gate"]["bits"] == 16
+ assert regex_config["mlp.gate"]["act_bits"] == 16
+ assert regex_config["mlp.gate"]["data_type"] == "float"
+ assert regex_config["mlp.gate"]["act_data_type"] == "float"
+ assert regex_config["mlp.gate"]["fixed_by_user"] is True
diff --git a/test/test_cpu/utils/test_utils.py b/test/test_cpu/utils/test_utils.py
index d15365024..f818e376e 100644
--- a/test/test_cpu/utils/test_utils.py
+++ b/test/test_cpu/utils/test_utils.py
@@ -156,12 +156,12 @@ class TestPredefinedIgnoreLayersBlockFilter:
return stub
@patch("auto_round.compressors.base.get_predefined_ignore_layers")
- @patch("auto_round.compressors.base.set_layer_config")
+ @patch("auto_round.compressors.base.resolve_layer_config")
@patch("auto_round.compressors.base._handle_special_schemes", return_value=None)
def test_kimi_k25_ignore_layers_preserved(self, mock_handle, mock_set_lc, mock_get_predefined):
"""KIMI K2.5: vision_tower and mm_projector must end up in ignore_layers."""
mock_get_predefined.return_value = ["vision_tower", "mm_projector"]
- mock_set_lc.return_value = ({}, False, None)
+ mock_set_lc.return_value = {}
stub = self._make_compressor_stub(
predefined_ignore_layers=["vision_tower", "mm_projector"],
@@ -177,13 +177,13 @@ class TestPredefinedIgnoreLayersBlockFilter:
), f"mm_projector should be in ignore_layers, got: '{stub.ignore_layers}'"
@patch("auto_round.compressors.base.get_predefined_ignore_layers")
- @patch("auto_round.compressors.base.set_layer_config")
+ @patch("auto_round.compressors.base.resolve_layer_config")
@patch("auto_round.compressors.base._handle_special_schemes", return_value=None)
def test_step3p5_ignore_layers_preserved(self, mock_handle, mock_set_lc, mock_get_predefined):
"""step3p5: short pattern names must not be dropped by block filter."""
predefined = ["g_proj", "moe.gate", "eh_proj", "shared_head", "layers.45"]
mock_get_predefined.return_value = predefined
- mock_set_lc.return_value = ({}, False, None)
+ mock_set_lc.return_value = {}
stub = self._make_compressor_stub(
predefined_ignore_layers=predefined,
@@ -195,12 +195,12 @@ class TestPredefinedIgnoreLayersBlockFilter:
assert name in stub.ignore_layers, f"'{name}' should be in ignore_layers, got: '{stub.ignore_layers}'"
@patch("auto_round.compressors.base.get_predefined_ignore_layers")
- @patch("auto_round.compressors.base.set_layer_config")
+ @patch("auto_round.compressors.base.resolve_layer_config")
@patch("auto_round.compressors.base._handle_special_schemes", return_value=None)
def test_longcat_classifier_preserved(self, mock_handle, mock_set_lc, mock_get_predefined):
"""Longcat: 'classifier' must not be dropped."""
mock_get_predefined.return_value = ["classifier"]
- mock_set_lc.return_value = ({}, False, None)
+ mock_set_lc.return_value = {}
stub = self._make_compressor_stub(
predefined_ignore_layers=["classifier"],
@@ -211,12 +211,12 @@ class TestPredefinedIgnoreLayersBlockFilter:
assert "classifier" in stub.ignore_layers, f"classifier should be in ignore_layers, got: '{stub.ignore_layers}'"
@patch("auto_round.compressors.base.get_predefined_ignore_layers")
- @patch("auto_round.compressors.base.set_layer_config")
+ @patch("auto_round.compressors.base.resolve_layer_config")
@patch("auto_round.compressors.base._handle_special_schemes", return_value=None)
def test_glm_flash_full_path_inside_block_preserved(self, mock_handle, mock_set_lc, mock_get_predefined):
"""GLM Flash: model.layers.0.mlp (full path inside block) must be kept."""
mock_get_predefined.return_value = ["model.layers.0.mlp"]
- mock_set_lc.return_value = ({}, False, None)
+ mock_set_lc.return_value = {}
stub = self._make_compressor_stub(
predefined_ignore_layers=["model.layers.0.mlp"],
@@ -229,12 +229,12 @@ class TestPredefinedIgnoreLayersBlockFilter:
), f"model.layers.0.mlp should be in ignore_layers, got: '{stub.ignore_layers}'"
@patch("auto_round.compressors.base.get_predefined_ignore_layers")
- @patch("auto_round.compressors.base.set_layer_config")
+ @patch("auto_round.compressors.base.resolve_layer_config")
@patch("auto_round.compressors.base._handle_special_schemes", return_value=None)
def test_mllm_with_multiple_block_groups(self, mock_handle, mock_set_lc, mock_get_predefined):
"""MLLM with vision + language blocks: ignore layers outside both are preserved."""
mock_get_predefined.return_value = ["vision_tower", "mm_projector"]
- mock_set_lc.return_value = ({}, False, None)
+ mock_set_lc.return_value = {}
stub = self._make_compressor_stub(
predefined_ignore_layers=["vision_tower", "mm_projector"],
diff --git a/test/test_cuda/algorithms/test_awq.py b/test/test_cuda/algorithms/test_awq.py
index f12e259c8..69644581b 100644
--- a/test/test_cuda/algorithms/test_awq.py
+++ b/test/test_cuda/algorithms/test_awq.py
@@ -29,7 +29,7 @@ import pytest
import torch
from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer
-from auto_round import AutoRound
+from auto_round import AutoRound, AWQConfig
from ...helpers import eval_generated_prompt, evaluate_accuracy, generate_prompt, get_model_path, opt_name_or_path
@@ -52,7 +52,7 @@ class TestAWQLLM:
ar = AutoRound(
tiny_opt_model_path,
scheme="W4A16",
- algorithm="awq",
+ alg_configs=AWQConfig(),
n_grid=4,
nsamples=2,
seqlen=32,
@@ -77,7 +77,7 @@ class TestAWQLLM:
bits=bits,
group_size=group_size,
sym=sym,
- algorithm="awq",
+ alg_configs=AWQConfig(),
n_grid=4,
nsamples=2,
seqlen=32,
@@ -99,7 +99,7 @@ class TestAWQLLM:
ar = AutoRound(
model_name,
scheme="W4A16",
- algorithm="awq",
+ alg_configs=AWQConfig(),
n_grid=4,
nsamples=2,
seqlen=32,
@@ -159,7 +159,7 @@ class TestAWQMoE:
ar = AutoRound(
tiny_qwen_moe_model_path,
scheme="W4A16",
- algorithm="awq",
+ alg_configs=AWQConfig(),
nsamples=2,
n_grid=4,
seqlen=32,
@@ -195,7 +195,7 @@ class TestAWQMoE:
ar = AutoRound(
tiny_qwen_moe_model_path,
scheme="W4A16",
- algorithm="awq",
+ alg_configs=AWQConfig(),
n_grid=4,
nsamples=2,
seqlen=32,
@@ -239,7 +239,7 @@ class TestAWQEval:
ar = AutoRound(
self.model_name,
scheme="W4A16",
- algorithm="awq",
+ alg_configs=AWQConfig(),
nsamples=32,
seqlen=32,
batch_size=8,
diff --git a/test/test_cuda/integrations/test_vllm.py b/test/test_cuda/integrations/test_vllm.py
index b58254c5e..c4aa4e5ab 100644
--- a/test/test_cuda/integrations/test_vllm.py
+++ b/test/test_cuda/integrations/test_vllm.py
@@ -15,7 +15,7 @@ import pytest
from vllm import LLM, SamplingParams
from vllm.platforms import current_platform
-from auto_round import AutoRound
+from auto_round import AutoRound, AWQConfig
from ...helpers import get_model_path
@@ -187,7 +187,7 @@ def test_vllm_awq_w8a8_llmc_inference(tiny_opt_model_path, tmp_path):
ar = AutoRound(
tiny_opt_model_path,
scheme="INT8",
- algorithm="awq",
+ alg_configs=AWQConfig(),
nsamples=2,
seqlen=32,
batch_size=2,
diff --git a/test/test_cuda/quantization/test_mxfp_nvfp.py b/test/test_cuda/quantization/test_mxfp_nvfp.py
index bdc79075e..ab9078718 100644
--- a/test/test_cuda/quantization/test_mxfp_nvfp.py
+++ b/test/test_cuda/quantization/test_mxfp_nvfp.py
@@ -11,20 +11,20 @@ from auto_round import AutoRound
from auto_round import schemes as ar_schemes
from auto_round.experimental import qmodules as ar_qmodules
from auto_round.export.export_to_autoround import qlinear_fp as ar_qlinear_fp
-from auto_round.formats import AutoRoundExportFormat
+from auto_round.export.formats import BackendDataType
from ...envs import has_module, require_awq, require_optimum
from ...helpers import get_model_path, save_tiny_model
testing_schemes = [
- AutoRoundExportFormat.MXFP8.value,
- AutoRoundExportFormat.MXFP4.value,
- AutoRoundExportFormat.NVFP4.value,
+ BackendDataType.MXFP8.value,
+ BackendDataType.MXFP4.value,
+ BackendDataType.NVFP4.value,
]
QMODULE_MAPPING = {
- AutoRoundExportFormat.MXFP8.value: ar_qmodules.MXFP8QuantLinear,
- AutoRoundExportFormat.MXFP4.value: ar_qmodules.MXFP4QuantLinear,
- AutoRoundExportFormat.NVFP4.value: ar_qmodules.NVFP4QuantLinear,
+ BackendDataType.MXFP8.value: ar_qmodules.MXFP8QuantLinear,
+ BackendDataType.MXFP4.value: ar_qmodules.MXFP4QuantLinear,
+ BackendDataType.NVFP4.value: ar_qmodules.NVFP4QuantLinear,
}
diff --git a/test/test_mlx/test_mlx_format.py b/test/test_mlx/test_mlx_format.py
index 81106ca6d..a302e3bd8 100644
--- a/test/test_mlx/test_mlx_format.py
+++ b/test/test_mlx/test_mlx_format.py
@@ -378,11 +378,11 @@ class TestMLXFormat:
config with ``false`` entries.
5. ``rope_parameters`` has been flattened (``rope_theta`` surfaced).
"""
- from auto_round import AutoRoundMLLM
+ from auto_round import AutoRound
save_dir = str(tmp_path / "qwen3_vl_9b_mlx")
try:
- ar = AutoRoundMLLM(
+ ar = AutoRound(
qwen_3_vl_9b_name_or_path,
scheme="W4A16",
bits=4,
@@ -467,11 +467,11 @@ class TestMLXFormat:
from mlx_vlm.prompt_utils import apply_chat_template
from mlx_vlm.utils import load_config
- from auto_round import AutoRoundMLLM
+ from auto_round import AutoRound
save_dir = str(tmp_path / "qwen3_vl_9b_mlx_run")
try:
- ar = AutoRoundMLLM(
+ ar = AutoRound(
qwen_3_vl_9b_name_or_path,
scheme="W4A16",
bits=4,