PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
refactor scheme,layer_config and formats (#2013)
Heng Guo77e8ea599361
diff --git a/auto_round/__init__.py b/auto_round/__init__.pyindex 031e205ff..d3f87c2c4 100644--- a/auto_round/__init__.py+++ b/auto_round/__init__.py@@ -11,10 +11,7 @@# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.# See the License for the specific language governing permissions and# limitations under the License.-from auto_round.autoround import AutoRound--# support for old api-from auto_round.autoround import AutoRoundLLM, AutoRoundMLLM, AutoRoundAdam, AutoRoundDiffusion+from auto_round.autoround import AutoRound, AutoRoundAdam, AutoRoundDiffusion, AutoRoundLLM, AutoRoundMLLMfrom auto_round.algorithms.quantization.rtn.config import OptimizedRTNConfig, RTNConfigfrom auto_round.algorithms.quantization.sign_round.config import (AdamRoundConfig,diff --git a/auto_round/algorithms/config_resolver.py b/auto_round/algorithms/config_resolver.pyindex 329e30ba8..cccf8055b 100644--- a/auto_round/algorithms/config_resolver.py+++ b/auto_round/algorithms/config_resolver.py@@ -148,3 +148,15 @@ def resolve_shared_config_values(configs: list[Any]) -> list[Any]:if getattr(config, attr_name) is None:setattr(config, attr_name, shared_value)return configs+++def sync_shared_config_from(source_config: Any, target_configs: list[Any]) -> None:+ """Propagate resolved source values to targets that already define matching attrs."""+ source_attrs = _public_config_attrs(source_config)+ for target in _quantization_configs(target_configs):+ if target is source_config:+ continue+ target_attrs = _public_config_attrs(target)+ for attr_name, source_value in source_attrs.items():+ if attr_name in target_attrs and source_value is not None:+ setattr(target, attr_name, source_value)diff --git a/auto_round/algorithms/quantization/config.py b/auto_round/algorithms/quantization/config.pyindex 9479f0952..055525c9b 100644--- a/auto_round/algorithms/quantization/config.py+++ b/auto_round/algorithms/quantization/config.py@@ -11,7 +11,6 @@# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.# See the License for the specific language governing permissions and# limitations under the License.-from enum import Enumfrom typing import ClassVarfrom auto_round.export.export_to_gguf.config import GGUF_INNER_CONFIG@@ -19,14 +18,6 @@ from auto_round.logger import loggerfrom auto_round.schemes import QuantizationScheme-class BackendDataType(str, Enum):- STANDARD_FP = "fp"- MX_FP = "mx_fp"- NV_FP = "nv_fp"- FP8_STATIC = "fp8_static"- FP8 = "fp8"--class QuantizationConfig:"""Common quantization configuration shared by block quantizers.@@ -186,71 +177,44 @@ class QuantizationConfig:@propertydef is_act_quantize(self) -> bool:- return self.act_bits is not None and self.act_bits <= 8+ return self.scheme.is_act_quantize()@propertydef is_nv_fp(self) -> bool:- return self.data_type is not None and BackendDataType.NV_FP in self.data_type+ return self.scheme.is_nv_fp()@propertydef is_act_nv_fp(self) -> bool:- return self.act_data_type is not None and BackendDataType.NV_FP in self.act_data_type+ return self.scheme.is_act_nv_fp()@propertydef is_mx_fp(self) -> bool:- return self.data_type is not None and BackendDataType.MX_FP in self.data_type+ return self.scheme.is_mx_fp()@propertydef is_act_mx_fp(self) -> bool:- return self.act_data_type is not None and BackendDataType.MX_FP in self.act_data_type+ return self.scheme.is_act_mx_fp()@propertydef is_dynamic_wint8aint8(self) -> bool:- if self.act_dynamic:- return True- if self.act_data_type is not None and self.data_type is not None:- if ("int8" in self.act_data_type or ("int" in self.act_data_type and self.act_bits == 8)) and (- "int8" in self.data_type or ("int" in self.data_type and self.bits == 8)- ):- return True- return False+ return self.scheme.is_dynamic_wint8aint8()@propertydef is_standard_fp(self) -> bool:- return (- self.data_type is not None- and BackendDataType.STANDARD_FP in self.data_type- and not self.is_mx_fp- and not self.is_nv_fp- )+ return self.scheme.is_standard_fp()@propertydef is_act_standard_fp(self) -> bool:- return (- self.act_data_type is not None- and BackendDataType.STANDARD_FP in self.act_data_type- and not self.is_act_mx_fp- and not self.is_act_nv_fp- )+ return self.scheme.is_act_standard_fp()@propertydef is_static_afp8(self) -> bool:- return self.act_data_type is not None and BackendDataType.FP8_STATIC in self.act_data_type+ return self.scheme.is_static_afp8()@property- def is_static_wfp8afp8(self) -> bool:- return self.data_type is not None and BackendDataType.FP8_STATIC in self.data_type and self.is_static_afp8+ def is_act_static(self) -> bool:+ return self.scheme.is_act_static()@propertydef is_wfp8afp8(self) -> bool:- if self.act_data_type is None or self.data_type is None:- return False- if (- ("fp8" in self.act_data_type or ("fp" in self.act_data_type and self.act_bits == 8))- and ("fp8" in self.data_type or ("fp" in self.data_type and self.bits == 8))- and self.is_act_standard_fp- and self.is_standard_fp- ):- return True- else:- return False+ return self.scheme.is_wfp8afp8()diff --git a/auto_round/algorithms/quantization/sign_round/config.py b/auto_round/algorithms/quantization/sign_round/config.pyindex 823b10e05..c3fb94e4e 100644--- a/auto_round/algorithms/quantization/sign_round/config.py+++ b/auto_round/algorithms/quantization/sign_round/config.py@@ -105,7 +105,7 @@ class SignRoundConfig(QuantizationConfig):def finalize_scheme(self) -> None:"""Resolve lr/minmax_lr once `bits` is known (low-bit schemes use a higher lr)."""- if self.lr is None:+ if self.lr is None and self.iters > 0:# TODO need to check 4 bits lr setting for auto-round-best, 3bits only validate on small modelsif self.iters >= 1000 and self.bits is not None and self.bits <= 3:self.lr = 2.0 / self.itersdiff --git a/auto_round/algorithms/registry.py b/auto_round/algorithms/registry.pyindex d56b0a497..081b09f0f 100644--- a/auto_round/algorithms/registry.py+++ b/auto_round/algorithms/registry.py@@ -90,6 +90,41 @@ def resolve_algorithm_alias(alias: str) -> str | None:return _ALIAS_TO_NAME.get(alias.strip().lower())+def resolve_algorithm_names(algorithms, *, ignore_unknown: bool = False) -> list[str]:+ """Resolve one algorithm string or sequence to ordered canonical names.++ A comma-separated string and a sequence use exactly the same semantics.+ Duplicate aliases are removed while preserving the first occurrence.+ """+ if isinstance(algorithms, str):+ raw_names = algorithms.split(",")+ else:+ raw_names = list(algorithms)++ canonical_names = []+ seen = set()+ for raw_name in raw_names:+ if not isinstance(raw_name, str):+ raise TypeError(f"Algorithm names must be strings, got {type(raw_name).__name__}.")+ name = raw_name.strip()+ if not name:+ continue+ canonical = resolve_algorithm_alias(name)+ if canonical is None:+ if ignore_unknown:+ continue+ supported_aliases = sorted(_ALIAS_TO_NAME.keys())+ raise ValueError(+ f"Unknown algorithm alias '{name}'. Supported aliases: {supported_aliases}. "+ "If you are adding a new algorithm, register it via "+ "auto_round.algorithms.registry.register_algorithm()."+ )+ if canonical not in seen:+ canonical_names.append(canonical)+ seen.add(canonical)+ return canonical_names++def get_algorithm_entry(name: str) -> AlgRegistryEntry:_ensure_builtin_algorithms_registered()canonical = resolve_algorithm_alias(name)diff --git a/auto_round/algorithms/transforms/awq/qdq.py b/auto_round/algorithms/transforms/awq/qdq.pyindex ccdbf5db3..4c98acdbb 100644--- a/auto_round/algorithms/transforms/awq/qdq.py+++ b/auto_round/algorithms/transforms/awq/qdq.py@@ -164,6 +164,7 @@ class QDQTool:)if init_scale is not None:quant_kwargs["init_scale"] = init_scale+ quant_kwargs["imatrix"] = imatrix if isinstance(imatrix, torch.Tensor) else torch.ones_like(weight)active_quant_func = opt_quant_funcqdq_weight, _, _ = active_quant_func(weight, **quant_kwargs)diff --git a/auto_round/algorithms/transforms/spinquant/apply.py b/auto_round/algorithms/transforms/spinquant/apply.pyindex e8e4c8f68..108820bb5 100644--- a/auto_round/algorithms/transforms/spinquant/apply.py+++ b/auto_round/algorithms/transforms/spinquant/apply.py@@ -32,7 +32,7 @@ Usage via the unified entry point::Usage via AutoRound pipeline::- from auto_round.compressors.entry import AutoRound+ from auto_round import AutoRoundfrom auto_round.algorithms.transforms.spinquant import SpinQuantConfigAutoRound(diff --git a/auto_round/auto_scheme/delta_loss.py b/auto_round/auto_scheme/delta_loss.pyindex e362687e3..035d9122b 100644--- a/auto_round/auto_scheme/delta_loss.py+++ b/auto_round/auto_scheme/delta_loss.py@@ -1184,7 +1184,7 @@ def get_score_for_scheme(# Unified mllm-aware forward (casts pixel_values/images to# model.dtype, handles dict-with-text/str/tuple paths the same- # way AutoRoundMLLM.calib does).+ # way the multimodal compressor calibration does).output, _prepared = model_forward(model, data_for_forward, labels=labels, use_cache=False)output.loss.backward()diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex c6c4b48f9..c4c58c318 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -11,339 +11,776 @@# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.# See the License for the specific language governing permissions and# limitations under the License.+from __future__ import annotations-from typing import TYPE_CHECKING, Optional, Union+from typing import TYPE_CHECKING, Any, Optional, Unionimport torchfrom auto_round.logger import deprecated, logger-from auto_round.schemes import QuantizationScheme+from auto_round.schemes import QuantizationScheme, parse_schemefrom auto_round.utils.device_manager import normalize_default_device_mapif TYPE_CHECKING:+ from auto_round.algorithms.quantization.config import QuantizationConfig+ from auto_round.algorithms.quantization.rtn.config import RTNConfigfrom auto_round.auto_scheme.gen_auto_scheme import AutoSchemefrom auto_round.compressors.base import BaseOrchestrator as BaseCompressor-_COMPAT_KWARGS = {- "format",- "bits",- "group_size",- "sym",- "data_type",- "act_bits",- "act_group_size",- "act_sym",- "act_data_type",- "act_dynamic",- "super_bits",- "super_group_size",- "scale_dtype",- "ignore_layers",- "quant_lm_head",- "to_quant_block_names",- "model_free",- "disable_model_free",- "model_dtype",- "trust_remote_code",- "amp",- "nblocks",+def _collect_config_scheme_overrides(config) -> dict:+ """Return the config's explicitly-set scheme fields as a ``{field: value}`` dict.++ These are exactly the per-field overrides layered on top of ``scheme=`` — the+ single mechanism through which ``bits`` / ``act_bits`` / ``data_type`` etc.+ reach the resolved scheme. Fields left as ``None`` are omitted so the scheme's+ own value wins.+ """+ return {k: getattr(config, k) for k in config._scheme_fields if getattr(config, k, None) is not None}+++def _preview_resolved_attrs(config, scheme=None) -> dict:+ from auto_round.auto_scheme.gen_auto_scheme import AutoScheme++ """Resolve scheme attributes without mutating config, for routing decisions.++ Called in ``AutoRound.__new__`` before the concrete compressor class is+ chosen. ``SchemeMixin.resolve_scheme()`` will do the authoritative+ resolution later; this is just a lightweight preview so routing logic+ (``enable_imatrix``, ``needs_act_calib``, etc.) can use the correct values+ even when the user specified only ``scheme=`` without explicit bit/dtype args.++ This is the single source of resolved scheme fields for entry-level routing:+ callers read from the returned dict and never re-read raw ``config`` attrs.+ When the scheme cannot be previewed (``AutoScheme``, or a deferred parse+ error), the config's own explicitly-set scheme overrides are returned so the+ values still reflect what the user passed.++ Returns:+ dict: resolved scheme attributes (config overrides when preview is skipped).+ """+ config_overrides = _collect_config_scheme_overrides(config)+ if isinstance(scheme, AutoScheme):+ # AutoScheme needs model info — cannot preview; fall back to raw config attrs.+ return config_overrides+ try:+ _, _, final_attrs = parse_scheme(scheme, config_overrides)+ return final_attrs+ except Exception:+ return config_overrides+++def _eager_validate_scheme(config, scheme=None) -> None:+ from auto_round.auto_scheme.gen_auto_scheme import AutoScheme++ """Eagerly validate scheme/config constraints at construction time.++ Mirrors the old-arch ``_check_configs()`` call in ``BaseCompressor.__init__``.+ Raises ``ValueError`` or ``NotImplementedError`` immediately if the scheme+ contains config-only invalid combinations (e.g. tuple group_size with non-fp8+ weight dtype) so that callers get a fast failure rather than a deferred error+ buried inside ``post_init()``.++ ``AutoScheme`` is skipped because it requires model information.+ """+ if isinstance(scheme, AutoScheme):+ return++ user_overrides = _collect_config_scheme_overrides(config)+ try:+ _, _, final_attrs = parse_scheme(scheme, user_overrides)+ except (ValueError, NotImplementedError):+ raise+ except Exception:+ return # Other parse errors are deferred to post_init++ import copy++ temp_config = copy.copy(config)+ if hasattr(config, "scheme"):+ temp_config.scheme = config.scheme.copy()+ temp_config._user_set_scheme_fields = set(getattr(config, "_user_set_scheme_fields", set()))+ for key, value in final_attrs.items():+ setattr(temp_config, key, value)+ temp_config.check_config() # raises ValueError / NotImplementedError if invalid+++# ---------------------------------------------------------------------------+# Compressor-class registry+# ---------------------------------------------------------------------------+# Maps (model_type, base_class_name) → combined class, created lazily.+_COMPRESSOR_REGISTRY: dict[tuple[str, str], type] = {}+++def _get_compressor_class(model_type: str, base_cls: type) -> type:+ """Return the compressor class for *base_cls* wired with the right model-type Mixin.++ For ``model_type == "llm"`` the bare *base_cls* is returned unchanged.+ For ``"mllm"`` and ``"diffusion"`` the corresponding Mixin is prepended via+ :func:`type` and the result is cached in ``_COMPRESSOR_REGISTRY`` so that+ each ``(model_type, base_cls)`` pair is created at most once per process.+ """+ if model_type == "llm":+ return base_cls+ key = (model_type, base_cls.__name__)+ if key in _COMPRESSOR_REGISTRY:+ return _COMPRESSOR_REGISTRY[key]+ if model_type == "mllm":+ from auto_round.compressors.mllm_mixin import MLLMMixin++ mixin = MLLMMixin+ elif model_type == "diffusion":+ from auto_round.compressors.diffusion_mixin import DiffusionMixin++ mixin = DiffusionMixin+ else:+ return base_cls+ combined = type(f"{model_type.capitalize()}{base_cls.__name__}", (mixin, base_cls), {})+ _COMPRESSOR_REGISTRY[key] = combined+ return combined+++def is_weight_scheme(scheme: Union[str, dict, object]) -> bool:+ if isinstance(scheme, str):+ return scheme.upper().startswith("W")+ if isinstance(scheme, dict):+ return all(isinstance(s, str) and s.upper().startswith("W") for s in scheme.values())+ from auto_round.auto_scheme.gen_auto_scheme import AutoScheme++ if isinstance(scheme, AutoScheme):+ opts = scheme.options+ if isinstance(opts, (list, tuple)):+ return all(isinstance(s, str) and s.upper().startswith("W") for s in opts)+ if isinstance(opts, str):+ return opts.upper().startswith("W")+ return False+++def is_gguf_k_target(value: Union[str, "AutoScheme", object]) -> bool:+ from auto_round.auto_scheme.gen_auto_scheme import AutoScheme++ if isinstance(value, str):+ normalized = value.strip().lower()+ return normalized.startswith("gguf:") and "_k" in normalized+ if isinstance(value, AutoScheme):+ opts = value.options+ if isinstance(opts, str):+ opts = [opts]+ if isinstance(opts, (list, tuple)):+ return any(isinstance(opt, str) and is_gguf_k_target(opt) for opt in opts)+ return False+++def _resolve_quant_config_for_routing(alg_configs) -> tuple[list, list, "QuantizationConfig"]:+ from auto_round.algorithms.config_resolver import split_quantization_configs+ from auto_round.algorithms.quantization.config import QuantizationConfig+ from auto_round.algorithms.quantization.rtn.config import RTNConfig++ preprocessor_configs, block_quant_configs = split_quantization_configs(alg_configs)+ if len(block_quant_configs) == 0 and preprocessor_configs:+ from auto_round.algorithms.quantization.rtn.config import RTNConfig as _RTNConfig++ return preprocessor_configs, block_quant_configs, _RTNConfig()+ if len(block_quant_configs) > 1:+ raise ValueError(+ f"Only one block-quantization config is allowed, but got {len(block_quant_configs)}: "+ f"{[type(c).__name__ for c in block_quant_configs]}"+ )+ if len(block_quant_configs) == 1:+ return preprocessor_configs, block_quant_configs, block_quant_configs[0]+ raise ValueError(+ "At least one quantization algorithm config is required. "+ "Pass a block quantizer such as RTNConfig or SignRoundConfig, "+ "or a quantization preprocessor such as AWQConfig."+ )+++def _build_model_free_compressor(+ model,+ scheme,+ layer_config,+ tokenizer,+ device_map,+ *,+ announced_via_flag: bool,+ **model_free_kwargs,+):+ """Construct a ``ModelFreeCompressor`` for the model-free auto-route.++ Shared by the unified entry and the model-free route so the string-model+ guard, the auto-routing info log, and the constructor call live in one place.+ ``announced_via_flag`` is truthy when the caller explicitly passed+ ``model_free=True`` (so the informational auto-routing message is suppressed).+ """+ from auto_round.compressors.model_free import ModelFreeCompressor++ if not isinstance(model, str):+ raise ValueError("model_free=True requires `model` to be a HuggingFace ID or local path string.")+ if not announced_via_flag:+ logger.info(+ "Auto-routing to model-free quantization "+ "(iters=0, disable_opt_rtn=True, supported scheme). "+ "Pass disable_model_free=True to use the regular flow."+ )+ return ModelFreeCompressor(+ model_name_or_path=model,+ scheme=scheme,+ layer_config=layer_config,+ tokenizer=tokenizer,+ device_map=device_map,+ **model_free_kwargs,+ )+++def _build_model_type_ctor_kwargs(model, base_kwargs, mllm_kwargs, diffusion_kwargs) -> tuple[str, dict[str, Any]]:+ from auto_round.utils.model import detect_model_type++ model_type = detect_model_type(model)+ has_multimodal_assets = mllm_kwargs.get("processor") is not None or mllm_kwargs.get("image_processor") is not None+ if has_multimodal_assets and model_type != "mllm":+ model_type = "mllm"++ ctor_kwargs = dict(base_kwargs)+ if model_type == "mllm":+ ctor_kwargs.update(mllm_kwargs)+ if model_type == "diffusion":+ ctor_kwargs.update(diffusion_kwargs)+ return model_type, ctor_kwargs+++def _select_rtn_compressor_base_cls(quant_config: "RTNConfig", scheme, format, base_kwargs) -> type:+ from auto_round.algorithms.quantization.rtn.config import OptimizedRTNConfig, RTNConfig+ from auto_round.auto_scheme.gen_auto_scheme import AutoScheme+ from auto_round.compressors.orchestrator import CompressionOrchestrator as Compressor+ from auto_round.compressors.utils import check_need_act_calibration++ enable_imatrix = False+ disable_opt_rtn = getattr(quant_config, "disable_opt_rtn", False)++ # Single resolved-scheme source for routing (SchemeMixin does the authoritative+ # resolution later; this preview only chooses the class). Computed once: neither+ # `quant_config`'s scheme fields nor `scheme` itself change within this function,+ # so the result is invariant across every use below — no need to recompute it.+ resolved_attrs = _preview_resolved_attrs(quant_config, scheme)++ # Auto-disable rtn optimization for W8A16/W8A8-equivalent resolved schemes,+ # unless the user already set disable_opt_rtn explicitly.+ if getattr(quant_config, "orig_disable_opt_rtn", None) is None:+ bits = resolved_attrs.get("bits")+ act_bits = resolved_attrs.get("act_bits")+ data_type = resolved_attrs.get("data_type")+ if bits is not None and bits >= 8 and act_bits is not None and act_bits >= 8 and data_type == "int":+ logger.warning("`disable_opt_rtn` is turned on for W8A16/W8A8 quantization to improve efficiency.")+ disable_opt_rtn = True+ quant_config.disable_opt_rtn = True++ if not disable_opt_rtn:+ has_gguf_k = is_gguf_k_target(format) or is_gguf_k_target(scheme)+ if has_gguf_k:+ enable_imatrix = True+ else:+ sym = resolved_attrs.get("sym")+ data_type = resolved_attrs.get("data_type") or ""+ bits = resolved_attrs.get("bits")+ if sym is not None and sym is False:+ enable_imatrix = False+ elif data_type == "int" and (bits is None or bits < 8):+ enable_imatrix = True+ elif is_weight_scheme(scheme):+ enable_imatrix = True++ act_bits = resolved_attrs.get("act_bits")+ act_data_type = resolved_attrs.get("act_data_type")+ act_dynamic = resolved_attrs.get("act_dynamic")+ is_act_quantize = act_bits is not None and act_bits <= 8+ needs_act_calib = is_act_quantize and check_need_act_calibration(+ act_dynamic,+ act_data_type,+ act_bits if act_bits is not None else 16,+ static_kv_dtype=base_kwargs.get("static_kv_dtype"),+ static_attention_dtype=base_kwargs.get("static_attention_dtype"),+ )++ # AutoScheme always requires calibration data for delta-loss based scheme+ # selection, regardless of whether imatrix is needed.+ quant_config.enable_imatrix = enable_imatrix+ needs_optimized_rtn = enable_imatrix or needs_act_calib or isinstance(scheme, AutoScheme)+ if needs_optimized_rtn:+ if not isinstance(quant_config, OptimizedRTNConfig):+ quant_config.__class__ = OptimizedRTNConfig+ else:+ # Pure zero-shot RTN: downgrade to basic RTNConfig+ if isinstance(quant_config, OptimizedRTNConfig):+ quant_config.__class__ = RTNConfig++ # Always use Compressor — it internally detects whether calibration+ # data is needed and falls back to the zero-shot (RTN) path when it is not.+ return Compressor+++_ENTRY_KWARG_OWNERS = {+ "model_free": "route",+ "disable_model_free": "route",+ "scale_dtype": "compressor",+ "ignore_layers": "compressor",+ "quant_lm_head": "compressor",+ "to_quant_block_names": "compressor",+ "format": "base",+ "dataset": "base",+ "batch_size": "base",+ "model_dtype": "base",+ "trust_remote_code": "base",+ "amp": "base",+ "disable_deterministic_algorithms": "base",+ "enable_deterministic_algorithms": "base",+ "static_kv_dtype": "base",+ "static_attention_dtype": "base",+ "processor": "mllm",+ "image_processor": "mllm",+ "template": "mllm",+ "extra_data_dir": "mllm",+ "quant_nontext_module": "mllm",+ "guidance_scale": "diffusion",+ "num_inference_steps": "diffusion",+ "generator_seed": "diffusion",+}++_SCHEME_FIELDS = set(QuantizationScheme.get_attributes())+_SIGNROUND_FIELDS = {+ "iters","lr","minmax_lr",+ "lr_scheduler",+ "momentum",+ "nblocks","enable_minmax_tuning","enable_norm_bias_tuning",+ "gradient_accumulate_steps",+ "enable_alg_ext",+ "not_use_best_mse",+ "dynamic_max_gap","enable_quanted_input",- "enable_opt_rtn",- "disable_deterministic_algorithms",- "enable_deterministic_algorithms",- "static_kv_dtype",- "static_attention_dtype",- "rotation_config",- "processor",- "image_processor",- "template",- "extra_data_dir",- "quant_nontext_module",- "guidance_scale",- "num_inference_steps",- "generator_seed",+ "optimizer",+ "enable_adam",+ "enable_lfq",+}+_RTN_FIELDS = {"disable_opt_rtn", "enable_opt_rtn"}+_AWQ_FIELDS = {"duo_scaling","n_grid",+ "seqlen",+ "nsamples",+ "batch_size",+ "apply_smooth",+ "smooth_iters",+ "apply_clip",+ "clip_as_init",+ "clip_n_grid",+ "clip_max_shrink",+ "clip_n_sample_token","mappings",- "algorithm",- "optimizer",- "lr_scheduler",- "not_use_best_mse",- "dynamic_max_gap",- "momentum",- "device",+}+_ROTATION_FIELDS = {+ "hadamard_type",+ "block_size",+ "fuse_online_to_weight",+ "allow_online_rotation",}-def _filter_supported_compat_kwargs(kwargs: dict) -> dict:- supported = {}- unknown = []- for key, value in kwargs.items():- if key in _COMPAT_KWARGS:- supported[key] = value- else:- unknown.append(key)+def _filter_supported_entry_kwargs(kwargs, *, context="AutoRound"):+ supported = {key: value for key, value in kwargs.items() if key in _ENTRY_KWARG_OWNERS}+ unknown = sorted(set(kwargs) - set(_ENTRY_KWARG_OWNERS))if unknown:logger.warning_once(- "AutoRound compatibility path received unsupported kwargs %s. They will be ignored.",- ", ".join(sorted(unknown)),+ "%s received unsupported runtime kwargs %s. They will be ignored.",+ context,+ ", ".join(unknown),)return supported-class AutoRound:- """Automatic weight rounding (Signed Gradient Descent) for LLM quantization-- Reference:- Cheng, Wenhua, et al., "Optimize weight rounding via signed gradient descent for- the quantization of LLMs." arXiv:2309.05516 (2023).-- Attributes:- model (torch.nn.Module | str): The loaded PyTorch model in eval mode.- tokenizer: Tokenizer used to prepare input text for calibration/tuning.- platform (str): The platform to load pretrained moded, options: ["hf", "model_scope"]- bits (int): Weight quantization bits.- group_size (int or tuple): Per-group size for weight quantization.- sym (bool): Whether to use symmetric weight quantization.- layer_config (dict): Per-layer quantization configuration.- nsamples (int): Number of calibration samples.- enable_torch_compile (bool): Whether to enable torch.compile for quant blocks/layers. Defaults to True except- on Windows, where it defaults to False.+def _split_entry_kwargs(kwargs, *, context="AutoRound"):+ buckets = {"route": {}, "compressor": {}, "base": {}, "mllm": {}, "diffusion": {}}+ for key, value in _filter_supported_entry_kwargs(kwargs, context=context).items():+ buckets[_ENTRY_KWARG_OWNERS[key]][key] = value+ return buckets+++def _config_fields(config):+ fields = set(_SCHEME_FIELDS)+ name = type(config).__name__.lower()+ if "awq" in name:+ fields.update(_AWQ_FIELDS)+ elif "rtn" in name:+ fields.update(_RTN_FIELDS)+ elif "signround" in name or "adamround" in name:+ fields.update(_SIGNROUND_FIELDS)+ elif "rotation" in name:+ fields.update(_ROTATION_FIELDS)+ return fields+++def _normalize_alg_configs(alg_configs, direct_kwargs=None):+ from auto_round.algorithms.config_resolver import split_quantization_configs+ from auto_round.algorithms.quantization.config import QuantizationConfig+ from auto_round.algorithms.quantization.rtn.config import RTNConfig+ from auto_round.algorithms.registry import normalize_algorithm_config, resolve_alg_config, resolve_algorithm_names+ from auto_round.algorithms.transforms import normalize_rotation_config+ from auto_round.algorithms.transforms.base import BaseRotationConfig++ direct_kwargs = dict(direct_kwargs or {})+ legacy_algorithm = direct_kwargs.pop("algorithm", None)+ if legacy_algorithm is not None:+ if alg_configs is not None:+ raise ValueError("`algorithm` and `alg_configs` cannot be used together.")+ alg_configs = legacy_algorithm+ logger.warning_once("`algorithm` is deprecated; use `alg_configs` instead.")+ if "backend" in direct_kwargs:+ raise ValueError(+ "Rotation backend selection must be nested in `rotation_config`; "+ "do not pass it as AutoRound(..., backend=...)."+ )+ rotation_config = direct_kwargs.pop("rotation_config", None)+ config_kwargs = {key: value for key, value in direct_kwargs.items() if key not in _ENTRY_KWARG_OWNERS}+ if alg_configs is None:+ # Preserve the legacy entry semantics: zero iterations are RTN, while+ # positive iterations use SignRound. RTN-only kwargs also select RTN+ # so they are not silently ignored by the default SignRound config.+ if direct_kwargs.get("iters") == 0:+ raw_configs = ["rtn"]+ else:+ raw_configs = ["signround"]+ elif isinstance(alg_configs, str):+ raw_configs = resolve_algorithm_names(alg_configs)+ if not raw_configs:+ raise ValueError("`algorithm`/`alg_configs` must contain at least one algorithm name.")+ elif isinstance(alg_configs, (list, tuple)):+ raw_configs = []+ seen_names = set()+ for raw_config in alg_configs:+ if not isinstance(raw_config, str):+ raw_configs.append(raw_config)+ continue+ for canonical_name in resolve_algorithm_names(raw_config):+ if canonical_name not in seen_names:+ raw_configs.append(canonical_name)+ seen_names.add(canonical_name)+ else:+ raw_configs = [alg_configs]++ configs = []+ for raw_config in raw_configs:+ config = resolve_alg_config(raw_config) if isinstance(raw_config, str) else raw_config+ if not isinstance(config, (QuantizationConfig, BaseRotationConfig)):+ raise TypeError(+ f"alg_configs entries must be algorithm or QuantizationConfig instances, "+ f"got {type(config).__name__}."+ )+ configs.append(normalize_algorithm_config(config))++ # ``iters=0`` has always selected RTN in the public entry and CLI. Apply+ # that rule after every input form has become a config so aliases, config+ # objects, and the deprecated ``algorithm`` argument cannot choose+ # different quantizer implementations.+ from auto_round.algorithms.quantization.sign_round.config import SignRoundConfig++ direct_iters = config_kwargs.get("iters")+ for index, config in enumerate(configs):+ effective_iters = direct_iters if direct_iters is not None else getattr(config, "iters", None)+ if isinstance(config, SignRoundConfig) and effective_iters == 0:+ rtn_config = RTNConfig(scheme=config.scheme.copy())+ rtn_config._user_set_scheme_fields = set(getattr(config, "_user_set_scheme_fields", set()))+ configs[index] = normalize_algorithm_config(rtn_config)++ if rotation_config is not None:+ normalized_rotation = normalize_rotation_config(rotation_config)+ if normalized_rotation is not None:+ configs.append(normalized_rotation)++ if not any(isinstance(config, QuantizationConfig) for config in configs):+ raise TypeError(+ "alg_configs entries must be algorithm aliases or QuantizationConfig instances, "+ "and must include at least one quantization algorithm config."+ )++ preprocessors, block_configs = split_quantization_configs(configs)+ if preprocessors and not block_configs:+ configs.append(normalize_algorithm_config(RTNConfig()))++ _, block_configs = split_quantization_configs(configs)+ for key, value in config_kwargs.items():+ if value is None:+ continue+ if key in _SCHEME_FIELDS:+ targets = block_configs+ else:+ targets = [config for config in configs if key in _config_fields(config)]+ if not targets:+ # ``iters`` is a legacy route selector. RTN intentionally has no+ # iterative parameter, so ``iters=0`` must not be reported as an+ # ignored algorithm-specific error after selecting RTN.+ if key == "iters" and any(isinstance(config, RTNConfig) for config in configs):+ continue+ owner = "AWQ" if key in _AWQ_FIELDS else "the selected algorithm"+ logger.error(+ "%s-specific parameter '%s' was provided, but %s is not enabled by alg_configs. "+ "The parameter is ignored.",+ owner,+ key,+ owner,+ )+ continue+ if len(targets) > 1:+ logger.error(+ "Parameter '%s' matches multiple algorithm configs. Pass it through the matching "+ "config object in 'alg_configs'; the direct value is ignored.",+ key,+ )+ continue+ target = targets[0]+ if key in ("disable_opt_rtn", "enable_opt_rtn"):+ if key == "disable_opt_rtn" or value:+ target.disable_opt_rtn = False if key == "enable_opt_rtn" else value+ target.orig_disable_opt_rtn = target.disable_opt_rtn+ else:+ setattr(target, key, value)+ logger.warning(+ "Passing '%s' directly to AutoRound is supported, but the recommended usage is "+ "'alg_configs=%sConfig(...)'.",+ key,+ type(targets[0]).__name__.replace("Config", ""),+ )+ return [normalize_algorithm_config(config) for config in configs]+++def _prepare_entry_kwargs(alg_configs, direct_kwargs):+ configs = _normalize_alg_configs(alg_configs, direct_kwargs)+ runtime_kwargs = {key: value for key, value in direct_kwargs.items() if key in _ENTRY_KWARG_OWNERS}+ return configs, runtime_kwargs+++class _CompressorBuilder(object):+ """Algorithm-config-driven entry point (``scheme`` + ``alg_configs``).++ This is the internal pipeline entry: it resolves the algorithm config(s),+ routes to the concrete :class:`BaseCompressor` subclass (ZeroShot / DataDriven+ / ModelFree / …) wired with the right model-type Mixin, and returns that+ compressor instance. It is distinct from the public dispatcher+ :class:`auto_round.AutoRound` (in ``auto_round/autoround.py``)."""- SKIP_ARGS = ("local_args", "kwargs", "cls", "model_cls", "dynamic_compressor", "alg_configs")+ @classmethod+ def _resolve_config(cls, config: Union[str, object, list]) -> Union[object, list[object]]:+ """Convert string alias(es) to the corresponding config instance(s) with default parameters."""+ from auto_round.algorithms.registry import resolve_alg_config++ if isinstance(config, str):+ return resolve_alg_config(config)+ if isinstance(config, list):+ return [cls._resolve_config(c) for c in config]+ return config- bits: int | None- group_size: int | tuple | None- sym: bool | None- data_type: str | None- act_bits: int | None- act_group_size: int | None- act_sym: bool | None- act_data_type: str | None- act_dynamic: bool | None- super_bits: int | None- super_group_size: int | None-- # all args in __new__ need be passed to the dynamic created class __init__def __new__(cls,model: Union[torch.nn.Module, str],+ scheme="W4A16",+ alg_configs: Union[str, object, list[Union[str, object]]] = None,tokenizer=None,- platform: str = "hf",- scheme: Union[str, dict, QuantizationScheme, "AutoScheme"] = "W4A16",- layer_config: dict[str, Union[str, dict, QuantizationScheme]] = None,- dataset: Optional[Union[str, list, tuple, torch.utils.data.DataLoader]] = None,- iters: int = 200,- seqlen: int = 2048,- nsamples: int = 128,- batch_size: int = 8,- gradient_accumulate_steps: int = 1,+ platform="hf",+ format=None,+ dataset="NeelNanda/pile-10k",low_gpu_mem_usage: bool = False,device_map: Union[str, torch.device, int, dict] = 0,- enable_torch_compile: Optional[bool] = None,+ iters: int = None,+ enable_torch_compile: bool = False,seed: int = 42,- enable_adam: bool = False,- enable_alg_ext: bool = False,- disable_opt_rtn: bool | None = None,low_cpu_mem_usage: bool = True,- alg_configs=None,+ layer_config=None,+ nsamples: int = None,+ seqlen: int = None,**kwargs,) -> "BaseCompressor":- """Initialize AutoRound with quantization and tuning configuration.-- Args:- model (torch.nn.Module | str): Model object or model name to load.- tokenizer: Tokenizer for text processing. Required if `model` is not a string and `iters > 0`.- platform: The platform to download pretrained model, options: ["hf", "model_scope"]- scheme (str| dict | QuantizationScheme ): A preset scheme that defines the quantization configurations- layer_config (dict, optional): Layer-wise quantization config. Defaults to None.- dataset (str | list | tuple | DataLoader, optional): Calibration data. Defaults to "NeelNanda/pile-10k".- iters (int, optional): Optimization iterations. Defaults to 200.- seqlen (int, optional): Calibration sequence length. Defaults to 2048.- nsamples (int, optional): Number of calibration samples. Defaults to 128.- batch_size (int, optional): Calibration batch size. Defaults to 8.- gradient_accumulate_steps (int, optional): Gradient accumulation steps. Defaults to 1.- low_gpu_mem_usage (bool, optional): Lower GPU memory mode. Defaults to False.- device_map (str | dict, optional): Device map for each module. Defaults to 0.- enable_torch_compile (bool, optional): Enable torch.compile for low cost in quantization. Defaults to True- except on Windows, where it defaults to False. Pass True to force it.- seed (int, optional): Random seed. Defaults to 42.- enable_adam (bool, optional): Enable Adam-based optimizer. Defaults to False.- enable_alg_ext (bool, optional): Enable algorithm extension (primarily for INT2)- for better accuracy. Defaults to False.- disable_opt_rtn (bool, optional): Disable RTN-mode optimization (iters=0) for fast quatnziation- with lower accuracy. Defaults to None.- low_cpu_mem_usage (bool, optional): Lower CPU memory mode. Defaults to False.-- bits (int, optional): Weight quantization bits. Defaults to 4.- group_size (int or tuple, optional): Weight quantization group size. Defaults to 128.- sym (bool, optional): Symmetric weight quantization. Defaults to True.- data_type (str, optional): Weight data type string, e.g., "int". Defaults to "int".- act_bits (int, optional): Activation quantization bits. Defaults to 16.- act_group_size (int, optional): Activation group size. Defaults to None.- act_sym (bool, optional): Symmetric activation quantization. Defaults to None.- act_data_type (str, optional): Activation data type; inherits weight dtype if None and act_bits < 16.- act_dynamic (bool, optional): Dynamic activation quantization. Defaults to True.- model_dtype (str): model dtype used to load pre-trained model.- amp (bool, optional): Use AMP for tuning. Defaults to True.- enable_minmax_tuning (bool, optional): Enable weight min-max tuning. Defaults to True.- lr (float, optional): Learning rate; if None, set to 1.0 / iters except when iters==0.- minmax_lr (float, optional): Learning rate for min-max tuning; defaults to `lr`.-- **kwargs: Backward compatible options:- - enable_alg_ext, quant_lm_head, lr, lr_scheduler, sampler, not_use_best_mse, dynamic_max_gap,- super_group_size, super_bits, scale_dtype ("fp16" etc.),- nblocks, to_quant_block_names,- enable_norm_bias_tuning, enable_quanted_input,- disable_deterministic_algorithms, vlm, static_kv_dtype- Raises:- ValueError: If invalid device is provided or tokenizer is missing for non-str model with iters > 0.- RuntimeError: If model parameters are on meta device.- Example:- Layer-wise configuration structure:-- >>> layer_config = {- ... "layer1": {- ... "bits": 3,- ... "group_size": 128,- ... "sym": True,- ... },- ... "layer2": {- ... "W8A16"- ... }- ... # ...- ... }- """+ from auto_round.algorithms.quantization.rtn.config import OptimizedRTNConfig, RTNConfig+ from auto_round.algorithms.quantization.sign_round.config import SignRoundConfig+ from auto_round.algorithms.registry import normalize_algorithm_config+ from auto_round.compressors.orchestrator import CompressionOrchestrator as Compressor+ from auto_round.compressors.utils import check_need_act_calibration+ from auto_round.utils.model import is_model_free_route++ if alg_configs is None:+ alg_configs = "signround"+ # TODO wenhuach if key in kwargs could override scheme and alg_config, we should pop and override,+ # e.g. gradient_accumulate_stepdevice_map = normalize_default_device_map(device_map)-- # Short-circuit: if alg_configs is provided, bypass AutoRoundCompatible and go directly- # to the new-arch entry point to avoid duplicate keyword argument errors.- if alg_configs is not None:- from auto_round.compressors.entry import AutoRound as _NewAutoRound- from auto_round.compressors.entry import filter_supported_entry_kwargs-- entry_kwargs = filter_supported_entry_kwargs(kwargs, context="AutoRound")-- return _NewAutoRound(+ split_kwargs = _split_entry_kwargs(kwargs)+ route_kwargs = dict(split_kwargs["route"])+ compressor_kwargs = dict(split_kwargs["compressor"])+ base_kwargs = dict(split_kwargs["base"])+ mllm_kwargs = dict(split_kwargs["mllm"])+ diffusion_kwargs = dict(split_kwargs["diffusion"])++ # Resolve string alias(es) to config instance(s) before routing.+ alg_configs = cls._resolve_config(alg_configs)+ if isinstance(alg_configs, list):+ alg_configs = [normalize_algorithm_config(cfg) for cfg in alg_configs]+ else:+ alg_configs = normalize_algorithm_config(alg_configs)+ configs_for_routing = alg_configs if isinstance(alg_configs, list) else [alg_configs]+ preprocessor_configs, _, quant_config = _resolve_quant_config_for_routing(configs_for_routing)++ # Model-free routing is now supported directly by the new entry path.+ model_free_iters = 0 if isinstance(quant_config, RTNConfig) else getattr(quant_config, "iters", None)+ model_free_disable_opt_rtn = getattr(quant_config, "disable_opt_rtn", None)+ # Model-free eligibility also depends on base-level options such as+ # static KV/attention quantization. Keep those options visible to the+ # route predicate; otherwise the fast path silently drops them and+ # cannot emit the required export metadata.+ route_decision_kwargs = dict(base_kwargs, **route_kwargs, format=format)+ route_scheme = (+ scheme+ if hasattr(scheme, "options") and hasattr(scheme, "avg_bits")+ else QuantizationScheme.from_dict(_preview_resolved_attrs(quant_config, scheme))+ )+ if is_model_free_route(+ model, route_scheme, model_free_iters, model_free_disable_opt_rtn, route_decision_kwargs+ ):+ # Direct scheme fields are consumed into ``quant_config`` during+ # entry normalization. Pass the fully resolved scheme onward so+ # model-free export does not silently fall back to preset defaults.+ model_free_scheme = scheme if hasattr(scheme, "options") and hasattr(scheme, "avg_bits") else route_scheme+ return _build_model_free_compressor(model,- scheme,- alg_configs,- tokenizer=tokenizer,- platform=platform,- format=entry_kwargs.pop("format", None),- low_gpu_mem_usage=low_gpu_mem_usage,- device_map=device_map,- iters=iters,- gradient_accumulate_steps=gradient_accumulate_steps,+ model_free_scheme,+ layer_config,+ tokenizer,+ device_map,+ announced_via_flag=bool(route_kwargs.get("model_free", False)),enable_torch_compile=enable_torch_compile,- seed=seed,- low_cpu_mem_usage=low_cpu_mem_usage,- layer_config=layer_config,- dataset=dataset,- nsamples=nsamples,- seqlen=seqlen,- batch_size=batch_size,- **entry_kwargs,+ **compressor_kwargs,+ **base_kwargs,+ **mllm_kwargs,+ **diffusion_kwargs,+ **route_kwargs,)- compat_kwargs = _filter_supported_compat_kwargs(kwargs)- compat_kwargs.update(- enable_adam=enable_adam,- enable_alg_ext=enable_alg_ext,- disable_opt_rtn=disable_opt_rtn,- )-- from auto_round.compressors.entry import AutoRoundCompatible+ # Eagerly validate scheme constraints that do not require model info.+ # This mirrors old-arch _check_configs() called at __init__ time so that+ # callers get ValueError/NotImplementedError on construction, not deferred.+ _eager_validate_scheme(quant_config, scheme)- return AutoRoundCompatible(+ local_args = dict(model=model,tokenizer=tokenizer,platform=platform,+ format=format,scheme=scheme,- layer_config=layer_config,dataset=dataset,- iters=iters,- seqlen=seqlen,- nsamples=nsamples,- batch_size=batch_size,- gradient_accumulate_steps=gradient_accumulate_steps,low_gpu_mem_usage=low_gpu_mem_usage,device_map=device_map,+ iters=iters,enable_torch_compile=enable_torch_compile,seed=seed,low_cpu_mem_usage=low_cpu_mem_usage,- **compat_kwargs,+ layer_config=layer_config,+ nsamples=nsamples,+ seqlen=seqlen,+ **compressor_kwargs,)+ model_type, ctor_kwargs = _build_model_type_ctor_kwargs(model, base_kwargs, mllm_kwargs, diffusion_kwargs)- @classmethod- @torch.no_grad()- def _sampling_inputs(+ # Preprocessor algorithms (AWQ, …) require a data-driven host so that+ # the per-block preprocessor lifecycle (prepare_block_group ->+ # block_forward_hooks -> pre_quantize_block -> post_quantize_block)+ # actually runs; the pipeline auto-appends RTN when no block_quantizer+ # is supplied. SignRound is itself data-driven and shares the same host.+ if preprocessor_configs or isinstance(quant_config, SignRoundConfig):+ return _get_compressor_class(model_type, Compressor)(alg_configs, **local_args, **ctor_kwargs)+ elif isinstance(quant_config, RTNConfig):+ base_cls = _select_rtn_compressor_base_cls(quant_config, scheme, format, base_kwargs)+ return _get_compressor_class(model_type, base_cls)(alg_configs, **local_args, **ctor_kwargs)+++class AutoRound:+ """Unified AutoRound entry point.++ alg_configs accepts an algorithm alias, one QuantizationConfig, or a+ sequence of either. When omitted, SignRound is selected. AWQ-only+ pipelines receive an RTN block quantizer by default.+ """++ SKIP_ARGS = ("local_args", "kwargs", "cls", "model_cls", "dynamic_compressor", "alg_configs")++ def __new__(cls,- input_ids: list[torch.Tensor],- input_others: dict,- indices: list[int],- seqlen: int,- batch_dim: int = 0,- share_cache_keys: tuple = (),- ):- """Samples inputs based on the given indices and sequence length.-- Args:- input_ids: The list of input tensor containing input_ids.- input_others: A dictionary containing other input data.- indices: The indices to sample from the input.- seqlen: The sequence length.-- Returns:- current_input_ids: The sampled input IDs.- current_input_others: The sampled other input data.- """- current_input_ids = [input_ids[i] for i in indices]-- current_input_ids = torch.cat(current_input_ids, dim=batch_dim)-- current_input_others = {"positional_inputs": input_others["positional_inputs"]}- for key in input_others.keys():- if "positional_inputs" in key:- continue- # Shared cache keys (e.g. position_embeddings, position_ids, cache_position) are stored- # directly as-is (not wrapped in a per-sample list) when batch_size > 1. Indexing such- # values by sample index would incorrectly decompose them (e.g. (cos, sin)[0] == cos).- # Always pass them through unchanged.- if key in share_cache_keys or isinstance(input_others[key], (str, bool, type(None))):- current_input_others[key] = input_others[key]- elif input_others[key] is not None:- current_input_others[key] = [input_others[key][i] for i in indices]- if len(indices) == 1:- current_input_others[key] = current_input_others[key][0]- else:- try:- current_input_others[key] = torch.cat(current_input_others[key], dim=0)- except TypeError as err:- logger.warning_once("Please check the model cache inputs or try setting batch_size to 1.")- else:- current_input_others[key] = None-- return current_input_ids, current_input_others+ model: Union[torch.nn.Module, str],+ tokenizer=None,+ platform: str = "hf",+ scheme: Union[str, dict, QuantizationScheme, "AutoScheme"] = "W4A16",+ layer_config: dict[str, Union[str, dict, QuantizationScheme]] = None,+ dataset: Optional[Union[str, list, tuple, torch.utils.data.DataLoader]] = None,+ iters: int | None = None,+ seqlen: int = 2048,+ nsamples: int = 128,+ batch_size: int = 8,+ gradient_accumulate_steps: int | None = None,+ low_gpu_mem_usage: bool = False,+ device_map: Union[str, torch.device, int, dict] = 0,+ enable_torch_compile: Optional[bool] = None,+ seed: int = 42,+ low_cpu_mem_usage: bool = True,+ alg_configs=None,+ algorithm: str | None = None,+ **kwargs,+ ) -> "BaseCompressor":+ direct_kwargs = dict(kwargs)+ legacy_device = direct_kwargs.pop("device", None)+ if legacy_device is not None:+ logger.warning_once("`device` is deprecated, please use `device_map` instead")+ if device_map in (None, 0):+ device_map = legacy_device+ if iters is not None:+ direct_kwargs["iters"] = iters+ if gradient_accumulate_steps is not None:+ direct_kwargs["gradient_accumulate_steps"] = gradient_accumulate_steps+ if algorithm is not None:+ direct_kwargs["algorithm"] = algorithm++ configs, runtime_kwargs = _prepare_entry_kwargs(alg_configs, direct_kwargs)+ runtime_kwargs["batch_size"] = batch_size++ return _CompressorBuilder(+ model,+ scheme,+ configs,+ tokenizer=tokenizer,+ platform=platform,+ format=runtime_kwargs.pop("format", None),+ dataset=dataset,+ low_gpu_mem_usage=low_gpu_mem_usage,+ device_map=normalize_default_device_map(device_map),+ iters=None,+ enable_torch_compile=enable_torch_compile,+ seed=seed,+ low_cpu_mem_usage=low_cpu_mem_usage,+ layer_config=layer_config,+ nsamples=nsamples,+ seqlen=seqlen,+ **runtime_kwargs,+ )+# Keep legacy entry points available for downstream integrations such as+# Neural Compressor. They delegate to the unified entry point.@deprecated("AutoRound")class AutoRoundLLM:-def __new__(cls, *args, **kwargs):return AutoRound(*args, **kwargs)@deprecated("AutoRound")class AutoRoundAdam:-def __new__(cls, *args, **kwargs):kwargs.setdefault("enable_adam", True)return AutoRound(*args, **kwargs)@@ -351,13 +788,11 @@ class AutoRoundAdam:@deprecated("AutoRound")class AutoRoundMLLM:-def __new__(cls, *args, **kwargs):return AutoRound(*args, **kwargs)@deprecated("AutoRound")class AutoRoundDiffusion:-def __new__(cls, *args, **kwargs):return AutoRound(*args, **kwargs)diff --git a/auto_round/cli/algorithms.py b/auto_round/cli/algorithms.pyindex 8291d1d82..193d70583 100644--- a/auto_round/cli/algorithms.py+++ b/auto_round/cli/algorithms.py@@ -22,6 +22,7 @@ from auto_round.algorithms.registry import (iter_algorithm_entries,register_algorithm,resolve_algorithm_alias,+ resolve_algorithm_names,)# ============================================================================@@ -108,19 +109,21 @@ class AlgorithmHandler(ABC):if getattr(args, "rotation_hadamard_type", None) and "hadamard" not in names:names.append("hadamard")- # Resolve aliases, drop unknowns, deduplicate preserving order- seen: set[str] = set()- canonical: list[str] = []- for n in names:- c = cls.resolve_alias(n)- if c and c not in seen:- canonical.append(c)- seen.add(c)+ # Resolve aliases, drop unknowns, deduplicate preserving order.+ # Python API strings use this same resolver so their composition+ # semantics cannot drift apart again.+ canonical = resolve_algorithm_names(names, ignore_unknown=True)+ seen = set(canonical)# Default quantization algorithm if none was specifiedif not ({"awq", "rtn", "auto_round"} & seen):canonical.append("rtn" if getattr(args, "iters", 0) == 0 else "auto_round")+ # Keep the legacy API rule even when the user explicitly spells out+ # ``--algorithm auto_round``: zero iterations select RTN.+ if getattr(args, "iters", None) == 0:+ canonical = ["rtn" if name == "auto_round" else name for name in canonical]+return [cls.get(name).build(args, common_kwargs) for name in canonical]@classmethoddiff --git a/auto_round/cli/main.py b/auto_round/cli/main.pyindex 50871dfb7..e99595119 100644--- a/auto_round/cli/main.py+++ b/auto_round/cli/main.py@@ -24,6 +24,8 @@ from auto_round.cli.parser import (build_quantize_parser,build_root_parser,)+from auto_round.compressors.config_resolution import ConfigResolutionError+from auto_round.logger import loggerdef _extract_common_quantization_kwargs(args) -> dict:@@ -142,7 +144,7 @@ RECIPES = {def list_item(argv=None):args = build_list_parser().parse_args(argv)if args.item in {"format", "formats"}:- from auto_round.formats import OutputFormat+ from auto_round.export.formats import OutputFormatprint("AutoRound supported output formats and quantization scheme:")print(OutputFormat.get_support_matrix())@@ -288,8 +290,8 @@ def tune(args):model_name = model_name[:-1]logger.info(f"start to quantize {model_name}")+ from auto_round import AutoRoundfrom auto_round.compressors.base import BaseOrchestrator as BaseCompressor- from auto_round.compressors.entry import AutoRound as PipelineAutoRoundif "bloom" in model_name:args.low_gpu_mem_usage = False@@ -383,10 +385,10 @@ def tune(args):from auto_round.utils import clear_memory- autoround: BaseCompressor = PipelineAutoRound(+ autoround: BaseCompressor = AutoRound(model_name,- scheme,- alg_configs if len(alg_configs) > 1 else alg_configs[0],+ scheme=scheme,+ alg_configs=alg_configs if len(alg_configs) > 1 else alg_configs[0],**_to_autoround_kwargs(args,low_cpu_mem_usage=low_cpu_mem_usage,@@ -481,20 +483,24 @@ def _print_help(topic=None):def run():- argv = list(sys.argv[1:])- command, command_argv = _normalize_cli_invocation(argv)-- if command == "help":- root_args = build_root_parser().parse_args(argv)- _print_help(root_args.topic)- return- if command == "list":- list_item(command_argv)- return- if command == "eval":- run_eval(command_argv)- return- start(argv=command_argv)+ try:+ argv = list(sys.argv[1:])+ command, command_argv = _normalize_cli_invocation(argv)++ if command == "help":+ root_args = build_root_parser().parse_args(argv)+ _print_help(root_args.topic)+ return+ if command == "list":+ list_item(command_argv)+ return+ if command == "eval":+ run_eval(command_argv)+ return+ start(argv=command_argv)+ except ConfigResolutionError as error:+ logger.error(str(error))+ raise SystemExit(2) from errordef run_best():diff --git a/auto_round/compressors/__init__.py b/auto_round/compressors/__init__.pyindex 833e764ec..7965acf4b 100644--- a/auto_round/compressors/__init__.py+++ b/auto_round/compressors/__init__.py@@ -18,29 +18,20 @@ from typing import TYPE_CHECKINGif TYPE_CHECKING:from auto_round.compressors.base import BaseOrchestrator, BaseCompressor- from auto_round.compressors.orchestrator import CompressionOrchestrator- from auto_round.compressors.entry import AutoRoundCompatible, AutoRoundfrom auto_round.compressors.model_free import ModelFreeCompressor+ from auto_round.compressors.orchestrator import CompressionOrchestrator__all__ = [- "AutoRound","BaseOrchestrator","BaseCompressor", # backward-compat alias"CompressionOrchestrator",- "AutoRoundCompatible","ModelFreeCompressor",]def __getattr__(name):"""Lazy import to avoid circular dependencies."""- if name in ("AutoRound", "AutoRoundCompatible"):- from auto_round.compressors.entry import AutoRound, AutoRoundCompatible-- if name == "AutoRound":- return AutoRound- return AutoRoundCompatible- elif name in ("BaseOrchestrator", "BaseCompressor"):+ if name in ("BaseOrchestrator", "BaseCompressor"):from auto_round.compressors.base import BaseOrchestratorreturn BaseOrchestrator@@ -48,8 +39,9 @@ def __getattr__(name):from auto_round.compressors.orchestrator import CompressionOrchestratorreturn CompressionOrchestrator- elif name in ("Compressor", "ZeroShotCompressor"):- # Backward-compat aliases+ elif name in ("Compressor", "ZeroShotCompressor", "DataDrivenCompressor", "CalibratedRTNCompressor"):+ # Backward-compat aliases: these classes were consolidated into+ # ``CompressionOrchestrator`` (main's #2039 compressor/quantizer refactor).from auto_round.compressors.orchestrator import CompressionOrchestratorreturn CompressionOrchestratordiff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.pyindex f8d6cfd9a..1da140d04 100644--- a/auto_round/compressors/base.py+++ b/auto_round/compressors/base.py@@ -15,7 +15,7 @@ import copyimport gcimport osimport sys-from dataclasses import asdict, dataclass, fields+from dataclasses import asdict, dataclass, fields, replacefrom typing import Any, Optional, Unionimport torch@@ -27,11 +27,23 @@ from auto_round.algorithms.transforms import (apply_rotation,)from auto_round.auto_scheme.gen_auto_scheme import AutoScheme+from auto_round.compressors.config_resolution import (+ FormatResolution,+ ResolvedScheme,+ resolve_quantization_config,+ thaw_mapping,+)+from auto_round.compressors.layer_config import (+ apply_plan_to_model,+ extract_regex_config,+ has_quantized_layer_outside_blocks,+ resolve_layer_config,+)from auto_round.compressors.shard_writer import ShardWriter-from auto_round.compressors.utils import _get_save_folder_name, is_act_static, is_mx_fp, is_nv_fp, set_layer_config+from auto_round.compressors.utils import _get_save_folder_name, is_mx_fp, is_nv_fpfrom auto_round.context.compress import CompressContextfrom auto_round.context.model import ModelContext-from auto_round.formats import OutputFormat, get_formats+from auto_round.export.formats import OutputFormat, resolve_formatsfrom auto_round.logger import loggerfrom auto_round.schemes import (QuantizationScheme,@@ -160,11 +172,6 @@ class BaseOrchestrator(object):is_auto_scheme: bool = Falseorig_scheme = Nonescheme = None- scale_dtype = None- layer_config = None- has_qlayer_outside_block: bool = False- regex_config: dict = None- quant_block_list: list = Noneto_quant_block_names = Noneignore_layers: str = ""quant_lm_head: bool = False@@ -523,6 +530,100 @@ class BaseOrchestrator(object):"""Convenience accessor for the tokenizer stored in ``model_context``."""return self.model_context.tokenizer+ def _replace_compression_plan(self, **changes) -> None:+ """Atomically update compatibility state once the immutable plan exists."""+ plan = self.__dict__.get("compression_plan")+ if plan is not None:+ self.__dict__["compression_plan"] = replace(plan, **changes)++ @property+ def scheme_context(self) -> Optional[QuantizationScheme]:+ plan = self.__dict__.get("compression_plan")+ return plan.scheme.value if plan is not None else self.__dict__.get("_scheme_context")++ @scheme_context.setter+ def scheme_context(self, value: Optional[QuantizationScheme]) -> None:+ self.__dict__["_scheme_context"] = value+ plan = self.__dict__.get("compression_plan")+ if plan is not None and value is not None:+ self._replace_compression_plan(+ scheme=ResolvedScheme.from_scheme(value, preset_name=plan.scheme.preset_name)+ )++ @property+ def formats(self):+ raw_value = self.__dict__.get("_formats")+ plan = self.__dict__.get("compression_plan")+ if plan is None or isinstance(raw_value, str) or (not plan.formats and raw_value is None):+ return raw_value+ return list(plan.formats)++ @formats.setter+ def formats(self, value) -> None:+ self.__dict__["_formats"] = value+ if value is not None and not isinstance(value, str):+ self._replace_compression_plan(formats=tuple(value))++ @property+ def layer_config(self) -> Optional[dict]:+ plan = self.__dict__.get("compression_plan")+ if plan is None:+ return self.__dict__.get("_layer_config")+ return {name: dict(config) for name, config in plan.layer_config.items()}++ @layer_config.setter+ def layer_config(self, value) -> None:+ self.__dict__["_layer_config"] = value+ if value is not None:+ self._replace_compression_plan(layer_config=value)++ @property+ def regex_config(self) -> Optional[dict]:+ plan = self.__dict__.get("compression_plan")+ if plan is None:+ return self.__dict__.get("_regex_config")+ return {name: dict(config) for name, config in plan.regex_config.items()}++ @regex_config.setter+ def regex_config(self, value) -> None:+ self.__dict__["_regex_config"] = value+ if value is not None:+ self._replace_compression_plan(regex_config=value)++ @property+ def has_qlayer_outside_block(self) -> bool:+ plan = self.__dict__.get("compression_plan")+ if plan is None:+ return self.__dict__.get("_has_qlayer_outside_block", False)+ return plan.has_qlayer_outside_block++ @has_qlayer_outside_block.setter+ def has_qlayer_outside_block(self, value: bool) -> None:+ self.__dict__["_has_qlayer_outside_block"] = value+ self._replace_compression_plan(has_qlayer_outside_block=value)++ @property+ def scale_dtype(self):+ plan = self.__dict__.get("compression_plan")+ return plan.scale_dtype if plan is not None else self.__dict__.get("_scale_dtype")++ @scale_dtype.setter+ def scale_dtype(self, value) -> None:+ self.__dict__["_scale_dtype"] = value+ self._replace_compression_plan(scale_dtype=value)++ @property+ def quant_block_list(self):+ plan = self.__dict__.get("compression_plan")+ if plan is None:+ return self.__dict__.get("_quant_block_list")+ return [list(group) for group in plan.quant_block_list] if plan.quant_block_list is not None else None++ @quant_block_list.setter+ def quant_block_list(self, value) -> None:+ self.__dict__["_quant_block_list"] = value+ self._replace_compression_plan(quant_block_list=value)+# ── Scheme resolution ─────────────────────────────────────────────────────def resolve_scheme(@@ -532,7 +633,7 @@ class BaseOrchestrator(object):) -> None:"""Phase-1 init: resolve scheme and bind config attrs (no model structure needed).- Must be called BEFORE ``get_formats()`` and BEFORE ``_scheme_post_init()``.+ Must be called BEFORE ``_resolve_formats()`` and BEFORE ``_scheme_post_init()``.Idempotent: safe to call multiple times."""if self._scheme_resolved:@@ -632,19 +733,53 @@ class BaseOrchestrator(object):"Please save the model using the `fake` format for now.")- layer_config, self.has_qlayer_outside_block, self.regex_config = set_layer_config(- self.model_context.model,- self.layer_config,- self.scheme,- self.scale_dtype,- self.supported_types,- self.inner_supported_types,- self.quant_block_list,- self.ignore_layers,- self.quant_lm_head,+ preset_name = self.scheme if isinstance(self.scheme, str) else None+ format_resolution = getattr(+ self,+ "_format_resolution",+ FormatResolution(+ formats=tuple(self.formats or ()),+ scheme=ResolvedScheme.from_scheme(self.scheme_context, preset_name=preset_name),+ scale_dtype=self.scale_dtype,+ quant_block_list=self.quant_block_list,+ ),+ )+ resolved_layer_config = resolve_layer_config(+ model=self.model_context.model,+ scheme=format_resolution.scheme,+ layer_config=self.layer_config,+ scale_dtype=self.scale_dtype,+ supported_types=self.supported_types,+ inner_supported_types=self.inner_supported_types,+ quant_block_list=self.quant_block_list,+ ignore_layers=self.ignore_layers,+ quant_lm_head=self.quant_lm_head,enable_gguf_official_mixed=False,is_mllm=self.model_context.is_mllm,)+ regex_config = extract_regex_config(+ model=self.model_context.model,+ scheme=format_resolution.scheme,+ layer_config=self.layer_config,+ scale_dtype=self.scale_dtype,+ supported_types=self.supported_types,+ inner_supported_types=self.inner_supported_types,+ ignore_layers=self.ignore_layers,+ )+ discovery_plan = resolve_quantization_config(+ (+ replace(format_resolution, layer_config_patch={})+ if format_resolution.layer_config_patch+ else format_resolution+ ),+ resolved_layer_config,+ regex_config=regex_config,+ has_qlayer_outside_block=has_quantized_layer_outside_blocks(resolved_layer_config),+ )+ apply_plan_to_model(self.model_context.model, discovery_plan)+ layer_config = {name: dict(config) for name, config in discovery_plan.layer_config.items()}+ self.has_qlayer_outside_block = discovery_plan.has_qlayer_outside_block+ self.regex_config = {name: dict(config) for name, config in discovery_plan.regex_config.items()}quant_layer_names = layer_config.keys()# ---- VLM: peel non-text sub-trees AutoScheme should not score ---- #@@ -731,6 +866,13 @@ class BaseOrchestrator(object):def configure_layer_config(self, enable_gguf_official_mixed: bool | None = True) -> None:"""Build ``self.layer_config`` from the resolved scheme on the patched model."""+ # External callers (e.g. llm-compressor's AutoRoundModifier) may invoke this+ # method directly without going through the normal post_init()/_scheme_post_init()+ # sequence. Make sure the scheme is resolved first so `self.scheme_context` (and+ # everything derived from it below) is never None -- resolve_scheme() is a no-op+ # if it already ran.+ if not self._scheme_resolved and hasattr(self, "_alg_configs"):+ self.resolve_scheme()_formats = getattr(self.compress_context, "formats", None)is_gguf_format = _formats is not None and any("gguf" in str(getattr(fmt, "output_format", "")) for fmt in _formats@@ -773,34 +915,64 @@ class BaseOrchestrator(object):quant_lm_head=self.quant_lm_head,mllm=self.model_context.is_mllm,)- _gguf_orig_fmt = getattr(self, "_gguf_original_format_name", None)- if _gguf_orig_fmt and "_MIXED" in _gguf_orig_fmt.upper():- self.layer_config = _handle_special_schemes(- _gguf_orig_fmt.lower(),- self.layer_config,- self.model_context.model,- supported_types=SUPPORTED_LAYER_TYPES,- inner_supported_types=INNER_SUPPORTED_LAYER_TYPES,- quant_lm_head=self.quant_lm_head,- mllm=self.model_context.is_mllm,- )fill_default_value = not self.is_auto_scheme- self.layer_config, self.has_qlayer_outside_block, self.regex_config = set_layer_config(- self.model_context.model,- self.layer_config,- self.scheme,- self.scale_dtype,- SUPPORTED_LAYER_TYPES,- INNER_SUPPORTED_LAYER_TYPES,- self.quant_block_list,- self.ignore_layers,- self.quant_lm_head,+ source_layer_config = self.layer_config+ format_resolution = getattr(+ self,+ "_format_resolution",+ FormatResolution(+ formats=tuple(self.formats or ()),+ scheme=ResolvedScheme.from_scheme(+ self.scheme_context,+ preset_name=self._resolve_gguf_preset_string(self.formats or []),+ ),+ scale_dtype=self.scale_dtype,+ quant_block_list=self.quant_block_list,+ ),+ )+ resolved_layer_config = resolve_layer_config(+ model=self.model_context.model,+ scheme=format_resolution.scheme,+ layer_config=source_layer_config,+ scale_dtype=self.scale_dtype,+ supported_types=SUPPORTED_LAYER_TYPES,+ inner_supported_types=INNER_SUPPORTED_LAYER_TYPES,+ quant_block_list=self.quant_block_list,+ ignore_layers=self.ignore_layers,+ quant_lm_head=self.quant_lm_head,enable_gguf_official_mixed=enable_gguf_official_mixed,is_mllm=self.model_context.is_mllm,fill_default_value=fill_default_value,- gguf_format_name=getattr(self, "_gguf_format_name", None),)+ regex_config = extract_regex_config(+ model=self.model_context.model,+ scheme=format_resolution.scheme,+ layer_config=source_layer_config,+ scale_dtype=self.scale_dtype,+ supported_types=SUPPORTED_LAYER_TYPES,+ inner_supported_types=INNER_SUPPORTED_LAYER_TYPES,+ ignore_layers=self.ignore_layers,+ fill_default_value=fill_default_value,+ )+ # ``resolved_layer_config`` already descends from (and fully subsumes)+ # ``format_resolution.layer_config_patch`` -- ``source_layer_config`` above was+ # seeded from that same patch before ``resolve_layer_config()`` expanded any+ # regex/partial keys (e.g. "self_attn") into concrete layer names. Re-merging the+ # patch here would reintroduce those now-stale, unexpanded keys into the final+ # plan, so drop it and rely solely on the fully-resolved layer configuration.+ if format_resolution.layer_config_patch:+ format_resolution = replace(format_resolution, layer_config_patch={})+ self.compression_plan = resolve_quantization_config(+ format_resolution,+ resolved_layer_config,+ regex_config=regex_config,+ has_qlayer_outside_block=has_quantized_layer_outside_blocks(resolved_layer_config),+ )+ self.layer_config = {name: dict(config) for name, config in self.compression_plan.layer_config.items()}+ self.regex_config = {name: dict(config) for name, config in self.compression_plan.regex_config.items()}+ self.has_qlayer_outside_block = self.compression_plan.has_qlayer_outside_block+ apply_plan_to_model(self.model_context.model, self.compression_plan)if self.is_auto_scheme:from auto_round.auto_scheme.utils import compute_avg_bits_for_model@@ -992,149 +1164,165 @@ class BaseOrchestrator(object):``model_context`` / ``compress_context`` / ``scale_dtype`` /``CalibrationContext`` from this compressor. ``quantizer.model``is a property that reads ``model_context.model``.- - Exposes ``self.quantizer`` as a ``@property`` (see below) that- transparently delegates to ``self.pipeline.block_quantizer`` so all- existing call-sites continue to work without modification.+ - Exposes ``self.alg_composer.block_quantizer`` so all quantization+ call-sites can reach the block quantizer directly.Postconditions:- - ``self.pipeline`` is an ``AlgorithmComposer`` wrapping the block quantizer.- - ``self.quantizer`` (via property) is ready and shares ``CalibrationContext``+ - ``self.alg_composer`` is an ``AlgorithmComposer`` wrapping the block quantizer.+ - ``self.alg_composer.block_quantizer`` is ready and shares ``CalibrationContext``with the compressor."""from auto_round.algorithms.composer import AlgorithmComposerself._alg_composer = AlgorithmComposer(self._alg_configs, orchestrator=self)+ # Sync the fully-resolved scheme state (built by _build_layer_config(), which+ # always runs before _build_composer() in post_init()) onto the block quantizer+ # so quantization methods (quantize_block, quantize_layer, etc.) have access to+ # layer_config, quant_block_list, etc. ``scale_dtype``/``scheme`` are NOT set here:+ # they are read-only properties backed by ``BaseAlgorithm.bind()``, which already+ # picked up the final ``self.scale_dtype``/``self.scheme_context`` above.+ plan = getattr(self, "compression_plan", None)+ if plan is not None:+ block_quantizer = self._alg_composer.block_quantizer+ block_quantizer.layer_config = {name: dict(config) for name, config in plan.layer_config.items()}+ block_quantizer.has_qlayer_outside_block = plan.has_qlayer_outside_block+ block_quantizer.regex_config = {name: dict(config) for name, config in plan.regex_config.items()}+ block_quantizer.quant_block_list = (+ [list(group) for group in plan.quant_block_list] if plan.quant_block_list is not None else None+ )+ block_quantizer.to_quant_block_names = self.to_quant_block_names+ block_quantizer.ignore_layers = self.ignore_layers++ from auto_round.algorithms.config_resolver import sync_shared_config_from++ sync_shared_config_from(block_quantizer.config, [pre.config for pre in self._alg_composer.preprocessors])++ # Also sync runtime-only state to all preprocessors so they have access to+ # per-layer quant config during pre-processing (e.g. AWQ grid search uses+ # layer_config to look up bits/group_size for each layer).+ for pre in self._alg_composer.preprocessors:+ pre.layer_config = block_quantizer.layer_config+@propertydef alg_composer(self) -> Any:"""The active :class:`~auto_round.algorithms.pipeline.AlgorithmComposer`."""return self._alg_composer+ @staticmethod+ def _resolve_gguf_preset_string(formats: list["OutputFormat"]) -> Optional[str]:+ """Return the precise GGUF preset string (e.g. ``"gguf:q4_k_m"``) for the+ single resolved GGUF format, or ``None`` if no GGUF format is present.++ Used by :meth:`_resolve_format_string` so ``self.scheme`` stays a string+ that :func:`auto_round.schemes.get_gguf_scheme` can short-circuit on,+ preserving alias disambiguation (Q4_K_S vs Q4_K_M, Q*_0/Q*_1, ...).+ ``_check_compatibility`` guarantees at most one GGUF format.+ """+ for fmt in formats or []:+ if not fmt.is_gguf():+ continue+ # The outer GGUFFormat reports output_format == "gguf"; the precise,+ # alias-correct, post-rewrite preset lives on backend.output_format+ # (e.g. "gguf:q4_k_m"). A standalone/inner GGUFFormat already carries+ # the precise string on output_format itself.+ backend = getattr(fmt, "backend", None)+ backend_fmt = getattr(backend, "output_format", None) if backend is not None else None+ precise = backend_fmt if backend_fmt and backend_fmt != "gguf" else fmt.output_format+ if precise and precise != "gguf":+ return precise+ return None++ def _resolve_format_string(self, format_str: str) -> list["OutputFormat"]:+ """Resolve one format string via resolve_formats(), then propagate any+ scheme/layer_config/scale_dtype/quant_block_list correction it makes+ (e.g. GGUF's gguf_args_check) back onto self.++ `scheme` may or may not be the object passed in — GGUF's "_mixed -> _s"+ rewrite can rebuild it wholesale — so this always re-pins self.scheme_context+ explicitly rather than relying on in-place mutation.+ """+ preset_name = self.scheme if isinstance(self.scheme, str) else None+ self._format_resolution = resolve_formats(+ ResolvedScheme.from_scheme(self.scheme_context, preset_name=preset_name),+ format=format_str,+ layer_config=self.layer_config or {},+ scale_dtype=self.scale_dtype,+ mllm=self.model_context.is_mllm,+ iters=getattr(self, "iters", 0),+ enable_alg_ext=getattr(self, "enable_alg_ext", False),+ quant_nontext_module=self.quant_nontext_module,+ quant_block_list=self.quant_block_list,+ platform=self.platform,+ is_auto_scheme=self.is_auto_scheme,+ model=self.model_context.model,+ )+ formats = list(self._format_resolution.formats)+ scheme = self._format_resolution.scheme.value+ self.layer_config = thaw_mapping(self._format_resolution.layer_config_patch)+ self.scale_dtype = self._format_resolution.scale_dtype+ self.quant_block_list = (+ [list(group) for group in self._format_resolution.quant_block_list]+ if self._format_resolution.quant_block_list is not None+ else None+ )+ self.scheme_context = scheme+ for config in self._alg_configs:+ if hasattr(config, "scheme"):+ config.scheme = self.scheme_context+ # self.scheme must stay an independent object from self.scheme_context+ # (the existing invariant from resolve_scheme(), Phase 1) — never assign+ # the same reference here even though it would often "work".+ #+ # Special-case GGUF: get_gguf_scheme() can only disambiguate presets that+ # share identical QuantizationScheme fields (e.g. GGUF:Q4_K_S vs GGUF:Q4_K_M,+ # or any GGUF:Q*_0 / GGUF:Q*_1 that its field-matching loop deliberately+ # skips) via its string short-circuit. If we hand it the resolved object+ # form, downstream resolve_layer_config() derives the WRONG (or empty) gguf+ # preset name, corrupting per-tensor qtype/embedding selection. So when the+ # resolved format is GGUF, pin self.scheme to the precise resolved preset+ # string (the outer GGUFFormat keeps it on .backend.output_format, already+ # past any "_mixed"->"_s" rewrite); _validate_format_combination guarantees at most+ # one GGUF format here.+ gguf_preset = self._format_resolution.scheme.preset_name+ if gguf_preset is not None and not gguf_preset.startswith("gguf:"):+ gguf_preset = None+ if gguf_preset is not None:+ self.scheme = gguf_preset+ else:+ self.scheme = copy.deepcopy(scheme)+ return formats+def _resolve_formats(self) -> None:- """Phase 2 – Format resolution and config attr sync.+ """Phase 2 - Format resolution and scheme/config sync.Preconditions:- - Phase 1 complete: the scheme is resolved (``data_type``, ``bits``,- ``sym`` etc. are set on both ``self`` and ``self.quantize_config``).+ - Phase 1 complete: ``self.scheme`` / ``self.scheme_context`` are resolved.Work performed:- Converts a string ``self.formats`` to a list of- :class:`~auto_round.formats.OutputFormat` objects via- :func:`~auto_round.formats.get_formats`.+ :class:`~auto_round.export.formats.OutputFormat` objects via+ :meth:`_resolve_format_string`, which also propagates any scheme+ correction (e.g. GGUF's ``gguf_args_check``) onto ``self.scheme``,+ ``self.scheme_context``, the algorithm configs that share it, and+ ``self.quantize_config``.- Initialises :class:`~auto_round.compressors.shard_writer.ShardWriter`when formats are present.- - **(2b)** Detects format-driven attribute mutations (``bits``, ``sym``,- ``data_type``, ``group_size``, etc.) that ``gguf_args_check`` may- have written onto ``self`` inside ``get_formats``, syncs them back- to ``self.quantize_config``, and rebuilds ``self.scheme`` accordingly.- - Merges any format-injected entries into ``self.layer_config``.Postconditions:- ``self.formats`` is a list (or ``None``).- ``self.compress_context.formats`` mirrors ``self.formats``.- - ``self.quantize_config`` and ``self.scheme`` reflect the final attrs.+ - ``self.scheme``, ``self.scheme_context`` and ``self.quantize_config``+ all reflect any format-driven corrections (e.g. GGUF)."""- # get_formats() inspects data_type / bits etc. that were just resolved.if isinstance(self.formats, str):- self.formats = get_formats(self.formats, self)+ self.formats = self._resolve_format_string(self.formats)if self.formats is not None:self.compress_context.formats = self.formatsShardWriter.reset()# Defer ShardWriter construction to _ensure_shard_writer() to avoid# heap fragmentation during post_init (parameter iteration).- # Snapshot the user-specified layer_config before format processing may- # inject extra per-layer entries (e.g. GGUF embedding / lm_head).- _pre_gguf_layer_config = copy.copy(self.layer_config) or {}-- # ── 2b: propagate format-adjusted attrs back to quantize_config ─────- # gguf_args_check (called inside get_formats) may have overridden- # bits / sym / data_type / super_bits / super_group_size / group_size- # on *this* BaseCompressor object via setattr(self, ...). Sync those- # changes to self.quantize_config before creating the quantizer so it is- # constructed with the definitive final values.- _gguf_forwarded_attrs = (- "bits",- "sym",- "data_type",- "super_bits",- "super_group_size",- "group_size",- "act_bits",- "scale_dtype",- )- # Skip this for AutoScheme — the format is for export only, and the- # per-layer quantization is already determined by orig_scheme.options.- # Restore scheme attrs from quantize_config for AutoScheme —- # gguf_args_check may have set __dict__ entries that shadow the proxy.- if self.is_auto_scheme:- for _attr in _gguf_forwarded_attrs:- if _attr in self.__dict__:- del self.__dict__[_attr]-- if not self.is_auto_scheme:- _any_gguf_attr_changed = False- for _attr in _gguf_forwarded_attrs:- if _attr not in self.__dict__:- continue- config_val = getattr(self.quantize_config, _attr, None)- self_val = self.__dict__[_attr]- if _attr not in ("scale_dtype", "act_bits") and config_val != self_val:- _any_gguf_attr_changed = True- if config_val != self_val:- setattr(self.quantize_config, _attr, self_val)- # If format resolution changed scheme attrs, rebuild self.scheme so that- # configure_layer_config() / set_layer_config() see the correct values.- if _any_gguf_attr_changed:- from auto_round.schemes import PRESET_SCHEMES- from auto_round.schemes import QuantizationScheme as _QS-- # Prefer to derive the scheme directly from the gguf format name to- # avoid ambiguity (e.g. Q4_K_S and Q4_K_M share identical weight attrs).- _gguf_preset_scheme = None- _gguf_fmt_name = None- _gguf_original_fmt_name = None- for _fmt in self.formats or []:- # GGUFFormat (outer) has output_format="gguf" but backend.output_format="gguf:q4_k_m"- # GGUFFormat (inner/standalone) has output_format="gguf:q4_k_m"- _of = getattr(_fmt, "output_format", "")- if "gguf" in str(_of):- if str(_of) == "gguf":- # outer GGUFFormat: full format in _original_format (e.g. "gguf:q2_k_mixed")- # or backend.output_format (e.g. "gguf:q2_k_s" after _mixed → _s conversion)- _orig = getattr(_fmt, "_original_format", None)- if _orig:- _gguf_original_fmt_name = str(_orig).upper()- _backend = getattr(_fmt, "backend", None)- _of = getattr(_backend, "output_format", _of) if _backend is not None else _of- _preset_key = str(_of).upper()- if _preset_key in PRESET_SCHEMES:- _gguf_preset_scheme = PRESET_SCHEMES[_preset_key]- _gguf_fmt_name = _preset_key- break- if _gguf_preset_scheme is not None:- # Update scheme on both compressor and quantizer.- self.scheme = _gguf_preset_scheme- # Store the exact gguf format name so configure_layer_config /- # set_layer_config can use it directly, avoiding Q4_K_S / Q4_K_M ambiguity.- self._gguf_format_name = _gguf_fmt_name- # Store original format name (may include _mixed) for _handle_special_schemes- if _gguf_original_fmt_name:- self._gguf_original_format_name = _gguf_original_fmt_name- else:- _new_scheme_dict = {f.name: getattr(self, f.name, None) for f in fields(_QS)}- _new_scheme = _QS.from_dict({k: v for k, v in _new_scheme_dict.items() if v is not None})- self.scheme = _new_scheme-- _gguf_layer_cfg = {- k: v for k, v in (self.__dict__.get("layer_config") or {}).items() if k not in (_pre_gguf_layer_config)- }- if _gguf_layer_cfg:- if self.layer_config is None:- self.layer_config = {}- for _lname, _lval in _gguf_layer_cfg.items():- self.layer_config.setdefault(_lname, _lval)-def _apply_rotations(self) -> None:"""Phase 4.5 – Apply Hadamard / rotation transforms to the model.@@ -1189,7 +1377,7 @@ class BaseOrchestrator(object):self.model_context.apply_patches(self.formats)def _build_layer_config(self) -> None:- """Phase 4 – Layer-config construction and quantizer sync.+ """Phase 4 – Layer-config construction.Preconditions:- Phase 3 complete: model topology is final.@@ -1198,17 +1386,12 @@ class BaseOrchestrator(object):Work performed:- Calls :meth:`_scheme_post_init` which walks the patched model to- build ``self.layer_config``, ``self.quant_block_list``, etc.- On the AutoScheme path this also runs delta-loss forward/backward- passes to select per-layer schemes.- - Syncs the fully-resolved ``layer_config`` and related attrs to- ``self.quantizer`` so quantization methods have the complete view.+ build ``self.layer_config``, ``self.quant_block_list``,+ ``self.compression_plan``, etc. On the AutoScheme path this also+ runs delta-loss forward/backward passes to select per-layer schemes.Postconditions:- - ``self.layer_config`` is fully populated.- - ``self.quantizer`` mirrors ``layer_config``, ``has_qlayer_outside_block``,- ``regex_config``, ``quant_block_list``, ``to_quant_block_names``,- ``scale_dtype``, and ``ignore_layers``.+ - ``self.layer_config`` and ``self.compression_plan`` are fully populated."""# configure_layer_config() walks the patched model; _gen_auto_scheme()# (AutoScheme path) runs delta-loss forward+backward passes.@@ -1273,14 +1456,14 @@ class BaseOrchestrator(object):if name.startswith("_"):raise AttributeError(f"'{type(self).__name__}' object has no attribute '{name}'")- # Delegate to block_quantizer: access _pipeline directly from __dict__ to- # avoid recursion (quantizer is now a @property backed by _pipeline; going- # through the property inside __getattr__ would re-trigger __getattr__- # if _pipeline itself isn't ready yet).- _pipeline = self.__dict__.get("_pipeline")- if _pipeline is not None:+ # Delegate to block_quantizer: access _alg_composer directly from __dict__+ # to avoid recursion (quantizer attribute access forwards through+ # _alg_composer.block_quantizer; going through a property inside+ # __getattr__ would re-trigger __getattr__ if _alg_composer isn't ready yet).+ _alg_composer = self.__dict__.get("_alg_composer")+ if _alg_composer is not None:try:- return object.__getattribute__(_pipeline.block_quantizer, name)+ return object.__getattribute__(_alg_composer.block_quantizer, name)except AttributeError:pass@@ -1449,7 +1632,7 @@ class BaseOrchestrator(object):self.compress_context.output_dir = output_dirif format is not None:if isinstance(format, str) and getattr(self, "formats", None) is None:- self.formats = get_formats(format, self)+ self.formats = self._resolve_format_string(format)self.compress_context.formats = self.formatsif not self.model_context.quantized:@@ -1460,7 +1643,7 @@ class BaseOrchestrator(object):logger.info("format is not set, using default auto_round format.")self.formats = "auto_round"if isinstance(self.formats, str):- self.formats = get_formats(self.formats, self)+ self.formats = self._resolve_format_string(self.formats)self.compress_context.formats = self.formatsfor format in self.formats:save_folder = _get_save_folder_name(format)@@ -1633,7 +1816,7 @@ class BaseOrchestrator(object):# self.formats to a default string above, resolve it into OutputFormat objects so that# quantize() and save_quantized() receive proper objects, not a raw string.if isinstance(self.formats, str):- self.formats = get_formats(self.formats, self)+ self.formats = self._resolve_format_string(self.formats)self.compress_context.formats = self.formats# Derive descriptive export dir after post_init so scheme-resolved attrs are available._fmt_str = format or (self.formats if isinstance(self.formats, str) else "")diff --git a/auto_round/compressors/config_resolution/__init__.py b/auto_round/compressors/config_resolution/__init__.pynew file mode 100644index 000000000..dcdbb3b74--- /dev/null+++ b/auto_round/compressors/config_resolution/__init__.py@@ -0,0 +1,40 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++from auto_round.compressors.config_resolution.contracts import (+ ResolvedQuantizationConfig,+ FormatResolution,+ ResolvedScheme,+ thaw_mapping,+)+from auto_round.compressors.config_resolution.errors import (+ ConfigResolutionError,+ FormatCompatibilityError,+ LayerConfigResolutionError,+ SchemeResolutionError,+)+from auto_round.compressors.config_resolution.resolve import resolve_quantization_config, resolve_scheme_value++__all__ = [+ "ResolvedQuantizationConfig",+ "FormatCompatibilityError",+ "FormatResolution",+ "LayerConfigResolutionError",+ "ConfigResolutionError",+ "ResolvedScheme",+ "SchemeResolutionError",+ "resolve_quantization_config",+ "resolve_scheme_value",+ "thaw_mapping",+]diff --git a/auto_round/compressors/config_resolution/contracts.py b/auto_round/compressors/config_resolution/contracts.pynew file mode 100644index 000000000..f095dc936--- /dev/null+++ b/auto_round/compressors/config_resolution/contracts.py@@ -0,0 +1,127 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++from __future__ import annotations++import copy+from dataclasses import dataclass, field+from types import MappingProxyType+from typing import Any, Mapping, Optional, Tuple++from auto_round.schemes import QuantizationScheme++LayerConfig = Mapping[str, Mapping[str, Any]]+BlockGroups = Tuple[Tuple[str, ...], ...]+++def _deepcopy_mapping_proxy(value: MappingProxyType, memo: dict) -> MappingProxyType:+ """``copy.deepcopy`` has no built-in support for ``types.MappingProxyType`` (it isn't+ picklable), so any object graph that happens to contain one -- e.g. a+ ``ResolvedQuantizationConfig``/``FormatResolution`` nested inside something a caller deep-copies+ wholesale -- would otherwise raise ``TypeError: cannot pickle 'mappingproxy' object``.+ Registering a dispatch entry here teaches ``copy.deepcopy`` to handle it everywhere,+ instead of requiring every call site that might transitively touch one of our frozen+ mappings to know about this quirk.+ """+ return MappingProxyType(copy.deepcopy(dict(value), memo))+++copy._deepcopy_dispatch[MappingProxyType] = _deepcopy_mapping_proxy+++def freeze_mapping(value: Optional[LayerConfig]) -> LayerConfig:+ """Return an isolated, read-only snapshot of a layer configuration mapping.++ Per-layer configuration values are usually dicts (e.g. ``{"bits": 4}``), but some+ callers store non-mapping values (e.g. a GGUF preset name string). Only dict-like+ values are deep-copied and wrapped in a read-only proxy; other values are copied+ as-is since they are already immutable/opaque to us.+ """+ frozen = {}+ for name, config in dict(value or {}).items():+ if isinstance(config, Mapping):+ frozen[name] = MappingProxyType(copy.deepcopy(dict(config)))+ else:+ frozen[name] = copy.deepcopy(config)+ return MappingProxyType(frozen)+++def thaw_mapping(value: Optional[LayerConfig]) -> dict:+ """Return a fully mutable, deep-copyable plain-dict snapshot of a frozen mapping.++ This is the inverse of :func:`freeze_mapping` and should be used instead of+ ``copy.deepcopy`` whenever a caller needs to mutate a previously frozen layer+ configuration mapping, since ``MappingProxyType`` instances cannot be deep-copied+ directly.+ """+ result = {}+ for name, config in dict(value or {}).items():+ result[name] = copy.deepcopy(dict(config)) if isinstance(config, Mapping) else copy.deepcopy(config)+ return result+++def freeze_block_groups(value: Optional[Tuple[Tuple[str, ...], ...]]) -> Optional[BlockGroups]:+ """Freeze the two-dimensional block grouping used by model traversal."""+ if value is None:+ return None+ return tuple(tuple(group) for group in value)+++@dataclass(frozen=True)+class ResolvedScheme:+ _value: QuantizationScheme+ preset_name: Optional[str] = None++ def __post_init__(self) -> None:+ object.__setattr__(self, "_value", copy.deepcopy(self._value))++ @property+ def value(self) -> QuantizationScheme:+ """Return an isolated scheme value so callers cannot mutate this DTO."""+ return copy.deepcopy(self._value)++ @classmethod+ def from_scheme(cls, value: QuantizationScheme, preset_name: Optional[str] = None) -> "ResolvedScheme":+ return cls(_value=value, preset_name=preset_name)+++@dataclass(frozen=True)+class FormatResolution:+ formats: Tuple[Any, ...]+ scheme: ResolvedScheme+ layer_config_patch: LayerConfig = field(default_factory=lambda: MappingProxyType({}))+ scale_dtype: Any = None+ quant_block_list: Optional[BlockGroups] = None++ def __post_init__(self) -> None:+ object.__setattr__(self, "formats", tuple(self.formats))+ object.__setattr__(self, "layer_config_patch", freeze_mapping(self.layer_config_patch))+ object.__setattr__(self, "quant_block_list", freeze_block_groups(self.quant_block_list))+++@dataclass(frozen=True)+class ResolvedQuantizationConfig:+ scheme: ResolvedScheme+ formats: Tuple[Any, ...]+ layer_config: LayerConfig+ regex_config: LayerConfig = field(default_factory=lambda: MappingProxyType({}))+ has_qlayer_outside_block: bool = False+ scale_dtype: Any = None+ quant_block_list: Optional[BlockGroups] = None++ def __post_init__(self) -> None:+ object.__setattr__(self, "formats", tuple(self.formats))+ object.__setattr__(self, "layer_config", freeze_mapping(self.layer_config))+ object.__setattr__(self, "regex_config", freeze_mapping(self.regex_config))+ object.__setattr__(self, "quant_block_list", freeze_block_groups(self.quant_block_list))diff --git a/auto_round/compressors/config_resolution/errors.py b/auto_round/compressors/config_resolution/errors.pynew file mode 100644index 000000000..438e2e582--- /dev/null+++ b/auto_round/compressors/config_resolution/errors.py@@ -0,0 +1,33 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+++class ConfigResolutionError(ValueError):+ """Base class for user-facing compression configuration resolution errors.++ Callers that want to surface a clean, no-traceback message (e.g. the CLI) should+ catch this common base instead of enumerating every specific subclass.+ """+++class SchemeResolutionError(ConfigResolutionError):+ """Raised when a quantization scheme cannot be resolved."""+++class FormatCompatibilityError(ConfigResolutionError):+ """Raised when requested export formats are mutually incompatible."""+++class LayerConfigResolutionError(ConfigResolutionError):+ """Raised when per-layer quantization configuration cannot be resolved."""diff --git a/auto_round/compressors/config_resolution/resolve.py b/auto_round/compressors/config_resolution/resolve.pynew file mode 100644index 000000000..4d501ae0c--- /dev/null+++ b/auto_round/compressors/config_resolution/resolve.py@@ -0,0 +1,69 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++from __future__ import annotations++import copy+from typing import Any, Mapping++from auto_round.compressors.config_resolution.contracts import (+ FormatResolution,+ LayerConfig,+ ResolvedQuantizationConfig,+ ResolvedScheme,+ thaw_mapping,+)+from auto_round.compressors.config_resolution.errors import SchemeResolutionError+from auto_round.schemes import QuantizationScheme, parse_scheme+++def resolve_scheme_value(scheme: Any, overrides: Mapping[str, Any]) -> ResolvedScheme:+ """Resolve a model-independent scheme without modifying caller-owned inputs."""+ from auto_round.auto_scheme.gen_auto_scheme import AutoScheme++ if isinstance(scheme, AutoScheme):+ raise SchemeResolutionError("AutoScheme requires model-aware resolution")++ try:+ _, _, final_attrs = parse_scheme(copy.deepcopy(scheme), copy.deepcopy(dict(overrides)))+ except (KeyError, NotImplementedError, TypeError, ValueError) as error:+ raise SchemeResolutionError(str(error)) from error++ preset_name = None+ if isinstance(scheme, str):+ normalized = scheme.strip("'\" ")+ preset_name = normalized.lower() if normalized.lower().startswith("gguf:") else normalized.upper()+ return ResolvedScheme.from_scheme(QuantizationScheme.from_dict(final_attrs), preset_name=preset_name)+++def resolve_quantization_config(+ format_resolution: FormatResolution,+ layer_config: LayerConfig,+ *,+ regex_config: LayerConfig | None = None,+ has_qlayer_outside_block: bool = False,+) -> ResolvedQuantizationConfig:+ """Combine immutable resolution results into the authoritative quantization config."""+ merged = thaw_mapping(format_resolution.layer_config_patch)+ for name, config in layer_config.items():+ merged.setdefault(name, {}).update(dict(config))+ return ResolvedQuantizationConfig(+ scheme=format_resolution.scheme,+ formats=format_resolution.formats,+ layer_config=merged,+ regex_config=regex_config or {},+ has_qlayer_outside_block=has_qlayer_outside_block,+ scale_dtype=format_resolution.scale_dtype,+ quant_block_list=format_resolution.quant_block_list,+ )diff --git a/auto_round/compressors/diffusion_mixin.py b/auto_round/compressors/diffusion_mixin.pyindex 1b9e53c66..7b22117e7 100644--- a/auto_round/compressors/diffusion_mixin.py+++ b/auto_round/compressors/diffusion_mixin.py@@ -519,9 +519,7 @@ class DiffusionMixin:# Handle multi-format (convert string to list if needed)_format = formatif isinstance(_format, str):- from auto_round.formats import get_formats-- _format = get_formats(_format, self)+ _format = self._resolve_format_string(_format)for name in pipe.components.keys():val = getattr(pipe, name)diff --git a/auto_round/compressors/entry.py b/auto_round/compressors/entry.pydeleted file mode 100644index 5c838be06..000000000--- a/auto_round/compressors/entry.py+++ /dev/null@@ -1,789 +0,0 @@-# # Copyright (C) 2026 Intel Corporation-# # SPDX-License-Identifier: Apache-2.0--import os-from typing import Any, Callable, Optional, Union--import torch--from auto_round.algorithms.config_resolver import split_quantization_configs-from auto_round.algorithms.quantization.config import QuantizationConfig-from auto_round.algorithms.quantization.rtn.config import OptimizedRTNConfig, RTNConfig-from auto_round.algorithms.quantization.sign_round.config import SignRoundConfig-from auto_round.algorithms.registry import normalize_algorithm_config, resolve_alg_config-from auto_round.algorithms.transforms import normalize_rotation_config as _normalize_rotation_alg_config-from auto_round.algorithms.transforms.awq.config import AWQConfig-from auto_round.algorithms.transforms.hadamard.config import RotationConfig as _NewArchRotationConfig-from auto_round.auto_scheme.gen_auto_scheme import AutoScheme-from auto_round.compressors.base import BaseOrchestrator as BaseCompressor-from auto_round.compressors.orchestrator import CompressionOrchestrator as Compressor-from auto_round.compressors.utils import check_need_act_calibration-from auto_round.logger import logger-from auto_round.schemes import QuantizationScheme, parse_scheme-from auto_round.utils.device_manager import normalize_default_device_map--_ENTRY_ROUTE_KWARGS = {"model_free", "disable_model_free", "disable_opt_rtn"}-_ENTRY_COMPRESSOR_KWARGS = {"scale_dtype", "ignore_layers", "quant_lm_head", "to_quant_block_names"}-_ENTRY_BASE_KWARGS = {- "format",- "dataset",- "batch_size",- "model_dtype",- "trust_remote_code",- "amp",- "nblocks",- "disable_deterministic_algorithms",- "enable_deterministic_algorithms",- "static_kv_dtype",- "static_attention_dtype",-}-_ENTRY_MLLM_KWARGS = {"processor", "image_processor", "template", "extra_data_dir", "quant_nontext_module"}-_ENTRY_DIFFUSION_KWARGS = {"guidance_scale", "num_inference_steps", "generator_seed"}-_ENTRY_ALLOWED_KWARGS = (- _ENTRY_ROUTE_KWARGS | _ENTRY_COMPRESSOR_KWARGS | _ENTRY_BASE_KWARGS | _ENTRY_MLLM_KWARGS | _ENTRY_DIFFUSION_KWARGS-)---def filter_supported_entry_kwargs(kwargs: dict[str, Any], *, context: str) -> dict[str, Any]:- """Return only kwargs supported by the new entry API.-- Unsupported kwargs are ignored with a warning so callers can cleanly migrate- without leaking old-API parameters into compressor constructors.- """-- supported = {}- unknown = []- for key, value in kwargs.items():- if key in _ENTRY_ALLOWED_KWARGS:- supported[key] = value- else:- unknown.append(key)- if unknown:- logger.warning_once(- "%s received unsupported kwargs %s. They will be ignored.",- context,- ", ".join(sorted(unknown)),- )- return supported---def _split_entry_kwargs(kwargs: dict[str, Any]) -> dict[str, dict[str, Any]]:- """Partition new-entry kwargs by ownership."""-- kwargs = filter_supported_entry_kwargs(kwargs, context="AutoRound entry")- buckets = {- "route": {},- "compressor": {},- "base": {},- "mllm": {},- "diffusion": {},- }- for key, value in kwargs.items():- if key in _ENTRY_ROUTE_KWARGS:- buckets["route"][key] = value- elif key in _ENTRY_COMPRESSOR_KWARGS:- buckets["compressor"][key] = value- elif key in _ENTRY_BASE_KWARGS:- buckets["base"][key] = value- elif key in _ENTRY_MLLM_KWARGS:- buckets["mllm"][key] = value- elif key in _ENTRY_DIFFUSION_KWARGS:- buckets["diffusion"][key] = value- return buckets---def _preview_resolved_attrs(config, scheme=None) -> dict:- """Resolve scheme attributes without mutating config, for routing decisions.-- Called in ``AutoRound.__new__`` before the concrete compressor class is- chosen. ``SchemeMixin.resolve_scheme()`` will do the authoritative- resolution later; this is just a lightweight preview so routing logic- (``enable_imatrix``, ``needs_act_calib``, etc.) can use the correct values- even when the user specified only ``scheme=`` without explicit bit/dtype args.-- Returns:- dict: resolved attributes (may be empty if scheme cannot be previewed).- """- if isinstance(scheme, AutoScheme):- # AutoScheme needs model info — cannot preview, rely on raw config attrs- return {}- scheme_attr_names = tuple(config._scheme_fields)- user_overrides = {k: getattr(config, k) for k in scheme_attr_names if getattr(config, k, None) is not None}- try:- _, _, final_attrs = parse_scheme(scheme, user_overrides)- return final_attrs- except Exception:- return {}---def _eager_validate_scheme(config, scheme=None) -> None:- """Eagerly validate scheme/config constraints at construction time.-- Mirrors the old-arch ``_check_configs()`` call in ``BaseCompressor.__init__``.- Raises ``ValueError`` or ``NotImplementedError`` immediately if the scheme- contains config-only invalid combinations (e.g. tuple group_size with non-fp8- weight dtype) so that callers get a fast failure rather than a deferred error- buried inside ``post_init()``.-- ``AutoScheme`` is skipped because it requires model information.- """- if isinstance(scheme, AutoScheme):- return-- scheme_attr_names = tuple(config._scheme_fields)- user_overrides = {k: getattr(config, k) for k in scheme_attr_names if getattr(config, k, None) is not None}- try:- _, _, final_attrs = parse_scheme(scheme, user_overrides)- except (ValueError, NotImplementedError):- raise- except Exception:- return # Other parse errors are deferred to post_init-- import copy-- temp_config = copy.copy(config)- if hasattr(config, "scheme"):- temp_config.scheme = config.scheme.copy()- temp_config._user_set_scheme_fields = set(getattr(config, "_user_set_scheme_fields", set()))- for key, value in final_attrs.items():- setattr(temp_config, key, value)- temp_config.check_config() # raises ValueError / NotImplementedError if invalid---# ----------------------------------------------------------------------------# Compressor-class registry-# ----------------------------------------------------------------------------# Maps (model_type, base_class_name) → combined class, created lazily.-_COMPRESSOR_REGISTRY: dict[tuple[str, str], type] = {}---def _get_compressor_class(model_type: str, base_cls: type) -> type:- """Return the compressor class for *base_cls* wired with the right model-type Mixin.-- For ``model_type == "llm"`` the bare *base_cls* is returned unchanged.- For ``"mllm"`` and ``"diffusion"`` the corresponding Mixin is prepended via- :func:`type` and the result is cached in ``_COMPRESSOR_REGISTRY`` so that- each ``(model_type, base_cls)`` pair is created at most once per process.- """- if model_type == "llm":- return base_cls- key = (model_type, base_cls.__name__)- if key in _COMPRESSOR_REGISTRY:- return _COMPRESSOR_REGISTRY[key]- if model_type == "mllm":- from auto_round.compressors.mllm_mixin import MLLMMixin-- mixin = MLLMMixin- elif model_type == "diffusion":- from auto_round.compressors.diffusion_mixin import DiffusionMixin-- mixin = DiffusionMixin- else:- return base_cls- combined = type(f"{model_type.capitalize()}{base_cls.__name__}", (mixin, base_cls), {})- _COMPRESSOR_REGISTRY[key] = combined- return combined---def is_weight_scheme(scheme: Union[str, dict, AutoScheme, object]) -> bool:- if isinstance(scheme, str):- return scheme.upper().startswith("W")- if isinstance(scheme, dict):- return all(isinstance(s, str) and s.upper().startswith("W") for s in scheme.values())- if isinstance(scheme, AutoScheme):- opts = scheme.options- if isinstance(opts, (list, tuple)):- return all(isinstance(s, str) and s.upper().startswith("W") for s in opts)- if isinstance(opts, str):- return opts.upper().startswith("W")- return False---def is_gguf_k_target(value: Union[str, AutoScheme, object]) -> bool:- if isinstance(value, str):- normalized = value.strip().lower()- return normalized.startswith("gguf:") and "_k" in normalized- if isinstance(value, AutoScheme):- opts = value.options- if isinstance(opts, str):- opts = [opts]- if isinstance(opts, (list, tuple)):- return any(isinstance(opt, str) and is_gguf_k_target(opt) for opt in opts)- return False---def _resolve_quant_config_for_routing(alg_configs) -> tuple[list, list, QuantizationConfig]:- preprocessor_configs, block_quant_configs = split_quantization_configs(alg_configs)- if len(block_quant_configs) == 0 and preprocessor_configs:- from auto_round.algorithms.quantization.rtn.config import RTNConfig as _RTNConfig-- return preprocessor_configs, block_quant_configs, _RTNConfig()- if len(block_quant_configs) > 1:- raise ValueError(- f"Only one block-quantization config is allowed, but got {len(block_quant_configs)}: "- f"{[type(c).__name__ for c in block_quant_configs]}"- )- if len(block_quant_configs) == 1:- return preprocessor_configs, block_quant_configs, block_quant_configs[0]- raise ValueError(- "At least one quantization algorithm config is required. "- "Pass a block quantizer such as RTNConfig or SignRoundConfig, "- "or a quantization preprocessor such as AWQConfig."- )---def _build_model_type_ctor_kwargs(model, base_kwargs, mllm_kwargs, diffusion_kwargs) -> tuple[str, dict[str, Any]]:- from auto_round.utils.model import detect_model_type-- model_type = detect_model_type(model)- has_multimodal_assets = mllm_kwargs.get("processor") is not None or mllm_kwargs.get("image_processor") is not None- if has_multimodal_assets and model_type != "mllm":- model_type = "mllm"-- ctor_kwargs = dict(base_kwargs)- if model_type == "mllm":- ctor_kwargs.update(mllm_kwargs)- if model_type == "diffusion":- ctor_kwargs.update(diffusion_kwargs)- return model_type, ctor_kwargs---def _select_rtn_compressor_base_cls(quant_config: RTNConfig, scheme, format, base_kwargs) -> type:- enable_imatrix = False- disable_opt_rtn = getattr(quant_config, "disable_opt_rtn", False)-- # Preview resolved scheme attrs once (authoritative resolution happens later).- resolved_attrs = _preview_resolved_attrs(quant_config, scheme)-- # Auto-disable rtn optimization for W8A16/W8A8-equivalent resolved schemes,- # unless the user already set disable_opt_rtn explicitly.- if getattr(quant_config, "orig_disable_opt_rtn", None) is None:- bits = resolved_attrs.get("bits", getattr(quant_config, "bits", None))- act_bits = resolved_attrs.get("act_bits", getattr(quant_config, "act_bits", None))- data_type = resolved_attrs.get("data_type", getattr(quant_config, "data_type", None))- if bits is not None and bits >= 8 and act_bits is not None and act_bits >= 8 and data_type == "int":- logger.warning("`disable_opt_rtn` is turned on for W8A16/W8A8 quantization to improve efficiency.")- disable_opt_rtn = True- quant_config.disable_opt_rtn = True-- if not disable_opt_rtn:- has_gguf_k = is_gguf_k_target(format) or is_gguf_k_target(scheme)- if has_gguf_k:- enable_imatrix = True- else:- sym = resolved_attrs.get("sym", getattr(quant_config, "sym", None))- data_type = resolved_attrs.get("data_type", getattr(quant_config, "data_type", "") or "")- bits = resolved_attrs.get("bits", getattr(quant_config, "bits", None))- if sym is not None and sym is False:- enable_imatrix = False- elif data_type == "int" and (bits is None or bits < 8):- enable_imatrix = True- elif is_weight_scheme(scheme):- enable_imatrix = True-- act_bits = resolved_attrs.get("act_bits", getattr(quant_config, "act_bits", None))- act_data_type = resolved_attrs.get("act_data_type", getattr(quant_config, "act_data_type", None))- act_dynamic = resolved_attrs.get("act_dynamic", getattr(quant_config, "act_dynamic", None))- is_act_quantize = act_bits is not None and act_bits <= 8- needs_act_calib = is_act_quantize and check_need_act_calibration(- act_dynamic,- act_data_type,- act_bits if act_bits is not None else 16,- static_kv_dtype=base_kwargs.get("static_kv_dtype"),- static_attention_dtype=base_kwargs.get("static_attention_dtype"),- )-- # AutoScheme always requires calibration data for delta-loss based scheme- # selection, regardless of whether imatrix is needed.- quant_config.enable_imatrix = enable_imatrix- if enable_imatrix or needs_act_calib or isinstance(scheme, AutoScheme):- if not isinstance(quant_config, OptimizedRTNConfig):- quant_config.__class__ = OptimizedRTNConfig-- if isinstance(quant_config, OptimizedRTNConfig):- pass # keep OptimizedRTNConfig as-is- elif not (enable_imatrix or needs_act_calib or isinstance(scheme, AutoScheme)):- # Pure zero-shot RTN: downgrade to basic RTNConfig- if isinstance(quant_config, OptimizedRTNConfig):- quant_config.__class__ = RTNConfig-- # Always use Compressor — it internally detects whether calibration- # data is needed and falls back to the zero-shot (RTN) path when it is not.- return Compressor---class AutoRound(object):-- @classmethod- def _resolve_config(cls, config: Union[str, object, list]) -> Union[object, list[object]]:- """Convert string alias(es) to the corresponding config instance(s) with default parameters."""- if isinstance(config, str):- return resolve_alg_config(config)- if isinstance(config, list):- return [cls._resolve_config(c) for c in config]- return config-- def __new__(- cls,- model: Union[torch.nn.Module, str],- scheme="W4A16",- alg_configs: Union[str, object, list[Union[str, object]]] = None,- tokenizer=None,- platform="hf",- format=None,- dataset=None,- low_gpu_mem_usage: bool = False,- device_map: Union[str, torch.device, int, dict] = 0,- iters: int = None,- enable_torch_compile: Optional[bool] = None,- seed: int = 42,- low_cpu_mem_usage: bool = True,- layer_config=None,- nsamples: int = None,- seqlen: int = None,- **kwargs,- ) -> "BaseCompressor":- from auto_round.utils.model import is_model_free_route-- if alg_configs is None:- alg_configs = "auto_round"- # TODO wenhuach if key in kwargs could override scheme and alg_config, we should pop and override,- # e.g. gradient_accumulate_step- device_map = normalize_default_device_map(device_map)- split_kwargs = _split_entry_kwargs(kwargs)- route_kwargs = dict(split_kwargs["route"])- compressor_kwargs = dict(split_kwargs["compressor"])- base_kwargs = dict(split_kwargs["base"])- mllm_kwargs = dict(split_kwargs["mllm"])- diffusion_kwargs = dict(split_kwargs["diffusion"])-- # Resolve string alias(es) to config instance(s) before routing.- alg_configs = cls._resolve_config(alg_configs)- if isinstance(alg_configs, list):- alg_configs = [normalize_algorithm_config(cfg) for cfg in alg_configs]- else:- alg_configs = normalize_algorithm_config(alg_configs)- configs_for_routing = alg_configs if isinstance(alg_configs, list) else [alg_configs]- preprocessor_configs, _, quant_config = _resolve_quant_config_for_routing(configs_for_routing)-- # Model-free routing is now supported directly by the new entry path.- model_free_iters = 0 if isinstance(quant_config, RTNConfig) else getattr(quant_config, "iters", None)- model_free_disable_opt_rtn = getattr(quant_config, "disable_opt_rtn", None)- route_decision_kwargs = dict(- route_kwargs,- format=format,- static_kv_dtype=base_kwargs.get("static_kv_dtype"),- static_attention_dtype=base_kwargs.get("static_attention_dtype"),- )- if is_model_free_route(model, scheme, model_free_iters, model_free_disable_opt_rtn, route_decision_kwargs):- from auto_round.compressors.model_free import ModelFreeCompressor-- if not isinstance(model, str):- raise ValueError("model_free=True requires `model` to be a HuggingFace ID or local path string.")- if not bool(route_kwargs.get("model_free", False)):- logger.info(- "Auto-routing to model-free quantization "- "(iters=0, disable_opt_rtn=True, supported scheme). "- "Pass disable_model_free=True to use the regular flow."- )- return ModelFreeCompressor(- model_name_or_path=model,- scheme=scheme,- layer_config=layer_config,- tokenizer=tokenizer,- device_map=device_map,- enable_torch_compile=enable_torch_compile,- **compressor_kwargs,- **base_kwargs,- **mllm_kwargs,- **diffusion_kwargs,- **route_kwargs,- )-- # Eagerly validate scheme constraints that do not require model info.- # This mirrors old-arch _check_configs() called at __init__ time so that- # callers get ValueError/NotImplementedError on construction, not deferred.- _eager_validate_scheme(quant_config, scheme)-- local_args = dict(- model=model,- tokenizer=tokenizer,- platform=platform,- format=format,- scheme=scheme,- dataset=dataset,- low_gpu_mem_usage=low_gpu_mem_usage,- device_map=device_map,- iters=iters,- enable_torch_compile=enable_torch_compile,- seed=seed,- low_cpu_mem_usage=low_cpu_mem_usage,- layer_config=layer_config,- nsamples=nsamples,- seqlen=seqlen,- **compressor_kwargs,- )- model_type, ctor_kwargs = _build_model_type_ctor_kwargs(model, base_kwargs, mllm_kwargs, diffusion_kwargs)-- # Preprocessor algorithms (AWQ, …) require a data-driven host so that- # the per-block preprocessor lifecycle (prepare_block_group ->- # block_forward_hooks -> pre_quantize_block -> pre_quantize_block ->- # post_quantize_block) actually runs.- # Preprocessor algorithms require Compressor for per-block lifecycle hooks.- # The pipeline auto-appends RTN when no block_quantizer is supplied.- if preprocessor_configs:- return _get_compressor_class(model_type, Compressor)(alg_configs, **local_args, **ctor_kwargs)-- if isinstance(quant_config, SignRoundConfig):- return _get_compressor_class(model_type, Compressor)(alg_configs, **local_args, **ctor_kwargs)-- elif isinstance(quant_config, RTNConfig):- base_cls = _select_rtn_compressor_base_cls(quant_config, scheme, format, base_kwargs)- return _get_compressor_class(model_type, base_cls)(alg_configs, **local_args, **ctor_kwargs)---class AutoRoundCompatible:- """AutoRoundCompatible wrapper class for backward compatibility.-- This class provides the same API as the old AutoRoundCompatible class but internally- uses the new AutoRound architecture with Mixin pattern.-- Args:- model: Model object or model name to load- tokenizer: Tokenizer for text processing- platform: Platform to download model ("hf" or "model_scope")- scheme: Quantization scheme (str, dict, or QuantizationScheme)- layer_config: Layer-wise quantization config- dataset: Calibration data- iters: Optimization iterations- seqlen: Calibration sequence length- nsamples: Number of calibration samples- batch_size: Calibration batch size- gradient_accumulate_steps: Gradient accumulation steps- low_gpu_mem_usage: Lower GPU memory mode- device_map: Device map for each module- enable_torch_compile: Enable torch.compile- seed: Random seed- low_cpu_mem_usage: Lower CPU memory mode- **kwargs: Additional arguments (bits, group_size, sym, etc.)-- Example:- >>> # Old API - still works- >>> from auto_round.compressors.entry import AutoRoundCompatible- >>> autoround = AutoRoundCompatible(- ... model="/models/opt-125m",- ... bits=4,- ... group_size=128,- ... iters=200,- ... )- >>> quantized_model, layer_config = autoround.quantize()- """-- SKIP_ARGS = ("local_args", "kwargs", "cls", "config")-- bits: int | None- group_size: int | None- sym: bool | None- data_type: str | None- act_bits: int | None- act_group_size: int | None- act_sym: bool | None- act_data_type: str | None- act_dynamic: bool | None- super_bits: int | None- super_group_size: int | None-- @staticmethod- def _pop_config_kwargs(kwargs: dict[str, Any]) -> tuple[dict[str, Any], dict[str, Any]]:- """Extract old-API config kwargs and split them by config type."""- common_keys = ("super_bits", "super_group_size")- auto_round_only_keys = (- "nblocks",- "enable_alg_ext",- "lr_scheduler",- "not_use_best_mse",- "dynamic_max_gap",- "optimizer",- "enable_adam",- "momentum",- )- common_kwargs = {}- auto_round_kwargs = {}- for key in common_keys:- if key in kwargs:- common_kwargs[key] = kwargs.pop(key)- for key in auto_round_only_keys:- if key in kwargs:- auto_round_kwargs[key] = kwargs.pop(key)- return common_kwargs, auto_round_kwargs-- @staticmethod- def _pop_compressor_only_kwargs(kwargs: dict[str, Any]) -> dict[str, Any]:- return {- "scale_dtype": kwargs.pop("scale_dtype", None),- "ignore_layers": kwargs.pop("ignore_layers", ""),- "quant_lm_head": kwargs.pop("quant_lm_head", False),- "to_quant_block_names": kwargs.pop("to_quant_block_names", None),- }-- @staticmethod- def _resolve_compat_algorithm(algorithm, iters) -> str:- if algorithm and algorithm.lower() == "awq":- return "awq"- if (algorithm and algorithm.lower() == "rtn") or iters == 0:- return "rtn"- return "signround"-- @staticmethod- def _pop_shared_quant_kwargs(kwargs: dict[str, Any]) -> dict[str, Any]:- return {- "bits": kwargs.pop("bits", None),- "group_size": kwargs.pop("group_size", None),- "sym": kwargs.pop("sym", None),- "data_type": kwargs.pop("data_type", None),- "act_bits": kwargs.pop("act_bits", None),- "act_group_size": kwargs.pop("act_group_size", None),- "act_sym": kwargs.pop("act_sym", None),- "act_data_type": kwargs.pop("act_data_type", None),- "act_dynamic": kwargs.pop("act_dynamic", None),- }-- @staticmethod- def _build_awq_config(- shared_quant_kwargs: dict[str, Any],- *,- seqlen,- nsamples,- batch_size,- kwargs,- common_config_kwargs,- ):- return AWQConfig(- **shared_quant_kwargs,- duo_scaling=kwargs.pop("duo_scaling", True),- n_grid=kwargs.pop("n_grid", 20),- seqlen=seqlen,- nsamples=nsamples,- batch_size=batch_size,- mappings=kwargs.pop("mappings", None),- **common_config_kwargs,- )-- @staticmethod- def _build_rtn_config(shared_quant_kwargs: dict[str, Any], *, kwargs, common_config_kwargs):- cfg = RTNConfig(- **shared_quant_kwargs,- disable_opt_rtn=kwargs.pop("disable_opt_rtn", None),- enable_opt_rtn=kwargs.pop("enable_opt_rtn", None),- **common_config_kwargs,- )- return normalize_algorithm_config(cfg)-- @staticmethod- def _build_signround_config(- shared_quant_kwargs: dict[str, Any],- *,- iters,- gradient_accumulate_steps,- kwargs,- common_config_kwargs,- auto_round_config_kwargs,- ):- cfg = SignRoundConfig(- **shared_quant_kwargs,- iters=iters,- gradient_accumulate_steps=gradient_accumulate_steps,- lr=kwargs.pop("lr", None),- minmax_lr=kwargs.pop("minmax_lr", None),- enable_minmax_tuning=kwargs.pop("enable_minmax_tuning", True),- enable_norm_bias_tuning=kwargs.pop("enable_norm_bias_tuning", False),- enable_quanted_input=kwargs.pop("enable_quanted_input", True),- **common_config_kwargs,- **auto_round_config_kwargs,- )- return normalize_algorithm_config(cfg)-- @classmethod- def _build_alg_config(- cls,- *,- algorithm,- iters,- gradient_accumulate_steps,- seqlen,- nsamples,- batch_size,- kwargs,- common_config_kwargs,- auto_round_config_kwargs,- ):- alg_name = cls._resolve_compat_algorithm(algorithm, iters)- shared_quant_kwargs = cls._pop_shared_quant_kwargs(kwargs)-- if alg_name == "awq":- return cls._build_awq_config(- shared_quant_kwargs,- seqlen=seqlen,- nsamples=nsamples,- batch_size=batch_size,- kwargs=kwargs,- common_config_kwargs=common_config_kwargs,- )- if alg_name == "rtn":- return cls._build_rtn_config(- shared_quant_kwargs,- kwargs=kwargs,- common_config_kwargs=common_config_kwargs,- )- return cls._build_signround_config(- shared_quant_kwargs,- iters=iters,- gradient_accumulate_steps=gradient_accumulate_steps,- kwargs=kwargs,- common_config_kwargs=common_config_kwargs,- auto_round_config_kwargs=auto_round_config_kwargs,- )-- @staticmethod- def _build_entry_forward_kwargs(kwargs: dict[str, Any]) -> dict[str, Any]:- format_name = kwargs.pop("format", None)- rotation_config = kwargs.pop("rotation_config", None)- mllm_kwargs = {- "processor": kwargs.pop("processor", None),- "image_processor": kwargs.pop("image_processor", None),- "template": kwargs.pop("template", None),- "extra_data_dir": kwargs.pop("extra_data_dir", None),- "quant_nontext_module": kwargs.pop("quant_nontext_module", False),- }- diffusion_kwargs = {- "guidance_scale": kwargs.pop("guidance_scale", 7.5),- "num_inference_steps": kwargs.pop("num_inference_steps", 50),- "generator_seed": kwargs.pop("generator_seed", None),- }- return {- "format": format_name,- "rotation_config": rotation_config,- **mllm_kwargs,- **diffusion_kwargs,- **kwargs,- }-- def __new__(- cls,- model: Union[torch.nn.Module, str],- tokenizer=None,- platform: str = "hf",- scheme: Union[str, dict, QuantizationScheme, AutoScheme] = "W4A16",- layer_config: dict[str, Union[str, dict, QuantizationScheme]] = None,- dataset: Optional[Union[str, list, tuple, torch.utils.data.DataLoader]] = None,- iters: int = 200,- seqlen: int = 2048,- nsamples: int = 128,- batch_size: int = 8,- gradient_accumulate_steps: int = 1,- low_gpu_mem_usage: bool = False,- device_map: Union[str, torch.device, int, dict] = 0,- enable_torch_compile: Optional[bool] = None,- seed: int = 42,- low_cpu_mem_usage: bool = True,- algorithm: str = None,- **kwargs,- ) -> "BaseCompressor":- """Create AutoRoundCompatible instance using new AutoRound architecture.-- This method translates old AutoRoundCompatible API to new AutoRound API.- """- from auto_round.utils import is_diffusion_model, is_mllm_model- from auto_round.utils.model import is_model_free_route-- device = kwargs.pop("device", None)- if device is not None:- logger.warning_once("`device` is deprecated, please use `device_map` instead")- if device_map in (None, 0):- device_map = device-- # ---- Model-free fast-path detection --------------------------------- if is_model_free_route(model, scheme, iters, kwargs.get("disable_opt_rtn"), kwargs):- from auto_round.compressors.model_free import ModelFreeCompressor-- compressor_only_kwargs = cls._pop_compressor_only_kwargs(kwargs)-- if not isinstance(model, str):- raise ValueError("model_free=True requires `model` to be a HuggingFace ID or local path string.")- if not bool(kwargs.get("model_free", False)):- logger.info(- "Auto-routing to model-free quantization "- "(iters=0, disable_opt_rtn=True, supported scheme). "- "Pass disable_model_free=True to use the regular flow."- )- return ModelFreeCompressor(- model_name_or_path=model,- scheme=scheme,- layer_config=layer_config,- tokenizer=tokenizer,- device_map=device_map,- low_cpu_mem_usage=low_cpu_mem_usage,- enable_torch_compile=enable_torch_compile,- **compressor_only_kwargs,- **kwargs,- )- # ---------------------------------------------------------------------- compressor_only_kwargs = cls._pop_compressor_only_kwargs(kwargs)- common_config_kwargs, auto_round_config_kwargs = cls._pop_config_kwargs(kwargs)-- config = cls._build_alg_config(- algorithm=algorithm,- iters=iters,- gradient_accumulate_steps=gradient_accumulate_steps,- seqlen=seqlen,- nsamples=nsamples,- batch_size=batch_size,- kwargs=kwargs,- common_config_kwargs=common_config_kwargs,- auto_round_config_kwargs=auto_round_config_kwargs,- )-- forward_kwargs = cls._build_entry_forward_kwargs(kwargs)- format_name = forward_kwargs.pop("format", None)- _rotation_config_raw = forward_kwargs.pop("rotation_config", None)- if _rotation_config_raw is not None:- _rc = _normalize_rotation_alg_config(_rotation_config_raw)- if _rc is None:- _rc = _NewArchRotationConfig()- config = [config, _rc]-- # Check model type for logging (use warning_once to avoid repeating for every block- # when called from LLM-Compressor which instantiates AutoRound per block)- if is_mllm_model(model, platform=platform):- logger.info("Using MLLM mode for multimodal model.")- elif is_diffusion_model(model):- logger.info("Using Diffusion mode for diffusion model.")- else:- logger.info("Using LLM mode.")-- # Create AutoRound instance using new architecture- compressor = AutoRound(- model,- scheme,- config,- tokenizer=tokenizer,- platform=platform,- format=format_name,- dataset=dataset,- iters=iters,- gradient_accumulate_steps=gradient_accumulate_steps,- low_gpu_mem_usage=low_gpu_mem_usage,- device_map=device_map,- enable_torch_compile=enable_torch_compile,- seed=seed,- low_cpu_mem_usage=low_cpu_mem_usage,- layer_config=layer_config,- nsamples=nsamples,- seqlen=seqlen,- batch_size=batch_size,- **compressor_only_kwargs,- **forward_kwargs,- )-- return compressordiff --git a/auto_round/compressors/layer_config/__init__.py b/auto_round/compressors/layer_config/__init__.pynew file mode 100644index 000000000..5b0adaf4f--- /dev/null+++ b/auto_round/compressors/layer_config/__init__.py@@ -0,0 +1,27 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++from auto_round.compressors.layer_config.resolver import (+ apply_plan_to_model,+ extract_regex_config,+ has_quantized_layer_outside_blocks,+ resolve_layer_config,+)++__all__ = [+ "apply_plan_to_model",+ "extract_regex_config",+ "has_quantized_layer_outside_blocks",+ "resolve_layer_config",+]diff --git a/auto_round/compressors/layer_config/resolver.py b/auto_round/compressors/layer_config/resolver.pynew file mode 100644index 000000000..60f347d2e--- /dev/null+++ b/auto_round/compressors/layer_config/resolver.py@@ -0,0 +1,497 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++import copy+import json+import os+import re+import sys+from dataclasses import asdict, fields+from typing import Union++import torch++from auto_round.compressors.config_resolution import ResolvedQuantizationConfig, ResolvedScheme+from auto_round.compressors.config_resolution.contracts import LayerConfig, freeze_mapping+from auto_round.logger import logger+from auto_round.schemes import QuantizationScheme, get_gguf_scheme, is_mx_fp, is_nv_fp+from auto_round.utils import (+ INNER_SUPPORTED_LAYER_TYPES,+ SUPPORTED_LAYER_TYPES,+ check_to_quantized,+ compress_layer_names,+ get_layer_names_in_block,+ infer_bits_by_data_type,+ to_standard_regex,+)+from auto_round.utils.model import get_module+++def apply_layer_config_special_cases(+ layer_config,+ model,+ default_dict,+ supported_types,+ inner_supported_types,+ quant_block_list,+ quant_lm_head,+ gguf_name,+) -> tuple[dict, bool, str | None, bool]:+ """Apply lm-head, shape-divisibility, and block-membership layer rules."""+ from auto_round.utils.model import get_lm_head_name++ lm_head_name = get_lm_head_name(model)+ tie_word_embeddings = bool(getattr(getattr(model, "config", None), "tie_word_embeddings", False))+ if lm_head_name in layer_config:+ quant_lm_head = True+ if quant_lm_head and tie_word_embeddings and not gguf_name:+ quant_lm_head = False+ logger.warning(+ "reset `quant_lm_head` to false as quantizing lm_head with tied weights has not been supported currently"+ )+ if lm_head_name not in layer_config and quant_lm_head:+ layer_config[lm_head_name] = copy.deepcopy(default_dict)+ if not quant_lm_head and not gguf_name:+ layer_config.pop(lm_head_name, None)++ if default_dict["data_type"] == "int" and default_dict["act_bits"] >= 16 and not gguf_name:+ for n, m in model.named_modules():+ if type(m) in supported_types or m.__class__.__name__ in inner_supported_types:+ if m.weight.shape[0] % 32 or m.weight.shape[1] % 32:+ layer_config.setdefault(n, copy.deepcopy(default_dict))+ layer_config[n].update({"bits": 16, "data_type": "fp", "fixed_by_user": True})++ if (is_nv_fp(default_dict["data_type"]) or is_mx_fp(default_dict["data_type"])) and not gguf_name:+ skipped_layers = []+ for n, m in model.named_modules():+ if type(m) in supported_types or m.__class__.__name__ in inner_supported_types:+ if m.weight.shape[1] % default_dict["group_size"]:+ layer_config.setdefault(n, copy.deepcopy(default_dict))+ layer_config[n].update(+ {"bits": 16, "data_type": "fp", "act_bits": 16, "act_data_type": "fp", "fixed_by_user": True}+ )+ skipped_layers.append(n)+ compressed_skipped_layers = compress_layer_names(skipped_layers)+ if compressed_skipped_layers:+ logger.warning_once(+ f"some layers are skipped quantization (shape not divisible by {default_dict['group_size']}): "+ f"{compressed_skipped_layers}"+ )++ for name in get_layer_names_in_block(model, supported_types, quant_block_list, inner_supported_types):+ if name not in layer_config:+ layer_config[name] = copy.deepcopy(default_dict)+ layer_config[name]["fixed_by_user"] = False+ layer_config[name]["in_blocks"] = True++ has_qlayer_outside_block = False+ for cfg in layer_config.values():+ if "in_blocks" not in cfg:+ cfg["in_blocks"] = False+ if not cfg["in_blocks"] and check_to_quantized(cfg):+ has_qlayer_outside_block = True+ return layer_config, has_qlayer_outside_block, lm_head_name, tie_word_embeddings+++def _get_safetensor_layer_names_not_in_model(model, all_module_names: list) -> list:+ """Collect layer names from safetensor files that are not loaded into the model.++ Some tensors (e.g. MTP layers) exist in the original checkpoint but are not+ instantiated by ``transformers``. This function discovers them so that regex+ patterns in ``layer_config`` can still match them.++ Returns:+ List of layer names (the path without the ``.weight`` suffix) for weight+ tensors present in the safetensor files but absent from *all_module_names*.+ """+ name_or_path = None+ if hasattr(model, "config") and hasattr(model.config, "name_or_path"):+ name_or_path = model.config.name_or_path+ if not name_or_path:+ return []++ if not os.path.isdir(name_or_path):+ # Resolution is intentionally local-only: model downloads belong to model loading,+ # not to the pure layer-config planning phase.+ return []++ try:+ from safetensors import safe_open+ except ImportError:+ return []++ # Build tensor-name list from the safetensors index or single file+ source_index_file = os.path.join(name_or_path, "model.safetensors.index.json")+ source_single_file = os.path.join(name_or_path, "model.safetensors")++ tensor_names: list = []+ if os.path.exists(source_index_file):+ with open(source_index_file) as f:+ src_index = json.load(f)+ tensor_names = list(src_index["weight_map"].keys())+ elif os.path.exists(source_single_file):+ with safe_open(source_single_file, framework="pt", device="cpu") as f:+ tensor_names = list(f.keys())+ else:+ return []++ module_name_set = set(all_module_names)+ extra_layer_names = []+ for tensor_name in tensor_names:+ if not tensor_name.endswith(".weight"):+ continue+ layer_name = tensor_name[: -len(".weight")]+ if layer_name not in module_name_set:+ extra_layer_names.append(layer_name)+ return extra_layer_names+++def _resolve_layer_config_presets(+ layer_config, model, ignore_layers, default_scheme, default_scale_dtype, fill_default_value+) -> tuple[dict, dict, tuple[str, ...], set[str]]:+ """Steps 1-4 of layer-config resolution: ignore-layer parsing, normalization,+ bits-inference, and default-filling."""+ from auto_round.schemes import QuantizationScheme, preset_name_to_scheme++ def normalize_item(item: Union[str, dict, "QuantizationScheme"], layer_name: str) -> dict:+ """Convert config entry into dict and validate keys."""+ if isinstance(item, str):+ config = asdict(preset_name_to_scheme(item.upper()))+ elif isinstance(item, QuantizationScheme):+ config = asdict(item)+ elif isinstance(item, dict):+ # "in_blocks" is an internal bookkeeping key injected by LLM-Compressor;+ # silently drop it before validation.+ item = {k: v for k, v in item.items() if k != "in_blocks"}+ scheme_name = item.pop("scheme", None)+ config = asdict(preset_name_to_scheme(scheme_name.upper())) if scheme_name is not None else {}+ invalid = set(item) - set(scheme_keys + ("fixed_by_user", "scale_dtype"))+ if invalid:+ raise ValueError(+ f"Invalid keys {invalid} in layer_config for '{layer_name}'. " f"Allowed keys: {scheme_keys}"+ )+ config.update(item)+ else:+ raise TypeError(+ f"Unsupported type for layer_config[{layer_name}]: {type(item)}. "+ f"Expected str, dict, or QuantizationScheme."+ )+ # Clean up+ config = {k: v for k, v in config.items() if v is not None}+ config["fixed_by_user"] = True+ return config++ extra_scheme_keys = ("scale_dtype",)+ scheme_keys = tuple(f.name for f in fields(QuantizationScheme)) + ("scale_dtype",)+ layer_config = copy.deepcopy(layer_config) or {}+ ignore_layer_patterns = set()+ if ignore_layers:+ ignore_layers = ignore_layers.replace(" ", "").split(",")+ ignore_layers = [name + "." if name[-1].isdigit() else name for name in ignore_layers]+ ignore_layer_patterns = set(ignore_layers)++ # 1. ignore_layers -> force 16+ for name in get_fp_layer_names(model, ignore_layers):+ layer_config[name] = {+ "bits": 16,+ "act_bits": 16,+ "data_type": "float",+ "act_data_type": "float",+ "fixed_by_user": True,+ }++ # 2. normalize+ layer_config = {k: normalize_item(v, k) for k, v in layer_config.items()}++ # 3. infer missing bits+ for cfg in layer_config.values():+ if "data_type" in cfg and "bits" not in cfg:+ if (b := infer_bits_by_data_type(cfg["data_type"])) is not None:+ cfg["bits"] = b+ if "act_data_type" in cfg and "act_bits" not in cfg:+ if (b := infer_bits_by_data_type(cfg["act_data_type"])) is not None:+ cfg["act_bits"] = b++ # 4. fill defaults+ if isinstance(default_scheme, str):+ default_dict = asdict(preset_name_to_scheme(default_scheme.upper()))+ elif default_scheme is None:+ # Defensive fallback: callers that build/patch layer configs without a resolved+ # scheme (e.g. external integrations invoking configure_layer_config() out of the+ # normal init sequence) would otherwise crash `asdict(None)` below.+ default_dict = asdict(QuantizationScheme())+ else:+ default_dict = asdict(default_scheme)+ default_dict["scale_dtype"] = default_scale_dtype++ # In AutoScheme with mixed gguf:q4_k_m, the super_group_size of gguf:q8_0 layer is None,+ # which should not be filled by default q4km again+ for cfg in layer_config.values():+ for key in scheme_keys:+ if fill_default_value:+ cfg.setdefault(key, copy.deepcopy(default_dict.get(key)))+ else:+ if key in extra_scheme_keys:+ cfg.setdefault(key, copy.deepcopy(default_dict.get(key)))+ else:+ cfg.setdefault(key, None)++ return layer_config, default_dict, scheme_keys, ignore_layer_patterns+++def _traverse_and_expand_layer_config(+ layer_config, model, supported_types, inner_supported_types, ignore_layer_patterns, scheme_keys, gguf_name+) -> tuple[dict, dict, list[str], tuple]:+ """Steps 5-6 of layer-config resolution: supported-module collection (incl. the+ GGUF embedding-type-detection branch) and regex-config expansion."""+ from auto_round.utils.model import get_module++ # 5. collect supported modules+ embedding_types = (torch.nn.Embedding,)+ if gguf_name:+ if torch.nn.Embedding not in supported_types:+ supported_types = (*supported_types, torch.nn.Embedding)++ # for some Embedding which type() is not torch.nn.Embedding+ # for example: transformers.models.gemma3.modeling_gemma3.Gemma3TextScaledWordEmbedding+ model_module_name = model.__class__.__module__+ module_cls = sys.modules[model_module_name]+ for name in module_cls.__dict__:+ if name.endswith("Embedding") and not name.endswith("RotaryEmbedding"):+ embedding_types = (*embedding_types, getattr(module_cls, name))+ supported_types = (*supported_types, *embedding_types)++ all_supported_layer_names, embedding_layer_names = [], []+ all_module_names = []+ for n, m in model.named_modules():+ all_module_names.append(n)+ if type(m) not in supported_types and m.__class__.__name__ not in inner_supported_types:+ continue+ all_supported_layer_names.append(n)+ if isinstance(m, embedding_types) or m.__class__.__name__.endswith("Embedding"):+ embedding_layer_names.append(n)++ # Also include layer names from safetensor files not loaded into the model+ # (e.g. MTP layers that transformers does not instantiate).+ safetensor_only_names = _get_safetensor_layer_names_not_in_model(model, all_module_names)++ # 6. expand regex configs+ regex_config = {}+ for name in list(layer_config.keys()):+ if name in all_supported_layer_names:+ continue+ if name in all_module_names:+ m = get_module(model, name)+ if len(list(m.children())) == 0 and type(m) not in supported_types:+ val = layer_config.pop(name)+ if name in ignore_layer_patterns:+ # Keep unsupported ignore_layers entries so export can serialize+ # them into regex-based extra_config for loaders like vLLM INC.+ regex_config[name] = val+ else:+ logger.warning(+ f"'{name}' exists in the model but is not a supported quantization target "+ f"in the current scheme, ignoring its setting in `layer_config`"+ )+ continue++ regex = re.compile(to_standard_regex(name))+ matched = [ln for ln in all_supported_layer_names if regex.search(ln)]+ safetensor_only_matched = [ln for ln in safetensor_only_names if regex.search(ln)]+ # skip it for mtp layers not loaded in transformers+ if not matched and not safetensor_only_matched:+ # type(mlp.gate) is Qwen3VLMoeTextTopKRouter instead of Linear+ logger.warning_once(+ f"Layer name or regex '{name}' in layer_config does not match any supported layers. "+ + "Please check for typos or update the regex pattern, ignore it for now"+ )+ val = layer_config.pop(name)+ regex_config[name] = val # keep regex config+ for match in matched:+ layer_config[match] = val++ return layer_config, regex_config, embedding_layer_names, supported_types+++def get_fp_layer_names(model: torch.nn.Module, ignore_layers: str):+ """Identifies and returns layers in the model to exclude from quantization.++ This function processes a comma-separated list of fully precision (FP) layers,+ matches them to the names of layers in the model, and returns a list of such+ layers to exclude from quantization.++ Args:+ model (torch.nn.Module): The model whose layers will be inspected.+ ignore_layers (str): A comma-separated string of layer names to be excluded+ from quantization. Whitespace is ignored in this string.++ Returns:+ list: A list of layer names that match the specified FP layers or are+ subcomponents of those layers.+ """+ from auto_round.utils import SUPPORTED_LAYER_TYPES++ if not ignore_layers:+ return []++ all_layer_names = []+ for n, m in model.named_modules():+ if type(m) in SUPPORTED_LAYER_TYPES:+ all_layer_names.append(n)+ not_to_quantized_layers = []++ for fp_layer in ignore_layers:+ if fp_layer == "":+ continue+ if fp_layer in all_layer_names:+ not_to_quantized_layers.append(fp_layer)+ continue+ for name in all_layer_names:+ if fp_layer in name:+ not_to_quantized_layers.append(name)+ not_to_quantized_layers.extend(ignore_layers) # keep regex name for later use+ if not_to_quantized_layers:+ logger.info(f"Ignored layers: {compress_layer_names(not_to_quantized_layers)}")+ return not_to_quantized_layers+++def resolve_layer_config(+ *,+ model,+ scheme: ResolvedScheme,+ layer_config,+ scale_dtype=None,+ supported_types=None,+ inner_supported_types=None,+ quant_block_list=None,+ ignore_layers: str = "",+ quant_lm_head: bool = False,+ enable_gguf_official_mixed: bool = True,+ is_mllm: bool = False,+ fill_default_value: bool = True,+) -> LayerConfig:+ """Resolve final per-layer configuration without writing model attributes."""+ supported_types = tuple(SUPPORTED_LAYER_TYPES if supported_types is None else supported_types)+ inner_supported_types = tuple(+ INNER_SUPPORTED_LAYER_TYPES if inner_supported_types is None else inner_supported_types+ )+ scheme_value = scheme.value+ gguf_name = scheme.preset_name if (scheme.preset_name or "").startswith("gguf:") else get_gguf_scheme(scheme_value)++ resolved, default_dict, scheme_keys, ignore_patterns = _resolve_layer_config_presets(+ layer_config,+ model,+ ignore_layers,+ scheme_value,+ scale_dtype,+ fill_default_value,+ )+ resolved, _, embedding_names, supported_types = _traverse_and_expand_layer_config(+ resolved,+ model,+ supported_types,+ inner_supported_types,+ ignore_patterns,+ scheme_keys,+ gguf_name,+ )+ resolved, has_outside, lm_head_name, tied = apply_layer_config_special_cases(+ resolved,+ model,+ default_dict,+ supported_types,+ inner_supported_types,+ quant_block_list,+ quant_lm_head,+ gguf_name,+ )+ if gguf_name:+ from auto_round.export.formats.backends.gguf import apply_gguf_layer_defaults++ resolved, _ = apply_gguf_layer_defaults(+ dict(resolved),+ model,+ gguf_name,+ lm_head_name,+ tied,+ embedding_names,+ scale_dtype,+ enable_gguf_official_mixed,+ is_mllm,+ has_outside,+ )+ return freeze_mapping(resolved)+++def extract_regex_config(+ *,+ model,+ scheme: ResolvedScheme,+ layer_config,+ scale_dtype=None,+ supported_types=None,+ inner_supported_types=None,+ ignore_layers: str = "",+ fill_default_value: bool = True,+) -> LayerConfig:+ """Resolve only the regex entries retained for export metadata."""+ supported_types = tuple(SUPPORTED_LAYER_TYPES if supported_types is None else supported_types)+ inner_supported_types = tuple(+ INNER_SUPPORTED_LAYER_TYPES if inner_supported_types is None else inner_supported_types+ )+ scheme_value = scheme.value+ gguf_name = scheme.preset_name if (scheme.preset_name or "").startswith("gguf:") else get_gguf_scheme(scheme_value)+ normalized, _, scheme_keys, ignore_patterns = _resolve_layer_config_presets(+ layer_config,+ model,+ ignore_layers,+ scheme_value,+ scale_dtype,+ fill_default_value,+ )+ _, regex_config, _, _ = _traverse_and_expand_layer_config(+ normalized,+ model,+ supported_types,+ inner_supported_types,+ ignore_patterns,+ scheme_keys,+ gguf_name,+ )+ return freeze_mapping(regex_config)+++def has_quantized_layer_outside_blocks(layer_config: LayerConfig) -> bool:+ """Derive the block-outside flag from a final layer mapping."""+ return any(+ not config.get("in_blocks", False) and check_to_quantized(dict(config)) for config in layer_config.values()+ )+++def apply_plan_to_model(model, plan: ResolvedQuantizationConfig) -> None:+ """Apply a resolved plan at the single explicit module-attribute write boundary."""+ scheme_keys = tuple(field.name for field in fields(QuantizationScheme)) + ("scale_dtype",)+ for module_name, module in model.named_modules():+ for key in scheme_keys:+ if module_name == "" and key == "rotation_config":+ continue+ if hasattr(module, key):+ delattr(module, key)++ for layer_name, config in plan.layer_config.items():+ module = get_module(model, layer_name)+ if module is None:+ continue+ for name, value in config.items():+ setattr(module, name, value)diff --git a/auto_round/compressors/model_free.py b/auto_round/compressors/model_free.pyindex be94664a8..3c59ba15b 100644--- a/auto_round/compressors/model_free.py+++ b/auto_round/compressors/model_free.py@@ -104,6 +104,7 @@ from typing import Any, Callable, Optional, Unionimport torchfrom auto_round import envs+from auto_round.compressors.config_resolution import thaw_mappingfrom auto_round.compressors.utils import is_mx_fpfrom auto_round.logger import loggerfrom auto_round.schemes import PRESET_SCHEMES, QuantizationScheme, preset_name_to_scheme@@ -1690,6 +1691,10 @@ def _validate_supported_scheme(# Activation quantization for MXFP is dynamic at inference time, so the# weight-only RTN path here is independent of act_bits.if is_mx_fp(data_type):+ if scheme_obj.act_data_type not in (None, "mx_fp"):+ raise ValueError(+ "Model-free MXFP supports only act_data_type='mx_fp', " f"but got '{scheme_obj.act_data_type}'."+ )# Restrict to the two explicitly supported MXFP presets when a string# name is provided. Variants such as MXFP4_RCEIL / MXFP8_RCEIL use a# different activation format; silently mapping them to "MXFP4" /@@ -2020,6 +2025,9 @@ class _ModelFreeCompressorCore:def _parse_scheme(self) -> None:scheme_in = self.scheme_input+ scheme_overrides = getattr(self, "user_scheme_overrides", None)+ if scheme_overrides:+ scheme_in = _apply_scheme_overrides(scheme_in, scheme_overrides)if isinstance(scheme_in, str) and scheme_in.upper() == "W4A16_MIXED":# Match regular-flow mixed recipe behavior in model-free mode:# default non-expert linear layers use 8-bit; expert overrides are@@ -2619,6 +2627,9 @@ class ModelFreeCompressor(_ModelFreeCompressorCore):quant_lm_head=quant_lm_head,low_cpu_mem_usage=low_cpu_mem_usage,)+ # Scheme fields are consumed above from ``kwargs``. Preserve them when+ # a later format check falls back to the regular AutoRound flow.+ fallback_init.update(self.user_scheme_overrides)self._fallback_init_kwargs = fallback_initif quant_nontext_module:@@ -2634,7 +2645,7 @@ class ModelFreeCompressor(_ModelFreeCompressorCore):logger.info("Format '%s' is not supported by model-free mode; falling back to the regular AutoRound flow.",- format,+ self.format,)logger.info("fallbacked_init_kwargs: %s",@@ -2718,7 +2729,7 @@ class ModelFreeCompressor(_ModelFreeCompressorCore):if not callable(post_init):raise RuntimeError("AutoScheme fallback compressor has no callable post_init().")post_init() # pylint: disable=E1102- layer_config = copy.deepcopy(getattr(compressor, "layer_config", {}) or {})+ layer_config = thaw_mapping(getattr(compressor, "layer_config", {}) or {})finally:# Release the model that was loaded only for scoring so the# packing phase keeps model-free's low memory footprint.diff --git a/auto_round/compressors/utils.py b/auto_round/compressors/utils.pyindex d7349166f..d247e8959 100644--- a/auto_round/compressors/utils.py+++ b/auto_round/compressors/utils.py@@ -11,139 +11,99 @@# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.# See the License for the specific language governing permissions and# limitations under the License.-import copy-import jsonimport osimport randomimport re-import sys-from dataclasses import asdict, fields-from enum import Enum-from typing import TYPE_CHECKING, Callable, Union+from typing import Unionimport torch-import transformersfrom torch.amp import autocast-from auto_round.export.export_to_gguf.config import GGML_QUANT_SIZES, GGUF_CONFIG, GGUF_INNER_CONFIG, QK_K, ModelType-from auto_round.export.export_to_gguf.gguf_dtype import GGUFDTypeSelector, gguf_format_to_ftype+from auto_round.compressors.config_resolution import LayerConfigResolutionErrorfrom auto_round.logger import logger+from auto_round.schemes import BackendDataType # re-exported: qlinear_fp/qlinear_int import it from here+from auto_round.schemes import (+ QuantizationScheme,+ is_mx_fp,+ is_mx_int,+ is_nv_fp,+ is_standard_fp,+)from auto_round.utils import (check_to_quantized,- compress_layer_names,get_layer_names_in_block,get_module,- to_standard_regex,)from auto_round.utils.device_manager import device_manager-if TYPE_CHECKING:- from auto_round.schemes import QuantizationScheme---class BackendDataType(str, Enum):- STANDARD_FP = "fp"- MX_FP = "mx_fp"- NV_FP = "nv_fp"- MX_INT = "mx_int"---def is_standard_fp(backend):- backend = backend.lower()- return BackendDataType.STANDARD_FP in backend and not is_mx_fp(backend) and not is_nv_fp(backend)---def is_mx_fp(backend):- backend = backend.lower()- return BackendDataType.MX_FP in backend+def _as_scheme(ar_or_scheme) -> "QuantizationScheme":+ """Resolve a compressor-like object or QuantizationScheme to a QuantizationScheme.-def is_mx_int(backend):- backend = backend.lower()- return BackendDataType.MX_INT in backend+ `ar` (the compressor) exposes the same flat attribute names as QuantizationScheme+ (bits, data_type, act_bits, ...), so QuantizationScheme.from_dict can read them+ directly without needing ar.scheme to already be resolved.+ """+ if isinstance(ar_or_scheme, QuantizationScheme):+ return ar_or_scheme+ return QuantizationScheme(+ bits=ar_or_scheme.bits,+ group_size=ar_or_scheme.group_size,+ sym=getattr(ar_or_scheme, "sym", True),+ data_type=ar_or_scheme.data_type,+ act_bits=ar_or_scheme.act_bits,+ act_group_size=getattr(ar_or_scheme, "act_group_size", None),+ act_sym=getattr(ar_or_scheme, "act_sym", None),+ act_data_type=ar_or_scheme.act_data_type,+ act_dynamic=getattr(ar_or_scheme, "act_dynamic", None),+ super_bits=getattr(ar_or_scheme, "super_bits", None),+ super_group_size=getattr(ar_or_scheme, "super_group_size", None),+ )-def is_nv_fp(backend):- backend = backend.lower()- return BackendDataType.NV_FP in backend+# ``is_standard_fp`` / ``is_mx_fp`` / ``is_nv_fp`` / ``is_mx_int`` (data_type-string+# classifiers) now live in ``auto_round.schemes`` as the single authority and are+# re-exported above so existing ``from auto_round.compressors.utils import is_mx_fp``+# call sites keep working.def is_wint_woq(ar):"""Returns True for integer weight-only quantization with non-quantized activations (`act_bits >= 16`)."""- return "int" in ar.data_type and ar.act_bits >= 16 and ar.super_group_size is None---def is_wint_a16(ar):- """Backward-compatible alias for `is_wint_woq()`."""- return is_wint_woq(ar)---def _is_weight_fp8_activation_static_fp8(- bit: int, group_size: int, sym: bool, data_type: str, act_dynamic: bool-) -> bool:- return bit == 8 and group_size == -1 and sym and data_type == "fp" and not act_dynamic+ return _as_scheme(ar).is_wint_woq()def is_wfp8afp8(ar):- if (- ("fp8" in ar.act_data_type or ("fp" in ar.act_data_type and ar.act_bits == 8))- and ("fp8" in ar.data_type or ("fp" in ar.data_type and ar.bits == 8))- and is_standard_fp(ar.act_data_type)- and is_standard_fp(ar.data_type)- ):- return True- else:- return False+ return _as_scheme(ar).is_wfp8afp8()def is_wint8aint8(ar):- if ("int8" in ar.act_data_type or ("int" in ar.act_data_type and ar.act_bits == 8)) and (- "int8" in ar.data_type or ("int" in ar.data_type and ar.bits == 8)- ):- return True- else:- return False+ return _as_scheme(ar).is_wint8aint8()-def is_act_static(ar_or_format: Union[str, Callable]) -> bool:- if isinstance(ar_or_format, str): # TODO this is not robust+def is_act_static(ar_or_format):+ if isinstance(ar_or_format, str):return "fp8_static" in ar_or_format.lower()- if not ar_or_format.act_dynamic:- return True- return False+ return _as_scheme(ar_or_format).is_act_static()-def is_dynamic_wint8aint8(ar_or_format: Union[str, Callable]) -> bool:+def is_dynamic_wint8aint8(ar_or_format):if isinstance(ar_or_format, str):return "int8_w8a8" in ar_or_format.lower()- if not ar_or_format.act_dynamic:- return False- if is_wint8aint8(ar_or_format):- return True- return False+ return _as_scheme(ar_or_format).is_dynamic_wint8aint8()-def is_wint4aint4(ar_or_scheme: Union[str, Callable]):+def is_wint4aint4(ar_or_scheme):if isinstance(ar_or_scheme, str):return "int4" in ar_or_scheme.lower()- elif (- "int4" in ar_or_scheme.act_data_type or ("int" in ar_or_scheme.act_data_type and ar_or_scheme.act_bits == 4)- ) and ("int4" in ar_or_scheme.data_type or ("int" in ar_or_scheme.data_type and ar_or_scheme.bits == 4)):- return True- return False+ return _as_scheme(ar_or_scheme).is_wint4aint4()-def is_dynamic_afp8(ar_or_format: Callable) -> bool:- return ar_or_format.act_dynamic and ar_or_format.act_data_type.startswith("fp") and ar_or_format.act_bits == 8+def is_dynamic_afp8(ar_or_format):+ return _as_scheme(ar_or_format).is_dynamic_afp8()-def is_block_wfp8(ar_or_format: Callable) -> bool:- return (- isinstance(ar_or_format.group_size, tuple)- and len(ar_or_format.group_size) == 2- and ar_or_format.data_type.startswith("fp")- and ar_or_format.bits == 8- )+def is_block_wfp8(ar_or_format):+ return _as_scheme(ar_or_format).is_block_wfp8()def block_forward(@@ -281,63 +241,6 @@ def infer_bits_by_data_type(data_type: str):return None-def _get_safetensor_layer_names_not_in_model(model, all_module_names: list) -> list:- """Collect layer names from safetensor files that are not loaded into the model.-- Some tensors (e.g. MTP layers) exist in the original checkpoint but are not- instantiated by ``transformers``. This function discovers them so that regex- patterns in ``layer_config`` can still match them.-- Returns:- List of layer names (the path without the ``.weight`` suffix) for weight- tensors present in the safetensor files but absent from *all_module_names*.- """- name_or_path = None- if hasattr(model, "config") and hasattr(model.config, "name_or_path"):- name_or_path = model.config.name_or_path- if not name_or_path:- return []-- if not os.path.isdir(name_or_path):- try:- from auto_round.utils.model import download_hf_model-- name_or_path = download_hf_model(name_or_path)- except Exception as e:- logger.debug(f"Could not resolve source model path to check for missing tensors: {e}")- return []-- try:- from safetensors import safe_open- except ImportError:- return []-- # Build tensor-name list from the safetensors index or single file- source_index_file = os.path.join(name_or_path, "model.safetensors.index.json")- source_single_file = os.path.join(name_or_path, "model.safetensors")-- tensor_names: list = []- if os.path.exists(source_index_file):- with open(source_index_file) as f:- src_index = json.load(f)- tensor_names = list(src_index["weight_map"].keys())- elif os.path.exists(source_single_file):- with safe_open(source_single_file, framework="pt", device="cpu") as f:- tensor_names = list(f.keys())- else:- return []-- module_name_set = set(all_module_names)- extra_layer_names = []- for tensor_name in tensor_names:- if not tensor_name.endswith(".weight"):- continue- layer_name = tensor_name[: -len(".weight")]- if layer_name not in module_name_set:- extra_layer_names.append(layer_name)- return extra_layer_names--def set_layer_config(model: torch.nn.Module,layer_config: dict[str, Union[str, dict, "QuantizationScheme"]],@@ -351,722 +254,84 @@ def set_layer_config(enable_gguf_official_mixed: bool = True,is_mllm: bool = False,fill_default_value=True,- gguf_format_name: str = None,) -> tuple[dict, bool, dict]:- """- Normalize, validate, and expand layer-specific quantization configs.- Returns (final_layer_config, has_quant_layer_outside_block)- """-- from auto_round.schemes import QuantizationScheme, get_gguf_scheme, preset_name_to_scheme- from auto_round.utils.model import get_layer_names_in_block, get_lm_head_name, get_module, is_separate_lm_head-- # ---- helpers -------------------------------------------------- def dispatch_layer_config(layer_config: dict[str, dict]) -> None:- """Assign scheme values as attributes to matched modules."""- for layer_name, scheme in layer_config.items():- module = get_module(model, layer_name)- if module is None:- # Layer exists in safetensor files but is not loaded into the model- # (e.g. MTP layers that transformers does not instantiate). Skip.- continue- for attr, value in scheme.items():- setattr(module, attr, value)-- def normalize_item(item: Union[str, dict, "QuantizationScheme"], layer_name: str) -> dict:- """Convert config entry into dict and validate keys."""- if isinstance(item, str):- config = asdict(preset_name_to_scheme(item.upper()))- elif isinstance(item, QuantizationScheme):- config = asdict(item)- elif isinstance(item, dict):- # "in_blocks" is an internal bookkeeping key injected by LLM-Compressor;- # silently drop it before validation.- item = {k: v for k, v in item.items() if k != "in_blocks"}- scheme_name = item.pop("scheme", None)- config = asdict(preset_name_to_scheme(scheme_name.upper())) if scheme_name is not None else {}- invalid = set(item) - set(scheme_keys + ("fixed_by_user", "scale_dtype"))- if invalid:- raise ValueError(- f"Invalid keys {invalid} in layer_config for '{layer_name}'. " f"Allowed keys: {scheme_keys}"- )- config.update(item)- else:- raise TypeError(- f"Unsupported type for layer_config[{layer_name}]: {type(item)}. "- f"Expected str, dict, or QuantizationScheme."- )- # Clean up- config = {k: v for k, v in config.items() if v is not None}- config["fixed_by_user"] = True- return config-- # ---- main logic ----------------------------------------------- extra_scheme_keys = ("scale_dtype",)- scheme_keys = tuple(f.name for f in fields(QuantizationScheme)) + ("scale_dtype",)- layer_config = copy.deepcopy(layer_config) or {}- ignore_layer_patterns = set()- if ignore_layers:- ignore_layers = ignore_layers.replace(" ", "").split(",")- ignore_layers = [name + "." if name[-1].isdigit() else name for name in ignore_layers]- ignore_layer_patterns = set(ignore_layers)-- # 1. ignore_layers -> force 16- for name in get_fp_layer_names(model, ignore_layers):- layer_config[name] = {- "bits": 16,- "act_bits": 16,- "data_type": "float",- "act_data_type": "float",- "fixed_by_user": True,- }-- # 2. normalize- layer_config = {k: normalize_item(v, k) for k, v in layer_config.items()}-- # 3. infer missing bits- for cfg in layer_config.values():- if "data_type" in cfg and "bits" not in cfg:- if (b := infer_bits_by_data_type(cfg["data_type"])) is not None:- cfg["bits"] = b- if "act_data_type" in cfg and "act_bits" not in cfg:- if (b := infer_bits_by_data_type(cfg["act_data_type"])) is not None:- cfg["act_bits"] = b-- # 4. fill defaults- if isinstance(default_scheme, str):- default_dict = asdict(preset_name_to_scheme(default_scheme.upper()))- else:- default_dict = asdict(default_scheme)- default_dict["scale_dtype"] = default_scale_dtype-- # In AutoScheme with mixed gguf:q4_k_m, the super_group_size of gguf:q8_0 layer is None,- # which should not be filled by default q4km again- for cfg in layer_config.values():- for key in scheme_keys:- if fill_default_value:- cfg.setdefault(key, copy.deepcopy(default_dict.get(key)))- else:- if key in extra_scheme_keys:- cfg.setdefault(key, copy.deepcopy(default_dict.get(key)))- else:- cfg.setdefault(key, None)-- # 5. collect supported modules- embedding_types = (torch.nn.Embedding,)- gguf_name = gguf_format_name or get_gguf_scheme(default_scheme)- if gguf_name:- if torch.nn.Embedding not in supported_types:- supported_types = (*supported_types, torch.nn.Embedding)-- # for some Embedding which type() is not torch.nn.Embedding- # for example: transformers.models.gemma3.modeling_gemma3.Gemma3TextScaledWordEmbedding- model_module_name = model.__class__.__module__- module_cls = sys.modules[model_module_name]- for name in module_cls.__dict__:- if name.endswith("Embedding") and not name.endswith("RotaryEmbedding"):- embedding_types = (*embedding_types, getattr(module_cls, name))- supported_types = (*supported_types, *embedding_types)-- all_supported_layer_names, embedding_layer_names = [], []- all_module_names = []- for n, m in model.named_modules():- all_module_names.append(n)- # cleanup stale attributes- for key in scheme_keys:- # `rotation_config` on the root model carries the active- # Hadamard rotation state (weights + hooks)- if n == "" and key == "rotation_config":- continue- if hasattr(m, key):- delattr(m, key)- if type(m) not in supported_types and m.__class__.__name__ not in inner_supported_types:- continue- all_supported_layer_names.append(n)- if isinstance(m, embedding_types) or m.__class__.__name__.endswith("Embedding"):- embedding_layer_names.append(n)-- # Also include layer names from safetensor files not loaded into the model- # (e.g. MTP layers that transformers does not instantiate).- safetensor_only_names = _get_safetensor_layer_names_not_in_model(model, all_module_names)-- # 6. expand regex configs- regex_config = {}- for name in list(layer_config.keys()):- if name in all_supported_layer_names:- continue- if name in all_module_names:- m = get_module(model, name)- if len(list(m.children())) == 0 and type(m) not in supported_types:- val = layer_config.pop(name)- if name in ignore_layer_patterns:- # Keep unsupported ignore_layers entries so export can serialize- # them into regex-based extra_config for loaders like vLLM INC.- regex_config[name] = val- else:- logger.warning(- f"'{name}' exists in the model but is not a supported quantization target "- f"in the current scheme, ignoring its setting in `layer_config`"- )- continue-- regex = re.compile(to_standard_regex(name))- matched = [ln for ln in all_supported_layer_names if regex.search(ln)]- safetensor_only_matched = [ln for ln in safetensor_only_names if regex.search(ln)]- # skip it for mtp layers not loaded in transformers- if not matched and not safetensor_only_matched:- # type(mlp.gate) is Qwen3VLMoeTextTopKRouter instead of Linear- logger.warning_once(- f"Layer name or regex '{name}' in layer_config does not match any supported layers. "- + "Please check for typos or update the regex pattern, ignore it for now"- )- val = layer_config.pop(name)- regex_config[name] = val # keep regex config- for match in matched:- layer_config[match] = val-- # 7. lm_head- lm_head_name = get_lm_head_name(model)- tie_word_embeddings = False- if hasattr(model, "config") and hasattr(model.config, "tie_word_embeddings"):- tie_word_embeddings = model.config.tie_word_embeddings-- if lm_head_name in layer_config:- quant_lm_head = True-- if quant_lm_head and tie_word_embeddings and not gguf_name:- quant_lm_head = False- logger.warning(- "reset `quant_lm_head` to false as quantizing " "lm_head with tied weights has not been supported currently"- )-- if lm_head_name not in layer_config and quant_lm_head:- layer_config[lm_head_name] = copy.deepcopy(default_dict)-- if not quant_lm_head and not gguf_name:- layer_config.pop(lm_head_name, None)-- # 8. enforce shape divisibility for int weight-only- if default_dict["data_type"] == "int" and default_dict["act_bits"] >= 16 and not gguf_name:- for n, m in model.named_modules():- if type(m) in supported_types or m.__class__.__name__ in inner_supported_types:- if m.weight.shape[0] % 32 or m.weight.shape[1] % 32:- layer_config.setdefault(n, copy.deepcopy(default_dict))- layer_config[n].update({"bits": 16, "data_type": "fp", "fixed_by_user": True})- # logger.warning_once(f"{n} skipped quantization (shape not divisible by 32).")- # enforce shape divisibility for mxfp/nvfp- if (is_nv_fp(default_dict["data_type"]) or is_mx_fp(default_dict["data_type"])) and not gguf_name:- skipped_layers = []- for n, m in model.named_modules():- if type(m) in supported_types or m.__class__.__name__ in inner_supported_types:- if m.weight.shape[1] % default_dict["group_size"]:- layer_config.setdefault(n, copy.deepcopy(default_dict))- layer_config[n].update(- {"bits": 16, "data_type": "fp", "act_bits": 16, "act_data_type": "fp", "fixed_by_user": True}- )- skipped_layers.append(n)-- compressed_skipped_layers = compress_layer_names(skipped_layers)- if compressed_skipped_layers:- logger.warning_once(- f"some layers are skipped quantization (shape not divisible by {default_dict['group_size']}): "- f"{compressed_skipped_layers}"- )-- # 9. block layers: mark as in_blocks=True- for name in get_layer_names_in_block(model, supported_types, quant_block_list, inner_supported_types):- if name not in layer_config:- layer_config[name] = copy.deepcopy(default_dict)- layer_config[name]["fixed_by_user"] = False- layer_config[name]["in_blocks"] = True-- # ---- restore: ensure missing in_blocks are set to False and compute flag ----- has_qlayer_outside_block = False- for cfg in layer_config.values():- if "in_blocks" not in cfg:- cfg["in_blocks"] = False- # mark layer outside block- if not cfg["in_blocks"] and check_to_quantized(cfg):- has_qlayer_outside_block = True-- # 10. GGUF handling- if not gguf_name:- dispatch_layer_config(layer_config)- return layer_config, has_qlayer_outside_block, regex_config-- # embed + lm_head defaults for gguf- tie_word_embeddings &= not is_separate_lm_head(model)- if lm_head_name not in layer_config and not tie_word_embeddings:- cfg = GGUF_INNER_CONFIG[GGUF_CONFIG[gguf_name.lower()]["lm_head"]]- cfg = {**cfg, "fixed_by_user": False, "scale_dtype": default_scale_dtype}- layer_config[lm_head_name] = cfg- has_qlayer_outside_block = True- for emd_name in embedding_layer_names:- if emd_name in layer_config:- continue- if not tie_word_embeddings:- cfg = GGUF_INNER_CONFIG[GGUF_CONFIG[gguf_name.lower()]["embedding"]]- else:- cfg = GGUF_INNER_CONFIG[GGUF_CONFIG[gguf_name.lower()]["lm_head"]]- cfg = {**cfg, "fixed_by_user": False, "scale_dtype": default_scale_dtype}- layer_config[emd_name] = cfg-- if enable_gguf_official_mixed:- model_type = ModelType.MMPROJ if is_mllm else ModelType.TEXT- layer_config, _ = get_layer_config_by_gguf_format(layer_config, gguf_name.lower(), model, model_type)-- _apply_gguf_shape_fallback(layer_config, model)- dispatch_layer_config(layer_config)- return layer_config, has_qlayer_outside_block, regex_config---def _use_more_bits(i_layer: int, n_layer: int):- return (i_layer < n_layer // 8) or (i_layer >= 7 * n_layer // 8) or ((i_layer - n_layer // 8) % 3 == 2)---def _search_gguf_type(gguf_type):- if gguf_type in GGUF_INNER_CONFIG:- return gguf_type- pattern = re.compile("gguf:q([0-9]{1,})_[01k]")- bits = re.search(pattern, gguf_type)- if not bits:- raise KeyError(f"{gguf_type} is not a correct gguf type, please check")-- for suffix in ["_k", "_0", "_1"]:- if gguf_type.endswith(suffix):- continue- if (tmp_type := re.sub("_[01k]", suffix, gguf_type)) in GGUF_INNER_CONFIG:- return tmp_type- return None---def gguf_type_fallback(gguf_type: str) -> str:- gguf_type = gguf_type.lower()- if gguf_type in ("gguf:q2_k", "gguf:q3_k", "gguf:q4_k"):- gguf_type = "gguf:q5_0"- elif gguf_type == "gguf:q5_k":- gguf_type = "gguf:q5_0"- elif gguf_type == "gguf:q6_k":- gguf_type = "gguf:q8_0"- return gguf_type---def get_gguf_qtype_by_layer_config(layer_config):- import gguf # pylint: disable=E0401-- if layer_config["bits"] >= 16:- return None- bits = layer_config["bits"]- super_bits = layer_config.get("super_bits", None)- sym = layer_config["sym"]- group_size = layer_config.get("group_size", None)- super_group_size = layer_config.get("super_group_size", None)- if bits == 2 and super_bits == 4 and not sym and group_size == 16 and super_group_size == 16:- return gguf.GGMLQuantizationType.Q2_K- if bits == 3 and super_bits == 6 and sym and group_size == 16 and super_group_size == 16:- return gguf.GGMLQuantizationType.Q3_K- if bits == 4:- if super_bits is not None and super_bits == 6 and not sym and group_size == 32 and super_group_size == 8:- return gguf.GGMLQuantizationType.Q4_K- if super_bits is None and sym and group_size == 32:- return gguf.GGMLQuantizationType.Q4_0- if super_bits is None and not sym and group_size == 32:- return gguf.GGMLQuantizationType.Q4_1- if bits == 5:- if super_bits == 6 and not sym and group_size == 32 and super_group_size == 8:- return gguf.GGMLQuantizationType.Q5_K- if super_bits is None and sym and group_size == 32:- return gguf.GGMLQuantizationType.Q5_0- if super_bits is None and not sym and group_size == 32:- return gguf.GGMLQuantizationType.Q5_1- if bits == 6 and super_bits == 8 and group_size == 16 and super_group_size == 16:- return gguf.GGMLQuantizationType.Q6_K- if bits == 8 and sym and group_size == 32:- return gguf.GGMLQuantizationType.Q8_0- raise ValueError("Unknown layer config")---def _apply_gguf_shape_fallback(layer_config, model):- from auto_round.utils.model import get_module-- for layer_name, config in layer_config.items():- if not check_to_quantized(config):- continue- layer = get_module(model, layer_name)- if layer is None or not hasattr(layer, "weight"):- continue- try:- qtype = get_gguf_qtype_by_layer_config(config)- except ValueError:- continue- if qtype is None:- continue-- gguf_type = f"gguf:{qtype.name.lower()}"- block_size = GGML_QUANT_SIZES[gguf_type.split(":")[-1]][0]- input_features = (- layer.weight.shape[0] if type(layer) == transformers.pytorch_utils.Conv1D else layer.weight.shape[-1]- )- if input_features % block_size == 0:- continue-- fallback_type = _gguf_type_fallback(gguf_type)- fallback_block_size = GGML_QUANT_SIZES[fallback_type.split(":")[-1]][0]- if input_features % fallback_block_size != 0:- fallback_type = "gguf:bf16"-- preserved = {key: config[key] for key in ("fixed_by_user", "scale_dtype") if key in config}- config.update(GGUF_INNER_CONFIG[fallback_type])- config.update(preserved)- logger.warning(- "fallback %s to %s before quantization, because input_features(%s) is not divisible by %s block_size(%s)",- layer_name,- fallback_type,- input_features,- gguf_type,- block_size,+ """Compatibility adapter for the pure layer-config resolver and explicit apply phase."""+ from auto_round.compressors.config_resolution import (+ ResolvedQuantizationConfig,+ ResolvedScheme,+ resolve_scheme_value,+ )+ from auto_round.compressors.layer_config import (+ apply_plan_to_model,+ extract_regex_config,+ has_quantized_layer_outside_blocks,+ resolve_layer_config,+ )+ from auto_round.schemes import get_gguf_scheme++ if isinstance(default_scheme, ResolvedScheme):+ resolved_scheme = default_scheme+ elif isinstance(default_scheme, QuantizationScheme):+ resolved_scheme = ResolvedScheme.from_scheme(+ default_scheme,+ preset_name=get_gguf_scheme(default_scheme) or None,)---def _get_digital_in_layer_name(layer_name):- pattern = re.compile(r"([a-zA-Z]+\.){1,}(\d+)")- res = re.search(pattern, layer_name)- if res:- return int(res[2])else:- return None---def _gguf_type_fallback(gguf_type: str) -> str:- gguf_type = gguf_type.lower()- if gguf_type in ("gguf:q2_k", "gguf:q3_k", "gguf:q4_k"):- gguf_type = "gguf:q5_0"- elif gguf_type == "gguf:q5_k":- gguf_type = "gguf:q5_0"- elif gguf_type == "gguf:q6_k":- gguf_type = "gguf:q8_0"- return gguf_type-+ resolved_scheme = resolve_scheme_value(default_scheme, {})-def _infer_gguf_n_layers_from_model(model, model_type=ModelType.TEXT):- if model is None or not hasattr(model, "named_modules"):- return None, None-- from auto_round.utils.common import MM_MODULE_KEYS-- text_layer_ids = set()- vision_layer_ids = set()- layer_pattern = re.compile(r"(?:^|\.)layers\.(\d+)(?:\.|$)")- for module_name, _ in model.named_modules():- match = layer_pattern.search(module_name)- if match is None:- continue- layer_id = int(match.group(1))- if model_type != ModelType.TEXT and any(key in module_name.lower() for key in MM_MODULE_KEYS):- vision_layer_ids.add(layer_id)- else:- text_layer_ids.add(layer_id)-- n_layer = max(text_layer_ids) + 1 if text_layer_ids else None- n_layer_vision = max(vision_layer_ids) + 1 if vision_layer_ids else None- return n_layer, n_layer_vision---def _resolve_gguf_n_layers(config, model_type=ModelType.TEXT, model=None):- """Resolve text (and vision) layer counts from config, then model structure.-- Multimodal configs (e.g. Qwen3.5, Qwen3-VL) keep the text hparams only in- ``config.text_config``, so both it and the top-level config must be checked.- If config naming changes, fall back to the loaded module hierarchy so GGUF- official mixed formats still receive a usable block count.- """- n_layer = None- n_layer_vision = None- text_config = getattr(config, "text_config", None)- vision_config = None- audio_config = getattr(config, "audio_config", None)- for name in ["n_layers", "num_hidden_layers", "n_layer", "num_layers", "depth"]:- if hasattr(config, name):- n_layer = getattr(config, name)- if text_config is not None and hasattr(text_config, name):- # the text hparams in text_config are authoritative for multimodal models;- # for text-only usage they still fill in when the top-level config lacks them- if model_type != ModelType.TEXT or n_layer is None:- n_layer = getattr(text_config, name)- if model_type != ModelType.TEXT:- for config_name in ["vision_config", "vision_encoder"]:- if hasattr(config, config_name):- vision_config = getattr(config, config_name)- if hasattr(vision_config, name):- n_layer_vision = getattr(vision_config, name)- break- if n_layer and n_layer_vision:- break-- inferred_n_layer, inferred_n_layer_vision = _infer_gguf_n_layers_from_model(model, model_type)- n_layer = n_layer or inferred_n_layer- n_layer_vision = n_layer_vision or inferred_n_layer_vision-- if (- model_type != ModelType.TEXT- and n_layer_vision is None- and vision_config is not None- and audio_config is not None- ):- # llama.cpp's Gemma4 mmproj converter uses a fixed tensor-name map size- # when vision and audio encoders coexist, because the two encoders do not- # expose a single transformer block count in config.json.- n_layer_vision = 128- return n_layer, n_layer_vision---##https://github.com/ggml-org/llama.cpp/blob/9e31bec4fd53634c9e5b04650488a09a055f5dab/src/llama-quant.cpp#L129-def get_layer_config_by_gguf_format(layer_config, target_gguf_format: str, model, model_type=ModelType.TEXT):- # # TODO: support for other format later- # target_gguf_format = next((fmt for fmt in gguf_format if fmt != "fake"), None)-- import gguf # pylint: disable=E0401-- from auto_round.export.export_to_gguf.llama_cpp_conversion import get_conversion- from auto_round.schemes import QuantizationScheme, get_gguf_scheme- from auto_round.utils.common import MM_MODULE_KEYS- from auto_round.utils.model import get_lm_head_name, get_module, is_separate_lm_head-- try:- hparams = model.config.to_dict()- conversion = get_conversion(hparams=hparams, model_type=model_type)- model_architecture = conversion.get_model_architecture(- hparams=hparams, model_type=conversion.model_type(model_type)- )- except AttributeError as e:- raise ImportError(- "Please use the latest gguf-py, you can use the following command to install it:\n"- "git clone https://github.com/ggml-org/llama.cpp.git && cd llama.cpp/gguf-py"- " && pip install . sentencepiece"- )- try:- if model_type != ModelType.TEXT:- model_class_vision = conversion.get_model_class(model_architecture, model_type=model_type)- model_class = conversion.get_model_class(model_architecture, model_type=ModelType.TEXT)-- except NotImplementedError:- return layer_config, {}-- n_layer, n_layer_vision = _resolve_gguf_n_layers(model.config, model_type, model=model)-- if n_layer is None:- return layer_config, {}-- tensor_map = gguf.get_tensor_name_map(model_class.model_arch, n_layer)- if model_type != ModelType.TEXT:- tensor_map_vision = gguf.get_tensor_name_map(model_class_vision.model_arch, n_layer_vision)-- def _set_config(config, target_config):- for k, v in target_config.items():- if isinstance(config, dict):- config[k] = v- else:- setattr(config, k, v)- return config-- gguf_format_config = {}- lm_head_name = get_lm_head_name(model)- inner_gguf_format = GGUF_CONFIG[target_gguf_format]["mostly"]- # ggml_type = getattr(gguf.GGMLQuantizationType,inner_gguf_format.split(":")[-1].upper())- block_size = GGML_QUANT_SIZES[inner_gguf_format.split(":")[-1].lower()][0]- tie_word_embeddings = True- if hasattr(model, "config") and hasattr(model.config, "tie_word_embeddings"):- tie_word_embeddings = model.config.tie_word_embeddings- tie_word_embeddings &= not is_separate_lm_head(model)-- dtype_selector = GGUFDTypeSelector(- hparams, gguf_format_to_ftype(target_gguf_format), model_class.model_arch, n_layer+ resolved = resolve_layer_config(+ model=model,+ scheme=resolved_scheme,+ layer_config=layer_config,+ scale_dtype=default_scale_dtype,+ supported_types=supported_types,+ inner_supported_types=inner_supported_types,+ quant_block_list=quant_block_list,+ ignore_layers=ignore_layers,+ quant_lm_head=quant_lm_head,+ enable_gguf_official_mixed=enable_gguf_official_mixed,+ is_mllm=is_mllm,+ fill_default_value=fill_default_value,)- layer_config_copy = copy.deepcopy(layer_config)- base_target_bits = None- if inner_gguf_format.startswith("gguf:q") and len(inner_gguf_format) >= 7 and (inner_gguf_format[6]).isdigit():- base_target_bits = int(inner_gguf_format[6])-- def _resolve_gguf_name(layer_name):- if model_type != ModelType.TEXT and any([key in layer_name for key in MM_MODULE_KEYS]):- gguf_layer_name = tensor_map_vision.get_name(layer_name)- if gguf_layer_name is None:- for key in MM_MODULE_KEYS:- gguf_layer_name = tensor_map_vision.get_name(layer_name.replace(f".{key}", ""))- if gguf_layer_name is not None:- break- else:- gguf_layer_name = tensor_map.get_name(layer_name)- if gguf_layer_name is None:- gguf_layer_name = tensor_map.get_name(layer_name.replace(".language_model", ""))- return gguf_layer_name-- dtype_selector.n_attention_wv = sum(- 1- for layer_name, config in layer_config_copy.items()- if check_to_quantized(config)- and (gguf_name := _resolve_gguf_name(layer_name))- and any(key in gguf_name for key in ("attn_v", "attn_qkv", "attn_kv_b"))+ regex_config = extract_regex_config(+ model=model,+ scheme=resolved_scheme,+ layer_config=layer_config,+ scale_dtype=default_scale_dtype,+ supported_types=supported_types,+ inner_supported_types=inner_supported_types,+ ignore_layers=ignore_layers,+ fill_default_value=fill_default_value,+ )+ has_outside = has_quantized_layer_outside_blocks(resolved)+ plan = ResolvedQuantizationConfig(+ scheme=resolved_scheme,+ formats=(),+ layer_config=resolved,+ regex_config=regex_config,+ has_qlayer_outside_block=has_outside,+ scale_dtype=default_scale_dtype,+ quant_block_list=quant_block_list,+ )+ apply_plan_to_model(model, plan)+ return (+ {name: dict(config) for name, config in plan.layer_config.items()},+ plan.has_qlayer_outside_block,+ {name: dict(config) for name, config in plan.regex_config.items()},)- for layer_name, config in layer_config_copy.items():- if not check_to_quantized(config):- continue- # Reset target_bits each iteration to prevent lm_head/embedding settings- # from bleeding into subsequent block layers and bypassing their special logic.- target_bits = base_target_bits- new_type = GGUF_CONFIG[target_gguf_format]["mostly"]- layer = get_module(model, layer_name)- if type(layer) == transformers.pytorch_utils.Conv1D:- input_features = layer.weight.shape[0]- else:- input_features = layer.weight.shape[-1]- i_layer = _get_digital_in_layer_name(layer_name)-- if lm_head_name is not None and layer_name == lm_head_name:- target_bits = int(re.search("gguf:q([0-9]{1,})_[01k]", GGUF_CONFIG[target_gguf_format]["lm_head"]).group(1))- if isinstance(layer, torch.nn.Embedding):- embedding_format_key = "lm_head" if tie_word_embeddings else "embedding"- target_bits = int(- re.search("gguf:q([0-9]{1,})_[01k]", GGUF_CONFIG[target_gguf_format][embedding_format_key]).group(1)- )-- gguf_name = _resolve_gguf_name(layer_name)- bits_index = 6- if config.get("fixed_by_user", False):- if "bits" not in config:- logger.warning(- f"Setting layer_config requires providing bits, {layer_name} has not bits,"- f" using bits={target_bits} instead."- )- new_type = new_type[:bits_index] + str(target_bits) + new_type[bits_index + 1 :]- else:- config_tmp = config.copy()- scheme_keys = [f.name for f in fields(QuantizationScheme)]- for key in config.keys():- if key not in scheme_keys:- config_tmp.pop(key, None)- matched_scheme = get_gguf_scheme(QuantizationScheme.from_dict(config_tmp)) # check matched- if not matched_scheme:- if config.get("super_group_size", None) is not None or config.get("super_bits", None) is not None:- new_type = new_type[:bits_index] + str(config["bits"]) + "_k"- if new_type not in GGUF_INNER_CONFIG:- prefix_idx = 0 if config.get("sym", True) else 1- new_type = new_type[:bits_index] + str(config["bits"]) + f"_{prefix_idx}"- if new_type not in GGUF_INNER_CONFIG:- new_type = new_type[:bits_index] + str(config["bits"]) + f"_{1-prefix_idx}"- if new_type not in GGUF_INNER_CONFIG:- raise ValueError(- f"the setting in layer_config {layer_name} "- f"could not match any supported gguf format, please have a check."- )-- new_type = new_type[:bits_index] + str(config["bits"]) + new_type[bits_index + 1 :]- new_type = _search_gguf_type(new_type)- if new_type is None:- raise ValueError(f"invalid bit setting for {layer_name}")- elif lm_head_name is not None and layer_name == lm_head_name and not tie_word_embeddings:- if gguf.MODEL_ARCH.FALCON == model_class.model_arch or input_features % block_size != 0:- new_type = "gguf:q8_0"- elif "lm_head" in GGUF_CONFIG[target_gguf_format]:- new_type = GGUF_CONFIG[target_gguf_format]["lm_head"]- elif new_type != "gguf:q8_0":- new_type = "gguf:q6_k"- elif lm_head_name is not None and layer_name == lm_head_name and tie_word_embeddings:- continue- elif isinstance(layer, torch.nn.Embedding):- embedding_format_key = "lm_head" if tie_word_embeddings else "embedding"- if embedding_format_key in GGUF_CONFIG[target_gguf_format]:- new_type = GGUF_CONFIG[target_gguf_format][embedding_format_key]- elif target_bits is not None and "bits" in config and config["bits"] != target_bits:- new_type = new_type[:bits_index] + str(config["bits"]) + new_type[bits_index + 1 :]- new_type = _search_gguf_type(new_type)- if new_type is None:- raise ValueError(f"invalid bit setting for {layer_name}")- elif gguf_name is not None:- gguf_weight_name = gguf_name if gguf_name.endswith(".weight") else f"{gguf_name}.weight"- new_type = dtype_selector.select_gguf_type(gguf_weight_name, len(layer.weight.shape), i_layer or 0)- new_block_size = GGML_QUANT_SIZES[new_type.split(":")[-1].lower()][0]- if input_features % new_block_size != 0:- new_type = _gguf_type_fallback(new_type)- new_block_size = GGML_QUANT_SIZES[new_type.split(":")[-1].lower()][0]- if input_features % new_block_size != 0:- new_type = "gguf:bf16"- logger.warning(- f"fallback {layer_name} to {new_type}, "- f"because input_features({input_features}) % block_size({block_size}) != 0"- )- # for deepseek v2- if layer_name.endswith("kv_b_proj") and new_type.endswith("_k") and "Deepseek" in model.config.architectures[0]:- fallback = False-- # calc if need fallback- qk_nope_head_dim = model.config.qk_nope_head_dim- kv_b_shape = get_module(model, layer_name).weight.shape-- if (- qk_nope_head_dim < QK_K- or qk_nope_head_dim % QK_K != 0- or kv_b_shape[-1] < QK_K- or kv_b_shape[-1] % QK_K != 0- ):- fallback = True- if fallback:- tmp_type = _gguf_type_fallback(new_type)- logger.warning_once(- f"self_attn.kv_b_proj does not support the use of {new_type}, replace it with {tmp_type}"- )- new_type = tmp_type-- target_config = GGUF_INNER_CONFIG[new_type]-- _set_config(layer_config[layer_name], target_config)- _set_config(layer, target_config)- gguf_format_config[layer_name] = new_type-- return layer_config, gguf_format_config---def get_fp_layer_names(model: torch.nn.Module, ignore_layers: str):- """Identifies and returns layers in the model to exclude from quantization.-- This function processes a comma-separated list of fully precision (FP) layers,- matches them to the names of layers in the model, and returns a list of such- layers to exclude from quantization.-- Args:- model (torch.nn.Module): The model whose layers will be inspected.- ignore_layers (str): A comma-separated string of layer names to be excluded- from quantization. Whitespace is ignored in this string.-- Returns:- list: A list of layer names that match the specified FP layers or are- subcomponents of those layers.- """- from auto_round.utils import SUPPORTED_LAYER_TYPES-- if not ignore_layers:- return []- all_layer_names = []- for n, m in model.named_modules():- if type(m) in SUPPORTED_LAYER_TYPES:- all_layer_names.append(n)- not_to_quantized_layers = []+from auto_round.compressors.layer_config.resolver import get_fp_layer_names- for fp_layer in ignore_layers:- if fp_layer == "":- continue- if fp_layer in all_layer_names:- not_to_quantized_layers.append(fp_layer)- continue- for name in all_layer_names:- if fp_layer in name:- not_to_quantized_layers.append(name)- not_to_quantized_layers.extend(ignore_layers) # keep regex name for later use- if not_to_quantized_layers:- logger.info(f"Ignored layers: {compress_layer_names(not_to_quantized_layers)}")- return not_to_quantized_layers+# Explicit compatibility exports for callers that historically imported GGUF and+# ignore-layer helpers from compressors.utils.+from auto_round.export.formats.backends.gguf import (+ _apply_gguf_shape_fallback,+ _infer_gguf_n_layers_from_model,+ _resolve_gguf_n_layers,+ get_layer_config_by_gguf_format,+ gguf_type_fallback,+)def get_shared_keys(model):@@ -1190,8 +455,7 @@ def _get_quantized_layer_names_outside_blocks(model, layer_config, supported_typcontinuelayer = get_module(model, key)if layer is None:- logger.error(f"could not find layer {key} in the model, exit...")- exit(-1)+ raise LayerConfigResolutionError(f"could not find layer '{key}' in the model")if type(layer) in supported_types and check_to_quantized(layer_config[key]):layer_names.append(key)diff --git a/auto_round/data_type/nvfp.py b/auto_round/data_type/nvfp.pyindex f9f8a88fe..bab5b5410 100644--- a/auto_round/data_type/nvfp.py+++ b/auto_round/data_type/nvfp.py@@ -393,7 +393,7 @@ def opt_rtn_fast_nvfp4(**kwargs,):tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)- if imatrix is None:+ if not isinstance(imatrix, torch.Tensor):qw = 1.0else:imatrix = imatrix.reshape(1, -1)diff --git a/auto_round/data_type/utils.py b/auto_round/data_type/utils.pyindex 4bbc35179..7e4be30db 100644--- a/auto_round/data_type/utils.py+++ b/auto_round/data_type/utils.py@@ -248,8 +248,8 @@ def search_optimized_init_scale(search_fn, _ = _resolve_optimized_dtype_funcs(data_type, q_scale_thresh)if search_fn is None:return None- if imatrix is None:- imatrix = 1.0+ if imatrix is None or not isinstance(imatrix, torch.Tensor):+ imatrix = torch.ones_like(weight_reshape)return search_fn(weight_reshape, bits, imatrix)@@ -270,10 +270,11 @@ def reshape_imatrix_for_weight(imatrix, weight_reshape: torch.Tensor, group_sizeEncapsulates the imatrix grouping logic shared by the SignRound optimizedwrapper and AWQ's internal QDQ so callers never handle the low-level reshape.- Returns the scalar ``1.0`` when no imatrix is available (uniform importance).+ Returns a tensor of ones when no imatrix is available (uniform importance),+ keeping every downstream optimized dtype implementation on its tensor path."""- if imatrix is None:- return 1.0+ if imatrix is None or not isinstance(imatrix, torch.Tensor):+ return torch.ones_like(weight_reshape)imatrix = imatrix.reshape(1, -1)imatrix = reshape_pad_tensor_by_group_size(imatrix, group_size, val=1e-5)[0].view(1, -1)imatrix = imatrix.expand(weight_reshape.numel() // imatrix.numel(), -1)diff --git a/auto_round/export/export_to_autoround/export.py b/auto_round/export/export_to_autoround/export.pyindex 4b9c9e30e..fd02d98d9 100644--- a/auto_round/export/export_to_autoround/export.py+++ b/auto_round/export/export_to_autoround/export.py@@ -28,6 +28,7 @@ from tqdm import tqdmfrom auto_round.compressors.utils import is_mx_fp, is_nv_fp, is_standard_fpfrom auto_round.export.export_to_autoround.utils import check_neq_config+from auto_round.export.formats import BackendDataTypefrom auto_round.export.utils import (filter_quantization_config,get_autogptq_packing_qlinear,@@ -37,7 +38,6 @@ from auto_round.export.utils import (save_model,save_pretrained_artifact,)-from auto_round.formats import AutoRoundExportFormatfrom auto_round.logger import loggerfrom auto_round.schemes import QuantizationSchemefrom auto_round.utils import (@@ -275,7 +275,7 @@ def save_quantized_as_autoround(if ((serialization_dict.get("sym") is None or serialization_dict.get("sym"))and ("gptq" not in backend and "awq" not in backend)- and (AutoRoundExportFormat.FP8_STATIC.value not in backend)+ and (BackendDataType.FP8_STATIC.value not in backend)):backend = backend.replace("auto_round", "auto_round:auto_gptq")diff --git a/auto_round/export/export_to_gguf/config.py b/auto_round/export/export_to_gguf/config.pyindex 56f2af953..f302949cd 100644--- a/auto_round/export/export_to_gguf/config.py+++ b/auto_round/export/export_to_gguf/config.py@@ -11,189 +11,42 @@# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.# See the License for the specific language governing permissions and# limitations under the License.+import copyfrom enum import IntEnum+from auto_round.schemes import GGUF_PRESET_ALIASES, GGUF_SCHEME_FACTS+class ModelType(IntEnum):TEXT = 1MMPROJ = 2-GGUF_INNER_CONFIG = {}--GGUF_INNER_CONFIG["gguf:q4_0"] = {- "bits": 4,- "act_bits": 16,- "group_size": 32,- "sym": True,- "data_type": "int",- "embedding": "gguf:q4_0",- "lm_head": "gguf:q6_k",- "super_bits": None,- "super_group_size": None,-}--GGUF_INNER_CONFIG["gguf:q4_1"] = {- "bits": 4,- "act_bits": 16,- "group_size": 32,- "sym": False,- "data_type": "int_asym_float_zp",- "embedding": "gguf:q4_1",- "lm_head": "gguf:q6_k",- "super_bits": None,- "super_group_size": None,-}--GGUF_INNER_CONFIG["gguf:q5_0"] = {- "bits": 5,- "act_bits": 16,- "group_size": 32,- "sym": True,- "data_type": "int",- "embedding": "gguf:q5_0",- "lm_head": "gguf:q6_k",- "super_bits": None,- "super_group_size": None,-}--GGUF_INNER_CONFIG["gguf:q5_1"] = {- "bits": 5,- "act_bits": 16,- "group_size": 32,- "sym": False,- "data_type": "int_asym_float_zp",- "embedding": "gguf:q5_1",- "lm_head": "gguf:q6_k",- "super_bits": None,- "super_group_size": None,+_GGUF_FORMAT_FIELDS = {+ "gguf:q4_0": {"embedding": "gguf:q4_0", "lm_head": "gguf:q6_k"},+ "gguf:q4_1": {"embedding": "gguf:q4_1", "lm_head": "gguf:q6_k"},+ "gguf:q5_0": {"embedding": "gguf:q5_0", "lm_head": "gguf:q6_k"},+ "gguf:q5_1": {"embedding": "gguf:q5_1", "lm_head": "gguf:q6_k"},+ "gguf:q8_0": {"embedding": "gguf:q8_0", "lm_head": "gguf:q8_0"},+ "gguf:q2_k": {"embedding": "gguf:q2_k", "lm_head": "gguf:q6_k"},+ "gguf:q3_k": {"embedding": "gguf:q3_k", "lm_head": "gguf:q6_k"},+ "gguf:q4_k": {"embedding": "gguf:q4_k", "lm_head": "gguf:q6_k"},+ "gguf:q5_k": {"embedding": "gguf:q5_k", "lm_head": "gguf:q6_k"},+ "gguf:q6_k": {"embedding": "gguf:q6_k", "lm_head": "gguf:q6_k"},+ "gguf:bf16": {"embedding": None, "lm_head": None},}+_GGUF_FORMAT_FIELDS["gguf:fp16"] = _GGUF_FORMAT_FIELDS["gguf:bf16"]-GGUF_INNER_CONFIG["gguf:q8_0"] = {- "bits": 8,- "act_bits": 16,- "group_size": 32,- "sym": True,- "data_type": "int",- "embedding": "gguf:q8_0",- "lm_head": "gguf:q8_0",- "super_bits": None,- "super_group_size": None,-}--GGUF_INNER_CONFIG["gguf:q2_k"] = {- "bits": 2,- "act_bits": 16,- "super_group_size": 16,- "super_bits": 4,- "group_size": 16,- "sym": False,- "data_type": "int_asym_dq",- "embedding": "gguf:q2_k",- "lm_head": "gguf:q6_k",-}--GGUF_INNER_CONFIG["gguf:q3_k"] = {- "bits": 3,- "act_bits": 16,- "super_group_size": 16,- "super_bits": 6,- "group_size": 16,- "sym": True,- "data_type": "int_sym_dq",- "embedding": "gguf:q3_k",- "lm_head": "gguf:q6_k",-}--GGUF_INNER_CONFIG["gguf:q4_k"] = {- "bits": 4,- "act_bits": 16,- "super_group_size": 8,- "super_bits": 6,- "group_size": 32,- "sym": False,- "data_type": "int_asym_dq",- "embedding": "gguf:q4_k",- "lm_head": "gguf:q6_k",-}--GGUF_INNER_CONFIG["gguf:q5_k"] = {- "bits": 5,- "act_bits": 16,- "super_group_size": 8,- "super_bits": 6,- "group_size": 32,- "sym": False,- "data_type": "int_asym_dq",- "embedding": "gguf:q5_k",- "lm_head": "gguf:q6_k",-}--GGUF_INNER_CONFIG["gguf:q6_k"] = {- "bits": 6,- "act_bits": 16,- "super_group_size": 16,- "super_bits": 8,- "group_size": 16,- "sym": True,- "data_type": "int_sym_dq",- "embedding": "gguf:q6_k",- "lm_head": "gguf:q6_k",-}--GGUF_INNER_CONFIG["gguf:bf16"] = GGUF_INNER_CONFIG["gguf:fp16"] = {- "bits": 16,- "act_bits": 16,- "super_group_size": None,- "super_bits": None,- "group_size": None,- "sym": True,- "data_type": "int_sym_dq",- "embedding": None,- "lm_head": None,-}+GGUF_INNER_CONFIG = {}+for key, facts in GGUF_SCHEME_FACTS.items():+ GGUF_INNER_CONFIG[key] = {**copy.deepcopy(facts), **_GGUF_FORMAT_FIELDS[key]}GGUF_CONFIG = {}-GGUF_CONFIG["gguf:q4_0"] = GGUF_INNER_CONFIG["gguf:q4_0"]-GGUF_CONFIG["gguf:q4_0"]["mostly"] = "gguf:q4_0"-GGUF_CONFIG["gguf:q4_1"] = GGUF_INNER_CONFIG["gguf:q4_1"]-GGUF_CONFIG["gguf:q4_1"]["mostly"] = "gguf:q4_1"-GGUF_CONFIG["gguf:q5_0"] = GGUF_INNER_CONFIG["gguf:q5_0"]-GGUF_CONFIG["gguf:q5_0"]["mostly"] = "gguf:q5_0"-GGUF_CONFIG["gguf:q5_1"] = GGUF_INNER_CONFIG["gguf:q5_1"]-GGUF_CONFIG["gguf:q5_1"]["mostly"] = "gguf:q5_1"-GGUF_CONFIG["gguf:q2_k_s"] = GGUF_INNER_CONFIG["gguf:q2_k"]-GGUF_CONFIG["gguf:q2_k_s"]["mostly"] = "gguf:q2_k"-# GGUF_CONFIG["gguf:q3_k"] = GGUF_INNER_CONFIG["gguf:q3_k"]-# GGUF_CONFIG["gguf:q3_k"]["mostly"] = "gguf:q3_k"-GGUF_CONFIG["gguf:q3_k_s"] = GGUF_INNER_CONFIG["gguf:q3_k"]-GGUF_CONFIG["gguf:q3_k_s"]["mostly"] = "gguf:q3_k"-GGUF_CONFIG["gguf:q3_k_m"] = GGUF_INNER_CONFIG["gguf:q3_k"]-GGUF_CONFIG["gguf:q3_k_m"]["mostly"] = "gguf:q3_k"-GGUF_CONFIG["gguf:q3_k_l"] = GGUF_INNER_CONFIG["gguf:q3_k"]-GGUF_CONFIG["gguf:q3_k_l"]["mostly"] = "gguf:q3_k"-# GGUF_CONFIG["gguf:q4_k"] = GGUF_INNER_CONFIG["gguf:q4_k"]-# GGUF_CONFIG["gguf:q4_k"]["mostly"]= "gguf:q4_k"-GGUF_CONFIG["gguf:q4_k_s"] = GGUF_INNER_CONFIG["gguf:q4_k"]-GGUF_CONFIG["gguf:q4_k_s"]["mostly"] = "gguf:q4_k"-GGUF_CONFIG["gguf:q4_k_m"] = GGUF_INNER_CONFIG["gguf:q4_k"]-GGUF_CONFIG["gguf:q4_k_m"]["mostly"] = "gguf:q4_k"-# GGUF_CONFIG["gguf:q5_k"] = GGUF_INNER_CONFIG["gguf:q5_k"]-# GGUF_CONFIG["gguf:q5_k"]["mostly"]= "gguf:q5_k"-GGUF_CONFIG["gguf:q5_k_s"] = GGUF_INNER_CONFIG["gguf:q5_k"]-GGUF_CONFIG["gguf:q5_k_s"]["mostly"] = "gguf:q5_k"-GGUF_CONFIG["gguf:q5_k_m"] = GGUF_INNER_CONFIG["gguf:q5_k"]-GGUF_CONFIG["gguf:q5_k_m"]["mostly"] = "gguf:q5_k"-GGUF_CONFIG["gguf:q6_k"] = GGUF_INNER_CONFIG["gguf:q6_k"]-GGUF_CONFIG["gguf:q6_k"]["mostly"] = "gguf:q6_k"-GGUF_CONFIG["gguf:q8_0"] = GGUF_INNER_CONFIG["gguf:q8_0"]-GGUF_CONFIG["gguf:q8_0"]["mostly"] = "gguf:q8_0"-# GGUF_CONFIG["gguf:fp16"] = GGUF_INNER_CONFIG["gguf:fp16"]-# GGUF_CONFIG["gguf:fp16"]["mostly"]= "gguf:fp16"-# GGUF_CONFIG["gguf:bf16"] = GGUF_INNER_CONFIG["gguf:fp16"]-# GGUF_CONFIG["gguf:bf16"]["mostly"]= "gguf:bf16"-GGUF_CONFIG["gguf:q2_k_mixed"] = GGUF_INNER_CONFIG["gguf:q2_k"]+for alias_key, facts_key in GGUF_PRESET_ALIASES.items():+ GGUF_CONFIG[alias_key] = GGUF_INNER_CONFIG[facts_key]+ if alias_key != "gguf:q2_k_mixed":+ GGUF_CONFIG[alias_key]["mostly"] = facts_keyQK_K = 256diff --git a/auto_round/export/formats/__init__.py b/auto_round/export/formats/__init__.pynew file mode 100644index 000000000..d401af8c8--- /dev/null+++ b/auto_round/export/formats/__init__.py@@ -0,0 +1,43 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++from auto_round.export.formats.backends import (+ AutoAWQFormat,+ AutoGPTQFormat,+ AutoRoundFormat,+ FakeFormat,+ FP8Format,+ GGUFFormat,+ LLMCompressorFormat,+ MLXFormat,+)+from auto_round.export.formats.base import (+ BackendDataType,+ OutputFormat,+)+from auto_round.export.formats.resolver import resolve_formats++__all__ = [+ "BackendDataType",+ "AutoRoundFormat",+ "AutoAWQFormat",+ "AutoGPTQFormat",+ "FakeFormat",+ "FP8Format",+ "GGUFFormat",+ "LLMCompressorFormat",+ "MLXFormat",+ "OutputFormat",+ "resolve_formats",+]diff --git a/auto_round/export/formats/backends/__init__.py b/auto_round/export/formats/backends/__init__.pynew file mode 100644index 000000000..263ecbf1d--- /dev/null+++ b/auto_round/export/formats/backends/__init__.py@@ -0,0 +1,33 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++from auto_round.export.formats.backends.autoround import AutoRoundFormat+from auto_round.export.formats.backends.fake import FakeFormat+from auto_round.export.formats.backends.fp8 import FP8Format+from auto_round.export.formats.backends.gguf import GGUFFormat+from auto_round.export.formats.backends.auto_awq import AutoAWQFormat+from auto_round.export.formats.backends.auto_gptq import AutoGPTQFormat+from auto_round.export.formats.backends.llm_compressor import LLMCompressorFormat+from auto_round.export.formats.backends.mlx import MLXFormat++__all__ = [+ "AutoAWQFormat",+ "AutoGPTQFormat",+ "AutoRoundFormat",+ "FakeFormat",+ "FP8Format",+ "GGUFFormat",+ "LLMCompressorFormat",+ "MLXFormat",+]diff --git a/auto_round/export/formats/backends/auto_awq.py b/auto_round/export/formats/backends/auto_awq.pynew file mode 100644index 000000000..1ab8c2831--- /dev/null+++ b/auto_round/export/formats/backends/auto_awq.py@@ -0,0 +1,127 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++import re+from typing import Callable, Union++import torch+import transformers++from auto_round.export.formats.base import OutputFormat, _check_divisible_by_32+from auto_round.logger import logger+from auto_round.schemes import QuantizationScheme++[email protected]("auto_awq")+class AutoAWQFormat(OutputFormat):+ support_schemes = ["W4A16"]+ format_name = "auto_awq"++ @classmethod+ def check_scheme_args(cls, scheme: QuantizationScheme) -> bool:+ error_logs = []+ if scheme.bits != 4:+ error_logs.append(f"bits={scheme.bits}")+ if not re.search("int", scheme.data_type):+ error_logs.append(f"data_type={scheme.data_type}")+ if scheme.super_bits:+ error_logs.append(f"super_bits={scheme.super_bits}")+ if scheme.super_group_size:+ error_logs.append(f"super_group_size={scheme.super_group_size}")+ if error_logs:+ raise ValueError(+ f"{cls.format_name} format support quantization scheme with {','.join(cls.support_schemes)} "+ f"but got {', '.join(error_logs)}, please have a check."+ )+ return True++ @staticmethod+ def check_awq_gemm_compatibility(model, bits, group_size, sym, layer_configs=None):+ """Check whether a model is compatible with the AutoAWQ GEMM kernel."""+ from auto_round.utils.model import get_layer_names_in_block, get_module++ if bits != 4:+ return False, "AutoAWQ GEMM kernel only supports 4 bits"+ for _, module in model.named_modules():+ if type(module) == transformers.pytorch_utils.Conv1D:+ return False, "AutoAWQ GEMM kernel does not support conv1d"++ layer_names = get_layer_names_in_block(model)+ for layer_name in layer_names:+ if (+ layer_configs is not None+ and layer_name in layer_configs.keys()+ and layer_configs[layer_name].get("bits", bits) > 8+ ):+ continue++ layer = get_module(model, layer_name)+ if layer.in_features % group_size != 0:+ return False, f"Layer {layer_name} in_features is not multiple of group_size {group_size}"+ if layer.out_features % (32 // bits) != 0:+ return False, f"Layer {layer_name} out_features is not multiple of 32 // bits"++ return True, ""++ def check_and_reset_format(self, scheme: QuantizationScheme, ctx):+ awq_supported, info = self.check_awq_gemm_compatibility(+ ctx.model, scheme.bits, scheme.group_size, scheme.sym, ctx.layer_config+ )+ if not awq_supported:+ logger.warning(f"The AutoAWQ format may not be supported due to {info}")+ if scheme.bits != 4:+ raise ValueError(f"auto_awq format support quantization scheme with W4A16 but got bits={scheme.bits}")++ if self.backend is None:+ ctx.layer_config = _check_divisible_by_32(scheme, ctx.model, ctx.layer_config)++ return super().check_and_reset_format(scheme, ctx)++ def pack_layer(self, layer_name, model, device=None, **kwargs):+ from auto_round.export.export_to_awq.export import pack_layer++ pack_layer(layer_name, model, backend=self.output_format, device=device)++ def save_quantized(+ self,+ output_dir: str,+ model: torch.nn.Module = None,+ tokenizer: Callable = None,+ layer_config: dict = None,+ inplace: bool = True,+ device: Union[str, torch.device] = "cpu",+ serialization_dict: dict = None,+ **kwargs,+ ) -> torch.nn.Module:+ backend = self.get_backend_name()+ if backend == "auto_round:auto_awq":+ from auto_round.export.export_to_autoround.export import save_quantized_as_autoround++ export_func = save_quantized_as_autoround+ else:+ from auto_round.export.export_to_awq.export import save_quantized_as_autoawq++ export_func = save_quantized_as_autoawq++ return export_func(+ output_dir=output_dir,+ model=model,+ tokenizer=tokenizer,+ layer_config=layer_config,+ inplace=inplace,+ backend=backend,+ device=device,+ serialization_dict=serialization_dict,+ **kwargs,+ )diff --git a/auto_round/export/formats/backends/auto_gptq.py b/auto_round/export/formats/backends/auto_gptq.pynew file mode 100644index 000000000..4fcaaca61--- /dev/null+++ b/auto_round/export/formats/backends/auto_gptq.py@@ -0,0 +1,100 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++import re+from typing import Callable, Union++import torch++from auto_round.export.formats.base import OutputFormat, _check_divisible_by_32+from auto_round.logger import logger+from auto_round.schemes import QuantizationScheme++[email protected]("auto_gptq", "gptqmodel")+class AutoGPTQFormat(OutputFormat):+ support_schemes = ["W4A16", "W2A16", "W3A16", "W8A16", "BF16", "W2A16G64", "W2A16G32", "W4A16_MIXED"]+ format_name = "auto_gptq"++ def check_and_reset_format(self, scheme: QuantizationScheme, ctx):+ if not scheme.sym:+ logger.warning(+ "the asymmetrical kernel of the GPTQ format may result in a noticeable accuracy drop,"+ " particularly for 2-bit quantization and smaller models."+ " We recommend exporting to either the AutoAWQ format ( only 4 bits) or "+ "the AutoRound format(2/3/4/8 bits)."+ )+ if self.backend is None:+ ctx.layer_config = _check_divisible_by_32(scheme, ctx.model, ctx.layer_config)+ return super().check_and_reset_format(scheme, ctx)++ @classmethod+ def check_scheme_args(cls, scheme: QuantizationScheme) -> bool:+ error_logs = []+ if scheme.bits not in [2, 3, 4, 8, 16]:+ error_logs.append(f"bits={scheme.bits}")+ if not re.search("int", scheme.data_type):+ error_logs.append(f"data_type={scheme.data_type}")+ if scheme.super_bits:+ error_logs.append(f"super_bits={scheme.super_bits}")+ if scheme.super_group_size:+ error_logs.append(f"super_group_size={scheme.super_group_size}")+ if error_logs:+ raise ValueError(+ f"{cls.format_name} format support quantization scheme with {','.join(cls.support_schemes)} "+ f"but got {', '.join(error_logs)}, please have a check."+ )+ return True++ def pack_layer(self, layer_name, model, device=None, **kwargs):+ if self.output_format.startswith("auto_round"):+ from auto_round.export.export_to_autoround.export import pack_layer++ pack_layer(layer_name, model, backend=self.output_format, device=device)+ else:+ from auto_round.export.export_to_autogptq.export import pack_layer++ pack_layer(layer_name, model, backend=self.output_format, device=device)++ def save_quantized(+ self,+ output_dir: str,+ model: torch.nn.Module = None,+ tokenizer: Callable = None,+ layer_config: dict = None,+ inplace: bool = True,+ device: Union[str, torch.device] = "cpu",+ serialization_dict: dict = None,+ **kwargs,+ ) -> torch.nn.Module:+ backend = self.get_backend_name()+ if backend == "auto_round:auto_gptq" or backend == "auto_round:gptqmodel":+ from auto_round.export.export_to_autoround.export import save_quantized_as_autoround++ export_func = save_quantized_as_autoround+ else:+ from auto_round.export.export_to_autogptq.export import save_quantized_as_autogptq++ export_func = save_quantized_as_autogptq+ return export_func(+ output_dir=output_dir,+ model=model,+ tokenizer=tokenizer,+ layer_config=layer_config,+ inplace=inplace,+ device=device,+ backend=backend,+ serialization_dict=serialization_dict,+ **kwargs,+ )diff --git a/auto_round/export/formats/backends/autoround.py b/auto_round/export/formats/backends/autoround.pynew file mode 100644index 000000000..21ab8daaa--- /dev/null+++ b/auto_round/export/formats/backends/autoround.py@@ -0,0 +1,217 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++import re+from typing import Any, Callable, Union++import torch++from auto_round.export.formats.backends.auto_awq import AutoAWQFormat+from auto_round.export.formats.backends.auto_gptq import AutoGPTQFormat+from auto_round.export.formats.backends.mlx import MLXFormat+from auto_round.export.formats.base import BackendDataType, OutputFormat, _check_divisible_by_32+from auto_round.logger import logger+from auto_round.schemes import QuantizationScheme++[email protected]("auto_round")[email protected]("auto_round:auto_awq")[email protected]("auto_round:llm_compressor")[email protected]("auto_round:gptqmodel", "auto_round:auto_gptq")[email protected]("auto_round:fp8")[email protected]("auto_round:mlx")+class AutoRoundFormat(OutputFormat):+ support_schemes = [+ "W4A16",+ "W4A16_MIXED",+ "W2A16",+ "W3A16",+ "W8A16",+ "MXFP4",+ "MXFP8",+ "NVFP4",+ "FPW8A16",+ "W2A16G64",+ "W2A16G32",+ "FP8_STATIC",+ "BF16",+ "FP8_BLOCK",+ "MXINT4",+ ]+ format_name = "auto_round"++ def __init__(self, format: str, scheme: QuantizationScheme, ctx: Any):+ self.output_format = "auto_round"+ self.backend = None++ if format == "auto_round":+ if scheme.sym and "int" in scheme.data_type and "mx" not in scheme.data_type:+ self.backend = AutoGPTQFormat("auto_round:auto_gptq", scheme, ctx)+ elif scheme.bits == 4 and not scheme.sym and "int" in scheme.data_type:+ if ctx.layer_config is None:+ enable_awq = True+ else:+ enable_awq = all(+ config["bits"] == scheme.bits or config["bits"] >= 16 for config in ctx.layer_config.values()+ )+ if enable_awq:+ self.backend = AutoAWQFormat("auto_round:auto_awq", scheme, ctx)+ elif scheme.is_nv_fp() or scheme.is_mx_fp():+ self.backend = AutoRoundFormat(scheme.data_type, scheme, ctx)+ elif scheme.is_mx_int() and scheme.bits == 4: # only add mx_int4 now+ self.backend = AutoRoundFormat(scheme.data_type, scheme, ctx)+ elif scheme.is_act_static(): # static wfp8afp8+ self.backend = AutoRoundFormat(BackendDataType.FP8_STATIC.value, scheme, ctx)+ elif scheme.data_type.startswith("fp") and scheme.bits == 8 and scheme.act_bits >= 16: # woq fp8+ self.backend = AutoRoundFormat(BackendDataType.FP8.value, scheme, ctx)+ elif scheme.data_type.startswith("fp") and scheme.bits == 8 and isinstance(scheme.group_size, tuple):+ self.backend = AutoRoundFormat("auto_round:fp8", scheme, ctx)+ elif scheme.act_bits < 16:+ raise ValueError(+ "AutoRound format does not support exporting "+ "for the current quantization configuration, "+ "please change to `fake` format for research purpose"+ )+ # for auto_round:fp8_static, auto_round:nv_fp etc.+ elif not format.startswith("auto_round"):+ if format == "mlx":+ self.backend = MLXFormat("mlx", scheme, ctx)+ elif format.upper() not in list(BackendDataType.__members__.keys()):+ raise KeyError(f"Unsupported backend format auto_round:{format}, please check")+ else:+ self.output_format = f"auto_round:{format}"+ self.backend = None+ elif format == "auto_round:mlx":+ self.backend = MLXFormat("mlx", scheme, ctx)+ else:+ backend = format.split(":")[1] if ":" in format else None+ self.backend = self._format_list.get(backend)(format, scheme, ctx) if backend else None++ if self.backend is not None:+ self.support_schemes = self.backend.support_schemes++ def check_and_reset_format(self, scheme: QuantizationScheme, ctx: Any):+ layer_config, quant_block_list = ctx.layer_config, ctx.quant_block_list+ if self.backend is not None:+ new_format, scheme, layer_config, quant_block_list = self.backend.check_and_reset_format(scheme, ctx)+ ctx.layer_config, ctx.quant_block_list = layer_config, quant_block_list+ self.backend = (+ OutputFormat._format_list[new_format](new_format, scheme, ctx) if new_format else self.backend+ )++ if scheme.act_bits <= 8:+ if scheme.is_act_standard_fp() and not scheme.act_dynamic:+ if (+ scheme.act_group_size != 0+ and not scheme.act_dynamic+ and self.get_backend_name() == f"auto_round:{BackendDataType.FP8.value}"+ ):+ logger.warning(+ f"Please note that quantize activation with act_group_size={scheme.act_group_size}"+ " may result in failure to export or import normally."+ )+ if self.backend is None:+ layer_config = _check_divisible_by_32(scheme, ctx.model, layer_config)+ return None, scheme, layer_config, quant_block_list++ def pack_layer(self, layer_name, model, device=None, **kwargs):+ if self.backend is not None:+ return self.backend.pack_layer(layer_name, model, device=device, **kwargs)++ backend = self.get_backend_name()++ if self.output_format in [+ f"auto_round:{BackendDataType.NV_FP.value}",+ f"auto_round:{BackendDataType.MX_FP.value}",+ f"auto_round:{BackendDataType.MX_FP_RCEIL.value}",+ f"auto_round:{BackendDataType.NV_FP4_WITH_STATIC_GS.value}",+ ]:+ from auto_round.export.export_to_autoround.export_to_nvfp_mx import pack_layer++ pack_func = pack_layer+ elif self.output_format in [f"auto_round:{BackendDataType.MX_INT.value}"]:+ from auto_round.export.export_to_autoround.export_to_nvfp_mx import pack_layer++ pack_func = pack_layer+ elif self.output_format in [+ f"auto_round:{BackendDataType.FP8.value}",+ f"auto_round:{BackendDataType.FP8_STATIC.value}",+ f"auto_round:{BackendDataType.FP8_STATIC.value}",+ ]:+ from auto_round.export.export_to_autoround.export_to_fp8 import pack_layer++ pack_func = pack_layer+ else:+ from auto_round.export.export_to_autoround.export import pack_layer++ pack_func = pack_layer+ return pack_func(layer_name, model, backend, device)++ def save_quantized(+ self,+ output_dir: str,+ model: torch.nn.Module = None,+ tokenizer: Callable = None,+ layer_config: dict = None,+ inplace: bool = True,+ device: Union[str, torch.device] = "cpu",+ serialization_dict: dict = None,+ **kwargs,+ ) -> torch.nn.Module:+ if self.backend is not None:+ extra_kwargs = {}+ if isinstance(self.backend, MLXFormat):+ extra_kwargs["autoround_format"] = True+ return self.backend.save_quantized(+ output_dir=output_dir,+ model=model,+ tokenizer=tokenizer,+ layer_config=layer_config,+ inplace=inplace,+ device=device,+ serialization_dict=serialization_dict,+ **extra_kwargs,+ **kwargs,+ )+ backend = self.get_backend_name()+ if re.search(f"{BackendDataType.MX_FP.value}|{BackendDataType.NV_FP.value}", backend):+ from auto_round.export.export_to_autoround.export_to_nvfp_mx import save_quantized_as_fp++ backend = "auto_round:llm_compressor"+ export_func = save_quantized_as_fp+ elif serialization_dict.get("data_type", "int") == "fp" and serialization_dict.get("bits", 16) == 8:+ from auto_round.export.export_to_autoround.export_to_fp8 import save_quantized_as_autoround++ backend = "auto_round:fp8_static" if serialization_dict.get("act_bits", 16) == 8 else None+ export_func = save_quantized_as_autoround+ elif re.search(f"{BackendDataType.MX_INT.value}", backend):+ from auto_round.export.export_to_autoround.export_to_nvfp_mx import save_quantized_as_fp++ backend = "auto_round"+ export_func = save_quantized_as_fp+ else:+ from auto_round.export.export_to_autoround.export import save_quantized_as_autoround++ export_func = save_quantized_as_autoround+ return export_func(+ output_dir=output_dir,+ model=model,+ tokenizer=tokenizer,+ layer_config=layer_config,+ inplace=inplace,+ device=device,+ backend=backend,+ serialization_dict=serialization_dict,+ **kwargs,+ )diff --git a/auto_round/export/formats/backends/fake.py b/auto_round/export/formats/backends/fake.pynew file mode 100644index 000000000..6c1e9a703--- /dev/null+++ b/auto_round/export/formats/backends/fake.py@@ -0,0 +1,65 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++from typing import Any, Callable, Union++import torch++from auto_round.export.formats.base import OutputFormat+from auto_round.logger import logger+from auto_round.schemes import QuantizationScheme+from auto_round.utils import copy_python_files_from_model_cache, unsupported_meta_device++[email protected]("fake")+class FakeFormat(OutputFormat):+ support_schemes = None+ format_name = "fake"++ def check_and_reset_format(+ self, scheme: QuantizationScheme, ctx: Any+ ) -> tuple[None, QuantizationScheme, dict, list]:+ return None, scheme, ctx.layer_config, ctx.quant_block_list++ # fake format will not execute pack_layer.+ def pack_layer(self, *args, **kwargs):+ pass++ def save_quantized(+ self,+ output_dir: str,+ model: torch.nn.Module = None,+ tokenizer: Callable = None,+ layer_config: dict = None,+ inplace: bool = True,+ device: Union[str, torch.device] = "cpu",+ serialization_dict: dict = None,+ **kwargs,+ ):+ if not unsupported_meta_device(model):+ model = model.to("cpu")+ model.save_pretrained(output_dir)+ elif hasattr(model, "config") and model.config is not None:+ model.config.save_pretrained(output_dir)++ if tokenizer is not None and hasattr(tokenizer, "save_pretrained"):+ tokenizer.save_pretrained(output_dir)+ processor = kwargs.get("processor", None)+ if processor is not None:+ processor.save_pretrained(output_dir)+ try:+ copy_python_files_from_model_cache(model, output_dir)+ except Exception as e:+ logger.warning("Skipping source model Python file copy due to error: %s", e)+ return modeldiff --git a/auto_round/export/formats/backends/fp8.py b/auto_round/export/formats/backends/fp8.pynew file mode 100644index 000000000..1ceea2089--- /dev/null+++ b/auto_round/export/formats/backends/fp8.py@@ -0,0 +1,92 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++from typing import Callable, Union++import torch++from auto_round.export.formats.base import OutputFormat+from auto_round.schemes import QuantizationScheme++[email protected]("fp8")+class FP8Format(OutputFormat):+ support_schemes = ["FP8_BLOCK"]+ format_name = "fp8"++ @classmethod+ def check_scheme_args(cls: OutputFormat, scheme: QuantizationScheme) -> bool:+ error_logs = []+ if scheme.bits != 8:+ error_logs.append(f"bits={scheme.bits}")+ if scheme.data_type != "fp":+ error_logs.append(f"data_type={scheme.data_type}")+ if not isinstance(scheme.group_size, tuple):+ error_logs.append(f"group_size={scheme.group_size}")+ if not scheme.act_dynamic:+ error_logs.append(f"act_dynamic={scheme.act_dynamic}")+ if not isinstance(scheme.act_group_size, int):+ error_logs.append(f"act_group_size={scheme.act_group_size}")+ if scheme.act_bits != 8:+ error_logs.append(f"act_bits={scheme.act_bits}")+ if scheme.act_data_type != "fp":+ error_logs.append(f"act_data_type={scheme.act_data_type}")+ if error_logs:+ raise ValueError(+ f"{cls.format_name} format support quantization scheme with {','.join(cls.support_schemes)} "+ f"but got {', '.join(error_logs)}, please have a check."+ )+ return True++ def pack_layer(self, layer_name, model, device=None, **kwargs):+ from auto_round.export.export_to_autoround.export_to_fp8 import pack_layer++ pack_layer(layer_name, model, self.output_format, device=device)++ def save_quantized(+ self,+ output_dir: str,+ model: torch.nn.Module = None,+ tokenizer: Callable = None,+ layer_config: dict = None,+ inplace: bool = True,+ device: Union[str, torch.device] = "cpu",+ serialization_dict: dict = None,+ **kwargs,+ ) -> torch.nn.Module:+ from auto_round.export.export_to_autoround.export_to_fp8 import save_quantized_as_autoround++ backend = self.get_backend_name()++ if isinstance(serialization_dict["group_size"], tuple):+ serialization_dict["weight_block_size"] = serialization_dict["group_size"]++ ignored_layers = []+ for layer_name, cfg in layer_config.items():+ if cfg["bits"] >= 16 and cfg["act_bits"] >= 16:+ ignored_layers.append(layer_name)+ if len(ignored_layers) > 0:+ serialization_dict["ignored_layers"] = ignored_layers++ return save_quantized_as_autoround(+ output_dir=output_dir,+ model=model,+ tokenizer=tokenizer,+ layer_config=layer_config,+ inplace=inplace,+ device=device,+ serialization_dict=serialization_dict,+ quant_method=backend,+ **kwargs,+ )diff --git a/auto_round/export/formats/backends/gguf.py b/auto_round/export/formats/backends/gguf.pynew file mode 100644index 000000000..cccb86719--- /dev/null+++ b/auto_round/export/formats/backends/gguf.py@@ -0,0 +1,801 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++import copy+import os+import re+from dataclasses import fields+from typing import Any, Callable, Union++import torch+import transformers++from auto_round.compressors.config_resolution.contracts import thaw_mapping+from auto_round.compressors.config_resolution.errors import FormatCompatibilityError+from auto_round.export.export_to_gguf.config import GGML_QUANT_SIZES, GGUF_CONFIG, GGUF_INNER_CONFIG, QK_K, ModelType+from auto_round.export.export_to_gguf.gguf_dtype import GGUFDTypeSelector, gguf_format_to_ftype+from auto_round.export.formats.base import OutputFormat+from auto_round.logger import logger+from auto_round.schemes import QuantizationScheme+from auto_round.utils import check_to_quantized, find_matching_blocks, get_block_names, get_module++[email protected]("gguf")+class GGUFFormat(OutputFormat):+ support_schemes = [+ "GGUF:Q4_0",+ "GGUF:Q4_1",+ "GGUF:Q5_0",+ "GGUF:Q5_1",+ "GGUF:Q2_K_S",+ "GGUF:Q3_K_S",+ "GGUF:Q3_K_M",+ "GGUF:Q3_K_L",+ "GGUF:Q4_K_S",+ "GGUF:Q4_K_M",+ "GGUF:Q5_K_S",+ "GGUF:Q5_K_M",+ "GGUF:Q6_K",+ "GGUF:Q8_0",+ "GGUF:Q2_K_MIXED",+ ]+ format_name = "gguf"++ def __init__(self, format: str, scheme: QuantizationScheme, ctx: Any):+ self.is_auto_scheme = ctx.is_auto_scheme+ layer_config = ctx.layer_config+ if format.startswith("gguf:"):+ self._original_format = format # preserve "gguf:q2_k_mixed" etc. for Phase 2b+ self.output_format = "gguf"+ self.backend_cls = GGUFFormat+ self.backend, scheme, layer_config = GGUFFormat.build(format.split(":")[-1], scheme, ctx)+ else:+ gguf_format = f"gguf:{format.lower()}"+ if format.lower().endswith("_mixed"):+ from auto_round.schemes import _handle_special_schemes+ from auto_round.utils.model import is_moe_model++ if format.lower() == "q2_k_mixed" and (ctx.iters or 0) > 0 and not is_moe_model(ctx.model):+ logger.warning(+ "gguf:q2_k_mixed only supports MoE models with iters>0. "+ "It is not an MoE model, falling back to gguf:q2_k_s."+ )+ gguf_format = "gguf:q2_k_s"+ else:+ layer_config = _handle_special_schemes(+ gguf_format, layer_config, ctx.model, quant_nontext_module=ctx.quant_nontext_module+ )+ gguf_format = gguf_format.lower().replace("_mixed", "_s")+ if isinstance(scheme, str) and scheme.lower() != gguf_format:+ # Defensive: no live call site passes a raw string scheme here today, but this+ # is kept for external callers that construct OutputFormat before scheme resolution.+ logger.warning(f"reset scheme {scheme.lower()} to {gguf_format} for gguf format export")+ scheme = gguf_format+ self.output_format = gguf_format+ self.backend = None+ from auto_round.schemes import GGUF_PRESET_ALIASES, GGUF_SCHEME_FACTS++ facts_name = GGUF_PRESET_ALIASES.get(gguf_format)+ if facts_name is not None:+ scheme = copy.deepcopy(scheme)+ for key, value in GGUF_SCHEME_FACTS[facts_name].items():+ setattr(scheme, key, value)+ self.mllm = ctx.mllm+ self._resolved_layer_config = layer_config+ self._resolved_scheme = scheme++ @classmethod+ def build(+ cls,+ format: str,+ scheme: QuantizationScheme,+ ctx: Any,+ ) -> tuple["GGUFFormat", QuantizationScheme, dict]:+ """Construct a GGUFFormat and surface the (possibly corrected) scheme/layer_config.++ Plain ``__init__`` cannot report a corrected ``scheme``/``layer_config`` back to its+ caller, but GGUF resolution can replace either (the ``_mixed`` -> ``_s`` rewrite, and the+ legacy string-scheme correction above). ``resolve_formats`` calls this instead of the bare+ constructor for every ``gguf:``-prefixed dispatch.+ """+ instance = cls(format, scheme, ctx)+ return instance, instance._resolved_scheme, instance._resolved_layer_config++ @classmethod+ def check_scheme_args(cls: OutputFormat, scheme: QuantizationScheme) -> bool:+ error_logs = []+ if not re.search("int", scheme.data_type):+ error_logs.append(f"data_type={scheme.data_type}")+ if error_logs:+ raise ValueError(+ f"{cls.format_name} format support quantization scheme with {','.join(cls.support_schemes)} "+ f"but got {', '.join(error_logs)}, please have a check."+ )+ return True++ def check_and_reset_format(self, scheme: QuantizationScheme, ctx: Any):+ if ctx.iters != 0 and scheme.bits != 3 and not ctx.enable_alg_ext:+ logger.warning_once(+ "`iters=0` is recommended when exporting to current GGUF format"+ " or add `enable_alg_ext` for better accuracy with much more tuning cost."+ " Please refer to https://github.com/intel/auto-round/tree/main/docs/gguf_alg_ext_acc.md"+ " for the accuracy results."+ )+ elif scheme.bits >= 8 and ctx.iters != 0:+ logger.warning_once("`iters=0` is recommended for bits>=8")++ if ctx.quant_nontext_module:+ # for gguf export, leave vl model for gguf itself+ all_blocks = get_block_names(ctx.model, False)+ ctx.quant_block_list = find_matching_blocks(ctx.model, all_blocks, None)+ return super().check_and_reset_format(scheme, ctx)++ def pack_layer(+ self,+ name,+ model,+ backend,+ output_dir,+ layer_config,+ tokenizer,+ processor=None,+ image_processor=None,+ model_type=ModelType.TEXT,+ device="cpu",+ quant_nontext_module=False,+ ):+ from auto_round.export.export_to_gguf.export import pack_gguf_layer++ pack_gguf_layer(+ name,+ model,+ backend,+ output_dir,+ layer_config,+ tokenizer,+ processor,+ image_processor,+ model_type,+ device,+ quant_nontext_module,+ is_auto_scheme=self.is_auto_scheme,+ )++ def save_quantized(+ self,+ output_dir: str,+ model: torch.nn.Module = None,+ tokenizer: Callable = None,+ layer_config: dict = None,+ inplace: bool = True,+ device: Union[str, torch.device] = "cpu",+ serialization_dict: dict = None,+ **kwargs,+ ) -> torch.nn.Module:+ from auto_round.export.export_to_gguf.export import save_quantized_as_gguf++ backend = self.get_backend_name()+ return save_quantized_as_gguf(+ output_dir=output_dir,+ model=model,+ backend=backend,+ layer_config=layer_config,+ mllm=self.mllm,+ device=device,+ serialization_dict=serialization_dict,+ is_auto_scheme=self.is_auto_scheme,+ **kwargs,+ )++ @staticmethod+ def gguf_args_check(+ scheme: QuantizationScheme,+ model,+ platform: str,+ formats: Union[str, list[str]] = None,+ model_type=ModelType.TEXT,+ ) -> QuantizationScheme:+ import argparse++ from auto_round.export.export_to_gguf.config import GGUF_CONFIG+ from auto_round.export.export_to_gguf.llama_cpp_conversion import get_conversion+ from auto_round.logger import logger+ from auto_round.utils.model import download_or_get_path, get_gguf_architecture++ formats = [formats] if isinstance(formats, str) else formats+ formats = sorted(formats, key=lambda x: len(x))+ export_gguf = False+ for f in formats:+ if f.startswith("gguf"):+ export_gguf = True++ if f.startswith("gguf") and f not in GGUF_CONFIG:+ logger.error(f"{f} is not supported, please check.")++ if export_gguf:+ if isinstance(model, str):+ model_path = model+ else:+ model_path = model.name_or_path+ if not os.path.isdir(model_path):+ model_path = download_or_get_path(model_path, platform)+ try:+ conversion = get_conversion(model_path, model_type=ModelType.TEXT)+ except AttributeError as e:+ print(f"[gguf conversion] error: {e}")+ raise ImportError(+ "Please use the latest gguf-py, you can use the following command to install it:\n"+ "git clone https://github.com/ggml-org/llama.cpp.git && cd llama.cpp/gguf-py"+ " && pip install . sentencepiece"+ )+ model_architecture = get_gguf_architecture(model_path, model_type=ModelType.TEXT)+ if not conversion.is_supported(model_architecture, ModelType.TEXT):+ raise FormatCompatibilityError(f"Model {model_architecture} is not supported to export gguf format.")++ pattern = re.compile(r"q\d_k")+ pre_dq_format = ""+ unsupported_list, reset_list = [], []+ for format in GGUF_CONFIG:+ if format in formats:+ if format == "q6_k_s":+ logger.warning("Please note that q6_k_s is q6_k.")++ if re.search(pattern, format):+ if pre_dq_format and re.search(pattern, format).group() not in pre_dq_format:+ raise FormatCompatibilityError(f"Cannot export {pre_dq_format} and {format} at the same time.")+ else:+ pre_dq_format = format++ unsupported_list, reset_list = [], []+ gguf_config = GGUF_CONFIG[format]+ for k, v in gguf_config.items():+ if not hasattr(scheme, k):+ continue+ if k == "data_type":+ if re.search(r"q\d_1", format) and len(formats) > 1:+ v = "int"+ # No caller passes an argparse.Namespace here; kept for dual-mode parity.+ if k == "sym" and isinstance(scheme, argparse.Namespace):+ k = "asym"+ v = not v+ if getattr(scheme, k) != v:+ unsupported_list.append(f"{k}={getattr(scheme, k)}")+ reset_list.append(f"{k}={v}")+ setattr(scheme, k, v)+ if len(unsupported_list) > 0:+ logger.info(+ f"format {format} does not support for {', '.join(unsupported_list)},"+ f" reset to {', '.join(reset_list)}."+ )+ return scheme++ def immediate_pack(+ self,+ name: str,+ model: torch.nn.Module,+ device: torch.device,+ output_dir: str = None,+ mllm: bool = False,+ layer_config: dict = None,+ tokenizer=None,+ processor=None,+ image_processor=None,+ quant_nontext_module: bool = False,+ **kwargs,+ ):+ m = get_module(model, name)+ if not check_to_quantized(m):+ return+ model_type = ModelType.MMPROJ if mllm else ModelType.TEXT+ self.pack_layer(+ name,+ model,+ self.get_backend_name(),+ output_dir,+ layer_config=layer_config,+ tokenizer=tokenizer,+ processor=processor,+ image_processor=image_processor,+ model_type=model_type,+ device=device,+ quant_nontext_module=quant_nontext_module,+ )+++def apply_gguf_layer_defaults(+ layer_config,+ model,+ gguf_name,+ lm_head_name,+ tie_word_embeddings,+ embedding_layer_names,+ default_scale_dtype,+ enable_gguf_official_mixed,+ is_mllm,+ has_qlayer_outside_block,+) -> tuple[dict, bool]:+ """Step 10 of layer-config resolution: embed/lm_head GGUF defaults plus the+ GGUF-format-specific per-layer mapping. Only called when `gguf_name` is truthy."""+ from auto_round.utils.model import is_separate_lm_head++ # embed + lm_head defaults for gguf+ tie_word_embeddings &= not is_separate_lm_head(model)+ if lm_head_name not in layer_config and not tie_word_embeddings:+ cfg = GGUF_INNER_CONFIG[GGUF_CONFIG[gguf_name.lower()]["lm_head"]]+ cfg = {**cfg, "fixed_by_user": False, "scale_dtype": default_scale_dtype}+ layer_config[lm_head_name] = cfg+ has_qlayer_outside_block = True+ for emd_name in embedding_layer_names:+ if emd_name in layer_config:+ continue+ if not tie_word_embeddings:+ cfg = GGUF_INNER_CONFIG[GGUF_CONFIG[gguf_name.lower()]["embedding"]]+ else:+ cfg = GGUF_INNER_CONFIG[GGUF_CONFIG[gguf_name.lower()]["lm_head"]]+ cfg = {**cfg, "fixed_by_user": False, "scale_dtype": default_scale_dtype}+ layer_config[emd_name] = cfg++ if enable_gguf_official_mixed:+ model_type = ModelType.MMPROJ if is_mllm else ModelType.TEXT+ layer_config, _ = get_layer_config_by_gguf_format(layer_config, gguf_name.lower(), model, model_type)++ _apply_gguf_shape_fallback(layer_config, model)+ return layer_config, has_qlayer_outside_block+++def _use_more_bits(i_layer: int, n_layer: int):+ return (i_layer < n_layer // 8) or (i_layer >= 7 * n_layer // 8) or ((i_layer - n_layer // 8) % 3 == 2)+++def _search_gguf_type(gguf_type):+ if gguf_type in GGUF_INNER_CONFIG:+ return gguf_type+ pattern = re.compile("gguf:q([0-9]{1,})_[01k]")+ bits = re.search(pattern, gguf_type)+ if not bits:+ raise KeyError(f"{gguf_type} is not a correct gguf type, please check")++ for suffix in ["_k", "_0", "_1"]:+ if gguf_type.endswith(suffix):+ continue+ if (tmp_type := re.sub("_[01k]", suffix, gguf_type)) in GGUF_INNER_CONFIG:+ return tmp_type+ return None+++def gguf_type_fallback(gguf_type: str) -> str:+ gguf_type = gguf_type.lower()+ if gguf_type in ("gguf:q2_k", "gguf:q3_k", "gguf:q4_k"):+ gguf_type = "gguf:q5_0"+ elif gguf_type == "gguf:q5_k":+ gguf_type = "gguf:q5_0"+ elif gguf_type == "gguf:q6_k":+ gguf_type = "gguf:q8_0"+ return gguf_type+++def get_gguf_qtype_by_layer_config(layer_config):+ import gguf # pylint: disable=E0401++ if layer_config["bits"] >= 16:+ return None+ bits = layer_config["bits"]+ super_bits = layer_config.get("super_bits", None)+ sym = layer_config["sym"]+ group_size = layer_config.get("group_size", None)+ super_group_size = layer_config.get("super_group_size", None)+ if bits == 2 and super_bits == 4 and not sym and group_size == 16 and super_group_size == 16:+ return gguf.GGMLQuantizationType.Q2_K+ if bits == 3 and super_bits == 6 and sym and group_size == 16 and super_group_size == 16:+ return gguf.GGMLQuantizationType.Q3_K+ if bits == 4:+ if super_bits is not None and super_bits == 6 and not sym and group_size == 32 and super_group_size == 8:+ return gguf.GGMLQuantizationType.Q4_K+ if super_bits is None and sym and group_size == 32:+ return gguf.GGMLQuantizationType.Q4_0+ if super_bits is None and not sym and group_size == 32:+ return gguf.GGMLQuantizationType.Q4_1+ if bits == 5:+ if super_bits == 6 and not sym and group_size == 32 and super_group_size == 8:+ return gguf.GGMLQuantizationType.Q5_K+ if super_bits is None and sym and group_size == 32:+ return gguf.GGMLQuantizationType.Q5_0+ if super_bits is None and not sym and group_size == 32:+ return gguf.GGMLQuantizationType.Q5_1+ if bits == 6 and super_bits == 8 and group_size == 16 and super_group_size == 16:+ return gguf.GGMLQuantizationType.Q6_K+ if bits == 8 and sym and group_size == 32:+ return gguf.GGMLQuantizationType.Q8_0+ raise ValueError("Unknown layer config")+++def _apply_gguf_shape_fallback(layer_config, model):+ from auto_round.utils.model import get_module++ for layer_name, config in layer_config.items():+ if not check_to_quantized(config):+ continue+ layer = get_module(model, layer_name)+ if layer is None or not hasattr(layer, "weight"):+ continue+ try:+ qtype = get_gguf_qtype_by_layer_config(config)+ except ValueError:+ continue+ if qtype is None:+ continue++ gguf_type = f"gguf:{qtype.name.lower()}"+ block_size = GGML_QUANT_SIZES[gguf_type.split(":")[-1]][0]+ input_features = (+ layer.weight.shape[0] if type(layer) == transformers.pytorch_utils.Conv1D else layer.weight.shape[-1]+ )+ if input_features % block_size == 0:+ continue++ fallback_type = _gguf_type_fallback(gguf_type)+ fallback_block_size = GGML_QUANT_SIZES[fallback_type.split(":")[-1]][0]+ if input_features % fallback_block_size != 0:+ fallback_type = "gguf:bf16"++ preserved = {key: config[key] for key in ("fixed_by_user", "scale_dtype") if key in config}+ config.update(GGUF_INNER_CONFIG[fallback_type])+ config.update(preserved)+ logger.warning(+ "fallback %s to %s before quantization, because input_features(%s) is not divisible by %s block_size(%s)",+ layer_name,+ fallback_type,+ input_features,+ gguf_type,+ block_size,+ )+++def _get_digital_in_layer_name(layer_name):+ pattern = re.compile(r"([a-zA-Z]+\.){1,}(\d+)")+ res = re.search(pattern, layer_name)+ if res:+ return int(res[2])+ else:+ return None+++def _gguf_type_fallback(gguf_type: str) -> str:+ gguf_type = gguf_type.lower()+ if gguf_type in ("gguf:q2_k", "gguf:q3_k", "gguf:q4_k"):+ gguf_type = "gguf:q5_0"+ elif gguf_type == "gguf:q5_k":+ gguf_type = "gguf:q5_0"+ elif gguf_type == "gguf:q6_k":+ gguf_type = "gguf:q8_0"+ return gguf_type+++def _select_gguf_layer_type(+ layer_name,+ config,+ layer,+ model,+ model_class,+ *,+ target_gguf_format,+ lm_head_name,+ tie_word_embeddings,+ block_size,+ base_target_bits,+ dtype_selector,+ gguf_name,+ i_layer,+) -> str | None:+ """Per-layer GGUF-type-selection cascade extracted from+ `get_layer_config_by_gguf_format`'s loop body.++ Returns the selected gguf type string, or ``None`` as the sentinel for the+ `lm_head_name == layer_name and tie_word_embeddings` skip case, which the+ caller must handle with `continue`.+ """+ import gguf # pylint: disable=E0401++ from auto_round.schemes import QuantizationScheme, get_gguf_scheme++ if type(layer) == transformers.pytorch_utils.Conv1D:+ input_features = layer.weight.shape[0]+ else:+ input_features = layer.weight.shape[-1]++ # Reset target_bits each iteration to prevent lm_head/embedding settings+ # from bleeding into subsequent block layers and bypassing their special logic.+ target_bits = base_target_bits+ new_type = GGUF_CONFIG[target_gguf_format]["mostly"]++ if lm_head_name is not None and layer_name == lm_head_name:+ target_bits = int(re.search("gguf:q([0-9]{1,})_[01k]", GGUF_CONFIG[target_gguf_format]["lm_head"]).group(1))+ if isinstance(layer, torch.nn.Embedding):+ embedding_format_key = "lm_head" if tie_word_embeddings else "embedding"+ target_bits = int(+ re.search("gguf:q([0-9]{1,})_[01k]", GGUF_CONFIG[target_gguf_format][embedding_format_key]).group(1)+ )++ bits_index = 6+ if config.get("fixed_by_user", False):+ if "bits" not in config:+ logger.warning(+ f"Setting layer_config requires providing bits, {layer_name} has not bits,"+ f" using bits={target_bits} instead."+ )+ new_type = new_type[:bits_index] + str(target_bits) + new_type[bits_index + 1 :]+ else:+ config_tmp = config.copy()+ scheme_keys = [f.name for f in fields(QuantizationScheme)]+ for key in config.keys():+ if key not in scheme_keys:+ config_tmp.pop(key, None)+ matched_scheme = get_gguf_scheme(QuantizationScheme.from_dict(config_tmp)) # check matched+ if not matched_scheme:+ if config.get("super_group_size", None) is not None or config.get("super_bits", None) is not None:+ new_type = new_type[:bits_index] + str(config["bits"]) + "_k"+ if new_type not in GGUF_INNER_CONFIG:+ prefix_idx = 0 if config.get("sym", True) else 1+ new_type = new_type[:bits_index] + str(config["bits"]) + f"_{prefix_idx}"+ if new_type not in GGUF_INNER_CONFIG:+ new_type = new_type[:bits_index] + str(config["bits"]) + f"_{1-prefix_idx}"+ if new_type not in GGUF_INNER_CONFIG:+ raise ValueError(+ f"the setting in layer_config {layer_name} "+ f"could not match any supported gguf format, please have a check."+ )++ new_type = new_type[:bits_index] + str(config["bits"]) + new_type[bits_index + 1 :]+ new_type = _search_gguf_type(new_type)+ if new_type is None:+ raise ValueError(f"invalid bit setting for {layer_name}")+ elif lm_head_name is not None and layer_name == lm_head_name and not tie_word_embeddings:+ if gguf.MODEL_ARCH.FALCON == model_class.model_arch or input_features % block_size != 0:+ new_type = "gguf:q8_0"+ elif "lm_head" in GGUF_CONFIG[target_gguf_format]:+ new_type = GGUF_CONFIG[target_gguf_format]["lm_head"]+ elif new_type != "gguf:q8_0":+ new_type = "gguf:q6_k"+ elif lm_head_name is not None and layer_name == lm_head_name and tie_word_embeddings:+ return None+ elif isinstance(layer, torch.nn.Embedding):+ embedding_format_key = "lm_head" if tie_word_embeddings else "embedding"+ if embedding_format_key in GGUF_CONFIG[target_gguf_format]:+ new_type = GGUF_CONFIG[target_gguf_format][embedding_format_key]+ elif target_bits is not None and "bits" in config and config["bits"] != target_bits:+ new_type = new_type[:bits_index] + str(config["bits"]) + new_type[bits_index + 1 :]+ new_type = _search_gguf_type(new_type)+ if new_type is None:+ raise ValueError(f"invalid bit setting for {layer_name}")+ elif gguf_name is not None:+ gguf_weight_name = gguf_name if gguf_name.endswith(".weight") else f"{gguf_name}.weight"+ new_type = dtype_selector.select_gguf_type(gguf_weight_name, len(layer.weight.shape), i_layer or 0)+ new_block_size = GGML_QUANT_SIZES[new_type.split(":")[-1].lower()][0]+ if input_features % new_block_size != 0:+ new_type = _gguf_type_fallback(new_type)+ new_block_size = GGML_QUANT_SIZES[new_type.split(":")[-1].lower()][0]+ if input_features % new_block_size != 0:+ new_type = "gguf:bf16"+ logger.warning(+ f"fallback {layer_name} to {new_type}, "+ f"because input_features({input_features}) % block_size({block_size}) != 0"+ )+ # for deepseek v2+ if layer_name.endswith("kv_b_proj") and new_type.endswith("_k") and "Deepseek" in model.config.architectures[0]:+ fallback = False++ # calc if need fallback+ qk_nope_head_dim = model.config.qk_nope_head_dim+ kv_b_shape = get_module(model, layer_name).weight.shape++ if (+ qk_nope_head_dim < QK_K+ or qk_nope_head_dim % QK_K != 0+ or kv_b_shape[-1] < QK_K+ or kv_b_shape[-1] % QK_K != 0+ ):+ fallback = True+ if fallback:+ tmp_type = _gguf_type_fallback(new_type)+ logger.warning_once(+ f"self_attn.kv_b_proj does not support the use of {new_type}, replace it with {tmp_type}"+ )+ new_type = tmp_type++ return new_type+++def _infer_gguf_n_layers_from_model(model, model_type=ModelType.TEXT):+ if model is None or not hasattr(model, "named_modules"):+ return None, None++ from auto_round.utils.common import MM_MODULE_KEYS++ text_layer_ids = set()+ vision_layer_ids = set()+ layer_pattern = re.compile(r"(?:^|\.)layers\.(\d+)(?:\.|$)")+ for module_name, _ in model.named_modules():+ match = layer_pattern.search(module_name)+ if match is None:+ continue+ layer_id = int(match.group(1))+ if model_type != ModelType.TEXT and any(key in module_name.lower() for key in MM_MODULE_KEYS):+ vision_layer_ids.add(layer_id)+ else:+ text_layer_ids.add(layer_id)++ n_layer = max(text_layer_ids) + 1 if text_layer_ids else None+ n_layer_vision = max(vision_layer_ids) + 1 if vision_layer_ids else None+ return n_layer, n_layer_vision+++def _resolve_gguf_n_layers(config, model_type=ModelType.TEXT, model=None):+ """Resolve text (and vision) layer counts from config, then model structure.++ Multimodal configs (e.g. Qwen3.5, Qwen3-VL) keep the text hparams only in+ ``config.text_config``, so both it and the top-level config must be checked.+ If config naming changes, fall back to the loaded module hierarchy so GGUF+ official mixed formats still receive a usable block count.+ """+ n_layer = None+ n_layer_vision = None+ text_config = getattr(config, "text_config", None)+ vision_config = None+ audio_config = getattr(config, "audio_config", None)+ for name in ["n_layers", "num_hidden_layers", "n_layer", "num_layers", "depth"]:+ if hasattr(config, name):+ n_layer = getattr(config, name)+ if text_config is not None and hasattr(text_config, name):+ # the text hparams in text_config are authoritative for multimodal models;+ # for text-only usage they still fill in when the top-level config lacks them+ if model_type != ModelType.TEXT or n_layer is None:+ n_layer = getattr(text_config, name)+ if model_type != ModelType.TEXT:+ for config_name in ["vision_config", "vision_encoder"]:+ if hasattr(config, config_name):+ vision_config = getattr(config, config_name)+ if hasattr(vision_config, name):+ n_layer_vision = getattr(vision_config, name)+ break+ if n_layer and n_layer_vision:+ break++ inferred_n_layer, inferred_n_layer_vision = _infer_gguf_n_layers_from_model(model, model_type)+ n_layer = n_layer or inferred_n_layer+ n_layer_vision = n_layer_vision or inferred_n_layer_vision++ if (+ model_type != ModelType.TEXT+ and n_layer_vision is None+ and vision_config is not None+ and audio_config is not None+ ):+ # llama.cpp's Gemma4 mmproj converter uses a fixed tensor-name map size+ # when vision and audio encoders coexist, because the two encoders do not+ # expose a single transformer block count in config.json.+ n_layer_vision = 128+ return n_layer, n_layer_vision+++##https://github.com/ggml-org/llama.cpp/blob/9e31bec4fd53634c9e5b04650488a09a055f5dab/src/llama-quant.cpp#L129+def get_layer_config_by_gguf_format(layer_config, target_gguf_format: str, model, model_type=ModelType.TEXT):+ import gguf # pylint: disable=E0401++ from auto_round.export.export_to_gguf.llama_cpp_conversion import get_conversion+ from auto_round.utils.common import MM_MODULE_KEYS+ from auto_round.utils.model import get_lm_head_name, get_module, is_separate_lm_head++ try:+ hparams = model.config.to_dict()+ conversion = get_conversion(hparams=hparams, model_type=model_type)+ model_architecture = conversion.get_model_architecture(+ hparams=hparams, model_type=conversion.model_type(model_type)+ )+ except AttributeError as e:+ raise ImportError(+ "Please use the latest gguf-py, you can use the following command to install it:\n"+ "git clone https://github.com/ggml-org/llama.cpp.git && cd llama.cpp/gguf-py"+ " && pip install . sentencepiece"+ )+ try:+ if model_type != ModelType.TEXT:+ model_class_vision = conversion.get_model_class(model_architecture, model_type=model_type)+ model_class = conversion.get_model_class(model_architecture, model_type=ModelType.TEXT)++ except NotImplementedError:+ return layer_config, {}++ n_layer, n_layer_vision = _resolve_gguf_n_layers(model.config, model_type, model=model)++ if n_layer is None:+ return layer_config, {}++ tensor_map = gguf.get_tensor_name_map(model_class.model_arch, n_layer)+ if model_type != ModelType.TEXT:+ tensor_map_vision = gguf.get_tensor_name_map(model_class_vision.model_arch, n_layer_vision)++ def _set_config(config, target_config):+ for k, v in target_config.items():+ config[k] = v+ return config++ gguf_format_config = {}+ lm_head_name = get_lm_head_name(model)+ inner_gguf_format = GGUF_CONFIG[target_gguf_format]["mostly"]+ # ggml_type = getattr(gguf.GGMLQuantizationType,inner_gguf_format.split(":")[-1].upper())+ block_size = GGML_QUANT_SIZES[inner_gguf_format.split(":")[-1].lower()][0]+ tie_word_embeddings = True+ if hasattr(model, "config") and hasattr(model.config, "tie_word_embeddings"):+ tie_word_embeddings = model.config.tie_word_embeddings+ tie_word_embeddings &= not is_separate_lm_head(model)++ dtype_selector = GGUFDTypeSelector(+ hparams, gguf_format_to_ftype(target_gguf_format), model_class.model_arch, n_layer+ )+ layer_config_copy = thaw_mapping(layer_config)+ base_target_bits = None+ if inner_gguf_format.startswith("gguf:q") and len(inner_gguf_format) >= 7 and (inner_gguf_format[6]).isdigit():+ base_target_bits = int(inner_gguf_format[6])++ def _resolve_gguf_name(layer_name):+ if model_type != ModelType.TEXT and any([key in layer_name for key in MM_MODULE_KEYS]):+ gguf_layer_name = tensor_map_vision.get_name(layer_name)+ if gguf_layer_name is None:+ for key in MM_MODULE_KEYS:+ gguf_layer_name = tensor_map_vision.get_name(layer_name.replace(f".{key}", ""))+ if gguf_layer_name is not None:+ break+ else:+ gguf_layer_name = tensor_map.get_name(layer_name)+ if gguf_layer_name is None:+ gguf_layer_name = tensor_map.get_name(layer_name.replace(".language_model", ""))+ return gguf_layer_name++ dtype_selector.n_attention_wv = sum(+ 1+ for layer_name, config in layer_config_copy.items()+ if check_to_quantized(config)+ and (gguf_name := _resolve_gguf_name(layer_name))+ and any(key in gguf_name for key in ("attn_v", "attn_qkv", "attn_kv_b"))+ )++ for layer_name, config in layer_config_copy.items():+ if not check_to_quantized(config):+ continue+ layer = get_module(model, layer_name)+ i_layer = _get_digital_in_layer_name(layer_name)+ new_type = _select_gguf_layer_type(+ layer_name,+ config,+ layer,+ model,+ model_class,+ target_gguf_format=target_gguf_format,+ lm_head_name=lm_head_name,+ tie_word_embeddings=tie_word_embeddings,+ block_size=block_size,+ base_target_bits=base_target_bits,+ dtype_selector=dtype_selector,+ gguf_name=_resolve_gguf_name(layer_name),+ i_layer=i_layer,+ )+ if new_type is None:+ continue++ target_config = GGUF_INNER_CONFIG[new_type]++ _set_config(layer_config[layer_name], target_config)+ gguf_format_config[layer_name] = new_type++ return layer_config, gguf_format_configdiff --git a/auto_round/export/formats/backends/llm_compressor.py b/auto_round/export/formats/backends/llm_compressor.pynew file mode 100644index 000000000..52d18037b--- /dev/null+++ b/auto_round/export/formats/backends/llm_compressor.py@@ -0,0 +1,220 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++import re+from typing import Any, Callable, Union++import torch++from auto_round.compressors.config_resolution.errors import FormatCompatibilityError+from auto_round.export.formats.base import BackendDataType, OutputFormat+from auto_round.logger import logger+from auto_round.schemes import QuantizationScheme++[email protected]("llm_compressor")+class LLMCompressorFormat(OutputFormat):+ support_schemes = [+ "MXFP4",+ "MXFP8",+ "NVFP4",+ "FPW8A16",+ "FP8_STATIC",+ "INT8",+ "INT8_W8A8",+ "FP8_BLOCK",+ "W4A16",+ "W8A16",+ ]+ format_name = "llm_compressor"++ def __init__(self, format: str, scheme: QuantizationScheme, ctx: Any):+ if not self.is_support_scheme(scheme):+ raise FormatCompatibilityError(+ f"Currently, the llm_compressor format only supports {self.support_schemes}, "+ f"but got scheme {scheme}, please change to fake or auto_round etc."+ )+ # if format.startswith("llm_compressor"):+ if re.search("^(auto_round:)?llm_compressor", format):+ self.output_format = format+ self.backend = None+ if scheme.is_nv_fp() or scheme.is_mx_fp():+ from auto_round.export.export_to_llmcompressor import check_compressed_tensors_supported++ check_compressed_tensors_supported(raise_error=True)+ self.backend = LLMCompressorFormat(scheme.data_type, scheme, ctx)+ elif scheme.is_dynamic_afp8() and scheme.is_block_wfp8():+ self.backend = LLMCompressorFormat(BackendDataType.FP8_BLOCK.value, scheme, ctx)+ elif scheme.is_act_static():+ self.backend = LLMCompressorFormat(BackendDataType.FP8_STATIC.value, scheme, ctx)+ if scheme.act_group_size != 0:+ logger.warning(+ f"scheme FP8_STATIC export to llm_compressor format only support for act_group_size 0,"+ f" ,but got act_group_size={scheme.act_group_size}, reset = 0"+ )+ scheme.act_group_size = 0+ if scheme.group_size > 0:+ logger.warning(+ f"please note that group_size={scheme.group_size}"+ " may not be supported for llm_compressor format, and cannot be loaded in llm_compressor"+ )+ elif scheme.is_dynamic_wint8aint8():+ from auto_round.export.export_to_llmcompressor import check_compressed_tensors_supported++ check_compressed_tensors_supported()+ self.backend = LLMCompressorFormat(BackendDataType.INT8.name, scheme, ctx)+ self.backend.output_format = f"llm_compressor:{BackendDataType.INT8.value}"+ elif scheme.is_wint_woq():+ from auto_round.export.export_to_llmcompressor import check_compressed_tensors_supported++ check_compressed_tensors_supported()+ self.backend = LLMCompressorFormat(BackendDataType.WINT_A16.value, scheme, ctx)+ else:+ if format.upper() not in list(BackendDataType.__members__.keys()):+ raise KeyError(f"Unsupported backend format llm_compressor:{format}, please check")+ self.output_format = f"llm_compressor:{format}"+ self.backend = None++ @classmethod+ def check_scheme_args(cls: OutputFormat, scheme: QuantizationScheme) -> bool:+ error_logs = []+ if scheme.bits not in [4, 8, 16]:+ error_logs.append(f"bits={scheme.bits}")+ if not re.search("mxfp|fp|nvfp|int", scheme.data_type):+ error_logs.append(f"data_type={scheme.data_type}")+ if scheme.data_type == "fp" and scheme.bits != 8:+ error_logs.append(f"data_type={scheme.data_type}, bits={scheme.bits}")+ if scheme.data_type == "int" and scheme.bits not in [4, 8]:+ error_logs.append(f"data_type={scheme.data_type}, bits={scheme.bits}")+ if scheme.super_bits:+ error_logs.append(f"super_bits={scheme.super_bits}")+ if scheme.super_group_size:+ error_logs.append(f"super_group_size={scheme.super_group_size}")+ if isinstance(scheme.group_size, tuple):+ if scheme.bits != 8:+ error_logs.append(f"bits={scheme.bits}")+ if scheme.data_type != "fp":+ error_logs.append(f"data_type={scheme.data_type}")+ if len(scheme.group_size) != 2:+ error_logs.append(f"group_size={scheme.group_size}")+ if not scheme.act_dynamic:+ error_logs.append(f"act_dynamic={scheme.act_dynamic}")+ if not isinstance(scheme.act_group_size, int):+ error_logs.append(f"act_group_size={scheme.act_group_size}")+ if scheme.act_bits != 8:+ error_logs.append(f"act_bits={scheme.act_bits}")+ if scheme.act_data_type != "fp":+ error_logs.append(f"act_data_type={scheme.act_data_type}")+ if error_logs:+ raise ValueError(+ f"LLMCompressor format support quantization scheme with {','.join(cls.support_schemes)} "+ f"but got {', '.join(error_logs)}, please have a check."+ )+ return True++ def check_and_reset_format(+ self, scheme: QuantizationScheme, ctx: Any+ ) -> tuple[str, QuantizationScheme, dict, list]:+ layer_config, quant_block_list = ctx.layer_config, ctx.quant_block_list+ if self.backend is not None:+ new_format, scheme, layer_config, quant_block_list = self.backend.check_and_reset_format(scheme, ctx)+ ctx.layer_config, ctx.quant_block_list = layer_config, quant_block_list+ self.backend = (+ OutputFormat._format_list[new_format](new_format, scheme, ctx) if new_format else self.backend+ )++ if scheme.act_bits <= 8 and (not scheme.is_act_standard_fp() or scheme.act_dynamic):+ if (scheme.is_act_nv_fp() and "static_gs" in scheme.act_data_type) or scheme.is_act_mx_fp():+ return None, scheme, layer_config, quant_block_list+ elif scheme.is_dynamic_afp8() and scheme.is_block_wfp8():+ return None, scheme, layer_config, quant_block_list+ else:+ bits, group_size, sym, act_bits = 8, -1, True, 8+ assert (+ scheme.bits == bits+ and scheme.group_size == group_size+ and scheme.sym == sym+ and scheme.act_bits == act_bits+ and scheme.act_dynamic+ ), (+ f"Currently only support to export llm_compressor format for sym dynamic quantized"+ f" W{scheme.bits}A{scheme.act_bits} model with group_size={group_size},"+ f" but got bits={scheme.bits}, group_size={scheme.group_size}, sym={scheme.sym},"+ f" act_bits={scheme.act_bits}"+ )+ return None, scheme, layer_config, quant_block_list+ return None, scheme, layer_config, quant_block_list++ def pack_layer(self, layer_name, model, device=None, **kwargs):+ if self.backend is not None:+ return self.backend.pack_layer(layer_name, model, device=device, **kwargs)+ if re.search(f"{BackendDataType.MX_FP.value}|{BackendDataType.NV_FP.value}", self.output_format):+ from auto_round.export.export_to_llmcompressor.export_to_fp import pack_layer++ return pack_layer(layer_name, model, device=device)+ elif re.search(f"{BackendDataType.FP8_STATIC.value}", self.output_format):+ from auto_round.export.export_to_llmcompressor.export_to_static_fp import pack_layer++ return pack_layer(layer_name, model, self.get_backend_name(), device=device)+ elif re.search(f"{BackendDataType.INT8.value}", self.output_format):+ from auto_round.export.export_to_llmcompressor.export import pack_layer++ return pack_layer(layer_name, model, device=device)+ elif re.search(f"{BackendDataType.FP8_BLOCK.value}", self.output_format):+ from auto_round.export.export_to_llmcompressor.export import pack_layer++ return pack_layer(layer_name, model, device=device)+ elif re.search(f"{BackendDataType.WINT_A16.value}", self.output_format):+ from auto_round.export.export_to_llmcompressor.export import pack_layer++ return pack_layer(layer_name, model, device=device)+ ## passed as no other llm_compressor format is supported yet+ logger.warning("No other llm_compressor packing format(except NVFP&MXFP) is supported yet, skip packing")+ return++ def save_quantized(+ self,+ output_dir: str,+ model: torch.nn.Module = None,+ tokenizer: Callable = None,+ layer_config: dict = None,+ inplace: bool = True,+ device: Union[str, torch.device] = "cpu",+ serialization_dict: dict = None,+ **kwargs,+ ) -> torch.nn.Module:+ backend = self.get_backend_name()+ if re.search(f"{BackendDataType.MX_FP.value}|{BackendDataType.NV_FP.value}", backend):+ from auto_round.export.export_to_llmcompressor.export_to_fp import save_quantized_as_fp++ export_func = save_quantized_as_fp+ elif re.search(f"{BackendDataType.FP8_STATIC.value}", backend):+ from auto_round.export.export_to_llmcompressor.export_to_static_fp import save_quantized_as_static_fp++ export_func = save_quantized_as_static_fp+ else:+ from auto_round.export.export_to_llmcompressor.export import save_quantized_as_llmcompressor++ export_func = save_quantized_as_llmcompressor+ return export_func(+ output_dir=output_dir,+ model=model,+ tokenizer=tokenizer,+ layer_config=layer_config,+ inplace=inplace,+ device=device,+ backend=backend,+ serialization_dict=serialization_dict,+ **kwargs,+ )diff --git a/auto_round/export/formats/backends/mlx.py b/auto_round/export/formats/backends/mlx.pynew file mode 100644index 000000000..37717a297--- /dev/null+++ b/auto_round/export/formats/backends/mlx.py@@ -0,0 +1,76 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++import re+from typing import Callable, Union++import torch++from auto_round.export.formats.base import OutputFormat+from auto_round.schemes import QuantizationScheme++[email protected]("mlx")+class MLXFormat(OutputFormat):+ support_schemes = ["W2A16", "W2A16G32", "W2A16G64", "W3A16", "W4A16", "W5A16", "W6A16", "W8A16", "BF16"]+ format_name = "mlx"++ @classmethod+ def check_scheme_args(cls: OutputFormat, scheme: QuantizationScheme) -> bool:+ error_logs = []+ if scheme.bits not in [2, 3, 4, 5, 6, 8, 16]:+ error_logs.append(f"bits={scheme.bits}")+ if scheme.act_bits != 16:+ error_logs.append(f"act_bits={scheme.act_bits}")+ if not re.search("int", scheme.data_type):+ error_logs.append(f"data_type={scheme.data_type}")+ if scheme.super_bits:+ error_logs.append(f"super_bits={scheme.super_bits}")+ if scheme.super_group_size:+ error_logs.append(f"super_group_size={scheme.super_group_size}")+ if error_logs:+ raise ValueError(+ f"MLX format support quantization scheme with {','.join(cls.support_schemes)} "+ f"but got {', '.join(error_logs)}, please have a check."+ )+ return True++ def pack_layer(self, layer_name, model, device=None, **kwargs):+ from auto_round.export.export_to_mlx.export import pack_layer++ pack_layer(layer_name, model, device=device, **kwargs)++ def save_quantized(+ self,+ output_dir: str,+ model: torch.nn.Module = None,+ tokenizer: Callable = None,+ layer_config: dict = None,+ inplace: bool = True,+ device: Union[str, torch.device] = "cpu",+ serialization_dict: dict = None,+ **kwargs,+ ) -> torch.nn.Module:+ from auto_round.export.export_to_mlx.export import save_quantized_as_mlx++ return save_quantized_as_mlx(+ output_dir=output_dir,+ model=model,+ tokenizer=tokenizer,+ layer_config=layer_config,+ inplace=inplace,+ device=device,+ serialization_dict=serialization_dict,+ **kwargs,+ )diff --git a/auto_round/export/formats/base.py b/auto_round/export/formats/base.pynew file mode 100644index 000000000..80636f99b--- /dev/null+++ b/auto_round/export/formats/base.py@@ -0,0 +1,237 @@+# Copyright (c) 2025 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++from __future__ import annotations++import copy+import os+import re+from abc import ABC, abstractmethod+from dataclasses import asdict+from enum import Enum+from typing import Any, Callable, Optional, Union++import torch+import transformers++from auto_round.compressors.config_resolution.errors import FormatCompatibilityError+from auto_round.export.export_to_gguf.config import ModelType+from auto_round.schemes import QuantizationScheme+from auto_round.utils import (+ INNER_SUPPORTED_LAYER_TYPES,+ SUPPORTED_LAYER_TYPES,+ check_to_quantized,+ compress_layer_names,+ copy_python_files_from_model_cache,+ find_matching_blocks,+ get_block_names,+ get_module,+ logger,+ unsupported_meta_device,+)+++class BackendDataType(str, Enum):+ """Quantized data-type/backend-variant identifiers used as the suffix after a format+ name (e.g. ``auto_round:fp8_static``, ``llm_compressor:int8_w8a8``). Shared across the+ ``auto_round`` and ``llm_compressor`` output formats -- not tied to a single format.+ """++ # Weight: FP8, per-channel, may be extended to per-tensor in future+ # Activation: FP8, per-tensor+ FP8_STATIC = "fp8_static"+ MXFP8 = "mxfp8"+ MXFP4 = "mxfp4"+ NVFP4 = "nvfp4"+ FP8 = "fp8"+ MX_FP = "mx_fp"+ NV_FP = "nv_fp"+ MX_FP_RCEIL = "mx_fp_rceil"+ NV_FP4_WITH_STATIC_GS = "nv_fp4_with_static_gs"+ INT8 = "int8_w8a8"+ FP8_BLOCK = "fp8_block"+ MXINT4 = "mxint4"+ MX_INT = "mx_int"+ WINT_A16 = "wint_a16"+++def _check_divisible_by_32(scheme: QuantizationScheme, model, layer_config: dict) -> dict:+ if model is None:+ return layer_config+ default_dict = asdict(scheme)+ skipped_layers = []+ if default_dict["data_type"] == "int" and default_dict["act_bits"] >= 16:+ for n, m in model.named_modules():+ if type(m) in SUPPORTED_LAYER_TYPES or m.__class__.__name__ in INNER_SUPPORTED_LAYER_TYPES:+ if m.weight.shape[0] % 32 or m.weight.shape[1] % 32:+ if layer_config is None:+ layer_config = {}+ if layer_config.get(n) is not None and layer_config[n]["bits"] >= 16:+ continue+ layer_config.setdefault(n, copy.deepcopy(default_dict))+ layer_config[n].update({"bits": 16, "data_type": "fp", "fixed_by_user": True})+ skipped_layers.append(n)+ compressed_skipped_layers = compress_layer_names(skipped_layers)+ if compressed_skipped_layers:+ logger.warning_once(+ f"some layers are skipped quantization (shape not divisible by 32): {compressed_skipped_layers}"+ )+ return layer_config+++class OutputFormat(ABC):+ """ "Base class for different output formats.++ format: determines which method from export module to use for exporting.+ For example, auto_round, gguf, llm_compressor etc.+ backend: determines the specific export process within the format.+ For example, auto_round:fp8_static, auto_round:auto_awq etc.+ """++ support_schemes: list = []+ _format_list: dict[str, OutputFormat] = {}+ format_name = "base"++ def __init__(self, format: str, scheme: QuantizationScheme, ctx: Any):+ """Initialize the OutputFormat class."""+ self.output_format = format+ self.backend = None+ self.mllm = ctx.mllm++ if not self.is_fake() and not self.is_support_scheme(scheme):+ raise FormatCompatibilityError(+ f"Currently, the {self.format_name} format only supports {self.support_schemes}, "+ f"but got scheme {scheme}, please change to fake or auto_round etc."+ )++ @classmethod+ def register(cls, *names: str) -> Callable[[OutputFormat], OutputFormat]:+ assert names++ def func(output_format: OutputFormat) -> OutputFormat:+ for name in names:+ cls._format_list[name] = output_format+ return output_format++ return func++ @classmethod+ def get_support_matrix(cls: OutputFormat) -> str:+ output_str = ""+ for k, v in sorted(cls._format_list.items()):+ if k == "fake":+ support_schemes = "All schemes"+ else:+ if ":" in k and k.split(":")[1] in cls._format_list:+ support_schemes = cls._format_list[k.split(":")[1]].support_schemes+ else:+ support_schemes = v.support_schemes+ support_schemes = ", ".join(support_schemes).rstrip(",")+ output_str += f"\x1b[31;1m{k}\x1b[0m support scheme:\n\t{support_schemes}\n"+ return output_str++ def get_backend_name(self) -> str:+ if self.backend is None:+ return self.output_format++ # auto_round:llm_compressor:fp8_static+ if self.backend.backend is not None:+ return f"{self.output_format}:{self.backend.get_backend_name()}"+ # auto_round:fp8_static, llm_compressor:fp8_static, auto_round:auto_awq+ else:+ return self.backend.get_backend_name()++ @classmethod+ def is_support_scheme(cls: OutputFormat, scheme: Union[str, QuantizationScheme]) -> bool:+ if isinstance(scheme, str) and scheme.upper() in cls.support_schemes:+ return True+ if isinstance(scheme, QuantizationScheme):+ return cls.check_scheme_args(scheme)+ return False++ @classmethod+ def check_scheme_args(cls: OutputFormat, scheme: QuantizationScheme) -> bool:+ return True++ def check_and_reset_format(+ self,+ scheme: QuantizationScheme,+ ctx: Any,+ ) -> tuple[Optional[str], QuantizationScheme, dict, list]:+ layer_config, quant_block_list = ctx.layer_config, ctx.quant_block_list+ if self.backend is not None:+ new_format, scheme, layer_config, quant_block_list = self.backend.check_and_reset_format(scheme, ctx)+ ctx.layer_config, ctx.quant_block_list = layer_config, quant_block_list+ self.backend = (+ OutputFormat._format_list[new_format](new_format, scheme, ctx) if new_format else self.backend+ )++ w_fp8 = scheme.data_type.startswith("fp") and scheme.bits == 8+ act_fp8 = scheme.act_data_type.startswith("fp") and scheme.act_bits == 8+ is_block_dynamic_fp8 = (+ self.format_name in ["fp8", "auto_round:fp8"]+ and isinstance(scheme.group_size, tuple)+ and scheme.act_dynamic+ )+ if (w_fp8 or act_fp8) and not is_block_dynamic_fp8:+ error_msg = (+ f"is only supported to export auto_round or llm_compressor format,"+ f" but got {self.format_name}, please check."+ )+ error_msg = ("act_data_type<fp8> " + error_msg) if act_fp8 else error_msg+ error_msg = ("data_type<fp8> " + error_msg) if w_fp8 else error_msg+ raise FormatCompatibilityError(error_msg)++ if (+ scheme.act_bits <= 8+ and (not scheme.is_act_standard_fp() or scheme.act_dynamic)+ and not is_block_dynamic_fp8+ ):+ logger.warning(+ f"{self.format_name} format does not support the current activation quantization configuration,"+ " reset to fake format and save."+ )+ return "fake", scheme, layer_config, quant_block_list++ return None, scheme, layer_config, quant_block_list++ @abstractmethod+ def pack_layer(self, *args, **kwargs):+ pass++ @abstractmethod+ def save_quantized(self, *args, **kwargs):+ pass++ def immediate_pack(self, name: str, model: torch.nn.Module, device: torch.device, **kwargs):+ m = get_module(model, name)+ if not check_to_quantized(m):+ return++ self.pack_layer(name, model, device=device)++ def is_gguf(self) -> bool:+ return "gguf" in self.output_format++ def is_fake(self) -> bool:+ return self.output_format == "fake"++ def is_gptq(self) -> bool:+ return "gptq" in self.output_format or (self.backend is not None and self.backend.is_gptq())++ def is_awq(self) -> bool:+ return "awq" in self.output_format or (self.backend is not None and self.backend.is_awq())++ def is_llm_compressor(self) -> bool:+ return "llm_compressor" in self.output_format or (self.backend is not None and self.backend.is_llm_compressor())diff --git a/auto_round/export/formats/resolver.py b/auto_round/export/formats/resolver.pynew file mode 100644index 000000000..73c6739c1--- /dev/null+++ b/auto_round/export/formats/resolver.py@@ -0,0 +1,174 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++from __future__ import annotations++from types import SimpleNamespace+from typing import Iterable++import torch++from auto_round.compressors.config_resolution import (+ FormatCompatibilityError,+ FormatResolution,+ ResolvedScheme,+ thaw_mapping,+)+from auto_round.export.formats.backends.gguf import GGUFFormat+from auto_round.export.formats.base import OutputFormat+from auto_round.schemes import get_gguf_scheme+from auto_round.utils import SUPPORTED_FORMATS, logger+++def _deduplicate(names: Iterable[str]) -> list[str]:+ return list(dict.fromkeys(names))+++def _normalize_format_names(value: str, bits: int) -> list[str]:+ names = value.lower().replace("q*_", f"q{bits}_").replace(" ", "").split(",")+ return _deduplicate(name for name in names if name)+++def _validate_format_combination(names: list[str]) -> None:+ has_gguf = any(name.startswith("gguf") for name in names)+ real_companions = [name for name in names if name != "fake" and not name.startswith("gguf")]+ if has_gguf and real_companions:+ raise FormatCompatibilityError(+ f"GGUF format is not compatible with other formats, but got {names}, please choose only one of them"+ )+++def _apply_scheme_format_constraint(names: list[str], scheme: ResolvedScheme) -> list[str]:+ preset_name = scheme.preset_name or ""+ gguf_name = preset_name.lower() if preset_name.lower().startswith("gguf:") else get_gguf_scheme(scheme.value)+ if not gguf_name:+ return names+ if gguf_name.endswith("_mixed"):+ gguf_name = gguf_name.replace("_mixed", "_s")+ if all(name in {"fake", gguf_name} for name in names):+ return names+ if any(name.startswith("gguf") for name in names if name != "fake"):+ logger.warning(+ "scheme %s is GGUF, but format %s specifies a different GGUF type. "+ "The scheme-driven per-layer quantization may differ from the file-level GGUF format type.",+ gguf_name,+ ",".join(names),+ )+ return names+ reset_names = [gguf_name]+ if "fake" in names:+ reset_names.append("fake")+ logger.warning(+ "reset format %s to %s since scheme %s can only be exported to format %s or fake",+ ",".join(names),+ ",".join(reset_names),+ gguf_name,+ gguf_name,+ )+ return reset_names+++def _precise_gguf_name(formats: list[OutputFormat]) -> str | None:+ for output_format in formats:+ if not output_format.is_gguf():+ continue+ backend = getattr(output_format, "backend", None)+ backend_name = getattr(backend, "output_format", None)+ precise = backend_name if backend_name and backend_name != "gguf" else output_format.output_format+ if precise and precise != "gguf":+ return precise.lower()+ return None+++def resolve_formats(+ scheme: ResolvedScheme,+ *,+ format: str = None,+ layer_config: dict = None,+ scale_dtype=None,+ quant_block_list=None,+ mllm: bool = False,+ iters: int = 0,+ enable_alg_ext: bool = False,+ quant_nontext_module: bool = False,+ platform: str = None,+ is_auto_scheme: bool = False,+ model=None,+) -> FormatResolution:+ """Resolve format selection using caller-isolated state and no exporter capability checks."""+ scheme_value = scheme.value+ names = _normalize_format_names(format or "auto_round", scheme_value.bits)+ _validate_format_combination(names)+ names = _deduplicate(_apply_scheme_format_constraint(names, scheme))++ for name in names:+ if name not in SUPPORTED_FORMATS:+ raise ValueError(f"{name} is not supported, we only support {SUPPORTED_FORMATS}")++ layer_config = thaw_mapping(layer_config)+ quant_block_list = [list(group) for group in quant_block_list] if quant_block_list is not None else None+ context = SimpleNamespace(+ model=model,+ layer_config=layer_config,+ scale_dtype=scale_dtype,+ mllm=mllm,+ iters=iters,+ enable_alg_ext=enable_alg_ext,+ quant_nontext_module=quant_nontext_module,+ quant_block_list=quant_block_list,+ platform=platform,+ is_auto_scheme=is_auto_scheme,+ )++ selected = []+ requested = format or "auto_round"+ for name in names:+ if name.startswith("gguf:"):+ output_format, scheme_value, context.layer_config = GGUFFormat.build(name, scheme_value, context)+ elif name in OutputFormat._format_list:+ output_format = OutputFormat._format_list[name](name, scheme_value, context)+ else:+ raise KeyError(f"Unsupported format {name}, please choose from {SUPPORTED_FORMATS}")++ new_name, scheme_value, context.layer_config, context.quant_block_list = output_format.check_and_reset_format(+ scheme_value, context+ )+ if new_name is not None:+ if new_name in requested:+ continue+ output_format = OutputFormat._format_list[new_name](new_name, scheme_value, context)+ selected.append(output_format)++ if any(output_format.is_gguf() for output_format in selected) and any(+ output_format.is_fake() for output_format in selected+ ):+ selected.sort(key=lambda output_format: 0 if output_format.is_fake() else 1)++ scale_dtype = context.scale_dtype+ if len(selected) == 1 and selected[0].is_gguf() and scale_dtype != torch.float32:+ scale_dtype = torch.float32+ logger.info("change `scale_dtype` to `torch.float32` for gguf format")++ precise_gguf_name = _precise_gguf_name(selected)+ resolved_scheme = ResolvedScheme.from_scheme(+ scheme_value,+ preset_name=precise_gguf_name or scheme.preset_name,+ )+ return FormatResolution(+ formats=tuple(selected),+ scheme=resolved_scheme,+ layer_config_patch=context.layer_config,+ scale_dtype=scale_dtype,+ quant_block_list=context.quant_block_list,+ )diff --git a/auto_round/formats.py b/auto_round/formats.pyindex 314ceb9cf..e28ed025c 100644--- a/auto_round/formats.py+++ b/auto_round/formats.py@@ -1,4 +1,4 @@-# Copyright (c) 2025 Intel Corporation+# Copyright (c) 2026 Intel Corporation## Licensed under the Apache License, Version 2.0 (the "License");# you may not use this file except in compliance with the License.@@ -12,56 +12,29 @@# See the License for the specific language governing permissions and# limitations under the License.-from __future__ import annotations+"""Backward-compatible imports for the relocated output-format API."""-import copy-import os-import re-import sys-from abc import ABC, abstractmethod-from dataclasses import asdictfrom enum import Enum-from typing import TYPE_CHECKING, Callable, Union-import torch-import transformers--from auto_round.compressors.utils import (- is_act_static,- is_block_wfp8,- is_dynamic_afp8,- is_dynamic_wint8aint8,- is_mx_fp,- is_mx_int,- is_nv_fp,- is_standard_fp,- is_wfp8afp8,- is_wint_woq,-)-from auto_round.export.export_to_gguf.config import ModelType-from auto_round.schemes import (- PRESET_SCHEMES,- QuantizationScheme,- get_gguf_scheme,-)-from auto_round.utils import (- INNER_SUPPORTED_LAYER_TYPES,- SUPPORTED_FORMATS,- SUPPORTED_LAYER_TYPES,- check_to_quantized,- compress_layer_names,- copy_python_files_from_model_cache,- find_matching_blocks,- get_block_names,- get_module,- logger,- unsupported_meta_device,+from auto_round.compressors.config_resolution import ResolvedScheme+from auto_round.export.formats import (+ AutoAWQFormat,+ AutoGPTQFormat,+ AutoRoundFormat,+ BackendDataType,+ FakeFormat,+ FP8Format,+ GGUFFormat,+ LLMCompressorFormat,+ MLXFormat,+ OutputFormat,+ resolve_formats,)class AutoRoundExportFormat(str, Enum):- # Weight: FP8, per-channel, may be extended to per-tensor in future- # Activation: FP8, per-tensor+ """Legacy names retained for callers importing the old module."""+FP8_STATIC = "fp8_static"MXFP8 = "mxfp8"MXFP4 = "mxfp4"@@ -78,1246 +51,37 @@ class AutoRoundExportFormat(str, Enum):WINT_A16 = "wint_a16"-if TYPE_CHECKING:- from auto_round.compressors.base import BaseOrchestrator as BaseCompressor---def _check_compatibility(formats: list[str], ar: BaseCompressor):- if (- any(["gguf" in f.lower() for f in formats])- and len([f for f in formats if f.lower() != "fake" and not f.lower().startswith("gguf")]) > 1- ):- raise ValueError(- f"GGUF format is not compatible with other formats, but got {formats}, please choose only one of them"- )- gguf_format_name = get_gguf_scheme(ar.scheme)- if gguf_format_name:- if gguf_format_name.lower().endswith("mixed"):- gguf_format_name = gguf_format_name.lower().replace("_mixed", "_s")- if any([f.lower() not in ["fake", gguf_format_name.lower()] for f in formats]):- has_gguf_format = any(f.lower().startswith("gguf") for f in formats if f.lower() != "fake")- if has_gguf_format:- logger.warning(- f"scheme {gguf_format_name} is GGUF, but format {','.join(formats)} specifies "- f"a different GGUF type. The scheme-driven per-layer quantization may differ from the "- f"file-level GGUF format type."- )- else:- tmp_format_name = (- gguf_format_name.lower() if "fake" not in formats else (f"{gguf_format_name.lower()},fake")- )- logger.warning(- f"reset format {','.join(formats)} to {tmp_format_name} "- f"since scheme {gguf_format_name} can only be exported to format "- f"{gguf_format_name.lower()} or fake"- )- formats = tmp_format_name.split(",")-- if isinstance(ar.group_size, tuple) and any(["auto_round" in f.lower() for f in formats]):- logger.warning(- "auto_round:fp8 format only supports vLLM inference for now. "- "We recommend using the FP8 format via `--format fp8` instead."- )-- return formats---def get_formats(- format: str,- ar: BaseCompressor,-) -> list[OutputFormat]:- """Get the list of OutputFormat instances based on the provided name."""-- def remove_duplicates(lst):- seen = set()- return [x for x in lst if not (x in seen or seen.add(x))]-- formats = format.lower().replace("q*_", f"q{ar.bits}_").replace(" ", "").split(",")- formats = remove_duplicates(formats) # need the keep origin order-- formats = _check_compatibility(formats, ar)-- formats = remove_duplicates(formats)-- for fmt in formats:- if fmt not in SUPPORTED_FORMATS:- raise ValueError(f"{fmt} is not supported, we only support {SUPPORTED_FORMATS}")-- for i in range(len(formats)):- if formats[i].startswith("gguf:"):- formats[i] = GGUFFormat(formats[i], ar)- elif formats[i] not in OutputFormat._format_list:- raise KeyError(f"Unsupported format {formats[i]}, please choose from {SUPPORTED_FORMATS}")- else:- formats[i] = OutputFormat._format_list[formats[i]](formats[i], ar)-- new_format = formats[i].check_and_reset_format(ar)- if new_format is not None:- if new_format not in format:- formats[i] = OutputFormat._format_list[new_format](new_format, ar)- else:- formats[i] = None-- formats = [fmt for fmt in formats if fmt is not None]-- # Ensure fake format is processed before GGUF — GGUF export may clear- # model weights via low_cpu_mem_usage, causing zeroed weights for fake.- if any(fmt.is_gguf() for fmt in formats) and any(fmt.is_fake() for fmt in formats):- formats.sort(key=lambda f: 0 if f.is_fake() else 1)-- if len(formats) == 1 and formats[0].is_gguf() and ar.scale_dtype != torch.float32:- ar.scale_dtype = torch.float32- logger.info("change `scale_dtype` to `torch.float32` for gguf format")-- return formats---def _check_divisible_by_32(ar):- from auto_round.schemes import preset_name_to_scheme-- if isinstance(ar.scheme, str):- default_dict = asdict(preset_name_to_scheme(ar.scheme.upper()))- else:- default_dict = asdict(ar.scheme)- skipped_layers = []- if default_dict["data_type"] == "int" and default_dict["act_bits"] >= 16:- for n, m in ar.model.named_modules():- if type(m) in SUPPORTED_LAYER_TYPES or m.__class__.__name__ in INNER_SUPPORTED_LAYER_TYPES:- if m.weight.shape[0] % 32 or m.weight.shape[1] % 32:- if ar.layer_config is None:- ar.layer_config = {}- if ar.layer_config.get(n) is not None and ar.layer_config[n]["bits"] >= 16:- continue- ar.layer_config.setdefault(n, copy.deepcopy(default_dict))- ar.layer_config[n].update({"bits": 16, "data_type": "fp", "fixed_by_user": True})- skipped_layers.append(n)- compressed_skipped_layers = compress_layer_names(skipped_layers)- if compressed_skipped_layers:- logger.warning_once(- f"some layers are skipped quantization (shape not divisible by 32): {compressed_skipped_layers}"- )---class OutputFormat(ABC):- """ "Base class for different output formats.-- format: determines which method from export module to use for exporting.- For example, auto_round, gguf, llm_compressor etc.- backend: determines the specific export process within the format.- For example, auto_round:fp8_static, auto_round:auto_awq etc.- """-- support_schemes: list = []- _format_list: dict[str, OutputFormat] = {}- format_name = "base"-- def __init__(self, format: str, ar: BaseCompressor):- """Initialize the OutputFormat class."""- self.output_format = format- self.backend = None-- if not self.is_fake() and not self.is_support_scheme(ar.scheme):- logger.error(- f"Currently, the {self.format_name} format only supports {self.support_schemes}, "- f"but got scheme {ar.scheme}, please change to fake or auto_round etc."- )- exit(-1)-- @classmethod- def register(cls, *names: str) -> Callable[[OutputFormat], OutputFormat]:- assert names-- def func(output_format: OutputFormat) -> OutputFormat:- for name in names:- cls._format_list[name] = output_format- return output_format-- return func-- @classmethod- def get_support_matrix(cls: OutputFormat) -> str:- output_str = ""- for k, v in sorted(cls._format_list.items()):- if k == "fake":- support_schemes = "All schemes"- else:- if ":" in k and k.split(":")[1] in cls._format_list:- support_schemes = cls._format_list[k.split(":")[1]].support_schemes- else:- support_schemes = v.support_schemes- support_schemes = ", ".join(support_schemes).rstrip(",")- output_str += f"\x1b[31;1m{k}\x1b[0m support scheme:\n\t{support_schemes}\n"- return output_str-- def get_backend_name(self) -> str:- if self.backend is None:- return self.output_format-- # auto_round:llm_compressor:fp8_static- if self.backend.backend is not None:- return f"{self.output_format}:{self.backend.get_backend_name()}"- # auto_round:fp8_static, llm_compressor:fp8_static, auto_round:auto_awq- else:- return self.backend.get_backend_name()-- @classmethod- def is_support_scheme(cls: OutputFormat, scheme: Union[str, QuantizationScheme]) -> bool:- if isinstance(scheme, str) and scheme.upper() in cls.support_schemes:- return True- if isinstance(scheme, QuantizationScheme):- return cls.check_scheme_args(scheme)- return False-- @classmethod- def check_scheme_args(cls: OutputFormat, scheme: QuantizationScheme) -> bool:- return True-- def check_and_reset_format(self, ar: BaseCompressor) -> str:- if self.backend is not None:- new_format = self.backend.check_and_reset_format(ar)- self.backend = OutputFormat._format_list[new_format](new_format, ar) if new_format else self.backend-- w_fp8 = ar.data_type.startswith("fp") and ar.bits == 8- act_fp8 = ar.act_data_type.startswith("fp") and ar.act_bits == 8- is_block_dynamic_fp8 = (- self.format_name in ["fp8", "auto_round:fp8"] and isinstance(ar.group_size, tuple) and ar.act_dynamic- )- if (w_fp8 or act_fp8) and not is_block_dynamic_fp8:- error_msg = (- f"is only supported to export auto_round or llm_compressor format,"- f" but got {self.format_name}, please check."- )- error_msg = ("act_data_type<fp8> " + error_msg) if act_fp8 else error_msg- error_msg = ("data_type<fp8> " + error_msg) if w_fp8 else error_msg- logger.error(error_msg)- sys.exit(-1)-- if ar.act_bits <= 8 and (not is_standard_fp(ar.act_data_type) or ar.act_dynamic) and not is_block_dynamic_fp8:- logger.warning(- f"{self.format_name} format does not support the current activation quantization configuration,"- " reset to fake format and save."- )- return "fake"-- return None-- @abstractmethod- def pack_layer(self, *args, **kwargs):- pass-- @abstractmethod- def save_quantized(self, *args, **kwargs):- pass-- def immediate_pack(self, name: str, model: torch.nn.Module, device: torch.device, **kwargs):- m = get_module(model, name)- if not check_to_quantized(m):- return-- self.pack_layer(name, model, device=device)-- def is_gguf(self) -> bool:- return "gguf" in self.output_format-- def is_fake(self) -> bool:- return self.output_format == "fake"-- def is_gptq(self) -> bool:- return "gptq" in self.output_format or (self.backend is not None and self.backend.is_gptq())-- def is_awq(self) -> bool:- return "awq" in self.output_format or (self.backend is not None and self.backend.is_awq())-- def is_llm_compressor(self) -> bool:- return "llm_compressor" in self.output_format or (self.backend is not None and self.backend.is_llm_compressor())--[email protected]("fake")-class FakeFormat(OutputFormat):- support_schemes = None- format_name = "fake"-- def check_and_reset_format(self, ar: BaseCompressor) -> str:- return None-- # fake format will not execute pack_layer.- def pack_layer(self, *args, **kwargs):- pass-- def save_quantized(- self,- output_dir: str,- model: torch.nn.Module = None,- tokenizer: Callable = None,- layer_config: dict = None,- inplace: bool = True,- device: Union[str, torch.device] = "cpu",- serialization_dict: dict = None,- **kwargs,- ):- if not unsupported_meta_device(model):- model = model.to("cpu")- model.save_pretrained(output_dir)- elif hasattr(model, "config") and model.config is not None:- model.config.save_pretrained(output_dir)-- if tokenizer is not None and hasattr(tokenizer, "save_pretrained"):- tokenizer.save_pretrained(output_dir)- processor = kwargs.get("processor", None)- if processor is not None:- processor.save_pretrained(output_dir)- try:- copy_python_files_from_model_cache(model, output_dir)- except Exception as e:- logger.warning("Skipping source model Python file copy due to error: %s", e)- return model--[email protected]("llm_compressor")-class LLMCompressorFormat(OutputFormat):- support_schemes = [- "MXFP4",- "MXFP8",- "NVFP4",- "FPW8A16",- "FP8_STATIC",- "INT8",- "INT8_W8A8",- "FP8_BLOCK",- "W4A16",- "W8A16",- ]- format_name = "llm_compressor"-- def __init__(self, format, ar):- if not self.is_support_scheme(ar.scheme):- logger.error(- f"Currently, the llm_compressor format only supports {self.support_schemes}, "- f"but got scheme {ar.scheme}, please change to fake or auto_round etc."- )- exit(-1)- # if format.startswith("llm_compressor"):- if re.search("^(auto_round:)?llm_compressor", format):- self.output_format = format- self.backend = None- if is_nv_fp(ar.data_type) or is_mx_fp(ar.data_type):- from auto_round.export.export_to_llmcompressor import check_compressed_tensors_supported-- check_compressed_tensors_supported(raise_error=True)- self.backend = LLMCompressorFormat(ar.data_type, ar)- elif is_dynamic_afp8(ar) and is_block_wfp8(ar):- self.backend = LLMCompressorFormat(AutoRoundExportFormat.FP8_BLOCK.value, ar)- elif is_act_static(ar):- self.backend = LLMCompressorFormat(AutoRoundExportFormat.FP8_STATIC.value, ar)- if ar.act_group_size != 0:- logger.warning(- f"scheme FP8_STATIC export to llm_compressor format only support for act_group_size 0,"- f" ,but got act_group_size={ar.act_group_size}, reset = 0"- )- ar.act_group_size = 0- if ar.group_size > 0:- logger.warning(- f"please note that group_size={ar.group_size}"- " may not be supported for llm_compressor format, and cannot be loaded in llm_compressor"- )- elif is_dynamic_wint8aint8(ar):- from auto_round.export.export_to_llmcompressor import check_compressed_tensors_supported-- check_compressed_tensors_supported()- self.backend = LLMCompressorFormat(AutoRoundExportFormat.INT8.name, ar)- self.backend.output_format = f"llm_compressor:{AutoRoundExportFormat.INT8.value}"- elif is_wint_woq(ar):- from auto_round.export.export_to_llmcompressor import check_compressed_tensors_supported-- check_compressed_tensors_supported()- self.backend = LLMCompressorFormat(AutoRoundExportFormat.WINT_A16.value, ar)- else:- if format.upper() not in list(AutoRoundExportFormat.__members__.keys()):- raise KeyError(f"Unsupported backend format llm_compressor:{format}, please check")- self.output_format = f"llm_compressor:{format}"- self.backend = None-- @classmethod- def check_scheme_args(cls: OutputFormat, scheme: QuantizationScheme) -> bool:- error_logs = []- if scheme.bits not in [4, 8, 16]:- error_logs.append(f"bits={scheme.bits}")- if not re.search("mxfp|fp|nvfp|int", scheme.data_type):- error_logs.append(f"data_type={scheme.data_type}")- if scheme.data_type == "fp" and scheme.bits != 8:- error_logs.append(f"data_type={scheme.data_type}, bits={scheme.bits}")- if scheme.data_type == "int" and scheme.bits not in [4, 8]:- error_logs.append(f"data_type={scheme.data_type}, bits={scheme.bits}")- if scheme.super_bits:- error_logs.append(f"super_bits={scheme.super_bits}")- if scheme.super_group_size:- error_logs.append(f"super_group_size={scheme.super_group_size}")- if isinstance(scheme.group_size, tuple):- if scheme.bits != 8:- error_logs.append(f"bits={scheme.bits}")- if scheme.data_type != "fp":- error_logs.append(f"data_type={scheme.data_type}")- if len(scheme.group_size) != 2:- error_logs.append(f"group_size={scheme.group_size}")- if not scheme.act_dynamic:- error_logs.append(f"act_dynamic={scheme.act_dynamic}")- if not isinstance(scheme.act_group_size, int):- error_logs.append(f"act_group_size={scheme.act_group_size}")- if scheme.act_bits != 8:- error_logs.append(f"act_bits={scheme.act_bits}")- if scheme.act_data_type != "fp":- error_logs.append(f"act_data_type={scheme.act_data_type}")- if error_logs:- raise ValueError(- f"LLMCompressor format support quantization scheme with {','.join(cls.support_schemes)} "- f"but got {', '.join(error_logs)}, please have a check."- )- return True-- def check_and_reset_format(self, ar: BaseCompressor) -> str | None:- if self.backend is not None:- new_format = self.backend.check_and_reset_format(ar)- self.backend = OutputFormat._format_list[new_format](new_format, ar) if new_format else self.backend-- if ar.act_bits <= 8 and (not is_standard_fp(ar.act_data_type) or ar.act_dynamic):- if (is_nv_fp(ar.act_data_type) and "static_gs" in ar.act_data_type) or (is_mx_fp(ar.act_data_type)):- return None- elif is_dynamic_afp8(ar) and is_block_wfp8(ar):- return None- else:- bits, group_size, sym, act_bits = 8, -1, True, 8- assert (- ar.bits == bits- and ar.group_size == group_size- and ar.sym == sym- and ar.act_bits == act_bits- and ar.act_dynamic- ), (- f"Currently only support to export llm_compressor format for sym dynamic quantized"- f" W{ar.bits}A{ar.act_bits} model with group_size={group_size},"- f" but got bits={ar.bits}, group_size={ar.group_size}, sym={ar.sym},"- f" act_bits={ar.act_bits}"- )- return None- return None-- def pack_layer(self, layer_name, model, device=None, **kwargs):- if self.backend is not None:- return self.backend.pack_layer(layer_name, model, device=device, **kwargs)- if re.search(f"{AutoRoundExportFormat.MX_FP.value}|{AutoRoundExportFormat.NV_FP.value}", self.output_format):- from auto_round.export.export_to_llmcompressor.export_to_fp import pack_layer-- return pack_layer(layer_name, model, device=device)- elif re.search(f"{AutoRoundExportFormat.FP8_STATIC.value}", self.output_format):- from auto_round.export.export_to_llmcompressor.export_to_static_fp import pack_layer-- return pack_layer(layer_name, model, self.get_backend_name(), device=device)- elif re.search(f"{AutoRoundExportFormat.INT8.value}", self.output_format):- from auto_round.export.export_to_llmcompressor.export import pack_layer-- return pack_layer(layer_name, model, device=device)- elif re.search(f"{AutoRoundExportFormat.FP8_BLOCK.value}", self.output_format):- from auto_round.export.export_to_llmcompressor.export import pack_layer-- return pack_layer(layer_name, model, device=device)- elif re.search(f"{AutoRoundExportFormat.WINT_A16.value}", self.output_format):- from auto_round.export.export_to_llmcompressor.export import pack_layer-- return pack_layer(layer_name, model, device=device)- ## passed as no other llm_compressor format is supported yet- logger.warning("No other llm_compressor packing format(except NVFP&MXFP) is supported yet, skip packing")- return-- def save_quantized(- self,- output_dir: str,- model: torch.nn.Module = None,- tokenizer: Callable = None,- layer_config: dict = None,- inplace: bool = True,- device: Union[str, torch.device] = "cpu",- serialization_dict: dict = None,- **kwargs,- ) -> torch.nn.Module:- backend = self.get_backend_name()- if re.search(f"{AutoRoundExportFormat.MX_FP.value}|{AutoRoundExportFormat.NV_FP.value}", backend):- from auto_round.export.export_to_llmcompressor.export_to_fp import save_quantized_as_fp-- export_func = save_quantized_as_fp- elif re.search(f"{AutoRoundExportFormat.FP8_STATIC.value}", backend):- from auto_round.export.export_to_llmcompressor.export_to_static_fp import save_quantized_as_static_fp-- export_func = save_quantized_as_static_fp- else:- from auto_round.export.export_to_llmcompressor.export import save_quantized_as_llmcompressor-- export_func = save_quantized_as_llmcompressor- return export_func(- output_dir=output_dir,- model=model,- tokenizer=tokenizer,- layer_config=layer_config,- inplace=inplace,- device=device,- backend=backend,- serialization_dict=serialization_dict,- **kwargs,- )--[email protected]("auto_gptq", "gptqmodel")-class AutoGPTQFormat(OutputFormat):- support_schemes = ["W4A16", "W2A16", "W3A16", "W8A16", "BF16", "W2A16G64", "W2A16G32", "W4A16_MIXED"]- format_name = "auto_gptq"-- def check_and_reset_format(self, ar):- if not ar.sym:- logger.warning(- "the asymmetrical kernel of the GPTQ format may result in a noticeable accuracy drop,"- " particularly for 2-bit quantization and smaller models."- " We recommend exporting to either the AutoAWQ format ( only 4 bits) or "- "the AutoRound format(2/3/4/8 bits)."- )- if self.backend is None:- _check_divisible_by_32(ar)- return super().check_and_reset_format(ar)-- @classmethod- def check_scheme_args(cls: OutputFormat, scheme: QuantizationScheme) -> bool:- error_logs = []- if scheme.bits not in [2, 3, 4, 8, 16]:- error_logs.append(f"bits={scheme.bits}")- if not re.search("int", scheme.data_type):- error_logs.append(f"data_type={scheme.data_type}")- if scheme.super_bits:- error_logs.append(f"super_bits={scheme.super_bits}")- if scheme.super_group_size:- error_logs.append(f"super_group_size={scheme.super_group_size}")- if error_logs:- raise ValueError(- f"{cls.format_name} format support quantization scheme with {','.join(cls.support_schemes)} "- f"but got {', '.join(error_logs)}, please have a check."- )- return True-- def pack_layer(self, layer_name, model, device=None, **kwargs):- if self.output_format.startswith("auto_round"):- from auto_round.export.export_to_autoround.export import pack_layer-- pack_layer(layer_name, model, backend=self.output_format, device=device)- else:- from auto_round.export.export_to_autogptq.export import pack_layer-- pack_layer(layer_name, model, backend=self.output_format, device=device)-- def save_quantized(- self,- output_dir: str,- model: torch.nn.Module = None,- tokenizer: Callable = None,- layer_config: dict = None,- inplace: bool = True,- device: Union[str, torch.device] = "cpu",- serialization_dict: dict = None,- **kwargs,- ) -> torch.nn.Module:- backend = self.get_backend_name()- if backend == "auto_round:auto_gptq" or backend == "auto_round:gptqmodel":- from auto_round.export.export_to_autoround.export import save_quantized_as_autoround-- export_func = save_quantized_as_autoround- else:- from auto_round.export.export_to_autogptq.export import save_quantized_as_autogptq-- export_func = save_quantized_as_autogptq- return export_func(- output_dir=output_dir,- model=model,- tokenizer=tokenizer,- layer_config=layer_config,- inplace=inplace,- device=device,- backend=backend,- serialization_dict=serialization_dict,- **kwargs,- )--[email protected]("auto_awq")-class AutoAWQFormat(OutputFormat):- support_schemes = ["W4A16"]- format_name = "auto_awq"-- @classmethod- def check_scheme_args(cls: OutputFormat, scheme: QuantizationScheme) -> bool:- error_logs = []- if scheme.bits != 4:- error_logs.append(f"bits={scheme.bits}")- if not re.search("int", scheme.data_type):- error_logs.append(f"data_type={scheme.data_type}")- if scheme.super_bits:- error_logs.append(f"super_bits={scheme.super_bits}")- if scheme.super_group_size:- error_logs.append(f"super_group_size={scheme.super_group_size}")- if error_logs:- raise ValueError(- f"{cls.format_name} format support quantization scheme with {','.join(cls.support_schemes)} "- f"but got {', '.join(error_logs)}, please have a check."- )- return True-- @staticmethod- def check_awq_gemm_compatibility(model, bits, group_size, sym, layer_configs=None):- """Checks if a model is compatible with the AutoAWQ GEMM kernel.-- Args:- model: The model object to evaluate, typically a PyTorch model.- bits (int): The number of bits for quantization (must be 4 for compatibility).- group_size (int): The group size for quantization.- sym (bool): Whether symmetric quantization is used (not utilized in the current function logic).- layer_configs (dict, optional): A dictionary mapping layer names to configurations, where each- configuration can specify a custom number of bits for the layer.-- Returns:- tuple: A tuple containing:- - bool: `True` if the model is compatible, `False` otherwise.- - str: An error message describing why the model is incompatible, or an empty string if compatible.- """- from auto_round.utils.model import get_layer_names_in_block, get_module-- if bits != 4:- return False, "AutoAWQ GEMM kernel only supports 4 bits"- for n, m in model.named_modules():- if type(m) == transformers.pytorch_utils.Conv1D:- return False, "AutoAWQ GEMM kernel does not support conv1d"-- layer_names = get_layer_names_in_block(model)- for layer_name in layer_names:- if (- layer_configs is not None- and layer_name in layer_configs.keys()- and layer_configs[layer_name].get("bits", bits) > 8- ):- continue-- layer = get_module(model, layer_name)- if layer.in_features % group_size != 0:- return False, f"Layer {layer_name} in_features is not multiple of group_size {group_size}"- if layer.out_features % (32 // bits) != 0:- return False, f"Layer {layer_name} out_features is not multiple of 32 // bits"-- return True, ""-- def check_and_reset_format(self, ar):- awq_supported, info = self.check_awq_gemm_compatibility(- ar.model, ar.bits, ar.group_size, ar.sym, ar.layer_config- )- if not awq_supported:- logger.warning(f"The AutoAWQ format may not be supported due to {info}")- if ar.bits != 4:- raise ValueError(f"auto_awq format support quantization scheme with W4A16 but got bits={ar.bits}")-- if self.backend is None:- _check_divisible_by_32(ar)-- return super().check_and_reset_format(ar)-- def pack_layer(self, layer_name, model, device=None, **kwargs):- from auto_round.export.export_to_awq.export import pack_layer-- pack_layer(layer_name, model, backend=self.output_format, device=device)-- def save_quantized(- self,- output_dir: str,- model: torch.nn.Module = None,- tokenizer: Callable = None,- layer_config: dict = None,- inplace: bool = True,- device: Union[str, torch.device] = "cpu",- serialization_dict: dict = None,- **kwargs,- ) -> torch.nn.Module:- backend = self.get_backend_name()- if backend == "auto_round:auto_awq":- from auto_round.export.export_to_autoround.export import save_quantized_as_autoround-- export_func = save_quantized_as_autoround- else:- from auto_round.export.export_to_awq.export import save_quantized_as_autoawq-- export_func = save_quantized_as_autoawq-- return export_func(- output_dir=output_dir,- model=model,- tokenizer=tokenizer,- layer_config=layer_config,- inplace=inplace,- backend=backend,- device=device,- serialization_dict=serialization_dict,- **kwargs,- )--[email protected]("gguf")-class GGUFFormat(OutputFormat):- support_schemes = [- "GGUF:Q4_0",- "GGUF:Q4_1",- "GGUF:Q5_0",- "GGUF:Q5_1",- "GGUF:Q2_K_S",- "GGUF:Q3_K_S",- "GGUF:Q3_K_M",- "GGUF:Q3_K_L",- "GGUF:Q4_K_S",- "GGUF:Q4_K_M",- "GGUF:Q5_K_S",- "GGUF:Q5_K_M",- "GGUF:Q6_K",- "GGUF:Q8_0",- "GGUF:Q2_K_MIXED",- ]- format_name = "gguf"-- def __init__(self, format: str, ar: BaseCompressor):- self.is_auto_scheme = bool(getattr(ar, "is_auto_scheme", False))- if format.startswith("gguf:"):- self._original_format = format # preserve "gguf:q2_k_mixed" etc. for Phase 2b- self.output_format = "gguf"- self.backend_cls = GGUFFormat- self.backend = GGUFFormat(format.split(":")[-1], ar)-- resolved_format = self.backend.output_format- self.gguf_args_check(ar, resolved_format, model_type=ModelType.TEXT)- if ar.mllm:- self.gguf_args_check(ar, resolved_format, model_type=ModelType.MMPROJ)- else:- scheme = ar.scheme- gguf_format = f"gguf:{format.lower()}"- if format.lower().endswith("_mixed"):- from auto_round.schemes import _handle_special_schemes- from auto_round.utils.model import is_moe_model-- if format.lower() == "q2_k_mixed" and (getattr(ar, "iters", 0) or 0) > 0 and not is_moe_model(ar.model):- logger.warning(- "gguf:q2_k_mixed only supports MoE models with iters>0. "- "It is not an MoE model, falling back to gguf:q2_k_s."- )- gguf_format = "gguf:q2_k_s"- else:- ar.layer_config = _handle_special_schemes(- gguf_format, ar.layer_config, ar.model, quant_nontext_module=ar.quant_nontext_module- )- gguf_format = gguf_format.lower().replace("_mixed", "_s")- if isinstance(scheme, str) and scheme.lower() != gguf_format and not getattr(ar, "is_auto_scheme", False):- logger.warning(f"reset scheme {scheme.lower()} to {gguf_format} for gguf format export")- ar.scheme = gguf_format- self.output_format = gguf_format- self.backend = None- self.mllm = ar.mllm-- @classmethod- def check_scheme_args(cls: OutputFormat, scheme: QuantizationScheme) -> bool:- error_logs = []- if not re.search("int", scheme.data_type):- error_logs.append(f"data_type={scheme.data_type}")- if error_logs:- raise ValueError(- f"{cls.format_name} format support quantization scheme with {','.join(cls.support_schemes)} "- f"but got {', '.join(error_logs)}, please have a check."- )- return True-- def check_and_reset_format(self, ar):- if getattr(ar, "iters", 0) != 0 and ar.bits != 3 and not ar.enable_alg_ext:- logger.warning_once(- "`iters=0` is recommended when exporting to current GGUF format"- " or add `enable_alg_ext` for better accuracy with much more tuning cost."- " Please refer to https://github.com/intel/auto-round/tree/main/docs/gguf_alg_ext_acc.md"- " for the accuracy results."- )- elif ar.bits >= 8 and getattr(ar, "iters", 0) != 0:- logger.warning_once("`iters=0` is recommended for bits>=8")-- if getattr(ar, "quant_nontext_module", False):- # for gguf export, leave vl model for gguf itself- all_blocks = get_block_names(ar.model, False)- ar.quant_block_list = find_matching_blocks(ar.model, all_blocks, None)- return super().check_and_reset_format(ar)-- def pack_layer(- self,- name,- model,- backend,- output_dir,- layer_config,- tokenizer,- processor=None,- image_processor=None,- model_type=ModelType.TEXT,- device="cpu",- quant_nontext_module=False,- ):- from auto_round.export.export_to_gguf.export import pack_gguf_layer-- pack_gguf_layer(- name,- model,- backend,- output_dir,- layer_config,- tokenizer,- processor,- image_processor,- model_type,- device,- quant_nontext_module,- is_auto_scheme=self.is_auto_scheme,- )-- def save_quantized(- self,- output_dir: str,- model: torch.nn.Module = None,- tokenizer: Callable = None,- layer_config: dict = None,- inplace: bool = True,- device: Union[str, torch.device] = "cpu",- serialization_dict: dict = None,- **kwargs,- ) -> torch.nn.Module:- from auto_round.export.export_to_gguf.export import save_quantized_as_gguf-- backend = self.get_backend_name()- return save_quantized_as_gguf(- output_dir=output_dir,- model=model,- backend=backend,- layer_config=layer_config,- mllm=self.mllm,- device=device,- serialization_dict=serialization_dict,- is_auto_scheme=self.is_auto_scheme,- **kwargs,- )-- @staticmethod- def gguf_args_check(args_or_ar, formats: Union[str, list[str]] = None, model_type=ModelType.TEXT):- import argparse-- from auto_round.export.export_to_gguf.config import GGUF_CONFIG- from auto_round.export.export_to_gguf.llama_cpp_conversion import get_conversion- from auto_round.logger import logger- from auto_round.utils.model import download_or_get_path, get_gguf_architecture-- formats = [formats] if isinstance(formats, str) else formats- formats = sorted(formats, key=lambda x: len(x))- export_gguf = False- for f in formats:- if f.startswith("gguf"):- export_gguf = True-- if f.startswith("gguf") and f not in GGUF_CONFIG:- logger.error(f"{f} is not supported, please check.")-- if export_gguf:- if isinstance(args_or_ar.model, str):- model_path = args_or_ar.model- else:- model_path = args_or_ar.model.name_or_path- if not os.path.isdir(model_path):- model_path = download_or_get_path(model_path, args_or_ar.platform)- conversion = get_conversion(model_path, model_type=ModelType.TEXT)- model_architecture = get_gguf_architecture(model_path, model_type=ModelType.TEXT)- if not conversion.is_supported(model_architecture, ModelType.TEXT):- logger.error(f"Model {model_architecture} is not supported to export gguf format.")- sys.exit(1)-- pattern = re.compile(r"q\d_k")- pre_dq_format = ""- unsupported_list, reset_list = [], []- for format in GGUF_CONFIG:- if format in formats:- if format == "q6_k_s":- logger.warning("Please note that q6_k_s is q6_k.")-- if re.search(pattern, format):- if pre_dq_format and re.search(pattern, format).group() not in pre_dq_format:- logger.error(f"Cannot export {pre_dq_format} and {format} at the same time.")- sys.exit(-1)- else:- pre_dq_format = format-- unsupported_list, reset_list = [], []- gguf_config = GGUF_CONFIG[format]- for k, v in gguf_config.items():- if not hasattr(args_or_ar, k):- continue- if k == "data_type":- if re.search(r"q\d_1", format) and len(formats) > 1:- v = "int"- if k == "sym" and isinstance(args_or_ar, argparse.Namespace):- k = "asym"- v = not v- if getattr(args_or_ar, k) != v:- unsupported_list.append(f"{k}={getattr(args_or_ar, k)}")- reset_list.append(f"{k}={v}")- setattr(args_or_ar, k, v)- if len(unsupported_list) > 0:- logger.info(- f"format {format} does not support for {', '.join(unsupported_list)},"- f" reset to {', '.join(reset_list)}."- )- # Removed obsolete commented-out block for improved readability and maintainability.- return args_or_ar-- def immediate_pack(- self,- name: str,- model: torch.nn.Module,- device: torch.device,- output_dir: str = None,- mllm: bool = False,- layer_config: dict = None,- tokenizer=None,- processor=None,- image_processor=None,- quant_nontext_module: bool = False,- **kwargs,- ):- m = get_module(model, name)- if not check_to_quantized(m):- return- model_type = ModelType.MMPROJ if mllm else ModelType.TEXT- self.pack_layer(- name,- model,- self.get_backend_name(),- output_dir,- layer_config=layer_config,- tokenizer=tokenizer,- processor=processor,- image_processor=image_processor,- model_type=model_type,- device=device,- quant_nontext_module=quant_nontext_module,- )--[email protected]("fp8")-class FP8Format(OutputFormat):- support_schemes = ["FP8_BLOCK"]- format_name = "fp8"-- @classmethod- def check_scheme_args(cls: OutputFormat, scheme: QuantizationScheme) -> bool:- error_logs = []- if scheme.bits != 8:- error_logs.append(f"bits={scheme.bits}")- if scheme.data_type != "fp":- error_logs.append(f"data_type={scheme.data_type}")- if not isinstance(scheme.group_size, tuple):- error_logs.append(f"group_size={scheme.group_size}")- if not scheme.act_dynamic:- error_logs.append(f"act_dynamic={scheme.act_dynamic}")- if not isinstance(scheme.act_group_size, int):- error_logs.append(f"act_group_size={scheme.act_group_size}")- if scheme.act_bits != 8:- error_logs.append(f"act_bits={scheme.act_bits}")- if scheme.act_data_type != "fp":- error_logs.append(f"act_data_type={scheme.act_data_type}")- if error_logs:- raise ValueError(- f"{cls.format_name} format support quantization scheme with {','.join(cls.support_schemes)} "- f"but got {', '.join(error_logs)}, please have a check."- )- return True-- def pack_layer(self, layer_name, model, device=None, **kwargs):- from auto_round.export.export_to_autoround.export_to_fp8 import pack_layer-- pack_layer(layer_name, model, self.output_format, device=device)-- def save_quantized(- self,- output_dir: str,- model: torch.nn.Module = None,- tokenizer: Callable = None,- layer_config: dict = None,- inplace: bool = True,- device: Union[str, torch.device] = "cpu",- serialization_dict: dict = None,- **kwargs,- ) -> torch.nn.Module:- from auto_round.export.export_to_autoround.export_to_fp8 import save_quantized_as_autoround-- backend = self.get_backend_name()-- if isinstance(serialization_dict["group_size"], tuple):- serialization_dict["weight_block_size"] = serialization_dict["group_size"]-- ignored_layers = []- for layer_name, cfg in layer_config.items():- if cfg["bits"] >= 16 and cfg["act_bits"] >= 16:- ignored_layers.append(layer_name)- if len(ignored_layers) > 0:- serialization_dict["ignored_layers"] = ignored_layers-- return save_quantized_as_autoround(- output_dir=output_dir,- model=model,- tokenizer=tokenizer,- layer_config=layer_config,- inplace=inplace,- device=device,- serialization_dict=serialization_dict,- quant_method=backend,- **kwargs,- )--[email protected]("mlx")-class MLXFormat(OutputFormat):- support_schemes = ["W2A16", "W2A16G32", "W2A16G64", "W3A16", "W4A16", "W5A16", "W6A16", "W8A16", "BF16"]- format_name = "mlx"-- @classmethod- def check_scheme_args(cls: OutputFormat, scheme: QuantizationScheme) -> bool:- error_logs = []- if scheme.bits not in [2, 3, 4, 5, 6, 8, 16]:- error_logs.append(f"bits={scheme.bits}")- if scheme.act_bits != 16:- error_logs.append(f"act_bits={scheme.act_bits}")- if not re.search("int", scheme.data_type):- error_logs.append(f"data_type={scheme.data_type}")- if scheme.super_bits:- error_logs.append(f"super_bits={scheme.super_bits}")- if scheme.super_group_size:- error_logs.append(f"super_group_size={scheme.super_group_size}")- if error_logs:- raise ValueError(- f"MLX format support quantization scheme with {','.join(cls.support_schemes)} "- f"but got {', '.join(error_logs)}, please have a check."- )- return True-- def pack_layer(self, layer_name, model, device=None, **kwargs):- from auto_round.export.export_to_mlx.export import pack_layer-- pack_layer(layer_name, model, device=device, **kwargs)-- def save_quantized(- self,- output_dir: str,- model: torch.nn.Module = None,- tokenizer: Callable = None,- layer_config: dict = None,- inplace: bool = True,- device: Union[str, torch.device] = "cpu",- serialization_dict: dict = None,- **kwargs,- ) -> torch.nn.Module:- from auto_round.export.export_to_mlx.export import save_quantized_as_mlx-- return save_quantized_as_mlx(- output_dir=output_dir,- model=model,- tokenizer=tokenizer,- layer_config=layer_config,- inplace=inplace,- device=device,- serialization_dict=serialization_dict,- **kwargs,- )--[email protected]("auto_round")[email protected]("auto_round:auto_awq")[email protected]("auto_round:llm_compressor")[email protected]("auto_round:gptqmodel", "auto_round:auto_gptq")[email protected]("auto_round:fp8")[email protected]("auto_round:mlx")-class AutoRoundFormat(OutputFormat):- support_schemes = [- "W4A16",- "W4A16_MIXED",- "W2A16",- "W3A16",- "W8A16",- "MXFP4",- "MXFP8",- "NVFP4",- "FPW8A16",- "W2A16G64",- "W2A16G32",- "FP8_STATIC",- "BF16",- "FP8_BLOCK",- "MXINT4",- ]- format_name = "auto_round"-- def __init__(self, format: str, ar: BaseCompressor):- self.output_format = "auto_round"- self.backend = None-- if format == "auto_round":- if ar.sym and "int" in ar.data_type and "mx" not in ar.data_type:- self.backend = AutoGPTQFormat("auto_round:auto_gptq", ar)- elif ar.bits == 4 and not ar.sym and "int" in ar.data_type:- if ar.layer_config is None:- enable_awq = True- else:- enable_awq = all(- config["bits"] == ar.bits or config["bits"] >= 16 for config in ar.layer_config.values()- )- if enable_awq:- self.backend = AutoAWQFormat("auto_round:auto_awq", ar)- elif is_nv_fp(ar.data_type) or is_mx_fp(ar.data_type):- self.backend = AutoRoundFormat(ar.data_type, ar)- elif is_mx_int(ar.data_type) and ar.bits == 4: # only add mx_int4 now- self.backend = AutoRoundFormat(ar.data_type, ar)- elif is_act_static(ar): # static wfp8afp8- self.backend = AutoRoundFormat(AutoRoundExportFormat.FP8_STATIC.value, ar)- elif ar.data_type.startswith("fp") and ar.bits == 8 and ar.act_bits >= 16: # woq fp8- self.backend = AutoRoundFormat(AutoRoundExportFormat.FP8.value, ar)- elif ar.data_type.startswith("fp") and ar.bits == 8 and isinstance(ar.group_size, tuple):- self.backend = AutoRoundFormat("auto_round:fp8", ar)- elif ar.act_bits < 16:- raise ValueError(- "AutoRound format does not support exporting "- "for the current quantization configuration, "- "please change to `fake` format for research purpose"- )- # for auto_round:fp8_static, auto_round:nv_fp etc.- elif not format.startswith("auto_round"):- if format == "mlx":- self.backend = MLXFormat("mlx", ar)- elif format.upper() not in list(AutoRoundExportFormat.__members__.keys()):- raise KeyError(f"Unsupported backend format auto_round:{format}, please check")- else:- self.output_format = f"auto_round:{format}"- self.backend = None- elif format == "auto_round:mlx":- self.backend = MLXFormat("mlx", ar)- else:- backend = format.split(":")[1] if ":" in format else None- self.backend = self._format_list.get(backend)(format, ar) if backend else None-- if self.backend is not None:- self.support_schemes = self.backend.support_schemes-- def check_and_reset_format(self, ar):- if self.backend is not None:- new_format = self.backend.check_and_reset_format(ar)- self.backend = OutputFormat._format_list[new_format](new_format, ar) if new_format else self.backend-- if ar.act_bits <= 8:- if is_standard_fp(ar.act_data_type) and not ar.act_dynamic:- if (- ar.act_group_size != 0- and not ar.act_dynamic- and self.get_backend_name() == f"auto_round:{AutoRoundExportFormat.FP8.value}"- ):- logger.warning(- f"Please note that quantize activation with act_group_size={ar.act_group_size}"- " may result in failure to export or import normally."- )- if self.backend is None:- _check_divisible_by_32(ar)- return None-- def pack_layer(self, layer_name, model, device=None, **kwargs):- if self.backend is not None:- return self.backend.pack_layer(layer_name, model, device=device, **kwargs)-- backend = self.get_backend_name()-- if self.output_format in [- f"auto_round:{AutoRoundExportFormat.NV_FP.value}",- f"auto_round:{AutoRoundExportFormat.MX_FP.value}",- f"auto_round:{AutoRoundExportFormat.MX_FP_RCEIL.value}",- f"auto_round:{AutoRoundExportFormat.NV_FP4_WITH_STATIC_GS.value}",- ]:- from auto_round.export.export_to_autoround.export_to_nvfp_mx import pack_layer-- pack_func = pack_layer- elif self.output_format in [f"auto_round:{AutoRoundExportFormat.MX_INT.value}"]:- from auto_round.export.export_to_autoround.export_to_nvfp_mx import pack_layer-- pack_func = pack_layer- elif self.output_format in [- f"auto_round:{AutoRoundExportFormat.FP8.value}",- f"auto_round:{AutoRoundExportFormat.FP8_STATIC.value}",- f"auto_round:{AutoRoundExportFormat.FP8_STATIC.value}",- ]:- from auto_round.export.export_to_autoround.export_to_fp8 import pack_layer-- pack_func = pack_layer- else:- from auto_round.export.export_to_autoround.export import pack_layer-- pack_func = pack_layer- return pack_func(layer_name, model, backend, device)-- def save_quantized(- self,- output_dir: str,- model: torch.nn.Module = None,- tokenizer: Callable = None,- layer_config: dict = None,- inplace: bool = True,- device: Union[str, torch.device] = "cpu",- serialization_dict: dict = None,- **kwargs,- ) -> torch.nn.Module:- if self.backend is not None:- extra_kwargs = {}- if isinstance(self.backend, MLXFormat):- extra_kwargs["autoround_format"] = True- return self.backend.save_quantized(- output_dir=output_dir,- model=model,- tokenizer=tokenizer,- layer_config=layer_config,- inplace=inplace,- device=device,- serialization_dict=serialization_dict,- **extra_kwargs,- **kwargs,- )- backend = self.get_backend_name()- if re.search(f"{AutoRoundExportFormat.MX_FP.value}|{AutoRoundExportFormat.NV_FP.value}", backend):- from auto_round.export.export_to_autoround.export_to_nvfp_mx import save_quantized_as_fp-- backend = "auto_round:llm_compressor"- export_func = save_quantized_as_fp- elif serialization_dict.get("data_type", "int") == "fp" and serialization_dict.get("bits", 16) == 8:- from auto_round.export.export_to_autoround.export_to_fp8 import save_quantized_as_autoround-- backend = "auto_round:fp8_static" if serialization_dict.get("act_bits", 16) == 8 else None- export_func = save_quantized_as_autoround- elif re.search(f"{AutoRoundExportFormat.MX_INT.value}", backend):- from auto_round.export.export_to_autoround.export_to_nvfp_mx import save_quantized_as_fp-- backend = "auto_round"- export_func = save_quantized_as_fp- else:- from auto_round.export.export_to_autoround.export import save_quantized_as_autoround-- export_func = save_quantized_as_autoround- return export_func(- output_dir=output_dir,- model=model,- tokenizer=tokenizer,- layer_config=layer_config,- inplace=inplace,- device=device,- backend=backend,- serialization_dict=serialization_dict,- **kwargs,- )+def get_formats(format: str, ar):+ """Resolve formats using the relocated resolver for the legacy API."""+ from auto_round.schemes import QuantizationScheme, parse_scheme++ scheme = ar.scheme+ if not isinstance(scheme, QuantizationScheme):+ _, _, attrs = parse_scheme(scheme, {})+ scheme = QuantizationScheme.from_dict(attrs)+ resolution = resolve_formats(+ ResolvedScheme.from_scheme(scheme),+ format=format,+ layer_config=getattr(ar, "layer_config", None),+ scale_dtype=getattr(ar, "scale_dtype", None),+ iters=getattr(ar, "iters", 0) or 0,+ model=getattr(ar, "model", None),+ )+ return list(resolution.formats)+++__all__ = [+ "AutoAWQFormat",+ "AutoRoundExportFormat",+ "AutoGPTQFormat",+ "AutoRoundFormat",+ "BackendDataType",+ "FakeFormat",+ "FP8Format",+ "GGUFFormat",+ "LLMCompressorFormat",+ "MLXFormat",+ "OutputFormat",+ "resolve_formats",+ "get_formats",+]diff --git a/auto_round/inference/convert_model.py b/auto_round/inference/convert_model.pyindex 722cfd318..b04c1bbaa 100644--- a/auto_round/inference/convert_model.py+++ b/auto_round/inference/convert_model.py@@ -21,7 +21,7 @@ from packaging.version import Versionfrom tqdm import tqdmfrom transformers.pytorch_utils import Conv1D-from auto_round.formats import AutoRoundExportFormat+from auto_round.export.formats import BackendDataTypefrom auto_round.inference.backend import (BackendInfos,dynamic_import_inference_linear,@@ -559,11 +559,11 @@ def _create_quant_layer(layer, layer_backend, config, in_features, out_features,bias=bias,)elif (- AutoRoundExportFormat.FP8_STATIC.value in layer_backend- or AutoRoundExportFormat.MXFP8.value in layer_backend- or AutoRoundExportFormat.MXFP4.value in layer_backend- or AutoRoundExportFormat.NVFP4.value in layer_backend- or AutoRoundExportFormat.MXINT4.value in layer_backend+ BackendDataType.FP8_STATIC.value in layer_backend+ or BackendDataType.MXFP8.value in layer_backend+ or BackendDataType.MXFP4.value in layer_backend+ or BackendDataType.NVFP4.value in layer_backend+ or BackendDataType.MXINT4.value in layer_backend):return QuantLinear.from_original(config, layer)diff --git a/auto_round/schemes.py b/auto_round/schemes.pyindex 8fa21d018..31da6d95c 100644--- a/auto_round/schemes.py+++ b/auto_round/schemes.py@@ -14,19 +14,186 @@import copyfrom copy import deepcopyfrom dataclasses import asdict, dataclass, fields+from enum import Enumfrom typing import TYPE_CHECKING, Any, Optional, Unionimport torch+GGUF_SCHEME_FACTS: dict[str, dict] = {+ "gguf:q4_0": {+ "bits": 4,+ "act_bits": 16,+ "group_size": 32,+ "sym": True,+ "data_type": "int",+ "super_bits": None,+ "super_group_size": None,+ },+ "gguf:q4_1": {+ "bits": 4,+ "act_bits": 16,+ "group_size": 32,+ "sym": False,+ "data_type": "int_asym_float_zp",+ "super_bits": None,+ "super_group_size": None,+ },+ "gguf:q5_0": {+ "bits": 5,+ "act_bits": 16,+ "group_size": 32,+ "sym": True,+ "data_type": "int",+ "super_bits": None,+ "super_group_size": None,+ },+ "gguf:q5_1": {+ "bits": 5,+ "act_bits": 16,+ "group_size": 32,+ "sym": False,+ "data_type": "int_asym_float_zp",+ "super_bits": None,+ "super_group_size": None,+ },+ "gguf:q8_0": {+ "bits": 8,+ "act_bits": 16,+ "group_size": 32,+ "sym": True,+ "data_type": "int",+ "super_bits": None,+ "super_group_size": None,+ },+ "gguf:q2_k": {+ "bits": 2,+ "act_bits": 16,+ "group_size": 16,+ "sym": False,+ "data_type": "int_asym_dq",+ "super_bits": 4,+ "super_group_size": 16,+ },+ "gguf:q3_k": {+ "bits": 3,+ "act_bits": 16,+ "group_size": 16,+ "sym": True,+ "data_type": "int_sym_dq",+ "super_bits": 6,+ "super_group_size": 16,+ },+ "gguf:q4_k": {+ "bits": 4,+ "act_bits": 16,+ "group_size": 32,+ "sym": False,+ "data_type": "int_asym_dq",+ "super_bits": 6,+ "super_group_size": 8,+ },+ "gguf:q5_k": {+ "bits": 5,+ "act_bits": 16,+ "group_size": 32,+ "sym": False,+ "data_type": "int_asym_dq",+ "super_bits": 6,+ "super_group_size": 8,+ },+ "gguf:q6_k": {+ "bits": 6,+ "act_bits": 16,+ "group_size": 16,+ "sym": True,+ "data_type": "int_sym_dq",+ "super_bits": 8,+ "super_group_size": 16,+ },+ "gguf:bf16": {+ "bits": 16,+ "act_bits": 16,+ "group_size": None,+ "sym": True,+ "data_type": "int_sym_dq",+ "super_bits": None,+ "super_group_size": None,+ },+}+GGUF_SCHEME_FACTS["gguf:fp16"] = GGUF_SCHEME_FACTS["gguf:bf16"]++GGUF_PRESET_ALIASES: dict[str, str] = {+ "gguf:q4_0": "gguf:q4_0",+ "gguf:q4_1": "gguf:q4_1",+ "gguf:q5_0": "gguf:q5_0",+ "gguf:q5_1": "gguf:q5_1",+ "gguf:q2_k_s": "gguf:q2_k",+ "gguf:q2_k_mixed": "gguf:q2_k",+ "gguf:q3_k_s": "gguf:q3_k",+ "gguf:q3_k_m": "gguf:q3_k",+ "gguf:q3_k_l": "gguf:q3_k",+ "gguf:q4_k_s": "gguf:q4_k",+ "gguf:q4_k_m": "gguf:q4_k",+ "gguf:q5_k_s": "gguf:q5_k",+ "gguf:q5_k_m": "gguf:q5_k",+ "gguf:q6_k": "gguf:q6_k",+ "gguf:q8_0": "gguf:q8_0",+}+from auto_round.logger import loggerfrom auto_round.utils import SUPPORTED_DTYPES, contain_any_mm_keys, infer_bits_by_data_type-__all__ = ["QuantizationScheme", "get_gguf_scheme", "preset_name_to_scheme"]+__all__ = [+ "QuantizationScheme",+ "BackendDataType",+ "GGUF_SCHEME_FACTS",+ "GGUF_PRESET_ALIASES",+ "is_standard_fp",+ "is_mx_fp",+ "is_nv_fp",+ "is_mx_int",+ "get_gguf_scheme",+ "preset_name_to_scheme",+]if TYPE_CHECKING:from auto_round.auto_scheme.gen_auto_scheme import AutoScheme+class BackendDataType(str, Enum):+ STANDARD_FP = "fp"+ MX_FP = "mx_fp"+ NV_FP = "nv_fp"+ MX_INT = "mx_int"+ FP8_STATIC = "fp8_static"+ FP8 = "fp8"+++# --- data_type string classifiers (single authority) ------------------------+# These classify a raw data_type / backend string (e.g. "mx_fp4", "nv_fp",+# "fp8_static"). They are the one place the substring rules live: both the+# ``QuantizationScheme.is_*`` methods and ``compressors.utils`` (which re-exports+# them for its many string call sites) delegate here, so there is no duplicate+# copy of the "what makes a dtype mx_fp/nv_fp/..." logic.+++def is_mx_fp(backend: str) -> bool:+ return BackendDataType.MX_FP in backend.lower()+++def is_nv_fp(backend: str) -> bool:+ return BackendDataType.NV_FP in backend.lower()+++def is_mx_int(backend: str) -> bool:+ return BackendDataType.MX_INT in backend.lower()+++def is_standard_fp(backend: str) -> bool:+ backend = backend.lower()+ return BackendDataType.STANDARD_FP in backend and not is_mx_fp(backend) and not is_nv_fp(backend)++@dataclassclass QuantizationScheme:bits: int = 4@@ -118,6 +285,78 @@ class QuantizationScheme:return Falsereturn True+ # --- classification predicates (single authority, see D1) ---++ def is_standard_fp(self) -> bool:+ return self.data_type is not None and is_standard_fp(self.data_type)++ def is_mx_fp(self) -> bool:+ return self.data_type is not None and is_mx_fp(self.data_type)++ def is_nv_fp(self) -> bool:+ return self.data_type is not None and is_nv_fp(self.data_type)++ def is_mx_int(self) -> bool:+ return self.data_type is not None and is_mx_int(self.data_type)++ def is_act_standard_fp(self) -> bool:+ return self.act_data_type is not None and is_standard_fp(self.act_data_type)++ def is_act_mx_fp(self) -> bool:+ return self.act_data_type is not None and is_mx_fp(self.act_data_type)++ def is_act_nv_fp(self) -> bool:+ return self.act_data_type is not None and is_nv_fp(self.act_data_type)++ def is_act_quantize(self) -> bool:+ return self.act_bits is not None and self.act_bits <= 8++ def is_wint_woq(self) -> bool:+ """Integer weight-only quantization with non-quantized activations (`act_bits >= 16`)."""+ return "int" in self.data_type and self.act_bits >= 16 and self.super_group_size is None++ def is_wfp8afp8(self) -> bool:+ if self.act_data_type is None or self.data_type is None:+ return False+ return (+ ("fp8" in self.act_data_type or ("fp" in self.act_data_type and self.act_bits == 8))+ and ("fp8" in self.data_type or ("fp" in self.data_type and self.bits == 8))+ and self.is_act_standard_fp()+ and self.is_standard_fp()+ )++ def is_wint8aint8(self) -> bool:+ return ("int8" in self.act_data_type or ("int" in self.act_data_type and self.act_bits == 8)) and (+ "int8" in self.data_type or ("int" in self.data_type and self.bits == 8)+ )++ def is_wint4aint4(self) -> bool:+ return ("int4" in self.act_data_type or ("int" in self.act_data_type and self.act_bits == 4)) and (+ "int4" in self.data_type or ("int" in self.data_type and self.bits == 4)+ )++ def is_dynamic_afp8(self) -> bool:+ return self.act_dynamic and self.act_data_type.startswith("fp") and self.act_bits == 8++ def is_block_wfp8(self) -> bool:+ return (+ isinstance(self.group_size, tuple)+ and len(self.group_size) == 2+ and self.data_type.startswith("fp")+ and self.bits == 8+ )++ def is_static_afp8(self) -> bool:+ return self.act_data_type is not None and BackendDataType.FP8_STATIC in self.act_data_type++ def is_act_static(self) -> bool:+ return not self.act_dynamic++ def is_dynamic_wint8aint8(self) -> bool:+ if not self.act_dynamic:+ return False+ return self.is_wint8aint8()+def preset_name_to_scheme(name: str) -> QuantizationScheme:"""Get a QuantizationScheme instance from a preset scheme name."""@@ -575,14 +814,9 @@ PRESET_SCHEMES = {"FP8_BLOCK": FP8_BLOCK,"MXINT4": MXINT4,}-from auto_round.export.export_to_gguf.config import GGUF_CONFIG-for key, val in GGUF_CONFIG.items():- value = copy.deepcopy(val)- value.pop("mostly", None)- value.pop("embedding", None)- value.pop("lm_head", None)- PRESET_SCHEMES[key.upper()] = QuantizationScheme.from_dict(value)+for alias_key, facts_key in GGUF_PRESET_ALIASES.items():+ PRESET_SCHEMES[alias_key.upper()] = QuantizationScheme.from_dict(GGUF_SCHEME_FACTS[facts_key])def _handle_special_schemes(@@ -653,6 +887,8 @@ def _handle_special_schemes(def get_gguf_scheme(scheme: Union[str, QuantizationScheme]) -> str:+ if scheme is None:+ return ""if isinstance(scheme, str) and scheme.upper().startswith("GGUF"):return schemeif isinstance(scheme, str):diff --git a/auto_round/special_model_handler.py b/auto_round/special_model_handler.pyindex bcddba7fd..c73e0e5c4 100644--- a/auto_round/special_model_handler.py+++ b/auto_round/special_model_handler.py@@ -17,7 +17,7 @@ from typing import Any, Callableimport torch-from auto_round.formats import OutputFormat+from auto_round.export.formats import OutputFormatfrom auto_round.modeling.fused_moe.replace_modules import apply_replacements, release_original_module_from auto_round.utils import is_moe_model_via_config, loggerdiff --git a/docs/step_by_step.md b/docs/step_by_step.mdindex 74a178834..f10466382 100644--- a/docs/step_by_step.md+++ b/docs/step_by_step.md@@ -363,12 +363,12 @@ AWQ-specific options:#### API Usage```python-from auto_round import AutoRound+from auto_round import AWQConfig, AutoRoundar = AutoRound("Qwen/Qwen3-0.6B",scheme="INT8",- algorithm="awq",+ alg_configs=AWQConfig(),)output_dir = "./tmp_awq"@@ -489,7 +489,7 @@ When using AutoScheme with `model_free=True`, only INT (`W2A16`/`W4A16`/`W8A16`)### AWQ Quantization Algorithm-AWQ (`algorithm="awq"`) is a pre-processing quantization algorithm that analyzes activation patterns and applies channel-wise scaling to protect salient weights. It runs BEFORE the actual quantization (RTN by default, or auto_round/SignRound).+AWQ (`alg_configs="awq"` or `alg_configs=AWQConfig()`) is a pre-processing quantization algorithm that analyzes activation patterns and applies channel-wise scaling to protect salient weights. It runs BEFORE the actual quantization (RTN by default, or auto_round/SignRound).#### CLI Usage```bash@@ -506,21 +506,22 @@ auto-round --model Qwen/Qwen3-0.6B --algorithm awq,auto_round --scheme W4A16#### API Usage```python-from auto_round import AutoRound-from auto_round.algorithms.quantization.awq.config import AWQConfig-from auto_round.algorithms.quantization.sign_round.config import SignRoundConfig+from auto_round import AWQConfig, AutoRound, SignRoundConfig++# String alias (AWQ defaults, with RTN appended automatically)+ar = AutoRound(model, tokenizer, alg_configs="awq", scheme="W4A16")# AWQ + default RTN (simplest)-ar = AutoRound(model, tokenizer, algorithm="awq", scheme="W4A16")+ar = AutoRound(model, tokenizer, alg_configs=AWQConfig(), scheme="W4A16")# AWQ + AutoRound via alg_configs (explicit pipeline)ar = AutoRound(model, tokenizer, alg_configs=[AWQConfig(), SignRoundConfig(iters=200)], scheme="W4A16")ar.quantize_and_save(output_dir="./qmodel")```-**Important Note**: `algorithm="awq"` (quantization algorithm) and `format="auto_awq"` (export format) are independent. You can use:-- `algorithm="awq"` + `format="auto_round"`: AWQ smoothing + AutoRound packing-- `algorithm="auto_round"` + `format="auto_awq"`: No AWQ smoothing + AutoAWQ packing+**Important Note**: `alg_configs="awq"` or `alg_configs=AWQConfig()` (quantization algorithm) and `format="auto_awq"` (export format) are independent. You can use:+- `alg_configs="awq"` + `format="auto_round"`: AWQ smoothing + AutoRound packing+- `alg_configs="signround"` + `format="auto_awq"`: No AWQ smoothing + AutoAWQ packing### OPT RTN ModeAutoRound also supports Optimized RTN (Round-To-Nearest) mode for fast, calibration-free baseline quantization. Setting `iters=0` tp enable it and we recommend using `group_size=32` for better results. Check [accuracy comparison](./opt_rtn.md) between RTN and OPT RTN modediff --git a/docs/step_by_step_CN.md b/docs/step_by_step_CN.mdindex 9163b93d8..77b5eedfa 100644--- a/docs/step_by_step_CN.md+++ b/docs/step_by_step_CN.md@@ -360,12 +360,12 @@ AWQ 专用选项:W8A8 搭配 AWQ 平滑化:```python-from auto_round import AutoRound+from auto_round import AWQConfig, AutoRoundar = AutoRound("Qwen/Qwen3-0.6B",scheme="INT8",- algorithm="awq",+ alg_configs=AWQConfig(),)output_dir = "./tmp_awq"@@ -487,7 +487,7 @@ AutoScheme 目前还**不支持对嵌入层(Embedding layer)进行自动量### AWQ 量化算法-AWQ(`algorithm="awq"`)是一种预处理量化算法,通过分析激活分布并应用通道缩放(channel-wise scaling)来保护重要的权重。它在实际量化(默认为 RTN,或使用 auto_round/SignRound)之前运行。+AWQ(`alg_configs="awq"` 或 `alg_configs=AWQConfig()`)是一种预处理量化算法,通过分析激活分布并应用通道缩放(channel-wise scaling)来保护重要的权重。它在实际量化(默认为 RTN,或使用 auto_round/SignRound)之前运行。#### 命令行用法```bash@@ -504,21 +504,22 @@ auto-round --model Qwen/Qwen3-0.6B --algorithm awq,auto_round --scheme W4A16#### API 用法```python-from auto_round import AutoRound-from auto_round.algorithms.quantization.awq.config import AWQConfig-from auto_round.algorithms.quantization.sign_round.config import SignRoundConfig+from auto_round import AWQConfig, AutoRound, SignRoundConfig++# 字符串别名(使用 AWQ 默认参数,并自动追加 RTN)+ar = AutoRound(model, tokenizer, alg_configs="awq", scheme="W4A16")# AWQ + 默认 RTN (最简用法)-ar = AutoRound(model, tokenizer, algorithm="awq", scheme="W4A16")+ar = AutoRound(model, tokenizer, alg_configs=AWQConfig(), scheme="W4A16")# 通过 alg_configs 指定 AWQ + AutoRound (显式流水线)ar = AutoRound(model, tokenizer, alg_configs=[AWQConfig(), SignRoundConfig(iters=200)], scheme="W4A16")ar.quantize_and_save(output_dir="./qmodel")```-**重要提示**:`algorithm="awq"`(量化算法)与 `format="auto_awq"`(导出格式)是相互独立的。你可以使用:-- `algorithm="awq"` + `format="auto_round"`:AWQ 平滑 + AutoRound 打包-- `algorithm="auto_round"` + `format="auto_awq"`:不使用 AWQ 平滑 + AutoAWQ 打包+**重要提示**:`alg_configs="awq"` 或 `alg_configs=AWQConfig()`(量化算法)与 `format="auto_awq"`(导出格式)是相互独立的。你可以使用:+- `alg_configs="awq"` + `format="auto_round"`:AWQ 平滑 + AutoRound 打包+- `alg_configs="signround"` + `format="auto_awq"`:不使用 AWQ 平滑 + AutoAWQ 打包### OPT-RTN 模式AutoRound 还提供优化版 RTN(Round-To-Nearest,就近舍入)模式,无需标定数据即可实现快速基线量化。**启用方式为 `iters=0`**。同时为获得更好的效果,推荐搭配 `group_size=32` 。RTN 与 OPT RTN 模式的精度对比详见[《精度对比报告》](./opt_rtn.md)。diff --git a/test/test_cpu/algorithms/test_awq.py b/test/test_cpu/algorithms/test_awq.pyindex 32966bd07..1bb27dbd7 100644--- a/test/test_cpu/algorithms/test_awq.py+++ b/test/test_cpu/algorithms/test_awq.py@@ -28,7 +28,7 @@ import pytestimport torchfrom transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer-from auto_round import AutoRound+from auto_round import AutoRound, AWQConfig, SignRoundConfigfrom ...helpers import generate_prompt, get_model_path, opt_name_or_path, save_tiny_model@@ -50,7 +50,7 @@ class TestAWQNormalLLM:ar = AutoRound(tiny_opt_model_path,scheme="W4A16",- algorithm="awq",+ alg_configs=AWQConfig(n_grid=1),n_grid=1,nsamples=2,seqlen=32,@@ -74,7 +74,7 @@ class TestAWQNormalLLM:ar = AutoRound(tiny_opt_model_path,scheme="W4A16",- algorithm="awq",+ alg_configs=AWQConfig(n_grid=1),n_grid=1,nsamples=2,seqlen=8,@@ -98,7 +98,7 @@ class TestAWQNormalLLM:bits=bits,group_size=group_size,sym=sym,- algorithm="awq",+ alg_configs=AWQConfig(n_grid=1),n_grid=1,nsamples=2,seqlen=8,@@ -117,7 +117,7 @@ class TestAWQNonIntegerSchemes:"""Regression: AWQ smoothing must run under non-integer schemes (MX/NV-FP).AWQ's grid-search / clip loss reproduces the block quantizer's weight QDQ.- The reported failure mode was an end-to-end ``algorithm='awq'`` run raising+ The reported failure mode was an end-to-end AWQ run raisingunder an MXFP/NVFP scheme."""@@ -126,11 +126,12 @@ class TestAWQNonIntegerSchemes:ar = AutoRound(tiny_opt_model_path,scheme=scheme,- algorithm="awq,signround",+ alg_configs=[AWQConfig(n_grid=1), SignRoundConfig()],n_grid=1,nsamples=2,seqlen=8,batch_size=2,+ dataset=["local AWQ calibration sample with enough tokens for quantization"] * 2,)model, layer_config = ar.quantize()@@ -158,7 +159,7 @@ class TestAWQW8A8LLMCompressor:ar = AutoRound(tiny_opt_model_path,scheme="INT8",- algorithm="awq",+ alg_configs=AWQConfig(n_grid=1),nsamples=2,seqlen=8,n_grid=1,@@ -246,7 +247,7 @@ class TestAWQMoE:ar = AutoRound(tiny_qwen_moe_model_path,scheme="W4A16",- algorithm="awq",+ alg_configs=AWQConfig(n_grid=1),n_grid=1,nsamples=2,seqlen=8,@@ -280,7 +281,7 @@ class TestAWQMoE:ar = AutoRound(tiny_qwen_moe_model_path,scheme="W4A16",- algorithm="awq",+ alg_configs=AWQConfig(n_grid=1),n_grid=1,nsamples=2,seqlen=32,@@ -456,6 +457,24 @@ class TestAWQUseV2ScaleSearch:assert init_scale is not None, dtassert init_scale.shape[0] == weight_reshape.shape[0]+ # Scalar imatrix sentinels mean uniform importance and must not enter+ # the tensor reshape path.+ scalar_init_scale = search_optimized_init_scale(weight_reshape, "mx_fp4", 4, 1.0)+ assert scalar_init_scale is not None+ assert scalar_init_scale.shape == init_scale.shape+# asym int and *_dq are not part of the optimized init-scale path.assert search_optimized_init_scale(torch.randn(4, 128), "int_asym", 4, None) is Noneassert search_optimized_init_scale(torch.randn(4, 128), "int_sym_dq", 4, None) is None++ def test_nvfp4_opt_rtn_accepts_uniform_imatrix_sentinel(self):+ """AWQ's SignRound-V1 QDQ may call optimized RTN without a collected imatrix."""+ import torch++ from auto_round.data_type.nvfp import opt_rtn_fast_nvfp4++ weight = torch.randn(8, 32)+ qdq_weight, _, _ = opt_rtn_fast_nvfp4(weight, bits=4, group_size=16, imatrix=1.0)++ assert qdq_weight.shape == weight.shape+ assert torch.isfinite(qdq_weight).all()diff --git a/test/test_cpu/config_resolution/test_config_snapshots.py b/test/test_cpu/config_resolution/test_config_snapshots.pynew file mode 100644index 000000000..fe36b1005--- /dev/null+++ b/test/test_cpu/config_resolution/test_config_snapshots.py@@ -0,0 +1,110 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++import pytest+import torch+import torch.nn as nn++from auto_round.compressors.config_resolution import resolve_quantization_config, resolve_scheme_value+from auto_round.export.formats import resolve_formats++EXPECTED_PLAN_SNAPSHOTS = {+ "w4_autoround": {+ "scheme_name": "W4A16",+ "format_name": "auto_round",+ "backend": "auto_round:auto_gptq",+ "bits": 4,+ "group_size": 128,+ "data_type": "int",+ "act_bits": 16,+ "act_data_type": "float",+ "scale_dtype": "torch.float16",+ },+ "fp8_static_llmc": {+ "scheme_name": "FP8_STATIC",+ "format_name": "llm_compressor",+ "backend": "llm_compressor:fp8_static",+ "bits": 8,+ "group_size": -1,+ "data_type": "fp",+ "act_bits": 8,+ "act_data_type": "fp",+ "scale_dtype": "torch.float16",+ },+ "gguf_q4_k_m": {+ "scheme_name": "GGUF:Q4_K_M",+ "format_name": "gguf:q4_k_m",+ "backend": "gguf:q4_k_m",+ "bits": 4,+ "group_size": 32,+ "data_type": "int_asym_dq",+ "act_bits": 16,+ "act_data_type": "float",+ "scale_dtype": "torch.float32",+ },+ "mxfp4_autoround": {+ "scheme_name": "MXFP4",+ "format_name": "auto_round",+ "backend": "auto_round:mx_fp",+ "bits": 4,+ "group_size": 32,+ "data_type": "mx_fp",+ "act_bits": 4,+ "act_data_type": "mx_fp",+ "scale_dtype": "torch.float16",+ },+ "nvfp4_autoround": {+ "scheme_name": "NVFP4",+ "format_name": "auto_round",+ "backend": "auto_round:nv_fp",+ "bits": 4,+ "group_size": 16,+ "data_type": "nv_fp",+ "act_bits": 4,+ "act_data_type": "nv_fp4_with_static_gs",+ "scale_dtype": "torch.float16",+ },+}++[email protected]("case_name", EXPECTED_PLAN_SNAPSHOTS)+def test_scheme_format_plan_snapshot(case_name):+ expected = EXPECTED_PLAN_SNAPSHOTS[case_name]+ scheme = resolve_scheme_value(expected["scheme_name"], {})+ resolution = resolve_formats(+ scheme,+ format=expected["format_name"],+ layer_config={"layer": {"bits": scheme.value.bits}},+ scale_dtype=torch.float16,+ model=nn.Sequential(nn.Linear(32, 32)),+ )+ plan = resolve_quantization_config(resolution, {"layer": {"bits": resolution.scheme.value.bits}})+ scheme_value = plan.scheme.value+ snapshot = {+ "scheme_name": expected["scheme_name"],+ "format_name": expected["format_name"],+ "backend": plan.formats[0].get_backend_name(),+ "bits": scheme_value.bits,+ "group_size": scheme_value.group_size,+ "data_type": scheme_value.data_type,+ "act_bits": scheme_value.act_bits,+ "act_data_type": scheme_value.act_data_type,+ "scale_dtype": str(plan.scale_dtype),+ }++ assert snapshot == expected+ assert {name: dict(config) for name, config in plan.layer_config.items()} == {"layer": {"bits": expected["bits"]}}+ assert dict(plan.regex_config) == {}+ assert plan.has_qlayer_outside_block is False+ assert plan.quant_block_list is Nonediff --git a/test/test_cpu/config_resolution/test_resolution.py b/test/test_cpu/config_resolution/test_resolution.pynew file mode 100644index 000000000..8fdc8781c--- /dev/null+++ b/test/test_cpu/config_resolution/test_resolution.py@@ -0,0 +1,85 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++from dataclasses import FrozenInstanceError+from types import SimpleNamespace++import pytest++from auto_round.auto_scheme.gen_auto_scheme import AutoScheme+from auto_round.cli import main as cli_main+from auto_round.compressors.config_resolution import (+ FormatCompatibilityError,+ ResolvedScheme,+ SchemeResolutionError,+ resolve_scheme_value,+)+from auto_round.schemes import QuantizationScheme+++def test_resolve_scheme_value_preserves_precise_gguf_identity():+ resolved = resolve_scheme_value("GGUF:Q4_K_M", {})++ assert resolved.preset_name == "gguf:q4_k_m"+ assert resolved.value.bits == 4+ assert resolved.value.data_type == "int_asym_dq"+ assert not isinstance(resolved.value, str)+++def test_resolve_scheme_value_applies_overrides_without_mutating_input():+ overrides = {"bits": 8}++ resolved = resolve_scheme_value("W4A16", overrides)++ assert resolved.preset_name == "W4A16"+ assert resolved.value.bits == 8+ assert overrides == {"bits": 8}+++def test_resolve_scheme_value_defers_model_aware_auto_scheme():+ scheme = AutoScheme(avg_bits=4.0, options=["W4A16", "W8A16"])++ with pytest.raises(SchemeResolutionError, match="model-aware"):+ resolve_scheme_value(scheme, {})++ assert scheme.options == ["W4A16", "W8A16"]+++def test_resolved_scheme_is_deeply_isolated_and_frozen():+ source = QuantizationScheme(bits=4, data_type="int", rotation_config={"mode": "hadamard"})+ resolved = ResolvedScheme.from_scheme(source, preset_name="W4A16")++ source.bits = 8+ source.rotation_config["mode"] = "random"+ leaked_copy = resolved.value+ leaked_copy.bits = 2+ leaked_copy.rotation_config["mode"] = "identity"++ assert resolved.value.bits == 4+ assert resolved.value.rotation_config == {"mode": "hadamard"}+ with pytest.raises(FrozenInstanceError):+ resolved.preset_name = "W8A16"+++def test_cli_converts_domain_error_to_nonzero_exit(monkeypatch):+ messages = []+ monkeypatch.setattr(cli_main.sys, "argv", ["auto-round"])+ monkeypatch.setattr(cli_main, "start", lambda argv=None: (_ for _ in ()).throw(FormatCompatibilityError("bad")))+ monkeypatch.setattr(cli_main, "logger", SimpleNamespace(error=messages.append))++ with pytest.raises(SystemExit) as error:+ cli_main.run()++ assert error.value.code != 0+ assert messages == ["bad"]diff --git a/test/test_cpu/core/test_autoopt.py b/test/test_cpu/core/test_autoopt.pyindex e65ff6436..7dfffea94 100644--- a/test/test_cpu/core/test_autoopt.py+++ b/test/test_cpu/core/test_autoopt.py@@ -6,7 +6,7 @@ import torchimport transformersfrom transformers import AutoModelForCausalLM, AutoTokenizer-from auto_round import AutoRoundAdam+from auto_round import AdamRoundConfig, AutoRoundclass TestAutoRound:@@ -29,13 +29,15 @@ class TestAutoRound:llm_block_names = get_block_names(tiny_opt_model, quant_vision=True)bits, group_size, sym, batch_size = 4, 128, False, 20- adamround = AutoRoundAdam(+ adamround = AutoRound(tiny_opt_model,opt_tokenizer,- bits=bits,- group_size=group_size,- sym=sym,- iters=2,+ alg_configs=AdamRoundConfig(+ bits=bits,+ group_size=group_size,+ sym=sym,+ iters=2,+ ),seqlen=2,batch_size=batch_size,dataset=dataloader,diff --git a/test/test_cpu/core/test_autoround_entry.py b/test/test_cpu/core/test_autoround_entry.pynew file mode 100644index 000000000..0b4d8c0ea--- /dev/null+++ b/test/test_cpu/core/test_autoround_entry.py@@ -0,0 +1,212 @@+import inspect++import pytest++from auto_round.algorithms.quantization.rtn.config import RTNConfig+from auto_round.algorithms.quantization.sign_round.config import SignRoundConfig, SignRoundV2Config+from auto_round.algorithms.transforms.awq.config import AWQConfig+from auto_round.algorithms.transforms.hadamard.config import RotationConfig+from auto_round.autoround import _normalize_alg_configs+++def _types(configs):+ return [type(config).__name__ for config in configs]+++def test_default_alg_configs_is_signround():+ configs = _normalize_alg_configs(None)++ assert _types(configs) == ["SignRoundConfig"]+++def test_public_entry_keeps_legacy_algorithm_parameter():+ from auto_round import AutoRound++ assert "algorithm" in inspect.signature(AutoRound.__new__).parameters+++ "value",+ [+ "signround",+ SignRoundConfig(iters=3),+ [SignRoundConfig(iters=3)],+ ("signround",),+ ],+)+def test_alg_configs_accepts_alias_config_and_sequence(value):+ configs = _normalize_alg_configs(value)++ assert len(configs) == 1+ assert isinstance(configs[0], SignRoundConfig)+++def test_awq_only_appends_rtn():+ configs = _normalize_alg_configs(AWQConfig())++ assert isinstance(configs[0], AWQConfig)+ assert isinstance(configs[1], RTNConfig)+++def test_awq_and_signround_does_not_append_rtn():+ configs = _normalize_alg_configs([AWQConfig(), SignRoundConfig(iters=3)])++ assert _types(configs) == ["AWQConfig", "SignRoundConfig"]+++def test_direct_signround_kwargs_warn_and_apply(monkeypatch):+ warnings = []+ monkeypatch.setattr("auto_round.autoround.logger.warning", lambda message, *args: warnings.append(message % args))++ configs = _normalize_alg_configs(None, direct_kwargs={"iters": 7, "lr": 0.2})++ assert isinstance(configs[0], SignRoundConfig)+ assert configs[0].iters == 7+ assert configs[0].lr == 0.2+ assert any("alg_configs" in message for message in warnings)+++def test_zero_iters_selects_rtn_and_forwards_disable_opt_rtn():+ configs = _normalize_alg_configs(None, direct_kwargs={"iters": 0, "disable_opt_rtn": True})++ assert len(configs) == 1+ assert isinstance(configs[0], RTNConfig)+ assert configs[0].disable_opt_rtn is True++[email protected]("algorithm", ["signround", "auto_round", "autoround"])+def test_legacy_signround_alias_with_zero_iters_selects_rtn(algorithm):+ legacy_configs = _normalize_alg_configs(None, direct_kwargs={"algorithm": algorithm, "iters": 0})+ unified_configs = _normalize_alg_configs(algorithm, direct_kwargs={"iters": 0})++ assert len(legacy_configs) == len(unified_configs) == 1+ assert isinstance(legacy_configs[0], RTNConfig)+ assert isinstance(unified_configs[0], RTNConfig)+++def test_explicit_zero_iter_signround_config_selects_rtn():+ configs = _normalize_alg_configs(SignRoundConfig(iters=0))++ assert len(configs) == 1+ assert isinstance(configs[0], RTNConfig)+++def test_combined_awq_zero_iters_keeps_awq_and_selects_rtn():+ configs = _normalize_alg_configs(None, direct_kwargs={"algorithm": "awq,signround", "iters": 0})++ assert _types(configs) == ["AWQConfig", "OptimizedRTNConfig"]+++ ("algorithm", "expected_types"),+ [+ ("rtn", ["OptimizedRTNConfig"]),+ ("awq", ["AWQConfig", "OptimizedRTNConfig"]),+ ("signround", ["SignRoundConfig"]),+ ("awq,signround", ["AWQConfig", "SignRoundConfig"]),+ ],+)+def test_legacy_algorithm_matches_alg_configs_aliases(algorithm, expected_types):+ legacy_configs = _normalize_alg_configs(None, direct_kwargs={"algorithm": algorithm})+ unified_configs = _normalize_alg_configs(algorithm)++ assert _types(legacy_configs) == expected_types+ assert _types(unified_configs) == expected_types+++def test_algorithm_and_alg_configs_are_mutually_exclusive():+ with pytest.raises(ValueError, match="cannot be used together"):+ _normalize_alg_configs("signround", direct_kwargs={"algorithm": "rtn"})+++def test_string_and_string_sequence_use_same_alias_deduplication():+ comma_configs = _normalize_alg_configs("awq,awq,signround")+ sequence_configs = _normalize_alg_configs(["awq", "awq", "signround"])++ assert _types(comma_configs) == ["AWQConfig", "SignRoundConfig"]+ assert _types(sequence_configs) == _types(comma_configs)+++def test_legacy_combined_algorithm_forwards_each_algorithm_option():+ configs = _normalize_alg_configs(+ None,+ direct_kwargs={"algorithm": "awq,signround", "n_grid": 7, "iters": 3, "enable_alg_ext": False},+ )++ assert _types(configs) == ["AWQConfig", "SignRoundConfig"]+ assert configs[0].n_grid == 7+ assert configs[1].iters == 3+ assert configs[1].enable_alg_ext is False+++ "configs",+ [+ lambda: _normalize_alg_configs(None, direct_kwargs={"algorithm": "awq,signround", "iters": 3}),+ lambda: _normalize_alg_configs([AWQConfig(), SignRoundConfig(iters=3)]),+ ],+)+def test_legacy_and_unified_composition_select_same_v1_quantizer(configs):+ from auto_round.algorithms.composer import AlgorithmComposer++ composer = AlgorithmComposer(configs())++ assert [type(pre).__name__ for pre in composer.preprocessors] == ["AWQTransform"]+ assert type(composer.block_quantizer).__name__ == "SignRoundQuantizer"+++def test_direct_enable_opt_rtn_forces_optimized_rtn():+ configs = _normalize_alg_configs(None, direct_kwargs={"iters": 0, "enable_opt_rtn": True})++ assert len(configs) == 1+ assert isinstance(configs[0], RTNConfig)+ assert configs[0].disable_opt_rtn is False+ assert configs[0].orig_disable_opt_rtn is False+++def test_rtn_switches_only_when_iters_is_zero():+ configs = _normalize_alg_configs(None, direct_kwargs={"disable_opt_rtn": True})++ assert isinstance(configs[0], SignRoundConfig)+++def test_direct_enable_alg_ext_normalizes_signround_variant():+ configs = _normalize_alg_configs(None, direct_kwargs={"iters": 1, "enable_alg_ext": True})++ assert len(configs) == 1+ assert isinstance(configs[0], SignRoundV2Config)+++def test_direct_rotation_config_is_added_to_pipeline():+ configs = _normalize_alg_configs(None, direct_kwargs={"rotation_config": "default"})++ assert any(isinstance(config, RotationConfig) for config in configs)+++def test_direct_enable_lfq_is_forwarded():+ configs = _normalize_alg_configs(None, direct_kwargs={"enable_lfq": True})++ assert isinstance(configs[0], SignRoundConfig)+ assert configs[0].enable_lfq is True+++def test_rotation_config_accepts_spinquant_shorthand():+ configs = _normalize_alg_configs(None, direct_kwargs={"rotation_config": "quarot"})++ assert any(type(config).__name__ == "SpinQuantConfig" for config in configs)+++def test_rotation_backend_must_be_nested_in_rotation_config():+ with pytest.raises(ValueError, match="rotation_config"):+ _normalize_alg_configs(None, direct_kwargs={"backend": "transform"})+++def test_awq_kwargs_without_awq_error_and_are_ignored(monkeypatch):+ errors = []+ monkeypatch.setattr("auto_round.autoround.logger.error", lambda message, *args: errors.append(message % args))++ configs = _normalize_alg_configs(None, direct_kwargs={"n_grid": 7, "apply_clip": True})++ assert isinstance(configs[0], SignRoundConfig)+ assert not hasattr(configs[0], "n_grid")+ assert any("AWQ" in message for message in errors)diff --git a/test/test_cpu/core/test_awq_autoround_smoke.py b/test/test_cpu/core/test_awq_autoround_smoke.pyindex d529c9c6e..0f7d53faf 100644--- a/test/test_cpu/core/test_awq_autoround_smoke.py+++ b/test/test_cpu/core/test_awq_autoround_smoke.py@@ -1,21 +1,24 @@"""Minimal runtime smoke for AWQ + AutoRound fusion."""+from auto_round import AutoRoundfrom auto_round.algorithms.quantization.sign_round.config import SignRoundConfigfrom auto_round.algorithms.transforms.awq.config import AWQConfig-from auto_round.compressors.entry import AutoRound-def test_awq_plus_autoround_quantize_smoke(tiny_opt_model_path):+def test_awq_plus_autoround_quantize_smoke(tiny_opt_model_path, dataloader):ar = AutoRound(tiny_opt_model_path,scheme="W4A16",alg_configs=[AWQConfig(n_grid=2), SignRoundConfig(iters=1)],nsamples=1,seqlen=8,+ dataset=dataloader,low_cpu_mem_usage=False,)model, layer_config = ar.quantize()+ assert [type(pre).__name__ for pre in ar.alg_composer.preprocessors] == ["AWQTransform"]+ assert type(ar.alg_composer.block_quantizer).__name__ == "SignRoundQuantizer"assert model is not Noneassert layer_configdiff --git a/test/test_cpu/core/test_compression_plan_state.py b/test/test_cpu/core/test_compression_plan_state.pynew file mode 100644index 000000000..38f0f3be2--- /dev/null+++ b/test/test_cpu/core/test_compression_plan_state.py@@ -0,0 +1,95 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++import inspect++from auto_round import AutoRound, RTNConfig+from auto_round.compressors.base import BaseCompressor+++def test_post_init_builds_authoritative_compression_plan(tiny_opt_model_path):+ compressor = AutoRound(+ tiny_opt_model_path,+ scheme="W4A16",+ alg_configs=RTNConfig(),+ format="auto_round",+ nsamples=1,+ seqlen=8,+ dataset=["local calibration sample"],+ low_cpu_mem_usage=False,+ )++ compressor.post_init()++ assert compressor._format_resolution.scheme.preset_name == compressor.compression_plan.scheme.preset_name+ assert compressor._format_resolution.formats == compressor.compression_plan.formats+ assert compressor.compression_plan.scheme.value == compressor.scheme_context+ assert compressor.compression_plan.formats == tuple(compressor.formats)+ assert dict(compressor.compression_plan.layer_config) == compressor.layer_config+ assert dict(compressor.compression_plan.regex_config) == compressor.regex_config+ assert compressor.compression_plan.has_qlayer_outside_block == compressor.has_qlayer_outside_block+++def test_legacy_state_views_cannot_mutate_authoritative_plan(tiny_opt_model_path):+ compressor = AutoRound(+ tiny_opt_model_path,+ scheme="W4A16",+ alg_configs=RTNConfig(),+ format="auto_round",+ nsamples=1,+ seqlen=8,+ dataset=["local calibration sample"],+ low_cpu_mem_usage=False,+ )+ compressor.post_init()++ compressor.scheme_context.bits = 2+ compressor.formats.clear()+ compressor.layer_config.clear()+ compressor.regex_config.clear()++ assert compressor.scheme_context.bits == compressor.compression_plan.scheme.value.bits == 4+ assert compressor.formats == list(compressor.compression_plan.formats)+ assert compressor.layer_config == {+ name: dict(config) for name, config in compressor.compression_plan.layer_config.items()+ }+ assert compressor.regex_config == {+ name: dict(config) for name, config in compressor.compression_plan.regex_config.items()+ }+++def test_quantizer_sync_reads_authoritative_plan_instead_of_parallel_state():+ source = inspect.getsource(BaseCompressor._build_layer_config)++ assert "self.quantizer.layer_config = self.layer_config" not in source+ assert "self.quantizer.regex_config = self.regex_config" not in source+ assert "self.quantizer.scale_dtype = self.scale_dtype" not in source+++def test_format_resolution_does_not_copy_scheme_fields_one_by_one():+ source = inspect.getsource(BaseCompressor._resolve_format_string)++ assert "fields(QuantizationScheme)" not in source+++def test_final_layer_config_path_does_not_call_legacy_mutating_adapter():+ source = inspect.getsource(BaseCompressor.configure_layer_config)++ assert "set_layer_config(" not in source+++def test_auto_scheme_layer_discovery_uses_pure_resolver():+ source = inspect.getsource(BaseCompressor._gen_auto_scheme)++ assert "set_layer_config(" not in sourcediff --git a/test/test_cpu/core/test_entry_contract.py b/test/test_cpu/core/test_entry_contract.pynew file mode 100644index 000000000..0ebda2063--- /dev/null+++ b/test/test_cpu/core/test_entry_contract.py@@ -0,0 +1,50 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++from auto_round import autoround+from auto_round.autoround import _split_entry_kwargs+++def test_split_entry_kwargs_partitions_owned_fields():+ processor = object()++ grouped = _split_entry_kwargs(+ {"dataset": ["sample"], "scale_dtype": "fp32", "processor": processor, "model_free": False}+ )++ assert grouped["base"]["dataset"] == ["sample"]+ assert grouped["compressor"]["scale_dtype"] == "fp32"+ assert grouped["mllm"]["processor"] is processor+ assert grouped["route"]["model_free"] is False+++def test_split_entry_kwargs_ignores_unknown_fields(monkeypatch):+ warnings = []+ monkeypatch.setattr(autoround.logger, "warning_once", lambda message, *args: warnings.append(message % args))++ grouped = _split_entry_kwargs({"unknown_option": 1}, context="test entry")++ assert all(not values for values in grouped.values())+ assert "unknown_option" in warnings[0]+++def test_cli_explicit_auto_round_with_zero_iters_selects_rtn():+ from argparse import Namespace++ from auto_round.cli.algorithms import AlgorithmHandler++ args = Namespace(algorithm="auto_round", iters=0)+ configs = AlgorithmHandler.build_configs(args, {})++ assert type(configs[0]).__name__ == "RTNConfig"diff --git a/test/test_cpu/core/test_entry_scheme_unification.py b/test/test_cpu/core/test_entry_scheme_unification.pynew file mode 100644index 000000000..b6362f383--- /dev/null+++ b/test/test_cpu/core/test_entry_scheme_unification.py@@ -0,0 +1,104 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+"""Entry-layer scheme unification: routing decisions in ``entry.py`` read scheme+fields from a single resolved-scheme source (``_preview_resolved_attrs``) rather+than double-reading raw config attrs. These lock that the resolved values — and+the resulting compressor-class routing — are identical whether the user passes+``scheme=`` alone or the equivalent bit/dtype overrides on the alg config.+"""++import ast+import inspect++from auto_round.algorithms.quantization.rtn.config import OptimizedRTNConfig, RTNConfig+from auto_round.autoround import (+ _collect_config_scheme_overrides,+ _preview_resolved_attrs,+ _select_rtn_compressor_base_cls,+)+from auto_round.compressors.orchestrator import CompressionOrchestrator+++def test_rtn_routing_does_not_fallback_to_raw_config_scheme_fields():+ tree = ast.parse(inspect.getsource(_select_rtn_compressor_base_cls))+ forbidden_fields = {"bits", "act_bits", "data_type", "sym", "act_data_type", "act_dynamic"}+ raw_reads = []+ for node in ast.walk(tree):+ if not isinstance(node, ast.Call) or not isinstance(node.func, ast.Name) or node.func.id != "getattr":+ continue+ if len(node.args) < 2 or not isinstance(node.args[0], ast.Name) or node.args[0].id != "quant_config":+ continue+ if isinstance(node.args[1], ast.Constant) and node.args[1].value in forbidden_fields:+ raw_reads.append(node.args[1].value)++ assert raw_reads == []+++def test_collect_config_scheme_overrides_omits_unset_fields():+ cfg = RTNConfig(bits=8, data_type="int")+ overrides = _collect_config_scheme_overrides(cfg)+ assert overrides["bits"] == 8+ assert overrides["data_type"] == "int"+ # Unset scheme fields must not appear (scheme's own value should win).+ assert "act_group_size" not in overrides+++def test_preview_resolves_scheme_only_and_override_to_same_attrs():+ # scheme="W8A16" vs bits=8 override on top of a W-generic scheme must resolve+ # to the same bits/data_type for routing.+ from_scheme = _preview_resolved_attrs(RTNConfig(), "W8A16")+ from_override = _preview_resolved_attrs(RTNConfig(bits=8), "W8A16")+ assert from_scheme.get("bits") == from_override.get("bits") == 8+++def test_preview_falls_back_to_config_overrides_when_preview_skipped():+ # An unknown scheme string makes parse_scheme raise; the resolver must then+ # surface the config's explicit overrides (not an empty dict) so routing still+ # sees the user's bits.+ cfg = RTNConfig(bits=4, data_type="int")+ resolved = _preview_resolved_attrs(cfg, "definitely-not-a-real-scheme-xyz")+ assert resolved.get("bits") == 4+ assert resolved.get("data_type") == "int"+++def test_routing_matches_between_scheme_only_and_equivalent_override():+ base_kwargs = {}+ # sym W4A16 (int4, sym) -> imatrix enabled, identical whether the bits/dtype+ # come from the scheme or from explicit config overrides. The compressor class+ # returned is always CompressionOrchestrator (it internally detects whether+ # calibration data is needed); the imatrix/zero-shot decision now surfaces via+ # ``enable_imatrix`` and the resolved config class.+ cfg_scheme = RTNConfig()+ cfg_override = RTNConfig(bits=4, data_type="int", sym=True)+ via_scheme = _select_rtn_compressor_base_cls(cfg_scheme, "W4A16", None, base_kwargs)+ via_override = _select_rtn_compressor_base_cls(cfg_override, "W4A16", None, base_kwargs)+ assert via_scheme is via_override is CompressionOrchestrator+ assert cfg_scheme.enable_imatrix is cfg_override.enable_imatrix is True+ assert isinstance(cfg_scheme, OptimizedRTNConfig) and isinstance(cfg_override, OptimizedRTNConfig)++ # asym W4A16 disables imatrix, again identical across paths.+ cfg_scheme_asym = RTNConfig(sym=False)+ cfg_override_asym = RTNConfig(bits=4, data_type="int", sym=False)+ via_scheme_asym = _select_rtn_compressor_base_cls(cfg_scheme_asym, "W4A16", None, base_kwargs)+ via_override_asym = _select_rtn_compressor_base_cls(cfg_override_asym, "W4A16", None, base_kwargs)+ assert via_scheme_asym is via_override_asym is CompressionOrchestrator+ assert cfg_scheme_asym.enable_imatrix is cfg_override_asym.enable_imatrix is False+++def test_w8a16_symmetric_routes_to_zero_shot():+ # sym W8A16 (bits>=8, sym True) -> no imatrix, no act calib -> zero-shot RTN path.+ cfg = RTNConfig()+ cls = _select_rtn_compressor_base_cls(cfg, "W8A16", None, {})+ assert cls is CompressionOrchestrator+ assert cfg.enable_imatrix is Falsediff --git a/test/test_cpu/core/test_format_decoupling.py b/test/test_cpu/core/test_format_decoupling.pynew file mode 100644index 000000000..b9dac1280--- /dev/null+++ b/test/test_cpu/core/test_format_decoupling.py@@ -0,0 +1,129 @@+# Copyright (c) 2025 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+"""Decoupling regression for the `export-format` capability.++Two behaviors that must survive the scheme/format/compressor decoupling:++1. Format-selection: ``resolve_formats(scheme, ...)`` (now scheme-centric, no ``ar``)+ resolves the same backend as pre-refactor for representative combos.+2. GGUF scheme correction propagates back onto ``self.scheme`` /+ ``self.scheme_context`` / ``self.quantize_config`` (the staleness fix), with+ ``self.scheme`` pinned to the precise preset *string* for alias+ disambiguation.+"""++from types import SimpleNamespace++import pytest+import torch+import torch.nn as nn++from auto_round.algorithms.quantization.config import QuantizationConfig+from auto_round.compressors.base import BaseCompressor+from auto_round.compressors.config_resolution import FormatResolution, ResolvedScheme+from auto_round.export.formats import resolve_formats+from auto_round.schemes import QuantizationScheme, parse_scheme+++def _resolved_scheme(scheme_name):+ # Real call sites always fully resolve the scheme (parse_scheme's third return,+ # final_attrs) before calling resolve_formats — e.g. act_data_type defaults to "float"+ # rather than staying None. Mirror that precondition here.+ _, _, final_attrs = parse_scheme(scheme_name, {})+ return QuantizationScheme.from_dict(final_attrs)+++# Harvested verbatim from a live run against pre-refactor code (HEAD=33b7df0e).+EXPECTED_FORMAT_SELECTION_BASELINE = {+ ("W4A16", "auto_round"): ["auto_round:auto_gptq"],+ ("FP8_STATIC", "llm_compressor"): ["llm_compressor:fp8_static"],+ ("MXFP4", "auto_round"): ["auto_round:mx_fp"],+ ("NVFP4", "auto_round"): ["auto_round:nv_fp"],+}+++def test_format_selection_baseline():+ results = {}+ for scheme_name, fmt in EXPECTED_FORMAT_SELECTION_BASELINE:+ scheme = _resolved_scheme(scheme_name)+ # _check_divisible_by_32 iterates model.named_modules() for int/act_bits>=16,+ # so a real nn.Module is required.+ model = nn.Sequential(nn.Linear(32, 32))+ resolution = resolve_formats(+ ResolvedScheme.from_scheme(scheme), format=fmt, model=model, scale_dtype=torch.float16+ )+ results[(scheme_name, fmt)] = [f.get_backend_name() for f in resolution.formats]+ assert results == EXPECTED_FORMAT_SELECTION_BASELINE+++def test_gguf_correction_propagates_to_scheme_and_quantize_config(monkeypatch):+ # W4A16's plain int-woq fields don't match every field gguf_args_check enforces+ # for a GGUF target; after resolving "gguf:q4_k_m" the scheme-bearing objects+ # must all agree, and self.scheme must be pinned to the precise preset string so+ # get_gguf_scheme()'s string short-circuit can disambiguate Q4_K_S vs Q4_K_M.+ from auto_round.schemes import get_gguf_scheme++ original_scheme = _resolved_scheme("W4A16")+ corrected_scheme = _resolved_scheme("GGUF:Q4_K_M")+ quantize_config = QuantizationConfig(scheme=original_scheme.copy())+ alg_config = QuantizationConfig(scheme=original_scheme.copy())+ resolved_format = SimpleNamespace(+ is_gguf=lambda: True,+ output_format="gguf",+ backend=SimpleNamespace(output_format="gguf:q4_k_m"),+ )++ def fake_resolve_formats(*args, **kwargs):+ return FormatResolution(+ formats=(resolved_format,),+ scheme=ResolvedScheme.from_scheme(corrected_scheme, preset_name="gguf:q4_k_m"),+ layer_config_patch={"layer": {"bits": 4}},+ scale_dtype=torch.float32,+ quant_block_list=(("model.layers.0",),),+ )++ monkeypatch.setattr("auto_round.compressors.base.resolve_formats", fake_resolve_formats)++ class FormatResolutionHost:+ _resolve_gguf_preset_string = staticmethod(BaseCompressor._resolve_gguf_preset_string)+ _resolve_format_string = BaseCompressor._resolve_format_string++ ar = FormatResolutionHost()+ ar.scheme_context = original_scheme+ ar.scheme = original_scheme.copy()+ ar.quantize_config = quantize_config+ ar._alg_configs = [alg_config, quantize_config]+ ar.model_context = SimpleNamespace(model=nn.Linear(1, 1), is_mllm=False)+ ar.layer_config = None+ ar.scale_dtype = torch.float16+ ar.quant_nontext_module = False+ ar.quant_block_list = None+ ar.platform = "hf"+ ar.is_auto_scheme = False++ formats = ar._resolve_format_string("gguf:q4_k_m")++ assert formats == [resolved_format]+ assert isinstance(ar.scheme, str)+ assert ar.scheme.lower() == "gguf:q4_k_m"+ assert get_gguf_scheme(ar.scheme) == ar.scheme # short-circuit preserved+ assert ar.scheme_context == corrected_scheme+ assert alg_config.scheme is ar.scheme_context+ assert quantize_config.scheme is ar.scheme_context+ assert ar.scheme_context.bits == ar.quantize_config.bits+ assert ar.scheme_context.data_type == ar.quantize_config.data_type+ assert ar.scheme_context.sym == ar.quantize_config.sym+ assert ar.layer_config == {"layer": {"bits": 4}}+ assert ar.scale_dtype is torch.float32+ assert ar.quant_block_list == [["model.layers.0"]]diff --git a/test/test_cpu/core/test_init.py b/test/test_cpu/core/test_init.pyindex 2741f9745..e9f0ccccc 100644--- a/test/test_cpu/core/test_init.py+++ b/test/test_cpu/core/test_init.py@@ -5,21 +5,15 @@ from auto_round import AutoRoundfrom auto_round.auto_scheme import AutoSchemefrom auto_round.cli.parser import build_quantize_parserfrom auto_round.compressors.base import BaseOrchestrator-from auto_round.compressors.entry import AutoRound as NewAutoRound-from auto_round.compressors.entry import AutoRoundCompatibledef test_torch_compile_platform_default_is_deferred():assert inspect.signature(AutoRound.__new__).parameters["enable_torch_compile"].default is None- assert inspect.signature(NewAutoRound.__new__).parameters["enable_torch_compile"].default is None- assert inspect.signature(AutoRoundCompatible.__new__).parameters["enable_torch_compile"].default is Noneassert inspect.signature(BaseOrchestrator.__init__).parameters["enable_torch_compile"].default is Nonedef test_dataset_default_is_deferred():assert inspect.signature(AutoRound.__new__).parameters["dataset"].default is None- assert inspect.signature(NewAutoRound.__new__).parameters["dataset"].default is None- assert inspect.signature(AutoRoundCompatible.__new__).parameters["dataset"].default is Noneassert inspect.signature(BaseOrchestrator.__init__).parameters["dataset"].default is None@@ -103,7 +97,7 @@ def test_argparse_check(tiny_opt_model_path):# so CalibrationState defaulted to 1 regardless of the user's value.steps = 8- ar = NewAutoRound(+ ar = AutoRound(tiny_opt_model_path,scheme="W4A16",gradient_accumulate_steps=steps,diff --git a/test/test_cpu/core/test_legacy_plan_parity.py b/test/test_cpu/core/test_legacy_plan_parity.pynew file mode 100644index 000000000..1ecd6b2a7--- /dev/null+++ b/test/test_cpu/core/test_legacy_plan_parity.py@@ -0,0 +1,35 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++from auto_round import AutoRound, SignRoundConfig+++def test_direct_and_algorithm_config_entries_build_equivalent_plans(tiny_opt_model_path):+ common = {+ "model": tiny_opt_model_path,+ "scheme": "W4A16",+ "iters": 0,+ "nsamples": 1,+ "seqlen": 8,+ "dataset": ["local calibration sample"],+ "low_cpu_mem_usage": False,+ }+ direct = AutoRound(bits=4, group_size=128, **common)+ configured = AutoRound(alg_configs=SignRoundConfig(bits=4, group_size=128, iters=0), **common)++ direct.post_init()+ configured.post_init()++ assert direct.compression_plan.scheme.value == configured.compression_plan.scheme.value+ assert direct.compression_plan.layer_config == configured.compression_plan.layer_configdiff --git a/test/test_cpu/core/test_pipeline_fail_fast.py b/test/test_cpu/core/test_pipeline_fail_fast.pyindex 59972e557..ceb1f8ce3 100644--- a/test/test_cpu/core/test_pipeline_fail_fast.py+++ b/test/test_cpu/core/test_pipeline_fail_fast.py@@ -2,6 +2,7 @@import pytest+from auto_round import AutoRound as NewAutoRoundfrom auto_round import AWQConfig, OptimizedRTNConfig, RotationConfig, RTNConfig, SignRoundConfig, SpinQuantConfigfrom auto_round.algorithms.composer import AlgorithmComposer, _can_compile_block_forwardfrom auto_round.algorithms.config_resolver import (@@ -11,9 +12,8 @@ from auto_round.algorithms.config_resolver import ()from auto_round.algorithms.quantization import registry as _rfrom auto_round.algorithms.quantization.rtn.quantizer import RTNQuantizer+from auto_round.autoround import _select_rtn_compressor_base_clsfrom auto_round.compressors.base import collect_user_scheme_overrides-from auto_round.compressors.entry import AutoRound as NewAutoRound-from auto_round.compressors.entry import _select_rtn_compressor_base_clsfrom auto_round.compressors.orchestrator import CompressionOrchestrator as Compressorfrom auto_round.logger import loggerfrom auto_round.schemes import QuantizationScheme@@ -108,78 +108,49 @@ def test_new_entry_defaults_to_autoround_config(monkeypatch):monkeypatch.setattr(Compressor, "__init__", _fake_init)monkeypatch.setattr("auto_round.utils.model.detect_model_type", lambda *args, **kwargs: "llm")- NewAutoRound("dummy-model", "W4A16", iters=1, seqlen=8, nsamples=1)+ NewAutoRound("dummy-model", scheme="W4A16", iters=1, seqlen=8, nsamples=1)- assert isinstance(captured["config"], SignRoundConfig)+ assert isinstance(captured["config"], list)+ assert isinstance(captured["config"][0], SignRoundConfig)-def test_entry_rejects_configs_without_quantization_members():- with pytest.raises(ValueError, match="At least one quantization algorithm config"):- NewAutoRound("dummy-model", "W4A16", [RotationConfig()])---def test_compat_entry_preserves_spinquant_dict_config(monkeypatch):+def test_new_entry_zero_iters_defaults_to_rtn_config(monkeypatch):captured = {}- rotation_config = {- "algorithm": "spinquant",- "r1": True,- "r2": True,- "r3": False,- "r4": False,- "rotation_size": 128,- "trainable_rotation": False,- "trainable_smooth": False,- }def _fake_init(self, config, **kwargs):captured["config"] = configmonkeypatch.setattr(Compressor, "__init__", _fake_init)- monkeypatch.setattr("auto_round.utils.is_mllm_model", lambda *args, **kwargs: False)- monkeypatch.setattr("auto_round.utils.is_diffusion_model", lambda *args, **kwargs: False)monkeypatch.setattr("auto_round.utils.model.detect_model_type", lambda *args, **kwargs: "llm")- from auto_round.autoround import AutoRound as CompatAutoRound-- CompatAutoRound(- "dummy-model",- scheme="W4A16",- iters=1,- seqlen=8,- nsamples=1,- rotation_config=rotation_config,- )+ NewAutoRound("dummy-model", scheme="W4A16", iters=0, disable_opt_rtn=True, disable_model_free=True)- configs = captured["config"] if isinstance(captured["config"], list) else [captured["config"]]- spinquant_cfg = next(cfg for cfg in configs if isinstance(cfg, SpinQuantConfig))- assert spinquant_cfg.rotation_size == rotation_config["rotation_size"]- assert spinquant_cfg.r1 is rotation_config["r1"]- assert spinquant_cfg.r2 is rotation_config["r2"]- assert spinquant_cfg.r3 is rotation_config["r3"]- assert spinquant_cfg.r4 is rotation_config["r4"]- assert spinquant_cfg.trainable_rotation is rotation_config["trainable_rotation"]- assert spinquant_cfg.trainable_smooth is rotation_config["trainable_smooth"]+ assert isinstance(captured["config"], list)+ assert isinstance(captured["config"][0], RTNConfig)+ assert captured["config"][0].disable_opt_rtn is True-def test_entry_warns_and_drops_unsupported_kwargs(monkeypatch, tiny_opt_model_path):- calls = []+def test_new_entry_accepts_rotation_config_in_algorithm_list(monkeypatch):+ captured = {}- def _record_warning(message, *args):- calls.append(message % args)+ def _fake_init(self, config, **kwargs):+ captured["config"] = config- monkeypatch.setattr(logger, "warning_once", _record_warning)+ monkeypatch.setattr(Compressor, "__init__", _fake_init)+ monkeypatch.setattr("auto_round.utils.model.detect_model_type", lambda *args, **kwargs: "llm")NewAutoRound(- tiny_opt_model_path,- "W4A16",- RTNConfig(disable_opt_rtn=True),- nsamples=1,- seqlen=8,- low_cpu_mem_usage=False,- nonsense_kwarg=123,+ "dummy-model",+ scheme="MXFP4",+ alg_configs=[SignRoundConfig(iters=1), RotationConfig()],)- assert any("unsupported kwargs nonsense_kwarg" in msg for msg in calls)+ assert any(isinstance(config, RotationConfig) for config in captured["config"])+++def test_entry_rejects_configs_without_quantization_members():+ with pytest.raises(TypeError, match="alg_configs entries must be algorithm aliases"):+ NewAutoRound("dummy-model", scheme="W4A16", alg_configs=[RotationConfig()])def test_shared_config_values_inherit_across_matching_attrs_only():diff --git a/test/test_cpu/export/test_export.py b/test/test_cpu/export/test_export.pyindex e23d2afbd..f8bb7cf21 100644--- a/test/test_cpu/export/test_export.py+++ b/test/test_cpu/export/test_export.py@@ -8,10 +8,12 @@ from packaging import versionfrom transformers import AutoModelForCausalLM, AutoRoundConfig, AutoTokenizerfrom auto_round import AutoRound+from auto_round.compressors.config_resolution import ResolvedSchemefrom auto_round.export.export_to_autogptq import export as autogptq_exportfrom auto_round.export.export_to_autoround import export as autoround_exportfrom auto_round.export.export_to_autoround import export_to_fp8 as autoround_fp8_exportfrom auto_round.export.export_to_awq import export as awq_export+from auto_round.export.formats import resolve_formatsfrom ...helpers import forbid_threaded_packing, get_model_path@@ -381,14 +383,18 @@ class TestAutoRound:print(res)def test_export_format(self):- from auto_round.formats import get_formats-autoround = AutoRound(self.model_name,scheme="FP8_STATIC",)autoround.post_init()- format_list = get_formats("auto_round, llm_compressor, auto_round:llm_compressor", autoround)+ resolution = resolve_formats(+ ResolvedScheme.from_scheme(autoround.scheme_context),+ format="auto_round, llm_compressor, auto_round:llm_compressor",+ model=autoround.model_context.model,+ scale_dtype=autoround.scale_dtype,+ )+ format_list = list(resolution.formats)assert len(format_list) == 3assert format_list[0].output_format == "auto_round"assert format_list[0].get_backend_name() == "auto_round:fp8_static"@@ -402,7 +408,13 @@ class TestAutoRound:scheme="W4A16",)autoround.post_init()- format_list = get_formats("auto_round:auto_awq, auto_gptq", autoround)+ resolution = resolve_formats(+ ResolvedScheme.from_scheme(autoround.scheme_context),+ format="auto_round:auto_awq, auto_gptq",+ model=autoround.model_context.model,+ scale_dtype=autoround.scale_dtype,+ )+ format_list = list(resolution.formats)assert format_list[0].output_format == "auto_round"assert format_list[0].get_backend_name() == "auto_round:auto_awq"assert format_list[1].output_format == "auto_gptq"@@ -413,7 +425,13 @@ class TestAutoRound:scheme="INT8",)autoround.post_init()- format_list = get_formats("llm_compressor, auto_round:llm_compressor", autoround)+ resolution = resolve_formats(+ ResolvedScheme.from_scheme(autoround.scheme_context),+ format="llm_compressor, auto_round:llm_compressor",+ model=autoround.model_context.model,+ scale_dtype=autoround.scale_dtype,+ )+ format_list = list(resolution.formats)assert format_list[0].output_format == "llm_compressor"assert format_list[0].get_backend_name() == "llm_compressor:int8_w8a8"assert format_list[1].output_format == "auto_round"@@ -425,15 +443,19 @@ class TestAutoRound:scheme="INT8_W8A8",)autoround_old.post_init()- format_list_old = get_formats("llm_compressor, auto_round:llm_compressor", autoround_old)+ resolution_old = resolve_formats(+ ResolvedScheme.from_scheme(autoround_old.scheme_context),+ format="llm_compressor, auto_round:llm_compressor",+ model=autoround_old.model_context.model,+ scale_dtype=autoround_old.scale_dtype,+ )+ format_list_old = list(resolution_old.formats)assert format_list_old[0].output_format == "llm_compressor"assert format_list_old[0].get_backend_name() == "llm_compressor:int8_w8a8"assert format_list_old[1].output_format == "auto_round"assert format_list_old[1].get_backend_name() == "auto_round:llm_compressor:int8_w8a8"def test_export_format_with_scheme(self, tiny_qwen_model_path):- from auto_round.formats import get_formats-ar = AutoRound(model=tiny_qwen_model_path,scheme="W4A16",@@ -443,10 +465,20 @@ class TestAutoRound:)ar.post_init()with pytest.raises(ValueError, match="auto_awq format support quantization scheme with W4A16 but got bits=2"):- get_formats("auto_round:auto_awq", ar)+ resolve_formats(+ ResolvedScheme.from_scheme(ar.scheme_context),+ format="auto_round:auto_awq",+ model=ar.model_context.model,+ scale_dtype=ar.scale_dtype,+ )with pytest.raises(ValueError, match="but got bits=2, data_type=int"):- get_formats("auto_round:llm_compressor", ar)+ resolve_formats(+ ResolvedScheme.from_scheme(ar.scheme_context),+ format="auto_round:llm_compressor",+ model=ar.model_context.model,+ scale_dtype=ar.scale_dtype,+ )ar = AutoRound(model=tiny_qwen_model_path,@@ -457,7 +489,12 @@ class TestAutoRound:)ar.post_init()with pytest.raises(ValueError, match="but got data_type=fp, bits=4"):- get_formats("auto_round:llm_compressor", ar)+ resolve_formats(+ ResolvedScheme.from_scheme(ar.scheme_context),+ format="auto_round:llm_compressor",+ model=ar.model_context.model,+ scale_dtype=ar.scale_dtype,+ )ar = AutoRound(model=tiny_qwen_model_path,@@ -467,7 +504,12 @@ class TestAutoRound:sym=True,)ar.post_init()- get_formats("auto_round:auto_awq", ar)+ resolve_formats(+ ResolvedScheme.from_scheme(ar.scheme_context),+ format="auto_round:auto_awq",+ model=ar.model_context.model,+ scale_dtype=ar.scale_dtype,+ )def test_autoawq_qwen3_vl_infer(self, dataloader):model_path = get_model_path("Qwen/Qwen3-VL-2B-Instruct")diff --git a/test/test_cpu/formats/test_format_resolver.py b/test/test_cpu/formats/test_format_resolver.pynew file mode 100644index 000000000..f2b922576--- /dev/null+++ b/test/test_cpu/formats/test_format_resolver.py@@ -0,0 +1,47 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++import pytest++from auto_round.compressors.config_resolution import FormatCompatibilityError, resolve_scheme_value+from auto_round.export.formats import resolve_formats+++def test_resolve_formats_does_not_mutate_inputs():+ layer_config = {"layer": {"bits": 4}}+ scheme = resolve_scheme_value("W4A16", {})++ result = resolve_formats(scheme, format="auto_round", layer_config=layer_config, model=None)++ layer_config["layer"]["bits"] = 8+ assert scheme.value.bits == 4+ assert result.scheme.value.bits == 4+ assert result.layer_config_patch["layer"]["bits"] == 4+++def test_resolve_formats_rejects_gguf_with_real_companion():+ scheme = resolve_scheme_value("W4A16", {})++ with pytest.raises(FormatCompatibilityError):+ resolve_formats(scheme, format="gguf:q4_k_m,auto_round", model=None)+++def test_resolve_formats_allows_gguf_with_fake_companion():+ scheme = resolve_scheme_value("W4A16", {})++ result = resolve_formats(scheme, format="gguf:q4_k_m,fake", model=None)++ assert sorted(f.output_format for f in result.formats) == ["fake", "gguf"]+ assert any(f.is_gguf() for f in result.formats)+ assert any(f.is_fake() for f in result.formats)diff --git a/test/test_cpu/integrations/test_llmc_integration.py b/test/test_cpu/integrations/test_llmc_integration.pyindex 9a3082be6..c297dc95a 100644--- a/test/test_cpu/integrations/test_llmc_integration.py+++ b/test/test_cpu/integrations/test_llmc_integration.py@@ -7,7 +7,7 @@ from llmcompressor import oneshotfrom llmcompressor.modifiers.autoround import AutoRoundModifierfrom transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer-from auto_round import AutoRound+from auto_round import AutoRound, AWQConfigfrom auto_round.calib_dataset import get_datasetfrom ...envs import multi_card@@ -253,7 +253,7 @@ def test_llmc_awq_w8a8_export_config_args(tiny_opt_model_path, tmp_path):ar = AutoRound(tiny_opt_model_path,scheme="INT8",- algorithm="awq",+ alg_configs=AWQConfig(),nsamples=2,seqlen=32,batch_size=2,diff --git a/test/test_cpu/layer_config/test_apply.py b/test/test_cpu/layer_config/test_apply.pynew file mode 100644index 000000000..976cabdbc--- /dev/null+++ b/test/test_cpu/layer_config/test_apply.py@@ -0,0 +1,33 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++import torch.nn as nn++from auto_round.compressors.config_resolution import ResolvedQuantizationConfig, ResolvedScheme+from auto_round.compressors.layer_config import apply_plan_to_model+from auto_round.schemes import QuantizationScheme+++def test_apply_plan_is_the_explicit_model_write_boundary():+ model = nn.Sequential(nn.Linear(32, 32))+ plan = ResolvedQuantizationConfig(+ scheme=ResolvedScheme.from_scheme(QuantizationScheme()),+ formats=(),+ layer_config={"0": {"bits": 4, "data_type": "int"}},+ )++ apply_plan_to_model(model, plan)++ assert model[0].bits == 4+ assert model[0].data_type == "int"diff --git a/test/test_cpu/layer_config/test_layer_config_resolver.py b/test/test_cpu/layer_config/test_layer_config_resolver.pynew file mode 100644index 000000000..ad369cd48--- /dev/null+++ b/test/test_cpu/layer_config/test_layer_config_resolver.py@@ -0,0 +1,60 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++import torch.nn as nn++from auto_round.compressors.config_resolution import ResolvedScheme+from auto_round.compressors.layer_config import (+ extract_regex_config,+ has_quantized_layer_outside_blocks,+ resolve_layer_config,+)+from auto_round.schemes import QuantizationScheme+++def test_resolver_does_not_write_quantization_attributes_to_modules():+ model = nn.Sequential(nn.Linear(32, 32))+ scheme = ResolvedScheme.from_scheme(QuantizationScheme(act_bits=16, act_data_type="float"))++ resolved = resolve_layer_config(+ model=model,+ scheme=scheme,+ layer_config={"0": {}},+ supported_types=(nn.Linear,),+ inner_supported_types=(),+ )++ assert "0" in resolved+ assert not hasattr(model[0], "bits")+++def test_extract_regex_config_keeps_pattern_separate_from_expanded_layers():+ model = nn.Sequential(nn.Linear(32, 32))+ scheme = ResolvedScheme.from_scheme(QuantizationScheme(act_bits=16, act_data_type="float"))++ regex_config = extract_regex_config(+ model=model,+ scheme=scheme,+ layer_config={"0": {"bits": 4}, "missing.*": {"bits": 16}},+ supported_types=(nn.Linear,),+ inner_supported_types=(),+ )++ assert set(regex_config) == {"missing.*"}+ assert regex_config["missing.*"]["bits"] == 16+++def test_has_quantized_layer_outside_blocks_is_derived_from_final_mapping():+ assert has_quantized_layer_outside_blocks({"layer": {"bits": 4, "in_blocks": False}})+ assert not has_quantized_layer_outside_blocks({"layer": {"bits": 16, "in_blocks": False}})diff --git a/test/test_cpu/models/test_moe_model.py b/test/test_cpu/models/test_moe_model.pyindex cb21213f7..d97c28663 100644--- a/test/test_cpu/models/test_moe_model.py+++ b/test/test_cpu/models/test_moe_model.py@@ -61,9 +61,10 @@ def test_gptoss(scheme, tiny_gpt_oss_model_path, tmp_path):assert (single_expert_cnt == config.num_local_experts), f"Expected {config.num_local_experts} GPTOssSingleExpert modules, found {single_expert_cnt}."+ expected_quant_linear_cnt = config.num_hidden_layers * 3 * config.num_local_expertsassert (- quant_linear_cnt == config.num_hidden_layers * 3 * config.num_local_experts- ), f"Expected {config.num_hidden_layers * 3 * config.num_local_experts} QuantLinear modules, found {quant_linear_cnt}."+ quant_linear_cnt == expected_quant_linear_cnt+ ), f"Expected {expected_quant_linear_cnt} QuantLinear modules, found {quant_linear_cnt}."# verify the quantized model can be loaded and run inferenceloaded_model = GptOssForCausalLM.from_pretrained(save_folder)diff --git a/test/test_cpu/quantization/test_model_free.py b/test/test_cpu/quantization/test_model_free.pyindex 8bba4b9fd..46ea0d35e 100644--- a/test/test_cpu/quantization/test_model_free.py+++ b/test/test_cpu/quantization/test_model_free.py@@ -74,6 +74,33 @@ from auto_round.compressors.model_free import (from auto_round.schemes import QuantizationSchemefrom auto_round.utils.model import is_model_free_route++def test_model_free_preserves_explicit_scheme_overrides():+ from auto_round.compressors.model_free import ModelFreeCompressor++ compressor = ModelFreeCompressor("unused-model-path", scheme="W8A16", sym=False)+ compressor._parse_scheme()++ assert compressor.default_scheme["sym"] is False+++def test_model_free_entry_passes_resolved_scheme_overrides(tiny_opt_model_path):+ from auto_round import AutoRound++ compressor = AutoRound(+ tiny_opt_model_path,+ scheme="W4A16",+ sym=False,+ iters=0,+ disable_opt_rtn=True,+ device_map="cpu",+ enable_torch_compile=False,+ )++ assert type(compressor).__name__ == "ModelFreeCompressor"+ assert compressor.scheme_input.sym is False++from ...envs import require_compressed_tensors# ---------------------------------------------------------------------------diff --git a/test/test_cpu/quantization/test_mx_quant_linear.py b/test/test_cpu/quantization/test_mx_quant_linear.pyindex 1ec5cb729..63733946d 100644--- a/test/test_cpu/quantization/test_mx_quant_linear.py+++ b/test/test_cpu/quantization/test_mx_quant_linear.py@@ -5,15 +5,15 @@ from auto_round.data_type.utils import get_quant_funcfrom auto_round.experimental import qmodules as ar_qmodulesfrom auto_round.export.export_to_autoround.qlinear_fp import QuantLinear as _MXFPLinearfrom auto_round.export.export_to_autoround.qlinear_int import QuantLinear as _MXINTLinear-from auto_round.formats import AutoRoundExportFormat+from auto_round.export.formats import BackendDataTypefrom auto_round.schemes import PRESET_SCHEMES-mx_schemes = [AutoRoundExportFormat.MXFP8.value, AutoRoundExportFormat.MXFP4.value]-mx_int_schemes = [AutoRoundExportFormat.MXINT4.value]+mx_schemes = [BackendDataType.MXFP8.value, BackendDataType.MXFP4.value]+mx_int_schemes = [BackendDataType.MXINT4.value]QMODULE_MAPPING = {- AutoRoundExportFormat.MXFP8.value: ar_qmodules.MXFP8QuantLinear,- AutoRoundExportFormat.MXFP4.value: ar_qmodules.MXFP4QuantLinear,- AutoRoundExportFormat.MXINT4.value: ar_qmodules.MXINT4QuantLinear,+ BackendDataType.MXFP8.value: ar_qmodules.MXFP8QuantLinear,+ BackendDataType.MXFP4.value: ar_qmodules.MXFP4QuantLinear,+ BackendDataType.MXINT4.value: ar_qmodules.MXINT4QuantLinear,}diff --git a/test/test_cpu/quantization/test_mxfp_save_load.py b/test/test_cpu/quantization/test_mxfp_save_load.pyindex ea7e4478d..d2b0a69e2 100644--- a/test/test_cpu/quantization/test_mxfp_save_load.py+++ b/test/test_cpu/quantization/test_mxfp_save_load.py@@ -10,23 +10,23 @@ from auto_round import AutoRoundfrom auto_round import schemes as ar_schemesfrom auto_round.experimental import qmodules as ar_qmodulesfrom auto_round.export.export_to_autoround import qlinear_fp as ar_qlinear_fp-from auto_round.formats import AutoRoundExportFormat+from auto_round.export.formats import BackendDataTypefrom auto_round.inference.backend import MX_TENSOR_DATA_TYPESfrom ...envs import has_modulefrom ...helpers import get_model_pathtesting_scheme_name_lst = [- AutoRoundExportFormat.MXFP8.value,- AutoRoundExportFormat.MXFP4.value,+ BackendDataType.MXFP8.value,+ BackendDataType.MXFP4.value,]QMODULE_MAPPING = {- AutoRoundExportFormat.MXFP8.value: ar_qmodules.MXFP8QuantLinear,- AutoRoundExportFormat.MXFP4.value: ar_qmodules.MXFP4QuantLinear,+ BackendDataType.MXFP8.value: ar_qmodules.MXFP8QuantLinear,+ BackendDataType.MXFP4.value: ar_qmodules.MXFP4QuantLinear,}SCHEMES_MAPPING = {- AutoRoundExportFormat.MXFP8.value: ar_schemes.MXFP8,- AutoRoundExportFormat.MXFP4.value: ar_schemes.MXFP4,+ BackendDataType.MXFP8.value: ar_schemes.MXFP8,+ BackendDataType.MXFP4.value: ar_schemes.MXFP4,}MX_TENSOR_DATA_TYPES_FP = [i for i in MX_TENSOR_DATA_TYPES if "int" not in i]diff --git a/test/test_cpu/quantization/test_nvfp4_quant_linear.py b/test/test_cpu/quantization/test_nvfp4_quant_linear.pyindex 9dc84fe15..860ef0ef0 100644--- a/test/test_cpu/quantization/test_nvfp4_quant_linear.py+++ b/test/test_cpu/quantization/test_nvfp4_quant_linear.py@@ -5,11 +5,11 @@ from auto_round.data_type.nvfp import calculate_gparamfrom auto_round.data_type.utils import get_quant_funcfrom auto_round.experimental import qmodules as ar_qmodulesfrom auto_round.export.export_to_autoround.qlinear_fp import QuantLinear as _FPLinear-from auto_round.formats import AutoRoundExportFormat+from auto_round.export.formats import BackendDataTypefrom auto_round.schemes import PRESET_SCHEMESQMODULE_MAPPING = {- AutoRoundExportFormat.NVFP4.value: ar_qmodules.NVFP4QuantLinear,+ BackendDataType.NVFP4.value: ar_qmodules.NVFP4QuantLinear,}@@ -35,7 +35,7 @@ def test_calculate_gparam_with_float8_input():assert torch.isfinite(global_scale)[email protected]("scheme", [AutoRoundExportFormat.NVFP4.value])[email protected]("scheme", [BackendDataType.NVFP4.value])@torch.inference_mode()def test_nvfp4_quantlinear_from_original_and_forward(scheme):"""diff --git a/test/test_cpu/schemes/test_scheme_decoupling.py b/test/test_cpu/schemes/test_scheme_decoupling.pynew file mode 100644index 000000000..136eece0b--- /dev/null+++ b/test/test_cpu/schemes/test_scheme_decoupling.py@@ -0,0 +1,133 @@+# Copyright (c) 2025 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+"""Decoupling regression for the `quantization-scheme` capability.++Covers the single-authority classification predicates on ``QuantizationScheme``+(D1), the two user-confirmed drift-point semantics (D2, Option A), and the GGUF+scheme-facts source that was pushed down into ``schemes.py`` (D4).+"""++import copy++from auto_round.algorithms.quantization.config import QuantizationConfig+from auto_round.schemes import GGUF_PRESET_ALIASES, GGUF_SCHEME_FACTS, BackendDataType, QuantizationScheme+++def _scheme(**overrides):+ base = dict(+ bits=4,+ group_size=128,+ sym=True,+ data_type="int",+ act_bits=16,+ act_group_size=None,+ act_sym=None,+ act_data_type=None,+ act_dynamic=None,+ super_bits=None,+ super_group_size=None,+ )+ base.update(overrides)+ return QuantizationScheme(**base)+++# --- D1: classification predicates are single-authority on QuantizationScheme ---+++def test_backend_data_type_members():+ assert BackendDataType.STANDARD_FP == "fp"+ assert BackendDataType.MX_FP == "mx_fp"+ assert BackendDataType.NV_FP == "nv_fp"+ assert BackendDataType.MX_INT == "mx_int"+ assert BackendDataType.FP8_STATIC == "fp8_static"+ assert BackendDataType.FP8 == "fp8"+++def test_standard_and_mx_and_nv_predicates():+ assert _scheme(data_type="fp").is_standard_fp() is True+ assert _scheme(data_type="mx_fp").is_standard_fp() is False+ assert _scheme(data_type="nv_fp").is_standard_fp() is False+ assert _scheme(data_type="mx_fp").is_mx_fp() is True+ assert _scheme(data_type="nv_fp").is_nv_fp() is True+ assert _scheme(data_type="mx_int").is_mx_int() is True+ assert _scheme(act_data_type="fp").is_act_standard_fp() is True+ assert _scheme(act_data_type="mx_fp").is_act_mx_fp() is True+ assert _scheme(act_data_type="nv_fp").is_act_nv_fp() is True+++def test_int_and_fp8_shape_predicates():+ assert _scheme(data_type="int", act_bits=16, super_group_size=None).is_wint_woq() is True+ assert _scheme(data_type="int", act_bits=8).is_wint_woq() is False+ assert _scheme(data_type="int", act_bits=16, super_group_size=8).is_wint_woq() is False+ assert _scheme(data_type="fp", bits=8, act_data_type="fp", act_bits=8).is_wfp8afp8() is True+ assert _scheme(data_type="int8", act_data_type="int8").is_wint8aint8() is True+ assert _scheme(data_type="int4", bits=4, act_data_type="int4", act_bits=4).is_wint4aint4() is True+ assert _scheme(act_dynamic=True, act_data_type="fp", act_bits=8).is_dynamic_afp8() is True+ assert _scheme(group_size=(128, 128), data_type="fp", bits=8).is_block_wfp8() is True+ assert _scheme(act_data_type="fp8_static").is_static_afp8() is True+ assert _scheme(act_bits=8).is_act_quantize() is True+ assert _scheme(act_bits=16).is_act_quantize() is False+++# --- D2: the two confirmed drift points use Option A (utils.py) semantics ---+++def test_is_act_static_matches_fp8_static_preset_option_a():+ # FP8_STATIC preset itself has data_type="fp"/act_data_type="fp" (no literal+ # "fp8_static"); Option A must still classify it as static.+ assert _scheme(data_type="fp", bits=8, act_data_type="fp", act_bits=8, act_dynamic=False).is_act_static()+ assert not _scheme(data_type="fp", bits=8, act_data_type="fp", act_bits=8, act_dynamic=True).is_act_static()+++def test_is_dynamic_wint8aint8_requires_int8_shape_option_a():+ # Option A: act_dynamic alone is not sufficient (AND, not OR).+ assert _scheme(data_type="int8", act_data_type="int8", act_dynamic=True).is_dynamic_wint8aint8()+ assert not _scheme(data_type="fp", act_data_type="fp", act_dynamic=True, bits=8, act_bits=8).is_dynamic_wint8aint8()+++def test_quantization_config_predicates_delegate_to_scheme_option_a():+ # config.py properties must delegate to the scheme's single-authority predicates.+ assert (+ QuantizationConfig(+ bits=8, data_type="fp", act_bits=8, act_data_type="fp", act_dynamic=True+ ).is_dynamic_wint8aint8+ is False+ )+ assert (+ QuantizationConfig(bits=8, data_type="fp", act_bits=8, act_data_type="fp", act_dynamic=False).is_act_static+ is True+ )+++# --- D4: GGUF scheme facts pushed down into schemes.py ---+++def test_gguf_scheme_facts_only_hold_scheme_fields():+ scheme_field_names = {"bits", "act_bits", "group_size", "sym", "data_type", "super_bits", "super_group_size"}+ for key, facts in GGUF_SCHEME_FACTS.items():+ assert set(facts.keys()) <= scheme_field_names, f"{key} leaks format fields: {facts.keys()}"+ assert not ({"embedding", "lm_head", "mostly"} & set(facts.keys()))+++def test_gguf_preset_aliases_and_facts_are_independent_objects():+ # Aliases resolve to a base facts key...+ assert GGUF_PRESET_ALIASES["gguf:q4_k_m"] == "gguf:q4_k"+ assert GGUF_PRESET_ALIASES["gguf:q2_k_mixed"] == "gguf:q2_k"+ # ...and mutating one facts entry must not bleed into a sibling (no shared refs).+ facts_copy = copy.deepcopy(GGUF_SCHEME_FACTS)+ GGUF_SCHEME_FACTS["gguf:q4_k"]["bits"] = 999+ try:+ assert GGUF_SCHEME_FACTS["gguf:q3_k"]["bits"] == facts_copy["gguf:q3_k"]["bits"]+ finally:+ GGUF_SCHEME_FACTS["gguf:q4_k"]["bits"] = facts_copy["gguf:q4_k"]["bits"]diff --git a/test/test_cpu/utils/test_layer_config_resolution.py b/test/test_cpu/utils/test_layer_config_resolution.pynew file mode 100644index 000000000..60d77ff41--- /dev/null+++ b/test/test_cpu/utils/test_layer_config_resolution.py@@ -0,0 +1,239 @@+# Copyright (c) 2025 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+"""Characterization for the `layer-config-resolution` capability.++Locks the per-layer output of ``set_layer_config`` (the public entry point that+the ``_resolve_*`` / ``_apply_*`` helpers were split out of) against verbatim+pre-refactor baselines for the W4A16, mixed-precision, and GGUF paths, plus the+ignore-layer regex-export behavior. Baselines were harvested from HEAD=33b7df0e.+"""++import shutil+import tempfile+from types import SimpleNamespace++import torch+import torch.nn as nn+from safetensors.torch import save_file++from auto_round.compressors.utils import set_layer_config+from auto_round.schemes import PRESET_SCHEMES+++class _TinyModel(nn.Module):+ def __init__(self):+ super().__init__()+ self.model = nn.Module()+ self.model.layers = nn.ModuleList([nn.Module() for _ in range(2)])+ for i, layer in enumerate(self.model.layers):+ layer.self_attn = nn.Module()+ layer.self_attn.q_proj = nn.Linear(32, 32)+ layer.mlp = nn.Module()+ layer.mlp.up_proj = nn.Linear(32, 64)+ self.lm_head = nn.Linear(32, 100)+++class _Router(nn.Module):+ def __init__(self, size: int = 32):+ super().__init__()+ self.weight = nn.Parameter(torch.randn(size, size))+++class _UnsupportedGateModel(nn.Module):+ def __init__(self, size: int = 32):+ super().__init__()+ self.mlp = nn.Module()+ self.mlp.gate = _Router(size)+ self.mlp.up_proj = nn.Linear(size, size)+++def _layer_config_for(scheme_name, layer_config=None, name_or_path=None):+ model = _TinyModel()+ if name_or_path is not None:+ # GGUF path calls is_separate_lm_head(model), which reads model.name_or_path+ # and inspects a real on-disk safetensors file — a local dir is required to+ # avoid a network download.+ model.name_or_path = name_or_path+ scheme = PRESET_SCHEMES[scheme_name]+ # set_layer_config returns (layer_config, has_qlayer_outside_block, regex_config).+ return set_layer_config(+ model=model,+ layer_config=layer_config,+ default_scheme=scheme,+ default_scale_dtype=torch.float16,+ supported_types=(nn.Linear,),+ inner_supported_types=(),+ quant_block_list=None,+ quant_lm_head=False,+ enable_gguf_official_mixed=False,+ is_mllm=False,+ )+++_COMMON_NON_LM_HEAD = {+ "act_bits": 16,+ "act_data_type": None,+ "act_dynamic": None,+ "act_group_size": None,+ "act_sym": None,+ "bits": 4,+ "data_type": "int",+ "group_size": 128,+ "in_blocks": True,+ "rotation_config": None,+ "scale_dtype": torch.float16,+ "super_bits": None,+ "super_group_size": None,+ "sym": True,+}++_LM_HEAD_FP16 = {+ "act_bits": 16,+ "act_data_type": None,+ "act_dynamic": None,+ "act_group_size": None,+ "act_sym": None,+ "bits": 16,+ "data_type": "fp",+ "fixed_by_user": True,+ "group_size": 128,+ "in_blocks": False,+ "rotation_config": None,+ "scale_dtype": torch.float16,+ "super_bits": None,+ "super_group_size": None,+ "sym": True,+}++EXPECTED_W4A16_BASELINE = (+ {+ "lm_head": dict(_LM_HEAD_FP16),+ "model.layers.0.self_attn.q_proj": {**_COMMON_NON_LM_HEAD, "fixed_by_user": False},+ "model.layers.0.mlp.up_proj": {**_COMMON_NON_LM_HEAD, "fixed_by_user": False},+ "model.layers.1.self_attn.q_proj": {**_COMMON_NON_LM_HEAD, "fixed_by_user": False},+ "model.layers.1.mlp.up_proj": {**_COMMON_NON_LM_HEAD, "fixed_by_user": False},+ },+ False,+ {},+)++EXPECTED_MIXED_BASELINE = (+ {+ "lm_head": dict(_LM_HEAD_FP16),+ "model.layers.0.self_attn.q_proj": {**_COMMON_NON_LM_HEAD, "bits": 8, "fixed_by_user": True},+ "model.layers.0.mlp.up_proj": {**_COMMON_NON_LM_HEAD, "fixed_by_user": False},+ "model.layers.1.self_attn.q_proj": {**_COMMON_NON_LM_HEAD, "fixed_by_user": False},+ "model.layers.1.mlp.up_proj": {**_COMMON_NON_LM_HEAD, "fixed_by_user": False},+ },+ False,+ {},+)++_GGUF_NON_LM_HEAD = {+ "act_bits": 16,+ "act_data_type": None,+ "act_dynamic": None,+ "act_group_size": None,+ "act_sym": None,+ "bits": 5,+ "data_type": "int",+ "embedding": "gguf:q5_0",+ "fixed_by_user": False,+ "group_size": 32,+ "in_blocks": True,+ "lm_head": "gguf:q6_k",+ "mostly": "gguf:q5_0",+ "rotation_config": None,+ "scale_dtype": torch.float16,+ "super_bits": None,+ "super_group_size": None,+ "sym": True,+}++# NOTE: q_proj/up_proj (in_features=32) and lm_head (in_features=32) no longer keep+# their plain gguf:q4_k_m / gguf:q6_k config: `_apply_gguf_shape_fallback` (merged from+# main's #1960, "fix few bugs cause less accuracy of gguf format") falls each layer back+# to the largest gguf type whose block_size (32 / 32) evenly divides in_features=32 —+# gguf:q5_0 for the K-quant body layers, gguf:q8_0 for the K-quant lm_head — since+# gguf:q4_k_m / gguf:q6_k both require block_size=256. Rebaselined post-merge+# (2026-07-08); see docs/behavior-changes/decouple-scheme-format-layer-config.md.+EXPECTED_GGUF_BASELINE = (+ {+ "model.layers.0.self_attn.q_proj": dict(_GGUF_NON_LM_HEAD),+ "model.layers.0.mlp.up_proj": dict(_GGUF_NON_LM_HEAD),+ "model.layers.1.self_attn.q_proj": dict(_GGUF_NON_LM_HEAD),+ "model.layers.1.mlp.up_proj": dict(_GGUF_NON_LM_HEAD),+ "lm_head": {+ "act_bits": 16,+ "bits": 8,+ "data_type": "int",+ "embedding": "gguf:q8_0",+ "fixed_by_user": False,+ "group_size": 32,+ "lm_head": "gguf:q8_0",+ "mostly": "gguf:q8_0",+ "scale_dtype": torch.float16,+ "super_bits": None,+ "super_group_size": None,+ "sym": True,+ },+ },+ True,+ {},+)+++def test_w4a16_plain_baseline():+ assert _layer_config_for("W4A16") == EXPECTED_W4A16_BASELINE+++def test_mixed_precision_override_baseline():+ override = {"model.layers.0.self_attn.q_proj": {"bits": 8, "data_type": "int"}}+ assert _layer_config_for("W4A16", layer_config=override) == EXPECTED_MIXED_BASELINE+++def test_gguf_q4_k_m_baseline():+ tmp_dir = tempfile.mkdtemp(prefix="ar_characterization_gguf_")+ try:+ # Empty safetensors file so is_separate_lm_head() resolves locally (tied head).+ save_file({}, f"{tmp_dir}/model.safetensors")+ result = _layer_config_for("GGUF:Q4_K_M", name_or_path=tmp_dir)+ finally:+ shutil.rmtree(tmp_dir, ignore_errors=True)+ assert result == EXPECTED_GGUF_BASELINE+++def test_ignore_layer_unsupported_module_kept_as_regex_export_entry():+ # An ignore target on an unsupported module type must not land in layer_config+ # but must be preserved verbatim in regex_config for export.+ layer_config, _, regex_config = _layer_config_for("W4A16") # warm-up sanity+ model = _UnsupportedGateModel()+ layer_config, _, regex_config = set_layer_config(+ model=model,+ layer_config={},+ default_scheme="W4A16",+ default_scale_dtype=torch.float16,+ supported_types=(nn.Linear,),+ inner_supported_types=(),+ ignore_layers="mlp.gate",+ quant_lm_head=False,+ enable_gguf_official_mixed=False,+ is_mllm=False,+ )+ assert "mlp.gate" not in layer_config+ assert regex_config["mlp.gate"]["bits"] == 16+ assert regex_config["mlp.gate"]["act_bits"] == 16+ assert regex_config["mlp.gate"]["data_type"] == "float"+ assert regex_config["mlp.gate"]["act_data_type"] == "float"+ assert regex_config["mlp.gate"]["fixed_by_user"] is Truediff --git a/test/test_cpu/utils/test_utils.py b/test/test_cpu/utils/test_utils.pyindex d15365024..f818e376e 100644--- a/test/test_cpu/utils/test_utils.py+++ b/test/test_cpu/utils/test_utils.py@@ -156,12 +156,12 @@ class TestPredefinedIgnoreLayersBlockFilter:return stub@patch("auto_round.compressors.base.get_predefined_ignore_layers")- @patch("auto_round.compressors.base.set_layer_config")+ @patch("auto_round.compressors.base.resolve_layer_config")@patch("auto_round.compressors.base._handle_special_schemes", return_value=None)def test_kimi_k25_ignore_layers_preserved(self, mock_handle, mock_set_lc, mock_get_predefined):"""KIMI K2.5: vision_tower and mm_projector must end up in ignore_layers."""mock_get_predefined.return_value = ["vision_tower", "mm_projector"]- mock_set_lc.return_value = ({}, False, None)+ mock_set_lc.return_value = {}stub = self._make_compressor_stub(predefined_ignore_layers=["vision_tower", "mm_projector"],@@ -177,13 +177,13 @@ class TestPredefinedIgnoreLayersBlockFilter:), f"mm_projector should be in ignore_layers, got: '{stub.ignore_layers}'"@patch("auto_round.compressors.base.get_predefined_ignore_layers")- @patch("auto_round.compressors.base.set_layer_config")+ @patch("auto_round.compressors.base.resolve_layer_config")@patch("auto_round.compressors.base._handle_special_schemes", return_value=None)def test_step3p5_ignore_layers_preserved(self, mock_handle, mock_set_lc, mock_get_predefined):"""step3p5: short pattern names must not be dropped by block filter."""predefined = ["g_proj", "moe.gate", "eh_proj", "shared_head", "layers.45"]mock_get_predefined.return_value = predefined- mock_set_lc.return_value = ({}, False, None)+ mock_set_lc.return_value = {}stub = self._make_compressor_stub(predefined_ignore_layers=predefined,@@ -195,12 +195,12 @@ class TestPredefinedIgnoreLayersBlockFilter:assert name in stub.ignore_layers, f"'{name}' should be in ignore_layers, got: '{stub.ignore_layers}'"@patch("auto_round.compressors.base.get_predefined_ignore_layers")- @patch("auto_round.compressors.base.set_layer_config")+ @patch("auto_round.compressors.base.resolve_layer_config")@patch("auto_round.compressors.base._handle_special_schemes", return_value=None)def test_longcat_classifier_preserved(self, mock_handle, mock_set_lc, mock_get_predefined):"""Longcat: 'classifier' must not be dropped."""mock_get_predefined.return_value = ["classifier"]- mock_set_lc.return_value = ({}, False, None)+ mock_set_lc.return_value = {}stub = self._make_compressor_stub(predefined_ignore_layers=["classifier"],@@ -211,12 +211,12 @@ class TestPredefinedIgnoreLayersBlockFilter:assert "classifier" in stub.ignore_layers, f"classifier should be in ignore_layers, got: '{stub.ignore_layers}'"@patch("auto_round.compressors.base.get_predefined_ignore_layers")- @patch("auto_round.compressors.base.set_layer_config")+ @patch("auto_round.compressors.base.resolve_layer_config")@patch("auto_round.compressors.base._handle_special_schemes", return_value=None)def test_glm_flash_full_path_inside_block_preserved(self, mock_handle, mock_set_lc, mock_get_predefined):"""GLM Flash: model.layers.0.mlp (full path inside block) must be kept."""mock_get_predefined.return_value = ["model.layers.0.mlp"]- mock_set_lc.return_value = ({}, False, None)+ mock_set_lc.return_value = {}stub = self._make_compressor_stub(predefined_ignore_layers=["model.layers.0.mlp"],@@ -229,12 +229,12 @@ class TestPredefinedIgnoreLayersBlockFilter:), f"model.layers.0.mlp should be in ignore_layers, got: '{stub.ignore_layers}'"@patch("auto_round.compressors.base.get_predefined_ignore_layers")- @patch("auto_round.compressors.base.set_layer_config")+ @patch("auto_round.compressors.base.resolve_layer_config")@patch("auto_round.compressors.base._handle_special_schemes", return_value=None)def test_mllm_with_multiple_block_groups(self, mock_handle, mock_set_lc, mock_get_predefined):"""MLLM with vision + language blocks: ignore layers outside both are preserved."""mock_get_predefined.return_value = ["vision_tower", "mm_projector"]- mock_set_lc.return_value = ({}, False, None)+ mock_set_lc.return_value = {}stub = self._make_compressor_stub(predefined_ignore_layers=["vision_tower", "mm_projector"],diff --git a/test/test_cuda/algorithms/test_awq.py b/test/test_cuda/algorithms/test_awq.pyindex f12e259c8..69644581b 100644--- a/test/test_cuda/algorithms/test_awq.py+++ b/test/test_cuda/algorithms/test_awq.py@@ -29,7 +29,7 @@ import pytestimport torchfrom transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer-from auto_round import AutoRound+from auto_round import AutoRound, AWQConfigfrom ...helpers import eval_generated_prompt, evaluate_accuracy, generate_prompt, get_model_path, opt_name_or_path@@ -52,7 +52,7 @@ class TestAWQLLM:ar = AutoRound(tiny_opt_model_path,scheme="W4A16",- algorithm="awq",+ alg_configs=AWQConfig(),n_grid=4,nsamples=2,seqlen=32,@@ -77,7 +77,7 @@ class TestAWQLLM:bits=bits,group_size=group_size,sym=sym,- algorithm="awq",+ alg_configs=AWQConfig(),n_grid=4,nsamples=2,seqlen=32,@@ -99,7 +99,7 @@ class TestAWQLLM:ar = AutoRound(model_name,scheme="W4A16",- algorithm="awq",+ alg_configs=AWQConfig(),n_grid=4,nsamples=2,seqlen=32,@@ -159,7 +159,7 @@ class TestAWQMoE:ar = AutoRound(tiny_qwen_moe_model_path,scheme="W4A16",- algorithm="awq",+ alg_configs=AWQConfig(),nsamples=2,n_grid=4,seqlen=32,@@ -195,7 +195,7 @@ class TestAWQMoE:ar = AutoRound(tiny_qwen_moe_model_path,scheme="W4A16",- algorithm="awq",+ alg_configs=AWQConfig(),n_grid=4,nsamples=2,seqlen=32,@@ -239,7 +239,7 @@ class TestAWQEval:ar = AutoRound(self.model_name,scheme="W4A16",- algorithm="awq",+ alg_configs=AWQConfig(),nsamples=32,seqlen=32,batch_size=8,diff --git a/test/test_cuda/integrations/test_vllm.py b/test/test_cuda/integrations/test_vllm.pyindex b58254c5e..c4aa4e5ab 100644--- a/test/test_cuda/integrations/test_vllm.py+++ b/test/test_cuda/integrations/test_vllm.py@@ -15,7 +15,7 @@ import pytestfrom vllm import LLM, SamplingParamsfrom vllm.platforms import current_platform-from auto_round import AutoRound+from auto_round import AutoRound, AWQConfigfrom ...helpers import get_model_path@@ -187,7 +187,7 @@ def test_vllm_awq_w8a8_llmc_inference(tiny_opt_model_path, tmp_path):ar = AutoRound(tiny_opt_model_path,scheme="INT8",- algorithm="awq",+ alg_configs=AWQConfig(),nsamples=2,seqlen=32,batch_size=2,diff --git a/test/test_cuda/quantization/test_mxfp_nvfp.py b/test/test_cuda/quantization/test_mxfp_nvfp.pyindex bdc79075e..ab9078718 100644--- a/test/test_cuda/quantization/test_mxfp_nvfp.py+++ b/test/test_cuda/quantization/test_mxfp_nvfp.py@@ -11,20 +11,20 @@ from auto_round import AutoRoundfrom auto_round import schemes as ar_schemesfrom auto_round.experimental import qmodules as ar_qmodulesfrom auto_round.export.export_to_autoround import qlinear_fp as ar_qlinear_fp-from auto_round.formats import AutoRoundExportFormat+from auto_round.export.formats import BackendDataTypefrom ...envs import has_module, require_awq, require_optimumfrom ...helpers import get_model_path, save_tiny_modeltesting_schemes = [- AutoRoundExportFormat.MXFP8.value,- AutoRoundExportFormat.MXFP4.value,- AutoRoundExportFormat.NVFP4.value,+ BackendDataType.MXFP8.value,+ BackendDataType.MXFP4.value,+ BackendDataType.NVFP4.value,]QMODULE_MAPPING = {- AutoRoundExportFormat.MXFP8.value: ar_qmodules.MXFP8QuantLinear,- AutoRoundExportFormat.MXFP4.value: ar_qmodules.MXFP4QuantLinear,- AutoRoundExportFormat.NVFP4.value: ar_qmodules.NVFP4QuantLinear,+ BackendDataType.MXFP8.value: ar_qmodules.MXFP8QuantLinear,+ BackendDataType.MXFP4.value: ar_qmodules.MXFP4QuantLinear,+ BackendDataType.NVFP4.value: ar_qmodules.NVFP4QuantLinear,}diff --git a/test/test_mlx/test_mlx_format.py b/test/test_mlx/test_mlx_format.pyindex 81106ca6d..a302e3bd8 100644--- a/test/test_mlx/test_mlx_format.py+++ b/test/test_mlx/test_mlx_format.py@@ -378,11 +378,11 @@ class TestMLXFormat:config with ``false`` entries.5. ``rope_parameters`` has been flattened (``rope_theta`` surfaced)."""- from auto_round import AutoRoundMLLM+ from auto_round import AutoRoundsave_dir = str(tmp_path / "qwen3_vl_9b_mlx")try:- ar = AutoRoundMLLM(+ ar = AutoRound(qwen_3_vl_9b_name_or_path,scheme="W4A16",bits=4,@@ -467,11 +467,11 @@ class TestMLXFormat:from mlx_vlm.prompt_utils import apply_chat_templatefrom mlx_vlm.utils import load_config- from auto_round import AutoRoundMLLM+ from auto_round import AutoRoundsave_dir = str(tmp_path / "qwen3_vl_9b_mlx_run")try:- ar = AutoRoundMLLM(+ ar = AutoRound(qwen_3_vl_9b_name_or_path,scheme="W4A16",bits=4,