PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
# Copyright (c) 2025 Intel Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
import functools
import inspect
from typing import TYPE_CHECKING, Any, Optional, Union
import torch
from auto_round.logger import deprecated, logger
from auto_round.recipes import Recipe, get_recipe, resolve_recipe
from auto_round.recipes.recipe import UNSET
from auto_round.recipes.resolver import merge_layer_defaults
from auto_round.scheme_entry import (
collect_config_scheme_overrides,
eager_validate_scheme,
is_gguf_k_target,
is_weight_scheme,
preview_resolved_attrs,
resolve_entry_scheme,
)
from auto_round.schemes import QuantizationScheme
from auto_round.utils.dataset_utils import CalibDataset, normalize_dataset_spec
from auto_round.utils.device_manager import normalize_default_device_map
if TYPE_CHECKING:
from auto_round.algorithms.quantization.config import QuantizationConfig
from auto_round.algorithms.quantization.rtn.config import RTNConfig
from auto_round.auto_scheme.gen_auto_scheme import AutoScheme
from auto_round.compressors.base import BaseCompressor
# ---------------------------------------------------------------------------
# Compressor-class registry
# ---------------------------------------------------------------------------
# Maps (model_type, base_class_name) → combined class, created lazily.
_COMPRESSOR_REGISTRY: dict[tuple[str, str], type] = {}
def _get_compressor_class(model_type: str, base_cls: type) -> type:
"""Return the compressor class for *base_cls* wired with the right model-type Mixin.
For ``model_type == "llm"`` the bare *base_cls* is returned unchanged.
For ``"mllm"`` and ``"diffusion"`` the corresponding Mixin is prepended via
:func:`type` and the result is cached in ``_COMPRESSOR_REGISTRY`` so that
each ``(model_type, base_cls)`` pair is created at most once per process.
"""
if model_type == "llm":
return base_cls
key = (model_type, base_cls.__name__)
if key in _COMPRESSOR_REGISTRY:
return _COMPRESSOR_REGISTRY[key]
if model_type == "mllm":
from auto_round.compressors.mllm_mixin import MLLMMixin
mixin = MLLMMixin
elif model_type == "diffusion":
from auto_round.compressors.diffusion_mixin import DiffusionMixin
mixin = DiffusionMixin
else:
return base_cls
combined = type(f"{model_type.capitalize()}{base_cls.__name__}", (mixin, base_cls), {})
_COMPRESSOR_REGISTRY[key] = combined
return combined
def _resolve_quant_config_for_routing(alg_configs) -> tuple[list, list, QuantizationConfig]:
from auto_round.algorithms.config_resolver import split_quantization_configs
from auto_round.algorithms.quantization.config import QuantizationConfig
from auto_round.algorithms.quantization.rtn.config import RTNConfig
preprocessor_configs, block_quant_configs = split_quantization_configs(alg_configs)
if len(block_quant_configs) == 0 and preprocessor_configs:
from auto_round.algorithms.quantization.rtn.config import RTNConfig as _RTNConfig
return preprocessor_configs, block_quant_configs, _RTNConfig()
if block_quant_configs:
# Stage ordering belongs to the composer. The last quantizer drives
# public scheme/format resolution, independently of algorithm identity.
return preprocessor_configs, block_quant_configs, block_quant_configs[-1]
raise ValueError(
"At least one quantization algorithm config is required. "
"Pass a block quantizer such as RTNConfig or SignRoundConfig, "
"or a quantization preprocessor such as AWQConfig."
)
def _build_model_free_compressor(
model,
scheme,
layer_config,
tokenizer,
device_map,
*,
announced_via_flag: bool,
**model_free_kwargs,
):
"""Construct a ``ModelFreeCompressor`` for the model-free auto-route.
Shared by the unified entry and the model-free route so the string-model
guard, the auto-routing info log, and the constructor call live in one place.
``announced_via_flag`` is truthy when the caller explicitly passed
``model_free=True`` (so the informational auto-routing message is suppressed).
"""
from auto_round.compressors.model_free import ModelFreeCompressor
if not isinstance(model, str):
raise ValueError("model_free=True requires `model` to be a HuggingFace ID or local path string.")
if not announced_via_flag:
logger.info(
"Auto-routing to model-free quantization "
"(iters=0, disable_opt_rtn=True, supported scheme). "
"Pass disable_model_free=True to use the regular flow."
)
return ModelFreeCompressor(
model_name_or_path=model,
scheme=scheme,
layer_config=layer_config,
tokenizer=tokenizer,
device_map=device_map,
**model_free_kwargs,
)
def _build_model_type_ctor_kwargs(model, base_kwargs, mllm_kwargs, diffusion_kwargs) -> tuple[str, dict[str, Any]]:
from auto_round.utils.model import detect_model_type
model_type = detect_model_type(model)
has_multimodal_assets = mllm_kwargs.get("processor") is not None or mllm_kwargs.get("image_processor") is not None
if has_multimodal_assets and model_type != "mllm":
model_type = "mllm"
ctor_kwargs = dict(base_kwargs)
if model_type == "mllm":
ctor_kwargs.update(mllm_kwargs)
if model_type == "diffusion":
ctor_kwargs.update(diffusion_kwargs)
return model_type, ctor_kwargs
def _select_rtn_compressor_base_cls(quant_config: RTNConfig, scheme, format, base_kwargs) -> type:
from auto_round.algorithms.quantization.rtn.config import OptimizedRTNConfig, RTNConfig
from auto_round.auto_scheme.gen_auto_scheme import AutoScheme
from auto_round.compressors.orchestrator import CompressionOrchestrator as Compressor
from auto_round.compressors.utils import check_need_act_calibration
enable_imatrix = False
disable_opt_rtn = getattr(quant_config, "disable_opt_rtn", False)
# Single resolved-scheme source for routing (SchemeMixin does the authoritative
# resolution later; this preview only chooses the class). Computed once: neither
# `quant_config`'s scheme fields nor `scheme` itself change within this function,
# so the result is invariant across every use below — no need to recompute it.
resolved_attrs = preview_resolved_attrs(quant_config, scheme, format=format)
# Auto-disable rtn optimization for W8A16/W8A8-equivalent resolved schemes,
# unless the user already set disable_opt_rtn explicitly.
if getattr(quant_config, "orig_disable_opt_rtn", None) is None:
bits = resolved_attrs.get("bits")
act_bits = resolved_attrs.get("act_bits")
data_type = resolved_attrs.get("data_type")
if bits is not None and bits >= 8 and act_bits is not None and act_bits >= 8 and data_type == "int":
logger.warning("`disable_opt_rtn` is turned on for W8A16/W8A8 quantization to improve efficiency.")
disable_opt_rtn = True
quant_config.disable_opt_rtn = True
if not disable_opt_rtn:
has_gguf_k = is_gguf_k_target(format) or is_gguf_k_target(scheme)
if has_gguf_k:
enable_imatrix = True
else:
sym = resolved_attrs.get("sym")
data_type = resolved_attrs.get("data_type") or ""
bits = resolved_attrs.get("bits")
if sym is not None and sym is False:
enable_imatrix = False
# enable_neuqi: the joint (scale, zero-point) search
# consumes the activation imatrix as a per-element weighting,
# so it must be collected even on the asymmetric path where
# the plain min/max initialization ignores it
if getattr(quant_config, "enable_neuqi", False):
enable_imatrix = True
elif (
(data_type == "int" and (bits is None or bits < 8))
or (data_type in ("mx_fp", "mx_int") and bits is not None and bits <= 8)
or (data_type == "nv_fp" and bits == 4)
or is_weight_scheme(scheme)
):
enable_imatrix = True
act_bits = resolved_attrs.get("act_bits")
act_data_type = resolved_attrs.get("act_data_type")
act_dynamic = resolved_attrs.get("act_dynamic")
is_act_quantize = act_bits is not None and act_bits <= 8
needs_act_calib = is_act_quantize and check_need_act_calibration(
act_dynamic,
act_data_type,
act_bits if act_bits is not None else 16,
static_kv_dtype=base_kwargs.get("static_kv_dtype"),
static_attention_dtype=base_kwargs.get("static_attention_dtype"),
)
# AutoScheme always requires calibration data for delta-loss based scheme
# selection, regardless of whether imatrix is needed.
quant_config.enable_imatrix = enable_imatrix
needs_optimized_rtn = enable_imatrix or needs_act_calib or isinstance(scheme, AutoScheme)
if needs_optimized_rtn:
if not isinstance(quant_config, OptimizedRTNConfig):
quant_config.__class__ = OptimizedRTNConfig
else:
# Pure zero-shot RTN: downgrade to basic RTNConfig
if isinstance(quant_config, OptimizedRTNConfig):
quant_config.__class__ = RTNConfig
# Always use Compressor — it internally detects whether calibration
# data is needed and falls back to the zero-shot (RTN) path when it is not.
return Compressor
_ENTRY_KWARG_OWNERS = {
"model_free": "route",
"disable_model_free": "route",
"scale_dtype": "compressor",
"ignore_layers": "compressor",
"quant_lm_head": "compressor",
"to_quant_block_names": "compressor",
"format": "base",
"dataset": "base",
"batch_size": "base",
"model_dtype": "base",
"trust_remote_code": "base",
"amp": "base",
"enable_deterministic_algorithms": "base",
"static_kv_dtype": "base",
"static_attention_dtype": "base",
"static_kv_granularity": "base",
"static_attention_granularity": "base",
"processor": "mllm",
"image_processor": "mllm",
"template": "mllm",
"extra_data_dir": "mllm",
"quant_nontext_module": "mllm",
"guidance_scale": "diffusion",
"num_inference_steps": "diffusion",
"calib_num_inference_steps": "diffusion",
"generator_seed": "diffusion",
"pipeline_call_kwargs": "diffusion",
"diffusion_tuning_cache_size": "diffusion",
}
_SCHEME_FIELDS = set(QuantizationScheme.get_attributes())
def _iter_registered_alg_configs() -> list[tuple[str, type]]:
"""Return each registered algorithm's canonical name and config class."""
from auto_round.algorithms.registry import iter_algorithm_entries
result = []
seen = set()
for entry in iter_algorithm_entries():
factory = entry.config_factory
if factory is None:
continue
config_cls = factory if isinstance(factory, type) else type(factory())
if config_cls not in seen:
seen.add(config_cls)
result.append((entry.name, config_cls))
return result
@functools.cache
def _discover_alg_config_fields(config_cls: type) -> frozenset:
"""Discover accepted config fields without maintaining per-algorithm lists."""
from pydantic import BaseModel
if issubclass(config_cls, BaseModel):
return frozenset(config_cls.model_fields.keys())
from auto_round.algorithms.config import AlgorithmConfig
from auto_round.algorithms.quantization.config import QuantizationConfig
fields = set()
for klass in config_cls.__mro__:
if klass in (QuantizationConfig, AlgorithmConfig, object):
break
init = klass.__dict__.get("__init__")
if init is None:
continue
for name, parameter in inspect.signature(init).parameters.items():
if name != "self" and parameter.kind not in (
inspect.Parameter.VAR_POSITIONAL,
inspect.Parameter.VAR_KEYWORD,
):
fields.add(name)
return frozenset(fields)
def _filter_supported_entry_kwargs(kwargs, *, context="AutoRound", strict=False):
supported = {key: value for key, value in kwargs.items() if key in _ENTRY_KWARG_OWNERS}
unknown = sorted(set(kwargs) - set(_ENTRY_KWARG_OWNERS))
if unknown:
if strict:
raise TypeError(f"{context} received unsupported runtime kwargs with model_free=True: {', '.join(unknown)}")
logger.warning_once(
"%s received unsupported runtime kwargs %s. They will be ignored.",
context,
", ".join(unknown),
)
return supported
def _split_entry_kwargs(kwargs, *, context="AutoRound", strict=False):
buckets = {"route": {}, "compressor": {}, "base": {}, "mllm": {}, "diffusion": {}}
for key, value in _filter_supported_entry_kwargs(kwargs, context=context, strict=strict).items():
buckets[_ENTRY_KWARG_OWNERS[key]][key] = value
return buckets
def _config_fields(config):
return set(_SCHEME_FIELDS) | set(_discover_alg_config_fields(type(config)))
def _owning_algorithm_names(field_name: str) -> list[str]:
return [cls.__name__ for _, cls in _iter_registered_alg_configs() if field_name in _discover_alg_config_fields(cls)]
def _normalize_alg_configs(alg_configs, direct_kwargs=None, *, strict=False):
from auto_round.algorithms.config_resolver import split_quantization_configs
from auto_round.algorithms.quantization.config import QuantizationConfig
from auto_round.algorithms.quantization.rtn.config import (
RTN_VARIANT_DISABLE_OPT_RTN,
RTNConfig,
validate_rtn_variant_names,
)
from auto_round.algorithms.registry import normalize_algorithm_config, resolve_alg_config, resolve_algorithm_names
from auto_round.algorithms.transforms import normalize_rotation_config
from auto_round.algorithms.transforms.awq.config import (
awq_disable_opt_rtn,
rtn_inherited_opt_kwargs,
sync_rtn_opt_rtn_from_awq,
)
from auto_round.algorithms.transforms.base import BaseRotationConfig
direct_kwargs = dict(direct_kwargs or {})
legacy_algorithm = direct_kwargs.pop("algorithm", None)
if legacy_algorithm is not None:
if alg_configs is not None:
raise ValueError("`algorithm` and `alg_configs` cannot be used together.")
alg_configs = legacy_algorithm
logger.warning_once("`algorithm` is deprecated; use `alg_configs` instead.")
if "backend" in direct_kwargs:
raise ValueError(
"Rotation backend selection must be nested in `rotation_config`; "
"do not pass it as AutoRound(..., backend=...)."
)
rotation_config = direct_kwargs.pop("rotation_config", None)
config_kwargs = {key: value for key, value in direct_kwargs.items() if key not in _ENTRY_KWARG_OWNERS}
if alg_configs is None:
# Preserve the legacy entry semantics: zero iterations are RTN, while
# positive iterations use SignRound. RTN-only kwargs also select RTN
# so they are not silently ignored by the default SignRound config.
if direct_kwargs.get("iters") == 0:
raw_configs = ["rtn"]
else:
raw_configs = ["signround"]
elif isinstance(alg_configs, str):
raw_configs = resolve_algorithm_names(alg_configs)
if not raw_configs:
raise ValueError("`algorithm`/`alg_configs` must contain at least one algorithm name.")
elif isinstance(alg_configs, (list, tuple)):
raw_configs = []
seen_names = set()
for raw_config in alg_configs:
if not isinstance(raw_config, str):
raw_configs.append(raw_config)
continue
for canonical_name in resolve_algorithm_names(raw_config):
if canonical_name not in seen_names:
raw_configs.append(canonical_name)
seen_names.add(canonical_name)
else:
raw_configs = [alg_configs]
validate_rtn_variant_names(raw_configs)
# An `rtn` / `opt-rtn` the caller actually named selects the RTN variant, the
# same way `--algorithm rtn` / `--algorithm opt-rtn` does on the CLI. When
# `alg_configs` is None the RTN config below is an implicit fallback, so it
# must keep the existing flag/heuristic behaviour.
named_rtn_variants = (
[name for name in raw_configs if isinstance(name, str) and name in RTN_VARIANT_DISABLE_OPT_RTN]
if alg_configs is not None
else []
)
configs = []
pending_rtn_indices = []
for raw_config in raw_configs:
if isinstance(raw_config, str) and raw_config == "rtn":
pending_rtn_indices.append(len(configs))
configs.append(None)
continue
else:
config = resolve_alg_config(raw_config) if isinstance(raw_config, str) else raw_config
if not isinstance(config, (QuantizationConfig, BaseRotationConfig)):
raise TypeError(
f"alg_configs entries must be algorithm or QuantizationConfig instances, "
f"got {type(config).__name__}."
)
configs.append(normalize_algorithm_config(config))
awq_opt_rtn_policy = awq_disable_opt_rtn(configs)
for index in pending_rtn_indices:
config = RTNConfig(**rtn_inherited_opt_kwargs(config_kwargs, awq_opt_rtn_policy))
configs[index] = normalize_algorithm_config(config)
# ``iters=0`` has always selected RTN in the public entry and CLI. Apply
# that rule after every input form has become a config so aliases, config
# objects, and the deprecated ``algorithm`` argument cannot choose
# different quantizer implementations.
from auto_round.algorithms.quantization.sign_round.config import SignRoundConfig
direct_iters = config_kwargs.get("iters")
for index, config in enumerate(configs):
effective_iters = direct_iters if direct_iters is not None else getattr(config, "iters", None)
if isinstance(config, SignRoundConfig) and effective_iters == 0:
rtn_kwargs = rtn_inherited_opt_kwargs(config_kwargs, awq_disable_opt_rtn(configs))
rtn_config = RTNConfig(scheme=config.scheme.copy(), **rtn_kwargs)
rtn_config._user_set_scheme_fields = set(getattr(config, "_user_set_scheme_fields", set()))
configs[index] = normalize_algorithm_config(rtn_config)
if rotation_config is not None:
normalized_rotation = normalize_rotation_config(rotation_config)
if normalized_rotation is not None:
configs.append(normalized_rotation)
if not any(isinstance(config, QuantizationConfig) for config in configs):
raise TypeError(
"alg_configs entries must be algorithm aliases or QuantizationConfig instances, "
"and must include at least one quantization algorithm config."
)
preprocessors, block_configs = split_quantization_configs(configs)
if preprocessors and not block_configs:
fallback_rtn_kwargs = rtn_inherited_opt_kwargs(config_kwargs, awq_disable_opt_rtn(configs))
configs.append(normalize_algorithm_config(RTNConfig(**fallback_rtn_kwargs)))
_, block_configs = split_quantization_configs(configs)
for key, value in config_kwargs.items():
if value is None:
continue
if key in ("disable_opt_rtn", "enable_opt_rtn"):
if key == "enable_opt_rtn" and not value:
continue
opt_rtn_value = False if key == "enable_opt_rtn" else value
targets = [config for config in configs if "disable_opt_rtn" in _config_fields(config)]
if not targets:
logger.warning_once(
"RTN-specific parameter '%s' was provided, but RTN/AWQ is not enabled by alg_configs. "
"The parameter is ignored.",
key,
)
continue
for target in targets:
target.disable_opt_rtn = opt_rtn_value
if hasattr(target, "orig_disable_opt_rtn"):
target.orig_disable_opt_rtn = opt_rtn_value
logger.warning(
"Passing '%s' directly to AutoRound is supported, but the recommended usage is "
"'alg_configs=\"awq\"' or 'alg_configs=\"rtn\"'.",
key,
)
continue
if key in _SCHEME_FIELDS:
targets = block_configs
else:
targets = [config for config in configs if key in _config_fields(config)]
if not targets:
# ``iters`` is a legacy route selector. RTN intentionally has no
# iterative parameter, so ``iters=0`` must not be reported as an
# ignored algorithm-specific error after selecting RTN.
if key == "iters" and any(isinstance(config, RTNConfig) for config in configs):
continue
if strict:
raise TypeError(f"Unsupported parameter '{key}' for the selected algorithm with model_free=True.")
owners = _owning_algorithm_names(key)
owner = "/".join(owners) if owners else "the selected algorithm"
logger.error(
"%s-specific parameter '%s' was provided, but %s is not enabled by alg_configs. "
"The parameter is ignored.",
owner,
key,
owner,
)
continue
if len(targets) > 1:
logger.error(
"Parameter '%s' matches multiple algorithm configs. Pass it through the matching "
"config object in 'alg_configs'; the direct value is ignored.",
key,
)
continue
target = targets[0]
setattr(target, key, value)
recommended_config_name = type(target).__name__.replace("Config", "")
logger.warning(
"Passing '%s' directly to AutoRound is supported, but the recommended usage is "
"'alg_configs=%sConfig(...)'.",
key,
recommended_config_name,
)
sync_rtn_opt_rtn_from_awq(configs)
if named_rtn_variants:
# Applied last, on RTN configs only (AWQ keeps its own policy), so the
# algorithm name is not overridden by the AWQ inheritance above or by a
# contradicting `disable_opt_rtn` argument. `orig_disable_opt_rtn` marks
# it as user policy, which is what stops the W8A16/W8A8 efficiency
# heuristic from silently switching the variant back.
forced_disable_opt_rtn = RTN_VARIANT_DISABLE_OPT_RTN[named_rtn_variants[0]]
for config in configs:
if isinstance(config, RTNConfig):
config.disable_opt_rtn = forced_disable_opt_rtn
if hasattr(config, "orig_disable_opt_rtn"):
config.orig_disable_opt_rtn = forced_disable_opt_rtn
return [normalize_algorithm_config(config) for config in configs]
def _prepare_entry_kwargs(alg_configs, direct_kwargs, *, strict=False):
configs = _normalize_alg_configs(alg_configs, direct_kwargs, strict=strict)
runtime_kwargs = {key: value for key, value in direct_kwargs.items() if key in _ENTRY_KWARG_OWNERS}
return configs, runtime_kwargs
class _CompressorBuilder:
"""Algorithm-config-driven entry point (``scheme`` + ``alg_configs``).
This is the internal pipeline entry: it resolves the algorithm config(s),
routes to the concrete :class:`BaseCompressor` subclass (ZeroShot / DataDriven
/ ModelFree / …) wired with the right model-type Mixin, and returns that
compressor instance. It is distinct from the public dispatcher
:class:`auto_round.AutoRound` (in ``auto_round/autoround.py``).
"""
@classmethod
def _resolve_config(cls, config: str | object | list) -> object | list[object]:
"""Convert string alias(es) to the corresponding config instance(s) with default parameters."""
from auto_round.algorithms.registry import resolve_alg_config
if isinstance(config, str):
return resolve_alg_config(config)
if isinstance(config, list):
return [cls._resolve_config(c) for c in config]
return config
def __new__(
cls,
model: torch.nn.Module | str,
scheme="W4A16",
alg_configs: str | object | list[str | object] = None,
tokenizer=None,
platform="hf",
format=None,
dataset: str | CalibDataset | list | tuple | torch.utils.data.DataLoader | None = None,
low_gpu_mem_usage: bool = False,
calibration_data_device: str = "auto",
device_map: str | torch.device | int | dict = 0,
iters: int | None = None,
enable_torch_compile: bool = False,
seed: int = 42,
low_cpu_mem_usage: bool = True,
layer_config=None,
nsamples: int | None = None,
seqlen: int | None = None,
**kwargs,
) -> BaseCompressor:
from auto_round.algorithms.quantization.rtn.config import OptimizedRTNConfig, RTNConfig
from auto_round.algorithms.quantization.sign_round.config import SignRoundConfig
from auto_round.algorithms.registry import normalize_algorithm_config
from auto_round.compressors.orchestrator import CompressionOrchestrator as Compressor
from auto_round.compressors.utils import check_need_act_calibration
from auto_round.utils.dataset_utils import CalibDataset
from auto_round.utils.model import is_model_free_route
# Normalize CalibDataset objects to spec strings
if isinstance(dataset, CalibDataset):
dataset = dataset.to_spec_string()
elif isinstance(dataset, (list, tuple)):
dataset = normalize_dataset_spec(dataset)
if alg_configs is None:
alg_configs = "signround"
# TODO wenhuach if key in kwargs could override scheme and alg_config, we should pop and override,
# e.g. gradient_accumulate_step
device_map = normalize_default_device_map(device_map)
split_kwargs = _split_entry_kwargs(kwargs, strict=bool(kwargs.get("model_free", False)))
route_kwargs = dict(split_kwargs["route"])
compressor_kwargs = dict(split_kwargs["compressor"])
base_kwargs = dict(split_kwargs["base"])
mllm_kwargs = dict(split_kwargs["mllm"])
diffusion_kwargs = dict(split_kwargs["diffusion"])
# Resolve string alias(es) to config instance(s) before routing.
alg_configs = cls._resolve_config(alg_configs)
if isinstance(alg_configs, list):
alg_configs = [normalize_algorithm_config(cfg) for cfg in alg_configs]
else:
alg_configs = normalize_algorithm_config(alg_configs)
configs_for_routing = alg_configs if isinstance(alg_configs, list) else [alg_configs]
preprocessor_configs, quantizer_configs, quant_config = _resolve_quant_config_for_routing(configs_for_routing)
is_svdquant = any(type(config).__name__ == "SVDQuantConfig" for config in preprocessor_configs)
if is_svdquant and "svdquant_omni" not in {name.strip().lower() for name in (format or "").split(",")}:
format = "svdquant_nunchaku"
# Any preprocessor that requires calibration data (e.g. AWQ, SVDQuant
# smoothing) must run on the regular model-loaded path; model-free RTN
# cannot replay their calibration.
calibration_preprocessors = [
type(config).__name__ for config in preprocessor_configs if getattr(config, "need_calib", False)
]
has_calibration_preprocessor = bool(calibration_preprocessors)
if has_calibration_preprocessor and route_kwargs.get("model_free", False):
raise ValueError(
"model_free=True is not supported with calibration-based preprocessor algorithms "
f"({', '.join(calibration_preprocessors)}). "
"Use the regular flow so the model can be loaded for calibration."
)
# Model-free routing is now supported directly by the new entry path.
model_free_iters = 0 if isinstance(quant_config, RTNConfig) else getattr(quant_config, "iters", None)
model_free_disable_opt_rtn = getattr(quant_config, "disable_opt_rtn", None)
# Model-free eligibility also depends on base-level options such as
# static KV/attention quantization. Keep those options visible to the
# route predicate; otherwise the fast path silently drops them and
# cannot emit the required export metadata.
route_decision_kwargs = dict(base_kwargs, **route_kwargs, format=format)
if len(quantizer_configs) > 1:
if route_kwargs.get("model_free", False):
raise ValueError("model_free=True is not supported with multiple quantizer stages.")
route_decision_kwargs["disable_model_free"] = True
is_svdquant_rtn = type(quant_config) is RTNConfig and is_svdquant
if is_svdquant_rtn and not route_kwargs.get("model_free", False):
# SVDQuant must run before plain RTN on the regular blockwise path.
route_decision_kwargs["disable_model_free"] = True
if has_calibration_preprocessor:
# Calibration-based preprocessors need the model loaded for
# calibration; model-free RTN cannot apply their transforms.
route_decision_kwargs["disable_model_free"] = True
route_scheme = (
scheme
if hasattr(scheme, "options") and hasattr(scheme, "avg_bits")
else QuantizationScheme.from_dict(preview_resolved_attrs(quant_config, scheme, format=format))
)
# Eagerly validate scheme constraints that do not require model info.
# This mirrors old-arch _check_configs() called at __init__ time so that
# callers get ValueError/NotImplementedError on construction, not deferred.
# Runs before the model-free early return so both routes enforce the
# same config-level constraints (e.g. the format-scoped 8-bit asym rule).
eager_validate_scheme(quant_config, scheme, format=format)
# NOTE: the W8-asym opt-in is the AR_ALLOW_W8_ASYM environment
# variable, read directly by parse_scheme and the generation-time
# gates; nothing is threaded through the compressor here.
if is_model_free_route(
model, route_scheme, model_free_iters, model_free_disable_opt_rtn, route_decision_kwargs
):
# Direct scheme fields are consumed into ``quant_config`` during
# entry normalization. Pass the fully resolved scheme onward so
# model-free export does not silently fall back to preset defaults.
model_free_scheme = scheme if hasattr(scheme, "options") and hasattr(scheme, "avg_bits") else route_scheme
return _build_model_free_compressor(
model,
model_free_scheme,
layer_config,
tokenizer,
device_map,
announced_via_flag=bool(route_kwargs.get("model_free", False)),
dataset=dataset,
nsamples=nsamples,
seqlen=seqlen,
seed=seed,
enable_torch_compile=enable_torch_compile,
disable_opt_rtn=model_free_disable_opt_rtn,
format=format,
**compressor_kwargs,
**base_kwargs,
**mllm_kwargs,
**diffusion_kwargs,
**route_kwargs,
)
local_args = dict(
model=model,
tokenizer=tokenizer,
platform=platform,
format=format,
scheme=scheme,
dataset=dataset,
low_gpu_mem_usage=low_gpu_mem_usage,
calibration_data_device=calibration_data_device,
device_map=device_map,
iters=iters,
enable_torch_compile=enable_torch_compile,
seed=seed,
low_cpu_mem_usage=low_cpu_mem_usage,
layer_config=layer_config,
nsamples=nsamples,
seqlen=seqlen,
**compressor_kwargs,
)
model_type, ctor_kwargs = _build_model_type_ctor_kwargs(model, base_kwargs, mllm_kwargs, diffusion_kwargs)
# Preprocessor algorithms (AWQ, …) require a data-driven host so that
# the per-block preprocessor lifecycle (prepare_block_group ->
# block_forward_hooks -> pre_quantize_block -> post_quantize_block)
# actually runs; the pipeline auto-appends RTN when no block_quantizer
# is supplied. SignRound is itself data-driven and shares the same host.
if preprocessor_configs or len(quantizer_configs) > 1 or not isinstance(quant_config, RTNConfig):
return _get_compressor_class(model_type, Compressor)(alg_configs, **local_args, **ctor_kwargs)
elif isinstance(quant_config, RTNConfig):
base_cls = _select_rtn_compressor_base_cls(quant_config, scheme, format, base_kwargs)
return _get_compressor_class(model_type, base_cls)(alg_configs, **local_args, **ctor_kwargs)
class AutoRound:
"""Unified AutoRound entry point.
``alg_configs`` accepts an algorithm alias, one config object, or a
sequence of aliases and config objects. A sequence may combine a
preprocessor with one block quantizer, for example
``["auto_round", "quarot"]``. Preprocessors execute in their listed
order; the block quantizer is always the final quantization stage,
regardless of its position in the sequence. When omitted, SignRound is
selected. AWQ-only pipelines receive an RTN block quantizer by default.
Args:
model: A model name/path or an already-loaded ``torch.nn.Module``.
tokenizer: Optional tokenizer used for calibration data.
scheme: Quantization scheme such as ``"W4A16"`` or ``"MXFP4"``.
schemes: Optional candidate quantization schemes for AutoScheme
(adaptive mixed-bit selection), e.g. ``("W4A16", "W8A16")`` or
``"W4A16,W8A16"``. Providing schemes enables AutoScheme; ``bits``
then sets the average target bits. Mutually exclusive with
``scheme``.
bits: Weight quantization bit width. When ``schemes`` is provided it
is the average target bits for AutoScheme (e.g. ``4.2``).
alg_configs: Algorithm alias, config instance, or sequence of either.
Use config instances to provide algorithm-specific options, such
as ``SignRoundConfig(iters=50)`` or ``AWQConfig(apply_clip=True)``.
recipe: Recommended quantization plan, defaulting to ``"auto-round"``.
Explicit alg_configs replaces its pipeline while inheriting calibration defaults.
layer_config: Optional per-layer quantization overrides.
dataset: Calibration dataset name, samples, dataloader, or a
:class:`~auto_round.utils.dataset_utils.CalibDataset` object.
seqlen: Calibration sequence length.
nsamples: Number of calibration samples.
batch_size: Calibration batch size.
low_gpu_mem_usage: Enable lower-memory calibration at the cost of speed.
calibration_data_device: Placement of the block calibration-data pools:
``"auto"`` (default; primary-first, shard across GPUs when needed),
``"off"`` (no placement machinery), ``"cpu"`` (park pools in host
RAM), or an explicit csv such as ``"cuda:1,cuda:2"``. Forwards still
run on the compute devices; only the at-rest pool placement changes.
device_map: Device or device mapping used for quantization.
enable_torch_compile: Whether to use ``torch.compile`` where supported.
seed: Random seed used by calibration and tuning.
**kwargs: Additional compressor, model-type, evaluation, and legacy
compatibility options. Unsupported options are ignored with a
warning; algorithm-specific options should be placed in the
corresponding config object.
"""
SKIP_ARGS = ("local_args", "kwargs", "cls", "model_cls", "dynamic_compressor", "alg_configs")
def __new__(
cls,
model: torch.nn.Module | str,
tokenizer=None,
platform: str = "hf",
scheme: str | dict | QuantizationScheme | AutoScheme = "W4A16",
schemes: str | list | tuple | None = None,
bits: float | None = None,
layer_config: dict[str, str | dict | QuantizationScheme] | None = None,
dataset: str | list | tuple | torch.utils.data.DataLoader | None = UNSET,
iters: int | None = None,
seqlen: int = UNSET,
nsamples: int = UNSET,
batch_size: int = UNSET,
gradient_accumulate_steps: int | None = None,
low_gpu_mem_usage: bool = False,
calibration_data_device: str = "auto",
device_map: str | torch.device | int | dict = 0,
enable_torch_compile: bool | None = None,
seed: int = 42,
low_cpu_mem_usage: bool = True,
alg_configs=None,
algorithm: str | None = None,
recipe: str | Recipe = "auto-round",
**kwargs,
) -> BaseCompressor:
direct_kwargs = dict(kwargs)
strict_model_free = bool(direct_kwargs.get("model_free", False))
legacy_device = direct_kwargs.pop("device", None)
if legacy_device is not None:
logger.warning_once("`device` is deprecated, please use `device_map` instead")
if device_map in (None, 0):
device_map = legacy_device
if iters is not None:
direct_kwargs["iters"] = iters
if gradient_accumulate_steps is not None:
direct_kwargs["gradient_accumulate_steps"] = gradient_accumulate_steps
if algorithm is not None:
direct_kwargs["algorithm"] = algorithm
if bits is not None:
direct_kwargs["bits"] = bits
scheme, direct_kwargs = resolve_entry_scheme(scheme, schemes, direct_kwargs)
definition = get_recipe(recipe) if isinstance(recipe, str) else recipe
plan = resolve_recipe(
definition,
scheme=scheme,
model=model,
trust_remote_code=direct_kwargs.get("trust_remote_code", True),
)
if alg_configs is None and algorithm is None:
alg_configs = plan.build_configs()
calibration = plan.calibration
dataset = calibration.get("dataset") if dataset is UNSET else dataset
seqlen = calibration["seqlen"] if seqlen is UNSET else seqlen
nsamples = calibration["nsamples"] if nsamples is UNSET else nsamples
batch_size = calibration["batch_size"] if batch_size is UNSET else batch_size
if plan.layer_config:
layer_config = merge_layer_defaults(plan.layer_config, layer_config)
configs, runtime_kwargs = _prepare_entry_kwargs(alg_configs, direct_kwargs, strict=strict_model_free)
runtime_kwargs["batch_size"] = batch_size
compressor = _CompressorBuilder(
model,
scheme,
configs,
tokenizer=tokenizer,
platform=platform,
format=runtime_kwargs.pop("format", None),
dataset=dataset,
low_gpu_mem_usage=low_gpu_mem_usage,
calibration_data_device=calibration_data_device,
device_map=normalize_default_device_map(device_map),
iters=None,
enable_torch_compile=enable_torch_compile,
seed=seed,
low_cpu_mem_usage=low_cpu_mem_usage,
layer_config=layer_config,
nsamples=nsamples,
seqlen=seqlen,
**runtime_kwargs,
)
compressor.resolved_recipe = plan
return compressor
# Keep legacy entry points available for downstream integrations such as
# Neural Compressor. They delegate to the unified entry point.
@deprecated("AutoRound")
class AutoRoundLLM:
def __new__(cls, *args, **kwargs):
return AutoRound(*args, **kwargs)
@deprecated("AutoRound")
class AutoRoundAdam:
def __new__(cls, *args, **kwargs):
kwargs.setdefault("enable_adam", True)
return AutoRound(*args, **kwargs)
@deprecated("AutoRound")
class AutoRoundMLLM:
def __new__(cls, *args, **kwargs):
return AutoRound(*args, **kwargs)
@deprecated("AutoRound")
class AutoRoundDiffusion:
def __new__(cls, *args, **kwargs):
return AutoRound(*args, **kwargs)