Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

support model_free WOQ quantization (#1699)

Xin Hef0013f09b2f3
diff --git a/auto_round/__main__.py b/auto_round/__main__.py
index 0fe576c75..a7e4a9569 100644
--- a/auto_round/__main__.py
+++ b/auto_round/__main__.py
@@ -17,11 +17,13 @@ import os
import re
import sys
+import torch
+
from auto_round.auto_scheme import AutoScheme
from auto_round.compressors import BaseCompressor
from auto_round.eval.eval_cli import EvalArgumentParser, eval, eval_task_by_task
from auto_round.eval.evaluation import run_model_evaluation
-from auto_round.schemes import PRESET_SCHEMES
+from auto_round.schemes import PRESET_SCHEMES, preset_name_to_scheme
from auto_round.utils import (
clear_memory,
get_device_and_parallelism,
@@ -183,7 +185,6 @@ class BasicArgumentParser(argparse.ArgumentParser):
help="Disable trusting remote code when loading models. "
"Use for security if you don't trust the model source.",
)
-
tuning = self.add_argument_group("Tuning Arguments")
tuning.add_argument(
"--ignore_scale_zp_bits",
@@ -287,7 +288,6 @@ class BasicArgumentParser(argparse.ArgumentParser):
help="Disable optimization for RTN (Round-To-Nearest) mode when iters=0. "
"RTN is fast but less accurate; keeping optimization enabled is recommended.",
)
-
group_opt_rtn.add_argument(
"--enable_opt_rtn",
action="store_const",
@@ -295,6 +295,22 @@ class BasicArgumentParser(argparse.ArgumentParser):
dest="disable_opt_rtn",
help="Enable optimization for RTN mode when iters=0.",
)
+ group_model_free = tuning.add_mutually_exclusive_group()
+ group_model_free.add_argument(
+ "--model_free",
+ action="store_true",
+ help="Force model-free quantization mode. "
+ "Downloads and quantizes safetensors files directly using RTN, "
+ "without loading the full model into memory. "
+ "Only supports auto_round output format.",
+ )
+ group_model_free.add_argument(
+ "--disable_model_free",
+ action="store_true",
+ help="Disable the automatic model-free routing that activates when "
+ "--iters 0 --disable_opt_rtn is combined with a supported INT WOQ scheme. "
+ "Use this to force the regular AutoRound flow.",
+ )
scheme = self.add_argument_group("Scheme Arguments")
scheme.add_argument("--bits", default=None, type=int, help="Number of bits for weight quantization. ")
@@ -595,9 +611,70 @@ def tune(args):
if "marlin" in args.format and args.asym is True:
raise RuntimeError("marlin backend only supports sym quantization, please remove --asym")
- device_str, use_auto_mapping = get_device_and_parallelism(args.device_map)
+ # ======================= Model-Free Mode =======================
+ # The model-free path is now integrated into AutoRound itself. We only
+ # need to forward the relevant flags; AutoRound handles auto-routing
+ # (when iters=0 + disable_opt_rtn + supported scheme) and explicit
+ # ``--model_free``. Layer config / ignore-layers / output-dir handling
+ # for model-free still needs special treatment because the model is
+ # never loaded here.
+ explicit_model_free = bool(getattr(args, "model_free", False))
+ from auto_round.compressors.model_free import is_model_free_supported_scheme
+
+ auto_model_free = (
+ not explicit_model_free
+ and not getattr(args, "disable_model_free", False)
+ and getattr(args, "iters", None) == 0
+ and getattr(args, "disable_opt_rtn", None) is True
+ and is_model_free_supported_scheme(args.scheme, vars(args))
+ and (
+ str(getattr(args, "format", "auto_round") or "auto_round").lower().replace(" ", "").split(",")[0]
+ == "auto_round"
+ or format.startswith("auto_round")
+ )
+ )
- import torch
+ if explicit_model_free or auto_model_free:
+ scheme = args.scheme.upper()
+ if scheme not in PRESET_SCHEMES:
+ raise ValueError(f"{scheme} is not supported. Only {list(PRESET_SCHEMES.keys())} are supported")
+ if not is_model_free_supported_scheme(scheme, vars(args)) and not explicit_model_free:
+ logger.info(
+ f"Auto-routing to model-free is skipped: scheme '{scheme}' is not in "
+ f"the model-free allowlist. Falling back to the regular AutoRound flow."
+ )
+ else:
+ layer_config = {}
+ if args.layer_config:
+ layer_config = parse_layer_config_arg(args.layer_config)
+
+ model_name = args.model.rstrip("/")
+ output_dir = args.output_dir
+ if output_dir == "./tmp_autoround" and model_name.split("/")[-1].strip(".") != "":
+ s = preset_name_to_scheme(scheme)
+ suffix = f"g{s.group_size}" if s.group_size > 0 else f"a{s.act_bits}"
+ output_dir = os.path.join(args.output_dir, model_name.split("/")[-1] + f"-w{s.bits}{suffix}")
+
+ from auto_round import AutoRound
+
+ ar = AutoRound(
+ model_name,
+ scheme=scheme,
+ sym=getattr(args, "sym", None),
+ group_size=getattr(args, "group_size", None),
+ iters=0,
+ disable_opt_rtn=True,
+ model_free=True,
+ layer_config=layer_config,
+ ignore_layers=args.ignore_layers,
+ quant_lm_head=getattr(args, "quant_lm_head", False),
+ quant_nontext_module=getattr(args, "quant_nontext_module", False),
+ device_map=args.device_map,
+ )
+ ar.quantize_and_save(output_dir=output_dir, format=args.format) # pylint: disable=E1101
+ return
+
+ device_str, use_auto_mapping = get_device_and_parallelism(args.device_map)
if args.enable_torch_compile:
logger.info(
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index b1844490f..7a4271647 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -27,9 +27,10 @@ from auto_round.compressors import (
MLLMCompressor,
)
from auto_round.compressors.diffusion.hybrid import HybridCompressor, is_hybrid_diffusion_model
+from auto_round.compressors.model_free import ModelFreeCompressor
from auto_round.logger import deprecated, logger
from auto_round.schemes import QuantizationScheme
-from auto_round.utils import is_diffusion_model, is_mllm_model
+from auto_round.utils import is_diffusion_model, is_mllm_model, is_model_free_route
if TYPE_CHECKING:
from auto_round.auto_scheme.gen_auto_scheme import AutoScheme
@@ -171,6 +172,22 @@ class AutoRound:
return AutoRoundCompatible(**local_args, **kwargs)
+ # ---- Model-free fast-path detection --------------------------------
+ if is_model_free_route(model, scheme, iters, disable_opt_rtn, kwargs):
+ if not isinstance(model, str):
+ raise ValueError("model_free=True requires `model` to be a HuggingFace ID or local path string.")
+ if not bool(kwargs.get("model_free", False)):
+ logger.info(
+ "Auto-routing to model-free quantization "
+ "(iters=0, disable_opt_rtn=True, supported scheme). "
+ "Pass disable_model_free=True to use the regular flow."
+ )
+ if extra_config is not None:
+ local_args.update(extra_config.to_dict())
+ local_args["model_name_or_path"] = local_args.pop("model")
+ return ModelFreeCompressor(**local_args, **kwargs)
+ # --------------------------------------------------------------------
+
model_cls = []
has_multimodal_assets = kwargs.get("processor") is not None or kwargs.get("image_processor") is not None
diff --git a/auto_round/compressors/__init__.py b/auto_round/compressors/__init__.py
index 15ec27ebe..07c763482 100644
--- a/auto_round/compressors/__init__.py
+++ b/auto_round/compressors/__init__.py
@@ -18,6 +18,7 @@ from auto_round.compressors.base import LLMCompressor
from auto_round.compressors.mllm.compressor import MLLMCompressor
from auto_round.compressors.diffusion.compressor import DiffusionCompressor
from auto_round.compressors.diffusion.hybrid import HybridCompressor
+from auto_round.compressors.model_free import ModelFreeCompressor
from auto_round.compressors.config import (
DiffusionExtraConfig,
ExtraConfig,
diff --git a/auto_round/compressors/model_free.py b/auto_round/compressors/model_free.py
new file mode 100644
index 000000000..6a22e2f4f
--- /dev/null
+++ b/auto_round/compressors/model_free.py
@@ -0,0 +1,1630 @@
+# Copyright (c) 2025 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Model-free RTN quantization (class-based).
+
+This module performs weight-only quantization (WOQ) using RTN (Round-To-Nearest)
+**without** loading the full model into memory. It reads safetensors files
+(from a Hugging Face repo or a local directory), quantizes eligible
+``nn.Linear`` weight tensors shard-by-shard, and writes the packed result to
+the output directory.
+
+The main entry point is the :class:`ModelFreeCompressor` class.
+
+Supported schemes
+-----------------
+Model-free mode currently supports **integer weight-only** quantization
+schemes packed in the ``auto_round:auto_gptq`` format only. Specifically:
+
+* Preset names: ``W2A16``, ``W2A16G32``, ``W2A16G64``, ``W3A16``, ``W4A16``,
+ ``W8A16``.
+* Custom :class:`~auto_round.schemes.QuantizationScheme` instances with
+ ``data_type="int"``, ``bits in {2, 3, 4, 8}``, ``act_bits >= 16``, and any
+ symmetric / asymmetric configuration.
+
+Schemes that require special packing (FP8, MXFP4, NVFP4, GGUF, INT8_W8A8,
+BF16, FPW8A16, ...) are **not** supported in model-free mode and will raise
+``ValueError``. Use the standard AutoRound flow for those.
+
+Usage (CLI)
+-----------
+::
+
+ auto_round facebook/opt-125m \\
+ --model_free \\
+ --scheme W4A16 \\
+ --output_dir int4-125m
+
+Usage (API)
+-----------
+::
+
+ from auto_round import AutoRound
+
+ AutoRound(
+ model="facebook/opt-125m",
+ scheme="W4A16",
+ model_free=True,
+ ).quantize_and_save("./int4-125m")
+"""
+
+from __future__ import annotations
+
+import copy
+import json
+import os
+import re
+import shutil
+import time
+from concurrent.futures import ThreadPoolExecutor
+from dataclasses import asdict, fields
+from typing import Optional, Union
+
+import torch
+
+from auto_round import envs
+from auto_round.logger import logger
+from auto_round.schemes import PRESET_SCHEMES, QuantizationScheme, preset_name_to_scheme
+from auto_round.utils.common import compress_layer_names, to_standard_regex
+from auto_round.utils.device import clear_memory, memory_monitor
+from auto_round.utils.missing_tensors import quantize_weight_rtn, split_fused_expert_tensors
+
+# ---------------------------------------------------------------------------
+# Constants
+# ---------------------------------------------------------------------------
+
+# add "embed", "conv" in case of auto detection failure in _check_conv1d_and_embedding
+_BLOCK_NAME_TO_IGNORE = ["shared_expert_gate.", ".gate.", "embed", "conv"]
+
+# Integer WOQ preset schemes that model-free mode can produce.
+# Other presets (FP8/MX/NV/GGUF/BF16/INT8_W8A8/FPW8A16) require different
+# packing kernels not implemented by ``quantize_weight_rtn``.
+#
+# Note: ``W3A16`` (3-bit) is intentionally excluded. 3-bit packing requires
+# in_features to be padded to a multiple of pack_factor=10, which the current
+# ``quantize_weight_rtn`` implementation does not handle correctly.
+SUPPORTED_PRESET_SCHEMES: tuple[str, ...] = (
+ "W2A16",
+ "W2A16G32",
+ "W2A16G64",
+ "W4A16",
+ "W4A16_MIXED",
+ "W8A16",
+)
+
+# Allowed ``bits`` values for integer WOQ.
+# 3-bit is excluded — see note above.
+_SUPPORTED_INT_BITS: tuple[int, ...] = (2, 4, 8)
+
+# Multimodal keywords kept in full precision by default.
+_NONTEXT_KEYWORDS: tuple[str, ...] = (
+ "vision",
+ "visual",
+ "image",
+ "img",
+ "audio",
+ "speech",
+ "wav",
+ "waveform",
+)
+
+
+# ---------------------------------------------------------------------------
+# Predefined ignore-layer rules
+# ---------------------------------------------------------------------------
+
+
+def get_predefined_ignore_layers_from_config(config: dict) -> list[str]:
+ """Return layers to ignore based on the model's config.json.
+
+ Delegates to the same rules registered via
+ :func:`~auto_round.special_model_handler.register_ignore_layers` by
+ wrapping the config dict in a lightweight pseudo-model object, so there
+ is no need to duplicate ignore-layer rule registrations here.
+ """
+ import types
+
+ from auto_round.special_model_handler import _PRE_DEFINED_IGNORE_LAYERS
+
+ # Build a pseudo-model whose .config attribute exposes the config fields.
+ cfg_ns = types.SimpleNamespace(**config)
+ wrapper = types.SimpleNamespace(config=cfg_ns)
+
+ layers: list[str] = []
+ for rule in _PRE_DEFINED_IGNORE_LAYERS:
+ if all(m(wrapper) for m in rule.matchers):
+ for ignore_layer in rule.ignore_layers:
+ if isinstance(ignore_layer, str):
+ layers.append(ignore_layer)
+ else:
+ # callable (e.g. get_glm_flash_ignore_layers)
+ res = ignore_layer(wrapper)
+ if isinstance(res, str):
+ layers.append(res)
+ elif isinstance(res, list):
+ layers.extend(res)
+
+ return list(dict.fromkeys(layers))
+
+
+# ---------------------------------------------------------------------------
+# I/O helpers (model resolution, shard discovery, downloads)
+# ---------------------------------------------------------------------------
+
+
+def _is_model_cached(model_name_or_path: str) -> bool:
+ """Return True if the model is already available locally or in HF cache."""
+ if os.path.isdir(model_name_or_path):
+ return True
+ try:
+ from huggingface_hub import try_to_load_from_cache
+
+ result = try_to_load_from_cache(model_name_or_path, "config.json")
+ return isinstance(result, str)
+ except Exception:
+ return False
+
+
+def _resolve_source_dir(model_name_or_path: str) -> str:
+ """Resolve model_name_or_path to a local directory (download if needed)."""
+ if os.path.isdir(model_name_or_path):
+ return model_name_or_path
+ from huggingface_hub import snapshot_download
+
+ return snapshot_download(model_name_or_path)
+
+
+def _load_config(source_dir: str) -> dict:
+ """Load config.json from model directory."""
+ config_path = os.path.join(source_dir, "config.json")
+ if not os.path.exists(config_path):
+ raise FileNotFoundError(f"config.json not found in {source_dir}")
+ with open(config_path) as f:
+ return json.load(f)
+
+
+def _list_weight_shards(source_dir: str) -> list[str]:
+ """Return list of weight shard filenames in order.
+
+ Safetensors shards are preferred. When no safetensors files are
+ found the function falls back to PyTorch ``.bin`` shards.
+
+ Handles both standard naming (``model.safetensors``,
+ ``pytorch_model.bin``) and custom prefixes such as
+ ``diffusion_pytorch_model-XXXXX-of-XXXXX.safetensors`` by scanning
+ all ``*.safetensors.index.json`` / ``*.bin.index.json`` index files
+ in the directory when no standard index is found.
+ """
+
+ def _shards_from_index(index_path: str) -> list[str]:
+ with open(index_path) as f:
+ index = json.load(f)
+ seen: set[str] = set()
+ shards: list[str] = []
+ for shard_file in index["weight_map"].values():
+ if shard_file not in seen:
+ seen.add(shard_file)
+ shards.append(shard_file)
+ return shards
+
+ # --- safetensors: standard index ---
+ st_index = os.path.join(source_dir, "model.safetensors.index.json")
+ if os.path.exists(st_index):
+ return _shards_from_index(st_index)
+
+ # --- safetensors: custom-prefix index (e.g. diffusion_pytorch_model.safetensors.index.json) ---
+ for fname in sorted(os.listdir(source_dir)):
+ if fname.endswith(".safetensors.index.json"):
+ return _shards_from_index(os.path.join(source_dir, fname))
+
+ # --- safetensors: any single .safetensors file ---
+ st_files = sorted(f for f in os.listdir(source_dir) if f.endswith(".safetensors"))
+ if len(st_files) == 1:
+ return st_files
+
+ # --- pytorch .bin: standard index ---
+ bin_index = os.path.join(source_dir, "pytorch_model.bin.index.json")
+ if os.path.exists(bin_index):
+ return _shards_from_index(bin_index)
+
+ # --- pytorch .bin: custom-prefix index ---
+ for fname in sorted(os.listdir(source_dir)):
+ if fname.endswith(".bin.index.json"):
+ return _shards_from_index(os.path.join(source_dir, fname))
+
+ # --- pytorch .bin: single file ---
+ bin_single = os.path.join(source_dir, "pytorch_model.bin")
+ if os.path.exists(bin_single):
+ return ["pytorch_model.bin"]
+
+ # --- pytorch .bin: any single .bin file ---
+ bin_files = sorted(f for f in os.listdir(source_dir) if f.endswith(".bin"))
+ if len(bin_files) == 1:
+ return bin_files
+
+ # --- safetensors: single file ---
+ return ["model.safetensors"]
+
+
+def _is_eligible_weight(tensor_name: str, tensor: torch.Tensor) -> bool:
+ """Check if a tensor is eligible for quantization (2D Linear weight)."""
+ return tensor_name.endswith(".weight") and tensor.dim() == 2
+
+
+def _is_weight_shard(fname: str) -> bool:
+ """Return True if *fname* is a weight shard (safetensors or .bin).
+
+ Excludes index files (``*.index.json``) so that they are copied to the
+ output directory as normal metadata.
+ """
+ if fname.endswith(".index.json"):
+ return False
+ return fname.endswith(".safetensors") or fname.endswith(".bin")
+
+
+# Keep old name as an alias for backward compatibility.
+_is_safetensors_shard = _is_weight_shard
+
+
+def _download_single_shard(
+ model_name_or_path: str,
+ shard_filename: str,
+ local_dir: str,
+) -> str:
+ """Download a single safetensors shard file. Returns the local path."""
+ local_path = os.path.join(local_dir, shard_filename)
+ if os.path.exists(local_path):
+ logger.info(f"Shard '{shard_filename}' already exists at '{local_path}', skipping download.")
+ return local_path
+
+ if os.path.isdir(model_name_or_path):
+ src = os.path.join(model_name_or_path, shard_filename)
+ if os.path.exists(src):
+ shutil.copy2(src, local_path)
+ return local_path
+ raise FileNotFoundError(f"{shard_filename} not found in {model_name_or_path}")
+
+ from huggingface_hub import hf_hub_download
+
+ return hf_hub_download(
+ repo_id=model_name_or_path,
+ filename=shard_filename,
+ local_dir=local_dir,
+ )
+
+
+def _download_metadata_files(
+ model_name_or_path: str,
+ local_dir: str,
+) -> str:
+ """Download all non-safetensors files from a model repo. Returns local dir."""
+ os.makedirs(local_dir, exist_ok=True)
+
+ if os.path.isdir(model_name_or_path):
+ for fname in os.listdir(model_name_or_path):
+ if _is_weight_shard(fname):
+ continue
+ src = os.path.join(model_name_or_path, fname)
+ dst = os.path.join(local_dir, fname)
+ if os.path.isdir(src):
+ if not os.path.exists(dst):
+ shutil.copytree(src, dst)
+ elif os.path.isfile(src) and not os.path.exists(dst):
+ shutil.copy2(src, dst)
+ return local_dir
+
+ from huggingface_hub import snapshot_download
+
+ snapshot_download(
+ repo_id=model_name_or_path,
+ local_dir=local_dir,
+ ignore_patterns=["*.safetensors", "*.bin", "*.pth", "*.pt"],
+ )
+ return local_dir
+
+
+# ---------------------------------------------------------------------------
+# Pattern matcher
+# ---------------------------------------------------------------------------
+
+
+class _PatternMatcher:
+ """Precompiled pattern matcher with result caching.
+
+ Merges *ignore_patterns* and ``_BLOCK_NAME_TO_IGNORE`` into single
+ compiled regexes, precompiles ``layer_config`` regex patterns, and
+ caches all match results so that repeated lookups (common across
+ shards) are O(1) dict hits.
+ """
+
+ __slots__ = (
+ "_ignore_re",
+ "_skip_re",
+ "_layer_config",
+ "_default_scheme",
+ "_compiled_lc",
+ "_ignore_cache",
+ "_scheme_cache",
+ )
+
+ def __init__(
+ self,
+ ignore_patterns: list[str],
+ layer_config: dict[str, dict],
+ default_scheme: dict,
+ ):
+ self._default_scheme = default_scheme
+ self._layer_config = layer_config
+
+ self._ignore_re: re.Pattern | None = self._build_ignore_regex(ignore_patterns)
+
+ skip_parts = [re.escape(b) for b in _BLOCK_NAME_TO_IGNORE]
+ self._skip_re: re.Pattern | None = re.compile("|".join(skip_parts)) if skip_parts else None
+
+ # Each entry: (compiled_regex | None, plain_string | None, cfg_dict)
+ self._compiled_lc: list[tuple[re.Pattern | None, str | None, dict]] = []
+ for pattern, cfg in layer_config.items():
+ try:
+ self._compiled_lc.append((re.compile(to_standard_regex(pattern)), None, cfg))
+ except re.error:
+ self._compiled_lc.append((None, pattern, cfg))
+
+ self._ignore_cache: dict[str, bool] = {}
+ self._scheme_cache: dict[str, dict | None] = {}
+
+ @staticmethod
+ def _build_ignore_regex(patterns: list[str]) -> re.Pattern | None:
+ """Merge ignore patterns into one compiled regex.
+
+ Uses :func:`~auto_round.utils.common.to_standard_regex` so that
+ plain names are automatically wrapped with ``.*`` on both sides
+ (substring matching) and regex meta-characters in user patterns
+ are preserved — consistent with ``set_layer_config``.
+ """
+ if not patterns:
+ return None
+ parts: list[str] = []
+ for p in patterns:
+ if p.endswith("."):
+ std = to_standard_regex(p.rstrip("."))
+ std = std.removesuffix(".*")
+ parts.append(f"{std}(?:\\.|$)")
+ else:
+ parts.append(to_standard_regex(p))
+ return re.compile("|".join(parts))
+
+ def should_ignore(self, tensor_name: str) -> bool:
+ """Check user-specified ignore patterns (merged regex + cache)."""
+ cached = self._ignore_cache.get(tensor_name)
+ if cached is not None:
+ return cached
+ layer_name = tensor_name.rsplit(".", 1)[0] if "." in tensor_name else tensor_name
+ result = bool(self._ignore_re and self._ignore_re.search(layer_name))
+ self._ignore_cache[tensor_name] = result
+ return result
+
+ def should_skip(self, tensor_name: str) -> bool:
+ """Check predefined skip patterns (routing gates, embeddings, etc.)."""
+ return bool(self._skip_re and self._skip_re.search(tensor_name))
+
+ def resolve_scheme(self, tensor_name: str) -> dict | None:
+ """Resolve quantization scheme for *tensor_name* (cached).
+
+ Returns ``None`` when the layer should stay in full precision.
+ """
+ if tensor_name in self._scheme_cache:
+ return self._scheme_cache[tensor_name]
+ result = self._resolve_uncached(tensor_name)
+ self._scheme_cache[tensor_name] = result
+ return result
+
+ def _resolve_uncached(self, tensor_name: str) -> dict | None:
+ layer_name = tensor_name.rsplit(".", 1)[0] if "." in tensor_name else tensor_name
+ default = self._default_scheme
+
+ if layer_name in self._layer_config:
+ cfg = self._layer_config[layer_name]
+ if cfg.get("bits", default.get("bits", 4)) >= 16:
+ return None
+ return {**default, **cfg}
+
+ for compiled, plain, cfg in self._compiled_lc:
+ if compiled is not None:
+ if compiled.search(layer_name):
+ if cfg.get("bits", default.get("bits", 4)) >= 16:
+ return None
+ return {**default, **cfg}
+ elif plain is not None and plain in layer_name:
+ if cfg.get("bits", default.get("bits", 4)) >= 16:
+ return None
+ return {**default, **cfg}
+
+ return default
+
+
+# ---------------------------------------------------------------------------
+# Per-tensor / per-shard helpers
+# ---------------------------------------------------------------------------
+
+
+def _quantize_single_tensor(
+ tensor_name: str,
+ tensor: torch.Tensor,
+ matcher: "_PatternMatcher",
+ device: str = "cpu",
+) -> tuple[str, dict[str, torch.Tensor], str | None, str | None]:
+ """Quantize one eligible weight tensor and return packed outputs.
+
+ Returns:
+ (layer_name, output_tensors_dict, quantized_layer_or_None, ignored_layer_or_None)
+ """
+ layer_name = tensor_name.rsplit(".", 1)[0]
+
+ if not _is_eligible_weight(tensor_name, tensor):
+ return layer_name, {tensor_name: tensor}, None, None
+
+ if matcher.should_ignore(tensor_name):
+ logger.debug(f"Ignoring (user-specified): {layer_name}")
+ return layer_name, {tensor_name: tensor}, None, layer_name
+
+ if matcher.should_skip(tensor_name):
+ logger.debug(f"Skipping (predefined): {layer_name}")
+ return layer_name, {tensor_name: tensor}, None, layer_name
+
+ scheme = matcher.resolve_scheme(tensor_name)
+ if scheme is None:
+ logger.debug(f"Keeping full precision: {layer_name}")
+ return layer_name, {tensor_name: tensor}, None, layer_name
+
+ bits = scheme["bits"]
+ group_size = scheme["group_size"]
+ sym = scheme.get("sym", True)
+
+ if bits >= 16:
+ return layer_name, {tensor_name: tensor}, None, layer_name
+
+ try:
+ qweight, qzeros, scales = quantize_weight_rtn(
+ weight=tensor,
+ bits=bits,
+ group_size=group_size,
+ sym=sym,
+ device=device,
+ )
+
+ out: dict[str, torch.Tensor] = {
+ f"{layer_name}.qweight": qweight,
+ f"{layer_name}.qzeros": qzeros,
+ f"{layer_name}.scales": scales,
+ }
+
+ logger.debug(f"Quantized: {layer_name} (bits={bits}, group_size={group_size}, sym={sym})")
+ return layer_name, out, layer_name, None
+
+ except Exception as e:
+ logger.warning(f"Failed to quantize {layer_name}: {e}. Keeping original weight.")
+ return layer_name, {tensor_name: tensor}, None, layer_name
+
+
+def _dequant_fp8_tensors(
+ raw_tensors: dict[str, torch.Tensor],
+ block_size: list | None = None,
+) -> dict[str, torch.Tensor]:
+ """Dequantize FP8 / FP4-packed weight tensors and remove their scale tensors.
+
+ Handles three storage conventions:
+
+ 1. **DeepSeek-V3 FP8** — weight dtype ``float8_e4m3fn`` + ``weight_scale_inv``
+ (per-block float32 scales, NOT E8M0).
+ 2. **FP8 + UE8M0 scale** — weight dtype ``float8_e4m3fn`` + ``.scale``
+ (F8_E8M0 / ``ue8m0`` format, e.g. ``scale_fmt=ue8m0``).
+ 3. **FP4-packed I8 + UE8M0 scale** — weight dtype ``torch.int8`` where each
+ byte stores two FP4 E2M1 nibbles, paired with a ``.scale`` tensor in
+ F8_E8M0 format. Shape relationship: ``weight[rows, cols/2]`` and
+ ``scale[rows, (cols/2)*2/block_size]`` where ``block_size`` is inferred
+ from the ratio ``weight.shape[1] * 2 / scale.shape[1]``.
+
+ All cases are converted to ``bfloat16`` so downstream RTN quantization can
+ proceed normally.
+ """
+ from auto_round.utils.weight_handler import _dequant_fp8_linear_weight
+
+ E8M0_EXPONENT_BIAS = 127
+
+ def _e8m0_to_float(scale_tensor: torch.Tensor) -> torch.Tensor:
+ """Convert E8M0 (power-of-2 exponent) scale tensor to float."""
+ raw = scale_tensor.view(torch.uint8).to(torch.int16) - E8M0_EXPONENT_BIAS
+ return torch.pow(2.0, raw.to(torch.float32)).to(torch.bfloat16)
+
+ def _dequant_fp4_packed_weight(
+ weight_i8: torch.Tensor,
+ scale_e8m0: torch.Tensor,
+ ) -> torch.Tensor:
+ """Dequantize FP4-packed int8 weight with E8M0 block scales to bfloat16.
+
+ Each ``int8`` byte stores two FP4 E2M1 values: the low nibble is the
+ first element and the high nibble is the second. Block size is inferred
+ from the shape ratio ``weight.shape[1] * 2 / scale.shape[1]``.
+
+ Args:
+ weight_i8: Packed weight tensor, shape ``[rows, cols_packed]``,
+ dtype ``torch.int8``. ``cols_packed = cols / 2``.
+ scale_e8m0: Per-block scale tensor, shape ``[rows, n_blocks]``,
+ stored as raw uint8 bytes in F8_E8M0 (ue8m0) format.
+
+ Returns:
+ Dequantized weight, shape ``[rows, cols]``, dtype ``bfloat16``.
+ """
+ # FP4 E2M1 lookup table — index is the 4-bit pattern (0..15)
+ # Layout: positive range at indices 0-7, negative at 8-15.
+ FP4_LUT = torch.tensor(
+ [0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0, -0.0, -0.5, -1.0, -1.5, -2.0, -3.0, -4.0, -6.0],
+ dtype=torch.bfloat16,
+ device=weight_i8.device,
+ )
+ rows = weight_i8.shape[0]
+
+ # Re-interpret int8 storage as uint8 for bit operations.
+ weight_u8 = weight_i8.view(torch.uint8)
+ lo = (weight_u8 & 0x0F).to(torch.long) # low nibble → first element
+ hi = ((weight_u8 >> 4) & 0x0F).to(torch.long) # high nibble → second element
+ # Interleave lo/hi along the last dim: [rows, cols_packed, 2] → [rows, cols]
+ unpacked = torch.stack([lo, hi], dim=-1).reshape(rows, -1)
+ cols = unpacked.shape[1]
+
+ # Decode 4-bit patterns to bfloat16 float values via the lookup table.
+ decoded = FP4_LUT[unpacked] # [rows, cols]
+
+ # Convert E8M0 per-block scales to float32 → bfloat16.
+ scale_f = _e8m0_to_float(scale_e8m0) # [rows, n_blocks]
+ n_blocks = scale_f.shape[1]
+ block_size_inner = cols // n_blocks # inferred block size (e.g. 32)
+
+ # Multiply each block of decoded values by its corresponding scale.
+ decoded = decoded.reshape(rows, n_blocks, block_size_inner)
+ return (decoded * scale_f.unsqueeze(-1)).reshape(rows, cols)
+
+ # ------------------------------------------------------------------
+ # Collect quantized weight entries.
+ # Tuple layout: (weight_name, scale_name, is_e8m0, is_fp4_packed)
+ # is_e8m0 – scale tensor is in F8_E8M0 format (needs _e8m0_to_float)
+ # is_fp4_packed – weight is int8-packed FP4; use _dequant_fp4_packed_weight
+ # ------------------------------------------------------------------
+ quant_entries: list[tuple[str, str, bool, bool]] = []
+
+ for name, tensor in raw_tensors.items():
+ if not name.endswith(".weight"):
+ continue
+
+ is_fp4_packed = tensor.dtype == torch.int8
+ is_fp8 = tensor.dtype == torch.float8_e4m3fn
+ # Also catch other 1-byte dtypes (e.g. float8_e5m2) by element size.
+ if not is_fp4_packed and not is_fp8:
+ if tensor.element_size() != 1:
+ continue
+ is_fp8 = True # treat remaining 1-byte dtypes as FP8
+
+ # Convention 1: .weight_scale_inv (DeepSeek-V3 style, FP8 only)
+ scale_inv_name = name.replace(".weight", ".weight_scale_inv")
+ if scale_inv_name in raw_tensors and not is_fp4_packed:
+ quant_entries.append((name, scale_inv_name, False, False))
+ continue
+
+ # Convention 2: .scale in F8_E8M0 / ue8m0 format
+ scale_name = name.replace(".weight", ".scale")
+ if scale_name in raw_tensors:
+ quant_entries.append((name, scale_name, True, is_fp4_packed))
+
+ if not quant_entries:
+ return raw_tensors
+
+ fp4_count = sum(1 for e in quant_entries if e[3])
+ fp8_count = len(quant_entries) - fp4_count
+ parts: list[str] = []
+ if fp8_count:
+ parts.append(f"{fp8_count} FP8")
+ if fp4_count:
+ parts.append(f"{fp4_count} FP4-packed (int8)")
+ logger.info(f"Dequantizing {' and '.join(parts)} weight tensor(s) to bfloat16.")
+
+ for weight_name, scale_name, is_e8m0, is_fp4_packed in quant_entries:
+ weight = raw_tensors[weight_name]
+ scale = raw_tensors.pop(scale_name)
+ if is_fp4_packed:
+ # FP4 E2M1 packed in int8 with UE8M0 per-block scale.
+ raw_tensors[weight_name] = _dequant_fp4_packed_weight(weight, scale)
+ else:
+ if is_e8m0:
+ scale = _e8m0_to_float(scale)
+ raw_tensors[weight_name] = _dequant_fp8_linear_weight(weight, scale, block_size=block_size)
+
+ return raw_tensors
+
+
+def _process_shard(
+ shard_path: str,
+ default_scheme: dict = None,
+ layer_config: dict = None,
+ ignore_patterns: list[str] = None,
+ device: str = "cpu",
+ *,
+ matcher: "_PatternMatcher | None" = None,
+ fp8_block_size: list | None = None,
+) -> tuple[dict[str, torch.Tensor], list[str], list[str]]:
+ """Quantize eligible weights in a single safetensors shard.
+
+ Returns:
+ (output_tensors, quantized_layer_names, ignored_layer_names)
+ """
+ if matcher is None:
+ matcher = _PatternMatcher(
+ ignore_patterns if ignore_patterns is not None else [],
+ layer_config if layer_config is not None else {},
+ default_scheme if default_scheme is not None else {},
+ )
+
+ output_tensors: dict[str, torch.Tensor] = {}
+ quantized_layers: list[str] = []
+ ignored_layers: list[str] = []
+
+ if shard_path.endswith(".bin"):
+ # PyTorch pickle checkpoint — load with weights_only where supported.
+ try:
+ raw_tensors = torch.load(shard_path, map_location="cpu", weights_only=True)
+ except TypeError:
+ # weights_only not available in older PyTorch versions
+ raw_tensors = torch.load(shard_path, map_location="cpu") # nosec
+ # Flatten nested state-dict wrappers if present.
+ if not isinstance(raw_tensors, dict):
+ raise ValueError(f"Expected a dict from {shard_path}, got {type(raw_tensors)}")
+ else:
+ from safetensors import safe_open
+
+ with safe_open(shard_path, framework="pt", device="cpu") as f:
+ raw_tensors = {name: f.get_tensor(name) for name in f.keys()}
+
+ raw_tensors = split_fused_expert_tensors(raw_tensors)
+ raw_tensors = _dequant_fp8_tensors(raw_tensors, block_size=fp8_block_size)
+
+ for tensor_name in list(raw_tensors.keys()):
+ tensor = raw_tensors.pop(tensor_name)
+ _layer_name, out_dict, q_layer, ig_layer = _quantize_single_tensor(
+ tensor_name,
+ tensor,
+ matcher,
+ device,
+ )
+ output_tensors.update(out_dict)
+ if q_layer:
+ quantized_layers.append(q_layer)
+ if ig_layer:
+ ignored_layers.append(ig_layer)
+
+ return output_tensors, quantized_layers, ignored_layers
+
+
+# ---------------------------------------------------------------------------
+# Output writers
+# ---------------------------------------------------------------------------
+
+
+def _build_quantization_config(
+ default_scheme: dict,
+ layer_config: dict,
+ ignore_patterns: list[str],
+ quantized_layers: list[str],
+ ignored_layers: list[str],
+ block_name_to_quantize: Optional[list[str]] = None,
+) -> dict:
+ """Build a quantization_config dict compatible with auto-round format."""
+ from auto_round.version import __version__
+
+ scheme_keys = [f.name for f in fields(QuantizationScheme)]
+
+ qconfig = {
+ "quant_method": "auto-round",
+ "packing_format": "auto_round:auto_gptq",
+ "bits": default_scheme["bits"],
+ "group_size": default_scheme["group_size"],
+ "sym": default_scheme.get("sym", True),
+ "data_type": default_scheme.get("data_type", "int"),
+ "iters": 0,
+ "model_free": True,
+ "autoround_version": __version__,
+ }
+
+ if block_name_to_quantize:
+ qconfig["block_name_to_quantize"] = block_name_to_quantize
+
+ extra_config = {}
+ for layer_name, cfg in layer_config.items():
+ if cfg.get("bits", default_scheme["bits"]) >= 16:
+ extra_config[layer_name] = {k: cfg.get(k) for k in scheme_keys if cfg.get(k) is not None}
+ continue
+ differs = False
+ for key in ("bits", "group_size", "sym"):
+ if cfg.get(key) is not None and cfg[key] != default_scheme.get(key):
+ differs = True
+ break
+ if differs:
+ extra_config[layer_name] = {k: cfg.get(k) for k in scheme_keys if cfg.get(k) is not None}
+
+ unique_ignored = list(dict.fromkeys(ignored_layers))
+ for layer_name in unique_ignored:
+ if layer_name not in extra_config:
+ extra_config[layer_name] = {"bits": 16, "data_type": "float"}
+
+ quantized_layer_set = set(quantized_layers)
+ if "lm_head" in quantized_layer_set and "lm_head" not in extra_config:
+ lm_head_cfg = layer_config.get("lm_head", default_scheme)
+ extra_config["lm_head"] = {k: lm_head_cfg.get(k) for k in scheme_keys if lm_head_cfg.get(k) is not None}
+
+ if extra_config:
+ qconfig["extra_config"] = extra_config
+
+ return qconfig
+
+
+def _write_output_shard(
+ output_dir: str,
+ shard_name: str,
+ tensors: dict[str, torch.Tensor],
+ weight_map: dict[str, str],
+):
+ """Write a single output shard and update the weight_map."""
+ from safetensors.torch import save_file
+
+ shard_path = os.path.join(output_dir, shard_name)
+
+ # Detect shared-storage tensors (e.g. tie_word_embeddings: wte ↔ lm_head).
+ # safetensors refuses to serialise them as-is; clone the duplicates so each
+ # tensor occupies its own memory region. The first occurrence keeps the
+ # original storage; subsequent aliases are cloned.
+ seen_data_ptrs: set[int] = set()
+ deduped: dict[str, torch.Tensor] = {}
+ for k, v in tensors.items():
+ if not v.is_contiguous():
+ v = v.contiguous()
+ ptr = v.data_ptr()
+ if ptr in seen_data_ptrs:
+ v = v.clone()
+ else:
+ seen_data_ptrs.add(ptr)
+ deduped[k] = v
+
+ save_file(deduped, shard_path)
+ for tensor_name in tensors:
+ weight_map[tensor_name] = shard_name
+
+
+def _write_index_file(output_dir: str, weight_map: dict[str, str]):
+ """Write model.safetensors.index.json (or rename single shard)."""
+ if len(set(weight_map.values())) <= 1:
+ shard_names = list(set(weight_map.values()))
+ if shard_names and shard_names[0] != "model.safetensors":
+ src = os.path.join(output_dir, shard_names[0])
+ dst = os.path.join(output_dir, "model.safetensors")
+ if os.path.exists(src):
+ os.rename(src, dst)
+ weight_map = {k: "model.safetensors" for k in weight_map}
+ return
+
+ index = {"metadata": {"total_size": 0}, "weight_map": weight_map}
+ with open(os.path.join(output_dir, "model.safetensors.index.json"), "w") as f:
+ json.dump(index, f, indent=2)
+
+
+def _prefetch_shard(
+ model_name_or_path: str,
+ shard_name: str,
+ work_dir: str,
+ source_dir: str,
+ streaming: bool,
+) -> str | None:
+ """Return the local path of the next shard (download if needed)."""
+ try:
+ if streaming:
+ return _download_single_shard(model_name_or_path, shard_name, work_dir)
+ path = os.path.join(source_dir, shard_name)
+ return path if os.path.exists(path) else None
+ except Exception as e: # pragma: no cover
+ logger.warning(f"Prefetch failed for {shard_name}: {e}")
+ return None
+
+
+# ---------------------------------------------------------------------------
+# Scheme validation
+# ---------------------------------------------------------------------------
+
+
+def _normalize_scheme(scheme: Union[str, QuantizationScheme]) -> QuantizationScheme:
+ """Convert *scheme* to a :class:`QuantizationScheme` instance.
+
+ Raises ``ValueError`` for unknown preset names and ``TypeError`` for
+ unsupported types.
+ """
+ if isinstance(scheme, str):
+ scheme_name = scheme.upper()
+ if scheme_name not in PRESET_SCHEMES:
+ raise ValueError(f"Unknown scheme '{scheme}'. Available: {list(PRESET_SCHEMES.keys())}")
+ return preset_name_to_scheme(scheme_name)
+ if isinstance(scheme, QuantizationScheme):
+ return scheme
+ raise TypeError(f"Unsupported scheme type: {type(scheme)}")
+
+
+def _apply_scheme_overrides(
+ scheme: Union[str, QuantizationScheme],
+ scheme_overrides: Optional[dict] = None,
+) -> QuantizationScheme:
+ """Return the effective scheme after applying non-None overrides."""
+ scheme_obj = copy.deepcopy(_normalize_scheme(scheme))
+ if not scheme_overrides:
+ return scheme_obj
+
+ valid_fields = {field.name for field in fields(QuantizationScheme)}
+ for key, value in scheme_overrides.items():
+ if key in valid_fields and value is not None:
+ setattr(scheme_obj, key, value)
+ return scheme_obj
+
+
+def _validate_supported_scheme(
+ scheme_obj: QuantizationScheme,
+ scheme_input: Union[str, QuantizationScheme],
+) -> None:
+ """Raise ``ValueError`` if *scheme_obj* is not supported by model-free.
+
+ Model-free only supports integer weight-only quantization (sym/asym),
+ packed in the ``auto_round:auto_gptq`` format.
+ """
+ act_bits = scheme_obj.act_bits if scheme_obj.act_bits is not None else 16
+ if act_bits < 16:
+ raise ValueError(
+ f"Model-free mode only supports weight-only quantization (WOQ) schemes "
+ f"where act_bits >= 16, but '{scheme_input}' has act_bits={act_bits}. "
+ f"Supported preset schemes: {list(SUPPORTED_PRESET_SCHEMES)}."
+ )
+
+ data_type = (scheme_obj.data_type or "int").lower()
+ if data_type != "int":
+ raise ValueError(
+ f"Model-free mode only supports integer weight quantization "
+ f"(data_type='int'), but '{scheme_input}' has data_type='{data_type}'. "
+ f"FP8 / MXFP / NVFP / GGUF / BF16 schemes require the standard "
+ f"AutoRound flow. Supported preset schemes: "
+ f"{list(SUPPORTED_PRESET_SCHEMES)}."
+ )
+
+ bits = scheme_obj.bits
+ if bits is None or bits not in _SUPPORTED_INT_BITS:
+ raise ValueError(
+ f"Model-free mode supports bits in {_SUPPORTED_INT_BITS}, "
+ f"but '{scheme_input}' requests bits={bits}. "
+ f"Supported preset schemes: {list(SUPPORTED_PRESET_SCHEMES)}."
+ )
+
+
+def is_model_free_supported_scheme(
+ scheme: Union[str, QuantizationScheme],
+ scheme_overrides: Optional[dict] = None,
+) -> bool:
+ """Return True if *scheme* can be quantized via model-free mode.
+
+ Useful for CLI auto-routing logic. Never raises.
+ """
+ try:
+ scheme_obj = _apply_scheme_overrides(scheme, scheme_overrides)
+ _validate_supported_scheme(scheme_obj, scheme_obj)
+ return True
+ except (ValueError, TypeError):
+ return False
+
+
+# ---------------------------------------------------------------------------
+# Main class
+# ---------------------------------------------------------------------------
+
+
+class _ModelFreeCompressorCore:
+ """Class-based driver for model-free RTN quantization.
+
+ The lifecycle is:
+
+ 1. ``__init__`` — store user inputs.
+ 2. :meth:`run` — perform validation, IO, quantization and writing.
+
+ Internal helpers are split into focused methods so that the flow is
+ readable end-to-end.
+
+ Args:
+ model_name_or_path: HuggingFace model ID or local directory path.
+ output_dir: Directory to save the quantized model.
+ scheme: Quantization scheme name (e.g. ``"W4A16"``) or a
+ :class:`QuantizationScheme` instance.
+ layer_config: Per-layer quantization overrides. Keys are layer
+ names or regex patterns; values are dicts with ``bits``,
+ ``group_size``, ``sym`` etc.
+ ignore_layers: Comma-separated list of layer name patterns to keep
+ in full precision.
+ format: Output format (only ``"auto_round"`` is supported).
+ device: Device for quantization computation (``"cpu"`` or
+ ``"cuda"``).
+ quant_lm_head: If True, quantize ``lm_head`` as well. By default
+ ``lm_head`` and any layer containing ``embed`` are kept in
+ full precision.
+ quant_nontext_module: If True, quantize non-text modules
+ (vision/audio/image) as well. By default these multimodal
+ modules are kept in full precision.
+ """
+
+ SUPPORTED_FORMATS: tuple[str, ...] = ("auto_round",)
+
+ def __init__(
+ self,
+ model_name_or_path: str,
+ output_dir: str,
+ scheme: Union[str, QuantizationScheme] = "W4A16",
+ layer_config: Optional[dict] = None,
+ ignore_layers: str = "",
+ format: str = "auto_round",
+ device: str = "cpu",
+ quant_lm_head: bool = False,
+ quant_nontext_module: bool = False,
+ ):
+ # --- raw inputs ---
+ self.model_name_or_path = model_name_or_path
+ self.output_dir = output_dir
+ self.scheme_input = scheme
+ self.layer_config_input = layer_config
+ self.ignore_layers_input = ignore_layers
+ self.format = format
+ self.device = device
+ self.quant_lm_head = quant_lm_head
+ self.quant_nontext_module = quant_nontext_module
+
+ # --- derived state populated during run() ---
+ self.scheme_obj: QuantizationScheme | None = None
+ self.default_scheme: dict = {}
+ self.layer_config: dict = {}
+ self.ignore_patterns: list[str] = []
+ self.matcher: _PatternMatcher | None = None
+ self.config: dict = {}
+ self.fp8_block_size: list | None = None
+ self.is_streaming: bool = False
+ self.is_diffusion_model: bool = False
+ self.diffusion_root_dir: str = ""
+ self.work_dir: str = ""
+ self.source_dir: str = ""
+ self.shard_names: list[str] = []
+ self.all_quantized_layers: list[str] = []
+ self.all_ignored_layers: list[str] = []
+ self.output_weight_map: dict[str, str] = {}
+
+ # -------------------------------------------------------------------
+ # Validation / parsing
+ # -------------------------------------------------------------------
+
+ def _validate_format(self) -> None:
+ format_lower = self.format.lower().replace(" ", "").split(",")[0]
+ if format_lower not in self.SUPPORTED_FORMATS:
+ raise ValueError(
+ f"Model-free mode only supports {self.SUPPORTED_FORMATS} format, "
+ f"got '{self.format}'. Please use --format auto_round."
+ )
+
+ def _parse_scheme(self) -> None:
+ self.scheme_obj = _normalize_scheme(self.scheme_input)
+ _validate_supported_scheme(self.scheme_obj, self.scheme_input)
+ ds = asdict(self.scheme_obj)
+ self.default_scheme = {k: v for k, v in ds.items() if v is not None}
+
+ def _parse_layer_config(self) -> None:
+ lc = copy.deepcopy(self.layer_config_input) if self.layer_config_input else {}
+
+ # Append '.' to keys ending with a digit to avoid partial numeric matches.
+ for key in list(lc.keys()):
+ if key and key[-1].isdigit():
+ lc[key + "."] = lc.pop(key)
+
+ # Normalize values to dicts.
+ for key, val in list(lc.items()):
+ if isinstance(val, str):
+ parsed = asdict(preset_name_to_scheme(val.upper()))
+ lc[key] = {k: v for k, v in parsed.items() if v is not None}
+ elif isinstance(val, QuantizationScheme):
+ lc[key] = {k: v for k, v in asdict(val).items() if v is not None}
+ elif isinstance(val, dict):
+ # Resolve 'scheme' key inside dict values, e.g. {'scheme': 'W2A16'}
+ if "scheme" in val:
+ scheme_val = val.pop("scheme")
+ if isinstance(scheme_val, str):
+ parsed = asdict(preset_name_to_scheme(scheme_val.upper()))
+ resolved = {k: v for k, v in parsed.items() if v is not None}
+ elif isinstance(scheme_val, QuantizationScheme):
+ resolved = {k: v for k, v in asdict(scheme_val).items() if v is not None}
+ else:
+ resolved = {}
+ # Explicit keys in val override the resolved scheme values
+ resolved.update(val)
+ lc[key] = resolved
+ else:
+ raise TypeError(f"Unsupported layer_config value type for '{key}': {type(val)}")
+
+ self.layer_config = lc
+
+ def _build_ignore_patterns(self) -> None:
+ ignore_patterns: list[str] = []
+ if self.ignore_layers_input:
+ ignore_patterns = [p.strip() for p in self.ignore_layers_input.replace(" ", "").split(",") if p.strip()]
+ ignore_patterns = [p + "." if p and p[-1].isdigit() else p for p in ignore_patterns]
+
+ if not self.quant_lm_head and "lm_head" not in ignore_patterns:
+ ignore_patterns.append("lm_head")
+ ignore_patterns.append("head") # for deepseek v4
+
+ if not self.quant_nontext_module:
+ for kw in _NONTEXT_KEYWORDS:
+ if kw not in ignore_patterns:
+ ignore_patterns.append(kw)
+
+ self.ignore_patterns = ignore_patterns
+
+ # -------------------------------------------------------------------
+ # Source resolution and discovery
+ # -------------------------------------------------------------------
+
+ def _resolve_source(self) -> None:
+ os.makedirs(self.output_dir, exist_ok=True)
+ self.is_streaming = not _is_model_cached(self.model_name_or_path)
+ if self.is_streaming:
+ logger.info("Model not found locally or in cache — using streaming download mode.")
+ self.work_dir = self.output_dir
+ _download_metadata_files(self.model_name_or_path, self.work_dir)
+ transformer_work_dir = os.path.join(self.work_dir, "transformer")
+ if (
+ not os.path.exists(os.path.join(self.work_dir, "config.json"))
+ and os.path.isdir(transformer_work_dir)
+ and os.path.exists(os.path.join(transformer_work_dir, "config.json"))
+ ):
+ self.is_diffusion_model = True
+ self.diffusion_root_dir = self.work_dir
+ self.work_dir = transformer_work_dir
+ logger.info(
+ "Detected diffusion model (no root config.json, found transformer/ subfolder). "
+ "Only the transformer component will be quantized; other sub-components are skipped."
+ )
+ self.config = _load_config(self.work_dir)
+ else:
+ self.source_dir = _resolve_source_dir(self.model_name_or_path)
+ transformer_source_dir = os.path.join(self.source_dir, "transformer")
+ if (
+ not os.path.exists(os.path.join(self.source_dir, "config.json"))
+ and os.path.isdir(transformer_source_dir)
+ and os.path.exists(os.path.join(transformer_source_dir, "config.json"))
+ ):
+ self.is_diffusion_model = True
+ self.diffusion_root_dir = self.source_dir
+ self.source_dir = transformer_source_dir
+ logger.info(
+ "Detected diffusion model (no root config.json, found transformer/ subfolder). "
+ "Only the transformer component will be quantized; other sub-components are skipped."
+ )
+ self.config = _load_config(self.source_dir)
+
+ def _check_conv1d_and_embedding(self) -> None:
+ """Detect Conv1d and embedding layers and automatically add them to the ignore list."""
+ local_dir = self.work_dir if self.is_streaming else self.source_dir
+ if not local_dir or not os.path.isdir(local_dir):
+ return
+
+ try:
+ from auto_round.utils.model import find_layers_from_config
+
+ incompatible = find_layers_from_config(local_dir, class_names=["Embedding", "Conv1d", "Conv1D"])
+
+ if incompatible:
+ # Group by class for a cleaner warning message
+ incompatible_layers = []
+ for cls, layers in incompatible.items():
+ incompatible_layers.extend(layers)
+ summary = ", ".join(f"{cls}({len(layers)})" for cls, layers in sorted(incompatible.items()))
+ self.ignore_patterns.extend(incompatible_layers)
+ logger.warning(
+ f"Detected {len(incompatible)} layer(s) incompatible with model-free RTN"
+ f": {compress_layer_names(incompatible_layers)}.\n"
+ f"These layers have been automatically added to ignore_layers "
+ f"and will be kept in full precision.\n"
+ f"To override, pass --ignore_layers explicitly or disable "
+ f"model-free mode (remove --model_free)."
+ )
+
+ except Exception as exc:
+ logger.warning(
+ f"Could not check model architecture for incompatible layers: {exc}.\n"
+ f"Models with Embedding or Conv1d layers may be incorrectly quantized "
+ f"in model-free mode (non-2D weights cannot be packed by the RTN kernel).\n"
+ f"If affected, either disable model-free mode (remove --model_free) or "
+ f"add those layers to --ignore_layers."
+ )
+
+ def _apply_predefined_ignore_layers(self) -> None:
+ predefined = get_predefined_ignore_layers_from_config(self.config)
+ if predefined:
+ logger.info(f"Using predefined ignore_layers from config: " f"{compress_layer_names(predefined)}")
+ self.ignore_patterns.extend(predefined)
+
+ def _detect_fp8_source(self) -> None:
+ quant_config = self.config.get("quantization_config", {})
+ is_fp8 = (
+ quant_config.get("quant_method") == "fp8"
+ or quant_config.get("quantization_type") == "fp8"
+ or quant_config.get("fmt", "").startswith("e4m3")
+ )
+ if is_fp8:
+ self.fp8_block_size = quant_config.get("weight_block_size")
+ logger.info(
+ f"Detected FP8 source model (block_size={self.fp8_block_size}, "
+ f"scale_fmt={quant_config.get('scale_fmt', 'N/A')}). "
+ f"FP8 weights will be dequantized before quantization."
+ )
+
+ def _discover_shards(self) -> None:
+ search_dir = self.work_dir if self.is_streaming else self.source_dir
+ self.shard_names = _list_weight_shards(search_dir)
+
+ def _build_matcher(self) -> None:
+ self.matcher = _PatternMatcher(
+ self.ignore_patterns,
+ self.layer_config,
+ self.default_scheme,
+ )
+
+ @property
+ def _quant_output_dir(self) -> str:
+ """Effective output directory for quantized weight shards and config.
+
+ For diffusion models the quantized transformer component is written
+ to ``<output_dir>/transformer/``; for all other models the top-level
+ ``output_dir`` is used directly.
+ """
+ if self.is_diffusion_model:
+ return os.path.join(self.output_dir, "transformer")
+ return self.output_dir
+
+ # -------------------------------------------------------------------
+ # Shard processing pipeline
+ # -------------------------------------------------------------------
+
+ def _process_all_shards(self) -> None:
+ try:
+ from tqdm import tqdm as _tqdm
+ except ImportError:
+ _tqdm = None
+
+ prefetch_pool = ThreadPoolExecutor(max_workers=1)
+ write_pool = ThreadPoolExecutor(max_workers=1)
+ prefetch_future = None
+ write_future = None
+
+ if self.shard_names:
+ prefetch_future = prefetch_pool.submit(
+ _prefetch_shard,
+ self.model_name_or_path,
+ self.shard_names[0],
+ self.work_dir,
+ self.source_dir,
+ self.is_streaming,
+ )
+
+ shard_iter = (
+ _tqdm(
+ enumerate(self.shard_names),
+ total=len(self.shard_names),
+ desc="Processing shards",
+ unit="shard",
+ )
+ if _tqdm
+ else enumerate(self.shard_names)
+ )
+
+ for shard_idx, shard_name in shard_iter:
+ shard_path = prefetch_future.result() if prefetch_future else None
+
+ # Kick off prefetch of the next shard.
+ if shard_idx + 1 < len(self.shard_names):
+ prefetch_future = prefetch_pool.submit(
+ _prefetch_shard,
+ self.model_name_or_path,
+ self.shard_names[shard_idx + 1],
+ self.work_dir,
+ self.source_dir,
+ self.is_streaming,
+ )
+ else:
+ prefetch_future = None
+
+ if shard_path is None or not os.path.exists(shard_path):
+ logger.warning(f"Shard not found: {shard_name}, skipping")
+ continue
+
+ output_tensors, quantized, ignored = _process_shard(
+ shard_path=shard_path,
+ device=self.device,
+ matcher=self.matcher,
+ fp8_block_size=self.fp8_block_size,
+ )
+ memory_monitor.update()
+ clear_memory()
+ if len(self.shard_names) > 1:
+ logger.info(f"Memory usage: {memory_monitor.get_summary()}")
+
+ self.all_quantized_layers.extend(quantized)
+ self.all_ignored_layers.extend(ignored)
+
+ if write_future is not None:
+ write_future.result()
+
+ os.makedirs(self._quant_output_dir, exist_ok=True)
+ out_shard_name = f"model-{shard_idx + 1:05d}-of-{len(self.shard_names):05d}.safetensors"
+ write_future = write_pool.submit(
+ _write_output_shard,
+ self._quant_output_dir,
+ out_shard_name,
+ output_tensors,
+ self.output_weight_map,
+ )
+
+ if self.is_streaming:
+ try:
+ os.remove(shard_path)
+ logger.debug(f"Deleted source shard: {shard_path}")
+ except OSError as e:
+ logger.warning(f"Could not delete source shard {shard_path}: {e}")
+
+ prefetch_pool.shutdown(wait=False)
+ if write_future is not None:
+ write_future.result()
+ write_pool.shutdown(wait=True)
+
+ # -------------------------------------------------------------------
+ # Output
+ # -------------------------------------------------------------------
+
+ def _write_index(self) -> None:
+ _write_index_file(self._quant_output_dir, self.output_weight_map)
+
+ def _write_config_files(self) -> None:
+ quantization_config = _build_quantization_config(
+ default_scheme=self.default_scheme,
+ layer_config=self.layer_config,
+ ignore_patterns=self.ignore_patterns,
+ quantized_layers=self.all_quantized_layers,
+ ignored_layers=self.all_ignored_layers,
+ )
+
+ self.config["quantization_config"] = quantization_config
+ os.makedirs(self._quant_output_dir, exist_ok=True)
+ with open(os.path.join(self._quant_output_dir, "config.json"), "w") as f:
+ json.dump(self.config, f, indent=2)
+
+ with open(os.path.join(self._quant_output_dir, "quantization_config.json"), "w") as f:
+ json.dump(quantization_config, f, indent=2)
+
+ def _copy_metadata_files(self) -> None:
+ if self.is_streaming:
+ # Metadata was downloaded directly to output_dir (or output_dir/transformer/
+ # for diffusion models) — nothing to copy or clean up.
+ return
+
+ if self.is_diffusion_model:
+ # For diffusion models, copy only root-level non-weight metadata files
+ # (model_index.json, tokenizer files, etc.) to output_dir.
+ # Sub-components other than transformer (vae, scheduler, …) are skipped
+ # per the "only quantize transformer" policy; the quantized transformer
+ # component is already written to output_dir/transformer/ by the pipeline.
+ for fname in os.listdir(self.diffusion_root_dir):
+ src = os.path.join(self.diffusion_root_dir, fname)
+ dst = os.path.join(self.output_dir, fname)
+ if os.path.isdir(src):
+ continue
+ if os.path.isfile(src) and not os.path.exists(dst):
+ shutil.copy2(src, dst)
+ return
+
+ for fname in os.listdir(self.source_dir):
+ if _is_weight_shard(fname):
+ continue
+ src = os.path.join(self.source_dir, fname)
+ dst = os.path.join(self.output_dir, fname)
+ if os.path.isdir(src):
+ if not os.path.exists(dst):
+ shutil.copytree(src, dst)
+ elif os.path.isfile(src) and not os.path.exists(dst):
+ shutil.copy2(src, dst)
+
+ def _log_summary(self, total_time: float) -> None:
+ compressed_quantized = compress_layer_names(self.all_quantized_layers)
+ compressed_ignored = compress_layer_names(list(dict.fromkeys(self.all_ignored_layers)))
+ logger.info(
+ f"\nModel-free quantization complete.\n"
+ f" Output directory: {self.output_dir}\n"
+ f" Total time: {total_time:.2f} seconds\n"
+ f" Memory usage: {memory_monitor.get_summary()}\n"
+ f" Quantized layers ({len(self.all_quantized_layers)}): "
+ f"{compressed_quantized}\n"
+ f" Ignored layers ({len(set(self.all_ignored_layers))}): "
+ f"{compressed_ignored}\n"
+ )
+
+ # -------------------------------------------------------------------
+ # Public entry point
+ # -------------------------------------------------------------------
+
+ def run(self) -> str:
+ """Execute the full model-free quantization pipeline.
+
+ Returns:
+ Absolute path to the output directory.
+ """
+ # ---- preflight ----
+ self._validate_format()
+ self._parse_scheme()
+ self._parse_layer_config()
+ self._build_ignore_patterns()
+
+ # ---- source resolution ----
+ self._resolve_source()
+ self._check_conv1d_and_embedding()
+ self._apply_predefined_ignore_layers()
+ self._build_matcher()
+ self._detect_fp8_source()
+ self._discover_shards()
+
+ logger.info(
+ f"Model-free quantization: {self.model_name_or_path}\n"
+ f" Scheme: {self.scheme_obj}\n"
+ f" Output: {self.output_dir}\n"
+ f" Shards: {len(self.shard_names)}\n"
+ f" Streaming download: {self.is_streaming}\n"
+ f" Diffusion model: {self.is_diffusion_model}\n"
+ f" Quant lm_head: {self.quant_lm_head}\n"
+ f" Quant nontext module: {self.quant_nontext_module}\n"
+ f" Device: {self.device}"
+ )
+
+ start_time = time.time()
+ memory_monitor.reset()
+
+ # ---- main loop ----
+ self._process_all_shards()
+
+ # ---- write outputs ----
+ self._write_index()
+ self._write_config_files()
+ self._copy_metadata_files()
+
+ self._log_summary(time.time() - start_time)
+ return self.output_dir
+
+
+# ---------------------------------------------------------------------------
+# AutoRound-compatible compressor: ModelFreeCompressor doubles as the
+# compressor object returned by AutoRound.__new__ when model-free mode is
+# selected. It owns both the quantization pipeline (run()) AND the
+# AutoRound-facing interface (quantize_and_save()).
+# ---------------------------------------------------------------------------
+
+
+class ModelFreeCompressor(_ModelFreeCompressorCore):
+ """Model-free RTN quantizer that also acts as an AutoRound compressor.
+
+ When constructed via ``AutoRound(model_free=True, ...)`` the instance is
+ returned directly from ``AutoRound.__new__``. The caller then invokes
+ :meth:`quantize_and_save` exactly as they would on any other compressor.
+
+ When used as a pure-quantization driver (CLI / functional API) call
+ :meth:`run` instead.
+
+ Args:
+ model_name_or_path: HuggingFace model ID or local directory path.
+ In the AutoRound compressor role this is the ``model`` argument.
+ output_dir: Where to write the quantized model. May be ``None``
+ when used as a compressor (output_dir is passed to
+ :meth:`quantize_and_save` later).
+ scheme: Quantization scheme name or :class:`QuantizationScheme`.
+ layer_config: Per-layer overrides.
+ ignore_layers: Comma-separated layer name patterns to skip.
+ format: Output format (only ``"auto_round"`` is supported).
+ device: Compute device.
+ quant_lm_head: Whether to quantize ``lm_head``.
+ quant_nontext_module: Whether to quantize non-text modules.
+ **kwargs: When called from ``AutoRound.__new__`` the full AutoRound
+ kwargs are forwarded here. Unknown kwargs are silently ignored
+ so that calibration-only parameters (``nsamples``, ``iters``,
+ ``dataset``, …) do not cause errors.
+ """
+
+ def __init__(
+ self,
+ model_name_or_path: str,
+ output_dir: Optional[str] = None,
+ scheme: Union[str, QuantizationScheme] = "W4A16",
+ layer_config: Optional[dict] = None,
+ ignore_layers: str = "",
+ format: str = "auto_round",
+ device: str = "cpu",
+ quant_lm_head: bool = False,
+ quant_nontext_module: bool = False,
+ # --- AutoRound compressor-role aliases ---
+ tokenizer=None,
+ device_map=None,
+ **kwargs,
+ ):
+ import copy
+ from dataclasses import fields as dc_fields
+
+ fallback_kwargs = dict(kwargs)
+
+ # Collect per-field scheme overrides forwarded from AutoRound
+ # (e.g. bits=4, sym=False passed as individual kwargs).
+ self.user_scheme_overrides: dict = {}
+ for field in dc_fields(QuantizationScheme):
+ if field.name in kwargs:
+ val = kwargs.pop(field.name)
+ if val is not None:
+ self.user_scheme_overrides[field.name] = val
+
+ # Resolve device: AutoRound passes device_map; the core API uses device.
+ if device_map is not None:
+ from auto_round.utils import detect_device
+
+ device = detect_device(device_map)
+
+ # Initialise the core quantizer
+ super().__init__(
+ model_name_or_path=model_name_or_path,
+ output_dir=output_dir or "tmp_autoround",
+ scheme=scheme,
+ layer_config=layer_config,
+ ignore_layers=ignore_layers,
+ format=format,
+ device=device,
+ quant_lm_head=quant_lm_head,
+ quant_nontext_module=quant_nontext_module,
+ )
+
+ # Compressor-role state (mirrors BaseCompressor attributes used by
+ # AutoRound's post-processing code)
+ self._output_dir_override: Optional[str] = None # set by quantize_and_save
+ self.model = None
+ self.tokenizer = tokenizer
+ self.model_free = True
+ self.model_free_path = model_name_or_path
+ self.iters = 0
+ self.disable_opt_rtn = True
+ self.formats = None
+ self.quantized = False
+ self._fallback_compressor = None
+ # Start from the remaining user kwargs and explicitly set/override
+ # known compressor init parameters for clarity.
+ fallback_init = dict(fallback_kwargs)
+ fallback_init.update(
+ model=model_name_or_path,
+ iters=0,
+ disable_opt_rtn=True,
+ tokenizer=tokenizer,
+ scheme=copy.deepcopy(scheme),
+ layer_config=copy.deepcopy(layer_config),
+ ignore_layers=ignore_layers,
+ device_map=device_map,
+ quant_lm_head=quant_lm_head,
+ )
+
+ self._fallback_init_kwargs = fallback_init
+ if quant_nontext_module:
+ self._fallback_init_kwargs["quant_nontext_module"] = quant_nontext_module
+ # remaining kwargs intentionally consumed/ignored
+
+ def _fallback_to_base_compressor(self):
+ from auto_round.autoround import AutoRound
+
+ logger.info(
+ "Format '%s' is not supported by model-free mode; falling back to the regular AutoRound flow.",
+ format,
+ )
+ logger.info(
+ "fallbacked_init_kwargs: %s",
+ self._fallback_init_kwargs,
+ )
+ compressor = AutoRound(**self._fallback_init_kwargs, disable_model_free=True)
+ self._fallback_compressor = compressor
+
+ def _fallback_to_quantize_and_save(
+ self,
+ output_dir: str,
+ format: str,
+ inplace: bool,
+ **kwargs,
+ ):
+ self._fallback_to_base_compressor()
+ return self._fallback_compressor.quantize_and_save( # pylint: disable=E1101
+ output_dir=output_dir, format=format, inplace=inplace, **kwargs
+ )
+
+ def quantize(
+ self,
+ ):
+ """fallback to base compressor's quantize."""
+ self._fallback_to_base_compressor()
+ return self._fallback_compressor.quantize() # pylint: disable=E1101
+
+ def __getattribute__(self, name: str):
+ """Prefer attributes from the fallback compressor when available.
+
+ Once model-free flow falls back to the regular AutoRound compressor,
+ external attribute reads on this wrapper should observe the fallback
+ compressor's state first.
+ """
+ local_only_names = {
+ "_fallback_compressor",
+ "_fallback_init_kwargs",
+ "_fallback_to_base_compressor",
+ "__dict__",
+ "__class__",
+ "__getattribute__",
+ "__setattr__",
+ "__delattr__",
+ }
+
+ if name in local_only_names or name.startswith("__"):
+ return super().__getattribute__(name)
+
+ fallback = super().__getattribute__("__dict__").get("_fallback_compressor")
+ if fallback is not None:
+ if name == "compressor":
+ return fallback
+ try:
+ return getattr(fallback, name)
+ except AttributeError:
+ pass
+
+ return super().__getattribute__(name)
+
+ # ------------------------------------------------------------------
+ # AutoRound compressor interface
+ # ------------------------------------------------------------------
+
+ def quantize_and_save(
+ self,
+ output_dir: str = "tmp_autoround",
+ format: str = "auto_round",
+ inplace: bool = True,
+ **kwargs,
+ ):
+ """Quantize and save — AutoRound compressor entry point."""
+ if format not in ["auto_round", "auto_round:auto_gptq"]:
+ return self._fallback_to_quantize_and_save(output_dir=output_dir, format=format, inplace=inplace, **kwargs)
+
+ # Apply user scheme overrides before running
+ if self.user_scheme_overrides:
+ self.scheme_input = _apply_scheme_overrides(self.scheme_input, self.user_scheme_overrides)
+
+ # Temporarily point output_dir at what the caller requested
+ orig = self.output_dir
+ self.output_dir = output_dir
+ out_path = self.run()
+ self.output_dir = orig
+ self.quantized = True
+ return None, out_path
diff --git a/auto_round/compressors_new/entry.py b/auto_round/compressors_new/entry.py
index 7d519e007..e26691528 100644
--- a/auto_round/compressors_new/entry.py
+++ b/auto_round/compressors_new/entry.py
@@ -389,6 +389,29 @@ class AutoRoundCompatible:
This method translates old AutoRoundCompatible API to new AutoRound API.
"""
from auto_round.utils import is_diffusion_model, is_mllm_model
+ from auto_round.utils.model import is_model_free_route
+
+ # ---- Model-free fast-path detection --------------------------------
+ if is_model_free_route(model, scheme, iters, kwargs.get("disable_opt_rtn"), kwargs):
+ from auto_round.compressors_new.model_free import ModelFreeCompressor
+
+ if not isinstance(model, str):
+ raise ValueError("model_free=True requires `model` to be a HuggingFace ID or local path string.")
+ if not bool(kwargs.get("model_free", False)):
+ logger.info(
+ "Auto-routing to model-free quantization "
+ "(iters=0, disable_opt_rtn=True, supported scheme). "
+ "Pass disable_model_free=True to use the regular flow."
+ )
+ return ModelFreeCompressor(
+ model_name_or_path=model,
+ scheme=scheme,
+ layer_config=layer_config,
+ tokenizer=tokenizer,
+ device_map=device_map,
+ **kwargs,
+ )
+ # --------------------------------------------------------------------
common_config_kwargs, auto_round_config_kwargs = cls._pop_config_kwargs(kwargs)
diff --git a/auto_round/compressors_new/model_free.py b/auto_round/compressors_new/model_free.py
new file mode 120000
index 000000000..de605bdd6
--- /dev/null
+++ b/auto_round/compressors_new/model_free.py
@@ -0,0 +1 @@
+../compressors/model_free.py
\ No newline at end of file
diff --git a/auto_round/compressors_new/utils.py b/auto_round/compressors_new/utils.py
index fcebb45d7..864168ff3 100644
--- a/auto_round/compressors_new/utils.py
+++ b/auto_round/compressors_new/utils.py
@@ -343,6 +343,12 @@ def set_layer_config(
# "in_blocks" is an internal bookkeeping key injected by LLM-Compressor;
# silently drop it before validation.
item = {k: v for k, v in item.items() if k != "in_blocks"}
+ # Support "scheme" key inside dict: resolve the preset and merge overrides.
+ if "scheme" in item:
+ scheme_name = item.pop("scheme")
+ base = asdict(preset_name_to_scheme(scheme_name.upper()))
+ base.update(item)
+ item = base
invalid = set(item) - set(scheme_keys + ("fixed_by_user", "scale_dtype"))
if invalid:
raise ValueError(
diff --git a/auto_round/utils/common.py b/auto_round/utils/common.py
index 792f113a6..996ef8f47 100644
--- a/auto_round/utils/common.py
+++ b/auto_round/utils/common.py
@@ -985,52 +985,69 @@ def parse_layer_config_arg(s: str) -> dict:
def compress_layer_names(names: list) -> str:
- """Compress numbered layer names, e.g. layer.0, layer.1, layer.2 → layer.[0-2]."""
- import re as _re
+ """Compress numbered layer names, e.g. layer.0, layer.1, layer.2 → layer.[0-2].
+
+ Runs the compression pass repeatedly until stable so that multi-level
+ numbering is fully reduced. For example::
- # group by (prefix, suffix) where the number varies
+ layers.0.experts.[0-255].down_proj
+ layers.10.experts.[0-255].down_proj
+ → layers.[0,10].experts.[0-255].down_proj
+ """
+ import re as _re
from collections import defaultdict
- groups: dict = defaultdict(list)
- singles: list = []
- for name in names:
- m = _re.match(r"^(.*\.)?(\d+)(\..+)?$", name)
- if m:
- prefix = m.group(1) or ""
- num = int(m.group(2))
- suffix = m.group(3) or ""
- groups[(prefix, suffix)].append(num)
- else:
- singles.append(name)
- parts: list = []
- for (prefix, suffix), nums in groups.items():
- nums_sorted = sorted(set(nums))
- if len(nums_sorted) == 1:
- parts.append(f"{prefix}{nums_sorted[0]}{suffix}")
- else:
- # Build comma-separated contiguous ranges, e.g. [0,2-3,5]
- ranges = []
- start = prev = nums_sorted[0]
- for n in nums_sorted[1:]:
- if n == prev + 1:
- prev = n
- continue
- # Close the current range
+ def _compress_once(name_list: list) -> list:
+ groups: dict = defaultdict(list)
+ singles: list = []
+ for name in name_list:
+ m = _re.match(r"^(.*\.)?(\d+)(\..+)?$", name)
+ if m:
+ prefix = m.group(1) or ""
+ num = int(m.group(2))
+ suffix = m.group(3) or ""
+ groups[(prefix, suffix)].append(num)
+ else:
+ singles.append(name)
+ parts: list = []
+ for (prefix, suffix), nums in groups.items():
+ nums_sorted = sorted(set(nums))
+ if len(nums_sorted) == 1:
+ parts.append(f"{prefix}{nums_sorted[0]}{suffix}")
+ else:
+ # Build comma-separated contiguous ranges, e.g. [0,2-3,5]
+ ranges = []
+ start = prev = nums_sorted[0]
+ for n in nums_sorted[1:]:
+ if n == prev + 1:
+ prev = n
+ continue
+ # Close the current range
+ if start == prev:
+ ranges.append(str(start))
+ else:
+ ranges.append(f"{start}-{prev}")
+ start = prev = n
+ # Close the final range
if start == prev:
ranges.append(str(start))
else:
ranges.append(f"{start}-{prev}")
- start = prev = n
- # Close the final range
- if start == prev:
- ranges.append(str(start))
- else:
- ranges.append(f"{start}-{prev}")
- range_str = ",".join(ranges)
- parts.append(f"{prefix}[{range_str}]{suffix}")
- parts.extend(singles)
- parts.sort()
- return ", ".join(parts)
+ range_str = ",".join(ranges)
+ parts.append(f"{prefix}[{range_str}]{suffix}")
+ parts.extend(singles)
+ return parts
+
+ current = list(names)
+ while True:
+ compressed = _compress_once(current)
+ # Stop when no further merging occurred
+ if len(compressed) >= len(current):
+ break
+ current = compressed
+
+ current.sort()
+ return ", ".join(current)
def infer_bits_by_data_type(data_type: str):
diff --git a/auto_round/utils/missing_tensors.py b/auto_round/utils/missing_tensors.py
index b67372b34..255b7f92f 100644
--- a/auto_round/utils/missing_tensors.py
+++ b/auto_round/utils/missing_tensors.py
@@ -528,80 +528,64 @@ def quantize_weight_rtn(
weight = torch.nn.functional.pad(weight, (0, 0, 0, out_pad))
padded_out = weight.shape[0]
- # Reshape for group-wise ops: [padded_out, num_groups, group_size]
- w_grouped = weight.reshape(padded_out, num_groups, group_size)
+ # Use quantization functions from auto_round/data_type/int.py
+ from auto_round.data_type.int import quant_tensor_asym, quant_tensor_rtn_sym
+ from auto_round.data_type.utils import reshape_pad_tensor_by_group_size
q_scale_thresh = 1e-5 # match quant_tensor_sym / quant_tensor_asym threshold
if sym:
- # Full-range symmetric quantization — aligned with quant_tensor_sym
- # in auto_round/data_type/int.py:
- # maxq = 2^(bits-1) (e.g. 8 for 4-bit)
- # scale = signed_max(wmin, wmax) / maxq (can be negative)
- # q = clamp(round(w / scale), -maxq, maxq - 1)
+ # Full-range symmetric quantization via quant_tensor_rtn_sym
maxq = 1 << (bits - 1) # e.g. 8 for 4-bit
zero_point = maxq # unsigned offset for packing
- w_min = w_grouped.amin(dim=-1, keepdim=True)
- w_max = w_grouped.amax(dim=-1, keepdim=True)
- # clamp so that 0 is always representable
- w_min = torch.clamp(w_min, max=0)
- w_max = torch.clamp(w_max, min=0)
- wmin_abs = -w_min
- wmax_abs = w_max
- # signed max: preserves the sign of the dominant extreme
- max_v = (2 * (wmax_abs < wmin_abs).int() - 1) * torch.max(wmax_abs, wmin_abs)
- del w_min, w_max, wmin_abs, wmax_abs
-
- scales_full = (max_v / maxq).to(torch.float16)
- # Clamp small scales for numerical stability (match q_scale_thresh)
- scales_full = torch.where(
- scales_full < 0,
- torch.clamp(scales_full, max=-q_scale_thresh),
- torch.clamp(scales_full, min=q_scale_thresh),
- )
- scales_full = scales_full.float() # compute in fp32
- del max_v
+ # quant_tensor_rtn_sym returns (qdq_result, scale, maxq)
+ # scale shape: [padded_out * num_groups, 1]
+ _, scale, _ = quant_tensor_rtn_sym(weight, bits=bits, group_size=group_size)
+
+ # Reshape weight for group-wise quantization: [padded_out * num_groups, group_size]
+ w_grouped, _, _ = reshape_pad_tensor_by_group_size(weight, group_size)
+ w_grouped = w_grouped.to(device=device, dtype=torch.float32)
+ del weight
- # In-place ops to minimise peak GPU memory.
- q = w_grouped / scales_full
- del weight, w_grouped # free the (potentially large) float32 weight storage
- q.round_().clamp_(-maxq, maxq - 1)
+ # Compute integer values for packing
+ q = (w_grouped / scale).round_().clamp_(-maxq, maxq - 1)
+ del w_grouped
q += zero_point # shift to unsigned [0, 2*maxq - 1]
- q = q.to(torch.int32) # new int32 tensor; float32 q is freed
+ q = q.to(torch.int32)
+
+ # scale → [num_groups, padded_out] (float16)
+ scales_out = scale.squeeze(-1).reshape(padded_out, num_groups).t().contiguous().to(torch.float16)
+ del scale
zp = torch.full((num_groups, padded_out), zero_point, dtype=torch.int32, device=device)
else:
- # Asymmetric quantization — aligned with quant_tensor_asym
- # in auto_round/data_type/int.py:
- # maxq = 2^bits - 1
- # wmin = clamp(min, max=0), wmax = clamp(max, min=0)
- # scale = (wmax - wmin) / maxq
- # zp = round(-wmin / scale)
- # q = clamp(round(w / scale) + zp, 0, maxq)
+ # Asymmetric quantization via quant_tensor_asym
max_int = (1 << bits) - 1
- w_min = torch.clamp(w_grouped.amin(dim=-1, keepdim=True), max=0)
- w_max = torch.clamp(w_grouped.amax(dim=-1, keepdim=True), min=0)
-
- scales_full = ((w_max - w_min) / max_int).to(torch.float16)
- scales_full = torch.clamp(scales_full, min=q_scale_thresh)
- scales_full = scales_full.float() # compute in fp32
- zp_full = torch.round(-w_min / scales_full).clamp_(0, max_int).to(torch.int32)
- del w_min, w_max
-
- q = w_grouped / scales_full
- del weight, w_grouped
- q.round_() # round FIRST (matches regular path: round(w/scale) then + zp)
- q += zp_full.float() # then add zp
+
+ # quant_tensor_asym returns (qdq_result, scale, zp)
+ # scale shape: [padded_out * num_groups, 1], zp shape: [padded_out * num_groups, 1]
+ _, scale, zp_val = quant_tensor_asym(weight, bits=bits, group_size=group_size)
+
+ # Reshape weight for group-wise quantization
+ w_grouped, _, _ = reshape_pad_tensor_by_group_size(weight, group_size)
+ w_grouped = w_grouped.to(device=device, dtype=torch.float32)
+ del weight
+
+ # Compute integer values for packing
+ q = (w_grouped / scale).round_()
+ del w_grouped
+ q += zp_val
q.clamp_(0, max_int)
q = q.to(torch.int32)
- zp = zp_full.squeeze(-1).t().contiguous() # [num_groups, padded_out]
- del zp_full
+ # scale → [num_groups, padded_out] (float16)
+ scales_out = scale.squeeze(-1).reshape(padded_out, num_groups).t().contiguous().to(torch.float16)
+ del scale
- # scales → [num_groups, padded_out] (float16)
- scales_out = scales_full.squeeze(-1).t().contiguous().to(torch.float16)
- del scales_full
+ # zp → [num_groups, padded_out]
+ zp = zp_val.squeeze(-1).reshape(padded_out, num_groups).t().contiguous().to(torch.int32)
+ del zp_val
# q → [in_features, padded_out]
q = q.reshape(padded_out, in_features).t().contiguous()
diff --git a/auto_round/utils/model.py b/auto_round/utils/model.py
index 04a8b0395..575b7ada0 100644
--- a/auto_round/utils/model.py
+++ b/auto_round/utils/model.py
@@ -2018,3 +2018,121 @@ def hook_ngram_embeddings_on_cpu(model):
hook_input_output_device_for_cpu_module(raw_ngram_embeddings)
return has_ngram_embeddings, raw_ngram_embeddings if has_ngram_embeddings else None
+
+
+def is_model_free_route(
+ model,
+ scheme,
+ iters: int,
+ disable_opt_rtn,
+ kwargs: dict,
+) -> bool:
+ """Return True when the model-free fast-path should be taken.
+
+ Mirrors the ``is_diffusion_model`` / ``is_mllm_model`` helpers used in
+ ``AutoRound.__new__`` to select the right compressor class.
+
+ Model-free mode is activated when **either** of the following holds:
+
+ * ``model_free=True`` is explicitly set in *kwargs*.
+ * All of the following are true:
+
+ - ``disable_model_free`` is not set (or False) in *kwargs*
+ - *model* is a string (HF hub ID or local path)
+ - *iters* == 0
+ - *disable_opt_rtn* is exactly ``True``
+ - *scheme* is a supported model-free preset
+
+ Note: this function only *reads* kwargs; it does **not** pop any keys.
+ """
+ from auto_round.compressors.model_free import is_model_free_supported_scheme
+
+ explicit = bool(kwargs.pop("model_free", False))
+ disabled = bool(kwargs.pop("disable_model_free", False))
+ if explicit:
+ return True
+ # Only auto-route when format is auto_round (or not specified).
+ fmt = kwargs.get("format", "auto_round")
+ if fmt is None:
+ fmt = "auto_round"
+ fmt_first = str(fmt).lower().replace(" ", "").split(",")[0]
+ if fmt_first != "auto_round":
+ return False
+ return (
+ not disabled
+ and isinstance(model, str)
+ and iters == 0
+ and disable_opt_rtn is True
+ and is_model_free_supported_scheme(scheme, kwargs)
+ )
+
+
+def find_layers_from_config(model_dir: str, class_names: list[str] | None = None) -> dict[str, str]:
+ """Detect layers of given class names by loading the model on ``device='meta'``.
+
+ Only ``config.json`` is required — no weights are read.
+
+ For regular models the root directory is checked. For diffusion-style
+ repos (no root ``config.json`` but a ``transformer/`` subfolder), only the
+ ``transformer/`` subfolder is checked — other sub-components (``vae/``,
+ ``scheduler/``, …) are intentionally skipped because only the transformer
+ is quantized in model-free mode.
+
+ Args:
+ model_dir: Local directory containing ``config.json``, or a diffusion
+ repo root whose ``transformer/`` subfolder contains ``config.json``.
+ class_names: Class names to look for, matched against
+ ``type(module).__name__``. Defaults to
+ ``["Embedding", "Conv1d", "Conv1D"]`` — the types incompatible
+ with model-free RTN packing.
+
+ Returns:
+ ``{class_name: [layer_name, ...]}`` for every matched module.
+ Returns an empty dict on any failure.
+ """
+ from huggingface_hub import snapshot_download
+ from transformers import AutoConfig, AutoModel
+
+ if class_names is None:
+ class_names = ["Embedding", "Conv1d", "Conv1D"]
+ if isinstance(class_names, str):
+ class_names = [class_names]
+ target = set(class_names)
+
+ # download if not local, but only the config files (fast)
+ if not os.path.exists(model_dir):
+ model_dir = snapshot_download(
+ repo_id=model_dir,
+ allow_patterns=["**/config.json"],
+ )
+
+ # Build the list of (prefix, config_dir) pairs to inspect.
+ # For diffusion repos (no root config.json) only check transformer/.
+ # For regular repos only check the root directory.
+ dirs: list[tuple[str, str]] = []
+ if os.path.exists(os.path.join(model_dir, "config.json")):
+ dirs.append(("", model_dir))
+ else:
+ transformer_dir = os.path.join(model_dir, "transformer")
+ if os.path.isdir(transformer_dir) and os.path.exists(os.path.join(transformer_dir, "config.json")):
+ dirs.append(("", transformer_dir))
+
+ result: dict[str, str] = {}
+ for prefix, config_dir in dirs:
+ try:
+ with torch.device("meta"):
+ config = AutoConfig.from_pretrained(config_dir, trust_remote_code=True)
+ model = AutoModel.from_config(config, trust_remote_code=True)
+ except Exception as e:
+ logger.warning(f"Failed to load model from {config_dir} for layer detection. Skipping. Error: {e}")
+ continue # skip silently
+ for name, module in model.named_modules():
+ cls_name = type(module).__name__
+ if any(t.lower() in cls_name.lower() for t in target):
+ full_name = f"{prefix}.{name}" if prefix else name
+ if cls_name not in result:
+ result[cls_name] = [full_name]
+ else:
+ result[cls_name].append(full_name)
+ del model
+ return result
diff --git a/docs/step_by_step.md b/docs/step_by_step.md
index 9762659f6..b9ffc7749 100644
--- a/docs/step_by_step.md
+++ b/docs/step_by_step.md
@@ -27,6 +27,7 @@ This document presents step-by-step instructions for auto-round llm quantization
- [API Usage](#api-usage-1)
- [Hyperparameters in AutoScheme](#hyperparameters-in-autoscheme)
+ [OPT RTN mode](#opt-rtn-mode)
+ + [Model-Free Mode](#model-free-mode)
+ [GGUF format](#gguf-format)
+ [Quantization Costs](#quantization-costs)
+ [Device/Multi-GPU setting in Quantization](#devicemulti-gpu-setting-in-quantization)
@@ -445,6 +446,94 @@ output_dir = "./tmp_autoround"
ar.quantize_and_save(output_dir, format="auto_round")
```
+### Model-Free Mode
+
+Model-free mode performs RTN WOQ quantization **without loading the full model into memory**. It downloads safetensors files directly, quantizes each Linear weight tensor shard-by-shard, and saves the packed result. This is useful when you want fast, no-calibration quantization with minimal resource requirements.
+
+> **Auto-enabled by default.** As of v0.13, when you pass `--iters 0 --disable_opt_rtn` together with a supported INT WOQ scheme, the CLI automatically takes the model-free path. This is **bit-exactly equivalent** to the regular `--iters 0 --disable_opt_rtn` flow but uses far less memory. Use `--disable_model_free` to opt out and force the original flow.
+
+**Key features:**
+- **No model object required** – only `config.json` and safetensors files are needed
+- **Low disk memory required** (If no local model files) – downloads and quantizes one shard at a time, deleting the source shard after processing
+- **Per-layer configuration** – supports `--layer_config` for per-layer bit-width overrides and `--ignore_layers` to keep specific layers in full precision
+- **Predefined ignore layers** – automatically skips model-specific layers (e.g., MoE gates, MTP layers) based on config detection
+- **Bit-exact parity** with the standard `--iters 0 --disable_opt_rtn` flow for all supported schemes
+
+<details>
+ <summary>Click to expand supported schemes and examples</summary>
+
+**Supported schemes**
+
+Model-free mode currently supports the following **integer weight-only** preset schemes (packed in the `auto_round:auto_gptq` format):
+
+| Preset | Bits | Group size | Sym |
+| --- | --- | --- | --- |
+| `W2A16` | 2 | 128 | true |
+| `W2A16G32` | 2 | 32 | true |
+| `W2A16G64` | 2 | 64 | true |
+| `W4A16` (default) | 4 | 128 | true |
+| `W4A16_MIXED` | 4 | 128 | true |
+| `W8A16` | 8 | 128 | true |
+
+All of the above presets also support **asymmetric quantization** (`sym=False`) for 2-bit and 8-bit variants (`W2A16`, `W2A16G32`, `W2A16G64`, `W8A16`), producing `auto_round:auto_gptq`-packed output with bit-exact parity to the regular flow. For 4-bit asymmetric quantization the regular flow uses `auto_round:auto_awq` packing as suggested; use the standard AutoRound flow for that case.
+
+You can also pass a custom `QuantizationScheme(bits=N, group_size=G, sym=True/False, data_type="int", act_bits=16)` with `bits ∈ {2, 4, 8}` and any group_size / sym configuration.
+
+Schemes that require special packing kernels (`W3A16`, `FPW8A16`, `BF16`, `MXFP4`, `MXFP8`, `MXINT4`, `NVFP4`, `FP8_BLOCK`, `FP8_STATIC`, `INT8_W8A8`, `GGUF:*`, ...) are **not** supported in model-free mode and will raise `ValueError`. Use the regular AutoRound flow for those.
+
+#### CLI Usage
+
+```bash
+# Easiest: --iters 0 --disable_opt_rtn auto-routes to model-free
+auto_round meta-llama/Llama-3.2-1B-Instruct \
+ --scheme W4A16 \
+ --iters 0 --disable_opt_rtn \
+ --output_dir ./int4-llama
+
+# Equivalent explicit invocation
+auto_round meta-llama/Llama-3.2-1B-Instruct \
+ --model_free \
+ --scheme W4A16 \
+ --output_dir ./int4-llama
+
+# Opt out of auto-routing and use the regular flow instead
+auto_round meta-llama/Llama-3.2-1B-Instruct \
+ --scheme W4A16 \
+ --iters 0 --disable_opt_rtn --disable_model_free \
+ --output_dir ./int4-llama
+
+# With per-layer configuration and ignored layers
+auto_round meta-llama/Llama-3.2-1B-Instruct \
+ --model_free \
+ --scheme W4A16 \
+ --group_size 32 \
+ --asym \
+ --layer_config "{k_proj:{bits:8},v_proj:{bits:8}}" \
+ --ignore_layers "mlp" \
+ --output_dir ./int4-llama
+```
+
+#### API Usage
+
+```python
+from auto_round import AutoRound
+
+AutoRound(
+ model="meta-llama/Llama-3.2-1B-Instruct",
+ scheme="W4A16", # Or a QuantizationScheme instance for custom group_size / sym.
+ layer_config={
+ ".*k_proj": {"bits": 8, "group_size": 32},
+ ".*v_proj": {"bits": 8, "group_size": 32},
+ },
+ ignore_layers="mlp",
+ model_free=True,
+).quantize_and_save("./int4-llama")
+```
+
+> **Note:** Model-free mode only supports the `auto_round` output format and uses RTN (no calibration data, no iterative tuning). For higher-quality quantization or schemes outside the supported list, use the standard AutoRound flow.
+
+</details>
+
### GGUF format
Experimental feature. This format is well-suited for CPU devices and is widely adopted by the community.
diff --git a/docs/step_by_step_CN.md b/docs/step_by_step_CN.md
index ee0abe1ee..b990166d8 100644
--- a/docs/step_by_step_CN.md
+++ b/docs/step_by_step_CN.md
@@ -27,6 +27,7 @@
- [API 用法](#api-用法)
- [AutoScheme 中的超参数](#autoscheme-超参数说明)
+ [OPT RTN 模式](#opt-rtn-模式)
+ + [免模型量化模式](#免模型量化模式)
+ [GGUF 格式](#gguf-格式量化)
+ [量化成本](#量化成本)
+ [设备及多卡量化设置](#设备及多卡量化设置)
@@ -159,7 +160,7 @@ AutoRound 支持多种量化配置:
| **auto_round** | W4A16、W2A16、W3A16、W8A16、W2A16G64、W2A16G32、`MXFP4`、`MXFP8`、`MXFP4_RCEIL`、`MXFP8_RCEIL`、`NVFP4`、`FPW8A16`、`FP8_STATIC`、`FP8_BLOCK`、`BF16`, `MXINT4` |
| **auto_awq** | W4A16、BF16 |
| **auto_gptq** | W4A16、W2A16、W3A16、W8A16、W2A16G64、W2A16G32、BF16 |
-| **llm_compressor** | NVFP4、`MXFP4`、`MXFP8`、`FPW8A16`、`FP8_STATIC`、FP8_STATIC |
+| **llm_compressor** | NVFP4、`MXFP4`、`MXFP8`、`FPW8A16`、`FP8_STATIC`、FP8_BLOCK |
| **mlx** / **auto_round:mlx** | W2A16、W3A16、W4A16、W5A16、W6A16、W8A16、BF16、混合 bit / 混合 group_size(仅 Apple Silicon) |
| **gguf** | GGUF:Q4_K_M、GGUF:Q2_K_S、GGUF:Q3_K_S、GGUF:Q3_K_M、GGUF:Q3_K_L、GGUF:Q4_K_S、GGUF:Q5_K_S、GGUF:Q5_K_M、GGUF:Q6_K、GGUF:Q4_0、GGUF:Q4_1、GGUF:Q5_0、GGUF:Q5_1、GGUF:Q8_0 |
| **fp8** | FP8_BLOCK |
@@ -420,7 +421,7 @@ ar.quantize_and_save()
#### 局限性
AutoScheme 目前还**不支持对嵌入层(Embedding layer)进行自动量化**。该层将直接采用候选方案中精度最高的配置。
-## OPT-RTN 模式
+### OPT-RTN 模式
AutoRound 还提供优化版 RTN(Round-To-Nearest,就近舍入)模式,无需标定数据即可实现快速基线量化。**启用方式为 `iters=0`**。同时为获得更好的效果,推荐搭配 `group_size=32` 。RTN 与 OPT RTN 模式的精度对比详见[《精度对比报告》](./opt_rtn.md)。
对于 GGUF 格式,我们参考 llamacpp 的思路,优化了 RTN 算法。若需使用原始(非优化)RTN 算法,开启 `--disable_opt_rtn` 即可。
@@ -438,6 +439,94 @@ output_dir = "./tmp_autoround"
ar.quantize_and_save(output_dir, format="auto_round")
```
+### 免模型量化模式
+
+免模型量化模式(Model-Free Mode)可以**无需将完整模型加载到内存中**即可执行 RTN WOQ 量化。它直接下载 safetensors 文件,逐分片地对每个 Linear 权重张量进行量化并保存打包结果。当您需要快速、无标定数据的量化且资源有限时,该模式非常实用。
+
+> **默认自动启用。** 自 v0.13 起,当您同时传入 `--iters 0 --disable_opt_rtn` 与一个受支持的 INT WOQ scheme 时,CLI 会自动走免模型路径。该路径与原始 `--iters 0 --disable_opt_rtn` 流程**位级(bit-exact)等价**,但内存占用大幅降低。如需关闭自动路由、强制使用原始流程,可加 `--disable_model_free`。
+
+**主要特性:**
+- **无需模型对象** — 仅需 `config.json` 和 safetensors 文件
+- **低磁盘内存** (如果无本地模型) — 逐个下载并量化分片,处理完成后立即删除源分片
+- **逐层配置** — 支持 `--layer_config` 设置逐层位宽,以及 `--ignore_layers` 保持特定层全精度
+- **预定义忽略层** — 根据模型配置自动跳过特定层(如 MoE 门控层、MTP 层等)
+- 与标准 `--iters 0 --disable_opt_rtn` 流程对所有受支持的 scheme **位级等价**
+
+<details>
+ <summary>点击展开支持的 Scheme 与示例</summary>
+
+**支持的 Scheme**
+
+免模型模式当前支持以下整数权重量化预设(均使用 `auto_round:auto_gptq` 打包格式):
+
+| Preset | Bits | Group size | Sym |
+| --- | --- | --- | --- |
+| `W2A16` | 2 | 128 | true |
+| `W2A16G32` | 2 | 32 | true |
+| `W2A16G64` | 2 | 64 | true |
+| `W4A16`(默认) | 4 | 128 | true |
+| `W4A16_MIXED` | 4 | 128 | true |
+| `W8A16` | 8 | 128 | true |
+
+上述 2-bit 和 8-bit 预设(`W2A16`、`W2A16G32`、`W2A16G64`、`W8A16`)同样支持**非对称量化**(`sym=False`),输出使用 `auto_round:auto_gptq` 打包格式,并与标准流程**位级等价**。4-bit 非对称量化时标准流程建议使用 `auto_round:auto_awq` 打包格式,如需该场景请使用标准 AutoRound 流程。
+
+也可以传入自定义的 `QuantizationScheme(bits=N, group_size=G, sym=True/False, data_type="int", act_bits=16)`,其中 `bits ∈ {2, 4, 8}`,group_size / sym 可任意设置。
+
+需要特殊打包内核的 scheme(`W3A16`、`FPW8A16`、`BF16`、`MXFP4`、`MXFP8`、`MXINT4`、`NVFP4`、`FP8_BLOCK`、`FP8_STATIC`、`INT8_W8A8`、`GGUF:*` 等)**不被支持**,传入会抛 `ValueError`。这些请使用标准 AutoRound 流程。
+
+#### 命令行用法
+
+```bash
+# 最简单:--iters 0 --disable_opt_rtn 自动路由到免模型
+auto_round meta-llama/Llama-3.2-1B-Instruct \
+ --scheme W4A16 \
+ --iters 0 --disable_opt_rtn \
+ --output_dir ./int4-llama
+
+# 等价的显式调用
+auto_round meta-llama/Llama-3.2-1B-Instruct \
+ --model_free \
+ --scheme W4A16 \
+ --output_dir ./int4-llama
+
+# 关闭自动路由,强制使用原始流程
+auto_round meta-llama/Llama-3.2-1B-Instruct \
+ --scheme W4A16 \
+ --iters 0 --disable_opt_rtn --disable_model_free \
+ --output_dir ./int4-llama
+
+# 搭配逐层配置和忽略层
+auto_round meta-llama/Llama-3.2-1B-Instruct \
+ --model_free \
+ --scheme W4A16 \
+ --group_size 32 \
+ --asym \
+ --layer_config "{k_proj:{bits:8},v_proj:{bits:8}}" \
+ --ignore_layers "mlp" \
+ --output_dir ./int4-llama
+```
+
+#### API 用法
+
+```python
+from auto_round import AutoRound
+
+AutoRound(
+ model="meta-llama/Llama-3.2-1B-Instruct",
+ scheme="W4A16", # 也支持 QuantizationScheme 对象自定义 group_size / sym
+ layer_config={
+ ".*k_proj": {"bits": 8, "group_size": 32},
+ ".*v_proj": {"bits": 8, "group_size": 32},
+ },
+ ignore_layers="mlp",
+ model_free=True,
+).quantize_and_save("./int4-llama")
+```
+
+> **注意:** 免模型量化模式仅支持 `auto_round` 输出格式,并使用 RTN(无标定数据、无迭代调优)。如需更高质量的量化结果或使用受支持列表外的 scheme,请使用标准 AutoRound 流程。
+
+</details>
+
### GGUF 格式量化
实验性功能。该格式适用 CPU 设备,在社区应用广泛。
@@ -745,8 +834,8 @@ print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50, do_sample=Fal
| ark | xpu | 4、8 | FP32/FP16/BF16 | 6 | gptq/gptq_zp+-1 | auto-round-lib<br/>torch>=2.8.0 |
| ark | xpu | 4 | FP32/FP16/BF16 | 6 | awq | auto-round-lib<br/>torch>=2.8.0 |
| marlin | cuda | 4、8 | BF16/FP16 | 6 | gptq/gptq_zp+-1 | gptqmodel |
-| exllamav2/<br/>gptqmodel:exllamav2 | cuda | 4 | BF16/FP16 | 5 | gptq | gptqmodel |
-| exllamav2/<br/>gptq:exllamav2 | cuda | 4 | FP16 | 5 | gptq_zp+-1 | auto-gptq<br/>transformers<5.0.0 |
+| exllamav2/<br/>gptqmodel:exllamav2 | cuda | 4 | BF16/FP16 | 5 | gptq/gptq_zp+-1 | gptqmodel |
+| exllamav2/<br/>gptq:exllamav2 | cuda | 4 | FP16 | 3 | gptq_zp+-1 | auto-gptq<br/>transformers<5.0.0 |
| gptq:cuda | cuda | 2、3、4、8 | FP16 | 1 | gptq_zp+-1 | auto-gptq<br/>transformers<5.0.0 |
| triton | xpu/cuda | 2、4、8 | BF16/FP16 | 2 | gptq/gptq_zp+-1 | auto-round |
| awq | cuda | 4 | FP16 | 5 | awq | auto-awq<br/>transformers<4.57.0 |
diff --git a/test/test_cpu/core/test_low_cpu_mem_options.py b/test/test_cpu/core/test_low_cpu_mem_options.py
index b56ff1cea..d49a1f7af 100644
--- a/test/test_cpu/core/test_low_cpu_mem_options.py
+++ b/test/test_cpu/core/test_low_cpu_mem_options.py
@@ -34,7 +34,6 @@ class TestLowCpuMemUsage:
bits=4,
low_cpu_mem_usage=True,
iters=0,
- disable_opt_rtn=True,
nsamples=1,
seqlen=32,
)
@@ -45,7 +44,6 @@ class TestLowCpuMemUsage:
bits=4,
low_cpu_mem_usage=False,
iters=0,
- disable_opt_rtn=True,
nsamples=1,
seqlen=32,
)
@@ -58,7 +56,6 @@ class TestLowCpuMemUsage:
bits=4,
low_cpu_mem_usage=False,
iters=0,
- disable_opt_rtn=True,
nsamples=1,
seqlen=32,
)
@@ -79,6 +76,7 @@ class TestLowCpuMemUsage:
disable_opt_rtn=True,
nsamples=1,
seqlen=32,
+ disable_model_free=True, # Ensure offloading is enabled for this test
)
dummy_block = torch.nn.Linear(4, 4)
diff --git a/test/test_cpu/models/test_conv1d.py b/test/test_cpu/models/test_conv1d.py
index 717178e73..15acf2c94 100644
--- a/test/test_cpu/models/test_conv1d.py
+++ b/test/test_cpu/models/test_conv1d.py
@@ -43,3 +43,13 @@ class TestQuantizationConv1d:
model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="cpu", trust_remote_code=True)
model_infer(model, self.tokenizer)
+
+
+def test_find_layers_from_config_lamini():
+ from auto_round.utils.model import find_layers_from_config
+
+ res = find_layers_from_config(lamini_name_or_path, class_names="Conv1d")
+ print(res)
+ assert "Conv1D" in res, "Conv1D should be detected in the model config"
+ assert "h.0.attn.c_attn" in res["Conv1D"], "Conv1D should be detected in the model config with correct prefix"
+ assert len(res["Conv1D"]) == 48, "At least one Conv1D layer should be detected in the model config"
diff --git a/test/test_cpu/quantization/test_asym.py b/test/test_cpu/quantization/test_asym.py
index 0f792a3a1..1ac67b8cd 100644
--- a/test/test_cpu/quantization/test_asym.py
+++ b/test/test_cpu/quantization/test_asym.py
@@ -22,13 +22,21 @@ class TestAutoRoundAsym:
def teardown_class(cls):
shutil.rmtree("runs", ignore_errors=True)
- def test_asym_group_size(self, tiny_opt_model_path):
- for group_size in [32, 64, 128]:
- bits, sym = 4, False
+ # use parameters later
+ def test_asym_format(self, tiny_opt_model_path):
+ for format in ["auto_round", "auto_round:auto_gptq", "auto_round:gptqmodel"]:
+ bits, group_size, sym = 4, 128, False
ar = AutoRound(
- tiny_opt_model_path, bits=bits, group_size=group_size, sym=sym, iters=0, seqlen=2, nsamples=1
+ tiny_opt_model_path,
+ bits=bits,
+ group_size=group_size,
+ sym=sym,
+ iters=0,
+ seqlen=2,
+ nsamples=1,
+ disable_opt_rtn=True,
)
- _, quantized_model_path = ar.quantize_and_save(format="auto_round", output_dir=self.save_folder)
+ _, quantized_model_path = ar.quantize_and_save(format=format, output_dir=self.save_folder)
model = AutoModelForCausalLM.from_pretrained(
quantized_model_path,
@@ -39,9 +47,9 @@ class TestAutoRoundAsym:
tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
model_infer(model, tokenizer)
- def test_asym_bits(self, tiny_opt_model_path):
- for bits in [2, 8]:
- group_size, sym = 128, False
+ def test_asym_group_size(self, tiny_opt_model_path):
+ for group_size in [32, 64, 128]:
+ bits, sym = 4, False
ar = AutoRound(
tiny_opt_model_path, bits=bits, group_size=group_size, sym=sym, iters=0, seqlen=2, nsamples=1
)
@@ -56,21 +64,13 @@ class TestAutoRoundAsym:
tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
model_infer(model, tokenizer)
- # use parameters later
- def test_asym_format(self, tiny_opt_model_path):
- for format in ["auto_round", "auto_round:auto_gptq", "auto_round:gptqmodel"]:
- bits, group_size, sym = 4, 128, False
+ def test_asym_bits(self, tiny_opt_model_path):
+ for bits in [2, 8]:
+ group_size, sym = 128, False
ar = AutoRound(
- tiny_opt_model_path,
- bits=bits,
- group_size=group_size,
- sym=sym,
- iters=0,
- seqlen=2,
- nsamples=1,
- disable_opt_rtn=True,
+ tiny_opt_model_path, bits=bits, group_size=group_size, sym=sym, iters=0, seqlen=2, nsamples=1
)
- _, quantized_model_path = ar.quantize_and_save(format=format, output_dir=self.save_folder)
+ _, quantized_model_path = ar.quantize_and_save(format="auto_round", output_dir=self.save_folder)
model = AutoModelForCausalLM.from_pretrained(
quantized_model_path,
diff --git a/test/test_cpu/quantization/test_mix_bits.py b/test/test_cpu/quantization/test_mix_bits.py
index 6eab008c0..196466664 100644
--- a/test/test_cpu/quantization/test_mix_bits.py
+++ b/test/test_cpu/quantization/test_mix_bits.py
@@ -41,35 +41,6 @@ class TestAutoRound:
def teardown_class(cls):
shutil.rmtree("runs", ignore_errors=True)
- @require_gptqmodel
- def test_mixed_gptqmodel(self, dataloader):
- layer_config = {
- "k_proj": {"scheme": "W8A16"}, # part name
- "lm_head": {"scheme": "w4a16"}, # set lm_head quant
- "fc1": {"bits": 16},
- "model.decoder.layers.0.self_attn.v_proj": {"bits": 16},
- "model.decoder.layers.0.self_attn.q_proj": {"bits": 8}, # full name
- }
- autoround = AutoRound(
- model=self.model_name,
- scheme="W4A16",
- iters=2,
- seqlen=2,
- layer_config=layer_config,
- dataset=dataloader,
- )
- quantized_model_path = self.save_dir
- _, quantized_model_path = autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_gptq")
- # test original GPTQModel inference
- from gptqmodel import GPTQModel
-
- model = GPTQModel.load(quantized_model_path)
- assert model.model.model.decoder.layers[0].self_attn.k_proj.bits == 8
- assert model.model.model.decoder.layers[0].self_attn.q_proj.bits == 8
- assert model.model.model.decoder.layers[1].self_attn.v_proj.bits == 4
- result = model.generate("Uncovering deep insights begins with")[0] # tokens
- assert "!!!" not in model.tokenizer.decode(result) # string output
-
def test_mixed_gptqmodel_convert_to_ar(self, dataloader):
layer_config = {
"k_proj": {"bits": 8}, # part name
diff --git a/test/test_cpu/quantization/test_model_free.py b/test/test_cpu/quantization/test_model_free.py
new file mode 100644
index 000000000..1cfdd082d
--- /dev/null
+++ b/test/test_cpu/quantization/test_model_free.py
@@ -0,0 +1,410 @@
+# Copyright (c) 2025 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Unit tests for auto_round.compressors.model_free module."""
+
+import json
+import os
+
+import pytest
+import torch
+from safetensors import safe_open
+from safetensors.torch import save_file
+
+from auto_round import AutoRound
+from auto_round.compressors.model_free import (
+ _dequant_fp8_tensors,
+ _ModelFreeCompressorCore,
+ _PatternMatcher,
+ _process_shard,
+ get_predefined_ignore_layers_from_config,
+ is_model_free_supported_scheme,
+)
+from auto_round.schemes import QuantizationScheme
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+_LLAMA_CFG = {"architectures": ["LlamaForCausalLM"], "model_type": "llama"}
+_DEFAULT_SCHEME = {"bits": 4, "group_size": 128, "sym": True, "data_type": "int"}
+
+_SIMPLE_CONFIG = {
+ "architectures": ["OPTForCausalLM"],
+ "model_type": "opt",
+ "hidden_size": 128,
+ "num_hidden_layers": 2,
+}
+
+_SIMPLE_TENSORS = {
+ "model.decoder.layers.0.self_attn.q_proj.weight": torch.randn(128, 128),
+ "model.decoder.layers.0.self_attn.k_proj.weight": torch.randn(128, 128),
+ "model.decoder.layers.0.fc1.weight": torch.randn(512, 128),
+ "model.decoder.layers.0.fc2.weight": torch.randn(128, 512),
+ "model.decoder.layers.0.fc1.bias": torch.randn(512),
+ "model.decoder.embed_tokens.weight": torch.randn(1000, 128),
+ "lm_head.weight": torch.randn(1000, 128),
+}
+
+
+def _make_model_dir(tmp_path, config, tensors, *, multi_shard=False):
+ """Create a minimal local model directory with config.json and safetensors."""
+ model_dir = str(tmp_path / "source_model")
+ os.makedirs(model_dir, exist_ok=True)
+ with open(os.path.join(model_dir, "config.json"), "w") as f:
+ json.dump(config, f)
+
+ if not multi_shard:
+ save_file(tensors, os.path.join(model_dir, "model.safetensors"))
+ else:
+ keys = list(tensors.keys())
+ mid = max(1, len(keys) // 2)
+ shard1 = {k: tensors[k] for k in keys[:mid]}
+ shard2 = {k: tensors[k] for k in keys[mid:]}
+ save_file(shard1, os.path.join(model_dir, "model-00001-of-00002.safetensors"))
+ save_file(shard2, os.path.join(model_dir, "model-00002-of-00002.safetensors"))
+ weight_map = {}
+ for k in keys[:mid]:
+ weight_map[k] = "model-00001-of-00002.safetensors"
+ for k in keys[mid:]:
+ weight_map[k] = "model-00002-of-00002.safetensors"
+ with open(os.path.join(model_dir, "model.safetensors.index.json"), "w") as f:
+ json.dump({"metadata": {"total_size": 0}, "weight_map": weight_map}, f)
+
+ return model_dir
+
+
+def _matcher(ignore=None, layer_config=None, default=None):
+ return _PatternMatcher(ignore or [], layer_config or {}, default or {})
+
+
+def _read_output_keys(output_dir):
+ keys = set()
+ for f in os.listdir(output_dir):
+ if f.endswith(".safetensors"):
+ with safe_open(os.path.join(output_dir, f), framework="pt") as sf:
+ keys.update(sf.keys())
+ return keys
+
+
+def _read_qconfig(output_dir):
+ with open(os.path.join(output_dir, "config.json")) as f:
+ return json.load(f).get("quantization_config", {})
+
+
+# ===========================================================================
+# _PatternMatcher
+# ===========================================================================
+
+
+class TestPatternMatcher:
+ def test_ignore_substring(self):
+ m = _matcher(ignore=["mlp"])
+ assert m.should_ignore("model.layers.0.mlp.fc1.weight") is True
+ assert m.should_ignore("model.layers.0.self_attn.q_proj.weight") is False
+
+ def test_ignore_trailing_dot(self):
+ m = _matcher(ignore=["layers.4."])
+ assert m.should_ignore("model.layers.4.mlp.fc1.weight") is True
+ assert m.should_ignore("model.layers.45.mlp.fc1.weight") is False
+
+ def test_skip_predefined(self):
+ m = _matcher()
+ assert m.should_skip("model.layers.0.shared_expert_gate.weight") is True
+ assert m.should_skip("model.layers.0.mlp.gate.weight") is True
+ assert m.should_skip("model.embed_tokens.weight") is True
+ assert m.should_skip("model.layers.0.mlp.fc1.weight") is False
+
+ def test_resolve_scheme_exact_regex_and_default(self):
+ default = {"bits": 4, "group_size": 128, "sym": True}
+ lc = {
+ "model.layers.0.mlp.fc1": {"bits": 8, "group_size": 32},
+ r".*k_proj": {"bits": 8},
+ }
+ m = _matcher(layer_config=lc, default=default)
+ assert m.resolve_scheme("model.layers.0.mlp.fc1.weight")["bits"] == 8
+ assert m.resolve_scheme("model.layers.0.self_attn.k_proj.weight")["bits"] == 8
+ assert m.resolve_scheme("model.layers.0.mlp.fc2.weight") == default
+
+ def test_resolve_bits16_returns_none(self):
+ m = _matcher(layer_config={"model.layers.0.fc1": {"bits": 16}}, default={"bits": 4, "group_size": 128})
+ assert m.resolve_scheme("model.layers.0.fc1.weight") is None
+
+ def test_resolve_substring_pattern(self):
+ default = {"bits": 4, "group_size": 128, "sym": True}
+ m = _matcher(layer_config={".ffn.experts.": {"bits": 2, "group_size": 64}}, default=default)
+ r = m.resolve_scheme("model.layers.0.ffn.experts.3.gate_proj.weight")
+ assert r["bits"] == 2 and r["group_size"] == 64
+ assert m.resolve_scheme("model.layers.0.self_attn.q_proj.weight") == default
+
+
+# ===========================================================================
+# _parse_layer_config — scheme key resolution
+# ===========================================================================
+
+
+class TestParseLayerConfig:
+ @staticmethod
+ def _make_core(layer_config_input):
+ core = _ModelFreeCompressorCore(
+ model_name_or_path="dummy",
+ output_dir="dummy_out",
+ scheme="W4A16",
+ )
+ core.layer_config_input = layer_config_input
+ core._parse_scheme()
+ core._parse_layer_config()
+ return core
+
+ def test_scheme_key_resolves(self):
+ core = self._make_core({".ffn.experts.": {"scheme": "W2A16"}})
+ cfg = next(v for k, v in core.layer_config.items() if "ffn.experts" in k)
+ assert cfg["bits"] == 2 and "scheme" not in cfg
+
+ m = _matcher(layer_config=core.layer_config, default=core.default_scheme)
+ assert m.resolve_scheme("model.layers.0.ffn.experts.3.gate_proj.weight")["bits"] == 2
+
+ def test_scheme_key_with_overrides(self):
+ core = self._make_core({".ffn.experts.": {"scheme": "W2A16", "group_size": 32}})
+ cfg = next(v for k, v in core.layer_config.items() if "ffn.experts" in k)
+ assert cfg["bits"] == 2 and cfg["group_size"] == 32
+
+ def test_string_value(self):
+ core = self._make_core({".ffn.experts.": "W2A16"})
+ cfg = next(v for k, v in core.layer_config.items() if "ffn.experts" in k)
+ assert cfg["bits"] == 2
+
+ def test_quantization_scheme_value(self):
+ core = self._make_core({".ffn.experts.": QuantizationScheme(bits=2, group_size=64)})
+ cfg = next(v for k, v in core.layer_config.items() if "ffn.experts" in k)
+ assert cfg["bits"] == 2 and cfg["group_size"] == 64
+
+
+# ===========================================================================
+# _process_shard
+# ===========================================================================
+
+
+class TestProcessShard:
+ def test_quantizes_eligible_weights(self, tmp_path):
+ shard_path = str(tmp_path / "shard.safetensors")
+ save_file({"layer.fc1.weight": torch.randn(64, 128), "layer.fc1.bias": torch.randn(64)}, shard_path)
+ output, quantized, _ = _process_shard(shard_path, _DEFAULT_SCHEME, {}, [])
+ assert "layer.fc1" in quantized
+ assert "layer.fc1.qweight" in output and "layer.fc1.bias" in output
+
+ def test_ignores_and_skips(self, tmp_path):
+ shard_path = str(tmp_path / "shard.safetensors")
+ save_file(
+ {"lm_head.weight": torch.randn(100, 128), "model.layers.0.mlp.gate.weight": torch.randn(8, 128)},
+ shard_path,
+ )
+ _, quantized, ignored = _process_shard(shard_path, _DEFAULT_SCHEME, {}, ["lm_head"])
+ assert len(quantized) == 0
+ assert "lm_head" in ignored and "model.layers.0.mlp.gate" in ignored
+
+ def test_fused_expert_split(self, tmp_path):
+ N, I, H = 2, 64, 32
+ shard_path = str(tmp_path / "shard.safetensors")
+ save_file(
+ {
+ "model.layers.0.mlp.experts.gate_up_proj": torch.randn(N, 2 * I, H),
+ "model.layers.0.mlp.experts.down_proj": torch.randn(N, H, I),
+ },
+ shard_path,
+ )
+ output, quantized, _ = _process_shard(shard_path, _DEFAULT_SCHEME, {}, [])
+ for i in range(N):
+ for proj in ["gate_proj", "up_proj", "down_proj"]:
+ base = f"model.layers.0.mlp.experts.{i}.{proj}"
+ assert base in quantized and f"{base}.qweight" in output
+
+
+# ===========================================================================
+# FP8 source model
+# ===========================================================================
+
+
+class TestFP8Source:
+ def test_dequant_fp8(self):
+ w = torch.randn(64, 128, dtype=torch.bfloat16).to(torch.float8_e4m3fn)
+ raw = {"layer.weight": w, "layer.weight_scale_inv": torch.tensor(0.5), "layer.bias": torch.randn(64)}
+ result = _dequant_fp8_tensors(raw, block_size=None)
+ assert result["layer.weight"].dtype == torch.bfloat16 and "layer.weight_scale_inv" not in result
+
+ def test_no_fp8_noop(self):
+ raw = {"layer.weight": torch.randn(64, 128)}
+ assert _dequant_fp8_tensors(raw, block_size=None) is raw
+
+ def test_process_shard_fp8(self, tmp_path):
+ shard_path = str(tmp_path / "shard.safetensors")
+ w = torch.randn(64, 128, dtype=torch.bfloat16).to(torch.float8_e4m3fn)
+ save_file({"layer.weight": w, "layer.weight_scale_inv": torch.tensor(1.0)}, shard_path)
+ output, quantized, _ = _process_shard(shard_path, _DEFAULT_SCHEME, {}, [], device="cpu", fp8_block_size=None)
+ assert "layer" in quantized and "layer.qweight" in output
+
+
+# ===========================================================================
+# End-to-end ModelFreeQuantize
+# ===========================================================================
+
+
+class TestModelFreeQuantize:
+ def test_basic(self, tmp_path):
+ model_dir = _make_model_dir(tmp_path, _SIMPLE_CONFIG, _SIMPLE_TENSORS)
+ output_dir = str(tmp_path / "output")
+ AutoRound(model=model_dir, scheme="W4A16", model_free=True).quantize_and_save(output_dir)
+ qc = _read_qconfig(output_dir)
+ assert qc["quant_method"] == "auto-round" and qc["bits"] == 4 and qc["model_free"] is True
+ keys = _read_output_keys(output_dir)
+ assert "lm_head.weight" in keys and "lm_head.qweight" not in keys
+
+ def test_ignore_layers(self, tmp_path):
+ tensors = {
+ "model.layers.0.mlp.fc1.weight": torch.randn(512, 128),
+ "model.layers.0.self_attn.q_proj.weight": torch.randn(128, 128),
+ "lm_head.weight": torch.randn(1000, 128),
+ }
+ model_dir = _make_model_dir(tmp_path, _LLAMA_CFG, tensors)
+ output_dir = str(tmp_path / "output")
+ AutoRound(model=model_dir, scheme="W4A16", model_free=True, ignore_layers="mlp").quantize_and_save(output_dir)
+ keys = _read_output_keys(output_dir)
+ assert "model.layers.0.mlp.fc1.weight" in keys and "model.layers.0.mlp.fc1.qweight" not in keys
+ assert "model.layers.0.self_attn.q_proj.qweight" in keys
+
+ def test_multi_shard(self, tmp_path):
+ model_dir = _make_model_dir(tmp_path, _SIMPLE_CONFIG, _SIMPLE_TENSORS, multi_shard=True)
+ output_dir = str(tmp_path / "output")
+ AutoRound(model=model_dir, scheme="W4A16", model_free=True).quantize_and_save(output_dir)
+ assert os.path.exists(os.path.join(output_dir, "model.safetensors.index.json"))
+
+ def test_quant_lm_head(self, tmp_path):
+ model_dir = _make_model_dir(tmp_path, _SIMPLE_CONFIG, _SIMPLE_TENSORS)
+ output_dir = str(tmp_path / "output")
+ AutoRound(model=model_dir, scheme="W4A16", model_free=True, quant_lm_head=True).quantize_and_save(output_dir)
+ assert "lm_head.qweight" in _read_output_keys(output_dir)
+
+ def test_asym(self, tmp_path):
+ model_dir = _make_model_dir(tmp_path, _LLAMA_CFG, {"layer.weight": torch.randn(64, 128)})
+ output_dir = str(tmp_path / "output")
+ AutoRound(
+ model=model_dir, scheme=QuantizationScheme(bits=4, group_size=64, sym=False), model_free=True
+ ).quantize_and_save(output_dir)
+ qc = _read_qconfig(output_dir)
+ assert qc["sym"] is False and qc["group_size"] == 64
+
+
+# ===========================================================================
+# Scheme validation
+# ===========================================================================
+
+
+_SUPPORTED = ["W2A16", "W2A16G32", "W2A16G64", "W4A16", "W4A16_MIXED", "W8A16"]
+_UNSUPPORTED = ["W3A16", "FPW8A16", "BF16", "MXFP4", "MXFP8", "MXINT4", "NVFP4", "FP8_BLOCK", "FP8_STATIC", "INT8_W8A8"]
+
+
+class TestSchemeValidation:
+ @pytest.mark.parametrize("name", _SUPPORTED)
+ def test_supported(self, tmp_path, name):
+ model_dir = _make_model_dir(tmp_path, _LLAMA_CFG, {"model.layers.0.mlp.fc1.weight": torch.randn(64, 128)})
+ out = str(tmp_path / f"out_{name}")
+ AutoRound(model=model_dir, scheme=name, model_free=True).quantize_and_save(out)
+ assert "model.layers.0.mlp.fc1.qweight" in _read_output_keys(out)
+
+ @pytest.mark.parametrize("name", _UNSUPPORTED)
+ def test_unsupported_raises(self, tmp_path, name):
+ model_dir = _make_model_dir(tmp_path, _LLAMA_CFG, {"layer.weight": torch.randn(64, 128)})
+ with pytest.raises(ValueError):
+ AutoRound(model=model_dir, model_free=True, scheme=name).quantize_and_save(str(tmp_path / "out"))
+
+ def test_is_model_free_supported_scheme(self):
+ for name in _SUPPORTED:
+ assert is_model_free_supported_scheme(name) is True
+ for name in _UNSUPPORTED:
+ assert is_model_free_supported_scheme(name) is False
+ assert is_model_free_supported_scheme("DOES_NOT_EXIST") is False
+
+
+# ===========================================================================
+# CLI auto-routing
+# ===========================================================================
+
+
+class TestCliAutoRouting:
+ def test_auto_routes(self, tmp_path):
+ model_dir = _make_model_dir(tmp_path, _LLAMA_CFG, {"layer.weight": torch.randn(64, 128)})
+ out_dir = str(tmp_path / "out")
+ from auto_round.__main__ import BasicArgumentParser, tune
+
+ args = BasicArgumentParser().parse_args(
+ [
+ "--model",
+ model_dir,
+ "--scheme",
+ "W4A16",
+ "--iters",
+ "0",
+ "--disable_opt_rtn",
+ "--format",
+ "auto_round",
+ "--output_dir",
+ out_dir,
+ ]
+ )
+ tune(args)
+ assert _read_qconfig(out_dir).get("model_free") is True
+
+ def test_disable_model_free_flag(self):
+ from auto_round.__main__ import BasicArgumentParser
+
+ args = BasicArgumentParser().parse_args(
+ [
+ "--model",
+ "dummy",
+ "--scheme",
+ "W4A16",
+ "--iters",
+ "0",
+ "--disable_opt_rtn",
+ "--disable_model_free",
+ ]
+ )
+ auto_route = (
+ not args.model_free
+ and not args.disable_model_free
+ and args.iters == 0
+ and args.disable_opt_rtn is True
+ and is_model_free_supported_scheme(args.scheme)
+ )
+ assert auto_route is False
+
+
+# ===========================================================================
+# Predefined ignore layers
+# ===========================================================================
+
+
+class TestPredefinedIgnoreLayers:
+ def test_normal_model_empty(self):
+ assert get_predefined_ignore_layers_from_config({"architectures": ["LlamaForCausalLM"]}) == []
+
+ def test_step3p5_ignore_layers(self):
+ cfg = {"model_type": "step3p5"}
+ assert get_predefined_ignore_layers_from_config(cfg) == [
+ "g_proj",
+ "moe.gate",
+ "eh_proj",
+ "shared_head",
+ "layers.45",
+ ]
diff --git a/test/test_cpu/quantization/test_model_free_parity.py b/test/test_cpu/quantization/test_model_free_parity.py
new file mode 100644
index 000000000..008589552
--- /dev/null
+++ b/test/test_cpu/quantization/test_model_free_parity.py
@@ -0,0 +1,253 @@
+# Copyright (c) 2025 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Parity tests between ``AutoRound(model_free=True)`` and the regular
+``AutoRound(iters=0, disable_opt_rtn=True, disable_model_free=True)`` flow.
+
+Both code paths perform RTN integer weight-only quantization in the
+``auto_round:auto_gptq`` packing format. These tests assert that:
+
+1. The ``quantization_config`` keys ``bits``, ``group_size``, ``sym``,
+ ``data_type`` (family), ``quant_method``, ``packing_format`` agree.
+2. The set of tensor names ending in ``.qweight``, ``.qzeros``, ``.scales``
+ is identical (after unioning across shards).
+3. For every shared key, ``.qweight``/``.qzeros``/``.scales`` are bit-exact.
+
+This file is symlinked into ``test_cuda/quantization``, ``test_xpu/quantization``
+and ``test_hpu/quantization`` so the same test body runs on every backend.
+The active backend is selected by the ``BACKEND`` constant which is computed
+from this file's directory path. Tests are skipped when the corresponding
+accelerator is unavailable on the host.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+
+import pytest
+import torch
+from safetensors import safe_open
+
+# ---------------------------------------------------------------------------
+# Backend selection
+# ---------------------------------------------------------------------------
+
+
+def _detect_backend() -> str:
+ """Pick the active backend based on this test file's directory path."""
+ p = os.path.abspath(__file__)
+ for tag in ("test_cuda", "test_xpu", "test_hpu", "test_cpu"):
+ if f"/{tag}/" in p or p.endswith(f"/{tag}"):
+ return tag.split("_", 1)[1]
+ return "cpu"
+
+
+BACKEND = _detect_backend()
+
+
+def _device_str() -> str:
+ """Map the backend tag to the device string passed to AutoRound."""
+ return {
+ "cpu": "cpu",
+ "cuda": "cuda:0",
+ "xpu": "xpu:0",
+ "hpu": "hpu",
+ }.get(BACKEND, "cpu")
+
+
+# Schemes verified to be supported by both code paths.
+# Each entry is (test_id, preset_name, scheme_kwargs). ``preset_name`` is the
+# scheme string passed to AutoRound; ``test_id`` is the pytest parametrize id.
+_PARITY_SCHEMES = [
+ ("W4A16", "W4A16", {"bits": 4, "group_size": 128, "sym": True}),
+ ("W2A16", "W2A16", {"bits": 2, "group_size": 128, "sym": True}),
+ ("W2A16G64", "W2A16G64", {"bits": 2, "group_size": 64, "sym": True}),
+ ("W2A16G32", "W2A16G32", {"bits": 2, "group_size": 32, "sym": True}),
+ ("W8A16", "W8A16", {"bits": 8, "group_size": 128, "sym": True}),
+ # Asymmetric (sym=False) variants for non-4-bit: both the model-free path and
+ # the regular path use ``auto_round:auto_gptq`` packing, so bit-exact parity holds.
+ ("W4A16_ASYM", "W4A16", {"bits": 4, "group_size": 128, "sym": False}),
+ ("W2A16_ASYM", "W2A16", {"bits": 2, "group_size": 128, "sym": False}),
+ ("W2A16G64_ASYM", "W2A16G64", {"bits": 2, "group_size": 64, "sym": False}),
+ ("W8A16_ASYM", "W8A16", {"bits": 8, "group_size": 128, "sym": False}),
+ # Note: W4A16 asym is excluded because the regular path uses
+ # ``auto_round:auto_awq`` packing for 4-bit asym, which differs from
+ # model-free's ``auto_round:auto_gptq``.
+]
+
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+
+def _load_all_keys_and_tensors(directory: str) -> dict[str, torch.Tensor]:
+ """Union of all tensors across every safetensors shard in *directory*."""
+ out: dict[str, torch.Tensor] = {}
+ for fname in sorted(os.listdir(directory)):
+ if not fname.endswith(".safetensors"):
+ continue
+ with safe_open(os.path.join(directory, fname), framework="pt") as f:
+ for k in f.keys():
+ out[k] = f.get_tensor(k)
+ return out
+
+
+def _quant_keys(tensors: dict[str, torch.Tensor]) -> set[str]:
+ return {k for k in tensors if k.endswith((".qweight", ".qzeros", ".scales"))}
+
+
+def _read_qconfig(directory: str) -> dict:
+ with open(os.path.join(directory, "config.json")) as f:
+ return json.load(f)["quantization_config"]
+
+
+# ---------------------------------------------------------------------------
+# Parity tests
+# ---------------------------------------------------------------------------
+
+
[email protected]("scheme_name,scheme_preset,scheme_kwargs", _PARITY_SCHEMES)
+def test_parity_model_free_vs_disable_opt_rtn(tmp_path, tiny_opt_model_path, scheme_name, scheme_preset, scheme_kwargs):
+ """``AutoRound(model_free=True)`` must produce identical packed tensors
+ to ``AutoRound(iters=0, disable_opt_rtn=True, disable_model_free=True)``.
+ """
+ from auto_round import AutoRound
+
+ out_a = str(tmp_path / f"mf_{scheme_name}")
+ out_b = str(tmp_path / f"reg_{scheme_name}")
+ device = _device_str()
+
+ # ---- Path A: AutoRound(model_free=True) ----
+ ar_a = AutoRound(
+ tiny_opt_model_path,
+ scheme=scheme_preset,
+ bits=scheme_kwargs["bits"],
+ group_size=scheme_kwargs["group_size"],
+ sym=scheme_kwargs["sym"],
+ iters=0,
+ disable_opt_rtn=True,
+ model_free=True,
+ device_map=device,
+ )
+ assert getattr(ar_a, "model_free", False) is True
+ # Model must NOT be loaded into memory in the model-free path.
+ assert ar_a.model is None
+ ar_a.quantize_and_save(format="auto_round", output_dir=out_a)
+
+ # ---- Path B: regular ``--iters 0 --disable_opt_rtn`` flow ----
+ ar_b = AutoRound(
+ tiny_opt_model_path,
+ scheme=scheme_preset,
+ bits=scheme_kwargs["bits"],
+ group_size=scheme_kwargs["group_size"],
+ sym=scheme_kwargs["sym"],
+ iters=0,
+ disable_opt_rtn=True,
+ disable_model_free=True, # opt out of auto-routing
+ device_map=device,
+ amp=False, # disable_amp to ensure model is loaded in full precision
+ )
+ assert getattr(ar_b, "model_free", False) is False
+ # Confirm the regular path actually loaded the model on the requested
+ # device family (this proves that the corresponding accelerator is
+ # actually exercised by the symlinked CUDA/XPU/HPU test).
+ if BACKEND != "cpu":
+ weight_devices = {p.device.type for p in ar_b.model.parameters()}
+ assert (
+ BACKEND in weight_devices or "cpu" in weight_devices
+ ), f"Expected model parameters on '{BACKEND}' or 'cpu', got {weight_devices}"
+ # ar_b.quantize_and_save(format="auto_round", output_dir=out_b)
+ _, out_b = ar_b.quantize_and_save(format="auto_round:auto_gptq", output_dir=out_b)
+
+ # ---- 1. quantization_config core keys agree ----
+ qc_a = _read_qconfig(out_a)
+ qc_b = _read_qconfig(out_b)
+
+ for key in ("bits", "group_size", "sym", "quant_method"):
+ assert qc_a[key] == qc_b[key], f"qconfig[{key}] differs: model_free={qc_a[key]} regular={qc_b[key]}"
+ assert "int" in qc_a.get("data_type", "")
+ assert "int" in qc_b.get("data_type", "")
+ assert (
+ qc_a["packing_format"] == qc_b["packing_format"]
+ ), f"packing_format differs: {qc_a['packing_format']} vs {qc_b['packing_format']}"
+
+ # ---- 2. quantized tensor key sets agree ----
+ tensors_a = _load_all_keys_and_tensors(out_a)
+ tensors_b = _load_all_keys_and_tensors(out_b)
+ keys_a = _quant_keys(tensors_a)
+ keys_b = _quant_keys(tensors_b)
+
+ only_a = keys_a - keys_b
+ only_b = keys_b - keys_a
+ assert not only_a and not only_b, (
+ f"Quantized key sets differ.\n only in model_free: {sorted(only_a)[:10]}\n"
+ f" only in regular: {sorted(only_b)[:10]}"
+ )
+
+ # ---- 3. Bit-exact equality for shared qweight/qzeros/scales ----
+ mismatched: list[str] = []
+ for k in sorted(keys_a):
+ ta = tensors_a[k]
+ tb = tensors_b[k]
+ if ta.shape != tb.shape or ta.dtype != tb.dtype:
+ mismatched.append(f"{k}: shape/dtype differs ({ta.shape}/{ta.dtype} vs {tb.shape}/{tb.dtype})")
+ continue
+ if not torch.equal(ta, tb):
+ diff = (ta.float() - tb.float()).abs()
+ mismatched.append(
+ f"{k}: max|diff|={diff.max().item():.4g}, " f"#diff={int((ta != tb).sum().item())}/{ta.numel()}"
+ )
+
+ if mismatched:
+ pytest.fail(
+ f"RTN values differ between model_free and regular path "
+ f"({len(mismatched)} tensors differ). First few:\n " + "\n ".join(str(m) for m in mismatched[:5])
+ )
+
+
+def test_auto_routing_to_model_free(tiny_opt_model_path):
+ """When iters=0 + disable_opt_rtn=True + supported scheme, AutoRound
+ auto-routes to the model-free path even without explicit ``model_free=True``.
+ """
+ from auto_round import AutoRound
+
+ ar = AutoRound(
+ tiny_opt_model_path,
+ scheme="W4A16",
+ iters=0,
+ disable_opt_rtn=True,
+ device_map=_device_str(),
+ )
+ assert getattr(ar, "model_free", False) is True
+ assert ar.model is None
+
+
+def test_disable_model_free_opt_out(tiny_opt_model_path):
+ """``disable_model_free=True`` keeps the regular flow even with the
+ auto-routing trigger conditions.
+ """
+ from auto_round import AutoRound
+
+ ar = AutoRound(
+ tiny_opt_model_path,
+ scheme="W4A16",
+ iters=0,
+ disable_opt_rtn=True,
+ disable_model_free=True,
+ device_map=_device_str(),
+ )
+ assert getattr(ar, "model_free", False) is False
+ assert ar.model is not None
diff --git a/test/test_cpu/schemes/test_scheme.py b/test/test_cpu/schemes/test_scheme.py
index 41e9a9f7d..b011d7fc0 100644
--- a/test/test_cpu/schemes/test_scheme.py
+++ b/test/test_cpu/schemes/test_scheme.py
@@ -80,6 +80,7 @@ class TestAutoRound:
disable_opt_rtn=True,
dataset=dataloader,
low_cpu_mem_usage=False,
+ disable_model_free=True,
)
_, quantized_model_path = ar.quantize_and_save(self.save_folder)
model = transformers.Qwen2_5_VLForConditionalGeneration.from_pretrained(quantized_model_path)
diff --git a/test/test_cuda/export/test_auto_gptq_format.py b/test/test_cuda/export/test_auto_gptq_format.py
index f7a2891cf..4ab6c4ca1 100644
--- a/test/test_cuda/export/test_auto_gptq_format.py
+++ b/test/test_cuda/export/test_auto_gptq_format.py
@@ -83,3 +83,32 @@ class TestAutoRound:
)
tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
eval_generated_prompt(model, tokenizer)
+
+ @require_gptqmodel
+ def test_mixed_gptqmodel(self, tiny_opt_model_path, dataloader, tmp_path):
+ layer_config = {
+ "k_proj": {"scheme": "W8A16"}, # part name
+ "lm_head": {"scheme": "w4a16"}, # set lm_head quant, useless due to tied weights
+ "fc1": {"bits": 16},
+ "model.decoder.layers.0.self_attn.v_proj": {"bits": 16},
+ "model.decoder.layers.0.self_attn.q_proj": {"bits": 8}, # full name
+ }
+ autoround = AutoRound(
+ model=tiny_opt_model_path,
+ scheme="W4A16",
+ iters=2,
+ seqlen=2,
+ layer_config=layer_config,
+ dataset=dataloader,
+ )
+ quantized_model_path = tmp_path
+ _, quantized_model_path = autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_gptq")
+ # test original GPTQModel inference
+ from gptqmodel import GPTQModel
+
+ model = GPTQModel.load(quantized_model_path)
+ assert model.model.model.decoder.layers[0].self_attn.k_proj.bits == 8
+ assert model.model.model.decoder.layers[0].self_attn.q_proj.bits == 8
+ assert model.model.model.decoder.layers[1].self_attn.v_proj.bits == 4
+ result = model.generate("Uncovering deep insights begins with")[0] # tokens
+ assert "!!!" not in model.tokenizer.decode(result) # string output
diff --git a/test/test_cuda/quantization/test_model_free_parity.py b/test/test_cuda/quantization/test_model_free_parity.py
new file mode 120000
index 000000000..7c94ed661
--- /dev/null
+++ b/test/test_cuda/quantization/test_model_free_parity.py
@@ -0,0 +1 @@
+../../test_cpu/quantization/test_model_free_parity.py
\ No newline at end of file
diff --git a/test/test_xpu/quantization/test_model_free_parity.py b/test/test_xpu/quantization/test_model_free_parity.py
new file mode 120000
index 000000000..7c94ed661
--- /dev/null
+++ b/test/test_xpu/quantization/test_model_free_parity.py
@@ -0,0 +1 @@
+../../test_cpu/quantization/test_model_free_parity.py
\ No newline at end of file