PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
support model_free WOQ quantization (#1699)
Xin Hef0013f09b2f3
diff --git a/auto_round/__main__.py b/auto_round/__main__.pyindex 0fe576c75..a7e4a9569 100644--- a/auto_round/__main__.py+++ b/auto_round/__main__.py@@ -17,11 +17,13 @@ import osimport reimport sys+import torch+from auto_round.auto_scheme import AutoSchemefrom auto_round.compressors import BaseCompressorfrom auto_round.eval.eval_cli import EvalArgumentParser, eval, eval_task_by_taskfrom auto_round.eval.evaluation import run_model_evaluation-from auto_round.schemes import PRESET_SCHEMES+from auto_round.schemes import PRESET_SCHEMES, preset_name_to_schemefrom auto_round.utils import (clear_memory,get_device_and_parallelism,@@ -183,7 +185,6 @@ class BasicArgumentParser(argparse.ArgumentParser):help="Disable trusting remote code when loading models. ""Use for security if you don't trust the model source.",)-tuning = self.add_argument_group("Tuning Arguments")tuning.add_argument("--ignore_scale_zp_bits",@@ -287,7 +288,6 @@ class BasicArgumentParser(argparse.ArgumentParser):help="Disable optimization for RTN (Round-To-Nearest) mode when iters=0. ""RTN is fast but less accurate; keeping optimization enabled is recommended.",)-group_opt_rtn.add_argument("--enable_opt_rtn",action="store_const",@@ -295,6 +295,22 @@ class BasicArgumentParser(argparse.ArgumentParser):dest="disable_opt_rtn",help="Enable optimization for RTN mode when iters=0.",)+ group_model_free = tuning.add_mutually_exclusive_group()+ group_model_free.add_argument(+ "--model_free",+ action="store_true",+ help="Force model-free quantization mode. "+ "Downloads and quantizes safetensors files directly using RTN, "+ "without loading the full model into memory. "+ "Only supports auto_round output format.",+ )+ group_model_free.add_argument(+ "--disable_model_free",+ action="store_true",+ help="Disable the automatic model-free routing that activates when "+ "--iters 0 --disable_opt_rtn is combined with a supported INT WOQ scheme. "+ "Use this to force the regular AutoRound flow.",+ )scheme = self.add_argument_group("Scheme Arguments")scheme.add_argument("--bits", default=None, type=int, help="Number of bits for weight quantization. ")@@ -595,9 +611,70 @@ def tune(args):if "marlin" in args.format and args.asym is True:raise RuntimeError("marlin backend only supports sym quantization, please remove --asym")- device_str, use_auto_mapping = get_device_and_parallelism(args.device_map)+ # ======================= Model-Free Mode =======================+ # The model-free path is now integrated into AutoRound itself. We only+ # need to forward the relevant flags; AutoRound handles auto-routing+ # (when iters=0 + disable_opt_rtn + supported scheme) and explicit+ # ``--model_free``. Layer config / ignore-layers / output-dir handling+ # for model-free still needs special treatment because the model is+ # never loaded here.+ explicit_model_free = bool(getattr(args, "model_free", False))+ from auto_round.compressors.model_free import is_model_free_supported_scheme++ auto_model_free = (+ not explicit_model_free+ and not getattr(args, "disable_model_free", False)+ and getattr(args, "iters", None) == 0+ and getattr(args, "disable_opt_rtn", None) is True+ and is_model_free_supported_scheme(args.scheme, vars(args))+ and (+ str(getattr(args, "format", "auto_round") or "auto_round").lower().replace(" ", "").split(",")[0]+ == "auto_round"+ or format.startswith("auto_round")+ )+ )- import torch+ if explicit_model_free or auto_model_free:+ scheme = args.scheme.upper()+ if scheme not in PRESET_SCHEMES:+ raise ValueError(f"{scheme} is not supported. Only {list(PRESET_SCHEMES.keys())} are supported")+ if not is_model_free_supported_scheme(scheme, vars(args)) and not explicit_model_free:+ logger.info(+ f"Auto-routing to model-free is skipped: scheme '{scheme}' is not in "+ f"the model-free allowlist. Falling back to the regular AutoRound flow."+ )+ else:+ layer_config = {}+ if args.layer_config:+ layer_config = parse_layer_config_arg(args.layer_config)++ model_name = args.model.rstrip("/")+ output_dir = args.output_dir+ if output_dir == "./tmp_autoround" and model_name.split("/")[-1].strip(".") != "":+ s = preset_name_to_scheme(scheme)+ suffix = f"g{s.group_size}" if s.group_size > 0 else f"a{s.act_bits}"+ output_dir = os.path.join(args.output_dir, model_name.split("/")[-1] + f"-w{s.bits}{suffix}")++ from auto_round import AutoRound++ ar = AutoRound(+ model_name,+ scheme=scheme,+ sym=getattr(args, "sym", None),+ group_size=getattr(args, "group_size", None),+ iters=0,+ disable_opt_rtn=True,+ model_free=True,+ layer_config=layer_config,+ ignore_layers=args.ignore_layers,+ quant_lm_head=getattr(args, "quant_lm_head", False),+ quant_nontext_module=getattr(args, "quant_nontext_module", False),+ device_map=args.device_map,+ )+ ar.quantize_and_save(output_dir=output_dir, format=args.format) # pylint: disable=E1101+ return++ device_str, use_auto_mapping = get_device_and_parallelism(args.device_map)if args.enable_torch_compile:logger.info(diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex b1844490f..7a4271647 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -27,9 +27,10 @@ from auto_round.compressors import (MLLMCompressor,)from auto_round.compressors.diffusion.hybrid import HybridCompressor, is_hybrid_diffusion_model+from auto_round.compressors.model_free import ModelFreeCompressorfrom auto_round.logger import deprecated, loggerfrom auto_round.schemes import QuantizationScheme-from auto_round.utils import is_diffusion_model, is_mllm_model+from auto_round.utils import is_diffusion_model, is_mllm_model, is_model_free_routeif TYPE_CHECKING:from auto_round.auto_scheme.gen_auto_scheme import AutoScheme@@ -171,6 +172,22 @@ class AutoRound:return AutoRoundCompatible(**local_args, **kwargs)+ # ---- Model-free fast-path detection --------------------------------+ if is_model_free_route(model, scheme, iters, disable_opt_rtn, kwargs):+ if not isinstance(model, str):+ raise ValueError("model_free=True requires `model` to be a HuggingFace ID or local path string.")+ if not bool(kwargs.get("model_free", False)):+ logger.info(+ "Auto-routing to model-free quantization "+ "(iters=0, disable_opt_rtn=True, supported scheme). "+ "Pass disable_model_free=True to use the regular flow."+ )+ if extra_config is not None:+ local_args.update(extra_config.to_dict())+ local_args["model_name_or_path"] = local_args.pop("model")+ return ModelFreeCompressor(**local_args, **kwargs)+ # --------------------------------------------------------------------+model_cls = []has_multimodal_assets = kwargs.get("processor") is not None or kwargs.get("image_processor") is not Nonediff --git a/auto_round/compressors/__init__.py b/auto_round/compressors/__init__.pyindex 15ec27ebe..07c763482 100644--- a/auto_round/compressors/__init__.py+++ b/auto_round/compressors/__init__.py@@ -18,6 +18,7 @@ from auto_round.compressors.base import LLMCompressorfrom auto_round.compressors.mllm.compressor import MLLMCompressorfrom auto_round.compressors.diffusion.compressor import DiffusionCompressorfrom auto_round.compressors.diffusion.hybrid import HybridCompressor+from auto_round.compressors.model_free import ModelFreeCompressorfrom auto_round.compressors.config import (DiffusionExtraConfig,ExtraConfig,diff --git a/auto_round/compressors/model_free.py b/auto_round/compressors/model_free.pynew file mode 100644index 000000000..6a22e2f4f--- /dev/null+++ b/auto_round/compressors/model_free.py@@ -0,0 +1,1630 @@+# Copyright (c) 2025 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++"""Model-free RTN quantization (class-based).++This module performs weight-only quantization (WOQ) using RTN (Round-To-Nearest)+**without** loading the full model into memory. It reads safetensors files+(from a Hugging Face repo or a local directory), quantizes eligible+``nn.Linear`` weight tensors shard-by-shard, and writes the packed result to+the output directory.++The main entry point is the :class:`ModelFreeCompressor` class.++Supported schemes+-----------------+Model-free mode currently supports **integer weight-only** quantization+schemes packed in the ``auto_round:auto_gptq`` format only. Specifically:++* Preset names: ``W2A16``, ``W2A16G32``, ``W2A16G64``, ``W3A16``, ``W4A16``,+ ``W8A16``.+* Custom :class:`~auto_round.schemes.QuantizationScheme` instances with+ ``data_type="int"``, ``bits in {2, 3, 4, 8}``, ``act_bits >= 16``, and any+ symmetric / asymmetric configuration.++Schemes that require special packing (FP8, MXFP4, NVFP4, GGUF, INT8_W8A8,+BF16, FPW8A16, ...) are **not** supported in model-free mode and will raise+``ValueError``. Use the standard AutoRound flow for those.++Usage (CLI)+-----------+::++ auto_round facebook/opt-125m \\+ --model_free \\+ --scheme W4A16 \\+ --output_dir int4-125m++Usage (API)+-----------+::++ from auto_round import AutoRound++ AutoRound(+ model="facebook/opt-125m",+ scheme="W4A16",+ model_free=True,+ ).quantize_and_save("./int4-125m")+"""++from __future__ import annotations++import copy+import json+import os+import re+import shutil+import time+from concurrent.futures import ThreadPoolExecutor+from dataclasses import asdict, fields+from typing import Optional, Union++import torch++from auto_round import envs+from auto_round.logger import logger+from auto_round.schemes import PRESET_SCHEMES, QuantizationScheme, preset_name_to_scheme+from auto_round.utils.common import compress_layer_names, to_standard_regex+from auto_round.utils.device import clear_memory, memory_monitor+from auto_round.utils.missing_tensors import quantize_weight_rtn, split_fused_expert_tensors++# ---------------------------------------------------------------------------+# Constants+# ---------------------------------------------------------------------------++# add "embed", "conv" in case of auto detection failure in _check_conv1d_and_embedding+_BLOCK_NAME_TO_IGNORE = ["shared_expert_gate.", ".gate.", "embed", "conv"]++# Integer WOQ preset schemes that model-free mode can produce.+# Other presets (FP8/MX/NV/GGUF/BF16/INT8_W8A8/FPW8A16) require different+# packing kernels not implemented by ``quantize_weight_rtn``.+#+# Note: ``W3A16`` (3-bit) is intentionally excluded. 3-bit packing requires+# in_features to be padded to a multiple of pack_factor=10, which the current+# ``quantize_weight_rtn`` implementation does not handle correctly.+SUPPORTED_PRESET_SCHEMES: tuple[str, ...] = (+ "W2A16",+ "W2A16G32",+ "W2A16G64",+ "W4A16",+ "W4A16_MIXED",+ "W8A16",+)++# Allowed ``bits`` values for integer WOQ.+# 3-bit is excluded — see note above.+_SUPPORTED_INT_BITS: tuple[int, ...] = (2, 4, 8)++# Multimodal keywords kept in full precision by default.+_NONTEXT_KEYWORDS: tuple[str, ...] = (+ "vision",+ "visual",+ "image",+ "img",+ "audio",+ "speech",+ "wav",+ "waveform",+)+++# ---------------------------------------------------------------------------+# Predefined ignore-layer rules+# ---------------------------------------------------------------------------+++def get_predefined_ignore_layers_from_config(config: dict) -> list[str]:+ """Return layers to ignore based on the model's config.json.++ Delegates to the same rules registered via+ :func:`~auto_round.special_model_handler.register_ignore_layers` by+ wrapping the config dict in a lightweight pseudo-model object, so there+ is no need to duplicate ignore-layer rule registrations here.+ """+ import types++ from auto_round.special_model_handler import _PRE_DEFINED_IGNORE_LAYERS++ # Build a pseudo-model whose .config attribute exposes the config fields.+ cfg_ns = types.SimpleNamespace(**config)+ wrapper = types.SimpleNamespace(config=cfg_ns)++ layers: list[str] = []+ for rule in _PRE_DEFINED_IGNORE_LAYERS:+ if all(m(wrapper) for m in rule.matchers):+ for ignore_layer in rule.ignore_layers:+ if isinstance(ignore_layer, str):+ layers.append(ignore_layer)+ else:+ # callable (e.g. get_glm_flash_ignore_layers)+ res = ignore_layer(wrapper)+ if isinstance(res, str):+ layers.append(res)+ elif isinstance(res, list):+ layers.extend(res)++ return list(dict.fromkeys(layers))+++# ---------------------------------------------------------------------------+# I/O helpers (model resolution, shard discovery, downloads)+# ---------------------------------------------------------------------------+++def _is_model_cached(model_name_or_path: str) -> bool:+ """Return True if the model is already available locally or in HF cache."""+ if os.path.isdir(model_name_or_path):+ return True+ try:+ from huggingface_hub import try_to_load_from_cache++ result = try_to_load_from_cache(model_name_or_path, "config.json")+ return isinstance(result, str)+ except Exception:+ return False+++def _resolve_source_dir(model_name_or_path: str) -> str:+ """Resolve model_name_or_path to a local directory (download if needed)."""+ if os.path.isdir(model_name_or_path):+ return model_name_or_path+ from huggingface_hub import snapshot_download++ return snapshot_download(model_name_or_path)+++def _load_config(source_dir: str) -> dict:+ """Load config.json from model directory."""+ config_path = os.path.join(source_dir, "config.json")+ if not os.path.exists(config_path):+ raise FileNotFoundError(f"config.json not found in {source_dir}")+ with open(config_path) as f:+ return json.load(f)+++def _list_weight_shards(source_dir: str) -> list[str]:+ """Return list of weight shard filenames in order.++ Safetensors shards are preferred. When no safetensors files are+ found the function falls back to PyTorch ``.bin`` shards.++ Handles both standard naming (``model.safetensors``,+ ``pytorch_model.bin``) and custom prefixes such as+ ``diffusion_pytorch_model-XXXXX-of-XXXXX.safetensors`` by scanning+ all ``*.safetensors.index.json`` / ``*.bin.index.json`` index files+ in the directory when no standard index is found.+ """++ def _shards_from_index(index_path: str) -> list[str]:+ with open(index_path) as f:+ index = json.load(f)+ seen: set[str] = set()+ shards: list[str] = []+ for shard_file in index["weight_map"].values():+ if shard_file not in seen:+ seen.add(shard_file)+ shards.append(shard_file)+ return shards++ # --- safetensors: standard index ---+ st_index = os.path.join(source_dir, "model.safetensors.index.json")+ if os.path.exists(st_index):+ return _shards_from_index(st_index)++ # --- safetensors: custom-prefix index (e.g. diffusion_pytorch_model.safetensors.index.json) ---+ for fname in sorted(os.listdir(source_dir)):+ if fname.endswith(".safetensors.index.json"):+ return _shards_from_index(os.path.join(source_dir, fname))++ # --- safetensors: any single .safetensors file ---+ st_files = sorted(f for f in os.listdir(source_dir) if f.endswith(".safetensors"))+ if len(st_files) == 1:+ return st_files++ # --- pytorch .bin: standard index ---+ bin_index = os.path.join(source_dir, "pytorch_model.bin.index.json")+ if os.path.exists(bin_index):+ return _shards_from_index(bin_index)++ # --- pytorch .bin: custom-prefix index ---+ for fname in sorted(os.listdir(source_dir)):+ if fname.endswith(".bin.index.json"):+ return _shards_from_index(os.path.join(source_dir, fname))++ # --- pytorch .bin: single file ---+ bin_single = os.path.join(source_dir, "pytorch_model.bin")+ if os.path.exists(bin_single):+ return ["pytorch_model.bin"]++ # --- pytorch .bin: any single .bin file ---+ bin_files = sorted(f for f in os.listdir(source_dir) if f.endswith(".bin"))+ if len(bin_files) == 1:+ return bin_files++ # --- safetensors: single file ---+ return ["model.safetensors"]+++def _is_eligible_weight(tensor_name: str, tensor: torch.Tensor) -> bool:+ """Check if a tensor is eligible for quantization (2D Linear weight)."""+ return tensor_name.endswith(".weight") and tensor.dim() == 2+++def _is_weight_shard(fname: str) -> bool:+ """Return True if *fname* is a weight shard (safetensors or .bin).++ Excludes index files (``*.index.json``) so that they are copied to the+ output directory as normal metadata.+ """+ if fname.endswith(".index.json"):+ return False+ return fname.endswith(".safetensors") or fname.endswith(".bin")+++# Keep old name as an alias for backward compatibility.+_is_safetensors_shard = _is_weight_shard+++def _download_single_shard(+ model_name_or_path: str,+ shard_filename: str,+ local_dir: str,+) -> str:+ """Download a single safetensors shard file. Returns the local path."""+ local_path = os.path.join(local_dir, shard_filename)+ if os.path.exists(local_path):+ logger.info(f"Shard '{shard_filename}' already exists at '{local_path}', skipping download.")+ return local_path++ if os.path.isdir(model_name_or_path):+ src = os.path.join(model_name_or_path, shard_filename)+ if os.path.exists(src):+ shutil.copy2(src, local_path)+ return local_path+ raise FileNotFoundError(f"{shard_filename} not found in {model_name_or_path}")++ from huggingface_hub import hf_hub_download++ return hf_hub_download(+ repo_id=model_name_or_path,+ filename=shard_filename,+ local_dir=local_dir,+ )+++def _download_metadata_files(+ model_name_or_path: str,+ local_dir: str,+) -> str:+ """Download all non-safetensors files from a model repo. Returns local dir."""+ os.makedirs(local_dir, exist_ok=True)++ if os.path.isdir(model_name_or_path):+ for fname in os.listdir(model_name_or_path):+ if _is_weight_shard(fname):+ continue+ src = os.path.join(model_name_or_path, fname)+ dst = os.path.join(local_dir, fname)+ if os.path.isdir(src):+ if not os.path.exists(dst):+ shutil.copytree(src, dst)+ elif os.path.isfile(src) and not os.path.exists(dst):+ shutil.copy2(src, dst)+ return local_dir++ from huggingface_hub import snapshot_download++ snapshot_download(+ repo_id=model_name_or_path,+ local_dir=local_dir,+ ignore_patterns=["*.safetensors", "*.bin", "*.pth", "*.pt"],+ )+ return local_dir+++# ---------------------------------------------------------------------------+# Pattern matcher+# ---------------------------------------------------------------------------+++class _PatternMatcher:+ """Precompiled pattern matcher with result caching.++ Merges *ignore_patterns* and ``_BLOCK_NAME_TO_IGNORE`` into single+ compiled regexes, precompiles ``layer_config`` regex patterns, and+ caches all match results so that repeated lookups (common across+ shards) are O(1) dict hits.+ """++ __slots__ = (+ "_ignore_re",+ "_skip_re",+ "_layer_config",+ "_default_scheme",+ "_compiled_lc",+ "_ignore_cache",+ "_scheme_cache",+ )++ def __init__(+ self,+ ignore_patterns: list[str],+ layer_config: dict[str, dict],+ default_scheme: dict,+ ):+ self._default_scheme = default_scheme+ self._layer_config = layer_config++ self._ignore_re: re.Pattern | None = self._build_ignore_regex(ignore_patterns)++ skip_parts = [re.escape(b) for b in _BLOCK_NAME_TO_IGNORE]+ self._skip_re: re.Pattern | None = re.compile("|".join(skip_parts)) if skip_parts else None++ # Each entry: (compiled_regex | None, plain_string | None, cfg_dict)+ self._compiled_lc: list[tuple[re.Pattern | None, str | None, dict]] = []+ for pattern, cfg in layer_config.items():+ try:+ self._compiled_lc.append((re.compile(to_standard_regex(pattern)), None, cfg))+ except re.error:+ self._compiled_lc.append((None, pattern, cfg))++ self._ignore_cache: dict[str, bool] = {}+ self._scheme_cache: dict[str, dict | None] = {}++ @staticmethod+ def _build_ignore_regex(patterns: list[str]) -> re.Pattern | None:+ """Merge ignore patterns into one compiled regex.++ Uses :func:`~auto_round.utils.common.to_standard_regex` so that+ plain names are automatically wrapped with ``.*`` on both sides+ (substring matching) and regex meta-characters in user patterns+ are preserved — consistent with ``set_layer_config``.+ """+ if not patterns:+ return None+ parts: list[str] = []+ for p in patterns:+ if p.endswith("."):+ std = to_standard_regex(p.rstrip("."))+ std = std.removesuffix(".*")+ parts.append(f"{std}(?:\\.|$)")+ else:+ parts.append(to_standard_regex(p))+ return re.compile("|".join(parts))++ def should_ignore(self, tensor_name: str) -> bool:+ """Check user-specified ignore patterns (merged regex + cache)."""+ cached = self._ignore_cache.get(tensor_name)+ if cached is not None:+ return cached+ layer_name = tensor_name.rsplit(".", 1)[0] if "." in tensor_name else tensor_name+ result = bool(self._ignore_re and self._ignore_re.search(layer_name))+ self._ignore_cache[tensor_name] = result+ return result++ def should_skip(self, tensor_name: str) -> bool:+ """Check predefined skip patterns (routing gates, embeddings, etc.)."""+ return bool(self._skip_re and self._skip_re.search(tensor_name))++ def resolve_scheme(self, tensor_name: str) -> dict | None:+ """Resolve quantization scheme for *tensor_name* (cached).++ Returns ``None`` when the layer should stay in full precision.+ """+ if tensor_name in self._scheme_cache:+ return self._scheme_cache[tensor_name]+ result = self._resolve_uncached(tensor_name)+ self._scheme_cache[tensor_name] = result+ return result++ def _resolve_uncached(self, tensor_name: str) -> dict | None:+ layer_name = tensor_name.rsplit(".", 1)[0] if "." in tensor_name else tensor_name+ default = self._default_scheme++ if layer_name in self._layer_config:+ cfg = self._layer_config[layer_name]+ if cfg.get("bits", default.get("bits", 4)) >= 16:+ return None+ return {**default, **cfg}++ for compiled, plain, cfg in self._compiled_lc:+ if compiled is not None:+ if compiled.search(layer_name):+ if cfg.get("bits", default.get("bits", 4)) >= 16:+ return None+ return {**default, **cfg}+ elif plain is not None and plain in layer_name:+ if cfg.get("bits", default.get("bits", 4)) >= 16:+ return None+ return {**default, **cfg}++ return default+++# ---------------------------------------------------------------------------+# Per-tensor / per-shard helpers+# ---------------------------------------------------------------------------+++def _quantize_single_tensor(+ tensor_name: str,+ tensor: torch.Tensor,+ matcher: "_PatternMatcher",+ device: str = "cpu",+) -> tuple[str, dict[str, torch.Tensor], str | None, str | None]:+ """Quantize one eligible weight tensor and return packed outputs.++ Returns:+ (layer_name, output_tensors_dict, quantized_layer_or_None, ignored_layer_or_None)+ """+ layer_name = tensor_name.rsplit(".", 1)[0]++ if not _is_eligible_weight(tensor_name, tensor):+ return layer_name, {tensor_name: tensor}, None, None++ if matcher.should_ignore(tensor_name):+ logger.debug(f"Ignoring (user-specified): {layer_name}")+ return layer_name, {tensor_name: tensor}, None, layer_name++ if matcher.should_skip(tensor_name):+ logger.debug(f"Skipping (predefined): {layer_name}")+ return layer_name, {tensor_name: tensor}, None, layer_name++ scheme = matcher.resolve_scheme(tensor_name)+ if scheme is None:+ logger.debug(f"Keeping full precision: {layer_name}")+ return layer_name, {tensor_name: tensor}, None, layer_name++ bits = scheme["bits"]+ group_size = scheme["group_size"]+ sym = scheme.get("sym", True)++ if bits >= 16:+ return layer_name, {tensor_name: tensor}, None, layer_name++ try:+ qweight, qzeros, scales = quantize_weight_rtn(+ weight=tensor,+ bits=bits,+ group_size=group_size,+ sym=sym,+ device=device,+ )++ out: dict[str, torch.Tensor] = {+ f"{layer_name}.qweight": qweight,+ f"{layer_name}.qzeros": qzeros,+ f"{layer_name}.scales": scales,+ }++ logger.debug(f"Quantized: {layer_name} (bits={bits}, group_size={group_size}, sym={sym})")+ return layer_name, out, layer_name, None++ except Exception as e:+ logger.warning(f"Failed to quantize {layer_name}: {e}. Keeping original weight.")+ return layer_name, {tensor_name: tensor}, None, layer_name+++def _dequant_fp8_tensors(+ raw_tensors: dict[str, torch.Tensor],+ block_size: list | None = None,+) -> dict[str, torch.Tensor]:+ """Dequantize FP8 / FP4-packed weight tensors and remove their scale tensors.++ Handles three storage conventions:++ 1. **DeepSeek-V3 FP8** — weight dtype ``float8_e4m3fn`` + ``weight_scale_inv``+ (per-block float32 scales, NOT E8M0).+ 2. **FP8 + UE8M0 scale** — weight dtype ``float8_e4m3fn`` + ``.scale``+ (F8_E8M0 / ``ue8m0`` format, e.g. ``scale_fmt=ue8m0``).+ 3. **FP4-packed I8 + UE8M0 scale** — weight dtype ``torch.int8`` where each+ byte stores two FP4 E2M1 nibbles, paired with a ``.scale`` tensor in+ F8_E8M0 format. Shape relationship: ``weight[rows, cols/2]`` and+ ``scale[rows, (cols/2)*2/block_size]`` where ``block_size`` is inferred+ from the ratio ``weight.shape[1] * 2 / scale.shape[1]``.++ All cases are converted to ``bfloat16`` so downstream RTN quantization can+ proceed normally.+ """+ from auto_round.utils.weight_handler import _dequant_fp8_linear_weight++ E8M0_EXPONENT_BIAS = 127++ def _e8m0_to_float(scale_tensor: torch.Tensor) -> torch.Tensor:+ """Convert E8M0 (power-of-2 exponent) scale tensor to float."""+ raw = scale_tensor.view(torch.uint8).to(torch.int16) - E8M0_EXPONENT_BIAS+ return torch.pow(2.0, raw.to(torch.float32)).to(torch.bfloat16)++ def _dequant_fp4_packed_weight(+ weight_i8: torch.Tensor,+ scale_e8m0: torch.Tensor,+ ) -> torch.Tensor:+ """Dequantize FP4-packed int8 weight with E8M0 block scales to bfloat16.++ Each ``int8`` byte stores two FP4 E2M1 values: the low nibble is the+ first element and the high nibble is the second. Block size is inferred+ from the shape ratio ``weight.shape[1] * 2 / scale.shape[1]``.++ Args:+ weight_i8: Packed weight tensor, shape ``[rows, cols_packed]``,+ dtype ``torch.int8``. ``cols_packed = cols / 2``.+ scale_e8m0: Per-block scale tensor, shape ``[rows, n_blocks]``,+ stored as raw uint8 bytes in F8_E8M0 (ue8m0) format.++ Returns:+ Dequantized weight, shape ``[rows, cols]``, dtype ``bfloat16``.+ """+ # FP4 E2M1 lookup table — index is the 4-bit pattern (0..15)+ # Layout: positive range at indices 0-7, negative at 8-15.+ FP4_LUT = torch.tensor(+ [0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0, -0.0, -0.5, -1.0, -1.5, -2.0, -3.0, -4.0, -6.0],+ dtype=torch.bfloat16,+ device=weight_i8.device,+ )+ rows = weight_i8.shape[0]++ # Re-interpret int8 storage as uint8 for bit operations.+ weight_u8 = weight_i8.view(torch.uint8)+ lo = (weight_u8 & 0x0F).to(torch.long) # low nibble → first element+ hi = ((weight_u8 >> 4) & 0x0F).to(torch.long) # high nibble → second element+ # Interleave lo/hi along the last dim: [rows, cols_packed, 2] → [rows, cols]+ unpacked = torch.stack([lo, hi], dim=-1).reshape(rows, -1)+ cols = unpacked.shape[1]++ # Decode 4-bit patterns to bfloat16 float values via the lookup table.+ decoded = FP4_LUT[unpacked] # [rows, cols]++ # Convert E8M0 per-block scales to float32 → bfloat16.+ scale_f = _e8m0_to_float(scale_e8m0) # [rows, n_blocks]+ n_blocks = scale_f.shape[1]+ block_size_inner = cols // n_blocks # inferred block size (e.g. 32)++ # Multiply each block of decoded values by its corresponding scale.+ decoded = decoded.reshape(rows, n_blocks, block_size_inner)+ return (decoded * scale_f.unsqueeze(-1)).reshape(rows, cols)++ # ------------------------------------------------------------------+ # Collect quantized weight entries.+ # Tuple layout: (weight_name, scale_name, is_e8m0, is_fp4_packed)+ # is_e8m0 – scale tensor is in F8_E8M0 format (needs _e8m0_to_float)+ # is_fp4_packed – weight is int8-packed FP4; use _dequant_fp4_packed_weight+ # ------------------------------------------------------------------+ quant_entries: list[tuple[str, str, bool, bool]] = []++ for name, tensor in raw_tensors.items():+ if not name.endswith(".weight"):+ continue++ is_fp4_packed = tensor.dtype == torch.int8+ is_fp8 = tensor.dtype == torch.float8_e4m3fn+ # Also catch other 1-byte dtypes (e.g. float8_e5m2) by element size.+ if not is_fp4_packed and not is_fp8:+ if tensor.element_size() != 1:+ continue+ is_fp8 = True # treat remaining 1-byte dtypes as FP8++ # Convention 1: .weight_scale_inv (DeepSeek-V3 style, FP8 only)+ scale_inv_name = name.replace(".weight", ".weight_scale_inv")+ if scale_inv_name in raw_tensors and not is_fp4_packed:+ quant_entries.append((name, scale_inv_name, False, False))+ continue++ # Convention 2: .scale in F8_E8M0 / ue8m0 format+ scale_name = name.replace(".weight", ".scale")+ if scale_name in raw_tensors:+ quant_entries.append((name, scale_name, True, is_fp4_packed))++ if not quant_entries:+ return raw_tensors++ fp4_count = sum(1 for e in quant_entries if e[3])+ fp8_count = len(quant_entries) - fp4_count+ parts: list[str] = []+ if fp8_count:+ parts.append(f"{fp8_count} FP8")+ if fp4_count:+ parts.append(f"{fp4_count} FP4-packed (int8)")+ logger.info(f"Dequantizing {' and '.join(parts)} weight tensor(s) to bfloat16.")++ for weight_name, scale_name, is_e8m0, is_fp4_packed in quant_entries:+ weight = raw_tensors[weight_name]+ scale = raw_tensors.pop(scale_name)+ if is_fp4_packed:+ # FP4 E2M1 packed in int8 with UE8M0 per-block scale.+ raw_tensors[weight_name] = _dequant_fp4_packed_weight(weight, scale)+ else:+ if is_e8m0:+ scale = _e8m0_to_float(scale)+ raw_tensors[weight_name] = _dequant_fp8_linear_weight(weight, scale, block_size=block_size)++ return raw_tensors+++def _process_shard(+ shard_path: str,+ default_scheme: dict = None,+ layer_config: dict = None,+ ignore_patterns: list[str] = None,+ device: str = "cpu",+ *,+ matcher: "_PatternMatcher | None" = None,+ fp8_block_size: list | None = None,+) -> tuple[dict[str, torch.Tensor], list[str], list[str]]:+ """Quantize eligible weights in a single safetensors shard.++ Returns:+ (output_tensors, quantized_layer_names, ignored_layer_names)+ """+ if matcher is None:+ matcher = _PatternMatcher(+ ignore_patterns if ignore_patterns is not None else [],+ layer_config if layer_config is not None else {},+ default_scheme if default_scheme is not None else {},+ )++ output_tensors: dict[str, torch.Tensor] = {}+ quantized_layers: list[str] = []+ ignored_layers: list[str] = []++ if shard_path.endswith(".bin"):+ # PyTorch pickle checkpoint — load with weights_only where supported.+ try:+ raw_tensors = torch.load(shard_path, map_location="cpu", weights_only=True)+ except TypeError:+ # weights_only not available in older PyTorch versions+ raw_tensors = torch.load(shard_path, map_location="cpu") # nosec+ # Flatten nested state-dict wrappers if present.+ if not isinstance(raw_tensors, dict):+ raise ValueError(f"Expected a dict from {shard_path}, got {type(raw_tensors)}")+ else:+ from safetensors import safe_open++ with safe_open(shard_path, framework="pt", device="cpu") as f:+ raw_tensors = {name: f.get_tensor(name) for name in f.keys()}++ raw_tensors = split_fused_expert_tensors(raw_tensors)+ raw_tensors = _dequant_fp8_tensors(raw_tensors, block_size=fp8_block_size)++ for tensor_name in list(raw_tensors.keys()):+ tensor = raw_tensors.pop(tensor_name)+ _layer_name, out_dict, q_layer, ig_layer = _quantize_single_tensor(+ tensor_name,+ tensor,+ matcher,+ device,+ )+ output_tensors.update(out_dict)+ if q_layer:+ quantized_layers.append(q_layer)+ if ig_layer:+ ignored_layers.append(ig_layer)++ return output_tensors, quantized_layers, ignored_layers+++# ---------------------------------------------------------------------------+# Output writers+# ---------------------------------------------------------------------------+++def _build_quantization_config(+ default_scheme: dict,+ layer_config: dict,+ ignore_patterns: list[str],+ quantized_layers: list[str],+ ignored_layers: list[str],+ block_name_to_quantize: Optional[list[str]] = None,+) -> dict:+ """Build a quantization_config dict compatible with auto-round format."""+ from auto_round.version import __version__++ scheme_keys = [f.name for f in fields(QuantizationScheme)]++ qconfig = {+ "quant_method": "auto-round",+ "packing_format": "auto_round:auto_gptq",+ "bits": default_scheme["bits"],+ "group_size": default_scheme["group_size"],+ "sym": default_scheme.get("sym", True),+ "data_type": default_scheme.get("data_type", "int"),+ "iters": 0,+ "model_free": True,+ "autoround_version": __version__,+ }++ if block_name_to_quantize:+ qconfig["block_name_to_quantize"] = block_name_to_quantize++ extra_config = {}+ for layer_name, cfg in layer_config.items():+ if cfg.get("bits", default_scheme["bits"]) >= 16:+ extra_config[layer_name] = {k: cfg.get(k) for k in scheme_keys if cfg.get(k) is not None}+ continue+ differs = False+ for key in ("bits", "group_size", "sym"):+ if cfg.get(key) is not None and cfg[key] != default_scheme.get(key):+ differs = True+ break+ if differs:+ extra_config[layer_name] = {k: cfg.get(k) for k in scheme_keys if cfg.get(k) is not None}++ unique_ignored = list(dict.fromkeys(ignored_layers))+ for layer_name in unique_ignored:+ if layer_name not in extra_config:+ extra_config[layer_name] = {"bits": 16, "data_type": "float"}++ quantized_layer_set = set(quantized_layers)+ if "lm_head" in quantized_layer_set and "lm_head" not in extra_config:+ lm_head_cfg = layer_config.get("lm_head", default_scheme)+ extra_config["lm_head"] = {k: lm_head_cfg.get(k) for k in scheme_keys if lm_head_cfg.get(k) is not None}++ if extra_config:+ qconfig["extra_config"] = extra_config++ return qconfig+++def _write_output_shard(+ output_dir: str,+ shard_name: str,+ tensors: dict[str, torch.Tensor],+ weight_map: dict[str, str],+):+ """Write a single output shard and update the weight_map."""+ from safetensors.torch import save_file++ shard_path = os.path.join(output_dir, shard_name)++ # Detect shared-storage tensors (e.g. tie_word_embeddings: wte ↔ lm_head).+ # safetensors refuses to serialise them as-is; clone the duplicates so each+ # tensor occupies its own memory region. The first occurrence keeps the+ # original storage; subsequent aliases are cloned.+ seen_data_ptrs: set[int] = set()+ deduped: dict[str, torch.Tensor] = {}+ for k, v in tensors.items():+ if not v.is_contiguous():+ v = v.contiguous()+ ptr = v.data_ptr()+ if ptr in seen_data_ptrs:+ v = v.clone()+ else:+ seen_data_ptrs.add(ptr)+ deduped[k] = v++ save_file(deduped, shard_path)+ for tensor_name in tensors:+ weight_map[tensor_name] = shard_name+++def _write_index_file(output_dir: str, weight_map: dict[str, str]):+ """Write model.safetensors.index.json (or rename single shard)."""+ if len(set(weight_map.values())) <= 1:+ shard_names = list(set(weight_map.values()))+ if shard_names and shard_names[0] != "model.safetensors":+ src = os.path.join(output_dir, shard_names[0])+ dst = os.path.join(output_dir, "model.safetensors")+ if os.path.exists(src):+ os.rename(src, dst)+ weight_map = {k: "model.safetensors" for k in weight_map}+ return++ index = {"metadata": {"total_size": 0}, "weight_map": weight_map}+ with open(os.path.join(output_dir, "model.safetensors.index.json"), "w") as f:+ json.dump(index, f, indent=2)+++def _prefetch_shard(+ model_name_or_path: str,+ shard_name: str,+ work_dir: str,+ source_dir: str,+ streaming: bool,+) -> str | None:+ """Return the local path of the next shard (download if needed)."""+ try:+ if streaming:+ return _download_single_shard(model_name_or_path, shard_name, work_dir)+ path = os.path.join(source_dir, shard_name)+ return path if os.path.exists(path) else None+ except Exception as e: # pragma: no cover+ logger.warning(f"Prefetch failed for {shard_name}: {e}")+ return None+++# ---------------------------------------------------------------------------+# Scheme validation+# ---------------------------------------------------------------------------+++def _normalize_scheme(scheme: Union[str, QuantizationScheme]) -> QuantizationScheme:+ """Convert *scheme* to a :class:`QuantizationScheme` instance.++ Raises ``ValueError`` for unknown preset names and ``TypeError`` for+ unsupported types.+ """+ if isinstance(scheme, str):+ scheme_name = scheme.upper()+ if scheme_name not in PRESET_SCHEMES:+ raise ValueError(f"Unknown scheme '{scheme}'. Available: {list(PRESET_SCHEMES.keys())}")+ return preset_name_to_scheme(scheme_name)+ if isinstance(scheme, QuantizationScheme):+ return scheme+ raise TypeError(f"Unsupported scheme type: {type(scheme)}")+++def _apply_scheme_overrides(+ scheme: Union[str, QuantizationScheme],+ scheme_overrides: Optional[dict] = None,+) -> QuantizationScheme:+ """Return the effective scheme after applying non-None overrides."""+ scheme_obj = copy.deepcopy(_normalize_scheme(scheme))+ if not scheme_overrides:+ return scheme_obj++ valid_fields = {field.name for field in fields(QuantizationScheme)}+ for key, value in scheme_overrides.items():+ if key in valid_fields and value is not None:+ setattr(scheme_obj, key, value)+ return scheme_obj+++def _validate_supported_scheme(+ scheme_obj: QuantizationScheme,+ scheme_input: Union[str, QuantizationScheme],+) -> None:+ """Raise ``ValueError`` if *scheme_obj* is not supported by model-free.++ Model-free only supports integer weight-only quantization (sym/asym),+ packed in the ``auto_round:auto_gptq`` format.+ """+ act_bits = scheme_obj.act_bits if scheme_obj.act_bits is not None else 16+ if act_bits < 16:+ raise ValueError(+ f"Model-free mode only supports weight-only quantization (WOQ) schemes "+ f"where act_bits >= 16, but '{scheme_input}' has act_bits={act_bits}. "+ f"Supported preset schemes: {list(SUPPORTED_PRESET_SCHEMES)}."+ )++ data_type = (scheme_obj.data_type or "int").lower()+ if data_type != "int":+ raise ValueError(+ f"Model-free mode only supports integer weight quantization "+ f"(data_type='int'), but '{scheme_input}' has data_type='{data_type}'. "+ f"FP8 / MXFP / NVFP / GGUF / BF16 schemes require the standard "+ f"AutoRound flow. Supported preset schemes: "+ f"{list(SUPPORTED_PRESET_SCHEMES)}."+ )++ bits = scheme_obj.bits+ if bits is None or bits not in _SUPPORTED_INT_BITS:+ raise ValueError(+ f"Model-free mode supports bits in {_SUPPORTED_INT_BITS}, "+ f"but '{scheme_input}' requests bits={bits}. "+ f"Supported preset schemes: {list(SUPPORTED_PRESET_SCHEMES)}."+ )+++def is_model_free_supported_scheme(+ scheme: Union[str, QuantizationScheme],+ scheme_overrides: Optional[dict] = None,+) -> bool:+ """Return True if *scheme* can be quantized via model-free mode.++ Useful for CLI auto-routing logic. Never raises.+ """+ try:+ scheme_obj = _apply_scheme_overrides(scheme, scheme_overrides)+ _validate_supported_scheme(scheme_obj, scheme_obj)+ return True+ except (ValueError, TypeError):+ return False+++# ---------------------------------------------------------------------------+# Main class+# ---------------------------------------------------------------------------+++class _ModelFreeCompressorCore:+ """Class-based driver for model-free RTN quantization.++ The lifecycle is:++ 1. ``__init__`` — store user inputs.+ 2. :meth:`run` — perform validation, IO, quantization and writing.++ Internal helpers are split into focused methods so that the flow is+ readable end-to-end.++ Args:+ model_name_or_path: HuggingFace model ID or local directory path.+ output_dir: Directory to save the quantized model.+ scheme: Quantization scheme name (e.g. ``"W4A16"``) or a+ :class:`QuantizationScheme` instance.+ layer_config: Per-layer quantization overrides. Keys are layer+ names or regex patterns; values are dicts with ``bits``,+ ``group_size``, ``sym`` etc.+ ignore_layers: Comma-separated list of layer name patterns to keep+ in full precision.+ format: Output format (only ``"auto_round"`` is supported).+ device: Device for quantization computation (``"cpu"`` or+ ``"cuda"``).+ quant_lm_head: If True, quantize ``lm_head`` as well. By default+ ``lm_head`` and any layer containing ``embed`` are kept in+ full precision.+ quant_nontext_module: If True, quantize non-text modules+ (vision/audio/image) as well. By default these multimodal+ modules are kept in full precision.+ """++ SUPPORTED_FORMATS: tuple[str, ...] = ("auto_round",)++ def __init__(+ self,+ model_name_or_path: str,+ output_dir: str,+ scheme: Union[str, QuantizationScheme] = "W4A16",+ layer_config: Optional[dict] = None,+ ignore_layers: str = "",+ format: str = "auto_round",+ device: str = "cpu",+ quant_lm_head: bool = False,+ quant_nontext_module: bool = False,+ ):+ # --- raw inputs ---+ self.model_name_or_path = model_name_or_path+ self.output_dir = output_dir+ self.scheme_input = scheme+ self.layer_config_input = layer_config+ self.ignore_layers_input = ignore_layers+ self.format = format+ self.device = device+ self.quant_lm_head = quant_lm_head+ self.quant_nontext_module = quant_nontext_module++ # --- derived state populated during run() ---+ self.scheme_obj: QuantizationScheme | None = None+ self.default_scheme: dict = {}+ self.layer_config: dict = {}+ self.ignore_patterns: list[str] = []+ self.matcher: _PatternMatcher | None = None+ self.config: dict = {}+ self.fp8_block_size: list | None = None+ self.is_streaming: bool = False+ self.is_diffusion_model: bool = False+ self.diffusion_root_dir: str = ""+ self.work_dir: str = ""+ self.source_dir: str = ""+ self.shard_names: list[str] = []+ self.all_quantized_layers: list[str] = []+ self.all_ignored_layers: list[str] = []+ self.output_weight_map: dict[str, str] = {}++ # -------------------------------------------------------------------+ # Validation / parsing+ # -------------------------------------------------------------------++ def _validate_format(self) -> None:+ format_lower = self.format.lower().replace(" ", "").split(",")[0]+ if format_lower not in self.SUPPORTED_FORMATS:+ raise ValueError(+ f"Model-free mode only supports {self.SUPPORTED_FORMATS} format, "+ f"got '{self.format}'. Please use --format auto_round."+ )++ def _parse_scheme(self) -> None:+ self.scheme_obj = _normalize_scheme(self.scheme_input)+ _validate_supported_scheme(self.scheme_obj, self.scheme_input)+ ds = asdict(self.scheme_obj)+ self.default_scheme = {k: v for k, v in ds.items() if v is not None}++ def _parse_layer_config(self) -> None:+ lc = copy.deepcopy(self.layer_config_input) if self.layer_config_input else {}++ # Append '.' to keys ending with a digit to avoid partial numeric matches.+ for key in list(lc.keys()):+ if key and key[-1].isdigit():+ lc[key + "."] = lc.pop(key)++ # Normalize values to dicts.+ for key, val in list(lc.items()):+ if isinstance(val, str):+ parsed = asdict(preset_name_to_scheme(val.upper()))+ lc[key] = {k: v for k, v in parsed.items() if v is not None}+ elif isinstance(val, QuantizationScheme):+ lc[key] = {k: v for k, v in asdict(val).items() if v is not None}+ elif isinstance(val, dict):+ # Resolve 'scheme' key inside dict values, e.g. {'scheme': 'W2A16'}+ if "scheme" in val:+ scheme_val = val.pop("scheme")+ if isinstance(scheme_val, str):+ parsed = asdict(preset_name_to_scheme(scheme_val.upper()))+ resolved = {k: v for k, v in parsed.items() if v is not None}+ elif isinstance(scheme_val, QuantizationScheme):+ resolved = {k: v for k, v in asdict(scheme_val).items() if v is not None}+ else:+ resolved = {}+ # Explicit keys in val override the resolved scheme values+ resolved.update(val)+ lc[key] = resolved+ else:+ raise TypeError(f"Unsupported layer_config value type for '{key}': {type(val)}")++ self.layer_config = lc++ def _build_ignore_patterns(self) -> None:+ ignore_patterns: list[str] = []+ if self.ignore_layers_input:+ ignore_patterns = [p.strip() for p in self.ignore_layers_input.replace(" ", "").split(",") if p.strip()]+ ignore_patterns = [p + "." if p and p[-1].isdigit() else p for p in ignore_patterns]++ if not self.quant_lm_head and "lm_head" not in ignore_patterns:+ ignore_patterns.append("lm_head")+ ignore_patterns.append("head") # for deepseek v4++ if not self.quant_nontext_module:+ for kw in _NONTEXT_KEYWORDS:+ if kw not in ignore_patterns:+ ignore_patterns.append(kw)++ self.ignore_patterns = ignore_patterns++ # -------------------------------------------------------------------+ # Source resolution and discovery+ # -------------------------------------------------------------------++ def _resolve_source(self) -> None:+ os.makedirs(self.output_dir, exist_ok=True)+ self.is_streaming = not _is_model_cached(self.model_name_or_path)+ if self.is_streaming:+ logger.info("Model not found locally or in cache — using streaming download mode.")+ self.work_dir = self.output_dir+ _download_metadata_files(self.model_name_or_path, self.work_dir)+ transformer_work_dir = os.path.join(self.work_dir, "transformer")+ if (+ not os.path.exists(os.path.join(self.work_dir, "config.json"))+ and os.path.isdir(transformer_work_dir)+ and os.path.exists(os.path.join(transformer_work_dir, "config.json"))+ ):+ self.is_diffusion_model = True+ self.diffusion_root_dir = self.work_dir+ self.work_dir = transformer_work_dir+ logger.info(+ "Detected diffusion model (no root config.json, found transformer/ subfolder). "+ "Only the transformer component will be quantized; other sub-components are skipped."+ )+ self.config = _load_config(self.work_dir)+ else:+ self.source_dir = _resolve_source_dir(self.model_name_or_path)+ transformer_source_dir = os.path.join(self.source_dir, "transformer")+ if (+ not os.path.exists(os.path.join(self.source_dir, "config.json"))+ and os.path.isdir(transformer_source_dir)+ and os.path.exists(os.path.join(transformer_source_dir, "config.json"))+ ):+ self.is_diffusion_model = True+ self.diffusion_root_dir = self.source_dir+ self.source_dir = transformer_source_dir+ logger.info(+ "Detected diffusion model (no root config.json, found transformer/ subfolder). "+ "Only the transformer component will be quantized; other sub-components are skipped."+ )+ self.config = _load_config(self.source_dir)++ def _check_conv1d_and_embedding(self) -> None:+ """Detect Conv1d and embedding layers and automatically add them to the ignore list."""+ local_dir = self.work_dir if self.is_streaming else self.source_dir+ if not local_dir or not os.path.isdir(local_dir):+ return++ try:+ from auto_round.utils.model import find_layers_from_config++ incompatible = find_layers_from_config(local_dir, class_names=["Embedding", "Conv1d", "Conv1D"])++ if incompatible:+ # Group by class for a cleaner warning message+ incompatible_layers = []+ for cls, layers in incompatible.items():+ incompatible_layers.extend(layers)+ summary = ", ".join(f"{cls}({len(layers)})" for cls, layers in sorted(incompatible.items()))+ self.ignore_patterns.extend(incompatible_layers)+ logger.warning(+ f"Detected {len(incompatible)} layer(s) incompatible with model-free RTN"+ f": {compress_layer_names(incompatible_layers)}.\n"+ f"These layers have been automatically added to ignore_layers "+ f"and will be kept in full precision.\n"+ f"To override, pass --ignore_layers explicitly or disable "+ f"model-free mode (remove --model_free)."+ )++ except Exception as exc:+ logger.warning(+ f"Could not check model architecture for incompatible layers: {exc}.\n"+ f"Models with Embedding or Conv1d layers may be incorrectly quantized "+ f"in model-free mode (non-2D weights cannot be packed by the RTN kernel).\n"+ f"If affected, either disable model-free mode (remove --model_free) or "+ f"add those layers to --ignore_layers."+ )++ def _apply_predefined_ignore_layers(self) -> None:+ predefined = get_predefined_ignore_layers_from_config(self.config)+ if predefined:+ logger.info(f"Using predefined ignore_layers from config: " f"{compress_layer_names(predefined)}")+ self.ignore_patterns.extend(predefined)++ def _detect_fp8_source(self) -> None:+ quant_config = self.config.get("quantization_config", {})+ is_fp8 = (+ quant_config.get("quant_method") == "fp8"+ or quant_config.get("quantization_type") == "fp8"+ or quant_config.get("fmt", "").startswith("e4m3")+ )+ if is_fp8:+ self.fp8_block_size = quant_config.get("weight_block_size")+ logger.info(+ f"Detected FP8 source model (block_size={self.fp8_block_size}, "+ f"scale_fmt={quant_config.get('scale_fmt', 'N/A')}). "+ f"FP8 weights will be dequantized before quantization."+ )++ def _discover_shards(self) -> None:+ search_dir = self.work_dir if self.is_streaming else self.source_dir+ self.shard_names = _list_weight_shards(search_dir)++ def _build_matcher(self) -> None:+ self.matcher = _PatternMatcher(+ self.ignore_patterns,+ self.layer_config,+ self.default_scheme,+ )++ @property+ def _quant_output_dir(self) -> str:+ """Effective output directory for quantized weight shards and config.++ For diffusion models the quantized transformer component is written+ to ``<output_dir>/transformer/``; for all other models the top-level+ ``output_dir`` is used directly.+ """+ if self.is_diffusion_model:+ return os.path.join(self.output_dir, "transformer")+ return self.output_dir++ # -------------------------------------------------------------------+ # Shard processing pipeline+ # -------------------------------------------------------------------++ def _process_all_shards(self) -> None:+ try:+ from tqdm import tqdm as _tqdm+ except ImportError:+ _tqdm = None++ prefetch_pool = ThreadPoolExecutor(max_workers=1)+ write_pool = ThreadPoolExecutor(max_workers=1)+ prefetch_future = None+ write_future = None++ if self.shard_names:+ prefetch_future = prefetch_pool.submit(+ _prefetch_shard,+ self.model_name_or_path,+ self.shard_names[0],+ self.work_dir,+ self.source_dir,+ self.is_streaming,+ )++ shard_iter = (+ _tqdm(+ enumerate(self.shard_names),+ total=len(self.shard_names),+ desc="Processing shards",+ unit="shard",+ )+ if _tqdm+ else enumerate(self.shard_names)+ )++ for shard_idx, shard_name in shard_iter:+ shard_path = prefetch_future.result() if prefetch_future else None++ # Kick off prefetch of the next shard.+ if shard_idx + 1 < len(self.shard_names):+ prefetch_future = prefetch_pool.submit(+ _prefetch_shard,+ self.model_name_or_path,+ self.shard_names[shard_idx + 1],+ self.work_dir,+ self.source_dir,+ self.is_streaming,+ )+ else:+ prefetch_future = None++ if shard_path is None or not os.path.exists(shard_path):+ logger.warning(f"Shard not found: {shard_name}, skipping")+ continue++ output_tensors, quantized, ignored = _process_shard(+ shard_path=shard_path,+ device=self.device,+ matcher=self.matcher,+ fp8_block_size=self.fp8_block_size,+ )+ memory_monitor.update()+ clear_memory()+ if len(self.shard_names) > 1:+ logger.info(f"Memory usage: {memory_monitor.get_summary()}")++ self.all_quantized_layers.extend(quantized)+ self.all_ignored_layers.extend(ignored)++ if write_future is not None:+ write_future.result()++ os.makedirs(self._quant_output_dir, exist_ok=True)+ out_shard_name = f"model-{shard_idx + 1:05d}-of-{len(self.shard_names):05d}.safetensors"+ write_future = write_pool.submit(+ _write_output_shard,+ self._quant_output_dir,+ out_shard_name,+ output_tensors,+ self.output_weight_map,+ )++ if self.is_streaming:+ try:+ os.remove(shard_path)+ logger.debug(f"Deleted source shard: {shard_path}")+ except OSError as e:+ logger.warning(f"Could not delete source shard {shard_path}: {e}")++ prefetch_pool.shutdown(wait=False)+ if write_future is not None:+ write_future.result()+ write_pool.shutdown(wait=True)++ # -------------------------------------------------------------------+ # Output+ # -------------------------------------------------------------------++ def _write_index(self) -> None:+ _write_index_file(self._quant_output_dir, self.output_weight_map)++ def _write_config_files(self) -> None:+ quantization_config = _build_quantization_config(+ default_scheme=self.default_scheme,+ layer_config=self.layer_config,+ ignore_patterns=self.ignore_patterns,+ quantized_layers=self.all_quantized_layers,+ ignored_layers=self.all_ignored_layers,+ )++ self.config["quantization_config"] = quantization_config+ os.makedirs(self._quant_output_dir, exist_ok=True)+ with open(os.path.join(self._quant_output_dir, "config.json"), "w") as f:+ json.dump(self.config, f, indent=2)++ with open(os.path.join(self._quant_output_dir, "quantization_config.json"), "w") as f:+ json.dump(quantization_config, f, indent=2)++ def _copy_metadata_files(self) -> None:+ if self.is_streaming:+ # Metadata was downloaded directly to output_dir (or output_dir/transformer/+ # for diffusion models) — nothing to copy or clean up.+ return++ if self.is_diffusion_model:+ # For diffusion models, copy only root-level non-weight metadata files+ # (model_index.json, tokenizer files, etc.) to output_dir.+ # Sub-components other than transformer (vae, scheduler, …) are skipped+ # per the "only quantize transformer" policy; the quantized transformer+ # component is already written to output_dir/transformer/ by the pipeline.+ for fname in os.listdir(self.diffusion_root_dir):+ src = os.path.join(self.diffusion_root_dir, fname)+ dst = os.path.join(self.output_dir, fname)+ if os.path.isdir(src):+ continue+ if os.path.isfile(src) and not os.path.exists(dst):+ shutil.copy2(src, dst)+ return++ for fname in os.listdir(self.source_dir):+ if _is_weight_shard(fname):+ continue+ src = os.path.join(self.source_dir, fname)+ dst = os.path.join(self.output_dir, fname)+ if os.path.isdir(src):+ if not os.path.exists(dst):+ shutil.copytree(src, dst)+ elif os.path.isfile(src) and not os.path.exists(dst):+ shutil.copy2(src, dst)++ def _log_summary(self, total_time: float) -> None:+ compressed_quantized = compress_layer_names(self.all_quantized_layers)+ compressed_ignored = compress_layer_names(list(dict.fromkeys(self.all_ignored_layers)))+ logger.info(+ f"\nModel-free quantization complete.\n"+ f" Output directory: {self.output_dir}\n"+ f" Total time: {total_time:.2f} seconds\n"+ f" Memory usage: {memory_monitor.get_summary()}\n"+ f" Quantized layers ({len(self.all_quantized_layers)}): "+ f"{compressed_quantized}\n"+ f" Ignored layers ({len(set(self.all_ignored_layers))}): "+ f"{compressed_ignored}\n"+ )++ # -------------------------------------------------------------------+ # Public entry point+ # -------------------------------------------------------------------++ def run(self) -> str:+ """Execute the full model-free quantization pipeline.++ Returns:+ Absolute path to the output directory.+ """+ # ---- preflight ----+ self._validate_format()+ self._parse_scheme()+ self._parse_layer_config()+ self._build_ignore_patterns()++ # ---- source resolution ----+ self._resolve_source()+ self._check_conv1d_and_embedding()+ self._apply_predefined_ignore_layers()+ self._build_matcher()+ self._detect_fp8_source()+ self._discover_shards()++ logger.info(+ f"Model-free quantization: {self.model_name_or_path}\n"+ f" Scheme: {self.scheme_obj}\n"+ f" Output: {self.output_dir}\n"+ f" Shards: {len(self.shard_names)}\n"+ f" Streaming download: {self.is_streaming}\n"+ f" Diffusion model: {self.is_diffusion_model}\n"+ f" Quant lm_head: {self.quant_lm_head}\n"+ f" Quant nontext module: {self.quant_nontext_module}\n"+ f" Device: {self.device}"+ )++ start_time = time.time()+ memory_monitor.reset()++ # ---- main loop ----+ self._process_all_shards()++ # ---- write outputs ----+ self._write_index()+ self._write_config_files()+ self._copy_metadata_files()++ self._log_summary(time.time() - start_time)+ return self.output_dir+++# ---------------------------------------------------------------------------+# AutoRound-compatible compressor: ModelFreeCompressor doubles as the+# compressor object returned by AutoRound.__new__ when model-free mode is+# selected. It owns both the quantization pipeline (run()) AND the+# AutoRound-facing interface (quantize_and_save()).+# ---------------------------------------------------------------------------+++class ModelFreeCompressor(_ModelFreeCompressorCore):+ """Model-free RTN quantizer that also acts as an AutoRound compressor.++ When constructed via ``AutoRound(model_free=True, ...)`` the instance is+ returned directly from ``AutoRound.__new__``. The caller then invokes+ :meth:`quantize_and_save` exactly as they would on any other compressor.++ When used as a pure-quantization driver (CLI / functional API) call+ :meth:`run` instead.++ Args:+ model_name_or_path: HuggingFace model ID or local directory path.+ In the AutoRound compressor role this is the ``model`` argument.+ output_dir: Where to write the quantized model. May be ``None``+ when used as a compressor (output_dir is passed to+ :meth:`quantize_and_save` later).+ scheme: Quantization scheme name or :class:`QuantizationScheme`.+ layer_config: Per-layer overrides.+ ignore_layers: Comma-separated layer name patterns to skip.+ format: Output format (only ``"auto_round"`` is supported).+ device: Compute device.+ quant_lm_head: Whether to quantize ``lm_head``.+ quant_nontext_module: Whether to quantize non-text modules.+ **kwargs: When called from ``AutoRound.__new__`` the full AutoRound+ kwargs are forwarded here. Unknown kwargs are silently ignored+ so that calibration-only parameters (``nsamples``, ``iters``,+ ``dataset``, …) do not cause errors.+ """++ def __init__(+ self,+ model_name_or_path: str,+ output_dir: Optional[str] = None,+ scheme: Union[str, QuantizationScheme] = "W4A16",+ layer_config: Optional[dict] = None,+ ignore_layers: str = "",+ format: str = "auto_round",+ device: str = "cpu",+ quant_lm_head: bool = False,+ quant_nontext_module: bool = False,+ # --- AutoRound compressor-role aliases ---+ tokenizer=None,+ device_map=None,+ **kwargs,+ ):+ import copy+ from dataclasses import fields as dc_fields++ fallback_kwargs = dict(kwargs)++ # Collect per-field scheme overrides forwarded from AutoRound+ # (e.g. bits=4, sym=False passed as individual kwargs).+ self.user_scheme_overrides: dict = {}+ for field in dc_fields(QuantizationScheme):+ if field.name in kwargs:+ val = kwargs.pop(field.name)+ if val is not None:+ self.user_scheme_overrides[field.name] = val++ # Resolve device: AutoRound passes device_map; the core API uses device.+ if device_map is not None:+ from auto_round.utils import detect_device++ device = detect_device(device_map)++ # Initialise the core quantizer+ super().__init__(+ model_name_or_path=model_name_or_path,+ output_dir=output_dir or "tmp_autoround",+ scheme=scheme,+ layer_config=layer_config,+ ignore_layers=ignore_layers,+ format=format,+ device=device,+ quant_lm_head=quant_lm_head,+ quant_nontext_module=quant_nontext_module,+ )++ # Compressor-role state (mirrors BaseCompressor attributes used by+ # AutoRound's post-processing code)+ self._output_dir_override: Optional[str] = None # set by quantize_and_save+ self.model = None+ self.tokenizer = tokenizer+ self.model_free = True+ self.model_free_path = model_name_or_path+ self.iters = 0+ self.disable_opt_rtn = True+ self.formats = None+ self.quantized = False+ self._fallback_compressor = None+ # Start from the remaining user kwargs and explicitly set/override+ # known compressor init parameters for clarity.+ fallback_init = dict(fallback_kwargs)+ fallback_init.update(+ model=model_name_or_path,+ iters=0,+ disable_opt_rtn=True,+ tokenizer=tokenizer,+ scheme=copy.deepcopy(scheme),+ layer_config=copy.deepcopy(layer_config),+ ignore_layers=ignore_layers,+ device_map=device_map,+ quant_lm_head=quant_lm_head,+ )++ self._fallback_init_kwargs = fallback_init+ if quant_nontext_module:+ self._fallback_init_kwargs["quant_nontext_module"] = quant_nontext_module+ # remaining kwargs intentionally consumed/ignored++ def _fallback_to_base_compressor(self):+ from auto_round.autoround import AutoRound++ logger.info(+ "Format '%s' is not supported by model-free mode; falling back to the regular AutoRound flow.",+ format,+ )+ logger.info(+ "fallbacked_init_kwargs: %s",+ self._fallback_init_kwargs,+ )+ compressor = AutoRound(**self._fallback_init_kwargs, disable_model_free=True)+ self._fallback_compressor = compressor++ def _fallback_to_quantize_and_save(+ self,+ output_dir: str,+ format: str,+ inplace: bool,+ **kwargs,+ ):+ self._fallback_to_base_compressor()+ return self._fallback_compressor.quantize_and_save( # pylint: disable=E1101+ output_dir=output_dir, format=format, inplace=inplace, **kwargs+ )++ def quantize(+ self,+ ):+ """fallback to base compressor's quantize."""+ self._fallback_to_base_compressor()+ return self._fallback_compressor.quantize() # pylint: disable=E1101++ def __getattribute__(self, name: str):+ """Prefer attributes from the fallback compressor when available.++ Once model-free flow falls back to the regular AutoRound compressor,+ external attribute reads on this wrapper should observe the fallback+ compressor's state first.+ """+ local_only_names = {+ "_fallback_compressor",+ "_fallback_init_kwargs",+ "_fallback_to_base_compressor",+ "__dict__",+ "__class__",+ "__getattribute__",+ "__setattr__",+ "__delattr__",+ }++ if name in local_only_names or name.startswith("__"):+ return super().__getattribute__(name)++ fallback = super().__getattribute__("__dict__").get("_fallback_compressor")+ if fallback is not None:+ if name == "compressor":+ return fallback+ try:+ return getattr(fallback, name)+ except AttributeError:+ pass++ return super().__getattribute__(name)++ # ------------------------------------------------------------------+ # AutoRound compressor interface+ # ------------------------------------------------------------------++ def quantize_and_save(+ self,+ output_dir: str = "tmp_autoround",+ format: str = "auto_round",+ inplace: bool = True,+ **kwargs,+ ):+ """Quantize and save — AutoRound compressor entry point."""+ if format not in ["auto_round", "auto_round:auto_gptq"]:+ return self._fallback_to_quantize_and_save(output_dir=output_dir, format=format, inplace=inplace, **kwargs)++ # Apply user scheme overrides before running+ if self.user_scheme_overrides:+ self.scheme_input = _apply_scheme_overrides(self.scheme_input, self.user_scheme_overrides)++ # Temporarily point output_dir at what the caller requested+ orig = self.output_dir+ self.output_dir = output_dir+ out_path = self.run()+ self.output_dir = orig+ self.quantized = True+ return None, out_pathdiff --git a/auto_round/compressors_new/entry.py b/auto_round/compressors_new/entry.pyindex 7d519e007..e26691528 100644--- a/auto_round/compressors_new/entry.py+++ b/auto_round/compressors_new/entry.py@@ -389,6 +389,29 @@ class AutoRoundCompatible:This method translates old AutoRoundCompatible API to new AutoRound API."""from auto_round.utils import is_diffusion_model, is_mllm_model+ from auto_round.utils.model import is_model_free_route++ # ---- Model-free fast-path detection --------------------------------+ if is_model_free_route(model, scheme, iters, kwargs.get("disable_opt_rtn"), kwargs):+ from auto_round.compressors_new.model_free import ModelFreeCompressor++ if not isinstance(model, str):+ raise ValueError("model_free=True requires `model` to be a HuggingFace ID or local path string.")+ if not bool(kwargs.get("model_free", False)):+ logger.info(+ "Auto-routing to model-free quantization "+ "(iters=0, disable_opt_rtn=True, supported scheme). "+ "Pass disable_model_free=True to use the regular flow."+ )+ return ModelFreeCompressor(+ model_name_or_path=model,+ scheme=scheme,+ layer_config=layer_config,+ tokenizer=tokenizer,+ device_map=device_map,+ **kwargs,+ )+ # --------------------------------------------------------------------common_config_kwargs, auto_round_config_kwargs = cls._pop_config_kwargs(kwargs)diff --git a/auto_round/compressors_new/model_free.py b/auto_round/compressors_new/model_free.pynew file mode 120000index 000000000..de605bdd6--- /dev/null+++ b/auto_round/compressors_new/model_free.py@@ -0,0 +1 @@+../compressors/model_free.py\ No newline at end of filediff --git a/auto_round/compressors_new/utils.py b/auto_round/compressors_new/utils.pyindex fcebb45d7..864168ff3 100644--- a/auto_round/compressors_new/utils.py+++ b/auto_round/compressors_new/utils.py@@ -343,6 +343,12 @@ def set_layer_config(# "in_blocks" is an internal bookkeeping key injected by LLM-Compressor;# silently drop it before validation.item = {k: v for k, v in item.items() if k != "in_blocks"}+ # Support "scheme" key inside dict: resolve the preset and merge overrides.+ if "scheme" in item:+ scheme_name = item.pop("scheme")+ base = asdict(preset_name_to_scheme(scheme_name.upper()))+ base.update(item)+ item = baseinvalid = set(item) - set(scheme_keys + ("fixed_by_user", "scale_dtype"))if invalid:raise ValueError(diff --git a/auto_round/utils/common.py b/auto_round/utils/common.pyindex 792f113a6..996ef8f47 100644--- a/auto_round/utils/common.py+++ b/auto_round/utils/common.py@@ -985,52 +985,69 @@ def parse_layer_config_arg(s: str) -> dict:def compress_layer_names(names: list) -> str:- """Compress numbered layer names, e.g. layer.0, layer.1, layer.2 → layer.[0-2]."""- import re as _re+ """Compress numbered layer names, e.g. layer.0, layer.1, layer.2 → layer.[0-2].++ Runs the compression pass repeatedly until stable so that multi-level+ numbering is fully reduced. For example::- # group by (prefix, suffix) where the number varies+ layers.0.experts.[0-255].down_proj+ layers.10.experts.[0-255].down_proj+ → layers.[0,10].experts.[0-255].down_proj+ """+ import re as _refrom collections import defaultdict- groups: dict = defaultdict(list)- singles: list = []- for name in names:- m = _re.match(r"^(.*\.)?(\d+)(\..+)?$", name)- if m:- prefix = m.group(1) or ""- num = int(m.group(2))- suffix = m.group(3) or ""- groups[(prefix, suffix)].append(num)- else:- singles.append(name)- parts: list = []- for (prefix, suffix), nums in groups.items():- nums_sorted = sorted(set(nums))- if len(nums_sorted) == 1:- parts.append(f"{prefix}{nums_sorted[0]}{suffix}")- else:- # Build comma-separated contiguous ranges, e.g. [0,2-3,5]- ranges = []- start = prev = nums_sorted[0]- for n in nums_sorted[1:]:- if n == prev + 1:- prev = n- continue- # Close the current range+ def _compress_once(name_list: list) -> list:+ groups: dict = defaultdict(list)+ singles: list = []+ for name in name_list:+ m = _re.match(r"^(.*\.)?(\d+)(\..+)?$", name)+ if m:+ prefix = m.group(1) or ""+ num = int(m.group(2))+ suffix = m.group(3) or ""+ groups[(prefix, suffix)].append(num)+ else:+ singles.append(name)+ parts: list = []+ for (prefix, suffix), nums in groups.items():+ nums_sorted = sorted(set(nums))+ if len(nums_sorted) == 1:+ parts.append(f"{prefix}{nums_sorted[0]}{suffix}")+ else:+ # Build comma-separated contiguous ranges, e.g. [0,2-3,5]+ ranges = []+ start = prev = nums_sorted[0]+ for n in nums_sorted[1:]:+ if n == prev + 1:+ prev = n+ continue+ # Close the current range+ if start == prev:+ ranges.append(str(start))+ else:+ ranges.append(f"{start}-{prev}")+ start = prev = n+ # Close the final rangeif start == prev:ranges.append(str(start))else:ranges.append(f"{start}-{prev}")- start = prev = n- # Close the final range- if start == prev:- ranges.append(str(start))- else:- ranges.append(f"{start}-{prev}")- range_str = ",".join(ranges)- parts.append(f"{prefix}[{range_str}]{suffix}")- parts.extend(singles)- parts.sort()- return ", ".join(parts)+ range_str = ",".join(ranges)+ parts.append(f"{prefix}[{range_str}]{suffix}")+ parts.extend(singles)+ return parts++ current = list(names)+ while True:+ compressed = _compress_once(current)+ # Stop when no further merging occurred+ if len(compressed) >= len(current):+ break+ current = compressed++ current.sort()+ return ", ".join(current)def infer_bits_by_data_type(data_type: str):diff --git a/auto_round/utils/missing_tensors.py b/auto_round/utils/missing_tensors.pyindex b67372b34..255b7f92f 100644--- a/auto_round/utils/missing_tensors.py+++ b/auto_round/utils/missing_tensors.py@@ -528,80 +528,64 @@ def quantize_weight_rtn(weight = torch.nn.functional.pad(weight, (0, 0, 0, out_pad))padded_out = weight.shape[0]- # Reshape for group-wise ops: [padded_out, num_groups, group_size]- w_grouped = weight.reshape(padded_out, num_groups, group_size)+ # Use quantization functions from auto_round/data_type/int.py+ from auto_round.data_type.int import quant_tensor_asym, quant_tensor_rtn_sym+ from auto_round.data_type.utils import reshape_pad_tensor_by_group_sizeq_scale_thresh = 1e-5 # match quant_tensor_sym / quant_tensor_asym thresholdif sym:- # Full-range symmetric quantization — aligned with quant_tensor_sym- # in auto_round/data_type/int.py:- # maxq = 2^(bits-1) (e.g. 8 for 4-bit)- # scale = signed_max(wmin, wmax) / maxq (can be negative)- # q = clamp(round(w / scale), -maxq, maxq - 1)+ # Full-range symmetric quantization via quant_tensor_rtn_symmaxq = 1 << (bits - 1) # e.g. 8 for 4-bitzero_point = maxq # unsigned offset for packing- w_min = w_grouped.amin(dim=-1, keepdim=True)- w_max = w_grouped.amax(dim=-1, keepdim=True)- # clamp so that 0 is always representable- w_min = torch.clamp(w_min, max=0)- w_max = torch.clamp(w_max, min=0)- wmin_abs = -w_min- wmax_abs = w_max- # signed max: preserves the sign of the dominant extreme- max_v = (2 * (wmax_abs < wmin_abs).int() - 1) * torch.max(wmax_abs, wmin_abs)- del w_min, w_max, wmin_abs, wmax_abs-- scales_full = (max_v / maxq).to(torch.float16)- # Clamp small scales for numerical stability (match q_scale_thresh)- scales_full = torch.where(- scales_full < 0,- torch.clamp(scales_full, max=-q_scale_thresh),- torch.clamp(scales_full, min=q_scale_thresh),- )- scales_full = scales_full.float() # compute in fp32- del max_v+ # quant_tensor_rtn_sym returns (qdq_result, scale, maxq)+ # scale shape: [padded_out * num_groups, 1]+ _, scale, _ = quant_tensor_rtn_sym(weight, bits=bits, group_size=group_size)++ # Reshape weight for group-wise quantization: [padded_out * num_groups, group_size]+ w_grouped, _, _ = reshape_pad_tensor_by_group_size(weight, group_size)+ w_grouped = w_grouped.to(device=device, dtype=torch.float32)+ del weight- # In-place ops to minimise peak GPU memory.- q = w_grouped / scales_full- del weight, w_grouped # free the (potentially large) float32 weight storage- q.round_().clamp_(-maxq, maxq - 1)+ # Compute integer values for packing+ q = (w_grouped / scale).round_().clamp_(-maxq, maxq - 1)+ del w_groupedq += zero_point # shift to unsigned [0, 2*maxq - 1]- q = q.to(torch.int32) # new int32 tensor; float32 q is freed+ q = q.to(torch.int32)++ # scale → [num_groups, padded_out] (float16)+ scales_out = scale.squeeze(-1).reshape(padded_out, num_groups).t().contiguous().to(torch.float16)+ del scalezp = torch.full((num_groups, padded_out), zero_point, dtype=torch.int32, device=device)else:- # Asymmetric quantization — aligned with quant_tensor_asym- # in auto_round/data_type/int.py:- # maxq = 2^bits - 1- # wmin = clamp(min, max=0), wmax = clamp(max, min=0)- # scale = (wmax - wmin) / maxq- # zp = round(-wmin / scale)- # q = clamp(round(w / scale) + zp, 0, maxq)+ # Asymmetric quantization via quant_tensor_asymmax_int = (1 << bits) - 1- w_min = torch.clamp(w_grouped.amin(dim=-1, keepdim=True), max=0)- w_max = torch.clamp(w_grouped.amax(dim=-1, keepdim=True), min=0)-- scales_full = ((w_max - w_min) / max_int).to(torch.float16)- scales_full = torch.clamp(scales_full, min=q_scale_thresh)- scales_full = scales_full.float() # compute in fp32- zp_full = torch.round(-w_min / scales_full).clamp_(0, max_int).to(torch.int32)- del w_min, w_max-- q = w_grouped / scales_full- del weight, w_grouped- q.round_() # round FIRST (matches regular path: round(w/scale) then + zp)- q += zp_full.float() # then add zp++ # quant_tensor_asym returns (qdq_result, scale, zp)+ # scale shape: [padded_out * num_groups, 1], zp shape: [padded_out * num_groups, 1]+ _, scale, zp_val = quant_tensor_asym(weight, bits=bits, group_size=group_size)++ # Reshape weight for group-wise quantization+ w_grouped, _, _ = reshape_pad_tensor_by_group_size(weight, group_size)+ w_grouped = w_grouped.to(device=device, dtype=torch.float32)+ del weight++ # Compute integer values for packing+ q = (w_grouped / scale).round_()+ del w_grouped+ q += zp_valq.clamp_(0, max_int)q = q.to(torch.int32)- zp = zp_full.squeeze(-1).t().contiguous() # [num_groups, padded_out]- del zp_full+ # scale → [num_groups, padded_out] (float16)+ scales_out = scale.squeeze(-1).reshape(padded_out, num_groups).t().contiguous().to(torch.float16)+ del scale- # scales → [num_groups, padded_out] (float16)- scales_out = scales_full.squeeze(-1).t().contiguous().to(torch.float16)- del scales_full+ # zp → [num_groups, padded_out]+ zp = zp_val.squeeze(-1).reshape(padded_out, num_groups).t().contiguous().to(torch.int32)+ del zp_val# q → [in_features, padded_out]q = q.reshape(padded_out, in_features).t().contiguous()diff --git a/auto_round/utils/model.py b/auto_round/utils/model.pyindex 04a8b0395..575b7ada0 100644--- a/auto_round/utils/model.py+++ b/auto_round/utils/model.py@@ -2018,3 +2018,121 @@ def hook_ngram_embeddings_on_cpu(model):hook_input_output_device_for_cpu_module(raw_ngram_embeddings)return has_ngram_embeddings, raw_ngram_embeddings if has_ngram_embeddings else None+++def is_model_free_route(+ model,+ scheme,+ iters: int,+ disable_opt_rtn,+ kwargs: dict,+) -> bool:+ """Return True when the model-free fast-path should be taken.++ Mirrors the ``is_diffusion_model`` / ``is_mllm_model`` helpers used in+ ``AutoRound.__new__`` to select the right compressor class.++ Model-free mode is activated when **either** of the following holds:++ * ``model_free=True`` is explicitly set in *kwargs*.+ * All of the following are true:++ - ``disable_model_free`` is not set (or False) in *kwargs*+ - *model* is a string (HF hub ID or local path)+ - *iters* == 0+ - *disable_opt_rtn* is exactly ``True``+ - *scheme* is a supported model-free preset++ Note: this function only *reads* kwargs; it does **not** pop any keys.+ """+ from auto_round.compressors.model_free import is_model_free_supported_scheme++ explicit = bool(kwargs.pop("model_free", False))+ disabled = bool(kwargs.pop("disable_model_free", False))+ if explicit:+ return True+ # Only auto-route when format is auto_round (or not specified).+ fmt = kwargs.get("format", "auto_round")+ if fmt is None:+ fmt = "auto_round"+ fmt_first = str(fmt).lower().replace(" ", "").split(",")[0]+ if fmt_first != "auto_round":+ return False+ return (+ not disabled+ and isinstance(model, str)+ and iters == 0+ and disable_opt_rtn is True+ and is_model_free_supported_scheme(scheme, kwargs)+ )+++def find_layers_from_config(model_dir: str, class_names: list[str] | None = None) -> dict[str, str]:+ """Detect layers of given class names by loading the model on ``device='meta'``.++ Only ``config.json`` is required — no weights are read.++ For regular models the root directory is checked. For diffusion-style+ repos (no root ``config.json`` but a ``transformer/`` subfolder), only the+ ``transformer/`` subfolder is checked — other sub-components (``vae/``,+ ``scheduler/``, …) are intentionally skipped because only the transformer+ is quantized in model-free mode.++ Args:+ model_dir: Local directory containing ``config.json``, or a diffusion+ repo root whose ``transformer/`` subfolder contains ``config.json``.+ class_names: Class names to look for, matched against+ ``type(module).__name__``. Defaults to+ ``["Embedding", "Conv1d", "Conv1D"]`` — the types incompatible+ with model-free RTN packing.++ Returns:+ ``{class_name: [layer_name, ...]}`` for every matched module.+ Returns an empty dict on any failure.+ """+ from huggingface_hub import snapshot_download+ from transformers import AutoConfig, AutoModel++ if class_names is None:+ class_names = ["Embedding", "Conv1d", "Conv1D"]+ if isinstance(class_names, str):+ class_names = [class_names]+ target = set(class_names)++ # download if not local, but only the config files (fast)+ if not os.path.exists(model_dir):+ model_dir = snapshot_download(+ repo_id=model_dir,+ allow_patterns=["**/config.json"],+ )++ # Build the list of (prefix, config_dir) pairs to inspect.+ # For diffusion repos (no root config.json) only check transformer/.+ # For regular repos only check the root directory.+ dirs: list[tuple[str, str]] = []+ if os.path.exists(os.path.join(model_dir, "config.json")):+ dirs.append(("", model_dir))+ else:+ transformer_dir = os.path.join(model_dir, "transformer")+ if os.path.isdir(transformer_dir) and os.path.exists(os.path.join(transformer_dir, "config.json")):+ dirs.append(("", transformer_dir))++ result: dict[str, str] = {}+ for prefix, config_dir in dirs:+ try:+ with torch.device("meta"):+ config = AutoConfig.from_pretrained(config_dir, trust_remote_code=True)+ model = AutoModel.from_config(config, trust_remote_code=True)+ except Exception as e:+ logger.warning(f"Failed to load model from {config_dir} for layer detection. Skipping. Error: {e}")+ continue # skip silently+ for name, module in model.named_modules():+ cls_name = type(module).__name__+ if any(t.lower() in cls_name.lower() for t in target):+ full_name = f"{prefix}.{name}" if prefix else name+ if cls_name not in result:+ result[cls_name] = [full_name]+ else:+ result[cls_name].append(full_name)+ del model+ return resultdiff --git a/docs/step_by_step.md b/docs/step_by_step.mdindex 9762659f6..b9ffc7749 100644--- a/docs/step_by_step.md+++ b/docs/step_by_step.md@@ -27,6 +27,7 @@ This document presents step-by-step instructions for auto-round llm quantization- [API Usage](#api-usage-1)- [Hyperparameters in AutoScheme](#hyperparameters-in-autoscheme)+ [OPT RTN mode](#opt-rtn-mode)+ + [Model-Free Mode](#model-free-mode)+ [GGUF format](#gguf-format)+ [Quantization Costs](#quantization-costs)+ [Device/Multi-GPU setting in Quantization](#devicemulti-gpu-setting-in-quantization)@@ -445,6 +446,94 @@ output_dir = "./tmp_autoround"ar.quantize_and_save(output_dir, format="auto_round")```+### Model-Free Mode++Model-free mode performs RTN WOQ quantization **without loading the full model into memory**. It downloads safetensors files directly, quantizes each Linear weight tensor shard-by-shard, and saves the packed result. This is useful when you want fast, no-calibration quantization with minimal resource requirements.++> **Auto-enabled by default.** As of v0.13, when you pass `--iters 0 --disable_opt_rtn` together with a supported INT WOQ scheme, the CLI automatically takes the model-free path. This is **bit-exactly equivalent** to the regular `--iters 0 --disable_opt_rtn` flow but uses far less memory. Use `--disable_model_free` to opt out and force the original flow.++**Key features:**+- **No model object required** – only `config.json` and safetensors files are needed+- **Low disk memory required** (If no local model files) – downloads and quantizes one shard at a time, deleting the source shard after processing+- **Per-layer configuration** – supports `--layer_config` for per-layer bit-width overrides and `--ignore_layers` to keep specific layers in full precision+- **Predefined ignore layers** – automatically skips model-specific layers (e.g., MoE gates, MTP layers) based on config detection+- **Bit-exact parity** with the standard `--iters 0 --disable_opt_rtn` flow for all supported schemes++<details>+ <summary>Click to expand supported schemes and examples</summary>++**Supported schemes**++Model-free mode currently supports the following **integer weight-only** preset schemes (packed in the `auto_round:auto_gptq` format):++| Preset | Bits | Group size | Sym |+| --- | --- | --- | --- |+| `W2A16` | 2 | 128 | true |+| `W2A16G32` | 2 | 32 | true |+| `W2A16G64` | 2 | 64 | true |+| `W4A16` (default) | 4 | 128 | true |+| `W4A16_MIXED` | 4 | 128 | true |+| `W8A16` | 8 | 128 | true |++All of the above presets also support **asymmetric quantization** (`sym=False`) for 2-bit and 8-bit variants (`W2A16`, `W2A16G32`, `W2A16G64`, `W8A16`), producing `auto_round:auto_gptq`-packed output with bit-exact parity to the regular flow. For 4-bit asymmetric quantization the regular flow uses `auto_round:auto_awq` packing as suggested; use the standard AutoRound flow for that case.++You can also pass a custom `QuantizationScheme(bits=N, group_size=G, sym=True/False, data_type="int", act_bits=16)` with `bits ∈ {2, 4, 8}` and any group_size / sym configuration.++Schemes that require special packing kernels (`W3A16`, `FPW8A16`, `BF16`, `MXFP4`, `MXFP8`, `MXINT4`, `NVFP4`, `FP8_BLOCK`, `FP8_STATIC`, `INT8_W8A8`, `GGUF:*`, ...) are **not** supported in model-free mode and will raise `ValueError`. Use the regular AutoRound flow for those.++#### CLI Usage++```bash+# Easiest: --iters 0 --disable_opt_rtn auto-routes to model-free+auto_round meta-llama/Llama-3.2-1B-Instruct \+ --scheme W4A16 \+ --iters 0 --disable_opt_rtn \+ --output_dir ./int4-llama++# Equivalent explicit invocation+auto_round meta-llama/Llama-3.2-1B-Instruct \+ --model_free \+ --scheme W4A16 \+ --output_dir ./int4-llama++# Opt out of auto-routing and use the regular flow instead+auto_round meta-llama/Llama-3.2-1B-Instruct \+ --scheme W4A16 \+ --iters 0 --disable_opt_rtn --disable_model_free \+ --output_dir ./int4-llama++# With per-layer configuration and ignored layers+auto_round meta-llama/Llama-3.2-1B-Instruct \+ --model_free \+ --scheme W4A16 \+ --group_size 32 \+ --asym \+ --layer_config "{k_proj:{bits:8},v_proj:{bits:8}}" \+ --ignore_layers "mlp" \+ --output_dir ./int4-llama+```++#### API Usage++```python+from auto_round import AutoRound++AutoRound(+ model="meta-llama/Llama-3.2-1B-Instruct",+ scheme="W4A16", # Or a QuantizationScheme instance for custom group_size / sym.+ layer_config={+ ".*k_proj": {"bits": 8, "group_size": 32},+ ".*v_proj": {"bits": 8, "group_size": 32},+ },+ ignore_layers="mlp",+ model_free=True,+).quantize_and_save("./int4-llama")+```++> **Note:** Model-free mode only supports the `auto_round` output format and uses RTN (no calibration data, no iterative tuning). For higher-quality quantization or schemes outside the supported list, use the standard AutoRound flow.++</details>+### GGUF formatExperimental feature. This format is well-suited for CPU devices and is widely adopted by the community.diff --git a/docs/step_by_step_CN.md b/docs/step_by_step_CN.mdindex ee0abe1ee..b990166d8 100644--- a/docs/step_by_step_CN.md+++ b/docs/step_by_step_CN.md@@ -27,6 +27,7 @@- [API 用法](#api-用法)- [AutoScheme 中的超参数](#autoscheme-超参数说明)+ [OPT RTN 模式](#opt-rtn-模式)+ + [免模型量化模式](#免模型量化模式)+ [GGUF 格式](#gguf-格式量化)+ [量化成本](#量化成本)+ [设备及多卡量化设置](#设备及多卡量化设置)@@ -159,7 +160,7 @@ AutoRound 支持多种量化配置:| **auto_round** | W4A16、W2A16、W3A16、W8A16、W2A16G64、W2A16G32、`MXFP4`、`MXFP8`、`MXFP4_RCEIL`、`MXFP8_RCEIL`、`NVFP4`、`FPW8A16`、`FP8_STATIC`、`FP8_BLOCK`、`BF16`, `MXINT4` || **auto_awq** | W4A16、BF16 || **auto_gptq** | W4A16、W2A16、W3A16、W8A16、W2A16G64、W2A16G32、BF16 |-| **llm_compressor** | NVFP4、`MXFP4`、`MXFP8`、`FPW8A16`、`FP8_STATIC`、FP8_STATIC |+| **llm_compressor** | NVFP4、`MXFP4`、`MXFP8`、`FPW8A16`、`FP8_STATIC`、FP8_BLOCK || **mlx** / **auto_round:mlx** | W2A16、W3A16、W4A16、W5A16、W6A16、W8A16、BF16、混合 bit / 混合 group_size(仅 Apple Silicon) || **gguf** | GGUF:Q4_K_M、GGUF:Q2_K_S、GGUF:Q3_K_S、GGUF:Q3_K_M、GGUF:Q3_K_L、GGUF:Q4_K_S、GGUF:Q5_K_S、GGUF:Q5_K_M、GGUF:Q6_K、GGUF:Q4_0、GGUF:Q4_1、GGUF:Q5_0、GGUF:Q5_1、GGUF:Q8_0 || **fp8** | FP8_BLOCK |@@ -420,7 +421,7 @@ ar.quantize_and_save()#### 局限性AutoScheme 目前还**不支持对嵌入层(Embedding layer)进行自动量化**。该层将直接采用候选方案中精度最高的配置。-## OPT-RTN 模式+### OPT-RTN 模式AutoRound 还提供优化版 RTN(Round-To-Nearest,就近舍入)模式,无需标定数据即可实现快速基线量化。**启用方式为 `iters=0`**。同时为获得更好的效果,推荐搭配 `group_size=32` 。RTN 与 OPT RTN 模式的精度对比详见[《精度对比报告》](./opt_rtn.md)。对于 GGUF 格式,我们参考 llamacpp 的思路,优化了 RTN 算法。若需使用原始(非优化)RTN 算法,开启 `--disable_opt_rtn` 即可。@@ -438,6 +439,94 @@ output_dir = "./tmp_autoround"ar.quantize_and_save(output_dir, format="auto_round")```+### 免模型量化模式++免模型量化模式(Model-Free Mode)可以**无需将完整模型加载到内存中**即可执行 RTN WOQ 量化。它直接下载 safetensors 文件,逐分片地对每个 Linear 权重张量进行量化并保存打包结果。当您需要快速、无标定数据的量化且资源有限时,该模式非常实用。++> **默认自动启用。** 自 v0.13 起,当您同时传入 `--iters 0 --disable_opt_rtn` 与一个受支持的 INT WOQ scheme 时,CLI 会自动走免模型路径。该路径与原始 `--iters 0 --disable_opt_rtn` 流程**位级(bit-exact)等价**,但内存占用大幅降低。如需关闭自动路由、强制使用原始流程,可加 `--disable_model_free`。++**主要特性:**+- **无需模型对象** — 仅需 `config.json` 和 safetensors 文件+- **低磁盘内存** (如果无本地模型) — 逐个下载并量化分片,处理完成后立即删除源分片+- **逐层配置** — 支持 `--layer_config` 设置逐层位宽,以及 `--ignore_layers` 保持特定层全精度+- **预定义忽略层** — 根据模型配置自动跳过特定层(如 MoE 门控层、MTP 层等)+- 与标准 `--iters 0 --disable_opt_rtn` 流程对所有受支持的 scheme **位级等价**++<details>+ <summary>点击展开支持的 Scheme 与示例</summary>++**支持的 Scheme**++免模型模式当前支持以下整数权重量化预设(均使用 `auto_round:auto_gptq` 打包格式):++| Preset | Bits | Group size | Sym |+| --- | --- | --- | --- |+| `W2A16` | 2 | 128 | true |+| `W2A16G32` | 2 | 32 | true |+| `W2A16G64` | 2 | 64 | true |+| `W4A16`(默认) | 4 | 128 | true |+| `W4A16_MIXED` | 4 | 128 | true |+| `W8A16` | 8 | 128 | true |++上述 2-bit 和 8-bit 预设(`W2A16`、`W2A16G32`、`W2A16G64`、`W8A16`)同样支持**非对称量化**(`sym=False`),输出使用 `auto_round:auto_gptq` 打包格式,并与标准流程**位级等价**。4-bit 非对称量化时标准流程建议使用 `auto_round:auto_awq` 打包格式,如需该场景请使用标准 AutoRound 流程。++也可以传入自定义的 `QuantizationScheme(bits=N, group_size=G, sym=True/False, data_type="int", act_bits=16)`,其中 `bits ∈ {2, 4, 8}`,group_size / sym 可任意设置。++需要特殊打包内核的 scheme(`W3A16`、`FPW8A16`、`BF16`、`MXFP4`、`MXFP8`、`MXINT4`、`NVFP4`、`FP8_BLOCK`、`FP8_STATIC`、`INT8_W8A8`、`GGUF:*` 等)**不被支持**,传入会抛 `ValueError`。这些请使用标准 AutoRound 流程。++#### 命令行用法++```bash+# 最简单:--iters 0 --disable_opt_rtn 自动路由到免模型+auto_round meta-llama/Llama-3.2-1B-Instruct \+ --scheme W4A16 \+ --iters 0 --disable_opt_rtn \+ --output_dir ./int4-llama++# 等价的显式调用+auto_round meta-llama/Llama-3.2-1B-Instruct \+ --model_free \+ --scheme W4A16 \+ --output_dir ./int4-llama++# 关闭自动路由,强制使用原始流程+auto_round meta-llama/Llama-3.2-1B-Instruct \+ --scheme W4A16 \+ --iters 0 --disable_opt_rtn --disable_model_free \+ --output_dir ./int4-llama++# 搭配逐层配置和忽略层+auto_round meta-llama/Llama-3.2-1B-Instruct \+ --model_free \+ --scheme W4A16 \+ --group_size 32 \+ --asym \+ --layer_config "{k_proj:{bits:8},v_proj:{bits:8}}" \+ --ignore_layers "mlp" \+ --output_dir ./int4-llama+```++#### API 用法++```python+from auto_round import AutoRound++AutoRound(+ model="meta-llama/Llama-3.2-1B-Instruct",+ scheme="W4A16", # 也支持 QuantizationScheme 对象自定义 group_size / sym+ layer_config={+ ".*k_proj": {"bits": 8, "group_size": 32},+ ".*v_proj": {"bits": 8, "group_size": 32},+ },+ ignore_layers="mlp",+ model_free=True,+).quantize_and_save("./int4-llama")+```++> **注意:** 免模型量化模式仅支持 `auto_round` 输出格式,并使用 RTN(无标定数据、无迭代调优)。如需更高质量的量化结果或使用受支持列表外的 scheme,请使用标准 AutoRound 流程。++</details>+### GGUF 格式量化实验性功能。该格式适用 CPU 设备,在社区应用广泛。@@ -745,8 +834,8 @@ print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50, do_sample=Fal| ark | xpu | 4、8 | FP32/FP16/BF16 | 6 | gptq/gptq_zp+-1 | auto-round-lib<br/>torch>=2.8.0 || ark | xpu | 4 | FP32/FP16/BF16 | 6 | awq | auto-round-lib<br/>torch>=2.8.0 || marlin | cuda | 4、8 | BF16/FP16 | 6 | gptq/gptq_zp+-1 | gptqmodel |-| exllamav2/<br/>gptqmodel:exllamav2 | cuda | 4 | BF16/FP16 | 5 | gptq | gptqmodel |-| exllamav2/<br/>gptq:exllamav2 | cuda | 4 | FP16 | 5 | gptq_zp+-1 | auto-gptq<br/>transformers<5.0.0 |+| exllamav2/<br/>gptqmodel:exllamav2 | cuda | 4 | BF16/FP16 | 5 | gptq/gptq_zp+-1 | gptqmodel |+| exllamav2/<br/>gptq:exllamav2 | cuda | 4 | FP16 | 3 | gptq_zp+-1 | auto-gptq<br/>transformers<5.0.0 || gptq:cuda | cuda | 2、3、4、8 | FP16 | 1 | gptq_zp+-1 | auto-gptq<br/>transformers<5.0.0 || triton | xpu/cuda | 2、4、8 | BF16/FP16 | 2 | gptq/gptq_zp+-1 | auto-round || awq | cuda | 4 | FP16 | 5 | awq | auto-awq<br/>transformers<4.57.0 |diff --git a/test/test_cpu/core/test_low_cpu_mem_options.py b/test/test_cpu/core/test_low_cpu_mem_options.pyindex b56ff1cea..d49a1f7af 100644--- a/test/test_cpu/core/test_low_cpu_mem_options.py+++ b/test/test_cpu/core/test_low_cpu_mem_options.py@@ -34,7 +34,6 @@ class TestLowCpuMemUsage:bits=4,low_cpu_mem_usage=True,iters=0,- disable_opt_rtn=True,nsamples=1,seqlen=32,)@@ -45,7 +44,6 @@ class TestLowCpuMemUsage:bits=4,low_cpu_mem_usage=False,iters=0,- disable_opt_rtn=True,nsamples=1,seqlen=32,)@@ -58,7 +56,6 @@ class TestLowCpuMemUsage:bits=4,low_cpu_mem_usage=False,iters=0,- disable_opt_rtn=True,nsamples=1,seqlen=32,)@@ -79,6 +76,7 @@ class TestLowCpuMemUsage:disable_opt_rtn=True,nsamples=1,seqlen=32,+ disable_model_free=True, # Ensure offloading is enabled for this test)dummy_block = torch.nn.Linear(4, 4)diff --git a/test/test_cpu/models/test_conv1d.py b/test/test_cpu/models/test_conv1d.pyindex 717178e73..15acf2c94 100644--- a/test/test_cpu/models/test_conv1d.py+++ b/test/test_cpu/models/test_conv1d.py@@ -43,3 +43,13 @@ class TestQuantizationConv1d:model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="cpu", trust_remote_code=True)model_infer(model, self.tokenizer)+++def test_find_layers_from_config_lamini():+ from auto_round.utils.model import find_layers_from_config++ res = find_layers_from_config(lamini_name_or_path, class_names="Conv1d")+ print(res)+ assert "Conv1D" in res, "Conv1D should be detected in the model config"+ assert "h.0.attn.c_attn" in res["Conv1D"], "Conv1D should be detected in the model config with correct prefix"+ assert len(res["Conv1D"]) == 48, "At least one Conv1D layer should be detected in the model config"diff --git a/test/test_cpu/quantization/test_asym.py b/test/test_cpu/quantization/test_asym.pyindex 0f792a3a1..1ac67b8cd 100644--- a/test/test_cpu/quantization/test_asym.py+++ b/test/test_cpu/quantization/test_asym.py@@ -22,13 +22,21 @@ class TestAutoRoundAsym:def teardown_class(cls):shutil.rmtree("runs", ignore_errors=True)- def test_asym_group_size(self, tiny_opt_model_path):- for group_size in [32, 64, 128]:- bits, sym = 4, False+ # use parameters later+ def test_asym_format(self, tiny_opt_model_path):+ for format in ["auto_round", "auto_round:auto_gptq", "auto_round:gptqmodel"]:+ bits, group_size, sym = 4, 128, Falsear = AutoRound(- tiny_opt_model_path, bits=bits, group_size=group_size, sym=sym, iters=0, seqlen=2, nsamples=1+ tiny_opt_model_path,+ bits=bits,+ group_size=group_size,+ sym=sym,+ iters=0,+ seqlen=2,+ nsamples=1,+ disable_opt_rtn=True,)- _, quantized_model_path = ar.quantize_and_save(format="auto_round", output_dir=self.save_folder)+ _, quantized_model_path = ar.quantize_and_save(format=format, output_dir=self.save_folder)model = AutoModelForCausalLM.from_pretrained(quantized_model_path,@@ -39,9 +47,9 @@ class TestAutoRoundAsym:tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)model_infer(model, tokenizer)- def test_asym_bits(self, tiny_opt_model_path):- for bits in [2, 8]:- group_size, sym = 128, False+ def test_asym_group_size(self, tiny_opt_model_path):+ for group_size in [32, 64, 128]:+ bits, sym = 4, Falsear = AutoRound(tiny_opt_model_path, bits=bits, group_size=group_size, sym=sym, iters=0, seqlen=2, nsamples=1)@@ -56,21 +64,13 @@ class TestAutoRoundAsym:tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)model_infer(model, tokenizer)- # use parameters later- def test_asym_format(self, tiny_opt_model_path):- for format in ["auto_round", "auto_round:auto_gptq", "auto_round:gptqmodel"]:- bits, group_size, sym = 4, 128, False+ def test_asym_bits(self, tiny_opt_model_path):+ for bits in [2, 8]:+ group_size, sym = 128, Falsear = AutoRound(- tiny_opt_model_path,- bits=bits,- group_size=group_size,- sym=sym,- iters=0,- seqlen=2,- nsamples=1,- disable_opt_rtn=True,+ tiny_opt_model_path, bits=bits, group_size=group_size, sym=sym, iters=0, seqlen=2, nsamples=1)- _, quantized_model_path = ar.quantize_and_save(format=format, output_dir=self.save_folder)+ _, quantized_model_path = ar.quantize_and_save(format="auto_round", output_dir=self.save_folder)model = AutoModelForCausalLM.from_pretrained(quantized_model_path,diff --git a/test/test_cpu/quantization/test_mix_bits.py b/test/test_cpu/quantization/test_mix_bits.pyindex 6eab008c0..196466664 100644--- a/test/test_cpu/quantization/test_mix_bits.py+++ b/test/test_cpu/quantization/test_mix_bits.py@@ -41,35 +41,6 @@ class TestAutoRound:def teardown_class(cls):shutil.rmtree("runs", ignore_errors=True)- @require_gptqmodel- def test_mixed_gptqmodel(self, dataloader):- layer_config = {- "k_proj": {"scheme": "W8A16"}, # part name- "lm_head": {"scheme": "w4a16"}, # set lm_head quant- "fc1": {"bits": 16},- "model.decoder.layers.0.self_attn.v_proj": {"bits": 16},- "model.decoder.layers.0.self_attn.q_proj": {"bits": 8}, # full name- }- autoround = AutoRound(- model=self.model_name,- scheme="W4A16",- iters=2,- seqlen=2,- layer_config=layer_config,- dataset=dataloader,- )- quantized_model_path = self.save_dir- _, quantized_model_path = autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_gptq")- # test original GPTQModel inference- from gptqmodel import GPTQModel-- model = GPTQModel.load(quantized_model_path)- assert model.model.model.decoder.layers[0].self_attn.k_proj.bits == 8- assert model.model.model.decoder.layers[0].self_attn.q_proj.bits == 8- assert model.model.model.decoder.layers[1].self_attn.v_proj.bits == 4- result = model.generate("Uncovering deep insights begins with")[0] # tokens- assert "!!!" not in model.tokenizer.decode(result) # string output-def test_mixed_gptqmodel_convert_to_ar(self, dataloader):layer_config = {"k_proj": {"bits": 8}, # part namediff --git a/test/test_cpu/quantization/test_model_free.py b/test/test_cpu/quantization/test_model_free.pynew file mode 100644index 000000000..1cfdd082d--- /dev/null+++ b/test/test_cpu/quantization/test_model_free.py@@ -0,0 +1,410 @@+# Copyright (c) 2025 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++"""Unit tests for auto_round.compressors.model_free module."""++import json+import os++import pytest+import torch+from safetensors import safe_open+from safetensors.torch import save_file++from auto_round import AutoRound+from auto_round.compressors.model_free import (+ _dequant_fp8_tensors,+ _ModelFreeCompressorCore,+ _PatternMatcher,+ _process_shard,+ get_predefined_ignore_layers_from_config,+ is_model_free_supported_scheme,+)+from auto_round.schemes import QuantizationScheme++# ---------------------------------------------------------------------------+# Helpers+# ---------------------------------------------------------------------------++_LLAMA_CFG = {"architectures": ["LlamaForCausalLM"], "model_type": "llama"}+_DEFAULT_SCHEME = {"bits": 4, "group_size": 128, "sym": True, "data_type": "int"}++_SIMPLE_CONFIG = {+ "architectures": ["OPTForCausalLM"],+ "model_type": "opt",+ "hidden_size": 128,+ "num_hidden_layers": 2,+}++_SIMPLE_TENSORS = {+ "model.decoder.layers.0.self_attn.q_proj.weight": torch.randn(128, 128),+ "model.decoder.layers.0.self_attn.k_proj.weight": torch.randn(128, 128),+ "model.decoder.layers.0.fc1.weight": torch.randn(512, 128),+ "model.decoder.layers.0.fc2.weight": torch.randn(128, 512),+ "model.decoder.layers.0.fc1.bias": torch.randn(512),+ "model.decoder.embed_tokens.weight": torch.randn(1000, 128),+ "lm_head.weight": torch.randn(1000, 128),+}+++def _make_model_dir(tmp_path, config, tensors, *, multi_shard=False):+ """Create a minimal local model directory with config.json and safetensors."""+ model_dir = str(tmp_path / "source_model")+ os.makedirs(model_dir, exist_ok=True)+ with open(os.path.join(model_dir, "config.json"), "w") as f:+ json.dump(config, f)++ if not multi_shard:+ save_file(tensors, os.path.join(model_dir, "model.safetensors"))+ else:+ keys = list(tensors.keys())+ mid = max(1, len(keys) // 2)+ shard1 = {k: tensors[k] for k in keys[:mid]}+ shard2 = {k: tensors[k] for k in keys[mid:]}+ save_file(shard1, os.path.join(model_dir, "model-00001-of-00002.safetensors"))+ save_file(shard2, os.path.join(model_dir, "model-00002-of-00002.safetensors"))+ weight_map = {}+ for k in keys[:mid]:+ weight_map[k] = "model-00001-of-00002.safetensors"+ for k in keys[mid:]:+ weight_map[k] = "model-00002-of-00002.safetensors"+ with open(os.path.join(model_dir, "model.safetensors.index.json"), "w") as f:+ json.dump({"metadata": {"total_size": 0}, "weight_map": weight_map}, f)++ return model_dir+++def _matcher(ignore=None, layer_config=None, default=None):+ return _PatternMatcher(ignore or [], layer_config or {}, default or {})+++def _read_output_keys(output_dir):+ keys = set()+ for f in os.listdir(output_dir):+ if f.endswith(".safetensors"):+ with safe_open(os.path.join(output_dir, f), framework="pt") as sf:+ keys.update(sf.keys())+ return keys+++def _read_qconfig(output_dir):+ with open(os.path.join(output_dir, "config.json")) as f:+ return json.load(f).get("quantization_config", {})+++# ===========================================================================+# _PatternMatcher+# ===========================================================================+++class TestPatternMatcher:+ def test_ignore_substring(self):+ m = _matcher(ignore=["mlp"])+ assert m.should_ignore("model.layers.0.mlp.fc1.weight") is True+ assert m.should_ignore("model.layers.0.self_attn.q_proj.weight") is False++ def test_ignore_trailing_dot(self):+ m = _matcher(ignore=["layers.4."])+ assert m.should_ignore("model.layers.4.mlp.fc1.weight") is True+ assert m.should_ignore("model.layers.45.mlp.fc1.weight") is False++ def test_skip_predefined(self):+ m = _matcher()+ assert m.should_skip("model.layers.0.shared_expert_gate.weight") is True+ assert m.should_skip("model.layers.0.mlp.gate.weight") is True+ assert m.should_skip("model.embed_tokens.weight") is True+ assert m.should_skip("model.layers.0.mlp.fc1.weight") is False++ def test_resolve_scheme_exact_regex_and_default(self):+ default = {"bits": 4, "group_size": 128, "sym": True}+ lc = {+ "model.layers.0.mlp.fc1": {"bits": 8, "group_size": 32},+ r".*k_proj": {"bits": 8},+ }+ m = _matcher(layer_config=lc, default=default)+ assert m.resolve_scheme("model.layers.0.mlp.fc1.weight")["bits"] == 8+ assert m.resolve_scheme("model.layers.0.self_attn.k_proj.weight")["bits"] == 8+ assert m.resolve_scheme("model.layers.0.mlp.fc2.weight") == default++ def test_resolve_bits16_returns_none(self):+ m = _matcher(layer_config={"model.layers.0.fc1": {"bits": 16}}, default={"bits": 4, "group_size": 128})+ assert m.resolve_scheme("model.layers.0.fc1.weight") is None++ def test_resolve_substring_pattern(self):+ default = {"bits": 4, "group_size": 128, "sym": True}+ m = _matcher(layer_config={".ffn.experts.": {"bits": 2, "group_size": 64}}, default=default)+ r = m.resolve_scheme("model.layers.0.ffn.experts.3.gate_proj.weight")+ assert r["bits"] == 2 and r["group_size"] == 64+ assert m.resolve_scheme("model.layers.0.self_attn.q_proj.weight") == default+++# ===========================================================================+# _parse_layer_config — scheme key resolution+# ===========================================================================+++class TestParseLayerConfig:+ @staticmethod+ def _make_core(layer_config_input):+ core = _ModelFreeCompressorCore(+ model_name_or_path="dummy",+ output_dir="dummy_out",+ scheme="W4A16",+ )+ core.layer_config_input = layer_config_input+ core._parse_scheme()+ core._parse_layer_config()+ return core++ def test_scheme_key_resolves(self):+ core = self._make_core({".ffn.experts.": {"scheme": "W2A16"}})+ cfg = next(v for k, v in core.layer_config.items() if "ffn.experts" in k)+ assert cfg["bits"] == 2 and "scheme" not in cfg++ m = _matcher(layer_config=core.layer_config, default=core.default_scheme)+ assert m.resolve_scheme("model.layers.0.ffn.experts.3.gate_proj.weight")["bits"] == 2++ def test_scheme_key_with_overrides(self):+ core = self._make_core({".ffn.experts.": {"scheme": "W2A16", "group_size": 32}})+ cfg = next(v for k, v in core.layer_config.items() if "ffn.experts" in k)+ assert cfg["bits"] == 2 and cfg["group_size"] == 32++ def test_string_value(self):+ core = self._make_core({".ffn.experts.": "W2A16"})+ cfg = next(v for k, v in core.layer_config.items() if "ffn.experts" in k)+ assert cfg["bits"] == 2++ def test_quantization_scheme_value(self):+ core = self._make_core({".ffn.experts.": QuantizationScheme(bits=2, group_size=64)})+ cfg = next(v for k, v in core.layer_config.items() if "ffn.experts" in k)+ assert cfg["bits"] == 2 and cfg["group_size"] == 64+++# ===========================================================================+# _process_shard+# ===========================================================================+++class TestProcessShard:+ def test_quantizes_eligible_weights(self, tmp_path):+ shard_path = str(tmp_path / "shard.safetensors")+ save_file({"layer.fc1.weight": torch.randn(64, 128), "layer.fc1.bias": torch.randn(64)}, shard_path)+ output, quantized, _ = _process_shard(shard_path, _DEFAULT_SCHEME, {}, [])+ assert "layer.fc1" in quantized+ assert "layer.fc1.qweight" in output and "layer.fc1.bias" in output++ def test_ignores_and_skips(self, tmp_path):+ shard_path = str(tmp_path / "shard.safetensors")+ save_file(+ {"lm_head.weight": torch.randn(100, 128), "model.layers.0.mlp.gate.weight": torch.randn(8, 128)},+ shard_path,+ )+ _, quantized, ignored = _process_shard(shard_path, _DEFAULT_SCHEME, {}, ["lm_head"])+ assert len(quantized) == 0+ assert "lm_head" in ignored and "model.layers.0.mlp.gate" in ignored++ def test_fused_expert_split(self, tmp_path):+ N, I, H = 2, 64, 32+ shard_path = str(tmp_path / "shard.safetensors")+ save_file(+ {+ "model.layers.0.mlp.experts.gate_up_proj": torch.randn(N, 2 * I, H),+ "model.layers.0.mlp.experts.down_proj": torch.randn(N, H, I),+ },+ shard_path,+ )+ output, quantized, _ = _process_shard(shard_path, _DEFAULT_SCHEME, {}, [])+ for i in range(N):+ for proj in ["gate_proj", "up_proj", "down_proj"]:+ base = f"model.layers.0.mlp.experts.{i}.{proj}"+ assert base in quantized and f"{base}.qweight" in output+++# ===========================================================================+# FP8 source model+# ===========================================================================+++class TestFP8Source:+ def test_dequant_fp8(self):+ w = torch.randn(64, 128, dtype=torch.bfloat16).to(torch.float8_e4m3fn)+ raw = {"layer.weight": w, "layer.weight_scale_inv": torch.tensor(0.5), "layer.bias": torch.randn(64)}+ result = _dequant_fp8_tensors(raw, block_size=None)+ assert result["layer.weight"].dtype == torch.bfloat16 and "layer.weight_scale_inv" not in result++ def test_no_fp8_noop(self):+ raw = {"layer.weight": torch.randn(64, 128)}+ assert _dequant_fp8_tensors(raw, block_size=None) is raw++ def test_process_shard_fp8(self, tmp_path):+ shard_path = str(tmp_path / "shard.safetensors")+ w = torch.randn(64, 128, dtype=torch.bfloat16).to(torch.float8_e4m3fn)+ save_file({"layer.weight": w, "layer.weight_scale_inv": torch.tensor(1.0)}, shard_path)+ output, quantized, _ = _process_shard(shard_path, _DEFAULT_SCHEME, {}, [], device="cpu", fp8_block_size=None)+ assert "layer" in quantized and "layer.qweight" in output+++# ===========================================================================+# End-to-end ModelFreeQuantize+# ===========================================================================+++class TestModelFreeQuantize:+ def test_basic(self, tmp_path):+ model_dir = _make_model_dir(tmp_path, _SIMPLE_CONFIG, _SIMPLE_TENSORS)+ output_dir = str(tmp_path / "output")+ AutoRound(model=model_dir, scheme="W4A16", model_free=True).quantize_and_save(output_dir)+ qc = _read_qconfig(output_dir)+ assert qc["quant_method"] == "auto-round" and qc["bits"] == 4 and qc["model_free"] is True+ keys = _read_output_keys(output_dir)+ assert "lm_head.weight" in keys and "lm_head.qweight" not in keys++ def test_ignore_layers(self, tmp_path):+ tensors = {+ "model.layers.0.mlp.fc1.weight": torch.randn(512, 128),+ "model.layers.0.self_attn.q_proj.weight": torch.randn(128, 128),+ "lm_head.weight": torch.randn(1000, 128),+ }+ model_dir = _make_model_dir(tmp_path, _LLAMA_CFG, tensors)+ output_dir = str(tmp_path / "output")+ AutoRound(model=model_dir, scheme="W4A16", model_free=True, ignore_layers="mlp").quantize_and_save(output_dir)+ keys = _read_output_keys(output_dir)+ assert "model.layers.0.mlp.fc1.weight" in keys and "model.layers.0.mlp.fc1.qweight" not in keys+ assert "model.layers.0.self_attn.q_proj.qweight" in keys++ def test_multi_shard(self, tmp_path):+ model_dir = _make_model_dir(tmp_path, _SIMPLE_CONFIG, _SIMPLE_TENSORS, multi_shard=True)+ output_dir = str(tmp_path / "output")+ AutoRound(model=model_dir, scheme="W4A16", model_free=True).quantize_and_save(output_dir)+ assert os.path.exists(os.path.join(output_dir, "model.safetensors.index.json"))++ def test_quant_lm_head(self, tmp_path):+ model_dir = _make_model_dir(tmp_path, _SIMPLE_CONFIG, _SIMPLE_TENSORS)+ output_dir = str(tmp_path / "output")+ AutoRound(model=model_dir, scheme="W4A16", model_free=True, quant_lm_head=True).quantize_and_save(output_dir)+ assert "lm_head.qweight" in _read_output_keys(output_dir)++ def test_asym(self, tmp_path):+ model_dir = _make_model_dir(tmp_path, _LLAMA_CFG, {"layer.weight": torch.randn(64, 128)})+ output_dir = str(tmp_path / "output")+ AutoRound(+ model=model_dir, scheme=QuantizationScheme(bits=4, group_size=64, sym=False), model_free=True+ ).quantize_and_save(output_dir)+ qc = _read_qconfig(output_dir)+ assert qc["sym"] is False and qc["group_size"] == 64+++# ===========================================================================+# Scheme validation+# ===========================================================================+++_SUPPORTED = ["W2A16", "W2A16G32", "W2A16G64", "W4A16", "W4A16_MIXED", "W8A16"]+_UNSUPPORTED = ["W3A16", "FPW8A16", "BF16", "MXFP4", "MXFP8", "MXINT4", "NVFP4", "FP8_BLOCK", "FP8_STATIC", "INT8_W8A8"]+++class TestSchemeValidation:+ @pytest.mark.parametrize("name", _SUPPORTED)+ def test_supported(self, tmp_path, name):+ model_dir = _make_model_dir(tmp_path, _LLAMA_CFG, {"model.layers.0.mlp.fc1.weight": torch.randn(64, 128)})+ out = str(tmp_path / f"out_{name}")+ AutoRound(model=model_dir, scheme=name, model_free=True).quantize_and_save(out)+ assert "model.layers.0.mlp.fc1.qweight" in _read_output_keys(out)++ @pytest.mark.parametrize("name", _UNSUPPORTED)+ def test_unsupported_raises(self, tmp_path, name):+ model_dir = _make_model_dir(tmp_path, _LLAMA_CFG, {"layer.weight": torch.randn(64, 128)})+ with pytest.raises(ValueError):+ AutoRound(model=model_dir, model_free=True, scheme=name).quantize_and_save(str(tmp_path / "out"))++ def test_is_model_free_supported_scheme(self):+ for name in _SUPPORTED:+ assert is_model_free_supported_scheme(name) is True+ for name in _UNSUPPORTED:+ assert is_model_free_supported_scheme(name) is False+ assert is_model_free_supported_scheme("DOES_NOT_EXIST") is False+++# ===========================================================================+# CLI auto-routing+# ===========================================================================+++class TestCliAutoRouting:+ def test_auto_routes(self, tmp_path):+ model_dir = _make_model_dir(tmp_path, _LLAMA_CFG, {"layer.weight": torch.randn(64, 128)})+ out_dir = str(tmp_path / "out")+ from auto_round.__main__ import BasicArgumentParser, tune++ args = BasicArgumentParser().parse_args(+ [+ "--model",+ model_dir,+ "--scheme",+ "W4A16",+ "--iters",+ "0",+ "--disable_opt_rtn",+ "--format",+ "auto_round",+ "--output_dir",+ out_dir,+ ]+ )+ tune(args)+ assert _read_qconfig(out_dir).get("model_free") is True++ def test_disable_model_free_flag(self):+ from auto_round.__main__ import BasicArgumentParser++ args = BasicArgumentParser().parse_args(+ [+ "--model",+ "dummy",+ "--scheme",+ "W4A16",+ "--iters",+ "0",+ "--disable_opt_rtn",+ "--disable_model_free",+ ]+ )+ auto_route = (+ not args.model_free+ and not args.disable_model_free+ and args.iters == 0+ and args.disable_opt_rtn is True+ and is_model_free_supported_scheme(args.scheme)+ )+ assert auto_route is False+++# ===========================================================================+# Predefined ignore layers+# ===========================================================================+++class TestPredefinedIgnoreLayers:+ def test_normal_model_empty(self):+ assert get_predefined_ignore_layers_from_config({"architectures": ["LlamaForCausalLM"]}) == []++ def test_step3p5_ignore_layers(self):+ cfg = {"model_type": "step3p5"}+ assert get_predefined_ignore_layers_from_config(cfg) == [+ "g_proj",+ "moe.gate",+ "eh_proj",+ "shared_head",+ "layers.45",+ ]diff --git a/test/test_cpu/quantization/test_model_free_parity.py b/test/test_cpu/quantization/test_model_free_parity.pynew file mode 100644index 000000000..008589552--- /dev/null+++ b/test/test_cpu/quantization/test_model_free_parity.py@@ -0,0 +1,253 @@+# Copyright (c) 2025 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++"""Parity tests between ``AutoRound(model_free=True)`` and the regular+``AutoRound(iters=0, disable_opt_rtn=True, disable_model_free=True)`` flow.++Both code paths perform RTN integer weight-only quantization in the+``auto_round:auto_gptq`` packing format. These tests assert that:++1. The ``quantization_config`` keys ``bits``, ``group_size``, ``sym``,+ ``data_type`` (family), ``quant_method``, ``packing_format`` agree.+2. The set of tensor names ending in ``.qweight``, ``.qzeros``, ``.scales``+ is identical (after unioning across shards).+3. For every shared key, ``.qweight``/``.qzeros``/``.scales`` are bit-exact.++This file is symlinked into ``test_cuda/quantization``, ``test_xpu/quantization``+and ``test_hpu/quantization`` so the same test body runs on every backend.+The active backend is selected by the ``BACKEND`` constant which is computed+from this file's directory path. Tests are skipped when the corresponding+accelerator is unavailable on the host.+"""++from __future__ import annotations++import json+import os++import pytest+import torch+from safetensors import safe_open++# ---------------------------------------------------------------------------+# Backend selection+# ---------------------------------------------------------------------------+++def _detect_backend() -> str:+ """Pick the active backend based on this test file's directory path."""+ p = os.path.abspath(__file__)+ for tag in ("test_cuda", "test_xpu", "test_hpu", "test_cpu"):+ if f"/{tag}/" in p or p.endswith(f"/{tag}"):+ return tag.split("_", 1)[1]+ return "cpu"+++BACKEND = _detect_backend()+++def _device_str() -> str:+ """Map the backend tag to the device string passed to AutoRound."""+ return {+ "cpu": "cpu",+ "cuda": "cuda:0",+ "xpu": "xpu:0",+ "hpu": "hpu",+ }.get(BACKEND, "cpu")+++# Schemes verified to be supported by both code paths.+# Each entry is (test_id, preset_name, scheme_kwargs). ``preset_name`` is the+# scheme string passed to AutoRound; ``test_id`` is the pytest parametrize id.+_PARITY_SCHEMES = [+ ("W4A16", "W4A16", {"bits": 4, "group_size": 128, "sym": True}),+ ("W2A16", "W2A16", {"bits": 2, "group_size": 128, "sym": True}),+ ("W2A16G64", "W2A16G64", {"bits": 2, "group_size": 64, "sym": True}),+ ("W2A16G32", "W2A16G32", {"bits": 2, "group_size": 32, "sym": True}),+ ("W8A16", "W8A16", {"bits": 8, "group_size": 128, "sym": True}),+ # Asymmetric (sym=False) variants for non-4-bit: both the model-free path and+ # the regular path use ``auto_round:auto_gptq`` packing, so bit-exact parity holds.+ ("W4A16_ASYM", "W4A16", {"bits": 4, "group_size": 128, "sym": False}),+ ("W2A16_ASYM", "W2A16", {"bits": 2, "group_size": 128, "sym": False}),+ ("W2A16G64_ASYM", "W2A16G64", {"bits": 2, "group_size": 64, "sym": False}),+ ("W8A16_ASYM", "W8A16", {"bits": 8, "group_size": 128, "sym": False}),+ # Note: W4A16 asym is excluded because the regular path uses+ # ``auto_round:auto_awq`` packing for 4-bit asym, which differs from+ # model-free's ``auto_round:auto_gptq``.+]+++# ---------------------------------------------------------------------------+# Helpers+# ---------------------------------------------------------------------------+++def _load_all_keys_and_tensors(directory: str) -> dict[str, torch.Tensor]:+ """Union of all tensors across every safetensors shard in *directory*."""+ out: dict[str, torch.Tensor] = {}+ for fname in sorted(os.listdir(directory)):+ if not fname.endswith(".safetensors"):+ continue+ with safe_open(os.path.join(directory, fname), framework="pt") as f:+ for k in f.keys():+ out[k] = f.get_tensor(k)+ return out+++def _quant_keys(tensors: dict[str, torch.Tensor]) -> set[str]:+ return {k for k in tensors if k.endswith((".qweight", ".qzeros", ".scales"))}+++def _read_qconfig(directory: str) -> dict:+ with open(os.path.join(directory, "config.json")) as f:+ return json.load(f)["quantization_config"]+++# ---------------------------------------------------------------------------+# Parity tests+# ---------------------------------------------------------------------------++[email protected]("scheme_name,scheme_preset,scheme_kwargs", _PARITY_SCHEMES)+def test_parity_model_free_vs_disable_opt_rtn(tmp_path, tiny_opt_model_path, scheme_name, scheme_preset, scheme_kwargs):+ """``AutoRound(model_free=True)`` must produce identical packed tensors+ to ``AutoRound(iters=0, disable_opt_rtn=True, disable_model_free=True)``.+ """+ from auto_round import AutoRound++ out_a = str(tmp_path / f"mf_{scheme_name}")+ out_b = str(tmp_path / f"reg_{scheme_name}")+ device = _device_str()++ # ---- Path A: AutoRound(model_free=True) ----+ ar_a = AutoRound(+ tiny_opt_model_path,+ scheme=scheme_preset,+ bits=scheme_kwargs["bits"],+ group_size=scheme_kwargs["group_size"],+ sym=scheme_kwargs["sym"],+ iters=0,+ disable_opt_rtn=True,+ model_free=True,+ device_map=device,+ )+ assert getattr(ar_a, "model_free", False) is True+ # Model must NOT be loaded into memory in the model-free path.+ assert ar_a.model is None+ ar_a.quantize_and_save(format="auto_round", output_dir=out_a)++ # ---- Path B: regular ``--iters 0 --disable_opt_rtn`` flow ----+ ar_b = AutoRound(+ tiny_opt_model_path,+ scheme=scheme_preset,+ bits=scheme_kwargs["bits"],+ group_size=scheme_kwargs["group_size"],+ sym=scheme_kwargs["sym"],+ iters=0,+ disable_opt_rtn=True,+ disable_model_free=True, # opt out of auto-routing+ device_map=device,+ amp=False, # disable_amp to ensure model is loaded in full precision+ )+ assert getattr(ar_b, "model_free", False) is False+ # Confirm the regular path actually loaded the model on the requested+ # device family (this proves that the corresponding accelerator is+ # actually exercised by the symlinked CUDA/XPU/HPU test).+ if BACKEND != "cpu":+ weight_devices = {p.device.type for p in ar_b.model.parameters()}+ assert (+ BACKEND in weight_devices or "cpu" in weight_devices+ ), f"Expected model parameters on '{BACKEND}' or 'cpu', got {weight_devices}"+ # ar_b.quantize_and_save(format="auto_round", output_dir=out_b)+ _, out_b = ar_b.quantize_and_save(format="auto_round:auto_gptq", output_dir=out_b)++ # ---- 1. quantization_config core keys agree ----+ qc_a = _read_qconfig(out_a)+ qc_b = _read_qconfig(out_b)++ for key in ("bits", "group_size", "sym", "quant_method"):+ assert qc_a[key] == qc_b[key], f"qconfig[{key}] differs: model_free={qc_a[key]} regular={qc_b[key]}"+ assert "int" in qc_a.get("data_type", "")+ assert "int" in qc_b.get("data_type", "")+ assert (+ qc_a["packing_format"] == qc_b["packing_format"]+ ), f"packing_format differs: {qc_a['packing_format']} vs {qc_b['packing_format']}"++ # ---- 2. quantized tensor key sets agree ----+ tensors_a = _load_all_keys_and_tensors(out_a)+ tensors_b = _load_all_keys_and_tensors(out_b)+ keys_a = _quant_keys(tensors_a)+ keys_b = _quant_keys(tensors_b)++ only_a = keys_a - keys_b+ only_b = keys_b - keys_a+ assert not only_a and not only_b, (+ f"Quantized key sets differ.\n only in model_free: {sorted(only_a)[:10]}\n"+ f" only in regular: {sorted(only_b)[:10]}"+ )++ # ---- 3. Bit-exact equality for shared qweight/qzeros/scales ----+ mismatched: list[str] = []+ for k in sorted(keys_a):+ ta = tensors_a[k]+ tb = tensors_b[k]+ if ta.shape != tb.shape or ta.dtype != tb.dtype:+ mismatched.append(f"{k}: shape/dtype differs ({ta.shape}/{ta.dtype} vs {tb.shape}/{tb.dtype})")+ continue+ if not torch.equal(ta, tb):+ diff = (ta.float() - tb.float()).abs()+ mismatched.append(+ f"{k}: max|diff|={diff.max().item():.4g}, " f"#diff={int((ta != tb).sum().item())}/{ta.numel()}"+ )++ if mismatched:+ pytest.fail(+ f"RTN values differ between model_free and regular path "+ f"({len(mismatched)} tensors differ). First few:\n " + "\n ".join(str(m) for m in mismatched[:5])+ )+++def test_auto_routing_to_model_free(tiny_opt_model_path):+ """When iters=0 + disable_opt_rtn=True + supported scheme, AutoRound+ auto-routes to the model-free path even without explicit ``model_free=True``.+ """+ from auto_round import AutoRound++ ar = AutoRound(+ tiny_opt_model_path,+ scheme="W4A16",+ iters=0,+ disable_opt_rtn=True,+ device_map=_device_str(),+ )+ assert getattr(ar, "model_free", False) is True+ assert ar.model is None+++def test_disable_model_free_opt_out(tiny_opt_model_path):+ """``disable_model_free=True`` keeps the regular flow even with the+ auto-routing trigger conditions.+ """+ from auto_round import AutoRound++ ar = AutoRound(+ tiny_opt_model_path,+ scheme="W4A16",+ iters=0,+ disable_opt_rtn=True,+ disable_model_free=True,+ device_map=_device_str(),+ )+ assert getattr(ar, "model_free", False) is False+ assert ar.model is not Nonediff --git a/test/test_cpu/schemes/test_scheme.py b/test/test_cpu/schemes/test_scheme.pyindex 41e9a9f7d..b011d7fc0 100644--- a/test/test_cpu/schemes/test_scheme.py+++ b/test/test_cpu/schemes/test_scheme.py@@ -80,6 +80,7 @@ class TestAutoRound:disable_opt_rtn=True,dataset=dataloader,low_cpu_mem_usage=False,+ disable_model_free=True,)_, quantized_model_path = ar.quantize_and_save(self.save_folder)model = transformers.Qwen2_5_VLForConditionalGeneration.from_pretrained(quantized_model_path)diff --git a/test/test_cuda/export/test_auto_gptq_format.py b/test/test_cuda/export/test_auto_gptq_format.pyindex f7a2891cf..4ab6c4ca1 100644--- a/test/test_cuda/export/test_auto_gptq_format.py+++ b/test/test_cuda/export/test_auto_gptq_format.py@@ -83,3 +83,32 @@ class TestAutoRound:)tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)eval_generated_prompt(model, tokenizer)++ @require_gptqmodel+ def test_mixed_gptqmodel(self, tiny_opt_model_path, dataloader, tmp_path):+ layer_config = {+ "k_proj": {"scheme": "W8A16"}, # part name+ "lm_head": {"scheme": "w4a16"}, # set lm_head quant, useless due to tied weights+ "fc1": {"bits": 16},+ "model.decoder.layers.0.self_attn.v_proj": {"bits": 16},+ "model.decoder.layers.0.self_attn.q_proj": {"bits": 8}, # full name+ }+ autoround = AutoRound(+ model=tiny_opt_model_path,+ scheme="W4A16",+ iters=2,+ seqlen=2,+ layer_config=layer_config,+ dataset=dataloader,+ )+ quantized_model_path = tmp_path+ _, quantized_model_path = autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_gptq")+ # test original GPTQModel inference+ from gptqmodel import GPTQModel++ model = GPTQModel.load(quantized_model_path)+ assert model.model.model.decoder.layers[0].self_attn.k_proj.bits == 8+ assert model.model.model.decoder.layers[0].self_attn.q_proj.bits == 8+ assert model.model.model.decoder.layers[1].self_attn.v_proj.bits == 4+ result = model.generate("Uncovering deep insights begins with")[0] # tokens+ assert "!!!" not in model.tokenizer.decode(result) # string outputdiff --git a/test/test_cuda/quantization/test_model_free_parity.py b/test/test_cuda/quantization/test_model_free_parity.pynew file mode 120000index 000000000..7c94ed661--- /dev/null+++ b/test/test_cuda/quantization/test_model_free_parity.py@@ -0,0 +1 @@+../../test_cpu/quantization/test_model_free_parity.py\ No newline at end of filediff --git a/test/test_xpu/quantization/test_model_free_parity.py b/test/test_xpu/quantization/test_model_free_parity.pynew file mode 120000index 000000000..7c94ed661--- /dev/null+++ b/test/test_xpu/quantization/test_model_free_parity.py@@ -0,0 +1 @@+../../test_cpu/quantization/test_model_free_parity.py\ No newline at end of file