Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

[Experimental] Add NVFP4 E5M3 support and related components for model-free quantization (#2123)

Xin He0c0511b547a7
diff --git a/auto_round/compressors/model_free.py b/auto_round/compressors/model_free.py
old mode 100755
new mode 100644
index f44833419..531f78556
--- a/auto_round/compressors/model_free.py
+++ b/auto_round/compressors/model_free.py
@@ -40,7 +40,12 @@ Model-free mode supports the following quantization families:
* Preset names: ``MXFP4``, ``MXFP8``.
* ``data_type="mx_fp"``, ``group_size=32``, ``bits in {4, 8}``.
-Schemes that require special packing (FP8, NVFP4, GGUF, INT8_W8A8,
+**NVFP4 E5M3** (``auto_round`` or ``fake`` format):
+
+* Preset name: ``NVFP4_E5M3``.
+* ``data_type="nvfp4_v2"``, ``group_size=16``, with high-precision QDQ weights.
+
+Schemes that require special packing (FP8, standard NVFP4, GGUF, INT8_W8A8,
BF16, FPW8A16, ...) are **not** supported in model-free mode and will raise
``ValueError``. Use the standard AutoRound flow for those.
@@ -49,6 +54,9 @@ Output formats
* **INT schemes** → ``auto_round:auto_gptq`` packing format, ``quant_method="auto-round"``.
* **MXFP schemes** → ``mxfp4-pack-quantized`` or ``mxfp8-quantized`` format,
``quant_method="compressed-tensors"``, compatible with vLLM / llm-compressor.
+* **NVFP4_E5M3** → AutoRound format with packed ``.weight_packed`` and
+ ``.weight_scale`` tensors; use ``format="fake"`` explicitly for high-precision
+ QDQ ``.weight`` tensors.
Usage (CLI)
-----------
@@ -97,29 +105,58 @@ import re
import shutil
import sys
import time
-from concurrent.futures import ProcessPoolExecutor, as_completed
+from concurrent.futures import FIRST_COMPLETED, ProcessPoolExecutor, ThreadPoolExecutor, as_completed, wait
from dataclasses import asdict, fields
-from typing import Any, Callable, Optional, Union
+from typing import Any, Optional, Union
import torch
from auto_round import envs
from auto_round.compressors.config_resolution import thaw_mapping
-from auto_round.compressors.utils import is_mx_fp
+from auto_round.compressors.utils import is_mx_fp, is_nv_fp
from auto_round.logger import logger
-from auto_round.schemes import PRESET_SCHEMES, QuantizationScheme, preset_name_to_scheme
-from auto_round.utils.common import AUDIO_MM_KEYS, VISION_MM_KEYS, compress_layer_names, to_standard_regex
-from auto_round.utils.device import clear_memory, compile_func, memory_monitor
+from auto_round.schemes import QuantizationScheme, preset_name_to_scheme
+from auto_round.utils.common import AUDIO_MM_KEYS, VISION_MM_KEYS, compress_layer_names
+from auto_round.utils.device import clear_memory, memory_monitor
from auto_round.utils.device_manager import default_enable_torch_compile
-from auto_round.utils.missing_tensors import quantize_weight_rtn, split_fused_expert_tensors
+from auto_round.utils.model_free_utils import (
+ _apply_scheme_overrides,
+ _build_cross_shard_pairs_from_weight_map,
+ _build_quantization_config,
+ _convert_auto_scheme_layer_config,
+ _download_metadata_files,
+ _download_single_shard,
+ _get_model_cache_status,
+ _is_full_precision_default,
+ _is_weight_shard,
+ _layer_config_has_mxfp,
+ _layer_config_has_nvfp4,
+ _list_weight_shards,
+ _load_config,
+ _load_weight_map_from_index,
+ _looks_like_auto_scheme,
+ _normalize_scheme,
+ _PatternMatcher,
+ _process_shard,
+ _resolve_source_dir,
+ _validate_auto_scheme_options,
+ _validate_supported_scheme,
+ _write_index_file,
+ _write_output_shard,
+ handle_model_type_low_precision_source_tensors,
+ is_model_free_supported_scheme,
+ preprocess_model_type_source_tensors,
+)
+
+# Backward-compat aliases for internal/private helper names used in tests and
+# downstream imports.
+_preprocess_model_type_source_tensors = preprocess_model_type_source_tensors
+_handle_model_type_low_precision_source_tensors = handle_model_type_low_precision_source_tensors
# ---------------------------------------------------------------------------
# Constants
# ---------------------------------------------------------------------------
-# add "embed", "conv" in case of auto detection failure in _check_conv1d_and_embedding
-_BLOCK_NAME_TO_IGNORE = ["shared_expert_gate.", ".gate.", "embed", "conv"]
-
# Preset schemes that model-free mode can produce.
# INT presets use ``auto_round:auto_gptq`` packing; MXFP presets use
# ``mxfp4-pack-quantized`` or ``mxfp8-quantized`` (compressed-tensors) packing.
@@ -138,6 +175,7 @@ SUPPORTED_PRESET_SCHEMES: tuple[str, ...] = (
"W8A16",
"MXFP4",
"MXFP8",
+ "NVFP4_E5M3",
"BF16",
)
@@ -148,6 +186,8 @@ _SUPPORTED_INT_BITS: tuple[int, ...] = (2, 4, 8)
# Allowed ``bits`` values for MXFP weight quantization.
_SUPPORTED_MXFP_BITS: tuple[int, ...] = (4, 8)
+_NVFP4_E5M3_DATA_TYPE = "nvfp4_v2"
+
# Multimodal keywords kept in full precision by default.
_NONTEXT_KEYWORDS: tuple[str, ...] = VISION_MM_KEYS + AUDIO_MM_KEYS
@@ -195,1609 +235,14 @@ def get_predefined_ignore_layers_from_config(config: dict) -> list[str]:
# ---------------------------------------------------------------------------
-def _is_model_cached(model_name_or_path: str) -> bool:
- """Return True if the model is already available locally or in HF cache."""
- if os.path.isdir(model_name_or_path):
- return True
- try:
- from huggingface_hub import try_to_load_from_cache
-
- result = try_to_load_from_cache(model_name_or_path, "config.json")
- return isinstance(result, str)
- except Exception:
- return False
-
-
-def _resolve_source_dir(model_name_or_path: str) -> str:
- """Resolve model_name_or_path to a local directory (download if needed)."""
- if os.path.isdir(model_name_or_path):
- return model_name_or_path
- from huggingface_hub import snapshot_download
-
- return snapshot_download(model_name_or_path)
-
-
-def _load_config(source_dir: str) -> dict:
- """Load config.json from model directory."""
- config_path = os.path.join(source_dir, "config.json")
- if not os.path.exists(config_path):
- raise FileNotFoundError(f"config.json not found in {source_dir}")
- with open(config_path) as f:
- return json.load(f)
-
-
-def _list_weight_shards(source_dir: str) -> list[str]:
- """Return list of weight shard filenames in order.
-
- Safetensors shards are preferred. When no safetensors files are
- found the function falls back to PyTorch ``.bin`` shards.
-
- Handles both standard naming (``model.safetensors``,
- ``pytorch_model.bin``) and custom prefixes such as
- ``diffusion_pytorch_model-XXXXX-of-XXXXX.safetensors`` by scanning
- all ``*.safetensors.index.json`` / ``*.bin.index.json`` index files
- in the directory when no standard index is found.
- """
-
- def _shards_from_index(index_path: str) -> list[str]:
- with open(index_path) as f:
- index = json.load(f)
- seen: set[str] = set()
- shards: list[str] = []
- for shard_file in index["weight_map"].values():
- if shard_file not in seen:
- seen.add(shard_file)
- shards.append(shard_file)
- return shards
-
- # --- safetensors: standard index ---
- st_index = os.path.join(source_dir, "model.safetensors.index.json")
- if os.path.exists(st_index):
- return _shards_from_index(st_index)
-
- # --- safetensors: custom-prefix index (e.g. diffusion_pytorch_model.safetensors.index.json) ---
- for fname in sorted(os.listdir(source_dir)):
- if fname.endswith(".safetensors.index.json"):
- return _shards_from_index(os.path.join(source_dir, fname))
-
- # --- safetensors: single file or index-less multi-file shards ---
- st_files = sorted(f for f in os.listdir(source_dir) if f.endswith(".safetensors"))
- if len(st_files) >= 1:
- return st_files
-
- # --- pytorch .bin: standard index ---
- bin_index = os.path.join(source_dir, "pytorch_model.bin.index.json")
- if os.path.exists(bin_index):
- return _shards_from_index(bin_index)
-
- # --- pytorch .bin: custom-prefix index ---
- for fname in sorted(os.listdir(source_dir)):
- if fname.endswith(".bin.index.json"):
- return _shards_from_index(os.path.join(source_dir, fname))
-
- # --- pytorch .bin: single file ---
- bin_single = os.path.join(source_dir, "pytorch_model.bin")
- if os.path.exists(bin_single):
- return ["pytorch_model.bin"]
-
- # --- pytorch .bin: any single .bin file ---
- bin_files = sorted(f for f in os.listdir(source_dir) if f.endswith(".bin"))
- if len(bin_files) >= 1:
- return bin_files
-
-
-def _is_eligible_weight(tensor_name: str, tensor: torch.Tensor) -> bool:
- """Check if a tensor is eligible for quantization (2D Linear weight)."""
- return tensor_name.endswith(".weight") and tensor.dim() == 2
-
-
-def _is_weight_shard(fname: str) -> bool:
- """Return True if *fname* is a weight shard (safetensors or .bin).
-
- Excludes index files (``*.index.json``) so that they are copied to the
- output directory as normal metadata.
- """
- if fname.endswith(".index.json"):
- return False
- return fname.endswith(".safetensors") or fname.endswith(".bin")
-
-
-# Keep old name as an alias for backward compatibility.
-_is_safetensors_shard = _is_weight_shard
-
-
-def _download_single_shard(
- model_name_or_path: str,
- shard_filename: str,
- local_dir: str,
-) -> str:
- """Download a single safetensors shard file. Returns the local path."""
- os.makedirs(local_dir, exist_ok=True)
- local_path = os.path.join(local_dir, shard_filename)
- if os.path.exists(local_path):
- logger.info(f"Shard '{shard_filename}' already exists at '{local_path}', skipping download.")
- return local_path
-
- if os.path.isdir(model_name_or_path):
- src = os.path.join(model_name_or_path, shard_filename)
- if os.path.exists(src):
- shutil.copy2(src, local_path)
- return local_path
- raise FileNotFoundError(f"{shard_filename} not found in {model_name_or_path}")
-
- from huggingface_hub import hf_hub_download
-
- return hf_hub_download(
- repo_id=model_name_or_path,
- filename=shard_filename,
- local_dir=local_dir,
- )
-
-
-def _download_metadata_files(
- model_name_or_path: str,
- local_dir: str,
-) -> str:
- """Download all non-safetensors files from a model repo. Returns local dir."""
- os.makedirs(local_dir, exist_ok=True)
-
- if os.path.isdir(model_name_or_path):
- for fname in os.listdir(model_name_or_path):
- if _is_weight_shard(fname):
- continue
- src = os.path.join(model_name_or_path, fname)
- dst = os.path.join(local_dir, fname)
- if os.path.isdir(src):
- if not os.path.exists(dst):
- shutil.copytree(src, dst)
- elif os.path.isfile(src) and not os.path.exists(dst):
- shutil.copy2(src, dst)
- return local_dir
-
- from huggingface_hub import snapshot_download
-
- snapshot_download(
- repo_id=model_name_or_path,
- local_dir=local_dir,
- ignore_patterns=["*.safetensors", "*.bin", "*.pth", "*.pt"],
- )
- return local_dir
-
-
-# ---------------------------------------------------------------------------
-# Pattern matcher
-# ---------------------------------------------------------------------------
-
-
-class _PatternMatcher:
- """Precompiled pattern matcher with result caching.
-
- Merges *ignore_patterns* and ``_BLOCK_NAME_TO_IGNORE`` into single
- compiled regexes, precompiles ``layer_config`` regex patterns, and
- caches all match results so that repeated lookups (common across
- shards) are O(1) dict hits.
- """
-
- __slots__ = (
- "_ignore_re",
- "_skip_re",
- "_layer_config",
- "_default_scheme",
- "_compiled_lc",
- "_ignore_cache",
- "_scheme_cache",
- )
-
- def __init__(
- self,
- ignore_patterns: list[str],
- layer_config: dict[str, dict],
- default_scheme: dict,
- ) -> None:
- self._default_scheme = default_scheme
- self._layer_config = layer_config
-
- self._ignore_re: re.Pattern | None = self._build_ignore_regex(ignore_patterns)
-
- skip_parts = [re.escape(b) for b in _BLOCK_NAME_TO_IGNORE]
- self._skip_re: re.Pattern | None = re.compile("|".join(skip_parts)) if skip_parts else None
-
- # Each entry: (compiled_regex | None, plain_string | None, cfg_dict)
- self._compiled_lc: list[tuple[re.Pattern | None, str | None, dict]] = []
- for pattern, cfg in layer_config.items():
- try:
- self._compiled_lc.append((re.compile(to_standard_regex(pattern)), None, cfg))
- except re.error:
- self._compiled_lc.append((None, pattern, cfg))
-
- self._ignore_cache: dict[str, bool] = {}
- self._scheme_cache: dict[str, dict | None] = {}
-
- @staticmethod
- def _build_ignore_regex(patterns: list[str]) -> re.Pattern | None:
- """Merge ignore patterns into one compiled regex.
-
- Uses :func:`~auto_round.utils.common.to_standard_regex` so that
- plain names are automatically wrapped with ``.*`` on both sides
- (substring matching) and regex meta-characters in user patterns
- are preserved — consistent with ``set_layer_config``.
- """
- if not patterns:
- return None
- parts: list[str] = []
- for p in patterns:
- if p.endswith("."):
- std = to_standard_regex(p.rstrip("."))
- std = std.removesuffix(".*")
- parts.append(f"{std}(?:\\.|$)")
- else:
- parts.append(to_standard_regex(p))
- return re.compile("|".join(parts))
-
- def should_ignore(self, tensor_name: str) -> bool:
- """Check user-specified ignore patterns (merged regex + cache)."""
- cached = self._ignore_cache.get(tensor_name)
- if cached is not None:
- return cached
- layer_name = tensor_name.rsplit(".", 1)[0] if "." in tensor_name else tensor_name
- result = bool(self._ignore_re and self._ignore_re.search(layer_name))
- self._ignore_cache[tensor_name] = result
- return result
-
- def should_skip(self, tensor_name: str) -> bool:
- """Check predefined skip patterns (routing gates, embeddings, etc.)."""
- return bool(self._skip_re and self._skip_re.search(tensor_name))
-
- def resolve_scheme(self, tensor_name: str) -> dict | None:
- """Resolve quantization scheme for *tensor_name* (cached).
-
- Returns ``None`` when the layer should stay in full precision.
- """
- if tensor_name in self._scheme_cache:
- return self._scheme_cache[tensor_name]
- result = self._resolve_uncached(tensor_name)
- self._scheme_cache[tensor_name] = result
- return result
-
- def _resolve_uncached(self, tensor_name: str) -> dict | None:
- layer_name = tensor_name.rsplit(".", 1)[0] if "." in tensor_name else tensor_name
- default = self._default_scheme
-
- if layer_name in self._layer_config:
- cfg = self._layer_config[layer_name]
- if cfg.get("bits", default.get("bits", 4)) >= 16:
- return None
- return {**default, **cfg}
-
- for compiled, plain, cfg in self._compiled_lc:
- if compiled is not None:
- if compiled.search(layer_name):
- if cfg.get("bits", default.get("bits", 4)) >= 16:
- return None
- return {**default, **cfg}
- elif plain is not None and plain in layer_name:
- if cfg.get("bits", default.get("bits", 4)) >= 16:
- return None
- return {**default, **cfg}
-
- return default
-
-
-# ---------------------------------------------------------------------------
# Per-tensor / per-shard helpers
# ---------------------------------------------------------------------------
-def _quantize_weight_mxfp(
- weight: torch.Tensor,
- layer_name: str,
- bits: int,
- group_size: int,
- data_type: str,
- device: str = "cpu",
- disable_opt_rtn: bool = False,
-) -> dict[str, torch.Tensor]:
- """Quantize a 2D weight tensor to MXFP4 / MXFP8 and return packed outputs.
-
- Reuses :func:`auto_round.data_type.mxfp.quant_mx` to derive the per-block
- shared exponent (E8M0 scale), and :class:`auto_round.export.export_to_autoround.qlinear_fp.QuantLinear`
- to perform the same packing as :func:`auto_round.export.export_to_llmcompressor.export_to_fp.pack_layer`.
-
- Returns a dict with one of:
- * MXFP8: ``{layer_name+'.weight': float8_e4m3fn, layer_name+'.weight_scale': uint8}``
- * MXFP4: ``{layer_name+'.weight_packed': uint8, layer_name+'.weight_scale': uint8}``
- """
- import torch.nn as nn
-
- from auto_round.data_type.utils import get_quant_func
- from auto_round.export.export_to_autoround.qlinear_fp import QuantLinear
-
- if not is_mx_fp(data_type):
- data_type = "mx_fp4" if bits == 4 else "mx_fp8"
-
- out_features, in_features = weight.shape
- if in_features % group_size != 0:
- raise ValueError(
- f"in_features={in_features} for layer '{layer_name}' is not divisible "
- f"by MXFP group_size={group_size}; cannot pack."
- )
-
- weight_dev = weight.to(device)
- # Use get_quant_func (same as WrapperLinear) so that all registered MXFP
- # variants automatically get opt_rtn support via the QUANT_FUNC_WITH_DTYPE
- # registry (e.g. "opt_rtn_mx_fp4" -> quant_mx_opt_rtn, "mx_fp4" -> quant_mx).
- quant_func, _ = get_quant_func(data_type, bits, sym=True, disable_opt_rtn=disable_opt_rtn, iters=0)
- weight_dev, shared_exp, _ = quant_func(weight_dev, bits=bits, group_size=group_size, data_type=data_type)
- # Reshape to (out_features, n_groups) so the on-disk weight_scale matches
- # the llm-compressor convention (and QuantLinear's registered buffer shape).
- shared_exp = shared_exp.reshape(out_features, in_features // group_size)
- # Ensure shared_exp is a numeric float (not a storage-specific dtype like
- # float8) — QuantLinear.pack performs `2 ** scales` which dispatches to
- # torch.pow; some backends do not implement pow for float8 dtypes. Cast to
- # float32 here to avoid runtime errors like "pow_cuda not implemented for
- # 'Float8_e4m3fn'" while preserving numeric values.
- shared_exp = shared_exp.to(torch.float32)
-
- # Build a lightweight nn.Linear holding the original weight so we can
- # delegate packing to the existing QuantLinear.pack implementation.
- fake_linear = nn.Linear(in_features, out_features, bias=False)
- with torch.no_grad():
- fake_linear.weight = nn.Parameter(weight_dev, requires_grad=False)
-
- qlayer = QuantLinear(
- bits=bits,
- group_size=group_size,
- infeatures=in_features,
- outfeatures=out_features,
- bias=False,
- data_type="mx_fp4" if bits == 4 else "mx_fp8e4m3",
- sym=True,
- act_bits=bits,
- )
- qlayer.pack(fake_linear, shared_exp, device=device)
-
- if bits == 8:
- return {
- f"{layer_name}.weight": qlayer.weight.to("cpu"),
- f"{layer_name}.weight_scale": qlayer.weight_scale.to("cpu"),
- }
- return {
- f"{layer_name}.weight_packed": qlayer.weight_packed.to("cpu"),
- f"{layer_name}.weight_scale": qlayer.weight_scale.to("cpu"),
- }
-
-
-def _quantize_single_tensor(
- tensor_name: str,
- tensor: torch.Tensor,
- matcher: "_PatternMatcher",
- device: str = "cpu",
- quantize_func: Callable = quantize_weight_rtn,
- disable_opt_rtn: bool = False,
-) -> tuple[str, dict[str, torch.Tensor], str | None, str | None]:
- """Quantize one eligible weight tensor and return packed outputs.
-
- Returns:
- (layer_name, output_tensors_dict, quantized_layer_or_None, ignored_layer_or_None)
- """
- layer_name = tensor_name.rsplit(".", 1)[0]
-
- if not _is_eligible_weight(tensor_name, tensor):
- ignored_layer = layer_name if tensor_name.endswith(".weight") and tensor.dim() > 1 else None
- return layer_name, {tensor_name: tensor}, None, ignored_layer
-
- if matcher.should_ignore(tensor_name):
- logger.debug(f"Ignoring (user-specified): {layer_name}")
- return layer_name, {tensor_name: tensor}, None, layer_name
-
- if matcher.should_skip(tensor_name):
- logger.debug(f"Skipping (predefined): {layer_name}")
- return layer_name, {tensor_name: tensor}, None, layer_name
-
- scheme = matcher.resolve_scheme(tensor_name)
- if scheme is None:
- logger.debug(f"Keeping full precision: {layer_name}")
- return layer_name, {tensor_name: tensor}, None, layer_name
-
- bits = scheme["bits"]
- group_size = scheme["group_size"]
- sym = scheme.get("sym", True)
- data_type = (scheme.get("data_type") or "int").lower()
-
- if bits >= 16:
- return layer_name, {tensor_name: tensor}, None, layer_name
-
- # ---- MXFP path (MXFP4 / MXFP8) ----
- if is_mx_fp(data_type):
- try:
- out = _quantize_weight_mxfp(
- weight=tensor,
- layer_name=layer_name,
- bits=bits,
- group_size=group_size,
- data_type=data_type,
- device=device,
- disable_opt_rtn=disable_opt_rtn,
- )
- logger.debug(f"Quantized (MXFP): {layer_name} (bits={bits}, group_size={group_size})")
- return layer_name, out, layer_name, None
- except Exception as e:
- logger.warning(f"Failed to MXFP-quantize {layer_name}: {e}. Keeping original weight.")
- return layer_name, {tensor_name: tensor}, None, layer_name
-
- # ---- Integer WOQ path ----
- # opt_rtn is always disabled for integer WOQ in model-free mode because
- # the scale search does not improve accuracy for INT quantization here.
- try:
- qweight, qzeros, scales = quantize_func(
- weight=tensor,
- bits=bits,
- group_size=group_size,
- sym=sym,
- device=device,
- disable_opt_rtn=True,
- )
-
- out: dict[str, torch.Tensor] = {
- f"{layer_name}.qweight": qweight,
- f"{layer_name}.qzeros": qzeros,
- f"{layer_name}.scales": scales,
- }
-
- logger.debug(f"Quantized: {layer_name} (bits={bits}, group_size={group_size}, sym={sym})")
- return layer_name, out, layer_name, None
-
- except Exception as e:
- logger.warning(f"Failed to quantize {layer_name}: {e}. Keeping original weight.")
- return layer_name, {tensor_name: tensor}, None, layer_name
-
-
-def _collect_mxfp_source_entries(raw_tensors: dict[str, torch.Tensor]) -> list[tuple[str, str, str, int]]:
- """Collect MXFP source tensors present in a shard.
-
- Returns entries as ``(layer_name, weight_key, scale_key, bits)`` where
- ``bits`` is 8 for ``.weight`` (float8) and 4 for ``.weight_packed``.
- """
- entries: list[tuple[str, str, str, int]] = []
- for name, tensor in raw_tensors.items():
- if name.endswith(".weight") and tensor.dtype == torch.float8_e4m3fn:
- layer_name = name[: -len(".weight")]
- scale_key = f"{layer_name}.weight_scale"
- # MXFP scales are raw E8M0 bytes stored as uint8.
- if scale_key in raw_tensors and raw_tensors[scale_key].dtype == torch.uint8:
- entries.append((layer_name, name, scale_key, 8))
- elif name.endswith(".weight_packed") and tensor.dtype in (torch.int8, torch.uint8):
- layer_name = name[: -len(".weight_packed")]
- scale_key = f"{layer_name}.weight_scale"
- # Guard against incorrectly detecting non-MXFP INT4 sources that also use
- # ".weight_packed" + ".weight_scale" but with floating-point scales.
- if scale_key in raw_tensors and raw_tensors[scale_key].dtype == torch.uint8:
- entries.append((layer_name, name, scale_key, 4))
- return entries
-
-
-def _read_int_like(value: Any) -> int | None:
- """Best-effort conversion for int-like config values."""
- if isinstance(value, bool) or value is None:
- return None
- if isinstance(value, int):
- return value
- if isinstance(value, float) and value.is_integer():
- return int(value)
- if isinstance(value, str):
- v = value.strip()
- if v.isdigit() or (v.startswith("-") and v[1:].isdigit()):
- return int(v)
- return None
-
-
-def _read_bool_like(value: Any) -> bool | None:
- """Best-effort conversion for bool-like config values."""
- if isinstance(value, bool):
- return value
- if isinstance(value, str):
- v = value.strip().lower()
- if v in {"true", "1", "yes"}:
- return True
- if v in {"false", "0", "no"}:
- return False
- return None
-
-
-def _resolve_kimi_k25_int4_params(source_quant_config: dict | None) -> tuple[int | None, bool]:
- """Resolve default (group_size, sym) for kimi_k25 INT4 source dequant.
-
- The source tensor shapes are still treated as the source of truth, while
- config values are used as defaults or for logging consistency checks.
- """
- group_size: int | None = None
- sym = True
-
- if not isinstance(source_quant_config, dict):
- return group_size, sym
-
- candidates = [source_quant_config]
- weights_cfg = source_quant_config.get("weights")
- if isinstance(weights_cfg, dict):
- candidates.append(weights_cfg)
-
- for cfg in candidates:
- if group_size is None:
- for key in ("group_size", "weight_group_size"):
- maybe = _read_int_like(cfg.get(key))
- if maybe is not None and maybe > 0:
- group_size = maybe
- break
-
- sym_val = _read_bool_like(cfg.get("sym"))
- if sym_val is not None:
- sym = sym_val
-
- # Some configs store asymmetry as zero_point=True.
- zp_val = _read_bool_like(cfg.get("zero_point"))
- if zp_val is not None:
- sym = not zp_val
-
- return group_size, sym
-
-
-def _collect_kimi_k25_int4_source_entries(
- raw_tensors: dict[str, torch.Tensor],
-) -> list[tuple[str, str, str, str | None]]:
- """Collect kimi_k25 INT4 source tensors.
-
- Expected source layout per layer:
- * ``<layer>.weight_packed``: uint8/int8 packed 2x int4 values per byte,
- **or** int32 packed 8x int4 values per element (Kimi-K2.6 format)
- * ``<layer>.weight_scale``: floating-point per-group scales
- * optional ``<layer>.weight_zero_point`` or ``<layer>.zero_point``
- * optional ``<layer>.weight_shape``: int32 tensor storing original shape
- """
- entries: list[tuple[str, str, str, str | None]] = []
- for name, packed in raw_tensors.items():
- if not name.endswith(".weight_packed"):
- continue
- if packed.dtype not in (torch.uint8, torch.int8, torch.int32):
- continue
-
- layer_name = name[: -len(".weight_packed")]
- scale_key = f"{layer_name}.weight_scale"
- if scale_key not in raw_tensors:
- continue
-
- scale = raw_tensors[scale_key]
- if scale.dtype not in (torch.float16, torch.bfloat16, torch.float32, torch.float64):
- continue
- if packed.dim() != 2 or scale.dim() != 2:
- continue
-
- zp_key = None
- for candidate in (f"{layer_name}.weight_zero_point", f"{layer_name}.zero_point"):
- if candidate in raw_tensors:
- zp_key = candidate
- break
-
- entries.append((layer_name, name, scale_key, zp_key))
-
- return entries
-
-
-def _unpack_int4_weight_packed(packed: torch.Tensor) -> torch.Tensor:
- """Unpack packed int4 bytes/ints into int16 ``[out, in]`` nibbles.
-
- Supports two packing formats:
- * ``uint8`` / ``int8``: 2 nibbles per byte — ``[out, in/2]`` → ``[out, in]``
- * ``int32``: 8 nibbles per element (Kimi-K2.6) — ``[out, in/8]`` → ``[out, in]``
- Nibbles are stored LSB-first: bits 0-3 → nibble 0, bits 4-7 → nibble 1, …
- """
- if packed.dtype == torch.int32:
- # Each int32 packs 8 nibbles (4-bit values), LSB-first.
- packed_i32 = packed.view(torch.int32)
- rows, cols = packed_i32.shape
- # Extract all 8 nibbles per int32 element.
- nibble_list = [(packed_i32 >> (4 * i)) & 0xF for i in range(8)]
- # Stack into [rows, cols, 8] then reshape to [rows, cols*8].
- out = torch.stack(nibble_list, dim=-1).reshape(rows, cols * 8).to(torch.int16)
- return out
-
- packed_u8 = packed.view(torch.uint8)
- lo = packed_u8 & 0x0F
- hi = (packed_u8 >> 4) & 0x0F
-
- out = torch.empty(
- (packed_u8.shape[0], packed_u8.shape[1] * 2),
- dtype=torch.int16,
- device=packed_u8.device,
- )
- out[:, 0::2] = lo.to(torch.int16)
- out[:, 1::2] = hi.to(torch.int16)
- return out
-
-
-def _dequant_kimi_k25_int4_tensors(
- raw_tensors: dict[str, torch.Tensor],
- source_quant_config: dict | None = None,
- device: str = "cpu",
- shard_name: str | None = None,
-) -> dict[str, torch.Tensor]:
- """Dequantize kimi_k25-style INT4 packed weights to bfloat16 ``.weight``.
-
- This normalizes source tensors so the downstream model-free path can
- requantize them into the requested target format (for example MXFP4).
- """
- entries = _collect_kimi_k25_int4_source_entries(raw_tensors)
- if not entries:
- return raw_tensors
-
- default_group_size, default_sym = _resolve_kimi_k25_int4_params(source_quant_config)
- dequant_device = str(device or "cpu")
- shard_prefix = f"[{shard_name}] " if shard_name else ""
- logger.info(
- f"{shard_prefix}Dequantizing kimi_k25 INT4 tensor(s) to bfloat16 on {dequant_device}: " f"total={len(entries)}."
- )
-
- for layer_name, packed_key, scale_key, zp_key in entries:
- packed = raw_tensors.pop(packed_key)
- scale = raw_tensors.pop(scale_key)
- zero = raw_tensors.pop(zp_key) if zp_key is not None else None
- # Remove auxiliary weight_shape tensor if present (Kimi-K2.6 format).
- raw_tensors.pop(f"{layer_name}.weight_shape", None)
-
- # int32 packs 8 nibbles per element; uint8/int8 packs 2 nibbles per byte.
- nibbles_per_element = 8 if packed.dtype == torch.int32 else 2
- in_features = packed.shape[1] * nibbles_per_element
- groups = scale.shape[1]
- inferred_group_size = (in_features // groups) if groups > 0 and in_features % groups == 0 else None
- group_size = inferred_group_size or default_group_size
- if group_size is None or group_size <= 0:
- raise ValueError(
- f"Cannot resolve group_size for INT4 tensor '{layer_name}': "
- f"packed_shape={tuple(packed.shape)}, scale_shape={tuple(scale.shape)}"
- )
- if in_features % group_size != 0:
- raise ValueError(
- f"Invalid group_size={group_size} for INT4 tensor '{layer_name}': "
- f"in_features={in_features} is not divisible by group_size."
- )
-
- if (
- inferred_group_size is not None
- and default_group_size is not None
- and inferred_group_size != default_group_size
- ):
- logger.warning(
- f"{shard_prefix}k25 INT4 group_size mismatch for {layer_name}: "
- f"config={default_group_size}, inferred={inferred_group_size}. "
- "Using inferred value from tensor shapes."
- )
-
- n_groups = in_features // group_size
- if scale.shape[1] != n_groups:
- raise ValueError(
- f"Scale shape mismatch for INT4 tensor '{layer_name}': "
- f"scale.shape={tuple(scale.shape)}, expected second dim {n_groups}."
- )
-
- def _dequant_impl(target_device: str) -> torch.Tensor:
- q = _unpack_int4_weight_packed(packed.to(target_device, non_blocking=True))
- sc = scale.to(target_device, non_blocking=True).to(torch.float32)
-
- if zero is not None:
- zp = zero.to(target_device, non_blocking=True)
- if zp.dim() == 2 and zp.shape == packed.shape and zp.dtype in (torch.int8, torch.uint8):
- zp = _unpack_int4_weight_packed(zp)
- elif zp.dim() == 2 and zp.shape[1] != n_groups:
- raise ValueError(f"Unsupported zero-point shape for INT4 tensor '{layer_name}': {tuple(zp.shape)}")
- if zp.dim() == 2 and zp.shape[1] == n_groups:
- zp = zp.to(torch.int16).repeat_interleave(group_size, dim=1)
- zp = zp[:, : q.shape[1]].to(torch.int16)
- q = q - zp
- else:
- if default_sym:
- q = torch.where(q >= 8, q - 16, q)
- else:
- # Conservative fallback for asymmetric INT4 without explicit
- # zero-point tensor: recenter to [-8, 7].
- q = q - 8
-
- sc = sc.repeat_interleave(group_size, dim=1)[:, : q.shape[1]]
- return (q.to(torch.float32) * sc).to(torch.bfloat16)
-
- dq_weight = _dequantize_with_device_fallback(
- dequant_device=dequant_device,
- shard_prefix=shard_prefix,
- op_name="kimi_k25 INT4 dequant",
- tensor_label=layer_name,
- on_device=lambda: _dequant_impl(dequant_device).to("cpu"),
- on_cpu=lambda: _dequant_impl("cpu"),
- )
- raw_tensors[f"{layer_name}.weight"] = dq_weight
-
- return raw_tensors
-
-
-def _handle_model_type_low_precision_source_tensors(
- raw_tensors: dict[str, torch.Tensor],
- model_type: str | None,
- source_quant_config: dict | None = None,
- device: str = "cpu",
- shard_name: str | None = None,
-) -> dict[str, torch.Tensor]:
- """Handle model-type-specific low-precision source tensors.
-
- Currently supports:
- * ``model_type='kimi_k25'`` INT4 packed tensors.
- """
- if (model_type or "").lower() == "kimi_k25":
- return _dequant_kimi_k25_int4_tensors(
- raw_tensors,
- source_quant_config=source_quant_config,
- device=device,
- shard_name=shard_name,
- )
- return raw_tensors
-
-
-def _is_out_of_memory_error(exc: Exception) -> bool:
- if isinstance(exc, torch.OutOfMemoryError):
- return True
- message = str(exc).lower()
- return "out of memory" in message or "cuda error: out of memory" in message
-
-
-def _dequantize_with_device_fallback(
- *,
- dequant_device: str,
- shard_prefix: str,
- op_name: str,
- tensor_label: str,
- on_device: Callable[[], torch.Tensor],
- on_cpu: Callable[[], torch.Tensor],
-) -> torch.Tensor:
- """Run dequantization on ``dequant_device`` and fall back to CPU on errors."""
- if dequant_device != "cpu":
- try:
- return on_device()
- except Exception as e:
- if _is_out_of_memory_error(e):
- logger.warning(
- f"{shard_prefix}{op_name} on {dequant_device} ran OOM for {tensor_label}: {e}. "
- "Clearing accelerator memory and falling back to CPU for this tensor."
- )
- clear_memory()
- else:
- logger.warning(
- f"{shard_prefix}{op_name} on {dequant_device} failed for {tensor_label}: {e}. "
- "Falling back to CPU for this tensor."
- )
- return on_cpu()
-
-
-def _dequant_mxfp_tensors(
- raw_tensors: dict[str, torch.Tensor],
- device: str = "cpu",
- shard_name: str | None = None,
-) -> dict[str, torch.Tensor]:
- """Dequantize llm-compressor MXFP8 / MXFP4 weight tensors to bfloat16.
-
- Detection is purely by *name* and *dtype*, reusing the dequant kernels in
- :mod:`auto_round_extension.vllm_ext`:
-
- * ``<layer>.weight`` (``float8_e4m3fn``) + ``<layer>.weight_scale`` → MXFP8,
- dequantized via :func:`~auto_round_extension.vllm_ext.mxfp8_qdq_utils.dequant_mx_fp8`.
- * ``<layer>.weight_packed`` (``uint8``) + ``<layer>.weight_scale`` → MXFP4,
- dequantized via :func:`~auto_round_extension.vllm_ext.mxfp4_qdq_utils.to_dtype`.
-
- The dequantized weight is written back under ``<layer>.weight`` and the
- scale (and any ``weight_packed``) tensor is removed, so the downstream RTN
- path can requantize the layer to the requested target scheme.
- """
- from auto_round_extension.vllm_ext.mxfp4_qdq_utils import to_dtype
- from auto_round_extension.vllm_ext.mxfp8_qdq_utils import dequant_mx_fp8
-
- # Tuple layout: (layer_name, weight_key, scale_key, bits)
- entries = _collect_mxfp_source_entries(raw_tensors)
-
- if not entries:
- return raw_tensors
-
- n_mxfp8 = sum(1 for _layer_name, _weight_key, _scale_key, bits in entries if bits == 8)
- n_mxfp4 = len(entries) - n_mxfp8
- dequant_device = str(device or "cpu")
- shard_prefix = f"[{shard_name}] " if shard_name else ""
- logger.info(
- f"{shard_prefix}Dequantizing MXFP tensor(s) to bfloat16 on {dequant_device}: "
- f"MXFP8={n_mxfp8}, MXFP4={n_mxfp4}, total={len(entries)}."
- )
-
- for layer_name, weight_key, scale_key, bits in entries:
- weight = raw_tensors.pop(weight_key)
- scale = raw_tensors.pop(scale_key).view(torch.uint8)
- if bits == 8:
- dq_weight = _dequantize_with_device_fallback(
- dequant_device=dequant_device,
- shard_prefix=shard_prefix,
- op_name="MXFP dequant",
- tensor_label=layer_name,
- on_device=lambda: dequant_mx_fp8(
- weight_fp8=weight.to(dequant_device, non_blocking=True),
- scale_e8m0=scale.to(dequant_device, non_blocking=True),
- block_size=32,
- target_dtype=torch.bfloat16,
- ).to("cpu"),
- on_cpu=lambda: dequant_mx_fp8(
- weight_fp8=weight,
- scale_e8m0=scale,
- block_size=32,
- target_dtype=torch.bfloat16,
- ),
- )
- else:
- dq_weight = _dequantize_with_device_fallback(
- dequant_device=dequant_device,
- shard_prefix=shard_prefix,
- op_name="MXFP dequant",
- tensor_label=layer_name,
- on_device=lambda: to_dtype(
- data_lp=weight.view(torch.uint8).contiguous().to(dequant_device, non_blocking=True),
- scale_e8m0=scale.to(dequant_device, non_blocking=True),
- elem_dtype="fp4_e2m1",
- block_size=32,
- target_dtype=torch.bfloat16,
- ).to("cpu"),
- on_cpu=lambda: to_dtype(
- data_lp=weight.view(torch.uint8).contiguous(),
- scale_e8m0=scale,
- elem_dtype="fp4_e2m1",
- block_size=32,
- target_dtype=torch.bfloat16,
- ),
- )
- raw_tensors[f"{layer_name}.weight"] = dq_weight
-
- return raw_tensors
-
-
-def _handle_mxfp_source_tensors(
- raw_tensors: dict[str, torch.Tensor],
- matcher: "_PatternMatcher",
- source_state: dict[str, int] | None = None,
- device: str = "cpu",
- shard_name: str | None = None,
-) -> tuple[dict[str, torch.Tensor], dict[str, torch.Tensor], list[str]]:
- """Handle llm-compressor-style MXFP8/MXFP4 source tensors.
-
- Detects llm-compressor MXFP tensors purely by *name + dtype*:
-
- * ``<layer>.weight`` (``float8_e4m3fn``) + ``<layer>.weight_scale`` → MXFP8.
- * ``<layer>.weight_packed`` (``uint8``) + ``<layer>.weight_scale`` → MXFP4.
-
- For each detected layer the effective target scheme is resolved via *matcher*:
-
- * If the target is the **same MXFP format** (``data_type='mx_fp'``, matching
- ``bits``), the tensors are emitted directly as a passthrough — no
- dequantization is performed and the layer is recorded as already quantized.
- * Otherwise the tensors are dequantized to ``bfloat16`` via
- :func:`_dequant_mxfp_tensors` so the downstream RTN path can re-quantize
- them to the requested target scheme.
-
- Returns:
- ``(raw_tensors, passthrough_tensors, passthrough_layers)``.
- """
- entries = _collect_mxfp_source_entries(raw_tensors)
- if not entries:
- return raw_tensors, {}, []
-
- source_state = source_state or {}
-
- passthrough_tensors: dict[str, torch.Tensor] = {}
- passthrough_layers: list[str] = []
- n_dequant = 0
-
- for layer_name, weight_key, scale_key, bits in entries:
- scheme = matcher.resolve_scheme(f"{layer_name}.weight")
- target_is_same_mxfp = (
- scheme is not None and is_mx_fp((scheme.get("data_type") or "").lower()) and scheme.get("bits") == bits
- )
- if target_is_same_mxfp:
- passthrough_tensors[weight_key] = raw_tensors.pop(weight_key).to("cpu")
- passthrough_tensors[scale_key] = raw_tensors.pop(scale_key).to("cpu")
- passthrough_layers.append(layer_name)
- else:
- n_dequant += 1
-
- if n_dequant:
- raw_tensors = _dequant_mxfp_tensors(raw_tensors, device=device, shard_name=shard_name)
-
- parts: list[str] = []
- if passthrough_layers:
- parts.append(f"{len(passthrough_layers)} passthrough")
- if n_dequant:
- parts.append(f"{n_dequant} dequantized to bfloat16")
- if source_state:
- parts.append(f"{len(source_state)} model_type-normalized")
- logger.info(f"Handling MXFP source tensor(s): {', '.join(parts)}.")
-
- return raw_tensors, passthrough_tensors, passthrough_layers
-
-
-def _dequant_fp8_tensors(
- raw_tensors: dict[str, torch.Tensor],
- block_size: list | None = None,
- device: str = "cpu",
- shard_name: str | None = None,
-) -> dict[str, torch.Tensor]:
- """Dequantize DeepSeek-V3-style FP8 weight tensors to bfloat16.
-
- Handles the **DeepSeek-V3 FP8** convention: weight dtype ``float8_e4m3fn``
- paired with a ``.weight_scale_inv`` tensor (per-block float32 scales, NOT
- E8M0). The weights are converted to ``bfloat16`` so downstream RTN
- quantization can proceed normally.
-
- MXFP sources are handled separately by
- :func:`_preprocess_model_type_source_tensors` / :func:`_handle_mxfp_source_tensors`.
- """
- from auto_round.utils.weight_handler import _dequant_fp8_linear_weight
-
- quant_entries: list[tuple[str, str]] = []
- for name, tensor in raw_tensors.items():
- if not name.endswith(".weight"):
- continue
- if tensor.dtype != torch.float8_e4m3fn and tensor.element_size() != 1:
- continue
- # DeepSeek-V3 style: .weight_scale_inv (per-block float32 scales).
- scale_inv_name = name.replace(".weight", ".weight_scale_inv")
- if scale_inv_name in raw_tensors:
- quant_entries.append((name, scale_inv_name))
-
- if not quant_entries:
- return raw_tensors
-
- # device has already been resolved by the caller; use it directly here.
- dequant_device = str(device or "cpu")
- shard_prefix = f"[{shard_name}] " if shard_name else ""
-
- logger.info(
- f"{shard_prefix}Dequantizing {len(quant_entries)} FP8 weight tensor(s) to bfloat16 on {dequant_device}."
- )
-
- for weight_name, scale_name in quant_entries:
- weight = raw_tensors[weight_name]
- scale = raw_tensors.pop(scale_name)
-
- # Dequantize on GPU for throughput, then move back to CPU to keep
- # per-shard memory usage bounded before per-layer quantization.
- raw_tensors[weight_name] = _dequantize_with_device_fallback(
- dequant_device=dequant_device,
- shard_prefix=shard_prefix,
- op_name="FP8 dequant",
- tensor_label=weight_name,
- on_device=lambda: _dequant_fp8_linear_weight(
- weight.to(dequant_device, non_blocking=True),
- scale.to(dequant_device, non_blocking=True),
- block_size=block_size,
- ).to("cpu"),
- on_cpu=lambda: _dequant_fp8_linear_weight(weight, scale, block_size=block_size),
- )
-
- return raw_tensors
-
-
-def _process_shard(
- shard_path: str,
- default_scheme: dict = None,
- layer_config: dict = None,
- ignore_patterns: list[str] = None,
- device: str = "cpu",
- *,
- shard_name: str | None = None,
- matcher: "_PatternMatcher | None" = None,
- fp8_block_size: list | None = None,
- model_type: str | None = None,
- source_quant_config: dict | None = None,
- enable_torch_compile: bool = False,
- disable_opt_rtn: bool = False,
-) -> tuple[dict[str, torch.Tensor], list[str], list[str]]:
- """Quantize eligible weights in a single safetensors shard.
-
- Returns:
- (output_tensors, quantized_layer_names, ignored_layer_names)
-
- ``ignored_layer_names`` is derived by comparing the set of input ``.weight``
- layer names (collected after fused-expert splitting) with the final set of
- quantized layer names. Any layer that had a ``.weight`` tensor in the input
- but was NOT quantized is reported as ignored — this correctly captures
- user-ignored layers, predefined-skipped layers, non-eligible weights, and
- any other pass-through case without separate per-tensor tracking.
- """
- if matcher is None:
- matcher = _PatternMatcher(
- ignore_patterns if ignore_patterns is not None else [],
- layer_config if layer_config is not None else {},
- default_scheme if default_scheme is not None else {},
- )
-
- output_tensors: dict[str, torch.Tensor] = {}
- quantized_layers: list[str] = []
- quantize_func = compile_func(quantize_weight_rtn, device) if enable_torch_compile else quantize_weight_rtn
-
- if shard_path.endswith(".bin"):
- # PyTorch pickle checkpoint — load with weights_only where supported.
- try:
- raw_tensors = torch.load(shard_path, map_location="cpu", weights_only=True)
- except TypeError:
- # weights_only not available in older PyTorch versions
- raw_tensors = torch.load(shard_path, map_location="cpu") # nosec
- # Flatten nested state-dict wrappers if present.
- if not isinstance(raw_tensors, dict):
- raise ValueError(f"Expected a dict from {shard_path}, got {type(raw_tensors)}")
- else:
- from safetensors import safe_open
-
- with safe_open(shard_path, framework="pt", device="cpu") as f:
- raw_tensors = {name: f.get_tensor(name) for name in f.keys()}
-
- raw_tensors = split_fused_expert_tensors(raw_tensors)
-
- # Snapshot candidate weight layer names *before* any preprocessing. 1D
- # weights (for example LayerNorm) are not quantization targets, while 3D
- # weights remain tracked so unsupported layouts are visible as ignored.
- input_weight_layers: list[str] = list(
- dict.fromkeys(
- name.rsplit(".", 1)[0]
- for name, tensor in raw_tensors.items()
- if name.endswith(".weight") and tensor.dim() > 1
- )
- )
-
- # Preserve original tensors for ignored/skipped layers so that already-
- # quantized weights (FP8, FP4-packed, etc.) are NOT dequantized.
- # Check both ".weight" and ".weight_packed" so that layers whose primary
- # tensor uses non-standard naming (e.g. already-quantized FP4-packed layers
- # stored as ".weight_packed") are correctly captured.
- preserved_prefixes: set[str] = set()
- for tname in raw_tensors:
- if (tname.endswith(".weight") or tname.endswith(".weight_packed") or tname.endswith(".qweight")) and (
- matcher.should_ignore(tname) or matcher.should_skip(tname)
- ):
- preserved_prefixes.add(tname.rsplit(".", 1)[0])
-
- preserved_tensors: dict[str, torch.Tensor] = {}
- if preserved_prefixes:
- for key in list(raw_tensors.keys()):
- prefix = key.rsplit(".", 1)[0]
- if prefix in preserved_prefixes:
- preserved_tensors[key] = raw_tensors.pop(key)
-
- # 1) model-type-specific preprocessing (format conversion only)
- raw_tensors, source_state = _preprocess_model_type_source_tensors(raw_tensors, model_type=model_type)
-
- # 1.5) model-type-specific low-precision dequantization (e.g. kimi_k25 INT4)
- raw_tensors = _handle_model_type_low_precision_source_tensors(
- raw_tensors,
- model_type=model_type,
- source_quant_config=source_quant_config,
- device=device,
- shard_name=shard_name,
- )
-
- # 2) generic MXFP handling for both preprocessed and normal source models
- raw_tensors, passthrough_tensors, passthrough_layers = _handle_mxfp_source_tensors(
- raw_tensors,
- matcher,
- source_state=source_state,
- device=device,
- shard_name=shard_name,
- )
- output_tensors.update(passthrough_tensors)
- quantized_layers.extend(passthrough_layers)
-
- raw_tensors = _dequant_fp8_tensors(
- raw_tensors,
- block_size=fp8_block_size,
- device=device,
- shard_name=shard_name,
- )
- raw_tensors.update(preserved_tensors)
-
- for tensor_name in list(raw_tensors.keys()):
- tensor = raw_tensors.pop(tensor_name)
- _layer_name, out_dict, q_layer, _ig_layer = _quantize_single_tensor(
- tensor_name,
- tensor,
- matcher,
- device,
- quantize_func,
- disable_opt_rtn,
- )
- output_tensors.update(out_dict)
- if q_layer:
- quantized_layers.append(q_layer)
-
- # Derive ignored layers by comparing input weight layers with quantized set.
- quantized_set = set(quantized_layers)
- ignored_layers: list[str] = [l for l in input_weight_layers if l not in quantized_set]
-
- return output_tensors, quantized_layers, ignored_layers
-
-
-# ---------------------------------------------------------------------------
# Output writers
# ---------------------------------------------------------------------------
-def _build_mxfp_quantization_config(
- default_scheme: dict,
- quantized_layers: list[str],
- ignored_layers: list[str],
- layer_config: dict | None = None,
-) -> dict:
- """Build a compressed-tensors / llm-compressor style quantization_config
- dict for MXFP4 / MXFP8 model-free output, including mixed-precision cases.
-
- When *layer_config* contains layers that override the default bits (e.g.
- some layers are MXFP8 while the default is MXFP4), the function creates
- one ``config_group`` per distinct bit-width. Override groups list their
- layers explicitly; the default-bits group uses ``targets=["Linear"]`` as a
- catch-all. The top-level ``"format"`` is set to ``"mixed-precision"``
- when more than one group is produced.
-
- Mirrors the per-group format produced by
- :mod:`auto_round.export.export_to_llmcompressor.export_to_fp`.
- """
- from auto_round.export.export_to_llmcompressor.config import (
- initialize_quantization,
- )
-
- bits = default_scheme.get("bits", 4)
- is_fp_default = (bits or 0) >= 16 # BF16/FP16 full-precision default
-
- if not is_fp_default and bits not in _SUPPORTED_MXFP_BITS:
- raise ValueError(f"Unsupported MXFP bits={bits} for model-free output.")
-
- # Default ignore list: any layer present in ignored_layers (deduped) that
- # was NOT quantized.
- ignore = list(dict.fromkeys(ignored_layers))
- quant_set = set(quantized_layers)
- ignore = [n for n in ignore if n not in quant_set]
-
- # Resolve each quantized layer's effective bits using layer_config overrides.
- scheme_groups: dict[int, list[str]] = {} # bits -> [layer_names]
- if layer_config:
- temp_matcher = _PatternMatcher(
- ignore_patterns=[],
- layer_config=layer_config,
- default_scheme=default_scheme,
- )
- for layer in quantized_layers:
- scheme = temp_matcher.resolve_scheme(f"{layer}.weight")
- layer_bits = scheme.get("bits", bits) if scheme is not None else bits
- scheme_groups.setdefault(layer_bits, []).append(layer)
- else:
- if not is_fp_default:
- scheme_groups[bits] = list(quantized_layers)
- # else: BF16 default with no layer_config → no MXFP layers; scheme_groups stays {}
-
- if len(scheme_groups) <= 1:
- # Single scheme — use the actual MXFP bits from the group, not the
- # default bits (which may be 16 for a BF16 default scheme).
- actual_bits = next(iter(scheme_groups.keys())) if scheme_groups else bits
- if actual_bits not in _SUPPORTED_MXFP_BITS:
- raise ValueError(f"Unsupported MXFP bits={actual_bits} for model-free output.")
- scheme_name = "MXFP4" if actual_bits == 4 else "MXFP8"
- fmt = "mxfp4-pack-quantized" if actual_bits == 4 else "mxfp8-quantized"
- qconfig = initialize_quantization(scheme=scheme_name, ignore=ignore)
- if is_fp_default and scheme_groups:
- targets = list(quantized_layers)
- qconfig.config_groups["group_0"].targets = targets
- qconfig = qconfig.to_dict()
- qconfig["format"] = fmt
- qconfig["provider"] = "auto-round"
- return qconfig
-
- # Mixed MXFP: build one config_group per distinct bit-width.
- # Override groups (non-default bits) come first, default group last,
- # ordered by descending bit-width within each partition so that the
- # higher-precision group gets the lower group index.
- override_items = sorted(
- [(b, layers) for b, layers in scheme_groups.items() if b != bits],
- key=lambda x: x[0],
- reverse=True,
- )
- default_item = (bits, scheme_groups[bits]) if bits in scheme_groups else None
- ordered = override_items + ([default_item] if default_item else [])
-
- config_groups: dict = {}
- group_formats: dict[str, str] = {}
- for idx, (group_bits, layer_names) in enumerate(ordered):
- group_name = f"group_{idx}"
- scheme_name = "MXFP4" if group_bits == 4 else "MXFP8"
- fmt = "mxfp4-pack-quantized" if group_bits == 4 else "mxfp8-quantized"
- is_default_group = group_bits == bits
- targets = ["Linear"] if is_default_group else layer_names
- tmp_qconfig = initialize_quantization(scheme=scheme_name, ignore=ignore)
- group_scheme = tmp_qconfig.config_groups["group_0"]
- group_scheme.targets = targets
- config_groups[group_name] = group_scheme
- group_formats[group_name] = fmt
-
- full_qconfig = initialize_quantization(scheme=None, config_groups=config_groups, ignore=ignore)
- full_dict = full_qconfig.to_dict()
- full_dict["format"] = "mixed-precision"
- for group_name, fmt in group_formats.items():
- full_dict["config_groups"][group_name]["format"] = fmt
- full_dict["provider"] = "auto-round"
- return full_dict
-
-
-def _build_mxfp_autoround_quantization_config(
- default_scheme: dict,
- quantized_layers: list[str],
- ignored_layers: list[str],
- layer_config: dict | None = None,
- block_name_to_quantize: Optional[str] = None,
-) -> dict:
- """Build an auto-round style quantization_config for MXFP4 / MXFP8.
-
- Unlike :func:`_build_mxfp_quantization_config` which produces a
- compressed-tensors / llm-compressor style config, this function produces
- an ``auto-round`` style config (``quant_method="auto-round"``,
- ``packing_format="auto_round:llm_compressor"``). The on-disk weight
- tensors are identical; only the ``quantization_config`` metadata differs.
-
- The generated config mirrors the layout of the regular AutoRound MXFP
- export, including activation quantization fields (``act_bits`` /
- ``act_data_type`` / …) from the scheme, ``enable_quanted_input``, and
- ``lm_head`` in ``extra_config`` when it was quantized.
- """
- from collections import Counter
-
- from auto_round.version import __version__
-
- bits = default_scheme.get("bits", 4)
- group_size = default_scheme.get("group_size", 32)
- is_fp_default = (bits or 0) >= 16
-
- # For BF16 default + MXFP layer_config overrides, derive the dominant
- # MXFP bit-width from the layers that were actually quantized.
- if is_fp_default and layer_config and quantized_layers:
- temp_matcher = _PatternMatcher(
- ignore_patterns=[],
- layer_config=layer_config,
- default_scheme=default_scheme,
- )
- mxfp_counter: Counter = Counter()
- for layer in quantized_layers:
- scheme = temp_matcher.resolve_scheme(f"{layer}.weight")
- if scheme is None:
- continue
- lb = scheme.get("bits")
- ldt = (scheme.get("data_type") or "").lower()
- if lb and lb < 16 and is_mx_fp(ldt):
- mxfp_counter[lb] += 1
- if mxfp_counter:
- bits, _ = mxfp_counter.most_common(1)[0]
- group_size = 32
-
- if (bits or 0) < 1 or bits not in _SUPPORTED_MXFP_BITS:
- bits = 4 # safe fallback
-
- qconfig: dict = {
- "quant_method": "auto-round",
- "packing_format": "auto_round:llm_compressor",
- "bits": bits,
- "group_size": group_size or 32,
- "sym": True, # MXFP is always symmetric
- "data_type": "mx_fp",
- "iters": 0,
- "model_free": True,
- "autoround_version": __version__,
- "enable_quanted_input": False,
- }
- if block_name_to_quantize:
- qconfig["block_name_to_quantize"] = block_name_to_quantize
-
- # Carry activation quantization fields from the scheme (e.g. MXFP4 has
- # act_bits=4, act_data_type="mx_fp", act_dynamic=True, act_group_size=32,
- # act_sym=True). Including these keeps the config consistent with the
- # output produced by the regular AutoRound MXFP export flow.
- for act_key in ("act_bits", "act_data_type", "act_dynamic", "act_group_size", "act_sym"):
- val = default_scheme.get(act_key)
- if val is not None:
- qconfig[act_key] = val
-
- scheme_keys = [f.name for f in fields(QuantizationScheme)]
- extra_config: dict = {}
- non_linear_ops = ["embed", "conv"]
- non_linear_re = re.compile("|".join(re.escape(op) for op in non_linear_ops))
-
- if layer_config:
- from auto_round.export.export_to_autoround.utils import check_neq_config
-
- expected_scheme = {key: qconfig.get(key) for key in scheme_keys}
- for layer_name, cfg in layer_config.items():
- if not isinstance(cfg, dict):
- continue
- neq_keys = check_neq_config(cfg, **expected_scheme)
- if neq_keys:
- extra_config[layer_name] = {key: cfg[key] for key in neq_keys if cfg.get(key) is not None}
-
- quantized_set = set(quantized_layers)
- unique_ignored = list(dict.fromkeys(ignored_layers))
- for layer_name in unique_ignored:
- if layer_name in quantized_set or layer_name in extra_config:
- continue
- if non_linear_re.search(layer_name):
- continue
- extra_config[layer_name] = {
- "bits": 16,
- "data_type": "float",
- "act_bits": 16,
- "act_data_type": "float",
- }
-
- # lm_head: when explicitly quantized, record its full scheme in extra_config
- # (mirrors the regular AutoRound MXFP export behavior).
- if "lm_head" in quantized_set and "lm_head" not in extra_config:
- lm_head_cfg = (layer_config or {}).get("lm_head", default_scheme)
- extra_config["lm_head"] = {k: lm_head_cfg.get(k) for k in scheme_keys if lm_head_cfg.get(k) is not None}
-
- if extra_config:
- qconfig["extra_config"] = extra_config
-
- return qconfig
-
-
-def _layer_config_has_mxfp(layer_config: dict | None) -> bool:
- """Return True if any layer_config entry requests MXFP quantization.
-
- Handles values that are plain strings (preset names), dicts (possibly
- with a ``"scheme"`` key or a ``"data_type"`` key), or
- :class:`QuantizationScheme` instances.
- """
- if not layer_config:
- return False
- for val in layer_config.values():
- if isinstance(val, str):
- try:
- s = _normalize_scheme(val.upper())
- if is_mx_fp((s.data_type or "").lower()):
- return True
- except Exception:
- pass
- elif isinstance(val, dict):
- # Direct data_type field
- if is_mx_fp((val.get("data_type") or "").lower()):
- return True
- # Nested 'scheme' key (e.g. {"scheme": "MXFP4"})
- scheme_val = val.get("scheme")
- if isinstance(scheme_val, str):
- try:
- s = _normalize_scheme(scheme_val.upper())
- if is_mx_fp((s.data_type or "").lower()):
- return True
- except Exception:
- pass
- elif isinstance(val, QuantizationScheme):
- if is_mx_fp((val.data_type or "").lower()):
- return True
- return False
-
-
-def _is_full_precision_default(scheme_input: Any) -> bool:
- """Return True when *scheme_input* represents a full-precision default (bits >= 16).
-
- Used to detect BF16/FP16 schemes where no weights are quantized by
- default but layer_config overrides are still applied.
- """
- try:
- s = _normalize_scheme(scheme_input)
- return (s.bits or 0) >= 16 and (s.act_bits or 16) >= 16
- except Exception:
- return False
-
-
-def _derive_dominant_int_scheme(
- quantized_layers: list[str],
- layer_config: dict,
- fallback: dict,
-) -> dict | None:
- """Infer the dominant INT scheme from layers that were actually quantized.
-
- Used when the default scheme is full-precision (BF16/FP16) to produce a
- quantization_config where the most common INT scheme becomes the top-level
- default and the BF16 layers appear as ``bits=16`` exceptions in
- ``extra_config``. This matches the layout expected by AutoRound loaders
- (dominant quantized bits at the top, full-precision overrides below).
-
- Returns ``None`` when no INT layers were found in *quantized_layers*.
- """
- from collections import Counter
-
- temp_matcher = _PatternMatcher(
- ignore_patterns=[],
- layer_config=layer_config,
- default_scheme=fallback,
- )
- counter: "Counter[tuple]" = Counter()
- for layer in quantized_layers:
- scheme = temp_matcher.resolve_scheme(f"{layer}.weight")
- if scheme is None:
- continue
- bits = scheme.get("bits")
- if bits is None or bits >= 16:
- continue
- data_type = (scheme.get("data_type") or "int").lower()
- if is_mx_fp(data_type):
- continue # MXFP is handled by the dedicated path
- key = (bits, scheme.get("group_size") or fallback.get("group_size"), bool(scheme.get("sym", True)), data_type)
- counter[key] += 1
-
- if not counter:
- return None
-
- (bits, group_size, sym, data_type), _ = counter.most_common(1)[0]
- return {
- "bits": bits,
- "group_size": group_size,
- "sym": sym,
- "data_type": data_type,
- }
-
-
-def _build_quantization_config(
- default_scheme: dict,
- layer_config: dict,
- ignore_patterns: list[str],
- quantized_layers: list[str],
- ignored_layers: list[str],
- block_name_to_quantize: Optional[str] = None,
- format: str = "auto_round",
-) -> dict:
- """Build a quantization_config dict compatible with auto-round format."""
- # MXFP (mx_fp) supports two output styles depending on *format*:
- # - "llm_compressor" → compressed-tensors / llm-compressor style config
- # (produced by _build_mxfp_quantization_config).
- # - "auto_round" / "auto_round:auto_gptq" → auto-round style config
- # (produced by _build_mxfp_autoround_quantization_config).
- # Also route to MXFP config when the default is full-precision (BF16/FP16)
- # but layer_config contains MXFP overrides.
- data_type = (default_scheme.get("data_type") or "int").lower()
- default_bits = default_scheme.get("bits", 4)
- is_fp_default = (default_bits or 0) >= 16 and not is_mx_fp(data_type)
- if is_mx_fp(data_type) or (is_fp_default and _layer_config_has_mxfp(layer_config)):
- if format in ("auto_round", "auto_round:auto_gptq"):
- return _build_mxfp_autoround_quantization_config(
- default_scheme=default_scheme,
- quantized_layers=quantized_layers,
- ignored_layers=ignored_layers,
- layer_config=layer_config,
- block_name_to_quantize=block_name_to_quantize,
- )
- return _build_mxfp_quantization_config(
- default_scheme=default_scheme,
- quantized_layers=quantized_layers,
- ignored_layers=ignored_layers,
- layer_config=layer_config,
- )
-
- # When the default is full-precision (BF16/FP16) but INT layers were
- # quantized via layer_config, derive the dominant INT scheme and use it as
- # the top-level defaults so the output config reads as
- # "INT-quantized with BF16 exceptions" rather than the inverse.
- if is_fp_default and quantized_layers:
- dominant = _derive_dominant_int_scheme(quantized_layers, layer_config, default_scheme)
- if dominant is not None:
- default_scheme = dominant
-
- from auto_round.version import __version__
-
- scheme_keys = [f.name for f in fields(QuantizationScheme)]
- # vllm only support auto_round:auto_gptq, but transformers cannot load it correctly when sym=False.
- # So we keep auto_round for asymmetric quantization to maintain compatibility with both.
- packing_format = "auto_round:auto_gptq" if default_scheme.get("sym", True) else "auto_round"
-
- qconfig = {
- "quant_method": "auto-round",
- "packing_format": packing_format,
- "bits": default_scheme["bits"],
- "group_size": default_scheme["group_size"],
- "sym": default_scheme.get("sym", True),
- "data_type": default_scheme.get("data_type", "int"),
- "iters": 0,
- "model_free": True,
- "autoround_version": __version__,
- }
-
- if block_name_to_quantize:
- qconfig["block_name_to_quantize"] = block_name_to_quantize
-
- extra_config = {}
- for layer_name, cfg in layer_config.items():
- if cfg.get("bits", default_scheme["bits"]) >= 16:
- extra_config[layer_name] = {k: cfg.get(k) for k in scheme_keys if cfg.get(k) is not None}
- continue
- differs = False
- for key in ("bits", "group_size", "sym"):
- if cfg.get(key) is not None and cfg[key] != default_scheme.get(key):
- differs = True
- break
- if differs:
- extra_config[layer_name] = {k: cfg.get(k) for k in scheme_keys if cfg.get(k) is not None}
-
- # Filter out non-Linear ops (embed, conv) that don't need to be recorded in config.
- # Routing gates and other predefined patterns are still recorded.
- non_linear_ops = ["embed", "conv"]
- non_linear_re = re.compile("|".join(re.escape(op) for op in non_linear_ops))
-
- unique_ignored = list(dict.fromkeys(ignored_layers))
- for layer_name in unique_ignored:
- if layer_name not in extra_config:
- # Skip non-Linear ops (embed, conv) since they're not Linear layers
- if non_linear_re.search(layer_name):
- continue
- extra_config[layer_name] = {"bits": 16, "data_type": "float"}
-
- quantized_layer_set = set(quantized_layers)
- if "lm_head" in quantized_layer_set and "lm_head" not in extra_config:
- lm_head_cfg = layer_config.get("lm_head", default_scheme)
- extra_config["lm_head"] = {k: lm_head_cfg.get(k) for k in scheme_keys if lm_head_cfg.get(k) is not None}
-
- if extra_config:
- qconfig["extra_config"] = extra_config
-
- return qconfig
-
-
-def _write_output_shard(
- output_dir: str,
- shard_name: str,
- tensors: dict[str, torch.Tensor],
- weight_map: dict[str, str],
-):
- """Write a single output shard and update the weight_map."""
- from safetensors.torch import save_file
-
- shard_path = os.path.join(output_dir, shard_name)
-
- # Detect shared-storage tensors (e.g. tie_word_embeddings: wte ↔ lm_head).
- # safetensors refuses to serialise them as-is; clone the duplicates so each
- # tensor occupies its own memory region. The first occurrence keeps the
- # original storage; subsequent aliases are cloned.
- seen_data_ptrs: set[int] = set()
- deduped: dict[str, torch.Tensor] = {}
- for k, v in tensors.items():
- if not v.is_contiguous():
- v = v.contiguous()
- ptr = v.data_ptr()
- if ptr in seen_data_ptrs:
- v = v.clone()
- else:
- seen_data_ptrs.add(ptr)
- deduped[k] = v
-
- save_file(deduped, shard_path)
- for tensor_name in tensors:
- weight_map[tensor_name] = shard_name
-
-
-def _write_index_file(output_dir: str, weight_map: dict[str, str]):
- """Write model.safetensors.index.json (or rename single shard)."""
- if len(set(weight_map.values())) <= 1:
- shard_names = list(set(weight_map.values()))
- if shard_names and shard_names[0] != "model.safetensors":
- src = os.path.join(output_dir, shard_names[0])
- dst = os.path.join(output_dir, "model.safetensors")
- if os.path.exists(src):
- os.rename(src, dst)
- weight_map = {k: "model.safetensors" for k in weight_map}
- return
-
- index = {"metadata": {"total_size": 0}, "weight_map": weight_map}
- with open(os.path.join(output_dir, "model.safetensors.index.json"), "w") as f:
- json.dump(index, f, indent=2)
-
-
def _prefetch_shard(
model_name_or_path: str,
shard_name: str,
@@ -1833,11 +278,12 @@ def _process_single_shard_task(
ignore_patterns: list[str],
fp8_block_size: list | None,
model_type: str | None,
+ source_quantization_config: dict | None = None,
quant_output_dir: str,
total_shards: int,
- source_quant_config: dict | None = None,
enable_torch_compile: bool = False,
disable_opt_rtn: bool = False,
+ donor_tensors_to_exclude: list[str] | None = None,
) -> tuple[int, str, str | None, str | None, list[str] | None, list[str] | None, list[str] | None]:
"""Process one shard in an isolated subprocess task.
@@ -1854,6 +300,54 @@ def _process_single_shard_task(
if shard_path is None or not os.path.exists(shard_path):
return shard_idx, shard_name, None, None, None, None, None
+ return _quantize_local_shard_task(
+ shard_idx,
+ shard_name,
+ shard_path=shard_path,
+ device=device,
+ default_scheme=default_scheme,
+ layer_config=layer_config,
+ ignore_patterns=ignore_patterns,
+ fp8_block_size=fp8_block_size,
+ model_type=model_type,
+ source_quantization_config=source_quantization_config,
+ quant_output_dir=quant_output_dir,
+ total_shards=total_shards,
+ enable_torch_compile=enable_torch_compile,
+ disable_opt_rtn=disable_opt_rtn,
+ cleanup_source_shard=is_streaming,
+ donor_tensors_to_exclude=donor_tensors_to_exclude,
+ )
+
+
+def _quantize_local_shard_task(
+ shard_idx: int,
+ shard_name: str,
+ *,
+ shard_path: str,
+ device: str,
+ default_scheme: dict,
+ layer_config: dict,
+ ignore_patterns: list[str],
+ fp8_block_size: list | None,
+ model_type: str | None,
+ source_quantization_config: dict | None,
+ quant_output_dir: str,
+ total_shards: int,
+ enable_torch_compile: bool = False,
+ disable_opt_rtn: bool = False,
+ cleanup_source_shard: bool = False,
+ donor_tensors_to_exclude: list[str] | None = None,
+ index_dir: str | None = None,
+ donor_shard_dir: str | None = None,
+) -> tuple[int, str, str | None, str | None, list[str] | None, list[str] | None, list[str] | None]:
+ """Quantize one already-downloaded shard and write the output shard.
+
+ Returns lightweight metadata only so IPC does not transfer tensor storages.
+ """
+ if shard_path is None or not os.path.exists(shard_path):
+ return shard_idx, shard_name, None, None, None, None, None
+
output_tensors, quantized, ignored = _process_shard(
shard_path=shard_path,
shard_name=shard_name,
@@ -1863,9 +357,12 @@ def _process_single_shard_task(
device=device,
fp8_block_size=fp8_block_size,
model_type=model_type,
- source_quant_config=source_quant_config,
+ source_quantization_config=source_quantization_config,
enable_torch_compile=enable_torch_compile,
disable_opt_rtn=disable_opt_rtn,
+ index_dir=index_dir,
+ donor_shard_dir=donor_shard_dir,
+ donor_tensors_to_exclude=set(donor_tensors_to_exclude) if donor_tensors_to_exclude else None,
)
out_shard_name = f"model-{shard_idx + 1:05d}-of-{total_shards:05d}.safetensors"
@@ -1879,13 +376,12 @@ def _process_single_shard_task(
tensor_names = list(local_weight_map.keys())
clear_memory()
- if is_streaming:
+ if cleanup_source_shard:
try:
os.remove(shard_path)
except OSError:
pass
- # Return only lightweight metadata to avoid IPC transfer of tensor storages.
return shard_idx, shard_name, shard_path, out_shard_name, tensor_names, quantized, ignored
@@ -1921,293 +417,6 @@ def _force_cleanup_process_pool(pool: ProcessPoolExecutor | None) -> None:
# ---------------------------------------------------------------------------
-def _normalize_scheme(scheme: Union[str, QuantizationScheme]) -> QuantizationScheme:
- """Convert *scheme* to a :class:`QuantizationScheme` instance.
-
- Raises ``ValueError`` for unknown preset names and ``TypeError`` for
- unsupported types.
- """
- if isinstance(scheme, str):
- scheme_name = scheme.upper()
- if scheme_name not in PRESET_SCHEMES:
- raise ValueError(f"Unknown scheme '{scheme}'. Available: {list(PRESET_SCHEMES.keys())}")
- return preset_name_to_scheme(scheme_name)
- if isinstance(scheme, QuantizationScheme):
- return scheme
- raise TypeError(f"Unsupported scheme type: {type(scheme)}")
-
-
-def _apply_scheme_overrides(
- scheme: Union[str, QuantizationScheme],
- scheme_overrides: Optional[dict] = None,
-) -> QuantizationScheme:
- """Return the effective scheme after applying non-None overrides."""
- scheme_obj = copy.deepcopy(_normalize_scheme(scheme))
- if not scheme_overrides:
- return scheme_obj
-
- valid_fields = {field.name for field in fields(QuantizationScheme)}
- for key, value in scheme_overrides.items():
- if key in valid_fields and value is not None:
- setattr(scheme_obj, key, value)
- return scheme_obj
-
-
-def _validate_supported_scheme(
- scheme_obj: QuantizationScheme,
- scheme_input: Union[str, QuantizationScheme],
-) -> None:
- """Raise ``ValueError`` if *scheme_obj* is not supported by model-free.
-
- Model-free supports:
-
- * Integer weight-only quantization (sym/asym), ``bits ∈ {2, 4, 8}``,
- packed in the ``auto_round:auto_gptq`` format.
- * MXFP weight quantization (``data_type='mx_fp'``), ``bits ∈ {4, 8}``,
- ``group_size=32``, packed in ``mxfp4-pack-quantized`` / ``mxfp8-quantized``
- format (compressed-tensors compatible).
- """
- data_type = (scheme_obj.data_type or "int").lower()
- bits = scheme_obj.bits
- act_bits = scheme_obj.act_bits if scheme_obj.act_bits is not None else 16
-
- # Full-precision (BF16/FP16) default: all layers stay in full precision
- # unless layer_config provides lower-bit overrides.
- if (bits or 0) >= 16 and act_bits >= 16:
- return
-
- # MXFP weight-only path: accept mx_fp data type with bits in {4, 8}.
- # Activation quantization for MXFP is dynamic at inference time, so the
- # weight-only RTN path here is independent of act_bits.
- if is_mx_fp(data_type):
- if scheme_obj.act_data_type not in (None, "mx_fp"):
- raise ValueError(
- "Model-free MXFP supports only act_data_type='mx_fp', " f"but got '{scheme_obj.act_data_type}'."
- )
- # Restrict to the two explicitly supported MXFP presets when a string
- # name is provided. Variants such as MXFP4_RCEIL / MXFP8_RCEIL use a
- # different activation format; silently mapping them to "MXFP4" /
- # "MXFP8" in the output config would misrepresent the requested scheme.
- if isinstance(scheme_input, str) and scheme_input.upper() not in ("MXFP4", "MXFP8"):
- raise ValueError(
- f"Model-free mode only supports MXFP preset names 'MXFP4' and 'MXFP8', "
- f"but got '{scheme_input}'. "
- f"Supported preset schemes: {list(SUPPORTED_PRESET_SCHEMES)}."
- )
- if bits is None or bits not in _SUPPORTED_MXFP_BITS:
- raise ValueError(
- f"Model-free mode supports MXFP bits in {_SUPPORTED_MXFP_BITS}, "
- f"but '{scheme_input}' requests bits={bits}. "
- f"Supported preset schemes: {list(SUPPORTED_PRESET_SCHEMES)}."
- )
- group_size = scheme_obj.group_size
- if group_size not in (None, 32):
- raise ValueError(
- f"Model-free mode supports MXFP only with group_size=32, "
- f"but '{scheme_input}' requests group_size={group_size}."
- )
- return
-
- if act_bits < 16:
- raise ValueError(
- f"Model-free mode only supports weight-only quantization (WOQ) schemes "
- f"where act_bits >= 16, but '{scheme_input}' has act_bits={act_bits}. "
- f"Supported preset schemes: {list(SUPPORTED_PRESET_SCHEMES)}."
- )
-
- if data_type != "int":
- raise ValueError(
- f"Model-free mode only supports integer weight quantization "
- f"(data_type='int') or MXFP (data_type='mx_fp'), but '{scheme_input}' "
- f"has data_type='{data_type}'. FP8 / NVFP / GGUF / BF16 schemes require "
- f"the standard AutoRound flow. Supported preset schemes: "
- f"{list(SUPPORTED_PRESET_SCHEMES)}."
- )
-
- if bits is None or bits not in _SUPPORTED_INT_BITS:
- raise ValueError(
- f"Model-free mode supports bits in {_SUPPORTED_INT_BITS}, "
- f"but '{scheme_input}' requests bits={bits}. "
- f"Supported preset schemes: {list(SUPPORTED_PRESET_SCHEMES)}."
- )
-
-
-def is_model_free_supported_scheme(
- scheme: Union[str, QuantizationScheme],
- scheme_overrides: Optional[dict] = None,
-) -> bool:
- """Return True if *scheme* can be quantized via model-free mode.
-
- Useful for CLI auto-routing logic. Never raises.
- """
- try:
- scheme_obj = _apply_scheme_overrides(scheme, scheme_overrides)
- _validate_supported_scheme(scheme_obj, scheme)
- return True
- except (ValueError, TypeError):
- return False
-
-
-# ---------------------------------------------------------------------------
-# AutoScheme support (two-phase: delta-loss selection + model-free packing)
-# ---------------------------------------------------------------------------
-
-
-def _looks_like_auto_scheme(scheme: Any) -> bool:
- """Duck-typed check for an :class:`~auto_round.auto_scheme.AutoScheme`.
-
- Avoids importing ``AutoScheme`` at module scope (it pulls in exporter /
- compressor modules that would create an import cycle with this file).
- """
- return hasattr(scheme, "options") and hasattr(scheme, "avg_bits")
-
-
-def _validate_auto_scheme_options(auto_scheme: Any) -> str:
- """Validate AutoScheme options for model-free packing.
-
- Returns the single quantized data-type family shared by all options
- (``"int"`` or ``"mx_fp"``). ``BF16`` (or any >=16-bit no-op scheme) is
- allowed and treated as "keep layer in full precision" during conversion,
- so it does not contribute to the returned family.
-
- Raises ``ValueError`` when any non-BF16 option is unsupported or when INT
- and MXFP options are mixed (they use different packing formats and cannot
- be produced in one model-free run).
- """
- options = list(getattr(auto_scheme, "options", []) or [])
- if not options:
- raise ValueError("AutoScheme.options must be non-empty for model-free mode.")
-
- families: set[str] = set()
- unsupported: list[Any] = []
- for opt in options:
- # Preserve original string validation semantics so preset-name
- # restrictions (e.g. MXFP4/MXFP8 only) are enforced.
- if isinstance(opt, str):
- try:
- scheme_obj = _normalize_scheme(opt)
- except (ValueError, TypeError):
- scheme_obj = None
- elif isinstance(opt, QuantizationScheme):
- scheme_obj = opt
- else:
- scheme_obj = None
-
- # AutoScheme may include BF16/no-op options. In model-free flow these
- # layers become ignore_layers entries (full precision), so they are
- # allowed and excluded from family checks.
- if scheme_obj is not None:
- act_bits = scheme_obj.act_bits if scheme_obj.act_bits is not None else 16
- if (scheme_obj.bits or 0) >= 16 and act_bits >= 16:
- continue
-
- # GGUF k-quants carry super_bits and are not packable by the model-free
- # RTN kernel even though their data_type is nominally "int".
- if scheme_obj is None or getattr(scheme_obj, "super_bits", None) is not None:
- unsupported.append(opt)
- continue
- if not is_model_free_supported_scheme(opt):
- unsupported.append(opt)
- continue
-
- data_type = (scheme_obj.data_type or "int").lower()
- families.add("mx_fp" if is_mx_fp(data_type) else "int")
-
- if unsupported:
- raise ValueError(
- f"Model-free + AutoScheme received unsupported option(s): {unsupported}. "
- f"Model-free supports INT WOQ (bits in {_SUPPORTED_INT_BITS}) and MXFP "
- f"(bits in {_SUPPORTED_MXFP_BITS}); GGUF / NVFP4 / FP8 options are not "
- f"packable in model-free mode. Remove the unsupported options or pass "
- f"disable_model_free=True to use the regular flow."
- )
- if len(families) > 1:
- raise ValueError(
- "Model-free + AutoScheme cannot mix INT and MXFP options in a single run "
- f"(got families {sorted(families)}); INT and MXFP use different packing "
- "formats. Use a single data-type family, or pass disable_model_free=True."
- )
- return families.pop()
-
-
-def _convert_auto_scheme_layer_config(
- generated: dict[str, dict],
- preferred_base_scheme: Union[str, QuantizationScheme, None] = None,
-) -> tuple[QuantizationScheme, dict[str, dict], list[str]]:
- """Convert an AutoScheme-generated ``layer_config`` into model-free inputs.
-
- Returns ``(base_scheme, per_layer_overrides, fp16_layers)`` where:
-
- * ``base_scheme`` is ``preferred_base_scheme`` when provided, matching the
- primary scheme selected by the regular AutoRound path. Otherwise the
- most common quantized scheme is used as a fallback.
- * ``per_layer_overrides`` maps every quantized layer name to its resolved
- :class:`QuantizationScheme` fields.
- * ``fp16_layers`` lists layers AutoScheme kept at >= 16 bits (added to the
- model-free ignore list so they stay in full precision).
- """
- from collections import Counter
-
- scheme_keys = {f.name for f in fields(QuantizationScheme)}
- per_layer: dict[str, dict] = {}
- fp16_layers: list[str] = []
- counter: "Counter[tuple]" = Counter()
-
- for name, cfg in generated.items():
- if not isinstance(cfg, dict):
- continue
- bits = cfg.get("bits")
- data_type_raw = (cfg.get("data_type") or "").strip()
-
- # Infer bits from compact dtype aliases that embed bit-width in the name,
- # e.g. "mxfp8" → 8, "MXFP4" → 4, "mx_fp4" → 4.
- if bits is None and data_type_raw:
- dt_lower = data_type_raw.lower()
- for prefix in ("mxfp", "mx_fp", "nvfp", "nv_fp"):
- tail = dt_lower[len(prefix) :]
- if dt_lower.startswith(prefix) and tail.isdigit():
- bits = int(tail)
- break
-
- if bits is None:
- continue
-
- clean = {k: cfg[k] for k in scheme_keys if cfg.get(k) is not None}
- clean["bits"] = bits # honour inferred value when not explicit
-
- # Normalise compact dtype aliases to canonical forms understood by
- # the quantization kernels ("mxfp8" / "MXFP4" → "mx_fp").
- if data_type_raw:
- dt_lower = data_type_raw.lower()
- if dt_lower.startswith("mxfp") or dt_lower.startswith("mx_fp"):
- clean["data_type"] = "mx_fp"
- elif dt_lower.startswith("nvfp") or dt_lower.startswith("nv_fp"):
- clean["data_type"] = "nv_fp"
-
- if bits >= 16:
- fp16_layers.append(name)
- continue
- data_type = (clean.get("data_type") or "int").lower()
- per_layer[name] = clean
- counter[(clean.get("bits"), clean.get("group_size"), bool(clean.get("sym", True)), data_type)] += 1
-
- if not counter:
- raise ValueError("AutoScheme did not assign any quantizable layers for model-free mode.")
-
- if preferred_base_scheme is not None:
- base_scheme = copy.deepcopy(_normalize_scheme(preferred_base_scheme))
- else:
- (base_bits, base_group_size, base_sym, base_dtype), _ = counter.most_common(1)[0]
- base_scheme = QuantizationScheme(
- bits=base_bits,
- group_size=base_group_size,
- sym=base_sym,
- data_type=base_dtype,
- )
- return base_scheme, per_layer, fp16_layers
-
-
-# ---------------------------------------------------------------------------
# Main class
# ---------------------------------------------------------------------------
@@ -2250,6 +459,7 @@ class _ModelFreeCompressorCore:
"""
SUPPORTED_FORMATS: tuple[str, ...] = (
+ "fake",
"auto_round",
"auto_round:auto_gptq",
"llm_compressor",
@@ -2276,7 +486,7 @@ class _ModelFreeCompressorCore:
self.scheme_input = scheme
self.layer_config_input = layer_config
self.ignore_layers_input = ignore_layers
- self.format = format
+ self.format = format or "auto_round"
self.device = device
self.quant_lm_head = quant_lm_head
self.quant_nontext_module = quant_nontext_module
@@ -2305,6 +515,19 @@ class _ModelFreeCompressorCore:
self.all_ignored_layers: list[str] = []
self.output_weight_map: dict[str, str] = {}
self.shard_parallelism: int = 1
+ # Cross-shard FP8 scale_inv dependency maps built from index.json.
+ # cross_shard_deps: {recipient_shard: {donor_shard: [scale_inv_names]}}
+ # donor_shard_tensors: {donor_shard: set(scale_inv names it donates)}
+ self.cross_shard_deps: dict[str, dict[str, list[str]]] = {}
+ self.donor_shard_tensors: dict[str, set[str]] = {}
+ # Reference-counting state so donor shard cache files can be deleted as
+ # soon as they are no longer needed, instead of waiting until the end
+ # of the whole run. A donor file is safe to delete once (a) its own
+ # quantization task has read it, and (b) every recipient shard that
+ # depends on it has finished (and thus finished hydrating from it).
+ self._donor_remaining_recipients: dict[str, int] = {}
+ self._donor_self_consumed: dict[str, bool] = {}
+ self._donor_shard_paths: dict[str, str] = {}
# -------------------------------------------------------------------
# Validation / parsing
@@ -2333,6 +556,7 @@ class _ModelFreeCompressorCore:
_validate_supported_scheme(self.scheme_obj, self.scheme_input)
ds = asdict(self.scheme_obj)
self.default_scheme = {k: v for k, v in ds.items() if v is not None}
+ self.default_scheme["_output_format"] = self.format
def _parse_layer_config(self) -> None:
lc = copy.deepcopy(self.layer_config_input) if self.layer_config_input else {}
@@ -2402,9 +626,19 @@ class _ModelFreeCompressorCore:
def _resolve_source(self) -> None:
os.makedirs(self.output_dir, exist_ok=True)
- self.is_streaming = not _is_model_cached(self.model_name_or_path)
+ cached, cache_reason = _get_model_cache_status(self.model_name_or_path)
+ self.is_streaming = not cached
+ logger.info(
+ "Model-free source decision: %s (%s)",
+ "streaming" if self.is_streaming else "local/snapshot",
+ cache_reason,
+ )
if self.is_streaming:
- logger.info("Model not found locally or in cache — using streaming download mode.")
+ logger.info(
+ "Path selected: streaming mode. Will download metadata first "
+ "(snapshot_download with weight ignore patterns), then fetch "
+ "weight shards on demand (hf_hub_download)."
+ )
self.work_dir = self.output_dir
_download_metadata_files(self.model_name_or_path, self.work_dir)
transformer_work_dir = os.path.join(self.work_dir, "transformer")
@@ -2422,6 +656,10 @@ class _ModelFreeCompressorCore:
)
self.config = _load_config(self.work_dir)
else:
+ if os.path.isdir(self.model_name_or_path):
+ logger.info("Path selected: local directory mode. Reading shards from local path.")
+ else:
+ logger.info("Path selected: snapshot/local-cache mode. Resolving source dir via snapshot_download.")
self.source_dir = _resolve_source_dir(self.model_name_or_path)
transformer_source_dir = os.path.join(self.source_dir, "transformer")
if (
@@ -2505,6 +743,135 @@ class _ModelFreeCompressorCore:
search_dir = self.work_dir if self.is_streaming else self.source_dir
self.shard_names = _list_weight_shards(search_dir)
+ def _build_cross_shard_deps(self) -> None:
+ """Build cross-shard FP8 scale_inv dependency map from index.json.
+
+ Identifies triples (recipient_shard, donor_shard, scale_inv_tensor) by
+ comparing ``.weight`` and ``.weight_scale_inv`` entries in weight_map.
+ Stored in ``self.cross_shard_deps`` and ``self.donor_shard_tensors`` for
+ use by the shard scheduler and quantization workers.
+ """
+ search_dir = self.work_dir if self.is_streaming else self.source_dir
+ # Find the first *.safetensors.index.json in search_dir.
+ index_path: str | None = None
+ st_std = os.path.join(search_dir, "model.safetensors.index.json")
+ if os.path.exists(st_std):
+ index_path = st_std
+ else:
+ candidates = sorted(
+ os.path.join(search_dir, f) for f in os.listdir(search_dir) if f.endswith(".safetensors.index.json")
+ )
+ if candidates:
+ index_path = candidates[0]
+
+ if not index_path:
+ self.cross_shard_deps = {}
+ self.donor_shard_tensors = {}
+ return
+
+ try:
+ weight_map = _load_weight_map_from_index(index_path)
+ except Exception as e:
+ logger.warning(f"Could not load weight_map for cross-shard dep analysis: {e}")
+ self.cross_shard_deps = {}
+ self.donor_shard_tensors = {}
+ return
+
+ self.cross_shard_deps, self.donor_shard_tensors = _build_cross_shard_pairs_from_weight_map(weight_map)
+ if self.cross_shard_deps:
+ n_pairs = sum(len(t) for donors in self.cross_shard_deps.values() for t in donors.values())
+ logger.info(
+ f"Cross-shard FP8 dependencies: {n_pairs} scale_inv tensor(s), "
+ f"{len(self.cross_shard_deps)} recipient shard(s), "
+ f"{len(self.donor_shard_tensors)} donor shard(s)."
+ )
+
+ # Build reverse mapping (donor -> distinct recipient shards) so donor
+ # cache files can be reference-counted and deleted as soon as every
+ # dependent recipient has finished, instead of only at the very end.
+ donor_recipients: dict[str, set[str]] = {}
+ for recipient, donors in self.cross_shard_deps.items():
+ for donor in donors:
+ donor_recipients.setdefault(donor, set()).add(recipient)
+ self._donor_remaining_recipients = {donor: len(recipients) for donor, recipients in donor_recipients.items()}
+ self._donor_self_consumed = {donor: False for donor in self.donor_shard_tensors}
+
+ def _reorder_shards_by_dependency(self) -> None:
+ """Reorder shard_names via topological sort so donors precede recipients.
+
+ A single-pass "donors first, recipients last" partition is not enough:
+ a shard can simultaneously be a *recipient* (it needs a scale_inv from
+ another shard) and a *donor* (it donates a scale_inv to a third shard).
+ Such transitive chains (X donates to Y, Y donates to Z) must be
+ resolved with a proper topological sort, otherwise Y could still be
+ scheduled after Z, i.e. after one of its own dependents.
+
+ Ties are broken by original shard order (stable) so scheduling stays
+ as close as possible to the natural shard sequence. This ensures:
+ - Non-streaming: workers start on shards that need no cross-shard data.
+ - Streaming: donor shards are downloaded before recipient shards, so the
+ donor cache files are available when recipient workers run hydration.
+ """
+ if not self.cross_shard_deps:
+ return
+
+ original_index = {name: i for i, name in enumerate(self.shard_names)}
+ # recipient -> set(donors it depends on)
+ depends_on: dict[str, set[str]] = {
+ recipient: set(donors) for recipient, donors in self.cross_shard_deps.items()
+ }
+ # in-degree = number of unresolved donor dependencies for each shard.
+ in_degree: dict[str, int] = {name: len(depends_on.get(name, ())) for name in self.shard_names}
+ # donor -> shards that depend on it (only counting donors that are
+ # themselves known shards; unknown donors can't be waited on anyway).
+ dependents: dict[str, list[str]] = {}
+ for recipient, donors in depends_on.items():
+ for donor in donors:
+ if donor in original_index:
+ dependents.setdefault(donor, []).append(recipient)
+ elif recipient in in_degree:
+ # Donor shard is unknown/missing from discovery; it can
+ # never be satisfied here, so don't block scheduling on it.
+ in_degree[recipient] -= 1
+
+ import heapq
+
+ ready = [original_index[name] for name, deg in in_degree.items() if deg == 0]
+ heapq.heapify(ready)
+ ordered: list[str] = []
+ visited: set[str] = set()
+ index_to_name = {i: name for name, i in original_index.items()}
+
+ while ready:
+ idx = heapq.heappop(ready)
+ name = index_to_name[idx]
+ if name in visited:
+ continue
+ visited.add(name)
+ ordered.append(name)
+ for dependent in dependents.get(name, ()):
+ in_degree[dependent] -= 1
+ if in_degree[dependent] == 0:
+ heapq.heappush(ready, original_index[dependent])
+
+ if len(ordered) != len(self.shard_names):
+ # Cycle detected (shouldn't happen with well-formed indices) —
+ # fall back to appending any unresolved shards in original order.
+ remaining = [name for name in self.shard_names if name not in visited]
+ logger.warning(
+ f"Shard dependency graph has a cycle or unresolved node(s) "
+ f"({len(remaining)} shard(s)); appending them in original order."
+ )
+ ordered.extend(remaining)
+
+ n_recipients = len(self.cross_shard_deps)
+ if n_recipients:
+ logger.info(
+ f"Shard scheduling: topologically sorted {len(self.shard_names)} shard(s) "
+ f"({n_recipients} recipient shard(s) with cross-shard dependencies)."
+ )
+ self.shard_names = ordered
+
def _resolve_shard_parallelism(self) -> tuple[int, str]:
shard_count = len(self.shard_names)
# Auto policy: shard_count // 4, capped at 10, minimum 1.
@@ -2543,6 +910,10 @@ class _ModelFreeCompressorCore:
# -------------------------------------------------------------------
def _process_all_shards(self) -> None:
+ if self.is_streaming:
+ self._process_all_shards_streaming_pipeline()
+ return
+
try:
from tqdm import tqdm as _tqdm
except ImportError:
@@ -2559,6 +930,7 @@ class _ModelFreeCompressorCore:
try:
pool = ProcessPoolExecutor(max_workers=worker_count, mp_context=mp.get_context("spawn"))
for shard_idx, shard_name in enumerate(self.shard_names):
+ donor_tensors = list(self.donor_shard_tensors.get(shard_name, ())) or None
futures.append(
pool.submit(
_process_single_shard_task,
@@ -2574,11 +946,12 @@ class _ModelFreeCompressorCore:
ignore_patterns=self.ignore_patterns,
fp8_block_size=self.fp8_block_size,
model_type=self.model_type,
- source_quant_config=self.config.get("quantization_config"),
+ source_quantization_config=self.config.get("quantization_config", {}),
enable_torch_compile=self.enable_torch_compile,
disable_opt_rtn=self.disable_opt_rtn,
quant_output_dir=self._quant_output_dir,
total_shards=len(self.shard_names),
+ donor_tensors_to_exclude=donor_tensors,
)
)
@@ -2589,36 +962,8 @@ class _ModelFreeCompressorCore:
)
for future in shard_iter:
- shard_idx, shard_name, shard_path, out_shard_name, tensor_names, quantized, ignored = future.result()
-
- if (
- shard_path is None
- or out_shard_name is None
- or tensor_names is None
- or quantized is None
- or ignored is None
- ):
- logger.warning(f"Shard not found: {shard_name}, skipping")
- continue
-
- memory_monitor.update()
- clear_memory()
- if len(self.shard_names) > 1:
- logger.info(f"Memory usage: {memory_monitor.get_summary()}")
-
- compressed_quantized = compress_layer_names(quantized)
- compressed_ignored = compress_layer_names(ignored)
- logger.info(
- f"Shard {shard_idx + 1}/{len(self.shard_names)} ({shard_name}):\n"
- f" Quantized layers ({len(quantized)}): {compressed_quantized}\n"
- f" Ignored layers ({len(ignored)}): {compressed_ignored}"
- )
-
- self.all_quantized_layers.extend(quantized)
- self.all_ignored_layers.extend(ignored)
-
- for tensor_name in tensor_names:
- self.output_weight_map[tensor_name] = out_shard_name
+ result = future.result()
+ self._merge_shard_task_result(result)
except KeyboardInterrupt:
logger.warning("Interrupted by user; terminating model-free shard worker processes.")
_force_cleanup_process_pool(pool)
@@ -2629,6 +974,212 @@ class _ModelFreeCompressorCore:
finally:
_force_cleanup_process_pool(pool)
+ def _merge_shard_task_result(
+ self,
+ result: tuple[int, str, str | None, str | None, list[str] | None, list[str] | None, list[str] | None],
+ ) -> None:
+ """Merge one shard-task result into global stats and weight map."""
+ shard_idx, shard_name, shard_path, out_shard_name, tensor_names, quantized, ignored = result
+ if shard_path is None or out_shard_name is None or tensor_names is None or quantized is None or ignored is None:
+ logger.warning(f"Shard not found: {shard_name}, skipping")
+ return
+
+ memory_monitor.update()
+ clear_memory()
+ if len(self.shard_names) > 1:
+ logger.info(f"Memory usage: {memory_monitor.get_summary()}")
+
+ compressed_quantized = compress_layer_names(quantized)
+ compressed_ignored = compress_layer_names(ignored)
+ logger.info(
+ f"Shard {shard_idx + 1}/{len(self.shard_names)} ({shard_name}):\n"
+ f" Quantized layers ({len(quantized)}): {compressed_quantized}\n"
+ f" Ignored layers ({len(ignored)}): {compressed_ignored}"
+ )
+
+ self.all_quantized_layers.extend(quantized)
+ self.all_ignored_layers.extend(ignored)
+ for tensor_name in tensor_names:
+ self.output_weight_map[tensor_name] = out_shard_name
+
+ if self.is_streaming:
+ self._release_donor_dependency(shard_name)
+
+ def _release_donor_dependency(self, completed_shard_name: str) -> None:
+ """Update donor reference counts after *completed_shard_name* finishes.
+
+ Donor shard cache files are kept alive only as long as needed: once a
+ donor's own quantization task has read it AND every recipient shard
+ that depends on it has finished (and thus hydrated from it), the
+ cached file is deleted immediately rather than waiting for the whole
+ run to finish via :meth:`_cleanup_streaming_shard_cache`.
+ """
+ if completed_shard_name in self._donor_self_consumed:
+ self._donor_self_consumed[completed_shard_name] = True
+ self._maybe_cleanup_donor_shard(completed_shard_name)
+
+ for donor in self.cross_shard_deps.get(completed_shard_name, {}):
+ if donor in self._donor_remaining_recipients:
+ self._donor_remaining_recipients[donor] = max(0, self._donor_remaining_recipients[donor] - 1)
+ self._maybe_cleanup_donor_shard(donor)
+
+ def _maybe_cleanup_donor_shard(self, donor_name: str) -> None:
+ """Delete *donor_name*'s cached shard file once it is no longer needed."""
+ if not self._donor_self_consumed.get(donor_name, False):
+ return
+ if self._donor_remaining_recipients.get(donor_name, 0) > 0:
+ return
+ shard_path = self._donor_shard_paths.pop(donor_name, None)
+ # Remove tracking regardless of whether the file still exists so we
+ # never attempt to clean this donor up twice.
+ self._donor_self_consumed.pop(donor_name, None)
+ self._donor_remaining_recipients.pop(donor_name, None)
+ if shard_path and os.path.exists(shard_path):
+ try:
+ os.remove(shard_path)
+ logger.debug(f"Removed donor shard cache file (no longer needed): {donor_name}")
+ except OSError as e:
+ logger.warning(f"Failed to remove donor shard cache file {donor_name}: {e}")
+
+ def _process_all_shards_streaming_pipeline(self) -> None:
+ """Streaming-mode shard pipeline with dedicated downloader and quant workers.
+
+ Design:
+ - one downloader worker serializes network bandwidth usage;
+ - N quant workers consume ready shards independently;
+ - shards are assigned for quantization as soon as download completes.
+ """
+ try:
+ from tqdm import tqdm as _tqdm
+ except ImportError:
+ _tqdm = None
+
+ if not self.shard_names:
+ return
+
+ os.makedirs(self._quant_output_dir, exist_ok=True)
+
+ worker_count = max(1, min(self.shard_parallelism, len(self.shard_names)))
+ prefetch_depth = max(2, worker_count)
+ total_shards = len(self.shard_names)
+
+ download_pool: ThreadPoolExecutor | None = None
+ quant_pool: ProcessPoolExecutor | None = None
+ download_futures: dict = {}
+ quant_futures = set()
+ next_download_idx = 0
+ completed_quant = 0
+
+ def _submit_next_download() -> bool:
+ nonlocal next_download_idx
+ if next_download_idx >= total_shards:
+ return False
+ shard_idx = next_download_idx
+ shard_name = self.shard_names[shard_idx]
+ future = download_pool.submit(
+ _prefetch_shard,
+ self.model_name_or_path,
+ shard_name,
+ self.work_dir,
+ self.source_dir,
+ self.is_streaming,
+ )
+ download_futures[future] = (shard_idx, shard_name)
+ next_download_idx += 1
+ return True
+
+ try:
+ download_pool = ThreadPoolExecutor(max_workers=1)
+ quant_pool = ProcessPoolExecutor(max_workers=worker_count, mp_context=mp.get_context("spawn"))
+
+ for _ in range(min(prefetch_depth, total_shards)):
+ _submit_next_download()
+
+ progress = _tqdm(total=total_shards, desc="Processing shards", unit="shard") if _tqdm else None
+
+ while completed_quant < total_shards:
+ wait_set = set(download_futures.keys()) | set(quant_futures)
+ if not wait_set:
+ break
+
+ done, _ = wait(wait_set, return_when=FIRST_COMPLETED)
+ for future in done:
+ if future in download_futures:
+ shard_idx, shard_name = download_futures.pop(future)
+ shard_path = future.result()
+ if shard_path is None or not os.path.exists(shard_path):
+ logger.warning(f"Prefetch failed for shard {shard_name}, skipping")
+ completed_quant += 1
+ if progress is not None:
+ progress.update(1)
+ else:
+ # Donor shards must stay on disk until every recipient
+ # that depends on them has been processed (tracked via
+ # _donor_remaining_recipients / _release_donor_dependency),
+ # so that recipient shards can still read scale_inv
+ # tensors from them via
+ # _hydrate_missing_fp8_scales_from_index. They are
+ # deleted as soon as that's no longer needed; any
+ # stragglers are swept up at the end by
+ # _cleanup_streaming_shard_cache.
+ shard_cache_dir = os.path.join(self.work_dir, ".cache", "model_free_source_shards")
+ is_donor = shard_name in self.donor_shard_tensors
+ if is_donor:
+ self._donor_shard_paths[shard_name] = shard_path
+ donor_tensors = list(self.donor_shard_tensors.get(shard_name, ())) or None
+ qf = quant_pool.submit(
+ _quantize_local_shard_task,
+ shard_idx,
+ shard_name,
+ shard_path=shard_path,
+ device=self.device,
+ default_scheme=self.default_scheme,
+ layer_config=self.layer_config,
+ ignore_patterns=self.ignore_patterns,
+ fp8_block_size=self.fp8_block_size,
+ model_type=self.model_type,
+ source_quantization_config=self.config.get("quantization_config", {}),
+ quant_output_dir=self._quant_output_dir,
+ total_shards=total_shards,
+ enable_torch_compile=self.enable_torch_compile,
+ # Keep donor shards alive for recipient hydration.
+ cleanup_source_shard=not is_donor,
+ donor_tensors_to_exclude=donor_tensors,
+ # Pass the work_dir as index_dir so workers can
+ # find the index.json that was downloaded during
+ # metadata fetch (not in the shard cache sub-dir).
+ index_dir=self.work_dir,
+ donor_shard_dir=shard_cache_dir,
+ )
+ quant_futures.add(qf)
+
+ while len(download_futures) < prefetch_depth and _submit_next_download():
+ pass
+ elif future in quant_futures:
+ quant_futures.remove(future)
+ result = future.result()
+ self._merge_shard_task_result(result)
+ completed_quant += 1
+ if progress is not None:
+ progress.update(1)
+
+ if progress is not None:
+ progress.close()
+ except KeyboardInterrupt:
+ logger.warning("Interrupted by user; terminating model-free shard workers.")
+ _force_cleanup_process_pool(quant_pool)
+ raise
+ except Exception:
+ _force_cleanup_process_pool(quant_pool)
+ raise
+ finally:
+ _force_cleanup_process_pool(quant_pool)
+ if download_pool is not None:
+ try:
+ download_pool.shutdown(wait=False, cancel_futures=True)
+ except Exception:
+ pass
+
# -------------------------------------------------------------------
# Output
# -------------------------------------------------------------------
@@ -2646,6 +1197,7 @@ class _ModelFreeCompressorCore:
break
block_name_to_quantize = ",".join(dict.fromkeys(block_prefixes)) or None
+ os.makedirs(self._quant_output_dir, exist_ok=True)
quantization_config = _build_quantization_config(
default_scheme=self.default_scheme,
layer_config=self.layer_config,
@@ -2657,7 +1209,6 @@ class _ModelFreeCompressorCore:
)
self.config["quantization_config"] = quantization_config
- os.makedirs(self._quant_output_dir, exist_ok=True)
with open(os.path.join(self._quant_output_dir, "config.json"), "w") as f:
json.dump(self.config, f, indent=2)
@@ -2761,6 +1312,8 @@ class _ModelFreeCompressorCore:
self._detect_fp8_source()
self._resolve_model_type()
self._discover_shards()
+ self._build_cross_shard_deps()
+ self._reorder_shards_by_dependency()
self.shard_parallelism, shard_parallelism_source = self._resolve_shard_parallelism()
# Determine the output packing format based on scheme data type
@@ -2768,6 +1321,8 @@ class _ModelFreeCompressorCore:
if is_mx_fp(data_type):
bits = self.default_scheme.get("bits", 4)
packing_format = "mxfp4-pack-quantized" if bits == 4 else "mxfp8-quantized"
+ elif data_type == _NVFP4_E5M3_DATA_TYPE:
+ packing_format = "fake" if self.format == "fake" else "auto_round:llm_compressor_nvfp4_e5m3"
else:
packing_format = "auto_round:auto_gptq"
if is_mx_fp(data_type) or _layer_config_has_mxfp(self.layer_config):
@@ -3172,9 +1727,14 @@ class ModelFreeCompressor(_ModelFreeCompressorCore):
normalized_scheme is not None and is_mx_fp((normalized_scheme.data_type or "").lower())
) or self._auto_scheme_family == "mx_fp":
_accepted_formats = {"llm_compressor", "auto_round", "auto_round:auto_gptq"}
+ elif normalized_scheme is not None and (normalized_scheme.data_type or "").lower() == _NVFP4_E5M3_DATA_TYPE:
+ _accepted_formats = {"fake", "llm_compressor", "auto_round", "auto_round:auto_gptq"}
elif _is_full_precision_default(self.scheme_input) and _layer_config_has_mxfp(self.layer_config_input):
# BF16 default with MXFP layer_config overrides.
_accepted_formats = {"llm_compressor", "auto_round", "auto_round:auto_gptq"}
+ elif _is_full_precision_default(self.scheme_input) and _layer_config_has_nvfp4(self.layer_config_input):
+ # BF16 default with NVFP4_E5M3 layer_config overrides.
+ _accepted_formats = {"fake", "llm_compressor", "auto_round", "auto_round:auto_gptq"}
if format not in _accepted_formats:
logger.warning(
f"Format '{format}' is not supported by model-free mode for scheme '{self.scheme_input}'; "
@@ -3198,119 +1758,3 @@ class ModelFreeCompressor(_ModelFreeCompressorCore):
self.format = orig_fmt
self.quantized = True
return None, out_path
-
-
-# ---------------------------------------------------------------------------
-# Model-Type Specific Preprocessing Hooks (Extension Point)
-# ---------------------------------------------------------------------------
-#
-# Keep model-specific source-format adaptation functions at the end of this
-# file so the core quantization pipeline remains easy to read and maintain.
-# Add new model handlers here, keyed by `model_type`, and keep dequant/passthrough
-# decisions in the generic MXFP handlers above.
-
-
-def _expand_e8m0_block_scale(
- scale: torch.Tensor,
- out_features: int,
- in_features: int,
- group_size: int = 32,
-) -> torch.Tensor:
- """Expand a coarse 2D E8M0 block scale to the llm-compressor per-group layout.
-
- deepseek_v4 stores the per-block shared exponent in a *coarse* 2D shape
- ``[out_features // block_h, in_features // block_w]`` (e.g. ``[12, 56]`` for
- a ``[1536, 7168]`` weight, i.e. 128x128 blocks). llm-compressor expects a
- per-group scale of shape ``[out_features, in_features // group_size]``
- (e.g. ``[1536, 224]`` for ``group_size=32``).
-
- Because every fine MXFP group lies entirely inside a single coarse block,
- the expansion is a pure ``repeat_interleave`` along both axes (no
- interpolation). The returned tensor is ``uint8`` (raw E8M0 bytes), matching
- the ``U8`` dtype used by llm-compressor ``weight_scale`` tensors.
- """
- scale = scale.view(torch.uint8)
- if scale.dim() != 2:
- raise ValueError(f"Expected a 2D E8M0 block scale, got shape {tuple(scale.shape)}.")
-
- target_rows = out_features
- target_cols = in_features // group_size
- rows, cols = scale.shape
-
- if target_rows % rows != 0 or target_cols % cols != 0:
- raise ValueError(
- f"Cannot expand E8M0 block scale {tuple(scale.shape)} to "
- f"({target_rows}, {target_cols}); shapes are not divisible."
- )
-
- if target_rows != rows:
- scale = scale.repeat_interleave(target_rows // rows, dim=0)
- if target_cols != cols:
- scale = scale.repeat_interleave(target_cols // cols, dim=1)
- return scale.contiguous()
-
-
-def _preprocess_model_type_source_tensors(
- raw_tensors: dict[str, torch.Tensor],
- model_type: str | None,
- group_size: int = 32,
-) -> tuple[dict[str, torch.Tensor], dict[str, int]]:
- """Apply model-type-specific source tensor normalization.
-
- This step is intentionally limited to *format conversion* and does not do
- passthrough / dequant decisions. It marks converted layers in the returned
- ``source_state`` so downstream generic MXFP handling can treat them exactly
- like normal llm-compressor MXFP sources.
-
- Returns:
- ``(raw_tensors, source_state)`` where ``source_state[layer]`` is the
- source MXFP bits (4 or 8) for model-type preprocessed layers.
- """
- if (model_type or "").lower() != "deepseek_v4":
- return raw_tensors, {}
-
- entries: list[tuple[str, str, bool]] = [] # (weight_name, scale_name, is_fp8)
- for name, tensor in raw_tensors.items():
- if not name.endswith(".weight"):
- continue
- scale_name = name[: -len(".weight")] + ".scale"
- if scale_name not in raw_tensors:
- continue
- if tensor.dtype == torch.float8_e4m3fn:
- entries.append((name, scale_name, True))
- elif tensor.dtype in (torch.int8, torch.uint8):
- entries.append((name, scale_name, False))
-
- if not entries:
- return raw_tensors, {}
-
- source_state: dict[str, int] = {}
- n_fp8 = 0
- n_fp4 = 0
- for weight_name, scale_name, is_fp8 in entries:
- layer_name = weight_name[: -len(".weight")]
- weight = raw_tensors.pop(weight_name)
- scale = raw_tensors.pop(scale_name)
-
- if is_fp8:
- out_features, in_features = weight.shape
- weight_key = f"{layer_name}.weight"
- source_state[layer_name] = 8
- n_fp8 += 1
- else:
- out_features = weight.shape[0]
- in_features = weight.shape[1] * 2
- weight = weight.view(torch.uint8).contiguous()
- weight_key = f"{layer_name}.weight_packed"
- source_state[layer_name] = 4
- n_fp4 += 1
-
- weight_scale = _expand_e8m0_block_scale(scale, out_features, in_features, group_size=group_size)
- raw_tensors[weight_key] = weight
- raw_tensors[f"{layer_name}.weight_scale"] = weight_scale
-
- logger.info(
- "Applied model_type preprocessing for deepseek_v4: "
- f"{n_fp8} MXFP8 layer(s), {n_fp4} MXFP4 layer(s) converted to llm-compressor naming."
- )
- return raw_tensors, source_state
diff --git a/auto_round/data_type/nvfp.py b/auto_round/data_type/nvfp.py
index bab5b5410..f72c875ea 100644
--- a/auto_round/data_type/nvfp.py
+++ b/auto_round/data_type/nvfp.py
@@ -206,14 +206,19 @@ def ref_fp4_quant(x, global_scale, block_size=16, v=0, max_scale=1.0):
scale = global_scale * (vec_max * get_reciprocal(FLOAT4_E2M1_MAX))
scale = torch.clip(scale, 0, FLOAT8_UE5M3_MAX)
scale = cast_to_ue5m3_ste(scale).to(torch.float32)
- output_scale = get_reciprocal(scale * get_reciprocal(global_scale))
- scaled_x = x.to(torch.float32) * output_scale + v
+ dequant_scale = scale * get_reciprocal(global_scale)
+ scaled_x = torch.where(
+ dequant_scale == 0,
+ torch.zeros_like(x, dtype=torch.float32),
+ x.to(torch.float32) / dequant_scale,
+ )
+ scaled_x = scaled_x + v
clipped_x = torch.clamp(scaled_x, -6.0, 6.0)
- return (cast_to_fp4(clipped_x) * get_reciprocal(output_scale)).reshape(m, n), scale
+ return (cast_to_fp4(clipped_x) * dequant_scale).reshape(m, n), scale
-@register_dtype("fp4_v2_with_global_scale")
-def fp4_v2_with_global_scale(tensor, bits=4, group_size=16, v=0, tensor_max=None, max_scale=1.0, **kwargs):
+@register_dtype("nvfp4_v2_with_global_scale")
+def nvfp4_v2_with_global_scale(tensor, bits=4, group_size=16, v=0, tensor_max=None, max_scale=1.0, **kwargs):
assert group_size == 32 or group_size == 16
orig_dtype = tensor.dtype
tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)
@@ -230,8 +235,8 @@ def fp4_v2_with_global_scale(tensor, bits=4, group_size=16, v=0, tensor_max=None
return qdq_res.to(orig_dtype), scale, None
-@register_dtype("fp4_v2")
-def fp4_v2(tensor, bits=4, group_size=32, v=0, max_scale=1.0, **kwargs):
+@register_dtype("nvfp4_v2")
+def nvfp4_v2(tensor, bits=4, group_size=32, v=0, max_scale=1.0, **kwargs):
assert group_size == 32 or group_size == 16
orig_dtype = tensor.dtype
tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)
diff --git a/auto_round/envs.py b/auto_round/envs.py
index 97cda8a6b..7c727443c 100644
--- a/auto_round/envs.py
+++ b/auto_round/envs.py
@@ -28,6 +28,7 @@ if TYPE_CHECKING:
AR_AUTO_SCHEME_BATCH_SIZE: Optional[int] = None
AR_AUTO_SCHEME_CACHE: Optional[str] = None
AR_ENABLE_AUTO_SCHEME_PARALLEL: bool = True
+ AR_NVFP4_E5M3_CACHE_HP_WEIGHT: bool = False
AR_DISK_STREAM_MODEL: bool = False
AR_RESUME_DIR: Optional[str] = None
@@ -105,6 +106,13 @@ environment_variables: dict[str, Callable[[], Any]] = {
# set it to 0 when workers could exhaust host RAM or device memory.
"AR_ENABLE_AUTO_SCHEME_PARALLEL": lambda: os.getenv("AR_ENABLE_AUTO_SCHEME_PARALLEL", "1").lower()
in ("1", "true", "yes"),
+ # Controls whether NVFP4 E5M3 quant linear caches a dequantized high-
+ # precision weight after the first forward instead of dequantizing on
+ # every call. When enabled, the packed weight buffers are released after
+ # the cache is materialized, trading lower runtime overhead for higher
+ # steady-state memory usage.
+ "AR_NVFP4_E5M3_CACHE_HP_WEIGHT": lambda: os.getenv("AR_NVFP4_E5M3_CACHE_HP_WEIGHT", "0").lower()
+ in ("1", "true", "yes", "on"),
# When set, the model is built as a meta-device skeleton and streamed
# block-by-block from disk during quantization instead of being fully
# materialized on CPU RAM up front.
diff --git a/auto_round/experimental/qmodules/__init__.py b/auto_round/experimental/qmodules/__init__.py
index 4f5ff8e82..e638f95bb 100644
--- a/auto_round/experimental/qmodules/__init__.py
+++ b/auto_round/experimental/qmodules/__init__.py
@@ -19,5 +19,7 @@ from auto_round.experimental.qmodules.mx import (
MXINT4QuantLinear,
)
+from auto_round.experimental.qmodules.fake import FakeActQuantLinear
from auto_round.experimental.qmodules.nvfp4 import NVFP4QuantLinear
+from auto_round.experimental.qmodules.nvfp4_e5m3 import CuteNVFP4E5M3QuantLinear, NVFP4E5M3QuantLinear
from auto_round.experimental.qmodules.fp8_static import WeightFP8ActFP8StaticQuantLinear
diff --git a/auto_round/experimental/qmodules/fake.py b/auto_round/experimental/qmodules/fake.py
new file mode 100644
index 000000000..0089a2555
--- /dev/null
+++ b/auto_round/experimental/qmodules/fake.py
@@ -0,0 +1,87 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Optional
+
+import torch
+
+from auto_round.data_type.utils import get_quant_func
+from auto_round.experimental.qmodules.base import QModuleBase
+from auto_round.schemes import QuantizationScheme
+
+__all__ = ["FakeActQuantLinear"]
+
+
+class FakeActQuantLinear(QModuleBase):
+ """Linear with high-precision QDQ weights and runtime activation QDQ."""
+
+ def __init__(
+ self,
+ in_features: int,
+ out_features: int,
+ config: QuantizationScheme,
+ weight: Optional[torch.Tensor] = None,
+ bias: Optional[torch.Tensor] = None,
+ dtype: torch.dtype = torch.bfloat16,
+ ):
+ super().__init__()
+ self.in_features = in_features
+ self.out_features = out_features
+ self.config = config
+ if weight is None:
+ weight = torch.empty((out_features, in_features), dtype=dtype)
+ self.weight = torch.nn.Parameter(weight, requires_grad=False)
+ if bias is None:
+ self.register_parameter("bias", None)
+ else:
+ self.bias = torch.nn.Parameter(bias, requires_grad=False)
+
+ @classmethod
+ def from_original(cls, config: QuantizationScheme, original_layer: torch.nn.Linear):
+ return cls(
+ in_features=original_layer.in_features,
+ out_features=original_layer.out_features,
+ config=config,
+ weight=original_layer.weight,
+ bias=original_layer.bias,
+ dtype=original_layer.weight.dtype,
+ )
+
+ @classmethod
+ def get_min_capability(cls) -> int:
+ return 0
+
+ def process_weights_after_loading(self, layer: torch.nn.Module):
+ return
+
+ def post_init(self):
+ return
+
+ def qdq_input(self, activation: torch.Tensor) -> torch.Tensor:
+ quant_func, _ = get_quant_func(
+ dtype=self.config.act_data_type,
+ bits=self.config.act_bits,
+ sym=self.config.act_sym,
+ )
+ qdq_activation, _, _ = quant_func(
+ tensor=activation,
+ bits=self.config.act_bits,
+ group_size=self.config.act_group_size,
+ )
+ return qdq_activation.to(activation.dtype)
+
+ @torch.inference_mode()
+ def forward(self, activation: torch.Tensor) -> torch.Tensor:
+ qdq_activation = self.qdq_input(activation)
+ return torch.nn.functional.linear(qdq_activation, self.weight.to(qdq_activation.dtype), self.bias)
diff --git a/auto_round/experimental/qmodules/nvfp4_e5m3.py b/auto_round/experimental/qmodules/nvfp4_e5m3.py
new file mode 100644
index 000000000..c8b115e7e
--- /dev/null
+++ b/auto_round/experimental/qmodules/nvfp4_e5m3.py
@@ -0,0 +1,161 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+from typing import Optional, Union
+
+import torch
+
+from auto_round.data_type.nvfp import e5m3_to_float_tensor, nvfp4_v2
+from auto_round.experimental.qmodules.base import QModuleBase
+from auto_round.experimental.qmodules.fp4_utils import unpack_fp4_from_uint8
+from auto_round.logger import logger
+from auto_round.schemes import QuantizationScheme
+from auto_round_extension.cuda.cute_nvfp4_e5m3 import (
+ try_cute_nvfp4_e5m3_linear,
+ try_cute_nvfp4_e5m3_weight_dq,
+ try_cute_nvfp4_v2_qdq,
+)
+
+__all__ = ["CuteNVFP4E5M3QuantLinear", "NVFP4E5M3QuantLinear"]
+
+_CACHE_WEIGHT_ENV = "AR_NVFP4_E5M3_CACHE_HP_WEIGHT"
+
+
+def _resolve_cache_weight(cache_weight: Optional[bool], default: bool) -> bool:
+ if cache_weight is not None:
+ return cache_weight
+ value = os.getenv(_CACHE_WEIGHT_ENV)
+ if value is None:
+ return default
+ return value.strip().lower() not in {"0", "false", "no", "off"}
+
+
+class NVFP4E5M3QuantLinear(QModuleBase):
+ """FP4 E2M1 weights and activations with unsigned E5M3 block scales."""
+
+ SUPPORTED_COMPUTE_DTYPE = [torch.bfloat16, torch.float16, torch.float32]
+ DEFAULT_CACHE_WEIGHT = False
+
+ def __init__(
+ self,
+ in_features: int,
+ out_features: int,
+ config: QuantizationScheme,
+ weight: Optional[torch.Tensor] = None,
+ weight_scale: Optional[torch.Tensor] = None,
+ bias: Union[torch.Tensor, bool, None] = None,
+ dtype=torch.bfloat16,
+ cache_weight: Optional[bool] = None,
+ ):
+ super().__init__()
+ assert dtype in self.SUPPORTED_COMPUTE_DTYPE
+ assert config.group_size == 16 and config.act_group_size == 16
+ self.in_features = in_features
+ self.out_features = out_features
+ self.group_size = config.group_size
+ self.config = config
+ self.dtype = dtype
+ self.cache_weight = _resolve_cache_weight(cache_weight, self.DEFAULT_CACHE_WEIGHT)
+ self._cached_weight = None
+
+ packed_weight = torch.zeros((out_features, in_features // 2), dtype=torch.uint8) if weight is None else weight
+ self.register_buffer("weight_packed", packed_weight)
+ scale = (
+ torch.empty((out_features, in_features // self.group_size), dtype=torch.uint8)
+ if weight_scale is None
+ else weight_scale
+ )
+ self.register_buffer("weight_scale", scale)
+
+ if bias is not None:
+ if isinstance(bias, bool):
+ bias = torch.zeros((out_features,), dtype=dtype)
+ self.bias = torch.nn.Parameter(bias, requires_grad=False)
+ else:
+ self.register_parameter("bias", None)
+
+ @classmethod
+ def get_min_capability(cls) -> int:
+ logger.warning_once("NVFP4 E5M3 quantization uses reference PyTorch inference and may be slow.")
+ return 0
+
+ def dequant_weight_online(self) -> torch.Tensor:
+ unpacked = unpack_fp4_from_uint8(self.weight_packed, self.out_features, self.in_features, dtype=self.dtype).to(
+ torch.float32
+ )
+ scale = e5m3_to_float_tensor(self.weight_scale).reshape(-1, 1)
+ return (unpacked.reshape(-1, self.group_size) * scale).reshape(self.out_features, self.in_features)
+
+ @property
+ def weight(self) -> torch.Tensor:
+ if self._cached_weight is None:
+ self._cached_weight = self.dequant_weight_online()
+ if self.cache_weight:
+ self.weight_packed = None
+ self.weight_scale = None
+ return self._cached_weight
+
+ def clear_weight_cache(self) -> None:
+ if self.weight_packed is None:
+ raise RuntimeError("Cannot clear the cached weight after quantized weight buffers have been released.")
+ self._cached_weight = None
+
+ def qdq_input(self, activation: torch.Tensor) -> torch.Tensor:
+ original_dtype = activation.dtype
+ qdq_activation, _, _ = nvfp4_v2(
+ activation.to(torch.float32), bits=self.config.act_bits, group_size=self.config.act_group_size
+ )
+ return qdq_activation.to(original_dtype)
+
+ @torch.inference_mode()
+ def forward(self, input: torch.Tensor) -> torch.Tensor:
+ qdq_input = self.qdq_input(input)
+ weight = self.weight if self.cache_weight else self.dequant_weight_online()
+ return torch.nn.functional.linear(qdq_input, weight.to(qdq_input.dtype), self.bias)
+
+ @classmethod
+ def from_original(cls, config: QuantizationScheme, original_layer: torch.nn.Linear):
+ return cls(
+ in_features=original_layer.in_features,
+ out_features=original_layer.out_features,
+ config=config,
+ bias=original_layer.bias,
+ dtype=original_layer.weight.dtype,
+ )
+
+
+class CuteNVFP4E5M3QuantLinear(NVFP4E5M3QuantLinear):
+ """NVFP4 E5M3 linear that dispatches activation QDQ and GEMM to CuTe."""
+
+ def dequant_weight_online(self) -> torch.Tensor:
+ cute_weight = try_cute_nvfp4_e5m3_weight_dq(self.weight_packed, self.weight_scale, self.dtype)
+ if cute_weight is not None:
+ return cute_weight
+ return super().dequant_weight_online()
+
+ def qdq_input(self, activation: torch.Tensor) -> torch.Tensor:
+ cute_qdq_activation = try_cute_nvfp4_v2_qdq(activation, self.config.act_group_size)
+ if cute_qdq_activation is not None:
+ return cute_qdq_activation
+ return super().qdq_input(activation)
+
+ @torch.inference_mode()
+ def forward(self, input: torch.Tensor) -> torch.Tensor:
+ if self.cache_weight:
+ return super().forward(input)
+ fused_output = try_cute_nvfp4_e5m3_linear(input, self.weight_packed, self.weight_scale, self.bias)
+ if fused_output is not None:
+ return fused_output
+ return super().forward(input)
diff --git a/auto_round/export/export_to_autoround/qlinear_fp.py b/auto_round/export/export_to_autoround/qlinear_fp.py
index 338d624a7..039c5e0b3 100644
--- a/auto_round/export/export_to_autoround/qlinear_fp.py
+++ b/auto_round/export/export_to_autoround/qlinear_fp.py
@@ -36,7 +36,7 @@ import transformers
import auto_round.envs as envs
from auto_round.compressors.utils import BackendDataType, is_mx_fp, is_nv_fp
from auto_round.data_type.mxfp import FP32_EXPONENT_BIAS, FP32_MIN_NORMAL
-from auto_round.data_type.nvfp import cast_to_fp4, get_reciprocal
+from auto_round.data_type.nvfp import cast_to_fp4, float_to_e5m3_frexp, get_reciprocal
from auto_round.data_type.utils import reshape_pad_tensor_by_group_size, revert_tensor_by_pad
from auto_round.utils import get_packing_device, logger
@@ -73,6 +73,7 @@ class QuantLinear(nn.Module):
raise NotImplementedError("Only 4,8 bits are supported.")
self.is_mx = is_mx_fp(data_type)
self.is_nv = is_nv_fp(data_type)
+ self.is_nvfp4_e5m3 = data_type == "nvfp4_v2"
if self.is_mx:
if group_size != 32:
raise NotImplementedError(f"Only group_size 32 are supported for {BackendDataType.MX_FP} data type.")
@@ -80,7 +81,7 @@ class QuantLinear(nn.Module):
raise NotImplementedError(
f"in_feature must be divisible by {group_size} for {BackendDataType.MX_FP} data type."
)
- if self.is_nv:
+ if self.is_nv or self.is_nvfp4_e5m3:
if group_size % 16 != 0:
raise NotImplementedError(f"Only group_size 16 are supported for {BackendDataType.NV_FP} data type.")
if infeatures % group_size != 0:
@@ -159,11 +160,17 @@ class QuantLinear(nn.Module):
)
scaled_tensor.clamp_(-6.0, 6.0)
scaled_tensor = cast_to_fp4(scaled_tensor)
+ elif self.is_nvfp4_e5m3:
+ scaled_tensor = tensor / scales.reshape(tensor.shape[0], -1)
+ scaled_tensor.clamp_(-6.0, 6.0)
+ scaled_tensor = cast_to_fp4(scaled_tensor)
else:
scaled_tensor = tensor / (2 ** scales.reshape(tensor.shape[0], -1))
scaled_tensor = revert_tensor_by_pad(scaled_tensor, orig_shape=orig_shape, pad_len=pad_len)
if self.is_mx:
final_scale = (scales + E8M0_EXPONENT_BIAS).clamp(0, E8M0_EXPONENT_NAN_VAL).to(torch.uint8)
+ elif self.is_nvfp4_e5m3:
+ final_scale = float_to_e5m3_frexp(scales.to(torch.float32))
else:
final_scale = scales.to(torch.float8_e4m3fn)
diff --git a/auto_round/export/export_to_llmcompressor/config.py b/auto_round/export/export_to_llmcompressor/config.py
index 62017ec73..1f5090e6c 100644
--- a/auto_round/export/export_to_llmcompressor/config.py
+++ b/auto_round/export/export_to_llmcompressor/config.py
@@ -102,3 +102,38 @@ def initialize_quantization(scheme, targets=["Linear"], config_groups=None, kv_c
quantization_status=QuantizationStatus.COMPRESSED,
ignore=ignore,
)
+
+
+def initialize_nvfp4_e5m3_quantization(ignore=None):
+ """Build stable compressed-tensors metadata for global-scale-free NVFP4 E5M3."""
+
+ def quant_args(dynamic):
+ return {
+ "actorder": None,
+ "block_structure": None,
+ "dynamic": dynamic,
+ "group_size": 16,
+ "num_bits": 4,
+ "observer": "minmax",
+ "observer_kwargs": {},
+ "strategy": "tensor_group",
+ "symmetric": True,
+ "type": "float",
+ }
+
+ return {
+ "config_groups": {
+ "group_0": {
+ "input_activations": quant_args("local"),
+ "output_activations": None,
+ "targets": ["Linear"],
+ "weights": quant_args(False),
+ }
+ },
+ "format": "nvfp4-e5m3-pack-quantized",
+ "global_compression_ratio": None,
+ "ignore": list(dict.fromkeys(ignore or ["lm_head"])),
+ "kv_cache_scheme": None,
+ "quant_method": "compressed-tensors",
+ "quantization_status": "compressed",
+ }
diff --git a/auto_round/export/export_to_llmcompressor/export_to_fp.py b/auto_round/export/export_to_llmcompressor/export_to_fp.py
index c061095da..f207a035d 100644
--- a/auto_round/export/export_to_llmcompressor/export_to_fp.py
+++ b/auto_round/export/export_to_llmcompressor/export_to_fp.py
@@ -140,6 +140,8 @@ def _get_scheme(bits, data_type):
return "MXFP4" if bits == 4 else "MXFP8"
if is_nv_fp(data_type):
return "NVFP4"
+ if data_type == "nvfp4_v2":
+ return "NVFP4_E5M3"
return None
@@ -149,6 +151,8 @@ def _get_group_format(bits, data_type):
return "mxfp4-pack-quantized" if bits == 4 else "mxfp8-quantized"
if is_nv_fp(data_type):
return "nvfp4-pack-quantized"
+ if data_type == "nvfp4_v2":
+ return "nvfp4-e5m3-pack-quantized"
return "float-quantized"
@@ -349,6 +353,11 @@ def save_quantized_as_fp(
static_kv_granularity=static_kv_granularity,
static_attention_granularity=static_attention_granularity,
)
+ elif data_type == "nvfp4_v2":
+ from auto_round.export.export_to_llmcompressor.config import initialize_nvfp4_e5m3_quantization
+
+ quantization_config = initialize_nvfp4_e5m3_quantization(ignore=ignore)
+ quantization_config["provider"] = "auto-round"
else:
scheme = _get_scheme(bits, data_type)
if scheme is None:
diff --git a/auto_round/export/formats/backends/autoround.py b/auto_round/export/formats/backends/autoround.py
index 21ab8daaa..4d570b26f 100644
--- a/auto_round/export/formats/backends/autoround.py
+++ b/auto_round/export/formats/backends/autoround.py
@@ -41,6 +41,7 @@ class AutoRoundFormat(OutputFormat):
"MXFP4",
"MXFP8",
"NVFP4",
+ "NVFP4_E5M3",
"FPW8A16",
"W2A16G64",
"W2A16G32",
@@ -67,7 +68,7 @@ class AutoRoundFormat(OutputFormat):
)
if enable_awq:
self.backend = AutoAWQFormat("auto_round:auto_awq", scheme, ctx)
- elif scheme.is_nv_fp() or scheme.is_mx_fp():
+ elif scheme.is_nv_fp() or scheme.is_mx_fp() or scheme.data_type == BackendDataType.NVFP4_E5M3.value:
self.backend = AutoRoundFormat(scheme.data_type, scheme, ctx)
elif scheme.is_mx_int() and scheme.bits == 4: # only add mx_int4 now
self.backend = AutoRoundFormat(scheme.data_type, scheme, ctx)
@@ -87,7 +88,9 @@ class AutoRoundFormat(OutputFormat):
elif not format.startswith("auto_round"):
if format == "mlx":
self.backend = MLXFormat("mlx", scheme, ctx)
- elif format.upper() not in list(BackendDataType.__members__.keys()):
+ elif format.upper() not in list(BackendDataType.__members__.keys()) and format not in {
+ BackendDataType.NVFP4_E5M3.value
+ }:
raise KeyError(f"Unsupported backend format auto_round:{format}, please check")
else:
self.output_format = f"auto_round:{format}"
@@ -131,7 +134,11 @@ class AutoRoundFormat(OutputFormat):
backend = self.get_backend_name()
- if self.output_format in [
+ if self.output_format == f"auto_round:{BackendDataType.NVFP4_E5M3.value}":
+ from auto_round.export.export_to_llmcompressor.export_to_fp import pack_layer
+
+ return pack_layer(layer_name, model, device=device)
+ elif self.output_format in [
f"auto_round:{BackendDataType.NV_FP.value}",
f"auto_round:{BackendDataType.MX_FP.value}",
f"auto_round:{BackendDataType.MX_FP_RCEIL.value}",
@@ -185,7 +192,12 @@ class AutoRoundFormat(OutputFormat):
**kwargs,
)
backend = self.get_backend_name()
- if re.search(f"{BackendDataType.MX_FP.value}|{BackendDataType.NV_FP.value}", backend):
+ if backend == f"auto_round:{BackendDataType.NVFP4_E5M3.value}":
+ from auto_round.export.export_to_autoround.export_to_nvfp_mx import save_quantized_as_fp
+
+ backend = "auto_round:llm_compressor_nvfp4_e5m3"
+ export_func = save_quantized_as_fp
+ elif re.search(f"{BackendDataType.MX_FP.value}|{BackendDataType.NV_FP.value}", backend):
from auto_round.export.export_to_autoround.export_to_nvfp_mx import save_quantized_as_fp
backend = "auto_round:llm_compressor"
diff --git a/auto_round/export/formats/backends/fake.py b/auto_round/export/formats/backends/fake.py
index 6c1e9a703..069d1aeae 100644
--- a/auto_round/export/formats/backends/fake.py
+++ b/auto_round/export/formats/backends/fake.py
@@ -12,6 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+import copy
from typing import Any, Callable, Union
import torch
@@ -22,6 +23,16 @@ from auto_round.schemes import QuantizationScheme
from auto_round.utils import copy_python_files_from_model_cache, unsupported_meta_device
+def _serialize_quantization_config_value(value):
+ if isinstance(value, torch.dtype):
+ return str(value)
+ if isinstance(value, dict):
+ return {key: _serialize_quantization_config_value(item) for key, item in value.items()}
+ if isinstance(value, (list, tuple)):
+ return [_serialize_quantization_config_value(item) for item in value]
+ return value
+
+
@OutputFormat.register("fake")
class FakeFormat(OutputFormat):
support_schemes = None
@@ -47,7 +58,46 @@ class FakeFormat(OutputFormat):
serialization_dict: dict = None,
**kwargs,
):
- if not unsupported_meta_device(model):
+ has_meta_device = unsupported_meta_device(model)
+ if not inplace and not has_meta_device:
+ model = copy.deepcopy(model.to("cpu"))
+
+ from auto_round.utils.model import set_module
+ from auto_round.wrapper import WrapperLinear, WrapperWALayer
+
+ if not has_meta_device:
+ wrapped_modules = [
+ (name, module)
+ for name, module in model.named_modules()
+ if name and isinstance(module, (WrapperLinear, WrapperWALayer))
+ ]
+ wrapped_names = {name for name, _ in wrapped_modules}
+ for name, module in wrapped_modules:
+ if any(name.startswith(f"{parent}.") for parent in wrapped_names if parent != name):
+ continue
+ orig_layer = module.orig_layer
+ while isinstance(orig_layer, (WrapperLinear, WrapperWALayer)):
+ orig_layer = orig_layer.orig_layer
+ for attr_name in ("act_min_scale", "act_max_scale", "act_scale"):
+ orig_layer._parameters.pop(attr_name, None)
+ orig_layer._buffers.pop(attr_name, None)
+ if hasattr(orig_layer, attr_name):
+ delattr(orig_layer, attr_name)
+ set_module(model, name, orig_layer.to("cpu"))
+
+ is_nvfp4_v2 = (serialization_dict or {}).get("data_type") == "nvfp4_v2"
+ if is_nvfp4_v2:
+ quantization_config = _serialize_quantization_config_value(dict(serialization_dict or {}))
+ quantization_config["quant_method"] = "auto-round"
+ quantization_config["packing_format"] = "auto_round:fake"
+ quantization_config["block_name_to_quantize"] = quantization_config.pop("to_quant_block_names", None)
+ from auto_round.export.utils import filter_quantization_config
+
+ filter_quantization_config(quantization_config)
+ if hasattr(model, "config") and model.config is not None:
+ model.config.quantization_config = quantization_config
+
+ if not has_meta_device:
model = model.to("cpu")
model.save_pretrained(output_dir)
elif hasattr(model, "config") and model.config is not None:
diff --git a/auto_round/export/formats/backends/llm_compressor.py b/auto_round/export/formats/backends/llm_compressor.py
index e856a4947..9bc260684 100644
--- a/auto_round/export/formats/backends/llm_compressor.py
+++ b/auto_round/export/formats/backends/llm_compressor.py
@@ -29,6 +29,7 @@ class LLMCompressorFormat(OutputFormat):
"MXFP4",
"MXFP8",
"NVFP4",
+ "NVFP4_E5M3",
"FPW8A16",
"FP8_STATIC",
"INT8",
@@ -53,7 +54,12 @@ class LLMCompressorFormat(OutputFormat):
if re.search("^(auto_round:)?llm_compressor", format):
self.output_format = format
self.backend = None
- if scheme.is_nv_fp() or scheme.is_mx_fp():
+ if scheme.data_type == "nvfp4_v2":
+ from auto_round.export.export_to_llmcompressor import check_compressed_tensors_supported
+
+ check_compressed_tensors_supported(raise_error=True)
+ self.output_format = "llm_compressor:nvfp4_v2"
+ elif scheme.is_nv_fp() or scheme.is_mx_fp():
from auto_round.export.export_to_llmcompressor import check_compressed_tensors_supported
check_compressed_tensors_supported(raise_error=True)
@@ -139,6 +145,8 @@ class LLMCompressorFormat(OutputFormat):
)
if scheme.act_bits <= 8 and (not scheme.is_act_standard_fp() or scheme.act_dynamic):
+ if scheme.act_data_type == "nvfp4_v2":
+ return None, scheme, layer_config, quant_block_list
if (scheme.is_act_nv_fp() and "static_gs" in scheme.act_data_type) or scheme.is_act_mx_fp():
return None, scheme, layer_config, quant_block_list
elif scheme.is_dynamic_afp8() and scheme.is_block_wfp8():
@@ -163,7 +171,7 @@ class LLMCompressorFormat(OutputFormat):
def pack_layer(self, layer_name, model, device=None, **kwargs):
if self.backend is not None:
return self.backend.pack_layer(layer_name, model, device=device, **kwargs)
- if re.search(f"{BackendDataType.MX_FP.value}|{BackendDataType.NV_FP.value}", self.output_format):
+ if re.search(f"{BackendDataType.MX_FP.value}|{BackendDataType.NV_FP.value}|nvfp4_v2", self.output_format):
from auto_round.export.export_to_llmcompressor.export_to_fp import pack_layer
return pack_layer(layer_name, model, device=device)
@@ -199,7 +207,7 @@ class LLMCompressorFormat(OutputFormat):
**kwargs,
) -> torch.nn.Module:
backend = self.get_backend_name()
- if re.search(f"{BackendDataType.MX_FP.value}|{BackendDataType.NV_FP.value}", backend):
+ if re.search(f"{BackendDataType.MX_FP.value}|{BackendDataType.NV_FP.value}|nvfp4_v2", backend):
from auto_round.export.export_to_llmcompressor.export_to_fp import save_quantized_as_fp
export_func = save_quantized_as_fp
diff --git a/auto_round/export/formats/base.py b/auto_round/export/formats/base.py
index 47c386d3b..618e3e0fa 100644
--- a/auto_round/export/formats/base.py
+++ b/auto_round/export/formats/base.py
@@ -54,6 +54,7 @@ class BackendDataType(str, Enum):
MXFP8 = "mxfp8"
MXFP4 = "mxfp4"
NVFP4 = "nvfp4"
+ NVFP4_E5M3 = "nvfp4_v2"
FP8 = "fp8"
MX_FP = "mx_fp"
NV_FP = "nv_fp"
diff --git a/auto_round/inference/backend.py b/auto_round/inference/backend.py
index ade2ca1ce..289e163de 100644
--- a/auto_round/inference/backend.py
+++ b/auto_round/inference/backend.py
@@ -194,10 +194,25 @@ def fp8_static_scheme_checker(
return config == FP8_STATIC
+def cute_nvfp4_e5m3_checker(in_feature: int, out_feature: int, config: QuantizationScheme) -> bool:
+ """Require CuTe DSL before selecting the optional NVFP4 E5M3 CUDA path."""
+ del out_feature
+ if not in_feature_checker_group_size(in_feature, 0, config):
+ return False
+ try:
+ from auto_round_extension.cuda.cute_nvfp4_e5m3 import is_cute_dsl_available
+
+ return torch.cuda.is_available() and is_cute_dsl_available() and torch.cuda.get_device_capability()[0] >= 8
+ except (ImportError, RuntimeError):
+ return False
+
+
GPTQ_FORMAT = ["auto_round:auto_gptq"] # zp+-1
GPTQ_FORMAT_NO_ZP = ["auto_round", "auto_round:gptqmodel"]
AWQ_FORMAT = ["auto_round:auto_awq"]
LLM_COMPRESSOR_FORMAT = ["auto_round:llm_compressor"]
+FAKE_FORMAT = ["auto_round:fake"]
+NVFP4_E5M3_LLM_COMPRESSOR_FORMAT = ["auto_round:llm_compressor_nvfp4_e5m3"]
WOQ_DEFAULT_ACT_BITS = [None, 16, 32]
# CPU backends that target Intel/x86 (ark / auto_round_kernel) cannot
@@ -333,6 +348,24 @@ BackendInfos["auto_round:torch_mxint4"] = BackendInfo(
# NVFP4
+BackendInfos["auto_round:fake"] = BackendInfo(
+ device=["xpu", "cuda", "cpu"],
+ packing_format=FAKE_FORMAT,
+ sym=[True],
+ compute_dtype=["float32", "float16", "bfloat16"],
+ data_type=["nvfp4_v2"],
+ group_size=[16],
+ bits=[4],
+ act_bits=[4],
+ act_group_size=[16],
+ act_sym=[True],
+ act_data_type=["nvfp4_v2"],
+ priority=0,
+ checkers=[mxfp_nvfp_feature_checker],
+ alias=["auto_round", "torch"],
+ requirements=["auto-round>0.12.0"],
+)
+
BackendInfos["auto_round:torch_nvfp4"] = BackendInfo(
device=["xpu", "cuda", "cpu"],
packing_format=LLM_COMPRESSOR_FORMAT,
@@ -352,6 +385,44 @@ BackendInfos["auto_round:torch_nvfp4"] = BackendInfo(
requirements=["auto-round>0.7.0"],
)
+BackendInfos["auto_round:torch_nvfp4_e5m3"] = BackendInfo(
+ device=["xpu", "cuda", "cpu"],
+ packing_format=NVFP4_E5M3_LLM_COMPRESSOR_FORMAT,
+ sym=[True],
+ compute_dtype=["float32", "float16", "bfloat16"],
+ data_type=["nvfp4_v2"],
+ group_size=[16],
+ bits=[4],
+ act_bits=[4],
+ act_group_size=[16],
+ act_sym=[True],
+ act_data_type=["nvfp4_v2"],
+ act_dynamic=[True],
+ priority=3,
+ checkers=[mxfp_nvfp_feature_checker],
+ alias=["auto_round", "torch"],
+ requirements=["auto-round>0.12.0"],
+)
+
+BackendInfos["auto_round:cute_nvfp4_e5m3"] = BackendInfo(
+ device=["cuda"],
+ packing_format=NVFP4_E5M3_LLM_COMPRESSOR_FORMAT,
+ sym=[True],
+ compute_dtype=["float32", "float16", "bfloat16"],
+ data_type=["nvfp4_v2"],
+ group_size=[16],
+ bits=[4],
+ act_bits=[4],
+ act_group_size=[16],
+ act_sym=[True],
+ act_data_type=["nvfp4_v2"],
+ act_dynamic=[True],
+ priority=6,
+ checkers=[cute_nvfp4_e5m3_checker],
+ alias=["cute_nvfp4_e5m3"],
+ requirements=["auto-round>0.12.0"],
+)
+
BackendInfos["auto_round:tritonv2"] = BackendInfo(
device=["cuda", "xpu"],
data_type=["int"],
@@ -779,8 +850,14 @@ def dynamic_import_inference_linear(backend, config, packing_format=None):
return ar_qmodules.MXINT4QuantLinear
if "torch_mxfp4" in backend:
return ar_qmodules.MXFP4QuantLinear
+ if backend == "auto_round:cute_nvfp4_e5m3":
+ return ar_qmodules.CuteNVFP4E5M3QuantLinear
+ if backend == "auto_round:torch_nvfp4_e5m3":
+ return ar_qmodules.NVFP4E5M3QuantLinear
if "torch_nvfp4" in backend:
return ar_qmodules.NVFP4QuantLinear
+ if "auto_round:fake" in backend:
+ return ar_qmodules.FakeActQuantLinear
if "auto_round_kernel" in backend or "ark" in backend:
try:
diff --git a/auto_round/inference/convert_model.py b/auto_round/inference/convert_model.py
index 2a832d125..b3fc98de4 100644
--- a/auto_round/inference/convert_model.py
+++ b/auto_round/inference/convert_model.py
@@ -13,6 +13,7 @@
# limitations under the License.
import os
import re
+from types import SimpleNamespace
from typing import Union
import torch
@@ -70,7 +71,7 @@ def skip_not_convert_modules(model, quantization_config, layer_names, layer_conf
if modules_to_not_convert:
for layer_name in layer_names:
if any([re.search(re.compile(n), layer_name) for n in modules_to_not_convert]):
- layer_configs[layer_name] = {"bits": 16}
+ layer_configs[layer_name] = {"bits": 16, "act_bits": 16}
return layer_configs
@@ -571,6 +572,9 @@ def _create_quant_layer(layer, layer_backend, config, in_features, out_features,
or BackendDataType.MXFP4.value in layer_backend
or BackendDataType.NVFP4.value in layer_backend
or BackendDataType.MXINT4.value in layer_backend
+ or layer_backend == "auto_round:torch_nvfp4_e5m3"
+ or layer_backend == "auto_round:cute_nvfp4_e5m3"
+ or layer_backend == "auto_round:fake"
):
return QuantLinear.from_original(config, layer)
@@ -839,6 +843,30 @@ def convert_hf_model(model: nn.Module, target_device: str = "cpu") -> tuple[nn.M
if is_transformers_version_greater_or_equal_5():
disable_moe_conversion_mapping(model)
quantization_config = model.config.quantization_config
+ config_format = (
+ quantization_config.get("format")
+ if isinstance(quantization_config, dict)
+ else getattr(quantization_config, "format", None)
+ )
+ if config_format == "nvfp4-e5m3-pack-quantized":
+ config_dict = quantization_config if isinstance(quantization_config, dict) else quantization_config.to_dict()
+ ignored = config_dict.get("ignore") or []
+ # TODO: For experimental purpose, will delete when it's not necessary.
+ quantization_config = SimpleNamespace(
+ quant_method="auto-round",
+ packing_format="auto_round:llm_compressor_nvfp4_e5m3",
+ bits=4,
+ group_size=16,
+ sym=True,
+ data_type="nvfp4_v2",
+ act_bits=4,
+ act_group_size=16,
+ act_sym=True,
+ act_data_type="nvfp4_v2",
+ act_dynamic=True,
+ extra_config={name: {"bits": 16, "act_bits": 16} for name in ignored},
+ )
+ model.config.quantization_config = quantization_config
# Check desc_act + static_groups
if getattr(quantization_config, "desc_act", False):
@@ -879,6 +907,12 @@ def convert_hf_model(model: nn.Module, target_device: str = "cpu") -> tuple[nn.M
# Replace layers with quantized versions
layer_configs = get_layer_config(model, quantization_config)
used_backends = _replace_by_quant_layers(model, layer_configs, backend, target_device, packing_format)
+ logger.info(
+ "Inference backend selection: requested=%s, packing_format=%s, selected=%s",
+ backend,
+ packing_format,
+ ", ".join(used_backends),
+ )
# Apply rotation hooks (hadamard, spinquant, quarot, etc.) via unified dispatch.
_has_rotation = getattr(quantization_config, "rotation_config", None) or getattr(
diff --git a/auto_round/schemes.py b/auto_round/schemes.py
index 31da6d95c..def0f79af 100644
--- a/auto_round/schemes.py
+++ b/auto_round/schemes.py
@@ -706,6 +706,18 @@ NVFP4 = QuantizationScheme.from_dict(
}
)
+NVFP4_E5M3 = QuantizationScheme.from_dict(
+ {
+ "bits": 4,
+ "group_size": 16,
+ "data_type": "nvfp4_v2",
+ "act_bits": 4,
+ "act_data_type": "nvfp4_v2",
+ "act_group_size": 16,
+ "act_sym": True,
+ }
+)
+
FPW8A16 = QuantizationScheme.from_dict(
{
"bits": 8,
@@ -802,6 +814,7 @@ PRESET_SCHEMES = {
"MXFP8": MXFP8,
"MXFP8_RCEIL": MXFP8_RCEIL,
"NVFP4": NVFP4,
+ "NVFP4_E5M3": NVFP4_E5M3,
"FPW8A16": FPW8A16,
"W2A16G64": W2A16G64,
"W2A16G32": W2A16G32,
diff --git a/auto_round/utils/common.py b/auto_round/utils/common.py
index c23aad8eb..8b68351c8 100644
--- a/auto_round/utils/common.py
+++ b/auto_round/utils/common.py
@@ -353,12 +353,50 @@ def monkey_patch_transformers():
_patch_tensor_get_dtype_for_prequantized_loading()
_patch_default_rope_init()
_patch_rotary_embedding_init_for_legacy_remote_code()
+ _patch_nvfp4_e5m3_compressed_tensors_quantizer()
if parsed_version >= version.parse("4.56.0"):
_patch_classmethod_kwargs(transformers.AutoModelForCausalLM, "from_pretrained", torch_dtype="dtype")
else:
_patch_classmethod_kwargs(transformers.AutoModelForCausalLM, "from_pretrained", dtype="torch_dtype")
+def _patch_nvfp4_e5m3_compressed_tensors_quantizer():
+ """Route the global-scale-free E5M3 format through AutoRound qmodules."""
+ try:
+ from transformers.quantizers.quantizer_compressed_tensors import CompressedTensorsHfQuantizer
+ except ImportError:
+ return
+
+ original_before = CompressedTensorsHfQuantizer._process_model_before_weight_loading
+ if getattr(original_before, "_auto_round_nvfp4_e5m3_patch", False):
+ return
+ original_after = CompressedTensorsHfQuantizer._process_model_after_weight_loading
+
+ def is_nvfp4_e5m3(self):
+ compression_config = getattr(getattr(self, "compressor", None), "quantization_config", None)
+ return getattr(compression_config, "format", None) == "nvfp4-e5m3-pack-quantized"
+
+ def patched_before(self, model, **kwargs):
+ if not is_nvfp4_e5m3(self):
+ return original_before(self, model, **kwargs)
+ from auto_round.inference.convert_model import convert_hf_model
+
+ model.config.quantization_config = self.compressor.quantization_config
+ target_device = "cuda" if torch.cuda.is_available() else "cpu"
+ model, _ = convert_hf_model(model, target_device=target_device)
+ self.run_compressed = True
+ return model
+
+ def patched_after(self, model, **kwargs):
+ if is_nvfp4_e5m3(self):
+ return model
+ return original_after(self, model, **kwargs)
+
+ patched_before._auto_round_nvfp4_e5m3_patch = True
+ CompressedTensorsHfQuantizer._process_model_before_weight_loading = patched_before
+ CompressedTensorsHfQuantizer._process_model_after_weight_loading = patched_after
+
+
@lru_cache(None)
def monkey_patch():
monkey_patch_transformers()
diff --git a/auto_round/utils/model.py b/auto_round/utils/model.py
index 617f59b54..907d3bc90 100644
--- a/auto_round/utils/model.py
+++ b/auto_round/utils/model.py
@@ -2585,16 +2585,15 @@ def is_model_free_route(
return False
if fmt_first == "llm_compressor":
- # llm_compressor output format is only supported for MXFP schemes.
- from auto_round.compressors.utils import is_mx_fp
+ from auto_round.compressors.model_free import _apply_scheme_overrides
+ from auto_round.schemes import is_mx_fp as _is_mx_fp
try:
- from auto_round.compressors.model_free import _normalize_scheme
-
- scheme_obj = _normalize_scheme(scheme)
- return common_conditions and is_mx_fp((scheme_obj.data_type or "").lower())
- except (ValueError, TypeError):
- return False
+ scheme_obj = _apply_scheme_overrides(scheme, kwargs)
+ scheme_is_mx_fp = _is_mx_fp(scheme_obj.data_type or "")
+ except Exception:
+ scheme_is_mx_fp = False
+ return common_conditions and scheme_is_mx_fp and is_model_free_supported_scheme(scheme, kwargs)
if fmt_first != "auto_round":
return False
return common_conditions and is_model_free_supported_scheme(scheme, kwargs)
diff --git a/auto_round/utils/model_free_utils.py b/auto_round/utils/model_free_utils.py
new file mode 100644
index 000000000..e489f0ca0
--- /dev/null
+++ b/auto_round/utils/model_free_utils.py
@@ -0,0 +1,2710 @@
+# Copyright (c) 2025 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Utility helpers for model-free quantization.
+
+This module hosts generic helpers used by model-free flow and keeps
+model-special handlers at the end of the file.
+"""
+
+from __future__ import annotations
+
+import copy
+import json
+import os
+import re
+import shutil
+import warnings
+from dataclasses import fields
+from functools import lru_cache
+from typing import Any, Callable, Optional, Union
+
+import torch
+
+from auto_round.compressors.utils import is_mx_fp, is_nv_fp
+from auto_round.logger import logger
+from auto_round.schemes import PRESET_SCHEMES, QuantizationScheme, preset_name_to_scheme
+from auto_round.utils.common import to_standard_regex
+from auto_round.utils.device import clear_memory, compile_func
+from auto_round.utils.missing_tensors import quantize_weight_rtn, split_fused_expert_tensors
+
+_NVFP4_E5M3_DATA_TYPE = "nvfp4_v2"
+_BLOCK_NAME_TO_IGNORE = ("shared_expert_gate.", ".gate.", "embed", "conv")
+_SUPPORTED_MXFP_BITS = (4, 8)
+_SUPPORTED_INT_BITS = (2, 4, 8)
+SUPPORTED_PRESET_SCHEMES = (
+ "W2A16",
+ "W2A16G32",
+ "W2A16G64",
+ "W4A16",
+ "W4A16_MIXED",
+ "W8A16",
+ "MXFP4",
+ "MXFP8",
+ "NVFP4_E5M3",
+ "BF16",
+)
+
+
+# ---------------------------------------------------------------------------
+# Generic Utility Helpers
+# ---------------------------------------------------------------------------
+
+
+# Pattern Matching and Layer Scheme Resolution
+# ---------------------------------------------------------------------------
+
+
+class _PatternMatcher:
+ """Precompile ignore and layer-config patterns for shard processing."""
+
+ __slots__ = (
+ "_ignore_re",
+ "_skip_re",
+ "_layer_config",
+ "_default_scheme",
+ "_compiled_lc",
+ "_ignore_cache",
+ "_scheme_cache",
+ )
+
+ def __init__(
+ self,
+ ignore_patterns: list[str],
+ layer_config: dict[str, dict],
+ default_scheme: dict,
+ ) -> None:
+ self._default_scheme = default_scheme
+ self._layer_config = layer_config
+ self._ignore_re: re.Pattern | None = self._build_ignore_regex(ignore_patterns)
+ self._skip_re = re.compile("|".join(re.escape(name) for name in _BLOCK_NAME_TO_IGNORE))
+
+ self._compiled_lc: list[tuple[re.Pattern | None, str | None, dict]] = []
+ for pattern, config in layer_config.items():
+ try:
+ self._compiled_lc.append((re.compile(to_standard_regex(pattern)), None, config))
+ except re.error:
+ self._compiled_lc.append((None, pattern, config))
+
+ self._ignore_cache: dict[str, bool] = {}
+ self._scheme_cache: dict[str, dict | None] = {}
+
+ @staticmethod
+ def _build_ignore_regex(patterns: list[str]) -> re.Pattern | None:
+ if not patterns:
+ return None
+ normalized_patterns: list[str] = []
+ for pattern in patterns:
+ if pattern.endswith("."):
+ normalized = to_standard_regex(pattern.rstrip(".")).removesuffix(".*")
+ normalized_patterns.append(f"{normalized}(?:\\.|$)")
+ else:
+ normalized_patterns.append(to_standard_regex(pattern))
+ return re.compile("|".join(normalized_patterns))
+
+ def should_ignore(self, tensor_name: str) -> bool:
+ cached = self._ignore_cache.get(tensor_name)
+ if cached is not None:
+ return cached
+ layer_name = tensor_name.rsplit(".", 1)[0] if "." in tensor_name else tensor_name
+ result = bool(self._ignore_re and self._ignore_re.search(layer_name))
+ self._ignore_cache[tensor_name] = result
+ return result
+
+ def should_skip(self, tensor_name: str) -> bool:
+ return bool(self._skip_re.search(tensor_name))
+
+ def resolve_scheme(self, tensor_name: str) -> dict | None:
+ if tensor_name not in self._scheme_cache:
+ self._scheme_cache[tensor_name] = self._resolve_uncached(tensor_name)
+ return self._scheme_cache[tensor_name]
+
+ def _resolve_uncached(self, tensor_name: str) -> dict | None:
+ layer_name = tensor_name.rsplit(".", 1)[0] if "." in tensor_name else tensor_name
+ default = self._default_scheme
+ if layer_name in self._layer_config:
+ config = self._layer_config[layer_name]
+ return None if config.get("bits", default.get("bits", 4)) >= 16 else {**default, **config}
+
+ for compiled, plain, config in self._compiled_lc:
+ if (compiled is not None and compiled.search(layer_name)) or (plain is not None and plain in layer_name):
+ return None if config.get("bits", default.get("bits", 4)) >= 16 else {**default, **config}
+ return default
+
+
+# ---------------------------------------------------------------------------
+# Source Tensor Detection and Normalization
+# ---------------------------------------------------------------------------
+
+
+def _collect_mxfp_source_entries(raw_tensors: dict[str, torch.Tensor]) -> list[tuple[str, str, str, int]]:
+ """Collect MXFP source tensors present in a shard.
+
+ Returns entries as ``(layer_name, weight_key, scale_key, bits)`` where
+ ``bits`` is 8 for ``.weight`` (float8) and 4 for ``.weight_packed``.
+ """
+ entries: list[tuple[str, str, str, int]] = []
+ for name, tensor in raw_tensors.items():
+ if name.endswith(".weight") and tensor.dtype == torch.float8_e4m3fn:
+ layer_name = name[: -len(".weight")]
+ scale_key = f"{layer_name}.weight_scale"
+ if scale_key in raw_tensors and raw_tensors[scale_key].dtype == torch.uint8:
+ entries.append((layer_name, name, scale_key, 8))
+ elif name.endswith(".weight_packed") and tensor.dtype in (torch.int8, torch.uint8):
+ layer_name = name[: -len(".weight_packed")]
+ if (
+ f"{layer_name}.weight_global_scale" in raw_tensors
+ or f"{layer_name}.input_global_scale" in raw_tensors
+ or f"{layer_name}.weight_scale_2" in raw_tensors
+ or f"{layer_name}.input_scale" in raw_tensors
+ ):
+ continue
+ scale_key = f"{layer_name}.weight_scale"
+ if scale_key in raw_tensors and raw_tensors[scale_key].dtype == torch.uint8:
+ entries.append((layer_name, name, scale_key, 4))
+ return entries
+
+
+def _normalize_nvfp4_source_tensors(
+ raw_tensors: dict[str, torch.Tensor],
+ shard_name: str | None = None,
+) -> tuple[dict[str, torch.Tensor], list[str]]:
+ """Normalize legacy NVFP4 source naming to llm-compressor naming."""
+
+ def _reciprocal_global_scale(scale: torch.Tensor) -> torch.Tensor:
+ return (1.0 / scale.float()).to(torch.float32).reshape([1])
+
+ converted_layers: list[str] = []
+ candidates: list[str] = []
+ for name, tensor in list(raw_tensors.items()):
+ if name.endswith(".weight"):
+ layer_name = name[: -len(".weight")]
+ elif name.endswith(".weight_packed"):
+ layer_name = name[: -len(".weight_packed")]
+ else:
+ continue
+
+ if tensor.dtype not in (torch.uint8, torch.int8):
+ continue
+ if f"{layer_name}.weight_scale" not in raw_tensors:
+ continue
+
+ has_legacy_global = f"{layer_name}.weight_scale_2" in raw_tensors or f"{layer_name}.input_scale" in raw_tensors
+ has_new_global = (
+ f"{layer_name}.weight_global_scale" in raw_tensors or f"{layer_name}.input_global_scale" in raw_tensors
+ )
+ has_new_packed = f"{layer_name}.weight_packed" in raw_tensors
+ if has_legacy_global or has_new_global or has_new_packed:
+ candidates.append(layer_name)
+
+ if not candidates:
+ return raw_tensors, converted_layers
+
+ for layer_name in candidates:
+ weight_key = f"{layer_name}.weight"
+ weight_packed_key = f"{layer_name}.weight_packed"
+ weight_scale_2_key = f"{layer_name}.weight_scale_2"
+ input_scale_key = f"{layer_name}.input_scale"
+ weight_global_scale_key = f"{layer_name}.weight_global_scale"
+ input_global_scale_key = f"{layer_name}.input_global_scale"
+
+ if weight_packed_key not in raw_tensors and weight_key in raw_tensors:
+ raw_tensors[weight_packed_key] = raw_tensors.pop(weight_key).view(torch.uint8).contiguous()
+
+ if weight_scale_2_key in raw_tensors and weight_global_scale_key not in raw_tensors:
+ raw_tensors[weight_global_scale_key] = raw_tensors.pop(weight_scale_2_key)
+ elif weight_scale_2_key in raw_tensors:
+ raw_tensors.pop(weight_scale_2_key)
+
+ if input_scale_key in raw_tensors and input_global_scale_key not in raw_tensors:
+ raw_tensors[input_global_scale_key] = raw_tensors.pop(input_scale_key)
+ elif input_scale_key in raw_tensors:
+ raw_tensors.pop(input_scale_key)
+
+ if weight_global_scale_key in raw_tensors:
+ raw_tensors[weight_global_scale_key] = _reciprocal_global_scale(raw_tensors[weight_global_scale_key])
+ if input_global_scale_key in raw_tensors:
+ raw_tensors[input_global_scale_key] = _reciprocal_global_scale(raw_tensors[input_global_scale_key])
+
+ converted_layers.append(layer_name)
+
+ if converted_layers:
+ shard_prefix = f"[{shard_name}] " if shard_name else ""
+ logger.info(f"{shard_prefix}Normalized {len(converted_layers)} legacy NVFP4 layer(s) to llm-compressor naming.")
+ return raw_tensors, converted_layers
+
+
+def _handle_nvfp4_source_tensors(
+ raw_tensors: dict[str, torch.Tensor],
+ matcher: Any,
+) -> tuple[dict[str, torch.Tensor], dict[str, torch.Tensor], list[str]]:
+ """Passthrough NVFP4 source tensors when target scheme for the layer is NVFP4."""
+ passthrough_tensors: dict[str, torch.Tensor] = {}
+ passthrough_layers: list[str] = []
+
+ for name, tensor in list(raw_tensors.items()):
+ if not name.endswith(".weight_packed") or tensor.dtype not in (torch.uint8, torch.int8):
+ continue
+
+ layer_name = name[: -len(".weight_packed")]
+ scale_key = f"{layer_name}.weight_scale"
+ if scale_key not in raw_tensors:
+ continue
+
+ scheme = matcher.resolve_scheme(f"{layer_name}.weight")
+ if scheme is None:
+ continue
+ scheme_bits = scheme.get("bits")
+ scheme_data_type = (scheme.get("data_type") or "").lower()
+ if not (scheme_bits == 4 and (is_nv_fp(scheme_data_type) or scheme_data_type == _NVFP4_E5M3_DATA_TYPE)):
+ continue
+
+ keys_to_move = [name, scale_key]
+ weight_global_scale_key = f"{layer_name}.weight_global_scale"
+ input_global_scale_key = f"{layer_name}.input_global_scale"
+ if weight_global_scale_key in raw_tensors:
+ keys_to_move.append(weight_global_scale_key)
+ if input_global_scale_key in raw_tensors:
+ keys_to_move.append(input_global_scale_key)
+
+ for key in keys_to_move:
+ value = raw_tensors.pop(key).to("cpu")
+ if key.endswith("_global_scale"):
+ value = value.reshape([1])
+ passthrough_tensors[key] = value
+ passthrough_layers.append(layer_name)
+
+ if passthrough_layers:
+ logger.info(f"Handling NVFP4 source tensor(s): {len(passthrough_layers)} passthrough layer(s).")
+
+ return raw_tensors, passthrough_tensors, passthrough_layers
+
+
+def _is_out_of_memory_error(exc: Exception) -> bool:
+ if isinstance(exc, torch.OutOfMemoryError):
+ return True
+ message = str(exc).lower()
+ return "out of memory" in message or "cuda error: out of memory" in message
+
+
+def _dequantize_with_device_fallback(
+ *,
+ dequant_device: str,
+ shard_prefix: str,
+ op_name: str,
+ tensor_label: str,
+ on_device: Callable[[], torch.Tensor],
+ on_cpu: Callable[[], torch.Tensor],
+) -> torch.Tensor:
+ """Run dequantization on ``dequant_device`` and fall back to CPU on errors."""
+ if dequant_device != "cpu":
+ try:
+ return on_device()
+ except Exception as e:
+ if _is_out_of_memory_error(e):
+ logger.warning(
+ f"{shard_prefix}{op_name} on {dequant_device} ran OOM for {tensor_label}: {e}. "
+ "Clearing accelerator memory and falling back to CPU for this tensor."
+ )
+ clear_memory()
+ else:
+ logger.warning(
+ f"{shard_prefix}{op_name} on {dequant_device} failed for {tensor_label}: {e}. "
+ "Falling back to CPU for this tensor."
+ )
+ return on_cpu()
+
+
+def _normalize_scheme(scheme: Union[str, QuantizationScheme]) -> QuantizationScheme:
+ """Convert *scheme* to a :class:`QuantizationScheme` instance.
+
+ Raises ``ValueError`` for unknown preset names and ``TypeError`` for
+ unsupported types.
+ """
+ if isinstance(scheme, str):
+ scheme_name = scheme.upper()
+ if scheme_name not in PRESET_SCHEMES:
+ raise ValueError(f"Unknown scheme '{scheme}'. Available: {list(PRESET_SCHEMES.keys())}")
+ return preset_name_to_scheme(scheme_name)
+ if isinstance(scheme, QuantizationScheme):
+ return scheme
+ raise TypeError(f"Unsupported scheme type: {type(scheme)}")
+
+
+# ---------------------------------------------------------------------------
+# Shard Tensor Processing
+# ---------------------------------------------------------------------------
+
+
+def _is_eligible_weight(tensor_name: str, tensor: torch.Tensor) -> bool:
+ """Check if a tensor is eligible for quantization (2D Linear weight)."""
+ return tensor_name.endswith(".weight") and tensor.dim() == 2
+
+
+def _quantize_weight_mxfp(
+ weight: torch.Tensor,
+ layer_name: str,
+ bits: int,
+ group_size: int,
+ data_type: str,
+ device: str = "cpu",
+ disable_opt_rtn: bool = False,
+) -> dict[str, torch.Tensor]:
+ """Quantize a 2D weight tensor to MXFP4 / MXFP8 and return packed outputs.
+
+ Reuses :func:`auto_round.data_type.mxfp.quant_mx` to derive the per-block
+ shared exponent (E8M0 scale), and :class:`auto_round.export.export_to_autoround.qlinear_fp.QuantLinear`
+ to perform the same packing as :func:`auto_round.export.export_to_llmcompressor.export_to_fp.pack_layer`.
+
+ Returns a dict with one of:
+ * MXFP8: ``{layer_name+'.weight': float8_e4m3fn, layer_name+'.weight_scale': uint8}``
+ * MXFP4: ``{layer_name+'.weight_packed': uint8, layer_name+'.weight_scale': uint8}``
+ """
+ import torch.nn as nn
+
+ from auto_round.data_type.utils import get_quant_func
+ from auto_round.export.export_to_autoround.qlinear_fp import QuantLinear
+
+ if not is_mx_fp(data_type):
+ data_type = "mx_fp4" if bits == 4 else "mx_fp8"
+
+ out_features, in_features = weight.shape
+ if in_features % group_size != 0:
+ raise ValueError(
+ f"in_features={in_features} for layer '{layer_name}' is not divisible "
+ f"by MXFP group_size={group_size}; cannot pack."
+ )
+
+ weight_dev = weight.to(device)
+ # Use get_quant_func (same as WrapperLinear) so that all registered MXFP
+ # variants automatically get opt_rtn support via the QUANT_FUNC_WITH_DTYPE
+ # registry (e.g. "opt_rtn_mx_fp4" -> quant_mx_opt_rtn, "mx_fp4" -> quant_mx).
+ quant_func, _ = get_quant_func(data_type, bits, sym=True, disable_opt_rtn=disable_opt_rtn, iters=0)
+ weight_dev, shared_exp, _ = quant_func(weight_dev, bits=bits, group_size=group_size, data_type=data_type)
+ # Reshape to (out_features, n_groups) so the on-disk weight_scale matches
+ # the llm-compressor convention (and QuantLinear's registered buffer shape).
+ shared_exp = shared_exp.reshape(out_features, in_features // group_size)
+ # Ensure shared_exp is a numeric float (not a storage-specific dtype like
+ # float8) — QuantLinear.pack performs `2 ** scales` which dispatches to
+ # torch.pow; some backends do not implement pow for float8 dtypes. Cast to
+ # float32 here to avoid runtime errors like "pow_cuda not implemented for
+ # 'Float8_e4m3fn'" while preserving numeric values.
+ shared_exp = shared_exp.to(torch.float32)
+
+ # Build a lightweight nn.Linear holding the original weight so we can
+ # delegate packing to the existing QuantLinear.pack implementation.
+ fake_linear = nn.Linear(in_features, out_features, bias=False)
+ with torch.no_grad():
+ fake_linear.weight = nn.Parameter(weight_dev, requires_grad=False)
+
+ qlayer = QuantLinear(
+ bits=bits,
+ group_size=group_size,
+ infeatures=in_features,
+ outfeatures=out_features,
+ bias=False,
+ data_type="mx_fp4" if bits == 4 else "mx_fp8e4m3",
+ sym=True,
+ act_bits=bits,
+ )
+ qlayer.pack(fake_linear, shared_exp, device=device)
+
+ if bits == 8:
+ return {
+ f"{layer_name}.weight": qlayer.weight.to("cpu"),
+ f"{layer_name}.weight_scale": qlayer.weight_scale.to("cpu"),
+ }
+ return {
+ f"{layer_name}.weight_packed": qlayer.weight_packed.to("cpu"),
+ f"{layer_name}.weight_scale": qlayer.weight_scale.to("cpu"),
+ }
+
+
+def _quantize_weight_nvfp4_e5m3(
+ weight: torch.Tensor,
+ layer_name: str,
+ group_size: int = 16,
+ device: str = "cpu",
+) -> dict[str, torch.Tensor]:
+ """Fake-quantize a 2D weight tensor to NVFP4 E5M3 and return its high-precision QDQ weight."""
+ from auto_round.data_type.nvfp import nvfp4_v2
+
+ out_features, in_features = weight.shape
+ if group_size != 16:
+ raise ValueError(f"NVFP4_E5M3 requires group_size=16, got {group_size} for layer '{layer_name}'.")
+ if in_features % group_size != 0:
+ raise ValueError(
+ f"in_features={in_features} for layer '{layer_name}' is not divisible "
+ f"by NVFP4_E5M3 group_size={group_size}; cannot quantize."
+ )
+
+ weight_dev = weight.to(device)
+ qdq_weight, _, _ = nvfp4_v2(weight_dev, bits=4, group_size=group_size)
+ return {f"{layer_name}.weight": qdq_weight.to(dtype=weight.dtype, device="cpu")}
+
+
+def _pack_weight_nvfp4_e5m3(
+ weight: torch.Tensor,
+ layer_name: str,
+ group_size: int = 16,
+ device: str = "cpu",
+) -> dict[str, torch.Tensor]:
+ """Pack FP4 E2M1 weights with unsigned E5M3 block scales."""
+ from auto_round.data_type.nvfp import nvfp4_v2
+ from auto_round.export.export_to_autoround.qlinear_fp import QuantLinear
+
+ out_features, in_features = weight.shape
+ if group_size != 16 or in_features % group_size != 0:
+ raise ValueError(
+ f"NVFP4_E5M3 requires in_features divisible by group_size=16, got {in_features} for '{layer_name}'."
+ )
+ weight_dev = weight.to(device)
+ _, scale, _ = nvfp4_v2(weight_dev, bits=4, group_size=group_size)
+ # nvfp4_v2 may return a flattened per-group scale layout (e.g. [N, 1]);
+ # normalize to [out_features, in_features // group_size] before packing
+ # so serialized .weight_scale keeps the expected 2D shape.
+ scale = scale.reshape(out_features, in_features // group_size).to(torch.float32)
+ linear = torch.nn.Linear(in_features, out_features, bias=False, device=device, dtype=weight.dtype)
+ linear.weight = torch.nn.Parameter(weight_dev, requires_grad=False)
+ qlayer = QuantLinear(
+ 4, group_size, in_features, out_features, False, data_type="nvfp4_v2", act_bits=4, act_data_type="nvfp4_v2"
+ )
+ qlayer.pack(linear, scale, device=device)
+ return {
+ f"{layer_name}.weight_packed": qlayer.weight_packed.to("cpu"),
+ f"{layer_name}.weight_scale": qlayer.weight_scale.to("cpu"),
+ }
+
+
+def _quantize_single_tensor(
+ tensor_name: str,
+ tensor: torch.Tensor,
+ matcher: "_PatternMatcher",
+ device: str = "cpu",
+ quantize_func: Callable = quantize_weight_rtn,
+ disable_opt_rtn: bool = False,
+) -> tuple[str, dict[str, torch.Tensor], str | None, str | None]:
+ """Quantize one eligible weight tensor and return packed outputs.
+
+ Returns:
+ (layer_name, output_tensors_dict, quantized_layer_or_None, ignored_layer_or_None)
+ """
+ layer_name = tensor_name.rsplit(".", 1)[0]
+
+ if not _is_eligible_weight(tensor_name, tensor):
+ ignored_layer = layer_name if tensor_name.endswith(".weight") and tensor.dim() > 1 else None
+ return layer_name, {tensor_name: tensor}, None, ignored_layer
+
+ if matcher.should_ignore(tensor_name):
+ logger.debug(f"Ignoring (user-specified): {layer_name}")
+ return layer_name, {tensor_name: tensor}, None, layer_name
+
+ if matcher.should_skip(tensor_name):
+ logger.debug(f"Skipping (predefined): {layer_name}")
+ return layer_name, {tensor_name: tensor}, None, layer_name
+
+ scheme = matcher.resolve_scheme(tensor_name)
+ if scheme is None:
+ logger.debug(f"Keeping full precision: {layer_name}")
+ return layer_name, {tensor_name: tensor}, None, layer_name
+
+ bits = scheme["bits"]
+ group_size = scheme["group_size"]
+ sym = scheme.get("sym", True)
+ data_type = (scheme.get("data_type") or "int").lower()
+
+ if bits >= 16:
+ return layer_name, {tensor_name: tensor}, None, layer_name
+
+ # ---- MXFP path (MXFP4 / MXFP8) ----
+ if is_mx_fp(data_type):
+ try:
+ out = _quantize_weight_mxfp(
+ weight=tensor,
+ layer_name=layer_name,
+ bits=bits,
+ group_size=group_size,
+ data_type=data_type,
+ device=device,
+ disable_opt_rtn=disable_opt_rtn,
+ )
+ logger.debug(f"Quantized (MXFP): {layer_name} (bits={bits}, group_size={group_size})")
+ return layer_name, out, layer_name, None
+ except Exception as e:
+ logger.warning(f"Failed to MXFP-quantize {layer_name}: {e}. Keeping original weight.")
+ return layer_name, {tensor_name: tensor}, None, layer_name
+
+ # ---- NVFP4 E5M3 fake-quantization path ----
+ if data_type == _NVFP4_E5M3_DATA_TYPE:
+ try:
+ quantize_e5m3 = (
+ _quantize_weight_nvfp4_e5m3 if scheme.get("_output_format") == "fake" else _pack_weight_nvfp4_e5m3
+ )
+ out = quantize_e5m3(
+ weight=tensor,
+ layer_name=layer_name,
+ group_size=group_size,
+ device=device,
+ )
+ logger.debug(f"Quantized (NVFP4_E5M3): {layer_name} (bits=4, group_size={group_size})")
+ return layer_name, out, layer_name, None
+ except Exception as e:
+ logger.warning(f"Failed to NVFP4_E5M3-quantize {layer_name}: {e}. Keeping original weight.")
+ return layer_name, {tensor_name: tensor}, None, layer_name
+
+ # ---- Integer WOQ path ----
+ # opt_rtn is always disabled for integer WOQ in model-free mode because
+ # the scale search does not improve accuracy for INT quantization here.
+ try:
+ qweight, qzeros, scales = quantize_func(
+ weight=tensor,
+ bits=bits,
+ group_size=group_size,
+ sym=sym,
+ device=device,
+ disable_opt_rtn=True,
+ )
+
+ out: dict[str, torch.Tensor] = {
+ f"{layer_name}.qweight": qweight,
+ f"{layer_name}.qzeros": qzeros,
+ f"{layer_name}.scales": scales,
+ }
+
+ logger.debug(f"Quantized: {layer_name} (bits={bits}, group_size={group_size}, sym={sym})")
+ return layer_name, out, layer_name, None
+
+ except Exception as e:
+ logger.warning(f"Failed to quantize {layer_name}: {e}. Keeping original weight.")
+ return layer_name, {tensor_name: tensor}, None, layer_name
+
+
+@lru_cache(maxsize=32)
+def _load_weight_map_from_index(index_path: str) -> dict[str, str]:
+ """Load weight_map from an index file with a small process-local cache."""
+ with open(index_path) as f:
+ index = json.load(f)
+ weight_map = index.get("weight_map", {})
+ return weight_map if isinstance(weight_map, dict) else {}
+
+
+def _build_cross_shard_pairs_from_weight_map(
+ weight_map: dict[str, str],
+) -> tuple[dict[str, dict[str, list[str]]], dict[str, set[str]]]:
+ """Identify cross-shard FP8 (weight, weight_scale_inv) pairs from weight_map.
+
+ A *cross-shard pair* exists when ``<layer>.weight`` and
+ ``<layer>.weight_scale_inv`` reside in **different** shards. The shard
+ that holds ``weight_scale_inv`` is the *donor*; the shard that holds
+ ``weight`` is the *recipient*.
+
+ Returns:
+ recipient_to_donors:
+ ``{recipient_shard: {donor_shard: [scale_inv_tensor_names]}}``
+ donor_shard_tensors:
+ ``{donor_shard: set(scale_inv tensor names it donates to other shards)}``
+ """
+ recipient_to_donors: dict[str, dict[str, list[str]]] = {}
+ donor_shard_tensors: dict[str, set[str]] = {}
+
+ for tensor_name, shard in weight_map.items():
+ if not tensor_name.endswith(".weight_scale_inv"):
+ continue
+ # "layer.weight_scale_inv" -> "layer.weight"
+ weight_name = tensor_name[: -len("_scale_inv")]
+ weight_shard = weight_map.get(weight_name)
+ if not weight_shard or weight_shard == shard:
+ continue
+ # `shard` (donor) donates `tensor_name` to `weight_shard` (recipient)
+ recipient_to_donors.setdefault(weight_shard, {}).setdefault(shard, []).append(tensor_name)
+ donor_shard_tensors.setdefault(shard, set()).add(tensor_name)
+
+ return recipient_to_donors, donor_shard_tensors
+
+
+def _hydrate_missing_fp8_scales_from_index(
+ raw_tensors: dict[str, torch.Tensor],
+ shard_path: str,
+ *,
+ shard_name: str | None = None,
+ index_dir: str | None = None,
+ donor_shard_dir: str | None = None,
+) -> dict[str, torch.Tensor]:
+ """Populate missing ``.weight_scale_inv`` tensors from sibling shards.
+
+ Some checkpoints shard FP8 weight and its corresponding scale tensor into
+ different ``.safetensors`` files. Model-free processing is shard-local, so
+ this helper hydrates missing ``<layer>.weight_scale_inv`` tensors by looking
+ up ``weight_map`` in ``*.safetensors.index.json`` and loading only the
+ needed tensors from referenced shards.
+
+ Args:
+ index_dir: Directory that contains ``*.safetensors.index.json``. When
+ ``None`` (default) the directory of *shard_path* is used, which is
+ correct for non-streaming mode. Streaming mode should pass
+ ``work_dir`` here because downloaded shards live in a
+ ``.cache/model_free_source_shards/`` sub-directory that does not
+ contain the index file.
+ donor_shard_dir: Directory where donor shard files can be found. When
+ ``None`` (default) the same directory as *index_dir* is used.
+ Streaming mode should pass the local shard cache directory so that
+ already-downloaded donor shards are resolved correctly.
+ """
+ if not shard_path.endswith(".safetensors"):
+ return raw_tensors
+
+ weight_to_scale: dict[str, str] = {}
+ for name, tensor in raw_tensors.items():
+ if not name.endswith(".weight"):
+ continue
+ if tensor.dtype != torch.float8_e4m3fn:
+ continue
+ scale_inv_name = f"{name[: -len('.weight')]}.weight_scale_inv"
+ if scale_inv_name not in raw_tensors:
+ weight_to_scale[name] = scale_inv_name
+
+ if not weight_to_scale:
+ return raw_tensors
+
+ shard_dir = os.path.dirname(shard_path)
+ # Resolve the directory where index.json lives. In streaming mode the
+ # shard lives in a .cache/ sub-directory that has no index file; callers
+ # pass index_dir=work_dir to point at the correct location.
+ idx_dir = index_dir if index_dir is not None else shard_dir
+ # Donor shards are looked up in donor_shard_dir (defaults to idx_dir).
+ donor_dir = donor_shard_dir if donor_shard_dir is not None else idx_dir
+
+ index_path = os.path.join(idx_dir, "model.safetensors.index.json")
+ if not os.path.exists(index_path):
+ candidates = sorted(
+ os.path.join(idx_dir, f) for f in os.listdir(idx_dir) if f.endswith(".safetensors.index.json")
+ )
+ if not candidates:
+ return raw_tensors
+ index_path = candidates[0]
+
+ try:
+ weight_map = _load_weight_map_from_index(index_path)
+ except Exception:
+ return raw_tensors
+
+ current_shard = os.path.basename(shard_path)
+ scales_by_shard: dict[str, list[str]] = {}
+ for scale_name in weight_to_scale.values():
+ target_shard = weight_map.get(scale_name)
+ if not target_shard or target_shard == current_shard:
+ continue
+ scales_by_shard.setdefault(target_shard, []).append(scale_name)
+
+ if not scales_by_shard:
+ return raw_tensors
+
+ from safetensors import safe_open
+
+ hydrated = 0
+ shard_prefix = f"[{shard_name}] " if shard_name else ""
+ for target_shard, scale_names in scales_by_shard.items():
+ target_path = os.path.join(donor_dir, target_shard)
+ if not os.path.exists(target_path):
+ logger.warning(
+ f"{shard_prefix}Donor shard '{target_shard}' not found in '{donor_dir}' while hydrating "
+ f"{len(scale_names)} FP8 scale_inv tensor(s); the affected weight(s) will remain in "
+ f"float8_e4m3fn and may fail downstream quantization. This usually indicates a shard "
+ f"scheduling/ordering issue (donor shard processed/downloaded after its recipient)."
+ )
+ continue
+ try:
+ with safe_open(target_path, framework="pt", device="cpu") as sf:
+ for scale_name in scale_names:
+ if scale_name in raw_tensors:
+ continue
+ try:
+ raw_tensors[scale_name] = sf.get_tensor(scale_name)
+ hydrated += 1
+ except Exception:
+ # Tensor may be absent in this shard; skip lazily.
+ continue
+ except Exception:
+ continue
+
+ if hydrated:
+ logger.info(f"{shard_prefix}Hydrated {hydrated} FP8 scale tensor(s) from sibling shard(s) using index mapping.")
+
+ return raw_tensors
+
+
+def _dequant_mxfp_tensors(
+ raw_tensors: dict[str, torch.Tensor],
+ device: str = "cpu",
+ shard_name: str | None = None,
+) -> dict[str, torch.Tensor]:
+ """Dequantize llm-compressor MXFP8 / MXFP4 weight tensors to bfloat16.
+
+ Detection is purely by *name* and *dtype*, reusing the dequant kernels in
+ :mod:`auto_round_extension.vllm_ext`:
+
+ * ``<layer>.weight`` (``float8_e4m3fn``) + ``<layer>.weight_scale`` → MXFP8,
+ dequantized via :func:`~auto_round_extension.vllm_ext.mxfp8_qdq_utils.dequant_mx_fp8`.
+ * ``<layer>.weight_packed`` (``uint8``) + ``<layer>.weight_scale`` → MXFP4,
+ dequantized via :func:`~auto_round_extension.vllm_ext.mxfp4_qdq_utils.to_dtype`.
+
+ The dequantized weight is written back under ``<layer>.weight`` and the
+ scale (and any ``weight_packed``) tensor is removed, so the downstream RTN
+ path can requantize the layer to the requested target scheme.
+ """
+ from auto_round_extension.vllm_ext.mxfp4_qdq_utils import to_dtype
+ from auto_round_extension.vllm_ext.mxfp8_qdq_utils import dequant_mx_fp8
+
+ # Tuple layout: (layer_name, weight_key, scale_key, bits)
+ entries = _collect_mxfp_source_entries(raw_tensors)
+
+ if not entries:
+ return raw_tensors
+
+ n_mxfp8 = sum(1 for _layer_name, _weight_key, _scale_key, bits in entries if bits == 8)
+ n_mxfp4 = len(entries) - n_mxfp8
+ dequant_device = str(device or "cpu")
+ shard_prefix = f"[{shard_name}] " if shard_name else ""
+ logger.info(
+ f"{shard_prefix}Dequantizing MXFP tensor(s) to bfloat16 on {dequant_device}: "
+ f"MXFP8={n_mxfp8}, MXFP4={n_mxfp4}, total={len(entries)}."
+ )
+
+ for layer_name, weight_key, scale_key, bits in entries:
+ weight = raw_tensors.pop(weight_key)
+ scale = raw_tensors.pop(scale_key).view(torch.uint8)
+ if bits == 8:
+ dq_weight = _dequantize_with_device_fallback(
+ dequant_device=dequant_device,
+ shard_prefix=shard_prefix,
+ op_name="MXFP dequant",
+ tensor_label=layer_name,
+ on_device=lambda: dequant_mx_fp8(
+ weight_fp8=weight.to(dequant_device, non_blocking=True),
+ scale_e8m0=scale.to(dequant_device, non_blocking=True),
+ block_size=32,
+ target_dtype=torch.bfloat16,
+ ).to("cpu"),
+ on_cpu=lambda: dequant_mx_fp8(
+ weight_fp8=weight,
+ scale_e8m0=scale,
+ block_size=32,
+ target_dtype=torch.bfloat16,
+ ),
+ )
+ else:
+ dq_weight = _dequantize_with_device_fallback(
+ dequant_device=dequant_device,
+ shard_prefix=shard_prefix,
+ op_name="MXFP dequant",
+ tensor_label=layer_name,
+ on_device=lambda: to_dtype(
+ data_lp=weight.view(torch.uint8).contiguous().to(dequant_device, non_blocking=True),
+ scale_e8m0=scale.to(dequant_device, non_blocking=True),
+ elem_dtype="fp4_e2m1",
+ block_size=32,
+ target_dtype=torch.bfloat16,
+ ).to("cpu"),
+ on_cpu=lambda: to_dtype(
+ data_lp=weight.view(torch.uint8).contiguous(),
+ scale_e8m0=scale,
+ elem_dtype="fp4_e2m1",
+ block_size=32,
+ target_dtype=torch.bfloat16,
+ ),
+ )
+ raw_tensors[f"{layer_name}.weight"] = dq_weight
+
+ return raw_tensors
+
+
+def _handle_mxfp_source_tensors(
+ raw_tensors: dict[str, torch.Tensor],
+ matcher: "_PatternMatcher",
+ source_state: dict[str, int] | None = None,
+ device: str = "cpu",
+ shard_name: str | None = None,
+) -> tuple[dict[str, torch.Tensor], dict[str, torch.Tensor], list[str]]:
+ """Handle llm-compressor-style MXFP8/MXFP4 source tensors.
+
+ Detects llm-compressor MXFP tensors purely by *name + dtype*:
+
+ * ``<layer>.weight`` (``float8_e4m3fn``) + ``<layer>.weight_scale`` → MXFP8.
+ * ``<layer>.weight_packed`` (``uint8``) + ``<layer>.weight_scale`` → MXFP4.
+
+ For each detected layer the effective target scheme is resolved via *matcher*:
+
+ * If the target is the **same MXFP format** (``data_type='mx_fp'``, matching
+ ``bits``), the tensors are emitted directly as a passthrough — no
+ dequantization is performed and the layer is recorded as already quantized.
+ * Otherwise the tensors are dequantized to ``bfloat16`` via
+ :func:`_dequant_mxfp_tensors` so the downstream RTN path can re-quantize
+ them to the requested target scheme.
+
+ Returns:
+ ``(raw_tensors, passthrough_tensors, passthrough_layers)``.
+ """
+ entries = _collect_mxfp_source_entries(raw_tensors)
+ if not entries:
+ return raw_tensors, {}, []
+
+ source_state = source_state or {}
+
+ passthrough_tensors: dict[str, torch.Tensor] = {}
+ passthrough_layers: list[str] = []
+ n_dequant = 0
+
+ for layer_name, weight_key, scale_key, bits in entries:
+ scheme = matcher.resolve_scheme(f"{layer_name}.weight")
+ is_ignored = matcher.should_ignore(f"{layer_name}.weight")
+ target_is_same_mxfp = (
+ scheme is not None and is_mx_fp((scheme.get("data_type") or "").lower()) and scheme.get("bits") == bits
+ )
+ if target_is_same_mxfp and not is_ignored:
+ passthrough_tensors[weight_key] = raw_tensors.pop(weight_key).to("cpu")
+ passthrough_tensors[scale_key] = raw_tensors.pop(scale_key).to("cpu")
+ passthrough_layers.append(layer_name)
+ else:
+ n_dequant += 1
+
+ if n_dequant:
+ raw_tensors = _dequant_mxfp_tensors(raw_tensors, device=device, shard_name=shard_name)
+
+ parts: list[str] = []
+ if passthrough_layers:
+ parts.append(f"{len(passthrough_layers)} passthrough")
+ if n_dequant:
+ parts.append(f"{n_dequant} dequantized to bfloat16")
+ if source_state:
+ parts.append(f"{len(source_state)} model_type-normalized")
+ logger.info(f"Handling MXFP source tensor(s): {', '.join(parts)}.")
+
+ return raw_tensors, passthrough_tensors, passthrough_layers
+
+
+def _dequant_fp8_tensors(
+ raw_tensors: dict[str, torch.Tensor],
+ block_size: list | None = None,
+ device: str = "cpu",
+ shard_name: str | None = None,
+ shard_path: str | None = None,
+ *,
+ index_dir: str | None = None,
+ donor_shard_dir: str | None = None,
+) -> dict[str, torch.Tensor]:
+ """Dequantize DeepSeek-V3-style FP8 weight tensors to bfloat16.
+
+ Handles the **DeepSeek-V3 FP8** convention: weight dtype ``float8_e4m3fn``
+ paired with a ``.weight_scale_inv`` tensor (per-block float32 scales, NOT
+ E8M0). The weights are converted to ``bfloat16`` so downstream RTN
+ quantization can proceed normally.
+
+ MXFP sources are handled separately by
+ :func:`preprocess_model_type_source_tensors` / :func:`_handle_mxfp_source_tensors`.
+ """
+ from auto_round.utils.weight_handler import _dequant_fp8_linear_weight
+
+ if shard_path:
+ raw_tensors = _hydrate_missing_fp8_scales_from_index(
+ raw_tensors,
+ shard_path,
+ shard_name=shard_name,
+ index_dir=index_dir,
+ donor_shard_dir=donor_shard_dir,
+ )
+
+ quant_entries: list[tuple[str, str]] = []
+ for name, tensor in raw_tensors.items():
+ if not name.endswith(".weight"):
+ continue
+ if tensor.dtype != torch.float8_e4m3fn:
+ continue
+ # DeepSeek-V3 style: .weight_scale_inv (per-block float32 scales).
+ scale_inv_name = f"{name[: -len('.weight')]}.weight_scale_inv"
+ if scale_inv_name in raw_tensors:
+ quant_entries.append((name, scale_inv_name))
+
+ if not quant_entries:
+ return raw_tensors
+
+ # device has already been resolved by the caller; use it directly here.
+ dequant_device = str(device or "cpu")
+ shard_prefix = f"[{shard_name}] " if shard_name else ""
+
+ logger.info(
+ f"{shard_prefix}Dequantizing {len(quant_entries)} FP8 weight tensor(s) to bfloat16 on {dequant_device}."
+ )
+
+ for weight_name, scale_name in quant_entries:
+ weight = raw_tensors[weight_name]
+ scale = raw_tensors.pop(scale_name)
+
+ # Dequantize on GPU for throughput, then move back to CPU to keep
+ # per-shard memory usage bounded before per-layer quantization.
+ raw_tensors[weight_name] = _dequantize_with_device_fallback(
+ dequant_device=dequant_device,
+ shard_prefix=shard_prefix,
+ op_name="FP8 dequant",
+ tensor_label=weight_name,
+ on_device=lambda: _dequant_fp8_linear_weight(
+ weight.to(dequant_device, non_blocking=True),
+ scale.to(dequant_device, non_blocking=True),
+ block_size=block_size,
+ ).to("cpu"),
+ on_cpu=lambda: _dequant_fp8_linear_weight(weight, scale, block_size=block_size),
+ )
+
+ return raw_tensors
+
+
+def _process_shard(
+ shard_path: str,
+ default_scheme: dict = None,
+ layer_config: dict = None,
+ ignore_patterns: list[str] = None,
+ device: str = "cpu",
+ *,
+ shard_name: str | None = None,
+ matcher: "_PatternMatcher | None" = None,
+ fp8_block_size: list | None = None,
+ model_type: str | None = None,
+ source_quantization_config: dict | None = None,
+ enable_torch_compile: bool = False,
+ disable_opt_rtn: bool = False,
+ index_dir: str | None = None,
+ donor_shard_dir: str | None = None,
+ donor_tensors_to_exclude: set[str] | None = None,
+) -> tuple[dict[str, torch.Tensor], list[str], list[str]]:
+ """Quantize eligible weights in a single safetensors shard.
+
+ Returns:
+ (output_tensors, quantized_layer_names, ignored_layer_names)
+
+ ``ignored_layer_names`` is derived by comparing the set of input ``.weight``
+ layer names (collected after fused-expert splitting) with the final set of
+ quantized layer names. Any layer that had a ``.weight`` tensor in the input
+ but was NOT quantized is reported as ignored — this correctly captures
+ user-ignored layers, predefined-skipped layers, non-eligible weights, and
+ any other pass-through case without separate per-tensor tracking.
+
+ Args:
+ index_dir: Directory containing the model's ``*.safetensors.index.json``.
+ Used to resolve cross-shard FP8 scale_inv tensors. Defaults to the
+ directory of *shard_path* (correct for non-streaming mode).
+ donor_shard_dir: Directory where donor shards are cached. Defaults to
+ *index_dir*. In streaming mode pass the local shard cache directory.
+ donor_tensors_to_exclude: If provided, these ``weight_scale_inv`` tensor
+ names are removed from the final output tensors. Used when this
+ shard is a donor: its scale_inv tensors have already been consumed
+ by the recipient shard and must not appear in the quantized output.
+ """
+ if matcher is None:
+ matcher = _PatternMatcher(
+ ignore_patterns if ignore_patterns is not None else [],
+ layer_config if layer_config is not None else {},
+ default_scheme if default_scheme is not None else {},
+ )
+
+ output_tensors: dict[str, torch.Tensor] = {}
+ quantized_layers: list[str] = []
+ quantize_func = compile_func(quantize_weight_rtn, device) if enable_torch_compile else quantize_weight_rtn
+
+ if shard_path.endswith(".bin"):
+ # PyTorch pickle checkpoint — load with weights_only where supported.
+ try:
+ raw_tensors = torch.load(shard_path, map_location="cpu", weights_only=True)
+ except TypeError:
+ # weights_only not available in older PyTorch versions
+ raw_tensors = torch.load(shard_path, map_location="cpu") # nosec
+ # Flatten nested state-dict wrappers if present.
+ if not isinstance(raw_tensors, dict):
+ raise ValueError(f"Expected a dict from {shard_path}, got {type(raw_tensors)}")
+ else:
+ from safetensors import safe_open
+
+ with safe_open(shard_path, framework="pt", device="cpu") as f:
+ raw_tensors = {name: f.get_tensor(name) for name in f.keys()}
+
+ raw_tensors = split_fused_expert_tensors(raw_tensors)
+
+ # Hydrate cross-shard FP8 weight_scale_inv tensors *before* any
+ # preprocessing below. Otherwise a weight whose scale lives in a sibling
+ # shard would miss the model_type-specific "expand-scale" passthrough
+ # handling (step 1) simply because its scale wasn't loaded yet, and would
+ # incorrectly fall through to the generic dequant-to-bf16 + RTN-requantize
+ # path instead of being treated identically to same-shard weights.
+ if shard_path:
+ raw_tensors = _hydrate_missing_fp8_scales_from_index(
+ raw_tensors,
+ shard_path,
+ shard_name=shard_name,
+ index_dir=index_dir,
+ donor_shard_dir=donor_shard_dir,
+ )
+
+ # Snapshot candidate weight layer names *before* any preprocessing. 1D
+ # weights (for example LayerNorm) are not quantization targets, while 3D
+ # weights remain tracked so unsupported layouts are visible as ignored.
+ input_weight_layers: list[str] = list(
+ dict.fromkeys(
+ name.rsplit(".", 1)[0]
+ for name, tensor in raw_tensors.items()
+ if name.endswith(".weight") and tensor.dim() > 1
+ )
+ )
+
+ # Preserve original tensors for predefined skipped layers so that already-
+ # quantized weights (FP8, FP4-packed, etc.) are NOT dequantized.
+ # User-specified ignore layers should still flow through the dequant path
+ # so the saved model exports them in full precision.
+ preserved_prefixes: set[str] = set()
+ for tname in raw_tensors:
+ if (
+ tname.endswith(".weight") or tname.endswith(".weight_packed") or tname.endswith(".qweight")
+ ) and matcher.should_skip(tname):
+ preserved_prefixes.add(tname.rsplit(".", 1)[0])
+
+ preserved_tensors: dict[str, torch.Tensor] = {}
+ if preserved_prefixes:
+ for key in list(raw_tensors.keys()):
+ prefix = key.rsplit(".", 1)[0]
+ if prefix in preserved_prefixes:
+ preserved_tensors[key] = raw_tensors.pop(key)
+
+ # 1) model-type-specific preprocessing (format conversion only)
+ raw_tensors, source_state = preprocess_model_type_source_tensors(
+ raw_tensors,
+ model_type=model_type,
+ quantization_config=source_quantization_config,
+ shard_name=shard_name,
+ )
+
+ # 1.5) normalize legacy NVFP4 names to llm-compressor naming.
+ raw_tensors, _converted_nvfp4_layers = _normalize_nvfp4_source_tensors(raw_tensors, shard_name=shard_name)
+
+ # 1.5) model-type-specific low-precision dequantization (e.g. kimi_k25 INT4)
+ raw_tensors = handle_model_type_low_precision_source_tensors(
+ raw_tensors,
+ model_type=model_type,
+ source_quant_config=source_quantization_config,
+ device=device,
+ shard_name=shard_name,
+ dequantize_with_device_fallback=_dequantize_with_device_fallback,
+ )
+
+ # 2) generic MXFP handling for both preprocessed and normal source models
+ raw_tensors, passthrough_tensors, passthrough_layers = _handle_mxfp_source_tensors(
+ raw_tensors,
+ matcher,
+ source_state=source_state,
+ device=device,
+ shard_name=shard_name,
+ )
+ output_tensors.update(passthrough_tensors)
+ quantized_layers.extend(passthrough_layers)
+
+ # 3) NVFP4 passthrough for layers already stored in packed format.
+ raw_tensors, nvfp_passthrough_tensors, nvfp_passthrough_layers = _handle_nvfp4_source_tensors(
+ raw_tensors,
+ matcher,
+ )
+ output_tensors.update(nvfp_passthrough_tensors)
+ quantized_layers.extend(nvfp_passthrough_layers)
+
+ raw_tensors = _dequant_fp8_tensors(
+ raw_tensors,
+ block_size=fp8_block_size,
+ device=device,
+ shard_name=shard_name,
+ shard_path=shard_path,
+ index_dir=index_dir,
+ donor_shard_dir=donor_shard_dir,
+ )
+ raw_tensors.update(preserved_tensors)
+
+ for tensor_name in list(raw_tensors.keys()):
+ tensor = raw_tensors.pop(tensor_name)
+ _layer_name, out_dict, q_layer, _ig_layer = _quantize_single_tensor(
+ tensor_name,
+ tensor,
+ matcher,
+ device,
+ quantize_func,
+ disable_opt_rtn,
+ )
+ output_tensors.update(out_dict)
+ if q_layer:
+ quantized_layers.append(q_layer)
+
+ # Remove scale_inv tensors that this shard donates to other shards.
+ # These tensors have no corresponding weight in this shard; keeping them
+ # would pollute the quantized output with raw FP8 scale metadata.
+ if donor_tensors_to_exclude:
+ removed = {t for t in donor_tensors_to_exclude if t in output_tensors}
+ for t in removed:
+ del output_tensors[t]
+ if removed:
+ shard_prefix = f"[{shard_name}] " if shard_name else ""
+ logger.debug(
+ f"{shard_prefix}Excluded {len(removed)} donated cross-shard scale_inv "
+ f"tensor(s) from quantized shard output."
+ )
+
+ # Derive ignored layers by comparing input weight layers with quantized set.
+ quantized_set = set(quantized_layers)
+ ignored_layers: list[str] = [l for l in input_weight_layers if l not in quantized_set]
+
+ return output_tensors, quantized_layers, ignored_layers
+
+
+# ---------------------------------------------------------------------------
+# Model Source I/O and Shard Discovery
+# ---------------------------------------------------------------------------
+
+
+def _get_model_cache_status(model_name_or_path: str) -> tuple[bool, str]:
+ """Return cache decision and a short reason string.
+
+ Cached means local dir, or HF cache contains config plus at least one
+ weight entry file (index or single-file checkpoint).
+ """
+ if os.path.isdir(model_name_or_path):
+ return True, "input is an existing local directory"
+
+ try:
+ from huggingface_hub import try_to_load_from_cache
+
+ # ``config.json`` alone is not enough: many workflows prefetch only
+ # config/tokenizer files, which would otherwise route to full
+ # ``snapshot_download``.
+ config_cached = isinstance(try_to_load_from_cache(model_name_or_path, "config.json"), str)
+ if not config_cached:
+ return False, "HF cache miss: config.json not found"
+
+ weight_entry_candidates = (
+ "model.safetensors.index.json",
+ "pytorch_model.bin.index.json",
+ "model.safetensors",
+ "pytorch_model.bin",
+ )
+ hit_entries: list[str] = []
+ for fname in weight_entry_candidates:
+ if isinstance(try_to_load_from_cache(model_name_or_path, fname), str):
+ hit_entries.append(fname)
+
+ if hit_entries:
+ return True, f"HF cache hit: config.json + {', '.join(hit_entries)}"
+ return (
+ False,
+ "HF cache partial hit: config.json exists but no weight entry file found",
+ )
+ except Exception as e:
+ return False, f"cache probe failed: {type(e).__name__}: {e}"
+
+
+def _is_model_cached(model_name_or_path: str) -> bool:
+ """Return True if the model is already available locally or in HF cache."""
+ cached, _ = _get_model_cache_status(model_name_or_path)
+ return cached
+
+
+def _resolve_source_dir(model_name_or_path: str) -> str:
+ """Resolve model_name_or_path to a local directory (download if needed)."""
+ if os.path.isdir(model_name_or_path):
+ return model_name_or_path
+ from huggingface_hub import snapshot_download
+
+ return snapshot_download(model_name_or_path)
+
+
+def _load_config(source_dir: str) -> dict:
+ """Load config.json from model directory."""
+ config_path = os.path.join(source_dir, "config.json")
+ if not os.path.exists(config_path):
+ raise FileNotFoundError(f"config.json not found in {source_dir}")
+ with open(config_path) as f:
+ return json.load(f)
+
+
+def _list_weight_shards(source_dir: str) -> list[str]:
+ """Return list of weight shard filenames in order.
+
+ Safetensors shards are preferred. When no safetensors files are
+ found the function falls back to PyTorch ``.bin`` shards.
+
+ Handles both standard naming (``model.safetensors``,
+ ``pytorch_model.bin``) and custom prefixes such as
+ ``diffusion_pytorch_model-XXXXX-of-XXXXX.safetensors`` by scanning
+ all ``*.safetensors.index.json`` / ``*.bin.index.json`` index files
+ in the directory when no standard index is found.
+ """
+
+ def _shards_from_index(index_path: str) -> list[str]:
+ with open(index_path) as f:
+ index = json.load(f)
+ seen: set[str] = set()
+ shards: list[str] = []
+ for shard_file in index["weight_map"].values():
+ if shard_file not in seen:
+ seen.add(shard_file)
+ shards.append(shard_file)
+ return shards
+
+ # --- safetensors: standard index ---
+ st_index = os.path.join(source_dir, "model.safetensors.index.json")
+ if os.path.exists(st_index):
+ return _shards_from_index(st_index)
+
+ # --- safetensors: custom-prefix index (e.g. diffusion_pytorch_model.safetensors.index.json) ---
+ for fname in sorted(os.listdir(source_dir)):
+ if fname.endswith(".safetensors.index.json"):
+ return _shards_from_index(os.path.join(source_dir, fname))
+
+ # --- safetensors: single file or index-less multi-file shards ---
+ st_files = sorted(f for f in os.listdir(source_dir) if f.endswith(".safetensors"))
+ if len(st_files) >= 1:
+ return st_files
+
+ # --- pytorch .bin: standard index ---
+ bin_index = os.path.join(source_dir, "pytorch_model.bin.index.json")
+ if os.path.exists(bin_index):
+ return _shards_from_index(bin_index)
+
+ # --- pytorch .bin: custom-prefix index ---
+ for fname in sorted(os.listdir(source_dir)):
+ if fname.endswith(".bin.index.json"):
+ return _shards_from_index(os.path.join(source_dir, fname))
+
+ # --- pytorch .bin: single file ---
+ bin_single = os.path.join(source_dir, "pytorch_model.bin")
+ if os.path.exists(bin_single):
+ return ["pytorch_model.bin"]
+
+ # --- pytorch .bin: any single .bin file ---
+ bin_files = sorted(f for f in os.listdir(source_dir) if f.endswith(".bin"))
+ if len(bin_files) >= 1:
+ return bin_files
+
+
+def _is_weight_shard(fname: str) -> bool:
+ """Return True if *fname* is a weight shard (safetensors or .bin).
+
+ Excludes index files (``*.index.json``) so that they are copied to the
+ output directory as normal metadata.
+ """
+ if fname.endswith(".index.json"):
+ return False
+ return fname.endswith(".safetensors") or fname.endswith(".bin")
+
+
+# Keep old name as an alias for backward compatibility.
+_is_safetensors_shard = _is_weight_shard
+
+
+def _download_single_shard(
+ model_name_or_path: str,
+ shard_filename: str,
+ local_dir: str,
+) -> str:
+ """Download a single safetensors shard file. Returns the local path."""
+ os.makedirs(local_dir, exist_ok=True)
+ local_path = os.path.join(local_dir, shard_filename)
+ if os.path.exists(local_path):
+ logger.info(f"Shard '{shard_filename}' already exists at '{local_path}', skipping download.")
+ return local_path
+
+ if os.path.isdir(model_name_or_path):
+ src = os.path.join(model_name_or_path, shard_filename)
+ if os.path.exists(src):
+ shutil.copy2(src, local_path)
+ return local_path
+ raise FileNotFoundError(f"{shard_filename} not found in {model_name_or_path}")
+
+ from huggingface_hub import hf_hub_download
+
+ return hf_hub_download(
+ repo_id=model_name_or_path,
+ filename=shard_filename,
+ local_dir=local_dir,
+ )
+
+
+def _download_metadata_files(
+ model_name_or_path: str,
+ local_dir: str,
+) -> str:
+ """Download all non-safetensors files from a model repo. Returns local dir."""
+ os.makedirs(local_dir, exist_ok=True)
+
+ if os.path.isdir(model_name_or_path):
+ for fname in os.listdir(model_name_or_path):
+ if _is_weight_shard(fname):
+ continue
+ src = os.path.join(model_name_or_path, fname)
+ dst = os.path.join(local_dir, fname)
+ if os.path.isdir(src):
+ if not os.path.exists(dst):
+ shutil.copytree(src, dst)
+ elif os.path.isfile(src) and not os.path.exists(dst):
+ shutil.copy2(src, dst)
+ return local_dir
+
+ from huggingface_hub import snapshot_download
+
+ snapshot_download(
+ repo_id=model_name_or_path,
+ local_dir=local_dir,
+ ignore_patterns=["*.safetensors", "*.bin", "*.pth", "*.pt"],
+ )
+ return local_dir
+
+
+# ---------------------------------------------------------------------------
+# ---------------------------------------------------------------------------
+# Output Writers
+# ---------------------------------------------------------------------------
+
+
+def _write_output_shard(
+ output_dir: str,
+ shard_name: str,
+ tensors: dict[str, torch.Tensor],
+ weight_map: dict[str, str],
+):
+ """Write a single output shard and update the weight_map."""
+ from safetensors.torch import save_file
+
+ shard_path = os.path.join(output_dir, shard_name)
+
+ # Detect shared-storage tensors (e.g. tie_word_embeddings: wte ↔ lm_head).
+ # safetensors refuses to serialise them as-is; clone the duplicates so each
+ # tensor occupies its own memory region. The first occurrence keeps the
+ # original storage; subsequent aliases are cloned.
+ seen_data_ptrs: set[int] = set()
+ deduped: dict[str, torch.Tensor] = {}
+ for k, v in tensors.items():
+ if not v.is_contiguous():
+ v = v.contiguous()
+ ptr = v.data_ptr()
+ if ptr in seen_data_ptrs:
+ v = v.clone()
+ else:
+ seen_data_ptrs.add(ptr)
+ deduped[k] = v
+
+ save_file(deduped, shard_path)
+ for tensor_name in tensors:
+ weight_map[tensor_name] = shard_name
+
+
+def _write_index_file(output_dir: str, weight_map: dict[str, str]):
+ """Write model.safetensors.index.json (or rename single shard)."""
+ if len(set(weight_map.values())) <= 1:
+ shard_names = list(set(weight_map.values()))
+ if shard_names and shard_names[0] != "model.safetensors":
+ src = os.path.join(output_dir, shard_names[0])
+ dst = os.path.join(output_dir, "model.safetensors")
+ if os.path.exists(src):
+ os.rename(src, dst)
+ weight_map = {k: "model.safetensors" for k in weight_map}
+ return
+
+ index = {"metadata": {"total_size": 0}, "weight_map": weight_map}
+ with open(os.path.join(output_dir, "model.safetensors.index.json"), "w") as f:
+ json.dump(index, f, indent=2)
+
+
+# ---------------------------------------------------------------------------
+# Quantization Configuration
+# ---------------------------------------------------------------------------
+
+
+def _get_llm_compressor_metadata() -> dict[str, str]:
+ """Return AutoRound provenance for model-free llm-compressor output."""
+ # Keep metadata deterministic and installation-source agnostic.
+ return {"provider": "auto-round"}
+
+
+def _build_nvfp4_e5m3_quantization_config(ignored_layers: list[str]) -> dict:
+ """Build compressed-tensors metadata for NVFP4 E5M3 without global scales."""
+ from auto_round.export.export_to_llmcompressor.config import initialize_nvfp4_e5m3_quantization
+
+ qconfig = initialize_nvfp4_e5m3_quantization(ignore=ignored_layers)
+ qconfig.update(_get_llm_compressor_metadata())
+ return qconfig
+
+
+def _get_mxfp_group_scheme_and_format(
+ group_bits: int,
+ group_data_type: str,
+ ignore: list[str],
+):
+ """Return ``(QuantizationScheme, format_str)`` for a single (bits, data_type) group.
+
+ Handles MXFP (mx_fp), NVFP4 (nv_fp), and NVFP4_E5M3 (nvfp4_v2) groups.
+ """
+ from auto_round.export.export_to_llmcompressor.config import (
+ initialize_nvfp4_e5m3_quantization,
+ initialize_quantization,
+ )
+
+ if is_nv_fp(group_data_type):
+ # Standard NVFP4 uses global scales and has a dedicated preset in
+ # compressed-tensors. Reuse that preset so metadata matches tensor
+ # layout when model-free does NVFP4 passthrough.
+ tmp_qconfig = initialize_quantization(scheme="NVFP4", ignore=ignore)
+ return tmp_qconfig.config_groups["group_0"], "nvfp4-pack-quantized"
+
+ if group_data_type == _NVFP4_E5M3_DATA_TYPE:
+ # NVFP4_E5M3 is not a compressed-tensors preset; use its dedicated builder.
+ nvfp4_dict = initialize_nvfp4_e5m3_quantization(ignore=ignore)
+ # Extract the QuantizationScheme object from the dict config.
+ from compressed_tensors.quantization import QuantizationScheme # pylint: disable=E0401
+
+ raw_scheme = nvfp4_dict["config_groups"]["group_0"]
+ scheme_obj = QuantizationScheme.model_validate(raw_scheme)
+ return scheme_obj, "nvfp4-e5m3-pack-quantized"
+ else:
+ scheme_name = "MXFP4" if group_bits == 4 else "MXFP8"
+ fmt = "mxfp4-pack-quantized" if group_bits == 4 else "mxfp8-quantized"
+ tmp_qconfig = initialize_quantization(scheme=scheme_name, ignore=ignore)
+ return tmp_qconfig.config_groups["group_0"], fmt
+
+
+def _build_mxfp_quantization_config(
+ default_scheme: dict,
+ quantized_layers: list[str],
+ ignored_layers: list[str],
+ layer_config: dict | None = None,
+) -> dict:
+ """Build a compressed-tensors / llm-compressor style quantization_config
+ dict for MXFP4 / MXFP8 model-free output, including mixed-precision cases.
+
+ When *layer_config* contains layers that override the default bits or
+ data_type (e.g. some layers are MXFP8 while the default is MXFP4, or
+ some layers use NVFP4_E5M3 while the default is MXFP8), the function
+ creates one ``config_group`` per distinct ``(bits, data_type)`` pair.
+ Override groups list their layers explicitly; the default group uses
+ ``targets=["Linear"]`` as a catch-all. The top-level ``"format"`` is
+ set to ``"mixed-precision"`` when more than one group is produced.
+
+ Mirrors the per-group format produced by
+ :mod:`auto_round.export.export_to_llmcompressor.export_to_fp`.
+ """
+ from auto_round.export.export_to_llmcompressor.config import (
+ initialize_quantization,
+ )
+
+ bits = default_scheme.get("bits", 4)
+ default_data_type = (default_scheme.get("data_type") or "mx_fp").lower()
+ is_fp_default = (bits or 0) >= 16 # BF16/FP16 full-precision default
+
+ if not is_fp_default and bits not in _SUPPORTED_MXFP_BITS and default_data_type != _NVFP4_E5M3_DATA_TYPE:
+ raise ValueError(f"Unsupported MXFP bits={bits} for model-free output.")
+
+ # Default ignore list: any layer present in ignored_layers (deduped) that
+ # was NOT quantized.
+ ignore = list(dict.fromkeys(ignored_layers))
+ quant_set = set(quantized_layers)
+ ignore = [n for n in ignore if n not in quant_set]
+
+ # Resolve each quantized layer's effective (bits, data_type) using layer_config overrides.
+ # Key: (bits, data_type) to distinguish e.g. MXFP4 from NVFP4_E5M3 (both 4-bit).
+ scheme_groups: dict[tuple[int, str], list[str]] = {} # (bits, data_type) -> [layer_names]
+ if layer_config:
+ temp_matcher = _PatternMatcher(
+ ignore_patterns=[],
+ layer_config=layer_config,
+ default_scheme=default_scheme,
+ )
+ for layer in quantized_layers:
+ scheme = temp_matcher.resolve_scheme(f"{layer}.weight")
+ layer_bits = scheme.get("bits", bits) if scheme is not None else bits
+ layer_dt = (
+ (scheme.get("data_type") or default_data_type).lower() if scheme is not None else default_data_type
+ )
+ scheme_groups.setdefault((layer_bits, layer_dt), []).append(layer)
+ else:
+ if not is_fp_default:
+ scheme_groups[(bits, default_data_type)] = list(quantized_layers)
+ # else: BF16 default with no layer_config → no MXFP layers; scheme_groups stays {}
+
+ if len(scheme_groups) <= 1:
+ # Single scheme — use the actual (bits, data_type) from the group.
+ if scheme_groups:
+ actual_bits, actual_dt = next(iter(scheme_groups.keys()))
+ else:
+ actual_bits, actual_dt = bits, default_data_type
+ if actual_dt != _NVFP4_E5M3_DATA_TYPE and actual_bits not in _SUPPORTED_MXFP_BITS:
+ raise ValueError(f"Unsupported MXFP bits={actual_bits} for model-free output.")
+ group_scheme, fmt = _get_mxfp_group_scheme_and_format(actual_bits, actual_dt, ignore)
+ if actual_dt == _NVFP4_E5M3_DATA_TYPE:
+ from auto_round.export.export_to_llmcompressor.config import initialize_nvfp4_e5m3_quantization
+
+ qconfig = initialize_nvfp4_e5m3_quantization(ignore=ignore)
+ if is_fp_default and scheme_groups:
+ qconfig["config_groups"]["group_0"]["targets"] = list(quantized_layers)
+ qconfig["format"] = fmt
+ qconfig.update(_get_llm_compressor_metadata())
+ return qconfig
+ from auto_round.export.export_to_llmcompressor.config import initialize_quantization as _init_q
+
+ scheme_name = "MXFP4" if actual_bits == 4 else "MXFP8"
+ qconfig = _init_q(scheme=scheme_name, ignore=ignore)
+ if is_fp_default and scheme_groups:
+ targets = list(quantized_layers)
+ qconfig.config_groups["group_0"].targets = targets
+ qconfig = qconfig.to_dict()
+ qconfig["format"] = fmt
+ qconfig.update(_get_llm_compressor_metadata())
+ return qconfig
+
+ # Mixed precision: build one config_group per distinct (bits, data_type).
+ # Override groups (non-default key) come first, default group last,
+ # ordered by descending bit-width within each partition so that the
+ # higher-precision group gets the lower group index.
+ default_key = (bits, default_data_type)
+ override_items = sorted(
+ [(key, layers) for key, layers in scheme_groups.items() if key != default_key],
+ key=lambda x: x[0][0],
+ reverse=True,
+ )
+ default_item = (default_key, scheme_groups[default_key]) if default_key in scheme_groups else None
+ ordered = override_items + ([default_item] if default_item else [])
+
+ config_groups: dict = {}
+ group_formats: dict[str, str] = {}
+ for idx, ((group_bits, group_dt), layer_names) in enumerate(ordered):
+ group_name = f"group_{idx}"
+ is_default_group = (group_bits, group_dt) == default_key
+ targets = ["Linear"] if is_default_group else layer_names
+ group_scheme, fmt = _get_mxfp_group_scheme_and_format(group_bits, group_dt, ignore)
+ group_scheme.targets = targets
+ config_groups[group_name] = group_scheme
+ group_formats[group_name] = fmt
+
+ full_qconfig = initialize_quantization(scheme=None, config_groups=config_groups, ignore=ignore)
+ full_dict = full_qconfig.to_dict()
+ full_dict["format"] = "mixed-precision"
+ for group_name, fmt in group_formats.items():
+ full_dict["config_groups"][group_name]["format"] = fmt
+ full_dict.update(_get_llm_compressor_metadata())
+ return full_dict
+
+
+def _build_mxfp_autoround_quantization_config(
+ default_scheme: dict,
+ quantized_layers: list[str],
+ ignored_layers: list[str],
+ layer_config: dict | None = None,
+ block_name_to_quantize: Optional[str] = None,
+) -> dict:
+ """Build an auto-round style quantization_config for MXFP4 / MXFP8.
+
+ Unlike :func:`_build_mxfp_quantization_config` which produces a
+ compressed-tensors / llm-compressor style config, this function produces
+ an ``auto-round`` style config (``quant_method="auto-round"``,
+ ``packing_format="auto_round:llm_compressor"``). The on-disk weight
+ tensors are identical; only the ``quantization_config`` metadata differs.
+
+ The generated config mirrors the layout of the regular AutoRound MXFP
+ export, including activation quantization fields (``act_bits`` /
+ ``act_data_type`` / …) from the scheme, ``enable_quanted_input``, and
+ ``lm_head`` in ``extra_config`` when it was quantized.
+ """
+ from collections import Counter
+
+ from auto_round.version import __version__
+
+ bits = default_scheme.get("bits", 4)
+ group_size = default_scheme.get("group_size", 32)
+ is_fp_default = (bits or 0) >= 16
+
+ # For BF16 default + MXFP layer_config overrides, derive the dominant
+ # MXFP bit-width from the layers that were actually quantized.
+ if is_fp_default and layer_config and quantized_layers:
+ temp_matcher = _PatternMatcher(
+ ignore_patterns=[],
+ layer_config=layer_config,
+ default_scheme=default_scheme,
+ )
+ mxfp_counter: Counter = Counter()
+ for layer in quantized_layers:
+ scheme = temp_matcher.resolve_scheme(f"{layer}.weight")
+ if scheme is None:
+ continue
+ lb = scheme.get("bits")
+ ldt = (scheme.get("data_type") or "").lower()
+ if lb and lb < 16 and is_mx_fp(ldt):
+ mxfp_counter[lb] += 1
+ if mxfp_counter:
+ bits, _ = mxfp_counter.most_common(1)[0]
+ group_size = 32
+
+ if (bits or 0) < 1 or bits not in _SUPPORTED_MXFP_BITS:
+ bits = 4 # safe fallback
+
+ qconfig: dict = {
+ "quant_method": "auto-round",
+ "packing_format": "auto_round:llm_compressor",
+ "bits": bits,
+ "group_size": group_size or 32,
+ "sym": True, # MXFP is always symmetric
+ "data_type": "mx_fp",
+ "iters": 0,
+ "model_free": True,
+ "autoround_version": __version__,
+ "enable_quanted_input": False,
+ }
+ if block_name_to_quantize:
+ qconfig["block_name_to_quantize"] = block_name_to_quantize
+
+ # Carry activation quantization fields from the scheme (e.g. MXFP4 has
+ # act_bits=4, act_data_type="mx_fp", act_dynamic=True, act_group_size=32,
+ # act_sym=True). Including these keeps the config consistent with the
+ # output produced by the regular AutoRound MXFP export flow.
+ for act_key in ("act_bits", "act_data_type", "act_dynamic", "act_group_size", "act_sym"):
+ val = default_scheme.get(act_key)
+ if val is not None:
+ qconfig[act_key] = val
+
+ scheme_keys = [f.name for f in fields(QuantizationScheme)]
+ extra_config: dict = {}
+ non_linear_ops = ["embed", "conv"]
+ non_linear_re = re.compile("|".join(re.escape(op) for op in non_linear_ops))
+
+ if layer_config:
+ from auto_round.export.export_to_autoround.utils import check_neq_config
+
+ expected_scheme = {key: qconfig.get(key) for key in scheme_keys}
+ for layer_name, cfg in layer_config.items():
+ if not isinstance(cfg, dict):
+ continue
+ neq_keys = check_neq_config(cfg, **expected_scheme)
+ if neq_keys:
+ extra_config[layer_name] = {key: cfg[key] for key in neq_keys if cfg.get(key) is not None}
+
+ quantized_set = set(quantized_layers)
+ unique_ignored = list(dict.fromkeys(ignored_layers))
+ for layer_name in unique_ignored:
+ if layer_name in quantized_set or layer_name in extra_config:
+ continue
+ if non_linear_re.search(layer_name):
+ continue
+ extra_config[layer_name] = {
+ "bits": 16,
+ "data_type": "float",
+ "act_bits": 16,
+ "act_data_type": "float",
+ }
+
+ # lm_head: when explicitly quantized, record its full scheme in extra_config
+ # (mirrors the regular AutoRound MXFP export behavior).
+ if "lm_head" in quantized_set and "lm_head" not in extra_config:
+ lm_head_cfg = (layer_config or {}).get("lm_head", default_scheme)
+ extra_config["lm_head"] = {k: lm_head_cfg.get(k) for k in scheme_keys if lm_head_cfg.get(k) is not None}
+
+ if extra_config:
+ qconfig["extra_config"] = extra_config
+
+ return qconfig
+
+
+def _layer_config_has_mxfp(layer_config: dict | None) -> bool:
+ """Return True if any layer_config entry requests MXFP quantization.
+
+ Handles values that are plain strings (preset names), dicts (possibly
+ with a ``"scheme"`` key or a ``"data_type"`` key), or
+ :class:`QuantizationScheme` instances.
+ """
+ if not layer_config:
+ return False
+ for val in layer_config.values():
+ if isinstance(val, str):
+ try:
+ s = _normalize_scheme(val.upper())
+ if is_mx_fp((s.data_type or "").lower()):
+ return True
+ except Exception:
+ pass
+ elif isinstance(val, dict):
+ # Direct data_type field
+ if is_mx_fp((val.get("data_type") or "").lower()):
+ return True
+ # Nested 'scheme' key (e.g. {"scheme": "MXFP4"})
+ scheme_val = val.get("scheme")
+ if isinstance(scheme_val, str):
+ try:
+ s = _normalize_scheme(scheme_val.upper())
+ if is_mx_fp((s.data_type or "").lower()):
+ return True
+ except Exception:
+ pass
+ elif isinstance(val, QuantizationScheme):
+ if is_mx_fp((val.data_type or "").lower()):
+ return True
+ return False
+
+
+def _layer_config_has_nvfp4(layer_config: dict | None) -> bool:
+ """Return True if any layer_config entry requests NVFP4 quantization.
+
+ Detects both the standard NVFP4 (``data_type='nv_fp'``) and the
+ global-scale-free variant NVFP4_E5M3 (``data_type='nvfp4_v2'``).
+
+ Handles values that are plain strings (preset names), dicts (possibly
+ with a ``"scheme"`` key or a ``"data_type"`` key), or
+ :class:`QuantizationScheme` instances.
+ """
+ if not layer_config:
+ return False
+ for val in layer_config.values():
+ if isinstance(val, str):
+ try:
+ s = _normalize_scheme(val.upper())
+ dt = (s.data_type or "").lower()
+ if dt == _NVFP4_E5M3_DATA_TYPE or is_nv_fp(dt):
+ return True
+ except Exception:
+ pass
+ elif isinstance(val, dict):
+ dt = (val.get("data_type") or "").lower()
+ if dt == _NVFP4_E5M3_DATA_TYPE or is_nv_fp(dt):
+ return True
+ scheme_val = val.get("scheme")
+ if isinstance(scheme_val, str):
+ try:
+ s = _normalize_scheme(scheme_val.upper())
+ dt = (s.data_type or "").lower()
+ if dt == _NVFP4_E5M3_DATA_TYPE or is_nv_fp(dt):
+ return True
+ except Exception:
+ pass
+ elif isinstance(val, QuantizationScheme):
+ dt = (val.data_type or "").lower()
+ if dt == _NVFP4_E5M3_DATA_TYPE or is_nv_fp(dt):
+ return True
+ return False
+
+
+def _get_layer_config_nvfp4_dt(layer_config: dict | None) -> str | None:
+ """Return the first NVFP4 data_type found in *layer_config*.
+
+ Returns ``"nv_fp"`` for standard NVFP4, ``"nvfp4_v2"`` for NVFP4_E5M3,
+ or ``None`` when no NVFP4 entry is present.
+ """
+ if not layer_config:
+ return None
+ for val in layer_config.values():
+ if isinstance(val, str):
+ try:
+ s = _normalize_scheme(val.upper())
+ dt = (s.data_type or "").lower()
+ if dt == _NVFP4_E5M3_DATA_TYPE:
+ return _NVFP4_E5M3_DATA_TYPE
+ if is_nv_fp(dt):
+ return "nv_fp"
+ except Exception:
+ pass
+ elif isinstance(val, dict):
+ dt = (val.get("data_type") or "").lower()
+ if dt == _NVFP4_E5M3_DATA_TYPE:
+ return _NVFP4_E5M3_DATA_TYPE
+ if is_nv_fp(dt):
+ return "nv_fp"
+ scheme_val = val.get("scheme")
+ if isinstance(scheme_val, str):
+ try:
+ s = _normalize_scheme(scheme_val.upper())
+ dt = (s.data_type or "").lower()
+ if dt == _NVFP4_E5M3_DATA_TYPE:
+ return _NVFP4_E5M3_DATA_TYPE
+ if is_nv_fp(dt):
+ return "nv_fp"
+ except Exception:
+ pass
+ elif isinstance(val, QuantizationScheme):
+ dt = (val.data_type or "").lower()
+ if dt == _NVFP4_E5M3_DATA_TYPE:
+ return _NVFP4_E5M3_DATA_TYPE
+ if is_nv_fp(dt):
+ return "nv_fp"
+ return None
+
+
+def _is_full_precision_default(scheme_input: Any) -> bool:
+ """Return True when *scheme_input* represents a full-precision default (bits >= 16).
+
+ Used to detect BF16/FP16 schemes where no weights are quantized by
+ default but layer_config overrides are still applied.
+ """
+ try:
+ s = _normalize_scheme(scheme_input)
+ return (s.bits or 0) >= 16 and (s.act_bits or 16) >= 16
+ except Exception:
+ return False
+
+
+def _derive_dominant_int_scheme(
+ quantized_layers: list[str],
+ layer_config: dict,
+ fallback: dict,
+) -> dict | None:
+ """Infer the dominant INT scheme from layers that were actually quantized.
+
+ Used when the default scheme is full-precision (BF16/FP16) to produce a
+ quantization_config where the most common INT scheme becomes the top-level
+ default and the BF16 layers appear as ``bits=16`` exceptions in
+ ``extra_config``. This matches the layout expected by AutoRound loaders
+ (dominant quantized bits at the top, full-precision overrides below).
+
+ Returns ``None`` when no INT layers were found in *quantized_layers*.
+ """
+ from collections import Counter
+
+ temp_matcher = _PatternMatcher(
+ ignore_patterns=[],
+ layer_config=layer_config,
+ default_scheme=fallback,
+ )
+ counter: "Counter[tuple]" = Counter()
+ for layer in quantized_layers:
+ scheme = temp_matcher.resolve_scheme(f"{layer}.weight")
+ if scheme is None:
+ continue
+ bits = scheme.get("bits")
+ if bits is None or bits >= 16:
+ continue
+ data_type = (scheme.get("data_type") or "int").lower()
+ if is_mx_fp(data_type):
+ continue # MXFP is handled by the dedicated path
+ key = (bits, scheme.get("group_size") or fallback.get("group_size"), bool(scheme.get("sym", True)), data_type)
+ counter[key] += 1
+
+ if not counter:
+ return None
+
+ (bits, group_size, sym, data_type), _ = counter.most_common(1)[0]
+ return {
+ "bits": bits,
+ "group_size": group_size,
+ "sym": sym,
+ "data_type": data_type,
+ }
+
+
+def _build_quantization_config(
+ default_scheme: dict,
+ layer_config: dict,
+ ignore_patterns: list[str],
+ quantized_layers: list[str],
+ ignored_layers: list[str],
+ block_name_to_quantize: Optional[str] = None,
+ format: str = "auto_round",
+) -> dict:
+ """Build a quantization_config dict compatible with auto-round format."""
+ # MXFP (mx_fp) supports two output styles depending on *format*:
+ # - "llm_compressor" → compressed-tensors / llm-compressor style config
+ # (produced by _build_mxfp_quantization_config).
+ # - "auto_round" / "auto_round:auto_gptq" → auto-round style config
+ # (produced by _build_mxfp_autoround_quantization_config).
+ # Also route to MXFP config when the default is full-precision (BF16/FP16)
+ # but layer_config contains MXFP overrides.
+ data_type = (default_scheme.get("data_type") or "int").lower()
+ default_bits = default_scheme.get("bits", 4)
+ is_fp_default = (default_bits or 0) >= 16 and not is_mx_fp(data_type)
+ if data_type == _NVFP4_E5M3_DATA_TYPE and format == "llm_compressor":
+ warnings.warn(
+ "LLMC/llm-compressor does not currently support the NVFP4_E5M3 scheme. "
+ "Please refer to docs/nvfp4_e5m3.md for the recommended export and usage path.",
+ UserWarning,
+ stacklevel=2,
+ )
+ return _build_nvfp4_e5m3_quantization_config(ignored_layers)
+ # BF16/FP16 default with NVFP4 layer_config overrides and llm_compressor format:
+ # build an NVFP4 config that targets only the explicitly quantized layers.
+ if is_fp_default and _layer_config_has_nvfp4(layer_config) and format == "llm_compressor":
+ nvfp4_dt = _get_layer_config_nvfp4_dt(layer_config)
+ targets = list(dict.fromkeys(quantized_layers)) if quantized_layers else ["Linear"]
+ if nvfp4_dt == _NVFP4_E5M3_DATA_TYPE:
+ warnings.warn(
+ "LLMC/llm-compressor does not currently support the NVFP4_E5M3 scheme. "
+ "Please refer to docs/nvfp4_e5m3.md for the recommended export and usage path.",
+ UserWarning,
+ stacklevel=2,
+ )
+ # Global-scale-free variant (NVFP4_E5M3)
+ from auto_round.export.export_to_llmcompressor.config import initialize_nvfp4_e5m3_quantization
+
+ qconfig = initialize_nvfp4_e5m3_quantization(ignore=[])
+ qconfig["config_groups"]["group_0"]["targets"] = targets
+ qconfig["format"] = "nvfp4-e5m3-pack-quantized"
+ else:
+ # Standard NVFP4 (nv_fp, scale_dtype=float8_e4m3fn)
+ from auto_round.export.export_to_llmcompressor.config import initialize_quantization
+
+ qconfig_obj = initialize_quantization(scheme="NVFP4", ignore=[])
+ qconfig = qconfig_obj.to_dict()
+ qconfig["config_groups"]["group_0"]["targets"] = targets
+ qconfig["format"] = "nvfp4-pack-quantized"
+ qconfig.update(_get_llm_compressor_metadata())
+ return qconfig
+ if is_mx_fp(data_type) or (is_fp_default and _layer_config_has_mxfp(layer_config)):
+ if format in ("auto_round", "auto_round:auto_gptq"):
+ return _build_mxfp_autoround_quantization_config(
+ default_scheme=default_scheme,
+ quantized_layers=quantized_layers,
+ ignored_layers=ignored_layers,
+ layer_config=layer_config,
+ block_name_to_quantize=block_name_to_quantize,
+ )
+ return _build_mxfp_quantization_config(
+ default_scheme=default_scheme,
+ quantized_layers=quantized_layers,
+ ignored_layers=ignored_layers,
+ layer_config=layer_config,
+ )
+
+ # When the default is full-precision (BF16/FP16) but INT layers were
+ # quantized via layer_config, derive the dominant INT scheme and use it as
+ # the top-level defaults so the output config reads as
+ # "INT-quantized with BF16 exceptions" rather than the inverse.
+ if is_fp_default and quantized_layers:
+ dominant = _derive_dominant_int_scheme(quantized_layers, layer_config, default_scheme)
+ if dominant is not None:
+ default_scheme = dominant
+
+ from auto_round.version import __version__
+
+ scheme_keys = [f.name for f in fields(QuantizationScheme)]
+ # vllm only support auto_round:auto_gptq, but transformers cannot load it correctly when sym=False.
+ # So we keep auto_round for asymmetric quantization to maintain compatibility with both.
+ if data_type == _NVFP4_E5M3_DATA_TYPE:
+ packing_format = "auto_round:fake" if format == "fake" else "auto_round:llm_compressor_nvfp4_e5m3"
+ else:
+ packing_format = "auto_round:auto_gptq" if default_scheme.get("sym", True) else "auto_round"
+
+ qconfig = {
+ "quant_method": "auto-round",
+ "packing_format": packing_format,
+ "bits": default_scheme["bits"],
+ "group_size": default_scheme["group_size"],
+ "sym": default_scheme.get("sym", True),
+ "data_type": default_scheme.get("data_type", "int"),
+ "iters": 0,
+ "model_free": True,
+ "autoround_version": __version__,
+ }
+
+ if data_type == _NVFP4_E5M3_DATA_TYPE:
+ for act_key in ("act_bits", "act_data_type", "act_group_size", "act_sym", "act_dynamic"):
+ value = default_scheme.get(act_key)
+ if value is not None:
+ qconfig[act_key] = value
+
+ if block_name_to_quantize:
+ qconfig["block_name_to_quantize"] = block_name_to_quantize
+
+ extra_config = {}
+ for layer_name, cfg in layer_config.items():
+ if cfg.get("bits", default_scheme["bits"]) >= 16:
+ extra_config[layer_name] = {k: cfg.get(k) for k in scheme_keys if cfg.get(k) is not None}
+ continue
+ differs = False
+ for key in ("bits", "group_size", "sym"):
+ if cfg.get(key) is not None and cfg[key] != default_scheme.get(key):
+ differs = True
+ break
+ if differs:
+ extra_config[layer_name] = {k: cfg.get(k) for k in scheme_keys if cfg.get(k) is not None}
+
+ # Filter out non-Linear ops (embed, conv) that don't need to be recorded in config.
+ # Routing gates and other predefined patterns are still recorded.
+ non_linear_ops = ["embed", "conv"]
+ non_linear_re = re.compile("|".join(re.escape(op) for op in non_linear_ops))
+
+ unique_ignored = list(dict.fromkeys(ignored_layers))
+ for layer_name in unique_ignored:
+ if layer_name not in extra_config:
+ # Skip non-Linear ops (embed, conv) since they're not Linear layers
+ if non_linear_re.search(layer_name):
+ continue
+ extra_config[layer_name] = {"bits": 16, "data_type": "float"}
+ if data_type == _NVFP4_E5M3_DATA_TYPE:
+ extra_config[layer_name].update({"act_bits": 16, "act_data_type": "float"})
+
+ quantized_layer_set = set(quantized_layers)
+ if "lm_head" in quantized_layer_set and "lm_head" not in extra_config:
+ lm_head_cfg = layer_config.get("lm_head", default_scheme)
+ extra_config["lm_head"] = {k: lm_head_cfg.get(k) for k in scheme_keys if lm_head_cfg.get(k) is not None}
+
+ if extra_config:
+ qconfig["extra_config"] = extra_config
+
+ return qconfig
+
+
+# ---------------------------------------------------------------------------
+# Scheme Validation and Override Resolution
+# ---------------------------------------------------------------------------
+
+
+def _apply_scheme_overrides(
+ scheme: Union[str, QuantizationScheme],
+ scheme_overrides: Optional[dict] = None,
+) -> QuantizationScheme:
+ """Return the effective scheme after applying non-None overrides."""
+ scheme_obj = copy.deepcopy(_normalize_scheme(scheme))
+ if not scheme_overrides:
+ return scheme_obj
+
+ valid_fields = {field.name for field in fields(QuantizationScheme)}
+ for key, value in scheme_overrides.items():
+ if key in valid_fields and value is not None:
+ setattr(scheme_obj, key, value)
+ return scheme_obj
+
+
+def _validate_supported_scheme(
+ scheme_obj: QuantizationScheme,
+ scheme_input: Union[str, QuantizationScheme],
+) -> None:
+ """Raise ``ValueError`` if *scheme_obj* is not supported by model-free.
+
+ Model-free supports:
+
+ * Integer weight-only quantization (sym/asym), ``bits ∈ {2, 4, 8}``,
+ packed in the ``auto_round:auto_gptq`` format.
+ * MXFP weight quantization (``data_type='mx_fp'``), ``bits ∈ {4, 8}``,
+ ``group_size=32``, packed in ``mxfp4-pack-quantized`` / ``mxfp8-quantized``
+ format (compressed-tensors compatible).
+ """
+ data_type = (scheme_obj.data_type or "int").lower()
+ bits = scheme_obj.bits
+ act_bits = scheme_obj.act_bits if scheme_obj.act_bits is not None else 16
+
+ # Full-precision (BF16/FP16) default: all layers stay in full precision
+ # unless layer_config provides lower-bit overrides.
+ if (bits or 0) >= 16 and act_bits >= 16:
+ return
+
+ # MXFP weight-only path: accept mx_fp data type with bits in {4, 8}.
+ # Activation quantization for MXFP is dynamic at inference time, so the
+ # weight-only RTN path here is independent of act_bits.
+ if is_mx_fp(data_type):
+ if scheme_obj.act_data_type not in (None, "mx_fp"):
+ raise ValueError(
+ "Model-free MXFP supports only act_data_type='mx_fp', " f"but got '{scheme_obj.act_data_type}'."
+ )
+ # Restrict to the two explicitly supported MXFP presets when a string
+ # name is provided. Variants such as MXFP4_RCEIL / MXFP8_RCEIL use a
+ # different activation format; silently mapping them to "MXFP4" /
+ # "MXFP8" in the output config would misrepresent the requested scheme.
+ if isinstance(scheme_input, str) and scheme_input.upper() not in ("MXFP4", "MXFP8"):
+ raise ValueError(
+ f"Model-free mode only supports MXFP preset names 'MXFP4' and 'MXFP8', "
+ f"but got '{scheme_input}'. "
+ f"Supported preset schemes: {list(SUPPORTED_PRESET_SCHEMES)}."
+ )
+ if bits is None or bits not in _SUPPORTED_MXFP_BITS:
+ raise ValueError(
+ f"Model-free mode supports MXFP bits in {_SUPPORTED_MXFP_BITS}, "
+ f"but '{scheme_input}' requests bits={bits}. "
+ f"Supported preset schemes: {list(SUPPORTED_PRESET_SCHEMES)}."
+ )
+ group_size = scheme_obj.group_size
+ if group_size not in (None, 32):
+ raise ValueError(
+ f"Model-free mode supports MXFP only with group_size=32, "
+ f"but '{scheme_input}' requests group_size={group_size}."
+ )
+ return
+
+ if data_type == _NVFP4_E5M3_DATA_TYPE:
+ if bits != 4 or scheme_obj.group_size != 16 or act_bits != 4:
+ raise ValueError(
+ f"Model-free NVFP4_E5M3 requires bits=4, group_size=16, and act_bits=4, "
+ f"but '{scheme_input}' requests bits={bits}, group_size={scheme_obj.group_size}, "
+ f"act_bits={act_bits}."
+ )
+ if (scheme_obj.act_data_type or "").lower() != _NVFP4_E5M3_DATA_TYPE or scheme_obj.act_group_size != 16:
+ raise ValueError(
+ f"Model-free NVFP4_E5M3 requires act_data_type='nvfp4_v2' and act_group_size=16, "
+ f"but '{scheme_input}' requests act_data_type='{scheme_obj.act_data_type}', "
+ f"act_group_size={scheme_obj.act_group_size}."
+ )
+ return
+
+ if act_bits < 16:
+ raise ValueError(
+ f"Model-free mode only supports weight-only quantization (WOQ) schemes "
+ f"where act_bits >= 16, but '{scheme_input}' has act_bits={act_bits}. "
+ f"Supported preset schemes: {list(SUPPORTED_PRESET_SCHEMES)}."
+ )
+
+ if data_type != "int":
+ raise ValueError(
+ f"Model-free mode only supports integer weight quantization "
+ f"(data_type='int') or MXFP (data_type='mx_fp'), but '{scheme_input}' "
+ f"has data_type='{data_type}'. FP8 / NVFP / GGUF / BF16 schemes require "
+ f"the standard AutoRound flow. Supported preset schemes: "
+ f"{list(SUPPORTED_PRESET_SCHEMES)}."
+ )
+
+ if bits is None or bits not in _SUPPORTED_INT_BITS:
+ raise ValueError(
+ f"Model-free mode supports bits in {_SUPPORTED_INT_BITS}, "
+ f"but '{scheme_input}' requests bits={bits}. "
+ f"Supported preset schemes: {list(SUPPORTED_PRESET_SCHEMES)}."
+ )
+
+
+def is_model_free_supported_scheme(
+ scheme: Union[str, QuantizationScheme],
+ scheme_overrides: Optional[dict] = None,
+) -> bool:
+ """Return True if *scheme* can be quantized via model-free mode.
+
+ Useful for CLI auto-routing logic. Never raises.
+ """
+ try:
+ scheme_obj = _apply_scheme_overrides(scheme, scheme_overrides)
+ _validate_supported_scheme(scheme_obj, scheme)
+ return True
+ except (ValueError, TypeError):
+ return False
+
+
+# ---------------------------------------------------------------------------
+# AutoScheme support (two-phase: delta-loss selection + model-free packing)
+# ---------------------------------------------------------------------------
+
+
+def _looks_like_auto_scheme(scheme: Any) -> bool:
+ """Duck-typed check for an :class:`~auto_round.auto_scheme.AutoScheme`.
+
+ Avoids importing ``AutoScheme`` at module scope (it pulls in exporter /
+ compressor modules that would create an import cycle with this file).
+ """
+ return hasattr(scheme, "options") and hasattr(scheme, "avg_bits")
+
+
+def _validate_auto_scheme_options(auto_scheme: Any) -> str:
+ """Validate AutoScheme options for model-free packing.
+
+ Returns the single quantized data-type family shared by all options
+ (``"int"`` or ``"mx_fp"``). ``BF16`` (or any >=16-bit no-op scheme) is
+ allowed and treated as "keep layer in full precision" during conversion,
+ so it does not contribute to the returned family.
+
+ Raises ``ValueError`` when any non-BF16 option is unsupported or when INT
+ and MXFP options are mixed (they use different packing formats and cannot
+ be produced in one model-free run).
+ """
+ options = list(getattr(auto_scheme, "options", []) or [])
+ if not options:
+ raise ValueError("AutoScheme.options must be non-empty for model-free mode.")
+
+ families: set[str] = set()
+ unsupported: list[Any] = []
+ for opt in options:
+ # Preserve original string validation semantics so preset-name
+ # restrictions (e.g. MXFP4/MXFP8 only) are enforced.
+ if isinstance(opt, str):
+ try:
+ scheme_obj = _normalize_scheme(opt)
+ except (ValueError, TypeError):
+ scheme_obj = None
+ elif isinstance(opt, QuantizationScheme):
+ scheme_obj = opt
+ else:
+ scheme_obj = None
+
+ # AutoScheme may include BF16/no-op options. In model-free flow these
+ # layers become ignore_layers entries (full precision), so they are
+ # allowed and excluded from family checks.
+ if scheme_obj is not None:
+ act_bits = scheme_obj.act_bits if scheme_obj.act_bits is not None else 16
+ if (scheme_obj.bits or 0) >= 16 and act_bits >= 16:
+ continue
+
+ # GGUF k-quants carry super_bits and are not packable by the model-free
+ # RTN kernel even though their data_type is nominally "int".
+ if scheme_obj is None or getattr(scheme_obj, "super_bits", None) is not None:
+ unsupported.append(opt)
+ continue
+ if not is_model_free_supported_scheme(opt):
+ unsupported.append(opt)
+ continue
+
+ data_type = (scheme_obj.data_type or "int").lower()
+ families.add("mx_fp" if is_mx_fp(data_type) else "int")
+
+ if unsupported:
+ raise ValueError(
+ f"Model-free + AutoScheme received unsupported option(s): {unsupported}. "
+ f"Model-free supports INT WOQ (bits in {_SUPPORTED_INT_BITS}) and MXFP "
+ f"(bits in {_SUPPORTED_MXFP_BITS}); GGUF / NVFP4 / FP8 options are not "
+ f"packable in model-free mode. Remove the unsupported options or pass "
+ f"disable_model_free=True to use the regular flow."
+ )
+ if len(families) > 1:
+ raise ValueError(
+ "Model-free + AutoScheme cannot mix INT and MXFP options in a single run "
+ f"(got families {sorted(families)}); INT and MXFP use different packing "
+ "formats. Use a single data-type family, or pass disable_model_free=True."
+ )
+ return families.pop()
+
+
+def _convert_auto_scheme_layer_config(
+ generated: dict[str, dict],
+ preferred_base_scheme: Union[str, QuantizationScheme, None] = None,
+) -> tuple[QuantizationScheme, dict[str, dict], list[str]]:
+ """Convert an AutoScheme-generated ``layer_config`` into model-free inputs.
+
+ Returns ``(base_scheme, per_layer_overrides, fp16_layers)`` where:
+
+ * ``base_scheme`` is ``preferred_base_scheme`` when provided, matching the
+ primary scheme selected by the regular AutoRound path. Otherwise the
+ most common quantized scheme is used as a fallback.
+ * ``per_layer_overrides`` maps every quantized layer name to its resolved
+ :class:`QuantizationScheme` fields.
+ * ``fp16_layers`` lists layers AutoScheme kept at >= 16 bits (added to the
+ model-free ignore list so they stay in full precision).
+ """
+ from collections import Counter
+
+ scheme_keys = {f.name for f in fields(QuantizationScheme)}
+ per_layer: dict[str, dict] = {}
+ fp16_layers: list[str] = []
+ counter: "Counter[tuple]" = Counter()
+
+ for name, cfg in generated.items():
+ if not isinstance(cfg, dict):
+ continue
+ bits = cfg.get("bits")
+ data_type_raw = (cfg.get("data_type") or "").strip()
+
+ # Infer bits from compact dtype aliases that embed bit-width in the name,
+ # e.g. "mxfp8" → 8, "MXFP4" → 4, "mx_fp4" → 4.
+ if bits is None and data_type_raw:
+ dt_lower = data_type_raw.lower()
+ for prefix in ("mxfp", "mx_fp", "nvfp", "nv_fp"):
+ tail = dt_lower[len(prefix) :]
+ if dt_lower.startswith(prefix) and tail.isdigit():
+ bits = int(tail)
+ break
+
+ if bits is None:
+ continue
+
+ clean = {k: cfg[k] for k in scheme_keys if cfg.get(k) is not None}
+ clean["bits"] = bits # honour inferred value when not explicit
+
+ # Normalise compact dtype aliases to canonical forms understood by
+ # the quantization kernels ("mxfp8" / "MXFP4" → "mx_fp").
+ if data_type_raw:
+ dt_lower = data_type_raw.lower()
+ if dt_lower.startswith("mxfp") or dt_lower.startswith("mx_fp"):
+ clean["data_type"] = "mx_fp"
+ elif dt_lower.startswith("nvfp") or dt_lower.startswith("nv_fp"):
+ clean["data_type"] = "nv_fp"
+
+ if bits >= 16:
+ fp16_layers.append(name)
+ continue
+ data_type = (clean.get("data_type") or "int").lower()
+ per_layer[name] = clean
+ counter[(clean.get("bits"), clean.get("group_size"), bool(clean.get("sym", True)), data_type)] += 1
+
+ if not counter:
+ raise ValueError("AutoScheme did not assign any quantizable layers for model-free mode.")
+
+ if preferred_base_scheme is not None:
+ base_scheme = copy.deepcopy(_normalize_scheme(preferred_base_scheme))
+ else:
+ (base_bits, base_group_size, base_sym, base_dtype), _ = counter.most_common(1)[0]
+ base_scheme = QuantizationScheme(
+ bits=base_bits,
+ group_size=base_group_size,
+ sym=base_sym,
+ data_type=base_dtype,
+ )
+ return base_scheme, per_layer, fp16_layers
+
+
+# ---------------------------------------------------------------------------
+# Model-Special Handlers (Keep at End)
+# ---------------------------------------------------------------------------
+
+
+def _read_int_like(value: Any) -> int | None:
+ """Best-effort conversion for int-like config values."""
+ if isinstance(value, bool) or value is None:
+ return None
+ if isinstance(value, int):
+ return value
+ if isinstance(value, float) and value.is_integer():
+ return int(value)
+ if isinstance(value, str):
+ v = value.strip()
+ if v.isdigit() or (v.startswith("-") and v[1:].isdigit()):
+ return int(v)
+ return None
+
+
+def _read_bool_like(value: Any) -> bool | None:
+ """Best-effort conversion for bool-like config values."""
+ if isinstance(value, bool):
+ return value
+ if isinstance(value, str):
+ v = value.strip().lower()
+ if v in {"true", "1", "yes"}:
+ return True
+ if v in {"false", "0", "no"}:
+ return False
+ return None
+
+
+def _resolve_kimi_k25_int4_params(source_quant_config: dict | None) -> tuple[int | None, bool]:
+ """Resolve default (group_size, sym) for kimi_k25 INT4 source dequant."""
+ group_size: int | None = None
+ sym = True
+
+ if not isinstance(source_quant_config, dict):
+ return group_size, sym
+
+ candidates = [source_quant_config]
+ weights_cfg = source_quant_config.get("weights")
+ if isinstance(weights_cfg, dict):
+ candidates.append(weights_cfg)
+
+ for cfg in candidates:
+ if group_size is None:
+ for key in ("group_size", "weight_group_size"):
+ maybe = _read_int_like(cfg.get(key))
+ if maybe is not None and maybe > 0:
+ group_size = maybe
+ break
+
+ sym_val = _read_bool_like(cfg.get("sym"))
+ if sym_val is not None:
+ sym = sym_val
+
+ zp_val = _read_bool_like(cfg.get("zero_point"))
+ if zp_val is not None:
+ sym = not zp_val
+
+ return group_size, sym
+
+
+def _collect_kimi_k25_int4_source_entries(
+ raw_tensors: dict[str, torch.Tensor],
+) -> list[tuple[str, str, str, str | None]]:
+ """Collect kimi_k25 INT4 source tensors."""
+ entries: list[tuple[str, str, str, str | None]] = []
+ for name, packed in raw_tensors.items():
+ if not name.endswith(".weight_packed"):
+ continue
+ if packed.dtype not in (torch.uint8, torch.int8, torch.int32):
+ continue
+
+ layer_name = name[: -len(".weight_packed")]
+ scale_key = f"{layer_name}.weight_scale"
+ if scale_key not in raw_tensors:
+ continue
+
+ scale = raw_tensors[scale_key]
+ if scale.dtype not in (torch.float16, torch.bfloat16, torch.float32, torch.float64):
+ continue
+ if packed.dim() != 2 or scale.dim() != 2:
+ continue
+
+ zp_key = None
+ for candidate in (f"{layer_name}.weight_zero_point", f"{layer_name}.zero_point"):
+ if candidate in raw_tensors:
+ zp_key = candidate
+ break
+
+ entries.append((layer_name, name, scale_key, zp_key))
+
+ return entries
+
+
+def _unpack_int4_weight_packed(packed: torch.Tensor) -> torch.Tensor:
+ """Unpack packed int4 bytes/ints into int16 ``[out, in]`` nibbles."""
+ if packed.dtype == torch.int32:
+ packed_i32 = packed.view(torch.int32)
+ rows, cols = packed_i32.shape
+ nibble_list = [(packed_i32 >> (4 * i)) & 0xF for i in range(8)]
+ out = torch.stack(nibble_list, dim=-1).reshape(rows, cols * 8).to(torch.int16)
+ return out
+
+ packed_u8 = packed.view(torch.uint8)
+ lo = packed_u8 & 0x0F
+ hi = (packed_u8 >> 4) & 0x0F
+
+ out = torch.empty(
+ (packed_u8.shape[0], packed_u8.shape[1] * 2),
+ dtype=torch.int16,
+ device=packed_u8.device,
+ )
+ out[:, 0::2] = lo.to(torch.int16)
+ out[:, 1::2] = hi.to(torch.int16)
+ return out
+
+
+def _dequant_kimi_k25_int4_tensors(
+ raw_tensors: dict[str, torch.Tensor],
+ source_quant_config: dict | None = None,
+ device: str = "cpu",
+ shard_name: str | None = None,
+ dequantize_with_device_fallback: Callable[..., torch.Tensor] | None = None,
+) -> dict[str, torch.Tensor]:
+ """Dequantize kimi_k25-style INT4 packed weights to bfloat16 ``.weight``."""
+ if dequantize_with_device_fallback is None:
+ raise ValueError("dequantize_with_device_fallback callback is required")
+
+ entries = _collect_kimi_k25_int4_source_entries(raw_tensors)
+ if not entries:
+ return raw_tensors
+
+ default_group_size, default_sym = _resolve_kimi_k25_int4_params(source_quant_config)
+ dequant_device = str(device or "cpu")
+ shard_prefix = f"[{shard_name}] " if shard_name else ""
+ logger.info(
+ f"{shard_prefix}Dequantizing kimi_k25 INT4 tensor(s) to bfloat16 on {dequant_device}: total={len(entries)}."
+ )
+
+ for layer_name, packed_key, scale_key, zp_key in entries:
+ packed = raw_tensors.pop(packed_key)
+ scale = raw_tensors.pop(scale_key)
+ zero = raw_tensors.pop(zp_key) if zp_key is not None else None
+ raw_tensors.pop(f"{layer_name}.weight_shape", None)
+
+ nibbles_per_element = 8 if packed.dtype == torch.int32 else 2
+ in_features = packed.shape[1] * nibbles_per_element
+ groups = scale.shape[1]
+ inferred_group_size = (in_features // groups) if groups > 0 and in_features % groups == 0 else None
+ group_size = inferred_group_size or default_group_size
+ if group_size is None or group_size <= 0:
+ raise ValueError(
+ f"Cannot resolve group_size for INT4 tensor '{layer_name}': "
+ f"packed_shape={tuple(packed.shape)}, scale_shape={tuple(scale.shape)}"
+ )
+ if in_features % group_size != 0:
+ raise ValueError(
+ f"Invalid group_size={group_size} for INT4 tensor '{layer_name}': "
+ f"in_features={in_features} is not divisible by group_size."
+ )
+
+ if (
+ inferred_group_size is not None
+ and default_group_size is not None
+ and inferred_group_size != default_group_size
+ ):
+ logger.warning(
+ f"{shard_prefix}k25 INT4 group_size mismatch for {layer_name}: "
+ f"config={default_group_size}, inferred={inferred_group_size}. "
+ "Using inferred value from tensor shapes."
+ )
+
+ n_groups = in_features // group_size
+ if scale.shape[1] != n_groups:
+ raise ValueError(
+ f"Scale shape mismatch for INT4 tensor '{layer_name}': "
+ f"scale.shape={tuple(scale.shape)}, expected second dim {n_groups}."
+ )
+
+ def _dequant_impl(target_device: str) -> torch.Tensor:
+ q = _unpack_int4_weight_packed(packed.to(target_device, non_blocking=True))
+ sc = scale.to(target_device, non_blocking=True).to(torch.float32)
+
+ if zero is not None:
+ zp = zero.to(target_device, non_blocking=True)
+ if zp.dim() == 2 and zp.shape == packed.shape and zp.dtype in (torch.int8, torch.uint8):
+ zp = _unpack_int4_weight_packed(zp)
+ elif zp.dim() == 2 and zp.shape[1] != n_groups:
+ raise ValueError(f"Unsupported zero-point shape for INT4 tensor '{layer_name}': {tuple(zp.shape)}")
+ if zp.dim() == 2 and zp.shape[1] == n_groups:
+ zp = zp.to(torch.int16).repeat_interleave(group_size, dim=1)
+ zp = zp[:, : q.shape[1]].to(torch.int16)
+ q = q - zp
+ else:
+ if default_sym:
+ q = torch.where(q >= 8, q - 16, q)
+ else:
+ q = q - 8
+
+ sc = sc.repeat_interleave(group_size, dim=1)[:, : q.shape[1]]
+ return (q.to(torch.float32) * sc).to(torch.bfloat16)
+
+ dq_weight = dequantize_with_device_fallback(
+ dequant_device=dequant_device,
+ shard_prefix=shard_prefix,
+ op_name="kimi_k25 INT4 dequant",
+ tensor_label=layer_name,
+ on_device=lambda: _dequant_impl(dequant_device).to("cpu"),
+ on_cpu=lambda: _dequant_impl("cpu"),
+ )
+ raw_tensors[f"{layer_name}.weight"] = dq_weight
+
+ return raw_tensors
+
+
+def _expand_e8m0_block_scale(
+ scale: torch.Tensor,
+ out_features: int,
+ in_features: int,
+ group_size: int = 32,
+) -> torch.Tensor:
+ """Expand a coarse 2D E8M0 block scale to per-group llm-compressor layout."""
+ scale = scale.view(torch.uint8)
+ if scale.dim() != 2:
+ raise ValueError(f"Expected a 2D E8M0 block scale, got shape {tuple(scale.shape)}.")
+
+ target_rows = out_features
+ target_cols = in_features // group_size
+ rows, cols = scale.shape
+
+ if target_rows % rows == 0 and target_cols % cols == 0:
+ if target_rows != rows:
+ scale = scale.repeat_interleave(target_rows // rows, dim=0)
+ if target_cols != cols:
+ scale = scale.repeat_interleave(target_cols // cols, dim=1)
+ return scale.contiguous()
+
+ coarse_block = 128
+ expected_rows = (out_features + coarse_block - 1) // coarse_block
+ expected_cols = (in_features + coarse_block - 1) // coarse_block
+ if rows == expected_rows and cols == expected_cols:
+ if coarse_block % group_size != 0:
+ raise ValueError(
+ f"Cannot expand DeepSeek E8M0 block scale with group_size={group_size}; "
+ f"{coarse_block} is not divisible by group_size."
+ )
+ groups_per_block_col = coarse_block // group_size
+ scale = scale.repeat_interleave(coarse_block, dim=0)[:target_rows]
+ scale = scale.repeat_interleave(groups_per_block_col, dim=1)[:, :target_cols]
+ return scale.contiguous()
+
+ raise ValueError(
+ f"Cannot expand E8M0 block scale {tuple(scale.shape)} to "
+ f"({target_rows}, {target_cols}); unsupported coarse/block layout."
+ )
+
+
+def preprocess_model_type_source_tensors(
+ raw_tensors: dict[str, torch.Tensor],
+ model_type: str | None,
+ group_size: int = 32,
+ quantization_config: dict | None = None,
+ shard_name: str | None = None,
+) -> tuple[dict[str, torch.Tensor], dict[str, int]]:
+ """Apply model-type-specific source tensor normalization."""
+ model_type = (model_type or "").lower()
+ quantization_config = quantization_config or {}
+ is_deepseek_v4 = model_type == "deepseek_v4"
+ is_deepseek_v32_ue8m0 = (
+ model_type == "deepseek_v32"
+ and quantization_config.get("quant_method") == "fp8"
+ and str(quantization_config.get("fmt", "")).lower() == "e4m3"
+ and str(quantization_config.get("scale_fmt", "")).lower() == "ue8m0"
+ )
+ if not is_deepseek_v4 and not is_deepseek_v32_ue8m0:
+ return raw_tensors, {}
+
+ entries: list[tuple[str, str, bool]] = []
+ for name, tensor in raw_tensors.items():
+ if not name.endswith(".weight"):
+ continue
+ layer_name = name[: -len(".weight")]
+ scale_candidates = [f"{layer_name}.scale"]
+ if is_deepseek_v32_ue8m0:
+ scale_candidates.extend((f"{layer_name}.weight_scale", f"{layer_name}.weight_scale_inv"))
+ scale_name = next((candidate for candidate in scale_candidates if candidate in raw_tensors), None)
+ if scale_name is None:
+ continue
+ if tensor.dtype == torch.float8_e4m3fn:
+ entries.append((name, scale_name, True))
+ elif tensor.dtype in (torch.int8, torch.uint8):
+ entries.append((name, scale_name, False))
+
+ if not entries:
+ return raw_tensors, {}
+
+ source_state: dict[str, int] = {}
+ n_fp8 = 0
+ n_fp4 = 0
+ for weight_name, scale_name, is_fp8 in entries:
+ layer_name = weight_name[: -len(".weight")]
+ weight = raw_tensors.pop(weight_name)
+ scale = raw_tensors.pop(scale_name)
+
+ if is_fp8:
+ out_features, in_features = weight.shape
+ weight_key = f"{layer_name}.weight"
+ source_state[layer_name] = 8
+ n_fp8 += 1
+
+ if scale.dtype == torch.float32:
+ shard_prefix = f"[{shard_name}] " if shard_name else ""
+ logger.warning_once(
+ f"[{model_type}] Scale tensor pattern has dtype float32 "
+ f"with UE8M0 encoding (only the 8-bit exponent is significant). "
+ f"Extracting uint8 E8M0 exponent bytes from fp32 representation."
+ )
+ scale = ((scale.view(torch.int32) >> 23) & 0xFF).to(torch.uint8)
+ else:
+ out_features = weight.shape[0]
+ in_features = weight.shape[1] * 2
+ weight = weight.view(torch.uint8).contiguous()
+ weight_key = f"{layer_name}.weight_packed"
+ source_state[layer_name] = 4
+ n_fp4 += 1
+
+ weight_scale = _expand_e8m0_block_scale(scale, out_features, in_features, group_size=group_size)
+ raw_tensors[weight_key] = weight
+ raw_tensors[f"{layer_name}.weight_scale"] = weight_scale
+
+ shard_prefix = f"[{shard_name}] " if shard_name else ""
+ logger.info(
+ f"{shard_prefix}Applied model_type preprocessing for {model_type}: "
+ f"{n_fp8} MXFP8 layer(s), {n_fp4} MXFP4 layer(s) converted to llm-compressor naming."
+ )
+ return raw_tensors, source_state
+
+
+def handle_model_type_low_precision_source_tensors(
+ raw_tensors: dict[str, torch.Tensor],
+ model_type: str | None,
+ source_quant_config: dict | None = None,
+ device: str = "cpu",
+ shard_name: str | None = None,
+ dequantize_with_device_fallback: Callable[..., torch.Tensor] | None = None,
+) -> dict[str, torch.Tensor]:
+ """Handle model-type-specific low-precision source tensors."""
+ if dequantize_with_device_fallback is None:
+ dequantize_with_device_fallback = _dequantize_with_device_fallback
+
+ if (model_type or "").lower() == "kimi_k25":
+ return _dequant_kimi_k25_int4_tensors(
+ raw_tensors,
+ source_quant_config=source_quant_config,
+ device=device,
+ shard_name=shard_name,
+ dequantize_with_device_fallback=dequantize_with_device_fallback,
+ )
+ return raw_tensors
diff --git a/auto_round_extension/cuda/cute_nvfp4_e5m3.py b/auto_round_extension/cuda/cute_nvfp4_e5m3.py
new file mode 100644
index 000000000..b71ae4614
--- /dev/null
+++ b/auto_round_extension/cuda/cute_nvfp4_e5m3.py
@@ -0,0 +1,270 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Optional CuTe DSL dispatch for NVFP4 E5M3 activation QDQ."""
+
+from functools import lru_cache
+from importlib.util import find_spec
+from typing import Optional
+
+import torch
+
+from auto_round.logger import logger
+
+_SUPPORTED_DTYPES = (torch.float16, torch.bfloat16, torch.float32)
+_GROUP_SIZE = 16
+_THREADS_PER_BLOCK = 256
+_FAILED_KERNEL_KEYS = set()
+_FAILED_WEIGHT_DQ_KERNEL_KEYS = set()
+
+
+@lru_cache(maxsize=1)
+def is_cute_dsl_available() -> bool:
+ """Return whether the optional NVIDIA CuTe DSL package is installed."""
+ return find_spec("cutlass") is not None
+
+
+def can_use_cute_nvfp4_v2_qdq(activation: torch.Tensor, group_size: int) -> bool:
+ """Check whether an activation can use the CuTe QDQ kernel."""
+ if not is_cute_dsl_available() or not activation.is_cuda:
+ return False
+ if group_size != _GROUP_SIZE or activation.dtype not in _SUPPORTED_DTYPES:
+ return False
+ if not activation.is_contiguous() or activation.shape[-1] % group_size:
+ return False
+ return torch.cuda.get_device_capability(activation.device)[0] >= 8
+
+
+def _make_qdq_kernel():
+ import cutlass
+ import cutlass.cute as cute
+ from cutlass._mlir_helpers import math
+
+ @cute.kernel
+ def qdq_kernel(input_tensor: cute.Tensor, output_tensor: cute.Tensor):
+ thread_idx, _, _ = cute.arch.thread_idx()
+ block_idx, _, _ = cute.arch.block_idx()
+ group_idx = block_idx * _THREADS_PER_BLOCK + thread_idx
+
+ if group_idx < input_tensor.shape[0]:
+ amax = cutlass.Float32(0.0)
+ for column_idx in range(_GROUP_SIZE):
+ value = cutlass.Float32(input_tensor[group_idx, column_idx])
+ amax = cute.arch.fmax(amax, math.abs(value))
+
+ # This reproduces the E5M3 scale domain used by nvfp4_v2 without
+ # materializing scale or intermediate FP32 tensors in global memory.
+ scale = amax / cutlass.Float32(6.0)
+ if scale > cutlass.Float32(0.0):
+ exponent = math.floor(math.log2(scale)) + cutlass.Float32(1.0)
+ mantissa = scale / math.exp2(exponent)
+ mantissa_bits = math.roundeven((mantissa - cutlass.Float32(0.5)) * cutlass.Float32(16.0))
+ mantissa_bits = cute.arch.fmin(cute.arch.fmax(mantissa_bits, 0.0), 7.0)
+ scale = (cutlass.Float32(1.0) + mantissa_bits / cutlass.Float32(8.0)) * math.exp2(
+ exponent - cutlass.Float32(1.0)
+ )
+
+ for column_idx in range(_GROUP_SIZE):
+ value = cutlass.Float32(input_tensor[group_idx, column_idx])
+ scaled = value / scale if scale > cutlass.Float32(0.0) else cutlass.Float32(0.0)
+ magnitude = math.abs(scaled)
+ quantized = cutlass.Float32(0.0)
+ if magnitude < cutlass.Float32(2.0):
+ quantized = math.roundeven(magnitude * cutlass.Float32(2.0)) / cutlass.Float32(2.0)
+ elif magnitude < cutlass.Float32(4.0):
+ quantized = math.roundeven(magnitude)
+ else:
+ quantized = cutlass.Float32(2.0) * math.roundeven(magnitude / cutlass.Float32(2.0))
+ quantized = cute.arch.fmin(cute.arch.fmax(quantized, 0.0), 6.0)
+ if scaled < cutlass.Float32(0.0):
+ quantized = -quantized
+ output_tensor[group_idx, column_idx] = (quantized * scale).to(output_tensor.element_type)
+
+ @cute.jit
+ def launch_qdq(input_tensor: cute.Tensor, output_tensor: cute.Tensor):
+ groups = input_tensor.shape[0]
+ qdq_kernel(input_tensor, output_tensor).launch(
+ grid=((groups + _THREADS_PER_BLOCK - 1) // _THREADS_PER_BLOCK, 1, 1),
+ block=(_THREADS_PER_BLOCK, 1, 1),
+ )
+
+ return launch_qdq
+
+
+def _make_weight_dq_kernel():
+ import cutlass
+ import cutlass.cute as cute
+ from cutlass._mlir_helpers import math
+
+ @cute.kernel
+ def weight_dq_kernel(weight_packed: cute.Tensor, weight_scale: cute.Tensor, output_tensor: cute.Tensor):
+ thread_idx, _, _ = cute.arch.thread_idx()
+ block_idx, _, _ = cute.arch.block_idx()
+ group_idx = block_idx * _THREADS_PER_BLOCK + thread_idx
+
+ if group_idx < output_tensor.shape[0]:
+ scale_byte = cutlass.Float32(weight_scale[group_idx, 0])
+ exponent = math.floor(scale_byte / cutlass.Float32(8.0))
+ mantissa = scale_byte - exponent * cutlass.Float32(8.0)
+ scale = (cutlass.Float32(1.0) + mantissa / cutlass.Float32(8.0)) * math.exp2(
+ exponent - cutlass.Float32(15.0)
+ )
+ if exponent == cutlass.Float32(0.0):
+ scale = mantissa * math.exp2(cutlass.Float32(-17.0))
+
+ for column_idx in range(_GROUP_SIZE):
+ packed = cutlass.Float32(weight_packed[group_idx, column_idx // 2])
+ packed_high = math.floor(packed / cutlass.Float32(16.0))
+ nibble = packed - packed_high * cutlass.Float32(16.0)
+ if column_idx % 2:
+ nibble = packed_high
+
+ sign = cutlass.Float32(1.0)
+ if nibble >= cutlass.Float32(8.0):
+ sign = cutlass.Float32(-1.0)
+ nibble = nibble - cutlass.Float32(8.0)
+
+ value = cutlass.Float32(0.0)
+ if nibble == cutlass.Float32(1.0):
+ value = cutlass.Float32(0.5)
+ elif nibble == cutlass.Float32(2.0):
+ value = cutlass.Float32(1.0)
+ elif nibble == cutlass.Float32(3.0):
+ value = cutlass.Float32(1.5)
+ elif nibble == cutlass.Float32(4.0):
+ value = cutlass.Float32(2.0)
+ elif nibble == cutlass.Float32(5.0):
+ value = cutlass.Float32(3.0)
+ elif nibble == cutlass.Float32(6.0):
+ value = cutlass.Float32(4.0)
+ elif nibble == cutlass.Float32(7.0):
+ value = cutlass.Float32(6.0)
+
+ output_tensor[group_idx, column_idx] = (sign * value * scale).to(output_tensor.element_type)
+
+ @cute.jit
+ def launch_weight_dq(weight_packed: cute.Tensor, weight_scale: cute.Tensor, output_tensor: cute.Tensor):
+ groups = output_tensor.shape[0]
+ weight_dq_kernel(weight_packed, weight_scale, output_tensor).launch(
+ grid=((groups + _THREADS_PER_BLOCK - 1) // _THREADS_PER_BLOCK, 1, 1),
+ block=(_THREADS_PER_BLOCK, 1, 1),
+ )
+
+ return launch_weight_dq
+
+
+@lru_cache(maxsize=None)
+def _get_compiled_qdq_kernel(device_index: int, dtype: torch.dtype):
+ import cutlass.cute as cute
+ from cutlass.cute.runtime import from_dlpack
+
+ input_tensor = torch.empty((1, _GROUP_SIZE), device=f"cuda:{device_index}", dtype=dtype)
+ output_tensor = torch.empty_like(input_tensor)
+ return cute.compile(
+ _make_qdq_kernel(),
+ from_dlpack(input_tensor).mark_layout_dynamic(),
+ from_dlpack(output_tensor).mark_layout_dynamic(),
+ )
+
+
+@lru_cache(maxsize=None)
+def _get_compiled_weight_dq_kernel(device_index: int, dtype: torch.dtype):
+ import cutlass.cute as cute
+ from cutlass.cute.runtime import from_dlpack
+
+ weight_packed = torch.empty((2, _GROUP_SIZE // 2), device=f"cuda:{device_index}", dtype=torch.uint8)
+ weight_scale = torch.empty((2, 1), device=f"cuda:{device_index}", dtype=torch.uint8)
+ output_tensor = torch.empty((2, _GROUP_SIZE), device=f"cuda:{device_index}", dtype=dtype)
+ return cute.compile(
+ _make_weight_dq_kernel(),
+ from_dlpack(weight_packed).mark_layout_dynamic(),
+ from_dlpack(weight_scale).mark_layout_dynamic(),
+ from_dlpack(output_tensor).mark_layout_dynamic(),
+ )
+
+
+def try_cute_nvfp4_v2_qdq(activation: torch.Tensor, group_size: int) -> Optional[torch.Tensor]:
+ """Run a CuTe DSL group-size-16 FP4 QDQ kernel when eligible."""
+ if not can_use_cute_nvfp4_v2_qdq(activation, group_size):
+ return None
+ if torch.cuda.is_current_stream_capturing():
+ return None
+
+ kernel_key = (activation.device.index, activation.dtype)
+ if kernel_key in _FAILED_KERNEL_KEYS:
+ return None
+
+ try:
+ groups = activation.numel() // group_size
+ output = torch.empty_like(activation)
+ compiled_qdq = _get_compiled_qdq_kernel(*kernel_key)
+ compiled_qdq(activation.reshape(groups, group_size), output.reshape(groups, group_size))
+ return output
+ except Exception as error:
+ _FAILED_KERNEL_KEYS.add(kernel_key)
+ logger.warning_once("CuTe NVFP4 E5M3 QDQ failed; falling back to PyTorch reference: %s", error)
+ return None
+
+
+def try_cute_nvfp4_e5m3_weight_dq(
+ weight_packed: torch.Tensor, weight_scale: torch.Tensor, dtype: torch.dtype
+) -> Optional[torch.Tensor]:
+ """Dequantize packed FP4 E5M3 weights with CuTe."""
+ if (
+ not is_cute_dsl_available()
+ or not weight_packed.is_cuda
+ or weight_packed.dtype != torch.uint8
+ or weight_scale.dtype != torch.uint8
+ or weight_packed.ndim != 2
+ or weight_scale.shape != (weight_packed.shape[0], weight_packed.shape[1] // (_GROUP_SIZE // 2))
+ or dtype not in _SUPPORTED_DTYPES
+ or torch.cuda.get_device_capability(weight_packed.device)[0] < 8
+ or torch.cuda.is_current_stream_capturing()
+ ):
+ return None
+
+ kernel_key = (weight_packed.device.index, dtype)
+ if kernel_key in _FAILED_WEIGHT_DQ_KERNEL_KEYS:
+ return None
+
+ try:
+ out_features = weight_packed.shape[0]
+ in_features = weight_packed.shape[1] * 2
+ groups = weight_scale.numel()
+ packed_groups = weight_packed.reshape(groups, _GROUP_SIZE // 2)
+ scale_groups = weight_scale.reshape(groups, 1)
+ output = torch.empty((groups, _GROUP_SIZE), device=weight_packed.device, dtype=dtype)
+ compiled_weight_dq = _get_compiled_weight_dq_kernel(*kernel_key)
+ compiled_weight_dq(packed_groups, scale_groups, output)
+ return output.reshape(out_features, in_features)
+ except Exception as error:
+ _FAILED_WEIGHT_DQ_KERNEL_KEYS.add(kernel_key)
+ logger.warning_once("CuTe NVFP4 E5M3 weight DQ failed; falling back to PyTorch reference: %s", error)
+ return None
+
+
+def try_cute_nvfp4_e5m3_linear(
+ activation: torch.Tensor,
+ weight_packed: torch.Tensor,
+ weight_scale: torch.Tensor,
+ bias: Optional[torch.Tensor],
+) -> Optional[torch.Tensor]:
+ """Reserved second-stage dispatch point for fused QDQ, unpack, and GEMM.
+
+ Returning ``None`` keeps the reference Linear path active until the packed-weight
+ mainloop has been validated against the existing E5M3 checkpoint format.
+ """
+ del activation, weight_packed, weight_scale, bias
+ fused_output: Optional[torch.Tensor] = None
+ return fused_output
diff --git a/docs/environments.md b/docs/environments.md
index 4e77d2ce0..47f36cb30 100644
--- a/docs/environments.md
+++ b/docs/environments.md
@@ -171,6 +171,16 @@ export AR_AUTO_SCHEME_CACHE=/path/to/auto_scheme_cache
export AR_ENABLE_AUTO_SCHEME_PARALLEL=0
```
+### AR_NVFP4_E5M3_CACHE_HP_WEIGHT
+- **Description**: Controls whether `NVFP4E5M3QuantLinear` caches a dequantized high-precision weight after the first forward pass, instead of dequantizing the packed FP4 weight on every call.
+- **Default**: `False` (equivalent to `"0"`)
+- **Valid Values**: `"1"`, `"true"`, `"yes"`, `"on"` (case-insensitive) enable caching; any other value disables caching
+- **Usage**: Enable this when repeated inference throughput matters more than memory footprint. The current implementation releases `weight_packed` and `weight_scale` after materializing the cached high-precision weight, so steady-state memory usage increases and the cache cannot be cleared back to packed storage.
+
+```bash
+export AR_NVFP4_E5M3_CACHE_HP_WEIGHT=1
+```
+
### AR_DISK_STREAM_MODEL
- **Description**: When enabled, `AutoRound(model=<path>, ...)` builds the model as a meta-device skeleton instead of fully materializing the checkpoint on CPU RAM up front, and streams each decoder block's real weights from the checkpoint's safetensors shards on demand -- materializing right before a block is used (calibration, tuning, or `AutoScheme` sensitivity scoring) and freeing it back to meta right after. This keeps peak CPU RAM roughly flat regardless of checkpoint size, instead of proportional to it. Non-block parameters (embeddings, `lm_head`, final norm) are still loaded up front, since they are typically small. Text-model AutoScheme scoring also supports combining this with parallel scoring, which is enabled by default; each worker streams its own block copy.
- **Default**: `False`
diff --git a/docs/environments_CN.md b/docs/environments_CN.md
index 8684ccdcf..365b3b46d 100644
--- a/docs/environments_CN.md
+++ b/docs/environments_CN.md
@@ -171,6 +171,16 @@ export AR_AUTO_SCHEME_CACHE=/path/to/auto_scheme_cache
export AR_ENABLE_AUTO_SCHEME_PARALLEL=0
```
+### AR_NVFP4_E5M3_CACHE_HP_WEIGHT
+- **描述**:控制 `NVFP4E5M3QuantLinear` 是否在首次前向后缓存解量化得到的高精度权重,而不是每次调用都从打包的 FP4 权重重新解量化。
+- **默认值**:`False`(等价于 `"0"`)
+- **有效值**:`"1"`、`"true"`、`"yes"`、`"on"`(不区分大小写)表示启用缓存;其他值表示禁用缓存
+- **用途**:当重复推理吞吐比内存占用更重要时可启用。当前实现会在缓存高精度权重后释放 `weight_packed` 和 `weight_scale`,因此稳态内存占用会增大,且之后无法再切回打包存储。
+
+```bash
+export AR_NVFP4_E5M3_CACHE_HP_WEIGHT=1
+```
+
### AR_DISK_STREAM_MODEL
- **描述**:启用后,`AutoRound(model=<path>, ...)` 会将模型构建为 meta 设备骨架,而不是先把整个 checkpoint 完全加载到 CPU 内存;随后按需从 checkpoint 的 safetensors 分片中流式加载每个解码器块的真实权重——在该块被使用前(校准、调优或 `AutoScheme` 敏感度评分)才实体化,用完后立即释放回 meta。这样峰值 CPU 内存基本保持平稳,而不会随 checkpoint 大小成比例增长。非块参数(embedding、`lm_head`、最终归一化层)体积通常较小,仍会一次性加载。文本模型的 AutoScheme 评分也支持与默认启用的并行评分组合使用;每个 worker 会流式加载自己的 block 副本。
- **默认值**:`False`
diff --git a/docs/nvfp4_e5m3.md b/docs/nvfp4_e5m3.md
new file mode 100644
index 000000000..a88aeaa66
--- /dev/null
+++ b/docs/nvfp4_e5m3.md
@@ -0,0 +1,139 @@
+# NVFP4 E5M3 Quantization (Experimental)
+
+> **Note**: This is an experimental feature. APIs, formats, and behaviours may
+> change in future releases.
+
+## Overview
+
+NVFP4 E5M3 is a quantization scheme that stores weights in FP4 (E2M1) format
+with **unsigned E5M3 block scales** (instead of the global-scale approach used
+by standard NVFP4). Activations are quantized on-the-fly via a QDQ
+(Quantize-Dequantize) pass.
+
+Key properties:
+
+| Property | Value |
+|---|---|
+| Preset name | `NVFP4_E5M3` |
+| Weight dtype | FP4 E2M1 (`nvfp4_v2`) |
+| Scale dtype | UE5M3 (per-block) |
+| Group size | 16 |
+| Activation bits | 4 (dynamic QDQ) |
+
+---
+
+> Important: `llm_compressor` export is not supported for the `NVFP4_E5M3` scheme in LLMC/llm-compressor for now. Please prefer the AutoRound format or the `fake` format for current workflows, and keep following the vLLM usage docs for the latest supported path.
+
+## Supported Output Formats
+
+| Format flag | Description |
+|---|---|
+| `auto_round` (default) | Packed `.weight_packed` + `.weight_scale` tensors; loadable with AutoRound inference |
+| `llm_compressor` | `nvfp4-e5m3-pack-quantized` compressed-tensors layout |
+| `fake` | Full-precision (BF16) `.weight` tensors with activation QDQ at runtime; no packing overhead |
+
+---
+
+## Supported Inference Backends
+
+| Backend | Description |
+|---|---|
+| `auto_round:cute_nvfp4_e5m3` | CUDA CuTe DSL kernel (dequant weight + QDQ act). Requires `cutlass` package and SM80+. **Fastest**. |
+| `auto_round:torch_nvfp4_e5m3` | Pure PyTorch dequant + QDQ act. Works on any CUDA device. |
+| `auto_round:fake` | BF16 weight + activation QDQ only (no weight packing). For debugging / profiling. |
+
+---
+
+## Usage
+
+### Standard AutoRound Flow
+
+```python
+from auto_round import AutoRound
+from auto_round.schemes import NVFP4_E5M3
+
+model, tokenizer = ... # load your model
+
+autoround = AutoRound(model, tokenizer, scheme=NVFP4_E5M3)
+autoround.quantize()
+
+# Save as packed auto_round format (default)
+autoround.save_quantized("output_dir")
+
+# Or save as llm_compressor format
+autoround.save_quantized("output_dir", format="llm_compressor")
+
+# Or save as fake format (full-precision weights + QDQ at runtime)
+autoround.save_quantized("output_dir", format="fake")
+```
+
+### Model-Free Quantization (no GPU training required)
+
+```bash
+# Pack a pre-quantized model (from HuggingFace or local dir)
+auto-round \
+ --model <model_id_or_path> \
+ --scheme NVFP4_E5M3 \
+ --model_free \
+ --output_dir ./nvfp4_e5m3_model
+```
+
+```python
+from auto_round.compressors.model_free import ModelFreeCompressor
+from auto_round.schemes import NVFP4_E5M3
+
+compressor = ModelFreeCompressor(
+ model="<model_id_or_path>",
+ scheme=NVFP4_E5M3,
+ output_dir="./nvfp4_e5m3_model",
+)
+compressor.run()
+```
+
+### Loading a Quantized Model
+
+```python
+from transformers import AutoModelForCausalLM, AutoTokenizer
+import auto_round # registers the AutoRound inference hooks
+
+model = AutoModelForCausalLM.from_pretrained(
+ "./nvfp4_e5m3_model",
+ device_map="auto",
+)
+tokenizer = AutoTokenizer.from_pretrained("./nvfp4_e5m3_model")
+```
+
+---
+
+## Environment Variables
+
+### `AR_NVFP4_E5M3_CACHE_HP_WEIGHT`
+
+Controls whether NVFP4 E5M3 `QuantLinear` caches a dequantized high-precision
+(BF16) weight after the first forward pass instead of dequantizing on every
+call.
+
+| Value | Behaviour |
+|---|---|
+| `0` / `false` / `no` / `off` (default) | Dequantize on every forward pass (lower memory, higher compute) |
+| `1` / `true` / `yes` / `on` | Cache the dequantized weight after first forward (lower compute, higher memory) |
+
+When caching is enabled, the packed weight buffers are released after the
+cache is materialized, trading lower runtime overhead for higher steady-state
+memory usage.
+
+```bash
+export AR_NVFP4_E5M3_CACHE_HP_WEIGHT=1
+```
+
+---
+
+## Accuracy Results (Llama-3.1-8B-Instruct, LAMBADA OpenAI)
+
+| Configuration | Time | Accuracy |
+|---|---|---|
+| BF16 baseline | 1m 5s | 0.7213 |
+| `auto_round` + `auto_round:cute_nvfp4_e5m3` | 1m 48s | 0.7196 |
+| `auto_round` + `auto_round:torch_nvfp4_e5m3` | 4m 23s | 0.7204 |
+| `llm_compressor` + `auto_round:cute_nvfp4_e5m3` | 1m 48s | 0.7196 |
+| `fake` + `auto_round:fake` | 3m 26s | 0.7204 |
diff --git a/docs/nvfp4_e5m3_CN.md b/docs/nvfp4_e5m3_CN.md
new file mode 100644
index 000000000..f9d5a76a0
--- /dev/null
+++ b/docs/nvfp4_e5m3_CN.md
@@ -0,0 +1,131 @@
+# NVFP4 E5M3 量化(实验性功能)
+
+> **注意**:这是一个实验性功能,后续版本中 API、格式及行为可能发生变化。
+
+## 概述
+
+NVFP4 E5M3 是一种量化方案,以 FP4(E2M1)格式存储权重,并使用**无符号 E5M3 块级 scale**(区别于标准 NVFP4 使用全局 scale 的方式)。激活值通过 QDQ(量化-反量化)操作在推理时动态量化。
+
+主要参数:
+
+| 参数 | 值 |
+|---|---|
+| 预设名称 | `NVFP4_E5M3` |
+| 权重数据类型 | FP4 E2M1(`nvfp4_v2`) |
+| Scale 数据类型 | UE5M3(块级别) |
+| 分组大小 | 16 |
+| 激活量化位宽 | 4(动态 QDQ) |
+
+---
+
+> 重要:LLMC/llm-compressor 当前并不支持 `NVFP4_E5M3` 方案的导出。当前请优先使用 AutoRound 原生格式或 `fake` 格式,并持续关注 vLLM 使用文档中的最新支持路径。
+
+## 支持的输出格式
+
+| 格式标志 | 说明 |
+|---|---|
+| `auto_round`(默认) | 打包的 `.weight_packed` + `.weight_scale` 张量,可使用 AutoRound 推理加载 |
+| `llm_compressor` | `nvfp4-e5m3-pack-quantized` compressed-tensors 布局 |
+| `fake` | 全精度(BF16)`.weight` 张量,推理时仅做激活 QDQ,无打包开销 |
+
+---
+
+## 支持的推理后端
+
+| 后端 | 说明 |
+|---|---|
+| `auto_round:cute_nvfp4_e5m3` | CUDA CuTe DSL 内核(权重反量化 + 激活 QDQ),需要 `cutlass` 包及 SM80+,**速度最快** |
+| `auto_round:torch_nvfp4_e5m3` | 纯 PyTorch 反量化 + 激活 QDQ,适用于任意 CUDA 设备 |
+| `auto_round:fake` | BF16 权重 + 仅激活 QDQ(无权重打包),用于调试 / 性能分析 |
+
+---
+
+## 使用方法
+
+### 标准 AutoRound 流程
+
+```python
+from auto_round import AutoRound
+from auto_round.schemes import NVFP4_E5M3
+
+model, tokenizer = ... # 加载你的模型
+
+autoround = AutoRound(model, tokenizer, scheme=NVFP4_E5M3)
+autoround.quantize()
+
+# 以 auto_round 打包格式保存(默认)
+autoround.save_quantized("output_dir")
+
+# 或以 llm_compressor 格式保存
+autoround.save_quantized("output_dir", format="llm_compressor")
+
+# 或以 fake 格式保存(全精度权重 + 推理时 QDQ)
+autoround.save_quantized("output_dir", format="fake")
+```
+
+### 无模型量化(无需 GPU 训练)
+
+```bash
+# 对已有模型进行直接打包(来自 HuggingFace 或本地目录)
+auto-round \
+ --model <model_id_or_path> \
+ --scheme NVFP4_E5M3 \
+ --model_free \
+ --output_dir ./nvfp4_e5m3_model
+```
+
+```python
+from auto_round.compressors.model_free import ModelFreeCompressor
+from auto_round.schemes import NVFP4_E5M3
+
+compressor = ModelFreeCompressor(
+ model="<model_id_or_path>",
+ scheme=NVFP4_E5M3,
+ output_dir="./nvfp4_e5m3_model",
+)
+compressor.run()
+```
+
+### 加载量化后的模型
+
+```python
+from transformers import AutoModelForCausalLM, AutoTokenizer
+import auto_round # 注册 AutoRound 推理钩子
+
+model = AutoModelForCausalLM.from_pretrained(
+ "./nvfp4_e5m3_model",
+ device_map="auto",
+)
+tokenizer = AutoTokenizer.from_pretrained("./nvfp4_e5m3_model")
+```
+
+---
+
+## 环境变量
+
+### `AR_NVFP4_E5M3_CACHE_HP_WEIGHT`
+
+控制 NVFP4 E5M3 `QuantLinear` 是否在首次前向传播后缓存反量化的高精度(BF16)权重,而不是每次调用时重新反量化。
+
+| 取值 | 行为 |
+|---|---|
+| `0` / `false` / `no` / `off`(默认) | 每次前向传播都进行反量化(内存更少,计算量更多) |
+| `1` / `true` / `yes` / `on` | 首次前向后缓存反量化权重(计算量更少,内存占用更多) |
+
+启用缓存后,打包权重缓冲区在缓存生成后会被释放,以较高的稳态内存换取较低的运行时开销。
+
+```bash
+export AR_NVFP4_E5M3_CACHE_HP_WEIGHT=1
+```
+
+---
+
+## 精度结果(Llama-3.1-8B-Instruct,LAMBADA OpenAI)
+
+| 配置 | 耗时 | 准确率 |
+|---|---|---|
+| BF16 基线 | 1m 5s | 0.7213 |
+| `auto_round` + `auto_round:cute_nvfp4_e5m3` | 1m 48s | 0.7196 |
+| `auto_round` + `auto_round:torch_nvfp4_e5m3` | 4m 23s | 0.7204 |
+| `llm_compressor` + `auto_round:cute_nvfp4_e5m3` | 1m 48s | 0.7196 |
+| `fake` + `auto_round:fake` | 3m 26s | 0.7204 |
diff --git a/test/unit/test_cpu/core/test_format_decoupling.py b/test/unit/test_cpu/core/test_format_decoupling.py
index b9dac1280..2dea8725d 100644
--- a/test/unit/test_cpu/core/test_format_decoupling.py
+++ b/test/unit/test_cpu/core/test_format_decoupling.py
@@ -32,7 +32,9 @@ import torch.nn as nn
from auto_round.algorithms.quantization.config import QuantizationConfig
from auto_round.compressors.base import BaseCompressor
from auto_round.compressors.config_resolution import FormatResolution, ResolvedScheme
+from auto_round.experimental import qmodules as ar_qmodules
from auto_round.export.formats import resolve_formats
+from auto_round.inference.backend import BackendInfos, dynamic_import_inference_linear, get_layer_backend
from auto_round.schemes import QuantizationScheme, parse_scheme
@@ -50,6 +52,7 @@ EXPECTED_FORMAT_SELECTION_BASELINE = {
("FP8_STATIC", "llm_compressor"): ["llm_compressor:fp8_static"],
("MXFP4", "auto_round"): ["auto_round:mx_fp"],
("NVFP4", "auto_round"): ["auto_round:nv_fp"],
+ ("NVFP4_E5M3", "auto_round"): ["auto_round:nvfp4_v2"],
}
@@ -67,6 +70,30 @@ def test_format_selection_baseline():
assert results == EXPECTED_FORMAT_SELECTION_BASELINE
+def test_nvfp4_e5m3_autoround_uses_llm_compressor_packing_and_torch_fallback():
+ scheme = _resolved_scheme("NVFP4_E5M3")
+ resolution = resolve_formats(
+ ResolvedScheme.from_scheme(scheme), format="auto_round", model=nn.Sequential(nn.Linear(16, 16))
+ )
+
+ assert resolution.formats[0].get_backend_name() == "auto_round:nvfp4_v2"
+ assert BackendInfos["auto_round:torch_nvfp4_e5m3"].packing_format == ["auto_round:llm_compressor_nvfp4_e5m3"]
+ assert BackendInfos["auto_round:cute_nvfp4_e5m3"].priority > BackendInfos["auto_round:torch_nvfp4_e5m3"].priority
+ assert dynamic_import_inference_linear("auto_round:torch_nvfp4_e5m3", scheme) is ar_qmodules.NVFP4E5M3QuantLinear
+ assert dynamic_import_inference_linear("auto_round:cute_nvfp4_e5m3", scheme) is ar_qmodules.CuteNVFP4E5M3QuantLinear
+ assert (
+ get_layer_backend(
+ "cpu",
+ "auto",
+ "auto_round:llm_compressor_nvfp4_e5m3",
+ scheme,
+ in_features=16,
+ out_features=16,
+ )
+ == "auto_round:torch_nvfp4_e5m3"
+ )
+
+
def test_gguf_correction_propagates_to_scheme_and_quantize_config(monkeypatch):
# W4A16's plain int-woq fields don't match every field gguf_args_check enforces
# for a GGUF target; after resolving "gguf:q4_k_m" the scheme-bearing objects
diff --git a/test/unit/test_cpu/data_type/test_nvfp.py b/test/unit/test_cpu/data_type/test_nvfp.py
index a231a30ec..444a14aad 100644
--- a/test/unit/test_cpu/data_type/test_nvfp.py
+++ b/test/unit/test_cpu/data_type/test_nvfp.py
@@ -29,11 +29,11 @@ from auto_round.data_type.nvfp import (
cast_to_ue5m3_ste,
e5m3_to_float_tensor,
float_to_e5m3_frexp,
- fp4_v2,
- fp4_v2_with_global_scale,
get_reciprocal,
nv_fp4,
nv_fp4_with_static_gs,
+ nvfp4_v2,
+ nvfp4_v2_with_global_scale,
ref_fp4_quant,
ref_nvfp4_quant,
search_nvfp4_scale,
@@ -538,106 +538,106 @@ class TestRefFp4Quant:
# ---------------------------------------------------------------------------
-# fp4_v2_with_global_scale
+# nvfp4_v2_with_global_scale
# ---------------------------------------------------------------------------
class TestFp4V2WithGlobalScale:
- """Test fp4_v2_with_global_scale."""
+ """Test nvfp4_v2_with_global_scale."""
def test_group_size_16(self):
t = torch.randn(4, 32, dtype=torch.bfloat16)
- q, s, z = fp4_v2_with_global_scale(t, group_size=16)
+ q, s, z = nvfp4_v2_with_global_scale(t, group_size=16)
assert q.shape == t.shape
assert z is None
def test_group_size_32(self):
t = torch.randn(4, 64, dtype=torch.bfloat16)
- q, s, z = fp4_v2_with_global_scale(t, group_size=32)
+ q, s, z = nvfp4_v2_with_global_scale(t, group_size=32)
assert q.shape == t.shape
def test_invalid_group_size(self):
t = torch.randn(4, 32, dtype=torch.bfloat16)
with pytest.raises(AssertionError):
- fp4_v2_with_global_scale(t, group_size=64)
+ nvfp4_v2_with_global_scale(t, group_size=64)
def test_tensor_max_as_float(self):
t = torch.randn(4, 32, dtype=torch.bfloat16)
- q, s, z = fp4_v2_with_global_scale(t, group_size=16, tensor_max=2.0)
+ q, s, z = nvfp4_v2_with_global_scale(t, group_size=16, tensor_max=2.0)
assert q.shape == t.shape
def test_tensor_max_as_tensor(self):
t = torch.randn(4, 32, dtype=torch.bfloat16)
tm = torch.tensor(1.0, dtype=torch.float32)
- q, s, z = fp4_v2_with_global_scale(t, group_size=16, tensor_max=tm)
+ q, s, z = nvfp4_v2_with_global_scale(t, group_size=16, tensor_max=tm)
assert q.shape == t.shape
def test_tensor_max_multi_element(self):
t = torch.randn(4, 32, dtype=torch.bfloat16)
tm = torch.tensor([1.0, 2.0])
- q, s, z = fp4_v2_with_global_scale(t, group_size=16, tensor_max=tm)
+ q, s, z = nvfp4_v2_with_global_scale(t, group_size=16, tensor_max=tm)
assert q.shape == t.shape
def test_with_max_scale(self):
t = torch.randn(4, 32, dtype=torch.bfloat16)
- q, s, z = fp4_v2_with_global_scale(t, group_size=16, max_scale=1.5)
+ q, s, z = nvfp4_v2_with_global_scale(t, group_size=16, max_scale=1.5)
assert q.shape == t.shape
def test_float16_input(self):
t = torch.randn(4, 32, dtype=torch.float16)
- q, s, z = fp4_v2_with_global_scale(t, group_size=16)
+ q, s, z = nvfp4_v2_with_global_scale(t, group_size=16)
assert q.dtype == torch.float16
def test_with_v(self):
t = torch.randn(4, 32, dtype=torch.bfloat16)
- q, s, z = fp4_v2_with_global_scale(t, group_size=16, v=0.5)
+ q, s, z = nvfp4_v2_with_global_scale(t, group_size=16, v=0.5)
assert q.shape == t.shape
# ---------------------------------------------------------------------------
-# fp4_v2
+# nvfp4_v2
# ---------------------------------------------------------------------------
-class TestFp4V2:
- """Test fp4_v2."""
+class TestNvfp4V2:
+ """Test nvfp4_v2."""
def test_group_size_16(self):
t = torch.randn(4, 32, dtype=torch.bfloat16)
- q, s, z = fp4_v2(t, group_size=16)
+ q, s, z = nvfp4_v2(t, group_size=16)
assert q.shape == t.shape
assert z is None
def test_group_size_32(self):
t = torch.randn(4, 64, dtype=torch.bfloat16)
- q, s, z = fp4_v2(t, group_size=32)
+ q, s, z = nvfp4_v2(t, group_size=32)
assert q.shape == t.shape
def test_invalid_group_size(self):
t = torch.randn(4, 32, dtype=torch.bfloat16)
with pytest.raises(AssertionError):
- fp4_v2(t, group_size=128)
+ nvfp4_v2(t, group_size=128)
def test_with_max_scale(self):
t = torch.randn(4, 32, dtype=torch.bfloat16)
- q, s, z = fp4_v2(t, group_size=16, max_scale=1.5)
+ q, s, z = nvfp4_v2(t, group_size=16, max_scale=1.5)
assert q.shape == t.shape
def test_with_v(self):
t = torch.randn(4, 32, dtype=torch.bfloat16)
- q, s, z = fp4_v2(t, group_size=16, v=0.5)
+ q, s, z = nvfp4_v2(t, group_size=16, v=0.5)
assert q.shape == t.shape
def test_float16_input(self):
t = torch.randn(4, 32, dtype=torch.float16)
- q, s, z = fp4_v2(t, group_size=16)
+ q, s, z = nvfp4_v2(t, group_size=16)
assert q.dtype == torch.float16
def test_non_divisible_dim(self):
"""Last dim not divisible by group_size -> padding path."""
t = torch.randn(4, 50, dtype=torch.bfloat16)
# group_size=32 divides 50 with padding to 64
- q, s, z = fp4_v2(t, group_size=32)
+ q, s, z = nvfp4_v2(t, group_size=32)
assert q.shape == t.shape
@@ -654,8 +654,8 @@ class TestQuantizationProperties:
[
lambda t: nv_fp4(t),
lambda t: nv_fp4_with_static_gs(t),
- lambda t: fp4_v2(t),
- lambda t: fp4_v2_with_global_scale(t),
+ lambda t: nvfp4_v2(t),
+ lambda t: nvfp4_v2_with_global_scale(t),
],
)
def test_outputs_are_finite(self, fn):
@@ -670,8 +670,8 @@ class TestQuantizationProperties:
[
lambda t: nv_fp4(t),
lambda t: nv_fp4_with_static_gs(t),
- lambda t: fp4_v2(t),
- lambda t: fp4_v2_with_global_scale(t),
+ lambda t: nvfp4_v2(t),
+ lambda t: nvfp4_v2_with_global_scale(t),
],
)
def test_dtype_preserved(self, fn):
diff --git a/test/unit/test_cpu/export/test_fake_format.py b/test/unit/test_cpu/export/test_fake_format.py
new file mode 100644
index 000000000..10b064e4a
--- /dev/null
+++ b/test/unit/test_cpu/export/test_fake_format.py
@@ -0,0 +1,188 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import os
+from types import SimpleNamespace
+
+import torch
+from transformers import AutoModelForCausalLM, OPTConfig, OPTForCausalLM
+
+from auto_round.experimental.qmodules.fake import FakeActQuantLinear
+from auto_round.formats import FakeFormat
+from auto_round.inference.convert_model import convert_hf_model
+from auto_round.schemes import PRESET_SCHEMES
+from auto_round.wrapper import WrapperWALayer
+
+
+class _WrappedLinear(WrapperWALayer):
+ def __init__(self, linear):
+ torch.nn.Module.__init__(self)
+ self.orig_layer = linear
+ self.register_buffer("act_max_scale", torch.ones(1))
+
+ def forward(self, inputs):
+ return self.orig_layer(inputs)
+
+
+class _SaveableModel(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ linear = torch.nn.Linear(4, 3)
+ linear.register_parameter("act_max_scale", torch.nn.Parameter(torch.ones(1)))
+ self.linear = _WrappedLinear(linear)
+ self.config = SimpleNamespace()
+
+ def save_pretrained(self, output_dir):
+ os.makedirs(output_dir, exist_ok=True)
+ torch.save(self.state_dict(), os.path.join(output_dir, "pytorch_model.bin"))
+ with open(os.path.join(output_dir, "config.json"), "w") as config_file:
+ json.dump({"quantization_config": self.config.quantization_config}, config_file)
+
+
+def test_fake_format_unwraps_quantized_layers_before_save(tmp_path):
+ model = _SaveableModel()
+ expected_weight = model.linear.orig_layer.weight.detach().clone()
+ output_dir = str(tmp_path / "fake_model")
+
+ FakeFormat("fake", PRESET_SCHEMES["NVFP4_E5M3"], SimpleNamespace(mllm=False)).save_quantized(
+ output_dir=output_dir,
+ model=model,
+ inplace=False,
+ serialization_dict={
+ "bits": 4,
+ "group_size": 16,
+ "sym": True,
+ "data_type": "nvfp4_v2",
+ "act_bits": 4,
+ "act_group_size": 16,
+ "act_sym": True,
+ "act_data_type": "nvfp4_v2",
+ "supported_types": [torch.nn.Linear],
+ },
+ )
+
+ state_dict = torch.load(os.path.join(output_dir, "pytorch_model.bin"), weights_only=True)
+ assert set(state_dict) == {"linear.weight", "linear.bias"}
+ assert torch.equal(state_dict["linear.weight"], expected_weight)
+ assert hasattr(model.linear, "orig_layer")
+ with open(os.path.join(output_dir, "config.json")) as config_file:
+ quantization_config = json.load(config_file)["quantization_config"]
+ assert "supported_types" not in quantization_config
+ assert quantization_config["packing_format"] == "auto_round:fake"
+
+
+class _TinyLoadModel(torch.nn.Module):
+ def __init__(self, quantization_config):
+ super().__init__()
+ self.block = torch.nn.Module()
+ self.block.linear = torch.nn.Linear(16, 4)
+ self.lm_head = torch.nn.Linear(16, 4)
+ self.config = SimpleNamespace(quantization_config=quantization_config)
+
+
+def test_fake_config_replaces_linear_and_qdq_activation_on_load():
+ quantization_config = SimpleNamespace(
+ bits=4,
+ group_size=16,
+ sym=True,
+ data_type="nvfp4_v2",
+ act_bits=4,
+ act_group_size=16,
+ act_sym=True,
+ act_data_type="nvfp4_v2",
+ act_dynamic=True,
+ quant_method="auto-round",
+ packing_format="auto_round:fake",
+ block_name_to_quantize="block",
+ backend="auto",
+ extra_config={},
+ modules_to_not_convert=[],
+ )
+ model = _TinyLoadModel(quantization_config)
+ original_weight = model.block.linear.weight.detach().clone()
+ activation = torch.randn(2, 3, 16)
+
+ model, used_backends = convert_hf_model(model, target_device="cpu")
+
+ assert used_backends == ["auto_round:fake"]
+ assert isinstance(model.block.linear, FakeActQuantLinear)
+ assert torch.equal(model.block.linear.weight, original_weight)
+ qdq_activation = model.block.linear.qdq_input(activation)
+ assert not torch.equal(qdq_activation, activation)
+ expected = torch.nn.functional.linear(qdq_activation, model.block.linear.weight, model.block.linear.bias)
+ assert torch.equal(model.block.linear(activation), expected)
+
+
+def test_fake_config_keeps_modules_to_not_convert_in_full_precision():
+ quantization_config = SimpleNamespace(
+ bits=4,
+ group_size=16,
+ sym=True,
+ data_type="nvfp4_v2",
+ act_bits=4,
+ act_group_size=16,
+ act_sym=True,
+ act_data_type="nvfp4_v2",
+ act_dynamic=True,
+ quant_method="auto-round",
+ packing_format="auto_round:fake",
+ block_name_to_quantize="",
+ backend="auto",
+ extra_config={},
+ modules_to_not_convert=["lm_head"],
+ )
+ model = _TinyLoadModel(quantization_config)
+ original_lm_head = model.lm_head
+
+ model, used_backends = convert_hf_model(model, target_device="cpu")
+
+ assert used_backends == ["auto_round:fake"]
+ assert isinstance(model.block.linear, FakeActQuantLinear)
+ assert model.lm_head is original_lm_head
+
+
+def test_transformers_load_replaces_fake_linear(tmp_path):
+ config = OPTConfig(
+ vocab_size=32,
+ hidden_size=16,
+ ffn_dim=32,
+ num_hidden_layers=1,
+ num_attention_heads=2,
+ max_position_embeddings=32,
+ word_embed_proj_dim=16,
+ )
+ config.quantization_config = {
+ "bits": 4,
+ "group_size": 16,
+ "sym": True,
+ "data_type": "nvfp4_v2",
+ "act_bits": 4,
+ "act_group_size": 16,
+ "act_sym": True,
+ "act_data_type": "nvfp4_v2",
+ "act_dynamic": True,
+ "quant_method": "auto-round",
+ "packing_format": "auto_round:fake",
+ "block_name_to_quantize": "model.decoder.layers",
+ }
+ model_dir = str(tmp_path / "fake_opt")
+ OPTForCausalLM(config).save_pretrained(model_dir)
+
+ loaded_model = AutoModelForCausalLM.from_pretrained(model_dir, device_map="cpu")
+
+ q_proj = loaded_model.model.decoder.layers[0].self_attn.q_proj
+ assert isinstance(q_proj, FakeActQuantLinear)
+ activation = torch.randn(1, 2, 16)
+ assert not torch.equal(q_proj.qdq_input(activation), activation)
diff --git a/test/unit/test_cpu/quantization/test_model_free.py b/test/unit/test_cpu/quantization/test_model_free.py
index 5e39a42cf..4e0d8e6e9 100644
--- a/test/unit/test_cpu/quantization/test_model_free.py
+++ b/test/unit/test_cpu/quantization/test_model_free.py
@@ -52,6 +52,13 @@ from safetensors.torch import save_file
from auto_round import AutoRound
from auto_round.compressors.model_free import (
+ _ModelFreeCompressorCore,
+ _process_single_shard_task,
+ get_predefined_ignore_layers_from_config,
+)
+from auto_round.schemes import QuantizationScheme
+from auto_round.utils.model import is_model_free_route
+from auto_round.utils.model_free_utils import (
_build_mxfp_autoround_quantization_config,
_build_mxfp_quantization_config,
_build_quantization_config,
@@ -59,21 +66,23 @@ from auto_round.compressors.model_free import (
_dequant_fp8_tensors,
_dequant_mxfp_tensors,
_expand_e8m0_block_scale,
- _handle_model_type_low_precision_source_tensors,
_handle_mxfp_source_tensors,
_looks_like_auto_scheme,
- _ModelFreeCompressorCore,
_PatternMatcher,
- _preprocess_model_type_source_tensors,
_process_shard,
- _process_single_shard_task,
_quantize_weight_mxfp,
+ _quantize_weight_nvfp4_e5m3,
_validate_auto_scheme_options,
- get_predefined_ignore_layers_from_config,
+)
+from auto_round.utils.model_free_utils import (
+ handle_model_type_low_precision_source_tensors as _handle_model_type_low_precision_source_tensors,
+)
+from auto_round.utils.model_free_utils import (
is_model_free_supported_scheme,
)
-from auto_round.schemes import QuantizationScheme
-from auto_round.utils.model import is_model_free_route
+from auto_round.utils.model_free_utils import (
+ preprocess_model_type_source_tensors as _preprocess_model_type_source_tensors,
+)
def test_model_free_preserves_explicit_scheme_overrides():
@@ -310,7 +319,7 @@ class TestProcessShard:
shard_path = str(tmp_path / "shard.safetensors")
save_file({"layer.fc1.weight": torch.randn(64, 128)}, shard_path)
compile_mock = Mock(side_effect=lambda func, _device: func)
- monkeypatch.setattr("auto_round.compressors.model_free.compile_func", compile_mock)
+ monkeypatch.setattr("auto_round.utils.model_free_utils.compile_func", compile_mock)
_process_shard(shard_path, _DEFAULT_SCHEME, {}, [], enable_torch_compile=enabled)
@@ -323,6 +332,127 @@ class TestProcessShard:
assert "layer.fc1" in quantized
assert "layer.fc1.qweight" in output and "layer.fc1.bias" in output
+
+def test_nvfp4_e5m3_model_free_fake_quantization():
+ weight = torch.randn(8, 32)
+ output = _quantize_weight_nvfp4_e5m3(weight, "layer.fc", group_size=16)
+
+ assert set(output) == {"layer.fc.weight"}
+ assert output["layer.fc.weight"].shape == weight.shape
+ assert output["layer.fc.weight"].dtype == weight.dtype
+ assert not torch.equal(output["layer.fc.weight"], weight)
+ assert is_model_free_supported_scheme("NVFP4_E5M3")
+ assert not is_model_free_supported_scheme("UNVFP4")
+ assert not is_model_free_supported_scheme("NVFP4+")
+
+
+def test_nvfp4_e5m3_model_free_end_to_end(tmp_path):
+ tensors = {
+ "model.layers.0.self_attn.q_proj.weight": torch.randn(32, 32),
+ "lm_head.weight": torch.randn(64, 32),
+ }
+ model_dir = _make_model_dir(tmp_path, _LLAMA_CFG, tensors)
+ output_dir = str(tmp_path / "output")
+ os.makedirs(output_dir)
+ with open(os.path.join(output_dir, "quantization_config.json"), "w") as f:
+ json.dump({"stale": True}, f)
+
+ compressor = _ModelFreeCompressorCore(model_name_or_path=model_dir, output_dir=output_dir, scheme="NVFP4_E5M3")
+ compressor.run()
+
+ output_keys = _read_output_keys(output_dir)
+ assert "model.layers.0.self_attn.q_proj.weight" not in output_keys
+ assert "model.layers.0.self_attn.q_proj.weight_packed" in output_keys
+ assert "model.layers.0.self_attn.q_proj.weight_scale" in output_keys
+ assert "lm_head.weight" in output_keys
+ assert compressor.format == "auto_round"
+ quantization_config = _read_qconfig(output_dir)
+ assert quantization_config["packing_format"] == "auto_round:llm_compressor_nvfp4_e5m3"
+ assert quantization_config["data_type"] == "nvfp4_v2"
+ assert quantization_config["act_bits"] == 4
+ assert quantization_config["act_data_type"] == "nvfp4_v2"
+ assert quantization_config["act_group_size"] == 16
+ assert quantization_config["act_sym"] is True
+ assert quantization_config["extra_config"]["lm_head"] == {
+ "bits": 16,
+ "data_type": "float",
+ "act_bits": 16,
+ "act_data_type": "float",
+ }
+ assert os.path.exists(os.path.join(output_dir, "quantization_config.json"))
+
+
+def test_nvfp4_e5m3_model_free_llm_compressor(tmp_path):
+ tensors = {
+ "model.layers.0.self_attn.q_proj.weight": torch.randn(32, 32),
+ "lm_head.weight": torch.randn(64, 32),
+ }
+ model_dir = _make_model_dir(tmp_path, _LLAMA_CFG, tensors)
+ output_dir = str(tmp_path / "output")
+
+ compressor = _ModelFreeCompressorCore(
+ model_name_or_path=model_dir,
+ output_dir=output_dir,
+ scheme="NVFP4_E5M3",
+ format="llm_compressor",
+ )
+ compressor.run()
+
+ output_keys = _read_output_keys(output_dir)
+ prefix = "model.layers.0.self_attn.q_proj"
+ assert f"{prefix}.weight_packed" in output_keys
+ assert f"{prefix}.weight_scale" in output_keys
+ assert f"{prefix}.weight" not in output_keys
+ assert f"{prefix}.weight_global_scale" not in output_keys
+ assert f"{prefix}.input_global_scale" not in output_keys
+ quantization_config = _read_qconfig(output_dir)
+ group = quantization_config["config_groups"]["group_0"]
+ assert quantization_config["format"] == "nvfp4-e5m3-pack-quantized"
+ assert quantization_config["quant_method"] == "compressed-tensors"
+ assert quantization_config["provider"] == "auto-round"
+ assert group["weights"]["group_size"] == 16
+ assert group["input_activations"]["dynamic"] == "local"
+
+
+def test_model_free_legacy_nvfp4_is_normalized_and_passthrough(tmp_path):
+ prefix = "model.layers.0.mlp.down_proj"
+ tensors = {
+ f"{prefix}.weight": torch.randint(0, 256, (32, 64), dtype=torch.uint8),
+ f"{prefix}.weight_scale": torch.randint(0, 256, (32, 4), dtype=torch.uint8),
+ f"{prefix}.weight_scale_2": torch.tensor([2.0], dtype=torch.float32),
+ f"{prefix}.input_scale": torch.tensor([4.0], dtype=torch.float32),
+ "model.layers.0.self_attn.q_proj.weight": torch.randn(32, 32),
+ }
+ shard_path = str(tmp_path / "shard.safetensors")
+ save_file(tensors, shard_path)
+
+ layer_config = {
+ prefix: {
+ "bits": 4,
+ "group_size": 16,
+ "sym": True,
+ "data_type": "nv_fp",
+ }
+ }
+ output, quantized, ignored = _process_shard(shard_path, _DEFAULT_SCHEME, layer_config, [])
+
+ # Legacy naming should be normalized to llm-compressor-style global-scale keys.
+ assert f"{prefix}.weight_packed" in output
+ assert f"{prefix}.weight_scale" in output
+ assert f"{prefix}.weight_global_scale" in output
+ assert f"{prefix}.input_global_scale" in output
+ assert f"{prefix}.weight" not in output
+ assert f"{prefix}.weight_scale_2" not in output
+ assert f"{prefix}.input_scale" not in output
+ assert torch.allclose(output[f"{prefix}.weight_global_scale"], torch.tensor([0.5], dtype=torch.float32))
+ assert torch.allclose(output[f"{prefix}.input_global_scale"], torch.tensor([0.25], dtype=torch.float32))
+
+ # The NVFP4 layer is treated as already quantized (passthrough) while
+ # other Linear layers in the shard are still quantized by model-free RTN.
+ assert prefix in quantized
+ assert "model.layers.0.self_attn.q_proj" in quantized
+ assert prefix not in ignored
+
def test_ignores_and_skips(self, tmp_path):
shard_path = str(tmp_path / "shard.safetensors")
save_file(
@@ -412,24 +542,40 @@ class TestFP8Source:
output, quantized, _ = _process_shard(shard_path, _DEFAULT_SCHEME, {}, [], device="cpu", fp8_block_size=None)
assert "layer" in quantized and "layer.qweight" in output
- def test_ignored_layer_preserves_original_fp8(self, tmp_path):
- """Ignored layers keep their original quantized tensors (no dequant)."""
- shard_path = str(tmp_path / "shard.safetensors")
- w_fp8 = torch.randn(64, 128, dtype=torch.bfloat16).to(torch.float8_e4m3fn)
- scale = torch.tensor(0.5)
- save_file(
- {"lm_head.weight": w_fp8, "lm_head.weight_scale_inv": scale, "layer.weight": torch.randn(64, 128)},
- shard_path,
- )
- output, quantized, ignored = _process_shard(
- shard_path, _DEFAULT_SCHEME, {}, ["lm_head"], device="cpu", fp8_block_size=None
+ def test_dequant_fp8_hydrates_scale_from_sibling_shard(self, tmp_path):
+ """When scale_inv is sharded separately, dequant should hydrate it via index."""
+ shard_dir = tmp_path / "source"
+ shard_dir.mkdir(parents=True, exist_ok=True)
+
+ weight_name = "model.layers.0.mlp.experts.1.gate_proj.weight"
+ scale_name = "model.layers.0.mlp.experts.1.gate_proj.weight_scale_inv"
+
+ shard_a = shard_dir / "model-00001-of-00002.safetensors"
+ shard_b = shard_dir / "model-00002-of-00002.safetensors"
+ save_file({weight_name: torch.randn(2048, 7168, dtype=torch.bfloat16).to(torch.float8_e4m3fn)}, str(shard_a))
+ save_file({scale_name: torch.ones(16, 56, dtype=torch.float32)}, str(shard_b))
+
+ with open(shard_dir / "model.safetensors.index.json", "w") as f:
+ json.dump(
+ {
+ "metadata": {"total_size": 0},
+ "weight_map": {
+ weight_name: shard_a.name,
+ scale_name: shard_b.name,
+ },
+ },
+ f,
+ )
+
+ output, quantized, _ = _process_shard(
+ str(shard_a),
+ _DEFAULT_SCHEME,
+ {},
+ [],
+ fp8_block_size=[128, 128],
)
- # lm_head should be ignored and kept in original FP8 format
- assert "lm_head" in ignored
- assert output["lm_head.weight"].dtype == torch.float8_e4m3fn
- assert "lm_head.weight_scale_inv" in output
- # non-ignored layer should be quantized normally
- assert "layer" in quantized
+ assert "model.layers.0.mlp.experts.1.gate_proj" in quantized
+ assert "model.layers.0.mlp.experts.1.gate_proj.qweight" in output
# ===========================================================================
diff --git a/test/unit/test_cpu/quantization/test_nvfp4_quant_linear.py b/test/unit/test_cpu/quantization/test_nvfp4_quant_linear.py
index 860ef0ef0..82f403330 100644
--- a/test/unit/test_cpu/quantization/test_nvfp4_quant_linear.py
+++ b/test/unit/test_cpu/quantization/test_nvfp4_quant_linear.py
@@ -1,10 +1,15 @@
+from types import SimpleNamespace
+from unittest.mock import patch
+
import pytest
import torch
+from transformers.quantizers.auto import AutoHfQuantizer
-from auto_round.data_type.nvfp import calculate_gparam
+from auto_round.data_type.nvfp import calculate_gparam, nvfp4_v2
from auto_round.data_type.utils import get_quant_func
from auto_round.experimental import qmodules as ar_qmodules
from auto_round.export.export_to_autoround.qlinear_fp import QuantLinear as _FPLinear
+from auto_round.export.export_to_llmcompressor.config import initialize_nvfp4_e5m3_quantization
from auto_round.export.formats import BackendDataType
from auto_round.schemes import PRESET_SCHEMES
@@ -35,6 +40,137 @@ def test_calculate_gparam_with_float8_input():
assert torch.isfinite(global_scale)
+def test_nvfp4_e5m3_compressed_tensors_loading_uses_no_global_scales():
+ class TinyModel(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.config = SimpleNamespace(model_type="tiny")
+ self.model = torch.nn.Module()
+ self.model.layers = torch.nn.ModuleList([torch.nn.ModuleDict({"fc": torch.nn.Linear(16, 8)})])
+
+ quantizer = AutoHfQuantizer.from_config(initialize_nvfp4_e5m3_quantization([]))
+ model = quantizer._process_model_before_weight_loading(TinyModel())
+ layer = model.model.layers[0]["fc"]
+
+ assert isinstance(layer, ar_qmodules.NVFP4E5M3QuantLinear)
+ assert set(layer.state_dict()) == {"bias", "weight_packed", "weight_scale"}
+ assert quantizer._process_model_after_weight_loading(model) is model
+
+
+def test_nvfp4_e5m3_qdq_input_uses_reference_fallback_on_cpu():
+ config = PRESET_SCHEMES["NVFP4_E5M3"]
+ layer = ar_qmodules.NVFP4E5M3QuantLinear(16, 8, config, dtype=torch.float32)
+ activation = torch.randn(2, 3, 16)
+
+ expected, _, _ = nvfp4_v2(activation, bits=config.act_bits, group_size=config.act_group_size)
+
+ assert torch.equal(layer.qdq_input(activation), expected)
+
+
+def test_nvfp4_e5m3_forward_does_not_cache_dequantized_weight_by_default():
+ config = PRESET_SCHEMES["NVFP4_E5M3"]
+ layer = ar_qmodules.NVFP4E5M3QuantLinear(16, 8, config, dtype=torch.float32)
+ activation = torch.randn(2, 16)
+ dequantized_weight = torch.randn(8, 16)
+
+ with patch.object(layer, "dequant_weight_online", return_value=dequantized_weight) as dequant_weight_online:
+ layer(activation)
+ layer(activation)
+
+ assert layer._cached_weight is None
+ assert dequant_weight_online.call_count == 2
+
+
+def test_nvfp4_e5m3_forward_caches_dequantized_weight_when_enabled():
+ config = PRESET_SCHEMES["NVFP4_E5M3"]
+ layer = ar_qmodules.NVFP4E5M3QuantLinear(16, 8, config, dtype=torch.float32, cache_weight=True)
+ activation = torch.randn(2, 16)
+ dequantized_weight = torch.randn(8, 16)
+
+ with patch.object(layer, "dequant_weight_online", return_value=dequantized_weight) as dequant_weight_online:
+ layer(activation)
+ layer(activation)
+
+ dequant_weight_online.assert_called_once_with()
+ assert layer.weight_packed is None
+ assert layer.weight_scale is None
+
+
+def test_nvfp4_e5m3_cannot_clear_released_quantized_weight_buffers():
+ config = PRESET_SCHEMES["NVFP4_E5M3"]
+ layer = ar_qmodules.NVFP4E5M3QuantLinear(16, 8, config, dtype=torch.float32, cache_weight=True)
+
+ layer(torch.randn(2, 16))
+
+ with pytest.raises(RuntimeError, match="quantized weight buffers have been released"):
+ layer.clear_weight_cache()
+
+
+def test_cute_nvfp4_e5m3_does_not_cache_dequantized_weight_by_default(monkeypatch):
+ monkeypatch.delenv("AR_NVFP4_E5M3_CACHE_HP_WEIGHT", raising=False)
+ config = PRESET_SCHEMES["NVFP4_E5M3"]
+ layer = ar_qmodules.CuteNVFP4E5M3QuantLinear(16, 8, config, dtype=torch.float32)
+ activation = torch.randn(2, 16)
+ dequantized_weight = torch.randn(8, 16)
+
+ with patch.object(layer, "dequant_weight_online", return_value=dequantized_weight) as dequant_weight_online:
+ layer(activation)
+ layer(activation)
+
+ assert not layer.cache_weight
+ assert dequant_weight_online.call_count == 2
+
+
+def test_nvfp4_e5m3_cache_weight_environment_override(monkeypatch):
+ config = PRESET_SCHEMES["NVFP4_E5M3"]
+ monkeypatch.setenv("AR_NVFP4_E5M3_CACHE_HP_WEIGHT", "1")
+ assert ar_qmodules.NVFP4E5M3QuantLinear(16, 8, config, dtype=torch.float32).cache_weight
+ assert ar_qmodules.CuteNVFP4E5M3QuantLinear(16, 8, config, dtype=torch.float32).cache_weight
+ monkeypatch.setenv("AR_NVFP4_E5M3_CACHE_HP_WEIGHT", "0")
+ assert not ar_qmodules.CuteNVFP4E5M3QuantLinear(16, 8, config, dtype=torch.float32).cache_weight
+
+
+def test_nvfp4_e5m3_torch_forward_does_not_call_cute():
+ config = PRESET_SCHEMES["NVFP4_E5M3"]
+ layer = ar_qmodules.NVFP4E5M3QuantLinear(16, 8, config, dtype=torch.float32)
+ activation = torch.randn(2, 16)
+
+ with patch(
+ "auto_round.experimental.qmodules.nvfp4_e5m3.try_cute_nvfp4_e5m3_linear",
+ ) as cute_linear:
+ output = layer(activation)
+
+ assert output.shape == (2, 8)
+ cute_linear.assert_not_called()
+
+
+def test_nvfp4_e5m3_cute_forward_uses_fused_output_when_available():
+ config = PRESET_SCHEMES["NVFP4_E5M3"]
+ layer = ar_qmodules.CuteNVFP4E5M3QuantLinear(16, 8, config, dtype=torch.float32)
+ activation = torch.randn(2, 16)
+ fused_output = torch.randn(2, 8)
+
+ with patch("auto_round.experimental.qmodules.nvfp4_e5m3.try_cute_nvfp4_e5m3_linear", return_value=fused_output):
+ assert layer(activation) is fused_output
+
+
+def test_cute_nvfp4_e5m3_uses_cached_weight_when_enabled():
+ config = PRESET_SCHEMES["NVFP4_E5M3"]
+ layer = ar_qmodules.CuteNVFP4E5M3QuantLinear(16, 8, config, dtype=torch.float32, cache_weight=True)
+ activation = torch.randn(2, 16)
+ dequantized_weight = torch.randn(8, 16)
+
+ with (
+ patch.object(layer, "dequant_weight_online", return_value=dequantized_weight) as dequant_weight_online,
+ patch("auto_round.experimental.qmodules.nvfp4_e5m3.try_cute_nvfp4_e5m3_linear") as cute_linear,
+ ):
+ layer(activation)
+ layer(activation)
+
+ dequant_weight_online.assert_called_once_with()
+ cute_linear.assert_not_called()
+
+
@pytest.mark.parametrize("scheme", [BackendDataType.NVFP4.value])
@torch.inference_mode()
def test_nvfp4_quantlinear_from_original_and_forward(scheme):
diff --git a/test/unit/test_cpu/schemes/test_auto_scheme.py b/test/unit/test_cpu/schemes/test_auto_scheme.py
index e87093bde..43b90815d 100644
--- a/test/unit/test_cpu/schemes/test_auto_scheme.py
+++ b/test/unit/test_cpu/schemes/test_auto_scheme.py
@@ -280,7 +280,7 @@ class TestAutoScheme:
def teardown_class(self):
shutil.rmtree("runs", ignore_errors=True)
- @pytest.mark.timeout(60)
+ @pytest.mark.timeout(70)
def test_auto_scheme_export(self, tiny_opt_model_path):
model_name = tiny_opt_model_path
int_save_dir = os.path.join(self.save_dir, "int")
diff --git a/test/unit/test_cpu/utils/test_model_free_cross_shard.py b/test/unit/test_cpu/utils/test_model_free_cross_shard.py
new file mode 100644
index 000000000..e2f0c4700
--- /dev/null
+++ b/test/unit/test_cpu/utils/test_model_free_cross_shard.py
@@ -0,0 +1,270 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Unit tests for cross-shard FP8 scale handling in model_free_utils."""
+
+import json
+import os
+import tempfile
+
+import pytest
+import torch
+
+from auto_round.utils.model_free_utils import (
+ _build_cross_shard_pairs_from_weight_map,
+ _hydrate_missing_fp8_scales_from_index,
+)
+
+# ---------------------------------------------------------------------------
+# _build_cross_shard_pairs_from_weight_map
+# ---------------------------------------------------------------------------
+
+
+class TestBuildCrossShardPairs:
+ """Tests for _build_cross_shard_pairs_from_weight_map."""
+
+ def _make_weight_map(self, entries: dict[str, str]) -> dict[str, str]:
+ return dict(entries)
+
+ def test_no_fp8_entries_returns_empty(self):
+ weight_map = {
+ "model.layer.weight": "shard-00001.safetensors",
+ "model.layer.bias": "shard-00001.safetensors",
+ }
+ recipient_to_donors, donor_shard_tensors = _build_cross_shard_pairs_from_weight_map(weight_map)
+ assert recipient_to_donors == {}
+ assert donor_shard_tensors == {}
+
+ def test_same_shard_scale_not_cross(self):
+ """weight and weight_scale_inv in the same shard → not a cross-shard pair."""
+ weight_map = {
+ "model.layer.weight": "shard-00001.safetensors",
+ "model.layer.weight_scale_inv": "shard-00001.safetensors",
+ }
+ recipient_to_donors, donor_shard_tensors = _build_cross_shard_pairs_from_weight_map(weight_map)
+ assert recipient_to_donors == {}
+ assert donor_shard_tensors == {}
+
+ def test_cross_shard_single_pair(self):
+ """weight in shard-1, weight_scale_inv in shard-2."""
+ weight_map = {
+ "model.layer.weight": "shard-00001.safetensors",
+ "model.layer.weight_scale_inv": "shard-00002.safetensors",
+ }
+ recipient_to_donors, donor_shard_tensors = _build_cross_shard_pairs_from_weight_map(weight_map)
+
+ assert "shard-00001.safetensors" in recipient_to_donors
+ donor_map = recipient_to_donors["shard-00001.safetensors"]
+ assert "shard-00002.safetensors" in donor_map
+ assert "model.layer.weight_scale_inv" in donor_map["shard-00002.safetensors"]
+
+ assert "shard-00002.safetensors" in donor_shard_tensors
+ assert "model.layer.weight_scale_inv" in donor_shard_tensors["shard-00002.safetensors"]
+
+ def test_cross_shard_multiple_layers_one_donor(self):
+ """Multiple layers whose scale_inv all live in the same donor shard."""
+ weight_map = {
+ "model.layer0.weight": "shard-00001.safetensors",
+ "model.layer0.weight_scale_inv": "shard-00002.safetensors",
+ "model.layer1.weight": "shard-00001.safetensors",
+ "model.layer1.weight_scale_inv": "shard-00002.safetensors",
+ }
+ recipient_to_donors, donor_shard_tensors = _build_cross_shard_pairs_from_weight_map(weight_map)
+
+ donor_map = recipient_to_donors["shard-00001.safetensors"]
+ scales = donor_map["shard-00002.safetensors"]
+ assert "model.layer0.weight_scale_inv" in scales
+ assert "model.layer1.weight_scale_inv" in scales
+ assert len(donor_shard_tensors["shard-00002.safetensors"]) == 2
+
+ def test_cross_shard_multiple_donors(self):
+ """Recipient shard needs scales from two different donor shards."""
+ weight_map = {
+ "model.layerA.weight": "shard-00001.safetensors",
+ "model.layerA.weight_scale_inv": "shard-00002.safetensors",
+ "model.layerB.weight": "shard-00001.safetensors",
+ "model.layerB.weight_scale_inv": "shard-00003.safetensors",
+ }
+ recipient_to_donors, donor_shard_tensors = _build_cross_shard_pairs_from_weight_map(weight_map)
+
+ donor_map = recipient_to_donors["shard-00001.safetensors"]
+ assert "shard-00002.safetensors" in donor_map
+ assert "shard-00003.safetensors" in donor_map
+ assert len(donor_shard_tensors) == 2
+
+ def test_scale_inv_without_matching_weight_ignored(self):
+ """weight_scale_inv present in weight_map but no corresponding .weight → ignored."""
+ weight_map = {
+ "model.layer.weight_scale_inv": "shard-00002.safetensors",
+ # no "model.layer.weight" key at all
+ }
+ recipient_to_donors, donor_shard_tensors = _build_cross_shard_pairs_from_weight_map(weight_map)
+ assert recipient_to_donors == {}
+ assert donor_shard_tensors == {}
+
+ def test_empty_weight_map(self):
+ recipient_to_donors, donor_shard_tensors = _build_cross_shard_pairs_from_weight_map({})
+ assert recipient_to_donors == {}
+ assert donor_shard_tensors == {}
+
+
+# ---------------------------------------------------------------------------
+# _hydrate_missing_fp8_scales_from_index
+# ---------------------------------------------------------------------------
+
+
+def _write_fake_fp8_shard(path: str, tensors: dict[str, torch.Tensor]) -> None:
+ """Save a dict of tensors as a safetensors file."""
+ from safetensors.torch import save_file
+
+ save_file(tensors, path)
+
+
+def _write_index_json(directory: str, weight_map: dict[str, str]) -> str:
+ index_path = os.path.join(directory, "model.safetensors.index.json")
+ with open(index_path, "w") as f:
+ json.dump({"weight_map": weight_map}, f)
+ return index_path
+
+
+class TestHydrateMissingFp8Scales:
+ """Tests for _hydrate_missing_fp8_scales_from_index."""
+
+ def test_non_safetensors_shard_returns_unchanged(self, tmp_path):
+ raw = {"w": torch.zeros(4)}
+ result = _hydrate_missing_fp8_scales_from_index(raw, str(tmp_path / "model.bin"))
+ assert result is raw
+
+ def test_no_fp8_weights_returns_unchanged(self, tmp_path):
+ shard_path = str(tmp_path / "shard-00001.safetensors")
+ # BF16 weight — not FP8, so nothing to hydrate
+ raw = {"model.layer.weight": torch.zeros(4, dtype=torch.bfloat16)}
+ result = _hydrate_missing_fp8_scales_from_index(raw, shard_path)
+ assert result is raw
+
+ def test_all_scales_present_returns_unchanged(self, tmp_path):
+ shard_path = str(tmp_path / "shard-00001.safetensors")
+ raw = {
+ "model.layer.weight": torch.zeros(4, dtype=torch.float8_e4m3fn),
+ "model.layer.weight_scale_inv": torch.ones(1),
+ }
+ result = _hydrate_missing_fp8_scales_from_index(raw, shard_path)
+ assert "model.layer.weight_scale_inv" in result
+
+ def test_cross_shard_hydration_local_mode(self, tmp_path):
+ """Recipient shard gets scale_inv from donor shard in local (non-streaming) mode."""
+ donor_name = "shard-00002.safetensors"
+ recipient_name = "shard-00001.safetensors"
+ scale_name = "model.layer.weight_scale_inv"
+
+ # Write donor shard with scale_inv
+ donor_path = tmp_path / donor_name
+ _write_fake_fp8_shard(str(donor_path), {scale_name: torch.ones(1)})
+
+ # Write index.json in the same directory
+ weight_map = {
+ "model.layer.weight": recipient_name,
+ scale_name: donor_name,
+ }
+ _write_index_json(str(tmp_path), weight_map)
+
+ # Recipient shard has FP8 weight but no scale_inv
+ recipient_path = tmp_path / recipient_name
+ raw = {"model.layer.weight": torch.zeros(4, dtype=torch.float8_e4m3fn)}
+
+ result = _hydrate_missing_fp8_scales_from_index(raw, str(recipient_path))
+ assert scale_name in result, "scale_inv should be hydrated from donor shard"
+ assert result[scale_name].dtype == torch.float32 or result[scale_name].numel() == 1
+
+ def test_cross_shard_hydration_streaming_mode(self, tmp_path):
+ """In streaming mode, index.json lives in work_dir, shards in cache subdir."""
+ work_dir = tmp_path / "work_dir"
+ cache_dir = work_dir / ".cache" / "model_free_source_shards"
+ work_dir.mkdir()
+ cache_dir.mkdir(parents=True)
+
+ donor_name = "shard-00002.safetensors"
+ recipient_name = "shard-00001.safetensors"
+ scale_name = "model.layer.weight_scale_inv"
+
+ # Donor shard lives in cache subdir
+ _write_fake_fp8_shard(str(cache_dir / donor_name), {scale_name: torch.ones(1)})
+
+ # Index.json lives in work_dir (not in cache subdir)
+ weight_map = {
+ "model.layer.weight": recipient_name,
+ scale_name: donor_name,
+ }
+ _write_index_json(str(work_dir), weight_map)
+
+ # Recipient shard path is in cache dir
+ recipient_path = cache_dir / recipient_name
+ raw = {"model.layer.weight": torch.zeros(4, dtype=torch.float8_e4m3fn)}
+
+ result = _hydrate_missing_fp8_scales_from_index(
+ raw,
+ str(recipient_path),
+ index_dir=str(work_dir),
+ donor_shard_dir=str(cache_dir),
+ )
+ assert scale_name in result, "streaming mode: scale_inv should be hydrated via index_dir/donor_shard_dir params"
+
+ def test_missing_index_json_returns_unchanged(self, tmp_path):
+ """If no index.json exists, raw_tensors is returned as-is (no crash)."""
+ shard_path = str(tmp_path / "shard-00001.safetensors")
+ raw = {"model.layer.weight": torch.zeros(4, dtype=torch.float8_e4m3fn)}
+ result = _hydrate_missing_fp8_scales_from_index(raw, shard_path)
+ assert result is raw
+
+ def test_donor_shard_missing_on_disk_returns_unchanged(self, tmp_path):
+ """Index references a donor shard that doesn't exist on disk → graceful skip."""
+ scale_name = "model.layer.weight_scale_inv"
+ recipient_name = "shard-00001.safetensors"
+ donor_name = "shard-00002.safetensors" # not written
+
+ weight_map = {
+ "model.layer.weight": recipient_name,
+ scale_name: donor_name,
+ }
+ _write_index_json(str(tmp_path), weight_map)
+
+ raw = {"model.layer.weight": torch.zeros(4, dtype=torch.float8_e4m3fn)}
+ result = _hydrate_missing_fp8_scales_from_index(raw, str(tmp_path / recipient_name))
+ assert scale_name not in result # hydration skipped silently
+
+ def test_multiple_layers_hydrated_from_single_donor(self, tmp_path):
+ """Multiple missing scale_inv tensors are hydrated in a single donor open."""
+ donor_name = "shard-00002.safetensors"
+ recipient_name = "shard-00001.safetensors"
+ scales = {
+ "model.layerA.weight_scale_inv": torch.ones(1),
+ "model.layerB.weight_scale_inv": torch.ones(1),
+ }
+ _write_fake_fp8_shard(str(tmp_path / donor_name), scales)
+
+ weight_map = {
+ "model.layerA.weight": recipient_name,
+ "model.layerA.weight_scale_inv": donor_name,
+ "model.layerB.weight": recipient_name,
+ "model.layerB.weight_scale_inv": donor_name,
+ }
+ _write_index_json(str(tmp_path), weight_map)
+
+ raw = {
+ "model.layerA.weight": torch.zeros(4, dtype=torch.float8_e4m3fn),
+ "model.layerB.weight": torch.zeros(4, dtype=torch.float8_e4m3fn),
+ }
+ result = _hydrate_missing_fp8_scales_from_index(raw, str(tmp_path / recipient_name))
+ assert "model.layerA.weight_scale_inv" in result
+ assert "model.layerB.weight_scale_inv" in result
diff --git a/test/unit/test_cuda/quantization/test_nvfp4_e5m3_cute.py b/test/unit/test_cuda/quantization/test_nvfp4_e5m3_cute.py
new file mode 100644
index 000000000..e6a9e2b89
--- /dev/null
+++ b/test/unit/test_cuda/quantization/test_nvfp4_e5m3_cute.py
@@ -0,0 +1,66 @@
+import pytest
+import torch
+
+from auto_round.data_type.nvfp import cast_to_fp4, e5m3_to_float_tensor, nvfp4_v2
+from auto_round.experimental.qmodules.fp4_utils import unpack_fp4_from_uint8
+from auto_round_extension.cuda.cute_nvfp4_e5m3 import (
+ can_use_cute_nvfp4_v2_qdq,
+ try_cute_nvfp4_e5m3_weight_dq,
+ try_cute_nvfp4_v2_qdq,
+)
+
+
[email protected](not torch.cuda.is_available(), reason="CUDA is required")
+ ("dtype", "atol"),
+ [(torch.float32, 1e-6), (torch.float16, 2e-3), (torch.bfloat16, 2e-2)],
+)
+def test_cute_nvfp4_v2_qdq_matches_reference(dtype, atol):
+ if torch.cuda.get_device_capability()[0] < 8:
+ pytest.skip("CuTe QDQ requires SM80 or newer")
+
+ activation = torch.randn(32, 16, device="cuda", dtype=dtype)
+ if not can_use_cute_nvfp4_v2_qdq(activation, 16):
+ pytest.skip("CuTe DSL is not available")
+
+ expected, _, _ = nvfp4_v2(activation.float(), bits=4, group_size=16)
+ actual = try_cute_nvfp4_v2_qdq(activation, 16)
+
+ assert actual is not None
+ torch.testing.assert_close(actual, expected.to(dtype), rtol=0, atol=atol)
+
+
[email protected](not torch.cuda.is_available(), reason="CUDA is required")
[email protected]("dtype", [torch.float32, torch.float16, torch.bfloat16])
+def test_cute_nvfp4_e5m3_weight_dq_matches_reference_for_multiple_groups(dtype):
+ if torch.cuda.get_device_capability()[0] < 8:
+ pytest.skip("CuTe weight DQ requires SM80 or newer")
+
+ torch.manual_seed(42)
+ weight_packed = torch.randint(0, 256, (37, 64), device="cuda", dtype=torch.uint8)
+ weight_scale = torch.randint(0, 120, (37, 8), device="cuda", dtype=torch.uint8)
+ actual = try_cute_nvfp4_e5m3_weight_dq(weight_packed, weight_scale, dtype)
+ if actual is None:
+ pytest.skip("CuTe DSL is not available")
+
+ unpacked = unpack_fp4_from_uint8(weight_packed, 37, 128, dtype=dtype).float()
+ expected = (unpacked.reshape(-1, 16) * e5m3_to_float_tensor(weight_scale).reshape(-1, 1)).reshape(37, 128).to(dtype)
+ torch.testing.assert_close(actual, expected, rtol=0, atol=0)
+
+
+def test_nvfp4_v2_uses_direct_scale_at_rounding_boundary():
+ activation = torch.tensor([[1.25] + [1.0] * 15], dtype=torch.float32)
+
+ actual, scale, _ = nvfp4_v2(activation, bits=4, group_size=16)
+ expected = cast_to_fp4(activation / scale) * scale
+
+ torch.testing.assert_close(actual, expected)
+
+
+def test_nvfp4_v2_direct_scale_handles_zero_group():
+ activation = torch.zeros(1, 16, dtype=torch.float32)
+
+ actual, scale, _ = nvfp4_v2(activation, bits=4, group_size=16)
+
+ assert torch.equal(actual, activation)
+ assert torch.equal(scale, torch.zeros_like(scale))