PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
[Experimental] Add NVFP4 E5M3 support and related components for model-free quantization (#2123)
Xin He0c0511b547a7
diff --git a/auto_round/compressors/model_free.py b/auto_round/compressors/model_free.pyold mode 100755new mode 100644index f44833419..531f78556--- a/auto_round/compressors/model_free.py+++ b/auto_round/compressors/model_free.py@@ -40,7 +40,12 @@ Model-free mode supports the following quantization families:* Preset names: ``MXFP4``, ``MXFP8``.* ``data_type="mx_fp"``, ``group_size=32``, ``bits in {4, 8}``.-Schemes that require special packing (FP8, NVFP4, GGUF, INT8_W8A8,+**NVFP4 E5M3** (``auto_round`` or ``fake`` format):++* Preset name: ``NVFP4_E5M3``.+* ``data_type="nvfp4_v2"``, ``group_size=16``, with high-precision QDQ weights.++Schemes that require special packing (FP8, standard NVFP4, GGUF, INT8_W8A8,BF16, FPW8A16, ...) are **not** supported in model-free mode and will raise``ValueError``. Use the standard AutoRound flow for those.@@ -49,6 +54,9 @@ Output formats* **INT schemes** → ``auto_round:auto_gptq`` packing format, ``quant_method="auto-round"``.* **MXFP schemes** → ``mxfp4-pack-quantized`` or ``mxfp8-quantized`` format,``quant_method="compressed-tensors"``, compatible with vLLM / llm-compressor.+* **NVFP4_E5M3** → AutoRound format with packed ``.weight_packed`` and+ ``.weight_scale`` tensors; use ``format="fake"`` explicitly for high-precision+ QDQ ``.weight`` tensors.Usage (CLI)-----------@@ -97,29 +105,58 @@ import reimport shutilimport sysimport time-from concurrent.futures import ProcessPoolExecutor, as_completed+from concurrent.futures import FIRST_COMPLETED, ProcessPoolExecutor, ThreadPoolExecutor, as_completed, waitfrom dataclasses import asdict, fields-from typing import Any, Callable, Optional, Union+from typing import Any, Optional, Unionimport torchfrom auto_round import envsfrom auto_round.compressors.config_resolution import thaw_mapping-from auto_round.compressors.utils import is_mx_fp+from auto_round.compressors.utils import is_mx_fp, is_nv_fpfrom auto_round.logger import logger-from auto_round.schemes import PRESET_SCHEMES, QuantizationScheme, preset_name_to_scheme-from auto_round.utils.common import AUDIO_MM_KEYS, VISION_MM_KEYS, compress_layer_names, to_standard_regex-from auto_round.utils.device import clear_memory, compile_func, memory_monitor+from auto_round.schemes import QuantizationScheme, preset_name_to_scheme+from auto_round.utils.common import AUDIO_MM_KEYS, VISION_MM_KEYS, compress_layer_names+from auto_round.utils.device import clear_memory, memory_monitorfrom auto_round.utils.device_manager import default_enable_torch_compile-from auto_round.utils.missing_tensors import quantize_weight_rtn, split_fused_expert_tensors+from auto_round.utils.model_free_utils import (+ _apply_scheme_overrides,+ _build_cross_shard_pairs_from_weight_map,+ _build_quantization_config,+ _convert_auto_scheme_layer_config,+ _download_metadata_files,+ _download_single_shard,+ _get_model_cache_status,+ _is_full_precision_default,+ _is_weight_shard,+ _layer_config_has_mxfp,+ _layer_config_has_nvfp4,+ _list_weight_shards,+ _load_config,+ _load_weight_map_from_index,+ _looks_like_auto_scheme,+ _normalize_scheme,+ _PatternMatcher,+ _process_shard,+ _resolve_source_dir,+ _validate_auto_scheme_options,+ _validate_supported_scheme,+ _write_index_file,+ _write_output_shard,+ handle_model_type_low_precision_source_tensors,+ is_model_free_supported_scheme,+ preprocess_model_type_source_tensors,+)++# Backward-compat aliases for internal/private helper names used in tests and+# downstream imports.+_preprocess_model_type_source_tensors = preprocess_model_type_source_tensors+_handle_model_type_low_precision_source_tensors = handle_model_type_low_precision_source_tensors# ---------------------------------------------------------------------------# Constants# ----------------------------------------------------------------------------# add "embed", "conv" in case of auto detection failure in _check_conv1d_and_embedding-_BLOCK_NAME_TO_IGNORE = ["shared_expert_gate.", ".gate.", "embed", "conv"]-# Preset schemes that model-free mode can produce.# INT presets use ``auto_round:auto_gptq`` packing; MXFP presets use# ``mxfp4-pack-quantized`` or ``mxfp8-quantized`` (compressed-tensors) packing.@@ -138,6 +175,7 @@ SUPPORTED_PRESET_SCHEMES: tuple[str, ...] = ("W8A16","MXFP4","MXFP8",+ "NVFP4_E5M3","BF16",)@@ -148,6 +186,8 @@ _SUPPORTED_INT_BITS: tuple[int, ...] = (2, 4, 8)# Allowed ``bits`` values for MXFP weight quantization._SUPPORTED_MXFP_BITS: tuple[int, ...] = (4, 8)+_NVFP4_E5M3_DATA_TYPE = "nvfp4_v2"+# Multimodal keywords kept in full precision by default._NONTEXT_KEYWORDS: tuple[str, ...] = VISION_MM_KEYS + AUDIO_MM_KEYS@@ -195,1609 +235,14 @@ def get_predefined_ignore_layers_from_config(config: dict) -> list[str]:# ----------------------------------------------------------------------------def _is_model_cached(model_name_or_path: str) -> bool:- """Return True if the model is already available locally or in HF cache."""- if os.path.isdir(model_name_or_path):- return True- try:- from huggingface_hub import try_to_load_from_cache-- result = try_to_load_from_cache(model_name_or_path, "config.json")- return isinstance(result, str)- except Exception:- return False---def _resolve_source_dir(model_name_or_path: str) -> str:- """Resolve model_name_or_path to a local directory (download if needed)."""- if os.path.isdir(model_name_or_path):- return model_name_or_path- from huggingface_hub import snapshot_download-- return snapshot_download(model_name_or_path)---def _load_config(source_dir: str) -> dict:- """Load config.json from model directory."""- config_path = os.path.join(source_dir, "config.json")- if not os.path.exists(config_path):- raise FileNotFoundError(f"config.json not found in {source_dir}")- with open(config_path) as f:- return json.load(f)---def _list_weight_shards(source_dir: str) -> list[str]:- """Return list of weight shard filenames in order.-- Safetensors shards are preferred. When no safetensors files are- found the function falls back to PyTorch ``.bin`` shards.-- Handles both standard naming (``model.safetensors``,- ``pytorch_model.bin``) and custom prefixes such as- ``diffusion_pytorch_model-XXXXX-of-XXXXX.safetensors`` by scanning- all ``*.safetensors.index.json`` / ``*.bin.index.json`` index files- in the directory when no standard index is found.- """-- def _shards_from_index(index_path: str) -> list[str]:- with open(index_path) as f:- index = json.load(f)- seen: set[str] = set()- shards: list[str] = []- for shard_file in index["weight_map"].values():- if shard_file not in seen:- seen.add(shard_file)- shards.append(shard_file)- return shards-- # --- safetensors: standard index ---- st_index = os.path.join(source_dir, "model.safetensors.index.json")- if os.path.exists(st_index):- return _shards_from_index(st_index)-- # --- safetensors: custom-prefix index (e.g. diffusion_pytorch_model.safetensors.index.json) ---- for fname in sorted(os.listdir(source_dir)):- if fname.endswith(".safetensors.index.json"):- return _shards_from_index(os.path.join(source_dir, fname))-- # --- safetensors: single file or index-less multi-file shards ---- st_files = sorted(f for f in os.listdir(source_dir) if f.endswith(".safetensors"))- if len(st_files) >= 1:- return st_files-- # --- pytorch .bin: standard index ---- bin_index = os.path.join(source_dir, "pytorch_model.bin.index.json")- if os.path.exists(bin_index):- return _shards_from_index(bin_index)-- # --- pytorch .bin: custom-prefix index ---- for fname in sorted(os.listdir(source_dir)):- if fname.endswith(".bin.index.json"):- return _shards_from_index(os.path.join(source_dir, fname))-- # --- pytorch .bin: single file ---- bin_single = os.path.join(source_dir, "pytorch_model.bin")- if os.path.exists(bin_single):- return ["pytorch_model.bin"]-- # --- pytorch .bin: any single .bin file ---- bin_files = sorted(f for f in os.listdir(source_dir) if f.endswith(".bin"))- if len(bin_files) >= 1:- return bin_files---def _is_eligible_weight(tensor_name: str, tensor: torch.Tensor) -> bool:- """Check if a tensor is eligible for quantization (2D Linear weight)."""- return tensor_name.endswith(".weight") and tensor.dim() == 2---def _is_weight_shard(fname: str) -> bool:- """Return True if *fname* is a weight shard (safetensors or .bin).-- Excludes index files (``*.index.json``) so that they are copied to the- output directory as normal metadata.- """- if fname.endswith(".index.json"):- return False- return fname.endswith(".safetensors") or fname.endswith(".bin")---# Keep old name as an alias for backward compatibility.-_is_safetensors_shard = _is_weight_shard---def _download_single_shard(- model_name_or_path: str,- shard_filename: str,- local_dir: str,-) -> str:- """Download a single safetensors shard file. Returns the local path."""- os.makedirs(local_dir, exist_ok=True)- local_path = os.path.join(local_dir, shard_filename)- if os.path.exists(local_path):- logger.info(f"Shard '{shard_filename}' already exists at '{local_path}', skipping download.")- return local_path-- if os.path.isdir(model_name_or_path):- src = os.path.join(model_name_or_path, shard_filename)- if os.path.exists(src):- shutil.copy2(src, local_path)- return local_path- raise FileNotFoundError(f"{shard_filename} not found in {model_name_or_path}")-- from huggingface_hub import hf_hub_download-- return hf_hub_download(- repo_id=model_name_or_path,- filename=shard_filename,- local_dir=local_dir,- )---def _download_metadata_files(- model_name_or_path: str,- local_dir: str,-) -> str:- """Download all non-safetensors files from a model repo. Returns local dir."""- os.makedirs(local_dir, exist_ok=True)-- if os.path.isdir(model_name_or_path):- for fname in os.listdir(model_name_or_path):- if _is_weight_shard(fname):- continue- src = os.path.join(model_name_or_path, fname)- dst = os.path.join(local_dir, fname)- if os.path.isdir(src):- if not os.path.exists(dst):- shutil.copytree(src, dst)- elif os.path.isfile(src) and not os.path.exists(dst):- shutil.copy2(src, dst)- return local_dir-- from huggingface_hub import snapshot_download-- snapshot_download(- repo_id=model_name_or_path,- local_dir=local_dir,- ignore_patterns=["*.safetensors", "*.bin", "*.pth", "*.pt"],- )- return local_dir---# ----------------------------------------------------------------------------# Pattern matcher-# ------------------------------------------------------------------------------class _PatternMatcher:- """Precompiled pattern matcher with result caching.-- Merges *ignore_patterns* and ``_BLOCK_NAME_TO_IGNORE`` into single- compiled regexes, precompiles ``layer_config`` regex patterns, and- caches all match results so that repeated lookups (common across- shards) are O(1) dict hits.- """-- __slots__ = (- "_ignore_re",- "_skip_re",- "_layer_config",- "_default_scheme",- "_compiled_lc",- "_ignore_cache",- "_scheme_cache",- )-- def __init__(- self,- ignore_patterns: list[str],- layer_config: dict[str, dict],- default_scheme: dict,- ) -> None:- self._default_scheme = default_scheme- self._layer_config = layer_config-- self._ignore_re: re.Pattern | None = self._build_ignore_regex(ignore_patterns)-- skip_parts = [re.escape(b) for b in _BLOCK_NAME_TO_IGNORE]- self._skip_re: re.Pattern | None = re.compile("|".join(skip_parts)) if skip_parts else None-- # Each entry: (compiled_regex | None, plain_string | None, cfg_dict)- self._compiled_lc: list[tuple[re.Pattern | None, str | None, dict]] = []- for pattern, cfg in layer_config.items():- try:- self._compiled_lc.append((re.compile(to_standard_regex(pattern)), None, cfg))- except re.error:- self._compiled_lc.append((None, pattern, cfg))-- self._ignore_cache: dict[str, bool] = {}- self._scheme_cache: dict[str, dict | None] = {}-- @staticmethod- def _build_ignore_regex(patterns: list[str]) -> re.Pattern | None:- """Merge ignore patterns into one compiled regex.-- Uses :func:`~auto_round.utils.common.to_standard_regex` so that- plain names are automatically wrapped with ``.*`` on both sides- (substring matching) and regex meta-characters in user patterns- are preserved — consistent with ``set_layer_config``.- """- if not patterns:- return None- parts: list[str] = []- for p in patterns:- if p.endswith("."):- std = to_standard_regex(p.rstrip("."))- std = std.removesuffix(".*")- parts.append(f"{std}(?:\\.|$)")- else:- parts.append(to_standard_regex(p))- return re.compile("|".join(parts))-- def should_ignore(self, tensor_name: str) -> bool:- """Check user-specified ignore patterns (merged regex + cache)."""- cached = self._ignore_cache.get(tensor_name)- if cached is not None:- return cached- layer_name = tensor_name.rsplit(".", 1)[0] if "." in tensor_name else tensor_name- result = bool(self._ignore_re and self._ignore_re.search(layer_name))- self._ignore_cache[tensor_name] = result- return result-- def should_skip(self, tensor_name: str) -> bool:- """Check predefined skip patterns (routing gates, embeddings, etc.)."""- return bool(self._skip_re and self._skip_re.search(tensor_name))-- def resolve_scheme(self, tensor_name: str) -> dict | None:- """Resolve quantization scheme for *tensor_name* (cached).-- Returns ``None`` when the layer should stay in full precision.- """- if tensor_name in self._scheme_cache:- return self._scheme_cache[tensor_name]- result = self._resolve_uncached(tensor_name)- self._scheme_cache[tensor_name] = result- return result-- def _resolve_uncached(self, tensor_name: str) -> dict | None:- layer_name = tensor_name.rsplit(".", 1)[0] if "." in tensor_name else tensor_name- default = self._default_scheme-- if layer_name in self._layer_config:- cfg = self._layer_config[layer_name]- if cfg.get("bits", default.get("bits", 4)) >= 16:- return None- return {**default, **cfg}-- for compiled, plain, cfg in self._compiled_lc:- if compiled is not None:- if compiled.search(layer_name):- if cfg.get("bits", default.get("bits", 4)) >= 16:- return None- return {**default, **cfg}- elif plain is not None and plain in layer_name:- if cfg.get("bits", default.get("bits", 4)) >= 16:- return None- return {**default, **cfg}-- return default---# ---------------------------------------------------------------------------# Per-tensor / per-shard helpers# ----------------------------------------------------------------------------def _quantize_weight_mxfp(- weight: torch.Tensor,- layer_name: str,- bits: int,- group_size: int,- data_type: str,- device: str = "cpu",- disable_opt_rtn: bool = False,-) -> dict[str, torch.Tensor]:- """Quantize a 2D weight tensor to MXFP4 / MXFP8 and return packed outputs.-- Reuses :func:`auto_round.data_type.mxfp.quant_mx` to derive the per-block- shared exponent (E8M0 scale), and :class:`auto_round.export.export_to_autoround.qlinear_fp.QuantLinear`- to perform the same packing as :func:`auto_round.export.export_to_llmcompressor.export_to_fp.pack_layer`.-- Returns a dict with one of:- * MXFP8: ``{layer_name+'.weight': float8_e4m3fn, layer_name+'.weight_scale': uint8}``- * MXFP4: ``{layer_name+'.weight_packed': uint8, layer_name+'.weight_scale': uint8}``- """- import torch.nn as nn-- from auto_round.data_type.utils import get_quant_func- from auto_round.export.export_to_autoround.qlinear_fp import QuantLinear-- if not is_mx_fp(data_type):- data_type = "mx_fp4" if bits == 4 else "mx_fp8"-- out_features, in_features = weight.shape- if in_features % group_size != 0:- raise ValueError(- f"in_features={in_features} for layer '{layer_name}' is not divisible "- f"by MXFP group_size={group_size}; cannot pack."- )-- weight_dev = weight.to(device)- # Use get_quant_func (same as WrapperLinear) so that all registered MXFP- # variants automatically get opt_rtn support via the QUANT_FUNC_WITH_DTYPE- # registry (e.g. "opt_rtn_mx_fp4" -> quant_mx_opt_rtn, "mx_fp4" -> quant_mx).- quant_func, _ = get_quant_func(data_type, bits, sym=True, disable_opt_rtn=disable_opt_rtn, iters=0)- weight_dev, shared_exp, _ = quant_func(weight_dev, bits=bits, group_size=group_size, data_type=data_type)- # Reshape to (out_features, n_groups) so the on-disk weight_scale matches- # the llm-compressor convention (and QuantLinear's registered buffer shape).- shared_exp = shared_exp.reshape(out_features, in_features // group_size)- # Ensure shared_exp is a numeric float (not a storage-specific dtype like- # float8) — QuantLinear.pack performs `2 ** scales` which dispatches to- # torch.pow; some backends do not implement pow for float8 dtypes. Cast to- # float32 here to avoid runtime errors like "pow_cuda not implemented for- # 'Float8_e4m3fn'" while preserving numeric values.- shared_exp = shared_exp.to(torch.float32)-- # Build a lightweight nn.Linear holding the original weight so we can- # delegate packing to the existing QuantLinear.pack implementation.- fake_linear = nn.Linear(in_features, out_features, bias=False)- with torch.no_grad():- fake_linear.weight = nn.Parameter(weight_dev, requires_grad=False)-- qlayer = QuantLinear(- bits=bits,- group_size=group_size,- infeatures=in_features,- outfeatures=out_features,- bias=False,- data_type="mx_fp4" if bits == 4 else "mx_fp8e4m3",- sym=True,- act_bits=bits,- )- qlayer.pack(fake_linear, shared_exp, device=device)-- if bits == 8:- return {- f"{layer_name}.weight": qlayer.weight.to("cpu"),- f"{layer_name}.weight_scale": qlayer.weight_scale.to("cpu"),- }- return {- f"{layer_name}.weight_packed": qlayer.weight_packed.to("cpu"),- f"{layer_name}.weight_scale": qlayer.weight_scale.to("cpu"),- }---def _quantize_single_tensor(- tensor_name: str,- tensor: torch.Tensor,- matcher: "_PatternMatcher",- device: str = "cpu",- quantize_func: Callable = quantize_weight_rtn,- disable_opt_rtn: bool = False,-) -> tuple[str, dict[str, torch.Tensor], str | None, str | None]:- """Quantize one eligible weight tensor and return packed outputs.-- Returns:- (layer_name, output_tensors_dict, quantized_layer_or_None, ignored_layer_or_None)- """- layer_name = tensor_name.rsplit(".", 1)[0]-- if not _is_eligible_weight(tensor_name, tensor):- ignored_layer = layer_name if tensor_name.endswith(".weight") and tensor.dim() > 1 else None- return layer_name, {tensor_name: tensor}, None, ignored_layer-- if matcher.should_ignore(tensor_name):- logger.debug(f"Ignoring (user-specified): {layer_name}")- return layer_name, {tensor_name: tensor}, None, layer_name-- if matcher.should_skip(tensor_name):- logger.debug(f"Skipping (predefined): {layer_name}")- return layer_name, {tensor_name: tensor}, None, layer_name-- scheme = matcher.resolve_scheme(tensor_name)- if scheme is None:- logger.debug(f"Keeping full precision: {layer_name}")- return layer_name, {tensor_name: tensor}, None, layer_name-- bits = scheme["bits"]- group_size = scheme["group_size"]- sym = scheme.get("sym", True)- data_type = (scheme.get("data_type") or "int").lower()-- if bits >= 16:- return layer_name, {tensor_name: tensor}, None, layer_name-- # ---- MXFP path (MXFP4 / MXFP8) ----- if is_mx_fp(data_type):- try:- out = _quantize_weight_mxfp(- weight=tensor,- layer_name=layer_name,- bits=bits,- group_size=group_size,- data_type=data_type,- device=device,- disable_opt_rtn=disable_opt_rtn,- )- logger.debug(f"Quantized (MXFP): {layer_name} (bits={bits}, group_size={group_size})")- return layer_name, out, layer_name, None- except Exception as e:- logger.warning(f"Failed to MXFP-quantize {layer_name}: {e}. Keeping original weight.")- return layer_name, {tensor_name: tensor}, None, layer_name-- # ---- Integer WOQ path ----- # opt_rtn is always disabled for integer WOQ in model-free mode because- # the scale search does not improve accuracy for INT quantization here.- try:- qweight, qzeros, scales = quantize_func(- weight=tensor,- bits=bits,- group_size=group_size,- sym=sym,- device=device,- disable_opt_rtn=True,- )-- out: dict[str, torch.Tensor] = {- f"{layer_name}.qweight": qweight,- f"{layer_name}.qzeros": qzeros,- f"{layer_name}.scales": scales,- }-- logger.debug(f"Quantized: {layer_name} (bits={bits}, group_size={group_size}, sym={sym})")- return layer_name, out, layer_name, None-- except Exception as e:- logger.warning(f"Failed to quantize {layer_name}: {e}. Keeping original weight.")- return layer_name, {tensor_name: tensor}, None, layer_name---def _collect_mxfp_source_entries(raw_tensors: dict[str, torch.Tensor]) -> list[tuple[str, str, str, int]]:- """Collect MXFP source tensors present in a shard.-- Returns entries as ``(layer_name, weight_key, scale_key, bits)`` where- ``bits`` is 8 for ``.weight`` (float8) and 4 for ``.weight_packed``.- """- entries: list[tuple[str, str, str, int]] = []- for name, tensor in raw_tensors.items():- if name.endswith(".weight") and tensor.dtype == torch.float8_e4m3fn:- layer_name = name[: -len(".weight")]- scale_key = f"{layer_name}.weight_scale"- # MXFP scales are raw E8M0 bytes stored as uint8.- if scale_key in raw_tensors and raw_tensors[scale_key].dtype == torch.uint8:- entries.append((layer_name, name, scale_key, 8))- elif name.endswith(".weight_packed") and tensor.dtype in (torch.int8, torch.uint8):- layer_name = name[: -len(".weight_packed")]- scale_key = f"{layer_name}.weight_scale"- # Guard against incorrectly detecting non-MXFP INT4 sources that also use- # ".weight_packed" + ".weight_scale" but with floating-point scales.- if scale_key in raw_tensors and raw_tensors[scale_key].dtype == torch.uint8:- entries.append((layer_name, name, scale_key, 4))- return entries---def _read_int_like(value: Any) -> int | None:- """Best-effort conversion for int-like config values."""- if isinstance(value, bool) or value is None:- return None- if isinstance(value, int):- return value- if isinstance(value, float) and value.is_integer():- return int(value)- if isinstance(value, str):- v = value.strip()- if v.isdigit() or (v.startswith("-") and v[1:].isdigit()):- return int(v)- return None---def _read_bool_like(value: Any) -> bool | None:- """Best-effort conversion for bool-like config values."""- if isinstance(value, bool):- return value- if isinstance(value, str):- v = value.strip().lower()- if v in {"true", "1", "yes"}:- return True- if v in {"false", "0", "no"}:- return False- return None---def _resolve_kimi_k25_int4_params(source_quant_config: dict | None) -> tuple[int | None, bool]:- """Resolve default (group_size, sym) for kimi_k25 INT4 source dequant.-- The source tensor shapes are still treated as the source of truth, while- config values are used as defaults or for logging consistency checks.- """- group_size: int | None = None- sym = True-- if not isinstance(source_quant_config, dict):- return group_size, sym-- candidates = [source_quant_config]- weights_cfg = source_quant_config.get("weights")- if isinstance(weights_cfg, dict):- candidates.append(weights_cfg)-- for cfg in candidates:- if group_size is None:- for key in ("group_size", "weight_group_size"):- maybe = _read_int_like(cfg.get(key))- if maybe is not None and maybe > 0:- group_size = maybe- break-- sym_val = _read_bool_like(cfg.get("sym"))- if sym_val is not None:- sym = sym_val-- # Some configs store asymmetry as zero_point=True.- zp_val = _read_bool_like(cfg.get("zero_point"))- if zp_val is not None:- sym = not zp_val-- return group_size, sym---def _collect_kimi_k25_int4_source_entries(- raw_tensors: dict[str, torch.Tensor],-) -> list[tuple[str, str, str, str | None]]:- """Collect kimi_k25 INT4 source tensors.-- Expected source layout per layer:- * ``<layer>.weight_packed``: uint8/int8 packed 2x int4 values per byte,- **or** int32 packed 8x int4 values per element (Kimi-K2.6 format)- * ``<layer>.weight_scale``: floating-point per-group scales- * optional ``<layer>.weight_zero_point`` or ``<layer>.zero_point``- * optional ``<layer>.weight_shape``: int32 tensor storing original shape- """- entries: list[tuple[str, str, str, str | None]] = []- for name, packed in raw_tensors.items():- if not name.endswith(".weight_packed"):- continue- if packed.dtype not in (torch.uint8, torch.int8, torch.int32):- continue-- layer_name = name[: -len(".weight_packed")]- scale_key = f"{layer_name}.weight_scale"- if scale_key not in raw_tensors:- continue-- scale = raw_tensors[scale_key]- if scale.dtype not in (torch.float16, torch.bfloat16, torch.float32, torch.float64):- continue- if packed.dim() != 2 or scale.dim() != 2:- continue-- zp_key = None- for candidate in (f"{layer_name}.weight_zero_point", f"{layer_name}.zero_point"):- if candidate in raw_tensors:- zp_key = candidate- break-- entries.append((layer_name, name, scale_key, zp_key))-- return entries---def _unpack_int4_weight_packed(packed: torch.Tensor) -> torch.Tensor:- """Unpack packed int4 bytes/ints into int16 ``[out, in]`` nibbles.-- Supports two packing formats:- * ``uint8`` / ``int8``: 2 nibbles per byte — ``[out, in/2]`` → ``[out, in]``- * ``int32``: 8 nibbles per element (Kimi-K2.6) — ``[out, in/8]`` → ``[out, in]``- Nibbles are stored LSB-first: bits 0-3 → nibble 0, bits 4-7 → nibble 1, …- """- if packed.dtype == torch.int32:- # Each int32 packs 8 nibbles (4-bit values), LSB-first.- packed_i32 = packed.view(torch.int32)- rows, cols = packed_i32.shape- # Extract all 8 nibbles per int32 element.- nibble_list = [(packed_i32 >> (4 * i)) & 0xF for i in range(8)]- # Stack into [rows, cols, 8] then reshape to [rows, cols*8].- out = torch.stack(nibble_list, dim=-1).reshape(rows, cols * 8).to(torch.int16)- return out-- packed_u8 = packed.view(torch.uint8)- lo = packed_u8 & 0x0F- hi = (packed_u8 >> 4) & 0x0F-- out = torch.empty(- (packed_u8.shape[0], packed_u8.shape[1] * 2),- dtype=torch.int16,- device=packed_u8.device,- )- out[:, 0::2] = lo.to(torch.int16)- out[:, 1::2] = hi.to(torch.int16)- return out---def _dequant_kimi_k25_int4_tensors(- raw_tensors: dict[str, torch.Tensor],- source_quant_config: dict | None = None,- device: str = "cpu",- shard_name: str | None = None,-) -> dict[str, torch.Tensor]:- """Dequantize kimi_k25-style INT4 packed weights to bfloat16 ``.weight``.-- This normalizes source tensors so the downstream model-free path can- requantize them into the requested target format (for example MXFP4).- """- entries = _collect_kimi_k25_int4_source_entries(raw_tensors)- if not entries:- return raw_tensors-- default_group_size, default_sym = _resolve_kimi_k25_int4_params(source_quant_config)- dequant_device = str(device or "cpu")- shard_prefix = f"[{shard_name}] " if shard_name else ""- logger.info(- f"{shard_prefix}Dequantizing kimi_k25 INT4 tensor(s) to bfloat16 on {dequant_device}: " f"total={len(entries)}."- )-- for layer_name, packed_key, scale_key, zp_key in entries:- packed = raw_tensors.pop(packed_key)- scale = raw_tensors.pop(scale_key)- zero = raw_tensors.pop(zp_key) if zp_key is not None else None- # Remove auxiliary weight_shape tensor if present (Kimi-K2.6 format).- raw_tensors.pop(f"{layer_name}.weight_shape", None)-- # int32 packs 8 nibbles per element; uint8/int8 packs 2 nibbles per byte.- nibbles_per_element = 8 if packed.dtype == torch.int32 else 2- in_features = packed.shape[1] * nibbles_per_element- groups = scale.shape[1]- inferred_group_size = (in_features // groups) if groups > 0 and in_features % groups == 0 else None- group_size = inferred_group_size or default_group_size- if group_size is None or group_size <= 0:- raise ValueError(- f"Cannot resolve group_size for INT4 tensor '{layer_name}': "- f"packed_shape={tuple(packed.shape)}, scale_shape={tuple(scale.shape)}"- )- if in_features % group_size != 0:- raise ValueError(- f"Invalid group_size={group_size} for INT4 tensor '{layer_name}': "- f"in_features={in_features} is not divisible by group_size."- )-- if (- inferred_group_size is not None- and default_group_size is not None- and inferred_group_size != default_group_size- ):- logger.warning(- f"{shard_prefix}k25 INT4 group_size mismatch for {layer_name}: "- f"config={default_group_size}, inferred={inferred_group_size}. "- "Using inferred value from tensor shapes."- )-- n_groups = in_features // group_size- if scale.shape[1] != n_groups:- raise ValueError(- f"Scale shape mismatch for INT4 tensor '{layer_name}': "- f"scale.shape={tuple(scale.shape)}, expected second dim {n_groups}."- )-- def _dequant_impl(target_device: str) -> torch.Tensor:- q = _unpack_int4_weight_packed(packed.to(target_device, non_blocking=True))- sc = scale.to(target_device, non_blocking=True).to(torch.float32)-- if zero is not None:- zp = zero.to(target_device, non_blocking=True)- if zp.dim() == 2 and zp.shape == packed.shape and zp.dtype in (torch.int8, torch.uint8):- zp = _unpack_int4_weight_packed(zp)- elif zp.dim() == 2 and zp.shape[1] != n_groups:- raise ValueError(f"Unsupported zero-point shape for INT4 tensor '{layer_name}': {tuple(zp.shape)}")- if zp.dim() == 2 and zp.shape[1] == n_groups:- zp = zp.to(torch.int16).repeat_interleave(group_size, dim=1)- zp = zp[:, : q.shape[1]].to(torch.int16)- q = q - zp- else:- if default_sym:- q = torch.where(q >= 8, q - 16, q)- else:- # Conservative fallback for asymmetric INT4 without explicit- # zero-point tensor: recenter to [-8, 7].- q = q - 8-- sc = sc.repeat_interleave(group_size, dim=1)[:, : q.shape[1]]- return (q.to(torch.float32) * sc).to(torch.bfloat16)-- dq_weight = _dequantize_with_device_fallback(- dequant_device=dequant_device,- shard_prefix=shard_prefix,- op_name="kimi_k25 INT4 dequant",- tensor_label=layer_name,- on_device=lambda: _dequant_impl(dequant_device).to("cpu"),- on_cpu=lambda: _dequant_impl("cpu"),- )- raw_tensors[f"{layer_name}.weight"] = dq_weight-- return raw_tensors---def _handle_model_type_low_precision_source_tensors(- raw_tensors: dict[str, torch.Tensor],- model_type: str | None,- source_quant_config: dict | None = None,- device: str = "cpu",- shard_name: str | None = None,-) -> dict[str, torch.Tensor]:- """Handle model-type-specific low-precision source tensors.-- Currently supports:- * ``model_type='kimi_k25'`` INT4 packed tensors.- """- if (model_type or "").lower() == "kimi_k25":- return _dequant_kimi_k25_int4_tensors(- raw_tensors,- source_quant_config=source_quant_config,- device=device,- shard_name=shard_name,- )- return raw_tensors---def _is_out_of_memory_error(exc: Exception) -> bool:- if isinstance(exc, torch.OutOfMemoryError):- return True- message = str(exc).lower()- return "out of memory" in message or "cuda error: out of memory" in message---def _dequantize_with_device_fallback(- *,- dequant_device: str,- shard_prefix: str,- op_name: str,- tensor_label: str,- on_device: Callable[[], torch.Tensor],- on_cpu: Callable[[], torch.Tensor],-) -> torch.Tensor:- """Run dequantization on ``dequant_device`` and fall back to CPU on errors."""- if dequant_device != "cpu":- try:- return on_device()- except Exception as e:- if _is_out_of_memory_error(e):- logger.warning(- f"{shard_prefix}{op_name} on {dequant_device} ran OOM for {tensor_label}: {e}. "- "Clearing accelerator memory and falling back to CPU for this tensor."- )- clear_memory()- else:- logger.warning(- f"{shard_prefix}{op_name} on {dequant_device} failed for {tensor_label}: {e}. "- "Falling back to CPU for this tensor."- )- return on_cpu()---def _dequant_mxfp_tensors(- raw_tensors: dict[str, torch.Tensor],- device: str = "cpu",- shard_name: str | None = None,-) -> dict[str, torch.Tensor]:- """Dequantize llm-compressor MXFP8 / MXFP4 weight tensors to bfloat16.-- Detection is purely by *name* and *dtype*, reusing the dequant kernels in- :mod:`auto_round_extension.vllm_ext`:-- * ``<layer>.weight`` (``float8_e4m3fn``) + ``<layer>.weight_scale`` → MXFP8,- dequantized via :func:`~auto_round_extension.vllm_ext.mxfp8_qdq_utils.dequant_mx_fp8`.- * ``<layer>.weight_packed`` (``uint8``) + ``<layer>.weight_scale`` → MXFP4,- dequantized via :func:`~auto_round_extension.vllm_ext.mxfp4_qdq_utils.to_dtype`.-- The dequantized weight is written back under ``<layer>.weight`` and the- scale (and any ``weight_packed``) tensor is removed, so the downstream RTN- path can requantize the layer to the requested target scheme.- """- from auto_round_extension.vllm_ext.mxfp4_qdq_utils import to_dtype- from auto_round_extension.vllm_ext.mxfp8_qdq_utils import dequant_mx_fp8-- # Tuple layout: (layer_name, weight_key, scale_key, bits)- entries = _collect_mxfp_source_entries(raw_tensors)-- if not entries:- return raw_tensors-- n_mxfp8 = sum(1 for _layer_name, _weight_key, _scale_key, bits in entries if bits == 8)- n_mxfp4 = len(entries) - n_mxfp8- dequant_device = str(device or "cpu")- shard_prefix = f"[{shard_name}] " if shard_name else ""- logger.info(- f"{shard_prefix}Dequantizing MXFP tensor(s) to bfloat16 on {dequant_device}: "- f"MXFP8={n_mxfp8}, MXFP4={n_mxfp4}, total={len(entries)}."- )-- for layer_name, weight_key, scale_key, bits in entries:- weight = raw_tensors.pop(weight_key)- scale = raw_tensors.pop(scale_key).view(torch.uint8)- if bits == 8:- dq_weight = _dequantize_with_device_fallback(- dequant_device=dequant_device,- shard_prefix=shard_prefix,- op_name="MXFP dequant",- tensor_label=layer_name,- on_device=lambda: dequant_mx_fp8(- weight_fp8=weight.to(dequant_device, non_blocking=True),- scale_e8m0=scale.to(dequant_device, non_blocking=True),- block_size=32,- target_dtype=torch.bfloat16,- ).to("cpu"),- on_cpu=lambda: dequant_mx_fp8(- weight_fp8=weight,- scale_e8m0=scale,- block_size=32,- target_dtype=torch.bfloat16,- ),- )- else:- dq_weight = _dequantize_with_device_fallback(- dequant_device=dequant_device,- shard_prefix=shard_prefix,- op_name="MXFP dequant",- tensor_label=layer_name,- on_device=lambda: to_dtype(- data_lp=weight.view(torch.uint8).contiguous().to(dequant_device, non_blocking=True),- scale_e8m0=scale.to(dequant_device, non_blocking=True),- elem_dtype="fp4_e2m1",- block_size=32,- target_dtype=torch.bfloat16,- ).to("cpu"),- on_cpu=lambda: to_dtype(- data_lp=weight.view(torch.uint8).contiguous(),- scale_e8m0=scale,- elem_dtype="fp4_e2m1",- block_size=32,- target_dtype=torch.bfloat16,- ),- )- raw_tensors[f"{layer_name}.weight"] = dq_weight-- return raw_tensors---def _handle_mxfp_source_tensors(- raw_tensors: dict[str, torch.Tensor],- matcher: "_PatternMatcher",- source_state: dict[str, int] | None = None,- device: str = "cpu",- shard_name: str | None = None,-) -> tuple[dict[str, torch.Tensor], dict[str, torch.Tensor], list[str]]:- """Handle llm-compressor-style MXFP8/MXFP4 source tensors.-- Detects llm-compressor MXFP tensors purely by *name + dtype*:-- * ``<layer>.weight`` (``float8_e4m3fn``) + ``<layer>.weight_scale`` → MXFP8.- * ``<layer>.weight_packed`` (``uint8``) + ``<layer>.weight_scale`` → MXFP4.-- For each detected layer the effective target scheme is resolved via *matcher*:-- * If the target is the **same MXFP format** (``data_type='mx_fp'``, matching- ``bits``), the tensors are emitted directly as a passthrough — no- dequantization is performed and the layer is recorded as already quantized.- * Otherwise the tensors are dequantized to ``bfloat16`` via- :func:`_dequant_mxfp_tensors` so the downstream RTN path can re-quantize- them to the requested target scheme.-- Returns:- ``(raw_tensors, passthrough_tensors, passthrough_layers)``.- """- entries = _collect_mxfp_source_entries(raw_tensors)- if not entries:- return raw_tensors, {}, []-- source_state = source_state or {}-- passthrough_tensors: dict[str, torch.Tensor] = {}- passthrough_layers: list[str] = []- n_dequant = 0-- for layer_name, weight_key, scale_key, bits in entries:- scheme = matcher.resolve_scheme(f"{layer_name}.weight")- target_is_same_mxfp = (- scheme is not None and is_mx_fp((scheme.get("data_type") or "").lower()) and scheme.get("bits") == bits- )- if target_is_same_mxfp:- passthrough_tensors[weight_key] = raw_tensors.pop(weight_key).to("cpu")- passthrough_tensors[scale_key] = raw_tensors.pop(scale_key).to("cpu")- passthrough_layers.append(layer_name)- else:- n_dequant += 1-- if n_dequant:- raw_tensors = _dequant_mxfp_tensors(raw_tensors, device=device, shard_name=shard_name)-- parts: list[str] = []- if passthrough_layers:- parts.append(f"{len(passthrough_layers)} passthrough")- if n_dequant:- parts.append(f"{n_dequant} dequantized to bfloat16")- if source_state:- parts.append(f"{len(source_state)} model_type-normalized")- logger.info(f"Handling MXFP source tensor(s): {', '.join(parts)}.")-- return raw_tensors, passthrough_tensors, passthrough_layers---def _dequant_fp8_tensors(- raw_tensors: dict[str, torch.Tensor],- block_size: list | None = None,- device: str = "cpu",- shard_name: str | None = None,-) -> dict[str, torch.Tensor]:- """Dequantize DeepSeek-V3-style FP8 weight tensors to bfloat16.-- Handles the **DeepSeek-V3 FP8** convention: weight dtype ``float8_e4m3fn``- paired with a ``.weight_scale_inv`` tensor (per-block float32 scales, NOT- E8M0). The weights are converted to ``bfloat16`` so downstream RTN- quantization can proceed normally.-- MXFP sources are handled separately by- :func:`_preprocess_model_type_source_tensors` / :func:`_handle_mxfp_source_tensors`.- """- from auto_round.utils.weight_handler import _dequant_fp8_linear_weight-- quant_entries: list[tuple[str, str]] = []- for name, tensor in raw_tensors.items():- if not name.endswith(".weight"):- continue- if tensor.dtype != torch.float8_e4m3fn and tensor.element_size() != 1:- continue- # DeepSeek-V3 style: .weight_scale_inv (per-block float32 scales).- scale_inv_name = name.replace(".weight", ".weight_scale_inv")- if scale_inv_name in raw_tensors:- quant_entries.append((name, scale_inv_name))-- if not quant_entries:- return raw_tensors-- # device has already been resolved by the caller; use it directly here.- dequant_device = str(device or "cpu")- shard_prefix = f"[{shard_name}] " if shard_name else ""-- logger.info(- f"{shard_prefix}Dequantizing {len(quant_entries)} FP8 weight tensor(s) to bfloat16 on {dequant_device}."- )-- for weight_name, scale_name in quant_entries:- weight = raw_tensors[weight_name]- scale = raw_tensors.pop(scale_name)-- # Dequantize on GPU for throughput, then move back to CPU to keep- # per-shard memory usage bounded before per-layer quantization.- raw_tensors[weight_name] = _dequantize_with_device_fallback(- dequant_device=dequant_device,- shard_prefix=shard_prefix,- op_name="FP8 dequant",- tensor_label=weight_name,- on_device=lambda: _dequant_fp8_linear_weight(- weight.to(dequant_device, non_blocking=True),- scale.to(dequant_device, non_blocking=True),- block_size=block_size,- ).to("cpu"),- on_cpu=lambda: _dequant_fp8_linear_weight(weight, scale, block_size=block_size),- )-- return raw_tensors---def _process_shard(- shard_path: str,- default_scheme: dict = None,- layer_config: dict = None,- ignore_patterns: list[str] = None,- device: str = "cpu",- *,- shard_name: str | None = None,- matcher: "_PatternMatcher | None" = None,- fp8_block_size: list | None = None,- model_type: str | None = None,- source_quant_config: dict | None = None,- enable_torch_compile: bool = False,- disable_opt_rtn: bool = False,-) -> tuple[dict[str, torch.Tensor], list[str], list[str]]:- """Quantize eligible weights in a single safetensors shard.-- Returns:- (output_tensors, quantized_layer_names, ignored_layer_names)-- ``ignored_layer_names`` is derived by comparing the set of input ``.weight``- layer names (collected after fused-expert splitting) with the final set of- quantized layer names. Any layer that had a ``.weight`` tensor in the input- but was NOT quantized is reported as ignored — this correctly captures- user-ignored layers, predefined-skipped layers, non-eligible weights, and- any other pass-through case without separate per-tensor tracking.- """- if matcher is None:- matcher = _PatternMatcher(- ignore_patterns if ignore_patterns is not None else [],- layer_config if layer_config is not None else {},- default_scheme if default_scheme is not None else {},- )-- output_tensors: dict[str, torch.Tensor] = {}- quantized_layers: list[str] = []- quantize_func = compile_func(quantize_weight_rtn, device) if enable_torch_compile else quantize_weight_rtn-- if shard_path.endswith(".bin"):- # PyTorch pickle checkpoint — load with weights_only where supported.- try:- raw_tensors = torch.load(shard_path, map_location="cpu", weights_only=True)- except TypeError:- # weights_only not available in older PyTorch versions- raw_tensors = torch.load(shard_path, map_location="cpu") # nosec- # Flatten nested state-dict wrappers if present.- if not isinstance(raw_tensors, dict):- raise ValueError(f"Expected a dict from {shard_path}, got {type(raw_tensors)}")- else:- from safetensors import safe_open-- with safe_open(shard_path, framework="pt", device="cpu") as f:- raw_tensors = {name: f.get_tensor(name) for name in f.keys()}-- raw_tensors = split_fused_expert_tensors(raw_tensors)-- # Snapshot candidate weight layer names *before* any preprocessing. 1D- # weights (for example LayerNorm) are not quantization targets, while 3D- # weights remain tracked so unsupported layouts are visible as ignored.- input_weight_layers: list[str] = list(- dict.fromkeys(- name.rsplit(".", 1)[0]- for name, tensor in raw_tensors.items()- if name.endswith(".weight") and tensor.dim() > 1- )- )-- # Preserve original tensors for ignored/skipped layers so that already-- # quantized weights (FP8, FP4-packed, etc.) are NOT dequantized.- # Check both ".weight" and ".weight_packed" so that layers whose primary- # tensor uses non-standard naming (e.g. already-quantized FP4-packed layers- # stored as ".weight_packed") are correctly captured.- preserved_prefixes: set[str] = set()- for tname in raw_tensors:- if (tname.endswith(".weight") or tname.endswith(".weight_packed") or tname.endswith(".qweight")) and (- matcher.should_ignore(tname) or matcher.should_skip(tname)- ):- preserved_prefixes.add(tname.rsplit(".", 1)[0])-- preserved_tensors: dict[str, torch.Tensor] = {}- if preserved_prefixes:- for key in list(raw_tensors.keys()):- prefix = key.rsplit(".", 1)[0]- if prefix in preserved_prefixes:- preserved_tensors[key] = raw_tensors.pop(key)-- # 1) model-type-specific preprocessing (format conversion only)- raw_tensors, source_state = _preprocess_model_type_source_tensors(raw_tensors, model_type=model_type)-- # 1.5) model-type-specific low-precision dequantization (e.g. kimi_k25 INT4)- raw_tensors = _handle_model_type_low_precision_source_tensors(- raw_tensors,- model_type=model_type,- source_quant_config=source_quant_config,- device=device,- shard_name=shard_name,- )-- # 2) generic MXFP handling for both preprocessed and normal source models- raw_tensors, passthrough_tensors, passthrough_layers = _handle_mxfp_source_tensors(- raw_tensors,- matcher,- source_state=source_state,- device=device,- shard_name=shard_name,- )- output_tensors.update(passthrough_tensors)- quantized_layers.extend(passthrough_layers)-- raw_tensors = _dequant_fp8_tensors(- raw_tensors,- block_size=fp8_block_size,- device=device,- shard_name=shard_name,- )- raw_tensors.update(preserved_tensors)-- for tensor_name in list(raw_tensors.keys()):- tensor = raw_tensors.pop(tensor_name)- _layer_name, out_dict, q_layer, _ig_layer = _quantize_single_tensor(- tensor_name,- tensor,- matcher,- device,- quantize_func,- disable_opt_rtn,- )- output_tensors.update(out_dict)- if q_layer:- quantized_layers.append(q_layer)-- # Derive ignored layers by comparing input weight layers with quantized set.- quantized_set = set(quantized_layers)- ignored_layers: list[str] = [l for l in input_weight_layers if l not in quantized_set]-- return output_tensors, quantized_layers, ignored_layers---# ---------------------------------------------------------------------------# Output writers# ----------------------------------------------------------------------------def _build_mxfp_quantization_config(- default_scheme: dict,- quantized_layers: list[str],- ignored_layers: list[str],- layer_config: dict | None = None,-) -> dict:- """Build a compressed-tensors / llm-compressor style quantization_config- dict for MXFP4 / MXFP8 model-free output, including mixed-precision cases.-- When *layer_config* contains layers that override the default bits (e.g.- some layers are MXFP8 while the default is MXFP4), the function creates- one ``config_group`` per distinct bit-width. Override groups list their- layers explicitly; the default-bits group uses ``targets=["Linear"]`` as a- catch-all. The top-level ``"format"`` is set to ``"mixed-precision"``- when more than one group is produced.-- Mirrors the per-group format produced by- :mod:`auto_round.export.export_to_llmcompressor.export_to_fp`.- """- from auto_round.export.export_to_llmcompressor.config import (- initialize_quantization,- )-- bits = default_scheme.get("bits", 4)- is_fp_default = (bits or 0) >= 16 # BF16/FP16 full-precision default-- if not is_fp_default and bits not in _SUPPORTED_MXFP_BITS:- raise ValueError(f"Unsupported MXFP bits={bits} for model-free output.")-- # Default ignore list: any layer present in ignored_layers (deduped) that- # was NOT quantized.- ignore = list(dict.fromkeys(ignored_layers))- quant_set = set(quantized_layers)- ignore = [n for n in ignore if n not in quant_set]-- # Resolve each quantized layer's effective bits using layer_config overrides.- scheme_groups: dict[int, list[str]] = {} # bits -> [layer_names]- if layer_config:- temp_matcher = _PatternMatcher(- ignore_patterns=[],- layer_config=layer_config,- default_scheme=default_scheme,- )- for layer in quantized_layers:- scheme = temp_matcher.resolve_scheme(f"{layer}.weight")- layer_bits = scheme.get("bits", bits) if scheme is not None else bits- scheme_groups.setdefault(layer_bits, []).append(layer)- else:- if not is_fp_default:- scheme_groups[bits] = list(quantized_layers)- # else: BF16 default with no layer_config → no MXFP layers; scheme_groups stays {}-- if len(scheme_groups) <= 1:- # Single scheme — use the actual MXFP bits from the group, not the- # default bits (which may be 16 for a BF16 default scheme).- actual_bits = next(iter(scheme_groups.keys())) if scheme_groups else bits- if actual_bits not in _SUPPORTED_MXFP_BITS:- raise ValueError(f"Unsupported MXFP bits={actual_bits} for model-free output.")- scheme_name = "MXFP4" if actual_bits == 4 else "MXFP8"- fmt = "mxfp4-pack-quantized" if actual_bits == 4 else "mxfp8-quantized"- qconfig = initialize_quantization(scheme=scheme_name, ignore=ignore)- if is_fp_default and scheme_groups:- targets = list(quantized_layers)- qconfig.config_groups["group_0"].targets = targets- qconfig = qconfig.to_dict()- qconfig["format"] = fmt- qconfig["provider"] = "auto-round"- return qconfig-- # Mixed MXFP: build one config_group per distinct bit-width.- # Override groups (non-default bits) come first, default group last,- # ordered by descending bit-width within each partition so that the- # higher-precision group gets the lower group index.- override_items = sorted(- [(b, layers) for b, layers in scheme_groups.items() if b != bits],- key=lambda x: x[0],- reverse=True,- )- default_item = (bits, scheme_groups[bits]) if bits in scheme_groups else None- ordered = override_items + ([default_item] if default_item else [])-- config_groups: dict = {}- group_formats: dict[str, str] = {}- for idx, (group_bits, layer_names) in enumerate(ordered):- group_name = f"group_{idx}"- scheme_name = "MXFP4" if group_bits == 4 else "MXFP8"- fmt = "mxfp4-pack-quantized" if group_bits == 4 else "mxfp8-quantized"- is_default_group = group_bits == bits- targets = ["Linear"] if is_default_group else layer_names- tmp_qconfig = initialize_quantization(scheme=scheme_name, ignore=ignore)- group_scheme = tmp_qconfig.config_groups["group_0"]- group_scheme.targets = targets- config_groups[group_name] = group_scheme- group_formats[group_name] = fmt-- full_qconfig = initialize_quantization(scheme=None, config_groups=config_groups, ignore=ignore)- full_dict = full_qconfig.to_dict()- full_dict["format"] = "mixed-precision"- for group_name, fmt in group_formats.items():- full_dict["config_groups"][group_name]["format"] = fmt- full_dict["provider"] = "auto-round"- return full_dict---def _build_mxfp_autoround_quantization_config(- default_scheme: dict,- quantized_layers: list[str],- ignored_layers: list[str],- layer_config: dict | None = None,- block_name_to_quantize: Optional[str] = None,-) -> dict:- """Build an auto-round style quantization_config for MXFP4 / MXFP8.-- Unlike :func:`_build_mxfp_quantization_config` which produces a- compressed-tensors / llm-compressor style config, this function produces- an ``auto-round`` style config (``quant_method="auto-round"``,- ``packing_format="auto_round:llm_compressor"``). The on-disk weight- tensors are identical; only the ``quantization_config`` metadata differs.-- The generated config mirrors the layout of the regular AutoRound MXFP- export, including activation quantization fields (``act_bits`` /- ``act_data_type`` / …) from the scheme, ``enable_quanted_input``, and- ``lm_head`` in ``extra_config`` when it was quantized.- """- from collections import Counter-- from auto_round.version import __version__-- bits = default_scheme.get("bits", 4)- group_size = default_scheme.get("group_size", 32)- is_fp_default = (bits or 0) >= 16-- # For BF16 default + MXFP layer_config overrides, derive the dominant- # MXFP bit-width from the layers that were actually quantized.- if is_fp_default and layer_config and quantized_layers:- temp_matcher = _PatternMatcher(- ignore_patterns=[],- layer_config=layer_config,- default_scheme=default_scheme,- )- mxfp_counter: Counter = Counter()- for layer in quantized_layers:- scheme = temp_matcher.resolve_scheme(f"{layer}.weight")- if scheme is None:- continue- lb = scheme.get("bits")- ldt = (scheme.get("data_type") or "").lower()- if lb and lb < 16 and is_mx_fp(ldt):- mxfp_counter[lb] += 1- if mxfp_counter:- bits, _ = mxfp_counter.most_common(1)[0]- group_size = 32-- if (bits or 0) < 1 or bits not in _SUPPORTED_MXFP_BITS:- bits = 4 # safe fallback-- qconfig: dict = {- "quant_method": "auto-round",- "packing_format": "auto_round:llm_compressor",- "bits": bits,- "group_size": group_size or 32,- "sym": True, # MXFP is always symmetric- "data_type": "mx_fp",- "iters": 0,- "model_free": True,- "autoround_version": __version__,- "enable_quanted_input": False,- }- if block_name_to_quantize:- qconfig["block_name_to_quantize"] = block_name_to_quantize-- # Carry activation quantization fields from the scheme (e.g. MXFP4 has- # act_bits=4, act_data_type="mx_fp", act_dynamic=True, act_group_size=32,- # act_sym=True). Including these keeps the config consistent with the- # output produced by the regular AutoRound MXFP export flow.- for act_key in ("act_bits", "act_data_type", "act_dynamic", "act_group_size", "act_sym"):- val = default_scheme.get(act_key)- if val is not None:- qconfig[act_key] = val-- scheme_keys = [f.name for f in fields(QuantizationScheme)]- extra_config: dict = {}- non_linear_ops = ["embed", "conv"]- non_linear_re = re.compile("|".join(re.escape(op) for op in non_linear_ops))-- if layer_config:- from auto_round.export.export_to_autoround.utils import check_neq_config-- expected_scheme = {key: qconfig.get(key) for key in scheme_keys}- for layer_name, cfg in layer_config.items():- if not isinstance(cfg, dict):- continue- neq_keys = check_neq_config(cfg, **expected_scheme)- if neq_keys:- extra_config[layer_name] = {key: cfg[key] for key in neq_keys if cfg.get(key) is not None}-- quantized_set = set(quantized_layers)- unique_ignored = list(dict.fromkeys(ignored_layers))- for layer_name in unique_ignored:- if layer_name in quantized_set or layer_name in extra_config:- continue- if non_linear_re.search(layer_name):- continue- extra_config[layer_name] = {- "bits": 16,- "data_type": "float",- "act_bits": 16,- "act_data_type": "float",- }-- # lm_head: when explicitly quantized, record its full scheme in extra_config- # (mirrors the regular AutoRound MXFP export behavior).- if "lm_head" in quantized_set and "lm_head" not in extra_config:- lm_head_cfg = (layer_config or {}).get("lm_head", default_scheme)- extra_config["lm_head"] = {k: lm_head_cfg.get(k) for k in scheme_keys if lm_head_cfg.get(k) is not None}-- if extra_config:- qconfig["extra_config"] = extra_config-- return qconfig---def _layer_config_has_mxfp(layer_config: dict | None) -> bool:- """Return True if any layer_config entry requests MXFP quantization.-- Handles values that are plain strings (preset names), dicts (possibly- with a ``"scheme"`` key or a ``"data_type"`` key), or- :class:`QuantizationScheme` instances.- """- if not layer_config:- return False- for val in layer_config.values():- if isinstance(val, str):- try:- s = _normalize_scheme(val.upper())- if is_mx_fp((s.data_type or "").lower()):- return True- except Exception:- pass- elif isinstance(val, dict):- # Direct data_type field- if is_mx_fp((val.get("data_type") or "").lower()):- return True- # Nested 'scheme' key (e.g. {"scheme": "MXFP4"})- scheme_val = val.get("scheme")- if isinstance(scheme_val, str):- try:- s = _normalize_scheme(scheme_val.upper())- if is_mx_fp((s.data_type or "").lower()):- return True- except Exception:- pass- elif isinstance(val, QuantizationScheme):- if is_mx_fp((val.data_type or "").lower()):- return True- return False---def _is_full_precision_default(scheme_input: Any) -> bool:- """Return True when *scheme_input* represents a full-precision default (bits >= 16).-- Used to detect BF16/FP16 schemes where no weights are quantized by- default but layer_config overrides are still applied.- """- try:- s = _normalize_scheme(scheme_input)- return (s.bits or 0) >= 16 and (s.act_bits or 16) >= 16- except Exception:- return False---def _derive_dominant_int_scheme(- quantized_layers: list[str],- layer_config: dict,- fallback: dict,-) -> dict | None:- """Infer the dominant INT scheme from layers that were actually quantized.-- Used when the default scheme is full-precision (BF16/FP16) to produce a- quantization_config where the most common INT scheme becomes the top-level- default and the BF16 layers appear as ``bits=16`` exceptions in- ``extra_config``. This matches the layout expected by AutoRound loaders- (dominant quantized bits at the top, full-precision overrides below).-- Returns ``None`` when no INT layers were found in *quantized_layers*.- """- from collections import Counter-- temp_matcher = _PatternMatcher(- ignore_patterns=[],- layer_config=layer_config,- default_scheme=fallback,- )- counter: "Counter[tuple]" = Counter()- for layer in quantized_layers:- scheme = temp_matcher.resolve_scheme(f"{layer}.weight")- if scheme is None:- continue- bits = scheme.get("bits")- if bits is None or bits >= 16:- continue- data_type = (scheme.get("data_type") or "int").lower()- if is_mx_fp(data_type):- continue # MXFP is handled by the dedicated path- key = (bits, scheme.get("group_size") or fallback.get("group_size"), bool(scheme.get("sym", True)), data_type)- counter[key] += 1-- if not counter:- return None-- (bits, group_size, sym, data_type), _ = counter.most_common(1)[0]- return {- "bits": bits,- "group_size": group_size,- "sym": sym,- "data_type": data_type,- }---def _build_quantization_config(- default_scheme: dict,- layer_config: dict,- ignore_patterns: list[str],- quantized_layers: list[str],- ignored_layers: list[str],- block_name_to_quantize: Optional[str] = None,- format: str = "auto_round",-) -> dict:- """Build a quantization_config dict compatible with auto-round format."""- # MXFP (mx_fp) supports two output styles depending on *format*:- # - "llm_compressor" → compressed-tensors / llm-compressor style config- # (produced by _build_mxfp_quantization_config).- # - "auto_round" / "auto_round:auto_gptq" → auto-round style config- # (produced by _build_mxfp_autoround_quantization_config).- # Also route to MXFP config when the default is full-precision (BF16/FP16)- # but layer_config contains MXFP overrides.- data_type = (default_scheme.get("data_type") or "int").lower()- default_bits = default_scheme.get("bits", 4)- is_fp_default = (default_bits or 0) >= 16 and not is_mx_fp(data_type)- if is_mx_fp(data_type) or (is_fp_default and _layer_config_has_mxfp(layer_config)):- if format in ("auto_round", "auto_round:auto_gptq"):- return _build_mxfp_autoround_quantization_config(- default_scheme=default_scheme,- quantized_layers=quantized_layers,- ignored_layers=ignored_layers,- layer_config=layer_config,- block_name_to_quantize=block_name_to_quantize,- )- return _build_mxfp_quantization_config(- default_scheme=default_scheme,- quantized_layers=quantized_layers,- ignored_layers=ignored_layers,- layer_config=layer_config,- )-- # When the default is full-precision (BF16/FP16) but INT layers were- # quantized via layer_config, derive the dominant INT scheme and use it as- # the top-level defaults so the output config reads as- # "INT-quantized with BF16 exceptions" rather than the inverse.- if is_fp_default and quantized_layers:- dominant = _derive_dominant_int_scheme(quantized_layers, layer_config, default_scheme)- if dominant is not None:- default_scheme = dominant-- from auto_round.version import __version__-- scheme_keys = [f.name for f in fields(QuantizationScheme)]- # vllm only support auto_round:auto_gptq, but transformers cannot load it correctly when sym=False.- # So we keep auto_round for asymmetric quantization to maintain compatibility with both.- packing_format = "auto_round:auto_gptq" if default_scheme.get("sym", True) else "auto_round"-- qconfig = {- "quant_method": "auto-round",- "packing_format": packing_format,- "bits": default_scheme["bits"],- "group_size": default_scheme["group_size"],- "sym": default_scheme.get("sym", True),- "data_type": default_scheme.get("data_type", "int"),- "iters": 0,- "model_free": True,- "autoround_version": __version__,- }-- if block_name_to_quantize:- qconfig["block_name_to_quantize"] = block_name_to_quantize-- extra_config = {}- for layer_name, cfg in layer_config.items():- if cfg.get("bits", default_scheme["bits"]) >= 16:- extra_config[layer_name] = {k: cfg.get(k) for k in scheme_keys if cfg.get(k) is not None}- continue- differs = False- for key in ("bits", "group_size", "sym"):- if cfg.get(key) is not None and cfg[key] != default_scheme.get(key):- differs = True- break- if differs:- extra_config[layer_name] = {k: cfg.get(k) for k in scheme_keys if cfg.get(k) is not None}-- # Filter out non-Linear ops (embed, conv) that don't need to be recorded in config.- # Routing gates and other predefined patterns are still recorded.- non_linear_ops = ["embed", "conv"]- non_linear_re = re.compile("|".join(re.escape(op) for op in non_linear_ops))-- unique_ignored = list(dict.fromkeys(ignored_layers))- for layer_name in unique_ignored:- if layer_name not in extra_config:- # Skip non-Linear ops (embed, conv) since they're not Linear layers- if non_linear_re.search(layer_name):- continue- extra_config[layer_name] = {"bits": 16, "data_type": "float"}-- quantized_layer_set = set(quantized_layers)- if "lm_head" in quantized_layer_set and "lm_head" not in extra_config:- lm_head_cfg = layer_config.get("lm_head", default_scheme)- extra_config["lm_head"] = {k: lm_head_cfg.get(k) for k in scheme_keys if lm_head_cfg.get(k) is not None}-- if extra_config:- qconfig["extra_config"] = extra_config-- return qconfig---def _write_output_shard(- output_dir: str,- shard_name: str,- tensors: dict[str, torch.Tensor],- weight_map: dict[str, str],-):- """Write a single output shard and update the weight_map."""- from safetensors.torch import save_file-- shard_path = os.path.join(output_dir, shard_name)-- # Detect shared-storage tensors (e.g. tie_word_embeddings: wte ↔ lm_head).- # safetensors refuses to serialise them as-is; clone the duplicates so each- # tensor occupies its own memory region. The first occurrence keeps the- # original storage; subsequent aliases are cloned.- seen_data_ptrs: set[int] = set()- deduped: dict[str, torch.Tensor] = {}- for k, v in tensors.items():- if not v.is_contiguous():- v = v.contiguous()- ptr = v.data_ptr()- if ptr in seen_data_ptrs:- v = v.clone()- else:- seen_data_ptrs.add(ptr)- deduped[k] = v-- save_file(deduped, shard_path)- for tensor_name in tensors:- weight_map[tensor_name] = shard_name---def _write_index_file(output_dir: str, weight_map: dict[str, str]):- """Write model.safetensors.index.json (or rename single shard)."""- if len(set(weight_map.values())) <= 1:- shard_names = list(set(weight_map.values()))- if shard_names and shard_names[0] != "model.safetensors":- src = os.path.join(output_dir, shard_names[0])- dst = os.path.join(output_dir, "model.safetensors")- if os.path.exists(src):- os.rename(src, dst)- weight_map = {k: "model.safetensors" for k in weight_map}- return-- index = {"metadata": {"total_size": 0}, "weight_map": weight_map}- with open(os.path.join(output_dir, "model.safetensors.index.json"), "w") as f:- json.dump(index, f, indent=2)--def _prefetch_shard(model_name_or_path: str,shard_name: str,@@ -1833,11 +278,12 @@ def _process_single_shard_task(ignore_patterns: list[str],fp8_block_size: list | None,model_type: str | None,+ source_quantization_config: dict | None = None,quant_output_dir: str,total_shards: int,- source_quant_config: dict | None = None,enable_torch_compile: bool = False,disable_opt_rtn: bool = False,+ donor_tensors_to_exclude: list[str] | None = None,) -> tuple[int, str, str | None, str | None, list[str] | None, list[str] | None, list[str] | None]:"""Process one shard in an isolated subprocess task.@@ -1854,6 +300,54 @@ def _process_single_shard_task(if shard_path is None or not os.path.exists(shard_path):return shard_idx, shard_name, None, None, None, None, None+ return _quantize_local_shard_task(+ shard_idx,+ shard_name,+ shard_path=shard_path,+ device=device,+ default_scheme=default_scheme,+ layer_config=layer_config,+ ignore_patterns=ignore_patterns,+ fp8_block_size=fp8_block_size,+ model_type=model_type,+ source_quantization_config=source_quantization_config,+ quant_output_dir=quant_output_dir,+ total_shards=total_shards,+ enable_torch_compile=enable_torch_compile,+ disable_opt_rtn=disable_opt_rtn,+ cleanup_source_shard=is_streaming,+ donor_tensors_to_exclude=donor_tensors_to_exclude,+ )+++def _quantize_local_shard_task(+ shard_idx: int,+ shard_name: str,+ *,+ shard_path: str,+ device: str,+ default_scheme: dict,+ layer_config: dict,+ ignore_patterns: list[str],+ fp8_block_size: list | None,+ model_type: str | None,+ source_quantization_config: dict | None,+ quant_output_dir: str,+ total_shards: int,+ enable_torch_compile: bool = False,+ disable_opt_rtn: bool = False,+ cleanup_source_shard: bool = False,+ donor_tensors_to_exclude: list[str] | None = None,+ index_dir: str | None = None,+ donor_shard_dir: str | None = None,+) -> tuple[int, str, str | None, str | None, list[str] | None, list[str] | None, list[str] | None]:+ """Quantize one already-downloaded shard and write the output shard.++ Returns lightweight metadata only so IPC does not transfer tensor storages.+ """+ if shard_path is None or not os.path.exists(shard_path):+ return shard_idx, shard_name, None, None, None, None, None+output_tensors, quantized, ignored = _process_shard(shard_path=shard_path,shard_name=shard_name,@@ -1863,9 +357,12 @@ def _process_single_shard_task(device=device,fp8_block_size=fp8_block_size,model_type=model_type,- source_quant_config=source_quant_config,+ source_quantization_config=source_quantization_config,enable_torch_compile=enable_torch_compile,disable_opt_rtn=disable_opt_rtn,+ index_dir=index_dir,+ donor_shard_dir=donor_shard_dir,+ donor_tensors_to_exclude=set(donor_tensors_to_exclude) if donor_tensors_to_exclude else None,)out_shard_name = f"model-{shard_idx + 1:05d}-of-{total_shards:05d}.safetensors"@@ -1879,13 +376,12 @@ def _process_single_shard_task(tensor_names = list(local_weight_map.keys())clear_memory()- if is_streaming:+ if cleanup_source_shard:try:os.remove(shard_path)except OSError:pass- # Return only lightweight metadata to avoid IPC transfer of tensor storages.return shard_idx, shard_name, shard_path, out_shard_name, tensor_names, quantized, ignored@@ -1921,293 +417,6 @@ def _force_cleanup_process_pool(pool: ProcessPoolExecutor | None) -> None:# ----------------------------------------------------------------------------def _normalize_scheme(scheme: Union[str, QuantizationScheme]) -> QuantizationScheme:- """Convert *scheme* to a :class:`QuantizationScheme` instance.-- Raises ``ValueError`` for unknown preset names and ``TypeError`` for- unsupported types.- """- if isinstance(scheme, str):- scheme_name = scheme.upper()- if scheme_name not in PRESET_SCHEMES:- raise ValueError(f"Unknown scheme '{scheme}'. Available: {list(PRESET_SCHEMES.keys())}")- return preset_name_to_scheme(scheme_name)- if isinstance(scheme, QuantizationScheme):- return scheme- raise TypeError(f"Unsupported scheme type: {type(scheme)}")---def _apply_scheme_overrides(- scheme: Union[str, QuantizationScheme],- scheme_overrides: Optional[dict] = None,-) -> QuantizationScheme:- """Return the effective scheme after applying non-None overrides."""- scheme_obj = copy.deepcopy(_normalize_scheme(scheme))- if not scheme_overrides:- return scheme_obj-- valid_fields = {field.name for field in fields(QuantizationScheme)}- for key, value in scheme_overrides.items():- if key in valid_fields and value is not None:- setattr(scheme_obj, key, value)- return scheme_obj---def _validate_supported_scheme(- scheme_obj: QuantizationScheme,- scheme_input: Union[str, QuantizationScheme],-) -> None:- """Raise ``ValueError`` if *scheme_obj* is not supported by model-free.-- Model-free supports:-- * Integer weight-only quantization (sym/asym), ``bits ∈ {2, 4, 8}``,- packed in the ``auto_round:auto_gptq`` format.- * MXFP weight quantization (``data_type='mx_fp'``), ``bits ∈ {4, 8}``,- ``group_size=32``, packed in ``mxfp4-pack-quantized`` / ``mxfp8-quantized``- format (compressed-tensors compatible).- """- data_type = (scheme_obj.data_type or "int").lower()- bits = scheme_obj.bits- act_bits = scheme_obj.act_bits if scheme_obj.act_bits is not None else 16-- # Full-precision (BF16/FP16) default: all layers stay in full precision- # unless layer_config provides lower-bit overrides.- if (bits or 0) >= 16 and act_bits >= 16:- return-- # MXFP weight-only path: accept mx_fp data type with bits in {4, 8}.- # Activation quantization for MXFP is dynamic at inference time, so the- # weight-only RTN path here is independent of act_bits.- if is_mx_fp(data_type):- if scheme_obj.act_data_type not in (None, "mx_fp"):- raise ValueError(- "Model-free MXFP supports only act_data_type='mx_fp', " f"but got '{scheme_obj.act_data_type}'."- )- # Restrict to the two explicitly supported MXFP presets when a string- # name is provided. Variants such as MXFP4_RCEIL / MXFP8_RCEIL use a- # different activation format; silently mapping them to "MXFP4" /- # "MXFP8" in the output config would misrepresent the requested scheme.- if isinstance(scheme_input, str) and scheme_input.upper() not in ("MXFP4", "MXFP8"):- raise ValueError(- f"Model-free mode only supports MXFP preset names 'MXFP4' and 'MXFP8', "- f"but got '{scheme_input}'. "- f"Supported preset schemes: {list(SUPPORTED_PRESET_SCHEMES)}."- )- if bits is None or bits not in _SUPPORTED_MXFP_BITS:- raise ValueError(- f"Model-free mode supports MXFP bits in {_SUPPORTED_MXFP_BITS}, "- f"but '{scheme_input}' requests bits={bits}. "- f"Supported preset schemes: {list(SUPPORTED_PRESET_SCHEMES)}."- )- group_size = scheme_obj.group_size- if group_size not in (None, 32):- raise ValueError(- f"Model-free mode supports MXFP only with group_size=32, "- f"but '{scheme_input}' requests group_size={group_size}."- )- return-- if act_bits < 16:- raise ValueError(- f"Model-free mode only supports weight-only quantization (WOQ) schemes "- f"where act_bits >= 16, but '{scheme_input}' has act_bits={act_bits}. "- f"Supported preset schemes: {list(SUPPORTED_PRESET_SCHEMES)}."- )-- if data_type != "int":- raise ValueError(- f"Model-free mode only supports integer weight quantization "- f"(data_type='int') or MXFP (data_type='mx_fp'), but '{scheme_input}' "- f"has data_type='{data_type}'. FP8 / NVFP / GGUF / BF16 schemes require "- f"the standard AutoRound flow. Supported preset schemes: "- f"{list(SUPPORTED_PRESET_SCHEMES)}."- )-- if bits is None or bits not in _SUPPORTED_INT_BITS:- raise ValueError(- f"Model-free mode supports bits in {_SUPPORTED_INT_BITS}, "- f"but '{scheme_input}' requests bits={bits}. "- f"Supported preset schemes: {list(SUPPORTED_PRESET_SCHEMES)}."- )---def is_model_free_supported_scheme(- scheme: Union[str, QuantizationScheme],- scheme_overrides: Optional[dict] = None,-) -> bool:- """Return True if *scheme* can be quantized via model-free mode.-- Useful for CLI auto-routing logic. Never raises.- """- try:- scheme_obj = _apply_scheme_overrides(scheme, scheme_overrides)- _validate_supported_scheme(scheme_obj, scheme)- return True- except (ValueError, TypeError):- return False---# ----------------------------------------------------------------------------# AutoScheme support (two-phase: delta-loss selection + model-free packing)-# ------------------------------------------------------------------------------def _looks_like_auto_scheme(scheme: Any) -> bool:- """Duck-typed check for an :class:`~auto_round.auto_scheme.AutoScheme`.-- Avoids importing ``AutoScheme`` at module scope (it pulls in exporter /- compressor modules that would create an import cycle with this file).- """- return hasattr(scheme, "options") and hasattr(scheme, "avg_bits")---def _validate_auto_scheme_options(auto_scheme: Any) -> str:- """Validate AutoScheme options for model-free packing.-- Returns the single quantized data-type family shared by all options- (``"int"`` or ``"mx_fp"``). ``BF16`` (or any >=16-bit no-op scheme) is- allowed and treated as "keep layer in full precision" during conversion,- so it does not contribute to the returned family.-- Raises ``ValueError`` when any non-BF16 option is unsupported or when INT- and MXFP options are mixed (they use different packing formats and cannot- be produced in one model-free run).- """- options = list(getattr(auto_scheme, "options", []) or [])- if not options:- raise ValueError("AutoScheme.options must be non-empty for model-free mode.")-- families: set[str] = set()- unsupported: list[Any] = []- for opt in options:- # Preserve original string validation semantics so preset-name- # restrictions (e.g. MXFP4/MXFP8 only) are enforced.- if isinstance(opt, str):- try:- scheme_obj = _normalize_scheme(opt)- except (ValueError, TypeError):- scheme_obj = None- elif isinstance(opt, QuantizationScheme):- scheme_obj = opt- else:- scheme_obj = None-- # AutoScheme may include BF16/no-op options. In model-free flow these- # layers become ignore_layers entries (full precision), so they are- # allowed and excluded from family checks.- if scheme_obj is not None:- act_bits = scheme_obj.act_bits if scheme_obj.act_bits is not None else 16- if (scheme_obj.bits or 0) >= 16 and act_bits >= 16:- continue-- # GGUF k-quants carry super_bits and are not packable by the model-free- # RTN kernel even though their data_type is nominally "int".- if scheme_obj is None or getattr(scheme_obj, "super_bits", None) is not None:- unsupported.append(opt)- continue- if not is_model_free_supported_scheme(opt):- unsupported.append(opt)- continue-- data_type = (scheme_obj.data_type or "int").lower()- families.add("mx_fp" if is_mx_fp(data_type) else "int")-- if unsupported:- raise ValueError(- f"Model-free + AutoScheme received unsupported option(s): {unsupported}. "- f"Model-free supports INT WOQ (bits in {_SUPPORTED_INT_BITS}) and MXFP "- f"(bits in {_SUPPORTED_MXFP_BITS}); GGUF / NVFP4 / FP8 options are not "- f"packable in model-free mode. Remove the unsupported options or pass "- f"disable_model_free=True to use the regular flow."- )- if len(families) > 1:- raise ValueError(- "Model-free + AutoScheme cannot mix INT and MXFP options in a single run "- f"(got families {sorted(families)}); INT and MXFP use different packing "- "formats. Use a single data-type family, or pass disable_model_free=True."- )- return families.pop()---def _convert_auto_scheme_layer_config(- generated: dict[str, dict],- preferred_base_scheme: Union[str, QuantizationScheme, None] = None,-) -> tuple[QuantizationScheme, dict[str, dict], list[str]]:- """Convert an AutoScheme-generated ``layer_config`` into model-free inputs.-- Returns ``(base_scheme, per_layer_overrides, fp16_layers)`` where:-- * ``base_scheme`` is ``preferred_base_scheme`` when provided, matching the- primary scheme selected by the regular AutoRound path. Otherwise the- most common quantized scheme is used as a fallback.- * ``per_layer_overrides`` maps every quantized layer name to its resolved- :class:`QuantizationScheme` fields.- * ``fp16_layers`` lists layers AutoScheme kept at >= 16 bits (added to the- model-free ignore list so they stay in full precision).- """- from collections import Counter-- scheme_keys = {f.name for f in fields(QuantizationScheme)}- per_layer: dict[str, dict] = {}- fp16_layers: list[str] = []- counter: "Counter[tuple]" = Counter()-- for name, cfg in generated.items():- if not isinstance(cfg, dict):- continue- bits = cfg.get("bits")- data_type_raw = (cfg.get("data_type") or "").strip()-- # Infer bits from compact dtype aliases that embed bit-width in the name,- # e.g. "mxfp8" → 8, "MXFP4" → 4, "mx_fp4" → 4.- if bits is None and data_type_raw:- dt_lower = data_type_raw.lower()- for prefix in ("mxfp", "mx_fp", "nvfp", "nv_fp"):- tail = dt_lower[len(prefix) :]- if dt_lower.startswith(prefix) and tail.isdigit():- bits = int(tail)- break-- if bits is None:- continue-- clean = {k: cfg[k] for k in scheme_keys if cfg.get(k) is not None}- clean["bits"] = bits # honour inferred value when not explicit-- # Normalise compact dtype aliases to canonical forms understood by- # the quantization kernels ("mxfp8" / "MXFP4" → "mx_fp").- if data_type_raw:- dt_lower = data_type_raw.lower()- if dt_lower.startswith("mxfp") or dt_lower.startswith("mx_fp"):- clean["data_type"] = "mx_fp"- elif dt_lower.startswith("nvfp") or dt_lower.startswith("nv_fp"):- clean["data_type"] = "nv_fp"-- if bits >= 16:- fp16_layers.append(name)- continue- data_type = (clean.get("data_type") or "int").lower()- per_layer[name] = clean- counter[(clean.get("bits"), clean.get("group_size"), bool(clean.get("sym", True)), data_type)] += 1-- if not counter:- raise ValueError("AutoScheme did not assign any quantizable layers for model-free mode.")-- if preferred_base_scheme is not None:- base_scheme = copy.deepcopy(_normalize_scheme(preferred_base_scheme))- else:- (base_bits, base_group_size, base_sym, base_dtype), _ = counter.most_common(1)[0]- base_scheme = QuantizationScheme(- bits=base_bits,- group_size=base_group_size,- sym=base_sym,- data_type=base_dtype,- )- return base_scheme, per_layer, fp16_layers---# ---------------------------------------------------------------------------# Main class# ---------------------------------------------------------------------------@@ -2250,6 +459,7 @@ class _ModelFreeCompressorCore:"""SUPPORTED_FORMATS: tuple[str, ...] = (+ "fake","auto_round","auto_round:auto_gptq","llm_compressor",@@ -2276,7 +486,7 @@ class _ModelFreeCompressorCore:self.scheme_input = schemeself.layer_config_input = layer_configself.ignore_layers_input = ignore_layers- self.format = format+ self.format = format or "auto_round"self.device = deviceself.quant_lm_head = quant_lm_headself.quant_nontext_module = quant_nontext_module@@ -2305,6 +515,19 @@ class _ModelFreeCompressorCore:self.all_ignored_layers: list[str] = []self.output_weight_map: dict[str, str] = {}self.shard_parallelism: int = 1+ # Cross-shard FP8 scale_inv dependency maps built from index.json.+ # cross_shard_deps: {recipient_shard: {donor_shard: [scale_inv_names]}}+ # donor_shard_tensors: {donor_shard: set(scale_inv names it donates)}+ self.cross_shard_deps: dict[str, dict[str, list[str]]] = {}+ self.donor_shard_tensors: dict[str, set[str]] = {}+ # Reference-counting state so donor shard cache files can be deleted as+ # soon as they are no longer needed, instead of waiting until the end+ # of the whole run. A donor file is safe to delete once (a) its own+ # quantization task has read it, and (b) every recipient shard that+ # depends on it has finished (and thus finished hydrating from it).+ self._donor_remaining_recipients: dict[str, int] = {}+ self._donor_self_consumed: dict[str, bool] = {}+ self._donor_shard_paths: dict[str, str] = {}# -------------------------------------------------------------------# Validation / parsing@@ -2333,6 +556,7 @@ class _ModelFreeCompressorCore:_validate_supported_scheme(self.scheme_obj, self.scheme_input)ds = asdict(self.scheme_obj)self.default_scheme = {k: v for k, v in ds.items() if v is not None}+ self.default_scheme["_output_format"] = self.formatdef _parse_layer_config(self) -> None:lc = copy.deepcopy(self.layer_config_input) if self.layer_config_input else {}@@ -2402,9 +626,19 @@ class _ModelFreeCompressorCore:def _resolve_source(self) -> None:os.makedirs(self.output_dir, exist_ok=True)- self.is_streaming = not _is_model_cached(self.model_name_or_path)+ cached, cache_reason = _get_model_cache_status(self.model_name_or_path)+ self.is_streaming = not cached+ logger.info(+ "Model-free source decision: %s (%s)",+ "streaming" if self.is_streaming else "local/snapshot",+ cache_reason,+ )if self.is_streaming:- logger.info("Model not found locally or in cache — using streaming download mode.")+ logger.info(+ "Path selected: streaming mode. Will download metadata first "+ "(snapshot_download with weight ignore patterns), then fetch "+ "weight shards on demand (hf_hub_download)."+ )self.work_dir = self.output_dir_download_metadata_files(self.model_name_or_path, self.work_dir)transformer_work_dir = os.path.join(self.work_dir, "transformer")@@ -2422,6 +656,10 @@ class _ModelFreeCompressorCore:)self.config = _load_config(self.work_dir)else:+ if os.path.isdir(self.model_name_or_path):+ logger.info("Path selected: local directory mode. Reading shards from local path.")+ else:+ logger.info("Path selected: snapshot/local-cache mode. Resolving source dir via snapshot_download.")self.source_dir = _resolve_source_dir(self.model_name_or_path)transformer_source_dir = os.path.join(self.source_dir, "transformer")if (@@ -2505,6 +743,135 @@ class _ModelFreeCompressorCore:search_dir = self.work_dir if self.is_streaming else self.source_dirself.shard_names = _list_weight_shards(search_dir)+ def _build_cross_shard_deps(self) -> None:+ """Build cross-shard FP8 scale_inv dependency map from index.json.++ Identifies triples (recipient_shard, donor_shard, scale_inv_tensor) by+ comparing ``.weight`` and ``.weight_scale_inv`` entries in weight_map.+ Stored in ``self.cross_shard_deps`` and ``self.donor_shard_tensors`` for+ use by the shard scheduler and quantization workers.+ """+ search_dir = self.work_dir if self.is_streaming else self.source_dir+ # Find the first *.safetensors.index.json in search_dir.+ index_path: str | None = None+ st_std = os.path.join(search_dir, "model.safetensors.index.json")+ if os.path.exists(st_std):+ index_path = st_std+ else:+ candidates = sorted(+ os.path.join(search_dir, f) for f in os.listdir(search_dir) if f.endswith(".safetensors.index.json")+ )+ if candidates:+ index_path = candidates[0]++ if not index_path:+ self.cross_shard_deps = {}+ self.donor_shard_tensors = {}+ return++ try:+ weight_map = _load_weight_map_from_index(index_path)+ except Exception as e:+ logger.warning(f"Could not load weight_map for cross-shard dep analysis: {e}")+ self.cross_shard_deps = {}+ self.donor_shard_tensors = {}+ return++ self.cross_shard_deps, self.donor_shard_tensors = _build_cross_shard_pairs_from_weight_map(weight_map)+ if self.cross_shard_deps:+ n_pairs = sum(len(t) for donors in self.cross_shard_deps.values() for t in donors.values())+ logger.info(+ f"Cross-shard FP8 dependencies: {n_pairs} scale_inv tensor(s), "+ f"{len(self.cross_shard_deps)} recipient shard(s), "+ f"{len(self.donor_shard_tensors)} donor shard(s)."+ )++ # Build reverse mapping (donor -> distinct recipient shards) so donor+ # cache files can be reference-counted and deleted as soon as every+ # dependent recipient has finished, instead of only at the very end.+ donor_recipients: dict[str, set[str]] = {}+ for recipient, donors in self.cross_shard_deps.items():+ for donor in donors:+ donor_recipients.setdefault(donor, set()).add(recipient)+ self._donor_remaining_recipients = {donor: len(recipients) for donor, recipients in donor_recipients.items()}+ self._donor_self_consumed = {donor: False for donor in self.donor_shard_tensors}++ def _reorder_shards_by_dependency(self) -> None:+ """Reorder shard_names via topological sort so donors precede recipients.++ A single-pass "donors first, recipients last" partition is not enough:+ a shard can simultaneously be a *recipient* (it needs a scale_inv from+ another shard) and a *donor* (it donates a scale_inv to a third shard).+ Such transitive chains (X donates to Y, Y donates to Z) must be+ resolved with a proper topological sort, otherwise Y could still be+ scheduled after Z, i.e. after one of its own dependents.++ Ties are broken by original shard order (stable) so scheduling stays+ as close as possible to the natural shard sequence. This ensures:+ - Non-streaming: workers start on shards that need no cross-shard data.+ - Streaming: donor shards are downloaded before recipient shards, so the+ donor cache files are available when recipient workers run hydration.+ """+ if not self.cross_shard_deps:+ return++ original_index = {name: i for i, name in enumerate(self.shard_names)}+ # recipient -> set(donors it depends on)+ depends_on: dict[str, set[str]] = {+ recipient: set(donors) for recipient, donors in self.cross_shard_deps.items()+ }+ # in-degree = number of unresolved donor dependencies for each shard.+ in_degree: dict[str, int] = {name: len(depends_on.get(name, ())) for name in self.shard_names}+ # donor -> shards that depend on it (only counting donors that are+ # themselves known shards; unknown donors can't be waited on anyway).+ dependents: dict[str, list[str]] = {}+ for recipient, donors in depends_on.items():+ for donor in donors:+ if donor in original_index:+ dependents.setdefault(donor, []).append(recipient)+ elif recipient in in_degree:+ # Donor shard is unknown/missing from discovery; it can+ # never be satisfied here, so don't block scheduling on it.+ in_degree[recipient] -= 1++ import heapq++ ready = [original_index[name] for name, deg in in_degree.items() if deg == 0]+ heapq.heapify(ready)+ ordered: list[str] = []+ visited: set[str] = set()+ index_to_name = {i: name for name, i in original_index.items()}++ while ready:+ idx = heapq.heappop(ready)+ name = index_to_name[idx]+ if name in visited:+ continue+ visited.add(name)+ ordered.append(name)+ for dependent in dependents.get(name, ()):+ in_degree[dependent] -= 1+ if in_degree[dependent] == 0:+ heapq.heappush(ready, original_index[dependent])++ if len(ordered) != len(self.shard_names):+ # Cycle detected (shouldn't happen with well-formed indices) —+ # fall back to appending any unresolved shards in original order.+ remaining = [name for name in self.shard_names if name not in visited]+ logger.warning(+ f"Shard dependency graph has a cycle or unresolved node(s) "+ f"({len(remaining)} shard(s)); appending them in original order."+ )+ ordered.extend(remaining)++ n_recipients = len(self.cross_shard_deps)+ if n_recipients:+ logger.info(+ f"Shard scheduling: topologically sorted {len(self.shard_names)} shard(s) "+ f"({n_recipients} recipient shard(s) with cross-shard dependencies)."+ )+ self.shard_names = ordered+def _resolve_shard_parallelism(self) -> tuple[int, str]:shard_count = len(self.shard_names)# Auto policy: shard_count // 4, capped at 10, minimum 1.@@ -2543,6 +910,10 @@ class _ModelFreeCompressorCore:# -------------------------------------------------------------------def _process_all_shards(self) -> None:+ if self.is_streaming:+ self._process_all_shards_streaming_pipeline()+ return+try:from tqdm import tqdm as _tqdmexcept ImportError:@@ -2559,6 +930,7 @@ class _ModelFreeCompressorCore:try:pool = ProcessPoolExecutor(max_workers=worker_count, mp_context=mp.get_context("spawn"))for shard_idx, shard_name in enumerate(self.shard_names):+ donor_tensors = list(self.donor_shard_tensors.get(shard_name, ())) or Nonefutures.append(pool.submit(_process_single_shard_task,@@ -2574,11 +946,12 @@ class _ModelFreeCompressorCore:ignore_patterns=self.ignore_patterns,fp8_block_size=self.fp8_block_size,model_type=self.model_type,- source_quant_config=self.config.get("quantization_config"),+ source_quantization_config=self.config.get("quantization_config", {}),enable_torch_compile=self.enable_torch_compile,disable_opt_rtn=self.disable_opt_rtn,quant_output_dir=self._quant_output_dir,total_shards=len(self.shard_names),+ donor_tensors_to_exclude=donor_tensors,))@@ -2589,36 +962,8 @@ class _ModelFreeCompressorCore:)for future in shard_iter:- shard_idx, shard_name, shard_path, out_shard_name, tensor_names, quantized, ignored = future.result()-- if (- shard_path is None- or out_shard_name is None- or tensor_names is None- or quantized is None- or ignored is None- ):- logger.warning(f"Shard not found: {shard_name}, skipping")- continue-- memory_monitor.update()- clear_memory()- if len(self.shard_names) > 1:- logger.info(f"Memory usage: {memory_monitor.get_summary()}")-- compressed_quantized = compress_layer_names(quantized)- compressed_ignored = compress_layer_names(ignored)- logger.info(- f"Shard {shard_idx + 1}/{len(self.shard_names)} ({shard_name}):\n"- f" Quantized layers ({len(quantized)}): {compressed_quantized}\n"- f" Ignored layers ({len(ignored)}): {compressed_ignored}"- )-- self.all_quantized_layers.extend(quantized)- self.all_ignored_layers.extend(ignored)-- for tensor_name in tensor_names:- self.output_weight_map[tensor_name] = out_shard_name+ result = future.result()+ self._merge_shard_task_result(result)except KeyboardInterrupt:logger.warning("Interrupted by user; terminating model-free shard worker processes.")_force_cleanup_process_pool(pool)@@ -2629,6 +974,212 @@ class _ModelFreeCompressorCore:finally:_force_cleanup_process_pool(pool)+ def _merge_shard_task_result(+ self,+ result: tuple[int, str, str | None, str | None, list[str] | None, list[str] | None, list[str] | None],+ ) -> None:+ """Merge one shard-task result into global stats and weight map."""+ shard_idx, shard_name, shard_path, out_shard_name, tensor_names, quantized, ignored = result+ if shard_path is None or out_shard_name is None or tensor_names is None or quantized is None or ignored is None:+ logger.warning(f"Shard not found: {shard_name}, skipping")+ return++ memory_monitor.update()+ clear_memory()+ if len(self.shard_names) > 1:+ logger.info(f"Memory usage: {memory_monitor.get_summary()}")++ compressed_quantized = compress_layer_names(quantized)+ compressed_ignored = compress_layer_names(ignored)+ logger.info(+ f"Shard {shard_idx + 1}/{len(self.shard_names)} ({shard_name}):\n"+ f" Quantized layers ({len(quantized)}): {compressed_quantized}\n"+ f" Ignored layers ({len(ignored)}): {compressed_ignored}"+ )++ self.all_quantized_layers.extend(quantized)+ self.all_ignored_layers.extend(ignored)+ for tensor_name in tensor_names:+ self.output_weight_map[tensor_name] = out_shard_name++ if self.is_streaming:+ self._release_donor_dependency(shard_name)++ def _release_donor_dependency(self, completed_shard_name: str) -> None:+ """Update donor reference counts after *completed_shard_name* finishes.++ Donor shard cache files are kept alive only as long as needed: once a+ donor's own quantization task has read it AND every recipient shard+ that depends on it has finished (and thus hydrated from it), the+ cached file is deleted immediately rather than waiting for the whole+ run to finish via :meth:`_cleanup_streaming_shard_cache`.+ """+ if completed_shard_name in self._donor_self_consumed:+ self._donor_self_consumed[completed_shard_name] = True+ self._maybe_cleanup_donor_shard(completed_shard_name)++ for donor in self.cross_shard_deps.get(completed_shard_name, {}):+ if donor in self._donor_remaining_recipients:+ self._donor_remaining_recipients[donor] = max(0, self._donor_remaining_recipients[donor] - 1)+ self._maybe_cleanup_donor_shard(donor)++ def _maybe_cleanup_donor_shard(self, donor_name: str) -> None:+ """Delete *donor_name*'s cached shard file once it is no longer needed."""+ if not self._donor_self_consumed.get(donor_name, False):+ return+ if self._donor_remaining_recipients.get(donor_name, 0) > 0:+ return+ shard_path = self._donor_shard_paths.pop(donor_name, None)+ # Remove tracking regardless of whether the file still exists so we+ # never attempt to clean this donor up twice.+ self._donor_self_consumed.pop(donor_name, None)+ self._donor_remaining_recipients.pop(donor_name, None)+ if shard_path and os.path.exists(shard_path):+ try:+ os.remove(shard_path)+ logger.debug(f"Removed donor shard cache file (no longer needed): {donor_name}")+ except OSError as e:+ logger.warning(f"Failed to remove donor shard cache file {donor_name}: {e}")++ def _process_all_shards_streaming_pipeline(self) -> None:+ """Streaming-mode shard pipeline with dedicated downloader and quant workers.++ Design:+ - one downloader worker serializes network bandwidth usage;+ - N quant workers consume ready shards independently;+ - shards are assigned for quantization as soon as download completes.+ """+ try:+ from tqdm import tqdm as _tqdm+ except ImportError:+ _tqdm = None++ if not self.shard_names:+ return++ os.makedirs(self._quant_output_dir, exist_ok=True)++ worker_count = max(1, min(self.shard_parallelism, len(self.shard_names)))+ prefetch_depth = max(2, worker_count)+ total_shards = len(self.shard_names)++ download_pool: ThreadPoolExecutor | None = None+ quant_pool: ProcessPoolExecutor | None = None+ download_futures: dict = {}+ quant_futures = set()+ next_download_idx = 0+ completed_quant = 0++ def _submit_next_download() -> bool:+ nonlocal next_download_idx+ if next_download_idx >= total_shards:+ return False+ shard_idx = next_download_idx+ shard_name = self.shard_names[shard_idx]+ future = download_pool.submit(+ _prefetch_shard,+ self.model_name_or_path,+ shard_name,+ self.work_dir,+ self.source_dir,+ self.is_streaming,+ )+ download_futures[future] = (shard_idx, shard_name)+ next_download_idx += 1+ return True++ try:+ download_pool = ThreadPoolExecutor(max_workers=1)+ quant_pool = ProcessPoolExecutor(max_workers=worker_count, mp_context=mp.get_context("spawn"))++ for _ in range(min(prefetch_depth, total_shards)):+ _submit_next_download()++ progress = _tqdm(total=total_shards, desc="Processing shards", unit="shard") if _tqdm else None++ while completed_quant < total_shards:+ wait_set = set(download_futures.keys()) | set(quant_futures)+ if not wait_set:+ break++ done, _ = wait(wait_set, return_when=FIRST_COMPLETED)+ for future in done:+ if future in download_futures:+ shard_idx, shard_name = download_futures.pop(future)+ shard_path = future.result()+ if shard_path is None or not os.path.exists(shard_path):+ logger.warning(f"Prefetch failed for shard {shard_name}, skipping")+ completed_quant += 1+ if progress is not None:+ progress.update(1)+ else:+ # Donor shards must stay on disk until every recipient+ # that depends on them has been processed (tracked via+ # _donor_remaining_recipients / _release_donor_dependency),+ # so that recipient shards can still read scale_inv+ # tensors from them via+ # _hydrate_missing_fp8_scales_from_index. They are+ # deleted as soon as that's no longer needed; any+ # stragglers are swept up at the end by+ # _cleanup_streaming_shard_cache.+ shard_cache_dir = os.path.join(self.work_dir, ".cache", "model_free_source_shards")+ is_donor = shard_name in self.donor_shard_tensors+ if is_donor:+ self._donor_shard_paths[shard_name] = shard_path+ donor_tensors = list(self.donor_shard_tensors.get(shard_name, ())) or None+ qf = quant_pool.submit(+ _quantize_local_shard_task,+ shard_idx,+ shard_name,+ shard_path=shard_path,+ device=self.device,+ default_scheme=self.default_scheme,+ layer_config=self.layer_config,+ ignore_patterns=self.ignore_patterns,+ fp8_block_size=self.fp8_block_size,+ model_type=self.model_type,+ source_quantization_config=self.config.get("quantization_config", {}),+ quant_output_dir=self._quant_output_dir,+ total_shards=total_shards,+ enable_torch_compile=self.enable_torch_compile,+ # Keep donor shards alive for recipient hydration.+ cleanup_source_shard=not is_donor,+ donor_tensors_to_exclude=donor_tensors,+ # Pass the work_dir as index_dir so workers can+ # find the index.json that was downloaded during+ # metadata fetch (not in the shard cache sub-dir).+ index_dir=self.work_dir,+ donor_shard_dir=shard_cache_dir,+ )+ quant_futures.add(qf)++ while len(download_futures) < prefetch_depth and _submit_next_download():+ pass+ elif future in quant_futures:+ quant_futures.remove(future)+ result = future.result()+ self._merge_shard_task_result(result)+ completed_quant += 1+ if progress is not None:+ progress.update(1)++ if progress is not None:+ progress.close()+ except KeyboardInterrupt:+ logger.warning("Interrupted by user; terminating model-free shard workers.")+ _force_cleanup_process_pool(quant_pool)+ raise+ except Exception:+ _force_cleanup_process_pool(quant_pool)+ raise+ finally:+ _force_cleanup_process_pool(quant_pool)+ if download_pool is not None:+ try:+ download_pool.shutdown(wait=False, cancel_futures=True)+ except Exception:+ pass+# -------------------------------------------------------------------# Output# -------------------------------------------------------------------@@ -2646,6 +1197,7 @@ class _ModelFreeCompressorCore:breakblock_name_to_quantize = ",".join(dict.fromkeys(block_prefixes)) or None+ os.makedirs(self._quant_output_dir, exist_ok=True)quantization_config = _build_quantization_config(default_scheme=self.default_scheme,layer_config=self.layer_config,@@ -2657,7 +1209,6 @@ class _ModelFreeCompressorCore:)self.config["quantization_config"] = quantization_config- os.makedirs(self._quant_output_dir, exist_ok=True)with open(os.path.join(self._quant_output_dir, "config.json"), "w") as f:json.dump(self.config, f, indent=2)@@ -2761,6 +1312,8 @@ class _ModelFreeCompressorCore:self._detect_fp8_source()self._resolve_model_type()self._discover_shards()+ self._build_cross_shard_deps()+ self._reorder_shards_by_dependency()self.shard_parallelism, shard_parallelism_source = self._resolve_shard_parallelism()# Determine the output packing format based on scheme data type@@ -2768,6 +1321,8 @@ class _ModelFreeCompressorCore:if is_mx_fp(data_type):bits = self.default_scheme.get("bits", 4)packing_format = "mxfp4-pack-quantized" if bits == 4 else "mxfp8-quantized"+ elif data_type == _NVFP4_E5M3_DATA_TYPE:+ packing_format = "fake" if self.format == "fake" else "auto_round:llm_compressor_nvfp4_e5m3"else:packing_format = "auto_round:auto_gptq"if is_mx_fp(data_type) or _layer_config_has_mxfp(self.layer_config):@@ -3172,9 +1727,14 @@ class ModelFreeCompressor(_ModelFreeCompressorCore):normalized_scheme is not None and is_mx_fp((normalized_scheme.data_type or "").lower())) or self._auto_scheme_family == "mx_fp":_accepted_formats = {"llm_compressor", "auto_round", "auto_round:auto_gptq"}+ elif normalized_scheme is not None and (normalized_scheme.data_type or "").lower() == _NVFP4_E5M3_DATA_TYPE:+ _accepted_formats = {"fake", "llm_compressor", "auto_round", "auto_round:auto_gptq"}elif _is_full_precision_default(self.scheme_input) and _layer_config_has_mxfp(self.layer_config_input):# BF16 default with MXFP layer_config overrides._accepted_formats = {"llm_compressor", "auto_round", "auto_round:auto_gptq"}+ elif _is_full_precision_default(self.scheme_input) and _layer_config_has_nvfp4(self.layer_config_input):+ # BF16 default with NVFP4_E5M3 layer_config overrides.+ _accepted_formats = {"fake", "llm_compressor", "auto_round", "auto_round:auto_gptq"}if format not in _accepted_formats:logger.warning(f"Format '{format}' is not supported by model-free mode for scheme '{self.scheme_input}'; "@@ -3198,119 +1758,3 @@ class ModelFreeCompressor(_ModelFreeCompressorCore):self.format = orig_fmtself.quantized = Truereturn None, out_path---# ----------------------------------------------------------------------------# Model-Type Specific Preprocessing Hooks (Extension Point)-# ----------------------------------------------------------------------------#-# Keep model-specific source-format adaptation functions at the end of this-# file so the core quantization pipeline remains easy to read and maintain.-# Add new model handlers here, keyed by `model_type`, and keep dequant/passthrough-# decisions in the generic MXFP handlers above.---def _expand_e8m0_block_scale(- scale: torch.Tensor,- out_features: int,- in_features: int,- group_size: int = 32,-) -> torch.Tensor:- """Expand a coarse 2D E8M0 block scale to the llm-compressor per-group layout.-- deepseek_v4 stores the per-block shared exponent in a *coarse* 2D shape- ``[out_features // block_h, in_features // block_w]`` (e.g. ``[12, 56]`` for- a ``[1536, 7168]`` weight, i.e. 128x128 blocks). llm-compressor expects a- per-group scale of shape ``[out_features, in_features // group_size]``- (e.g. ``[1536, 224]`` for ``group_size=32``).-- Because every fine MXFP group lies entirely inside a single coarse block,- the expansion is a pure ``repeat_interleave`` along both axes (no- interpolation). The returned tensor is ``uint8`` (raw E8M0 bytes), matching- the ``U8`` dtype used by llm-compressor ``weight_scale`` tensors.- """- scale = scale.view(torch.uint8)- if scale.dim() != 2:- raise ValueError(f"Expected a 2D E8M0 block scale, got shape {tuple(scale.shape)}.")-- target_rows = out_features- target_cols = in_features // group_size- rows, cols = scale.shape-- if target_rows % rows != 0 or target_cols % cols != 0:- raise ValueError(- f"Cannot expand E8M0 block scale {tuple(scale.shape)} to "- f"({target_rows}, {target_cols}); shapes are not divisible."- )-- if target_rows != rows:- scale = scale.repeat_interleave(target_rows // rows, dim=0)- if target_cols != cols:- scale = scale.repeat_interleave(target_cols // cols, dim=1)- return scale.contiguous()---def _preprocess_model_type_source_tensors(- raw_tensors: dict[str, torch.Tensor],- model_type: str | None,- group_size: int = 32,-) -> tuple[dict[str, torch.Tensor], dict[str, int]]:- """Apply model-type-specific source tensor normalization.-- This step is intentionally limited to *format conversion* and does not do- passthrough / dequant decisions. It marks converted layers in the returned- ``source_state`` so downstream generic MXFP handling can treat them exactly- like normal llm-compressor MXFP sources.-- Returns:- ``(raw_tensors, source_state)`` where ``source_state[layer]`` is the- source MXFP bits (4 or 8) for model-type preprocessed layers.- """- if (model_type or "").lower() != "deepseek_v4":- return raw_tensors, {}-- entries: list[tuple[str, str, bool]] = [] # (weight_name, scale_name, is_fp8)- for name, tensor in raw_tensors.items():- if not name.endswith(".weight"):- continue- scale_name = name[: -len(".weight")] + ".scale"- if scale_name not in raw_tensors:- continue- if tensor.dtype == torch.float8_e4m3fn:- entries.append((name, scale_name, True))- elif tensor.dtype in (torch.int8, torch.uint8):- entries.append((name, scale_name, False))-- if not entries:- return raw_tensors, {}-- source_state: dict[str, int] = {}- n_fp8 = 0- n_fp4 = 0- for weight_name, scale_name, is_fp8 in entries:- layer_name = weight_name[: -len(".weight")]- weight = raw_tensors.pop(weight_name)- scale = raw_tensors.pop(scale_name)-- if is_fp8:- out_features, in_features = weight.shape- weight_key = f"{layer_name}.weight"- source_state[layer_name] = 8- n_fp8 += 1- else:- out_features = weight.shape[0]- in_features = weight.shape[1] * 2- weight = weight.view(torch.uint8).contiguous()- weight_key = f"{layer_name}.weight_packed"- source_state[layer_name] = 4- n_fp4 += 1-- weight_scale = _expand_e8m0_block_scale(scale, out_features, in_features, group_size=group_size)- raw_tensors[weight_key] = weight- raw_tensors[f"{layer_name}.weight_scale"] = weight_scale-- logger.info(- "Applied model_type preprocessing for deepseek_v4: "- f"{n_fp8} MXFP8 layer(s), {n_fp4} MXFP4 layer(s) converted to llm-compressor naming."- )- return raw_tensors, source_statediff --git a/auto_round/data_type/nvfp.py b/auto_round/data_type/nvfp.pyindex bab5b5410..f72c875ea 100644--- a/auto_round/data_type/nvfp.py+++ b/auto_round/data_type/nvfp.py@@ -206,14 +206,19 @@ def ref_fp4_quant(x, global_scale, block_size=16, v=0, max_scale=1.0):scale = global_scale * (vec_max * get_reciprocal(FLOAT4_E2M1_MAX))scale = torch.clip(scale, 0, FLOAT8_UE5M3_MAX)scale = cast_to_ue5m3_ste(scale).to(torch.float32)- output_scale = get_reciprocal(scale * get_reciprocal(global_scale))- scaled_x = x.to(torch.float32) * output_scale + v+ dequant_scale = scale * get_reciprocal(global_scale)+ scaled_x = torch.where(+ dequant_scale == 0,+ torch.zeros_like(x, dtype=torch.float32),+ x.to(torch.float32) / dequant_scale,+ )+ scaled_x = scaled_x + vclipped_x = torch.clamp(scaled_x, -6.0, 6.0)- return (cast_to_fp4(clipped_x) * get_reciprocal(output_scale)).reshape(m, n), scale+ return (cast_to_fp4(clipped_x) * dequant_scale).reshape(m, n), scale-@register_dtype("fp4_v2_with_global_scale")-def fp4_v2_with_global_scale(tensor, bits=4, group_size=16, v=0, tensor_max=None, max_scale=1.0, **kwargs):+@register_dtype("nvfp4_v2_with_global_scale")+def nvfp4_v2_with_global_scale(tensor, bits=4, group_size=16, v=0, tensor_max=None, max_scale=1.0, **kwargs):assert group_size == 32 or group_size == 16orig_dtype = tensor.dtypetensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)@@ -230,8 +235,8 @@ def fp4_v2_with_global_scale(tensor, bits=4, group_size=16, v=0, tensor_max=Nonereturn qdq_res.to(orig_dtype), scale, None-@register_dtype("fp4_v2")-def fp4_v2(tensor, bits=4, group_size=32, v=0, max_scale=1.0, **kwargs):+@register_dtype("nvfp4_v2")+def nvfp4_v2(tensor, bits=4, group_size=32, v=0, max_scale=1.0, **kwargs):assert group_size == 32 or group_size == 16orig_dtype = tensor.dtypetensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)diff --git a/auto_round/envs.py b/auto_round/envs.pyindex 97cda8a6b..7c727443c 100644--- a/auto_round/envs.py+++ b/auto_round/envs.py@@ -28,6 +28,7 @@ if TYPE_CHECKING:AR_AUTO_SCHEME_BATCH_SIZE: Optional[int] = NoneAR_AUTO_SCHEME_CACHE: Optional[str] = NoneAR_ENABLE_AUTO_SCHEME_PARALLEL: bool = True+ AR_NVFP4_E5M3_CACHE_HP_WEIGHT: bool = FalseAR_DISK_STREAM_MODEL: bool = FalseAR_RESUME_DIR: Optional[str] = None@@ -105,6 +106,13 @@ environment_variables: dict[str, Callable[[], Any]] = {# set it to 0 when workers could exhaust host RAM or device memory."AR_ENABLE_AUTO_SCHEME_PARALLEL": lambda: os.getenv("AR_ENABLE_AUTO_SCHEME_PARALLEL", "1").lower()in ("1", "true", "yes"),+ # Controls whether NVFP4 E5M3 quant linear caches a dequantized high-+ # precision weight after the first forward instead of dequantizing on+ # every call. When enabled, the packed weight buffers are released after+ # the cache is materialized, trading lower runtime overhead for higher+ # steady-state memory usage.+ "AR_NVFP4_E5M3_CACHE_HP_WEIGHT": lambda: os.getenv("AR_NVFP4_E5M3_CACHE_HP_WEIGHT", "0").lower()+ in ("1", "true", "yes", "on"),# When set, the model is built as a meta-device skeleton and streamed# block-by-block from disk during quantization instead of being fully# materialized on CPU RAM up front.diff --git a/auto_round/experimental/qmodules/__init__.py b/auto_round/experimental/qmodules/__init__.pyindex 4f5ff8e82..e638f95bb 100644--- a/auto_round/experimental/qmodules/__init__.py+++ b/auto_round/experimental/qmodules/__init__.py@@ -19,5 +19,7 @@ from auto_round.experimental.qmodules.mx import (MXINT4QuantLinear,)+from auto_round.experimental.qmodules.fake import FakeActQuantLinearfrom auto_round.experimental.qmodules.nvfp4 import NVFP4QuantLinear+from auto_round.experimental.qmodules.nvfp4_e5m3 import CuteNVFP4E5M3QuantLinear, NVFP4E5M3QuantLinearfrom auto_round.experimental.qmodules.fp8_static import WeightFP8ActFP8StaticQuantLineardiff --git a/auto_round/experimental/qmodules/fake.py b/auto_round/experimental/qmodules/fake.pynew file mode 100644index 000000000..0089a2555--- /dev/null+++ b/auto_round/experimental/qmodules/fake.py@@ -0,0 +1,87 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++from typing import Optional++import torch++from auto_round.data_type.utils import get_quant_func+from auto_round.experimental.qmodules.base import QModuleBase+from auto_round.schemes import QuantizationScheme++__all__ = ["FakeActQuantLinear"]+++class FakeActQuantLinear(QModuleBase):+ """Linear with high-precision QDQ weights and runtime activation QDQ."""++ def __init__(+ self,+ in_features: int,+ out_features: int,+ config: QuantizationScheme,+ weight: Optional[torch.Tensor] = None,+ bias: Optional[torch.Tensor] = None,+ dtype: torch.dtype = torch.bfloat16,+ ):+ super().__init__()+ self.in_features = in_features+ self.out_features = out_features+ self.config = config+ if weight is None:+ weight = torch.empty((out_features, in_features), dtype=dtype)+ self.weight = torch.nn.Parameter(weight, requires_grad=False)+ if bias is None:+ self.register_parameter("bias", None)+ else:+ self.bias = torch.nn.Parameter(bias, requires_grad=False)++ @classmethod+ def from_original(cls, config: QuantizationScheme, original_layer: torch.nn.Linear):+ return cls(+ in_features=original_layer.in_features,+ out_features=original_layer.out_features,+ config=config,+ weight=original_layer.weight,+ bias=original_layer.bias,+ dtype=original_layer.weight.dtype,+ )++ @classmethod+ def get_min_capability(cls) -> int:+ return 0++ def process_weights_after_loading(self, layer: torch.nn.Module):+ return++ def post_init(self):+ return++ def qdq_input(self, activation: torch.Tensor) -> torch.Tensor:+ quant_func, _ = get_quant_func(+ dtype=self.config.act_data_type,+ bits=self.config.act_bits,+ sym=self.config.act_sym,+ )+ qdq_activation, _, _ = quant_func(+ tensor=activation,+ bits=self.config.act_bits,+ group_size=self.config.act_group_size,+ )+ return qdq_activation.to(activation.dtype)++ @torch.inference_mode()+ def forward(self, activation: torch.Tensor) -> torch.Tensor:+ qdq_activation = self.qdq_input(activation)+ return torch.nn.functional.linear(qdq_activation, self.weight.to(qdq_activation.dtype), self.bias)diff --git a/auto_round/experimental/qmodules/nvfp4_e5m3.py b/auto_round/experimental/qmodules/nvfp4_e5m3.pynew file mode 100644index 000000000..c8b115e7e--- /dev/null+++ b/auto_round/experimental/qmodules/nvfp4_e5m3.py@@ -0,0 +1,161 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++import os+from typing import Optional, Union++import torch++from auto_round.data_type.nvfp import e5m3_to_float_tensor, nvfp4_v2+from auto_round.experimental.qmodules.base import QModuleBase+from auto_round.experimental.qmodules.fp4_utils import unpack_fp4_from_uint8+from auto_round.logger import logger+from auto_round.schemes import QuantizationScheme+from auto_round_extension.cuda.cute_nvfp4_e5m3 import (+ try_cute_nvfp4_e5m3_linear,+ try_cute_nvfp4_e5m3_weight_dq,+ try_cute_nvfp4_v2_qdq,+)++__all__ = ["CuteNVFP4E5M3QuantLinear", "NVFP4E5M3QuantLinear"]++_CACHE_WEIGHT_ENV = "AR_NVFP4_E5M3_CACHE_HP_WEIGHT"+++def _resolve_cache_weight(cache_weight: Optional[bool], default: bool) -> bool:+ if cache_weight is not None:+ return cache_weight+ value = os.getenv(_CACHE_WEIGHT_ENV)+ if value is None:+ return default+ return value.strip().lower() not in {"0", "false", "no", "off"}+++class NVFP4E5M3QuantLinear(QModuleBase):+ """FP4 E2M1 weights and activations with unsigned E5M3 block scales."""++ SUPPORTED_COMPUTE_DTYPE = [torch.bfloat16, torch.float16, torch.float32]+ DEFAULT_CACHE_WEIGHT = False++ def __init__(+ self,+ in_features: int,+ out_features: int,+ config: QuantizationScheme,+ weight: Optional[torch.Tensor] = None,+ weight_scale: Optional[torch.Tensor] = None,+ bias: Union[torch.Tensor, bool, None] = None,+ dtype=torch.bfloat16,+ cache_weight: Optional[bool] = None,+ ):+ super().__init__()+ assert dtype in self.SUPPORTED_COMPUTE_DTYPE+ assert config.group_size == 16 and config.act_group_size == 16+ self.in_features = in_features+ self.out_features = out_features+ self.group_size = config.group_size+ self.config = config+ self.dtype = dtype+ self.cache_weight = _resolve_cache_weight(cache_weight, self.DEFAULT_CACHE_WEIGHT)+ self._cached_weight = None++ packed_weight = torch.zeros((out_features, in_features // 2), dtype=torch.uint8) if weight is None else weight+ self.register_buffer("weight_packed", packed_weight)+ scale = (+ torch.empty((out_features, in_features // self.group_size), dtype=torch.uint8)+ if weight_scale is None+ else weight_scale+ )+ self.register_buffer("weight_scale", scale)++ if bias is not None:+ if isinstance(bias, bool):+ bias = torch.zeros((out_features,), dtype=dtype)+ self.bias = torch.nn.Parameter(bias, requires_grad=False)+ else:+ self.register_parameter("bias", None)++ @classmethod+ def get_min_capability(cls) -> int:+ logger.warning_once("NVFP4 E5M3 quantization uses reference PyTorch inference and may be slow.")+ return 0++ def dequant_weight_online(self) -> torch.Tensor:+ unpacked = unpack_fp4_from_uint8(self.weight_packed, self.out_features, self.in_features, dtype=self.dtype).to(+ torch.float32+ )+ scale = e5m3_to_float_tensor(self.weight_scale).reshape(-1, 1)+ return (unpacked.reshape(-1, self.group_size) * scale).reshape(self.out_features, self.in_features)++ @property+ def weight(self) -> torch.Tensor:+ if self._cached_weight is None:+ self._cached_weight = self.dequant_weight_online()+ if self.cache_weight:+ self.weight_packed = None+ self.weight_scale = None+ return self._cached_weight++ def clear_weight_cache(self) -> None:+ if self.weight_packed is None:+ raise RuntimeError("Cannot clear the cached weight after quantized weight buffers have been released.")+ self._cached_weight = None++ def qdq_input(self, activation: torch.Tensor) -> torch.Tensor:+ original_dtype = activation.dtype+ qdq_activation, _, _ = nvfp4_v2(+ activation.to(torch.float32), bits=self.config.act_bits, group_size=self.config.act_group_size+ )+ return qdq_activation.to(original_dtype)++ @torch.inference_mode()+ def forward(self, input: torch.Tensor) -> torch.Tensor:+ qdq_input = self.qdq_input(input)+ weight = self.weight if self.cache_weight else self.dequant_weight_online()+ return torch.nn.functional.linear(qdq_input, weight.to(qdq_input.dtype), self.bias)++ @classmethod+ def from_original(cls, config: QuantizationScheme, original_layer: torch.nn.Linear):+ return cls(+ in_features=original_layer.in_features,+ out_features=original_layer.out_features,+ config=config,+ bias=original_layer.bias,+ dtype=original_layer.weight.dtype,+ )+++class CuteNVFP4E5M3QuantLinear(NVFP4E5M3QuantLinear):+ """NVFP4 E5M3 linear that dispatches activation QDQ and GEMM to CuTe."""++ def dequant_weight_online(self) -> torch.Tensor:+ cute_weight = try_cute_nvfp4_e5m3_weight_dq(self.weight_packed, self.weight_scale, self.dtype)+ if cute_weight is not None:+ return cute_weight+ return super().dequant_weight_online()++ def qdq_input(self, activation: torch.Tensor) -> torch.Tensor:+ cute_qdq_activation = try_cute_nvfp4_v2_qdq(activation, self.config.act_group_size)+ if cute_qdq_activation is not None:+ return cute_qdq_activation+ return super().qdq_input(activation)++ @torch.inference_mode()+ def forward(self, input: torch.Tensor) -> torch.Tensor:+ if self.cache_weight:+ return super().forward(input)+ fused_output = try_cute_nvfp4_e5m3_linear(input, self.weight_packed, self.weight_scale, self.bias)+ if fused_output is not None:+ return fused_output+ return super().forward(input)diff --git a/auto_round/export/export_to_autoround/qlinear_fp.py b/auto_round/export/export_to_autoround/qlinear_fp.pyindex 338d624a7..039c5e0b3 100644--- a/auto_round/export/export_to_autoround/qlinear_fp.py+++ b/auto_round/export/export_to_autoround/qlinear_fp.py@@ -36,7 +36,7 @@ import transformersimport auto_round.envs as envsfrom auto_round.compressors.utils import BackendDataType, is_mx_fp, is_nv_fpfrom auto_round.data_type.mxfp import FP32_EXPONENT_BIAS, FP32_MIN_NORMAL-from auto_round.data_type.nvfp import cast_to_fp4, get_reciprocal+from auto_round.data_type.nvfp import cast_to_fp4, float_to_e5m3_frexp, get_reciprocalfrom auto_round.data_type.utils import reshape_pad_tensor_by_group_size, revert_tensor_by_padfrom auto_round.utils import get_packing_device, logger@@ -73,6 +73,7 @@ class QuantLinear(nn.Module):raise NotImplementedError("Only 4,8 bits are supported.")self.is_mx = is_mx_fp(data_type)self.is_nv = is_nv_fp(data_type)+ self.is_nvfp4_e5m3 = data_type == "nvfp4_v2"if self.is_mx:if group_size != 32:raise NotImplementedError(f"Only group_size 32 are supported for {BackendDataType.MX_FP} data type.")@@ -80,7 +81,7 @@ class QuantLinear(nn.Module):raise NotImplementedError(f"in_feature must be divisible by {group_size} for {BackendDataType.MX_FP} data type.")- if self.is_nv:+ if self.is_nv or self.is_nvfp4_e5m3:if group_size % 16 != 0:raise NotImplementedError(f"Only group_size 16 are supported for {BackendDataType.NV_FP} data type.")if infeatures % group_size != 0:@@ -159,11 +160,17 @@ class QuantLinear(nn.Module):)scaled_tensor.clamp_(-6.0, 6.0)scaled_tensor = cast_to_fp4(scaled_tensor)+ elif self.is_nvfp4_e5m3:+ scaled_tensor = tensor / scales.reshape(tensor.shape[0], -1)+ scaled_tensor.clamp_(-6.0, 6.0)+ scaled_tensor = cast_to_fp4(scaled_tensor)else:scaled_tensor = tensor / (2 ** scales.reshape(tensor.shape[0], -1))scaled_tensor = revert_tensor_by_pad(scaled_tensor, orig_shape=orig_shape, pad_len=pad_len)if self.is_mx:final_scale = (scales + E8M0_EXPONENT_BIAS).clamp(0, E8M0_EXPONENT_NAN_VAL).to(torch.uint8)+ elif self.is_nvfp4_e5m3:+ final_scale = float_to_e5m3_frexp(scales.to(torch.float32))else:final_scale = scales.to(torch.float8_e4m3fn)diff --git a/auto_round/export/export_to_llmcompressor/config.py b/auto_round/export/export_to_llmcompressor/config.pyindex 62017ec73..1f5090e6c 100644--- a/auto_round/export/export_to_llmcompressor/config.py+++ b/auto_round/export/export_to_llmcompressor/config.py@@ -102,3 +102,38 @@ def initialize_quantization(scheme, targets=["Linear"], config_groups=None, kv_cquantization_status=QuantizationStatus.COMPRESSED,ignore=ignore,)+++def initialize_nvfp4_e5m3_quantization(ignore=None):+ """Build stable compressed-tensors metadata for global-scale-free NVFP4 E5M3."""++ def quant_args(dynamic):+ return {+ "actorder": None,+ "block_structure": None,+ "dynamic": dynamic,+ "group_size": 16,+ "num_bits": 4,+ "observer": "minmax",+ "observer_kwargs": {},+ "strategy": "tensor_group",+ "symmetric": True,+ "type": "float",+ }++ return {+ "config_groups": {+ "group_0": {+ "input_activations": quant_args("local"),+ "output_activations": None,+ "targets": ["Linear"],+ "weights": quant_args(False),+ }+ },+ "format": "nvfp4-e5m3-pack-quantized",+ "global_compression_ratio": None,+ "ignore": list(dict.fromkeys(ignore or ["lm_head"])),+ "kv_cache_scheme": None,+ "quant_method": "compressed-tensors",+ "quantization_status": "compressed",+ }diff --git a/auto_round/export/export_to_llmcompressor/export_to_fp.py b/auto_round/export/export_to_llmcompressor/export_to_fp.pyindex c061095da..f207a035d 100644--- a/auto_round/export/export_to_llmcompressor/export_to_fp.py+++ b/auto_round/export/export_to_llmcompressor/export_to_fp.py@@ -140,6 +140,8 @@ def _get_scheme(bits, data_type):return "MXFP4" if bits == 4 else "MXFP8"if is_nv_fp(data_type):return "NVFP4"+ if data_type == "nvfp4_v2":+ return "NVFP4_E5M3"return None@@ -149,6 +151,8 @@ def _get_group_format(bits, data_type):return "mxfp4-pack-quantized" if bits == 4 else "mxfp8-quantized"if is_nv_fp(data_type):return "nvfp4-pack-quantized"+ if data_type == "nvfp4_v2":+ return "nvfp4-e5m3-pack-quantized"return "float-quantized"@@ -349,6 +353,11 @@ def save_quantized_as_fp(static_kv_granularity=static_kv_granularity,static_attention_granularity=static_attention_granularity,)+ elif data_type == "nvfp4_v2":+ from auto_round.export.export_to_llmcompressor.config import initialize_nvfp4_e5m3_quantization++ quantization_config = initialize_nvfp4_e5m3_quantization(ignore=ignore)+ quantization_config["provider"] = "auto-round"else:scheme = _get_scheme(bits, data_type)if scheme is None:diff --git a/auto_round/export/formats/backends/autoround.py b/auto_round/export/formats/backends/autoround.pyindex 21ab8daaa..4d570b26f 100644--- a/auto_round/export/formats/backends/autoround.py+++ b/auto_round/export/formats/backends/autoround.py@@ -41,6 +41,7 @@ class AutoRoundFormat(OutputFormat):"MXFP4","MXFP8","NVFP4",+ "NVFP4_E5M3","FPW8A16","W2A16G64","W2A16G32",@@ -67,7 +68,7 @@ class AutoRoundFormat(OutputFormat):)if enable_awq:self.backend = AutoAWQFormat("auto_round:auto_awq", scheme, ctx)- elif scheme.is_nv_fp() or scheme.is_mx_fp():+ elif scheme.is_nv_fp() or scheme.is_mx_fp() or scheme.data_type == BackendDataType.NVFP4_E5M3.value:self.backend = AutoRoundFormat(scheme.data_type, scheme, ctx)elif scheme.is_mx_int() and scheme.bits == 4: # only add mx_int4 nowself.backend = AutoRoundFormat(scheme.data_type, scheme, ctx)@@ -87,7 +88,9 @@ class AutoRoundFormat(OutputFormat):elif not format.startswith("auto_round"):if format == "mlx":self.backend = MLXFormat("mlx", scheme, ctx)- elif format.upper() not in list(BackendDataType.__members__.keys()):+ elif format.upper() not in list(BackendDataType.__members__.keys()) and format not in {+ BackendDataType.NVFP4_E5M3.value+ }:raise KeyError(f"Unsupported backend format auto_round:{format}, please check")else:self.output_format = f"auto_round:{format}"@@ -131,7 +134,11 @@ class AutoRoundFormat(OutputFormat):backend = self.get_backend_name()- if self.output_format in [+ if self.output_format == f"auto_round:{BackendDataType.NVFP4_E5M3.value}":+ from auto_round.export.export_to_llmcompressor.export_to_fp import pack_layer++ return pack_layer(layer_name, model, device=device)+ elif self.output_format in [f"auto_round:{BackendDataType.NV_FP.value}",f"auto_round:{BackendDataType.MX_FP.value}",f"auto_round:{BackendDataType.MX_FP_RCEIL.value}",@@ -185,7 +192,12 @@ class AutoRoundFormat(OutputFormat):**kwargs,)backend = self.get_backend_name()- if re.search(f"{BackendDataType.MX_FP.value}|{BackendDataType.NV_FP.value}", backend):+ if backend == f"auto_round:{BackendDataType.NVFP4_E5M3.value}":+ from auto_round.export.export_to_autoround.export_to_nvfp_mx import save_quantized_as_fp++ backend = "auto_round:llm_compressor_nvfp4_e5m3"+ export_func = save_quantized_as_fp+ elif re.search(f"{BackendDataType.MX_FP.value}|{BackendDataType.NV_FP.value}", backend):from auto_round.export.export_to_autoround.export_to_nvfp_mx import save_quantized_as_fpbackend = "auto_round:llm_compressor"diff --git a/auto_round/export/formats/backends/fake.py b/auto_round/export/formats/backends/fake.pyindex 6c1e9a703..069d1aeae 100644--- a/auto_round/export/formats/backends/fake.py+++ b/auto_round/export/formats/backends/fake.py@@ -12,6 +12,7 @@# See the License for the specific language governing permissions and# limitations under the License.+import copyfrom typing import Any, Callable, Unionimport torch@@ -22,6 +23,16 @@ from auto_round.schemes import QuantizationSchemefrom auto_round.utils import copy_python_files_from_model_cache, unsupported_meta_device+def _serialize_quantization_config_value(value):+ if isinstance(value, torch.dtype):+ return str(value)+ if isinstance(value, dict):+ return {key: _serialize_quantization_config_value(item) for key, item in value.items()}+ if isinstance(value, (list, tuple)):+ return [_serialize_quantization_config_value(item) for item in value]+ return value++@OutputFormat.register("fake")class FakeFormat(OutputFormat):support_schemes = None@@ -47,7 +58,46 @@ class FakeFormat(OutputFormat):serialization_dict: dict = None,**kwargs,):- if not unsupported_meta_device(model):+ has_meta_device = unsupported_meta_device(model)+ if not inplace and not has_meta_device:+ model = copy.deepcopy(model.to("cpu"))++ from auto_round.utils.model import set_module+ from auto_round.wrapper import WrapperLinear, WrapperWALayer++ if not has_meta_device:+ wrapped_modules = [+ (name, module)+ for name, module in model.named_modules()+ if name and isinstance(module, (WrapperLinear, WrapperWALayer))+ ]+ wrapped_names = {name for name, _ in wrapped_modules}+ for name, module in wrapped_modules:+ if any(name.startswith(f"{parent}.") for parent in wrapped_names if parent != name):+ continue+ orig_layer = module.orig_layer+ while isinstance(orig_layer, (WrapperLinear, WrapperWALayer)):+ orig_layer = orig_layer.orig_layer+ for attr_name in ("act_min_scale", "act_max_scale", "act_scale"):+ orig_layer._parameters.pop(attr_name, None)+ orig_layer._buffers.pop(attr_name, None)+ if hasattr(orig_layer, attr_name):+ delattr(orig_layer, attr_name)+ set_module(model, name, orig_layer.to("cpu"))++ is_nvfp4_v2 = (serialization_dict or {}).get("data_type") == "nvfp4_v2"+ if is_nvfp4_v2:+ quantization_config = _serialize_quantization_config_value(dict(serialization_dict or {}))+ quantization_config["quant_method"] = "auto-round"+ quantization_config["packing_format"] = "auto_round:fake"+ quantization_config["block_name_to_quantize"] = quantization_config.pop("to_quant_block_names", None)+ from auto_round.export.utils import filter_quantization_config++ filter_quantization_config(quantization_config)+ if hasattr(model, "config") and model.config is not None:+ model.config.quantization_config = quantization_config++ if not has_meta_device:model = model.to("cpu")model.save_pretrained(output_dir)elif hasattr(model, "config") and model.config is not None:diff --git a/auto_round/export/formats/backends/llm_compressor.py b/auto_round/export/formats/backends/llm_compressor.pyindex e856a4947..9bc260684 100644--- a/auto_round/export/formats/backends/llm_compressor.py+++ b/auto_round/export/formats/backends/llm_compressor.py@@ -29,6 +29,7 @@ class LLMCompressorFormat(OutputFormat):"MXFP4","MXFP8","NVFP4",+ "NVFP4_E5M3","FPW8A16","FP8_STATIC","INT8",@@ -53,7 +54,12 @@ class LLMCompressorFormat(OutputFormat):if re.search("^(auto_round:)?llm_compressor", format):self.output_format = formatself.backend = None- if scheme.is_nv_fp() or scheme.is_mx_fp():+ if scheme.data_type == "nvfp4_v2":+ from auto_round.export.export_to_llmcompressor import check_compressed_tensors_supported++ check_compressed_tensors_supported(raise_error=True)+ self.output_format = "llm_compressor:nvfp4_v2"+ elif scheme.is_nv_fp() or scheme.is_mx_fp():from auto_round.export.export_to_llmcompressor import check_compressed_tensors_supportedcheck_compressed_tensors_supported(raise_error=True)@@ -139,6 +145,8 @@ class LLMCompressorFormat(OutputFormat):)if scheme.act_bits <= 8 and (not scheme.is_act_standard_fp() or scheme.act_dynamic):+ if scheme.act_data_type == "nvfp4_v2":+ return None, scheme, layer_config, quant_block_listif (scheme.is_act_nv_fp() and "static_gs" in scheme.act_data_type) or scheme.is_act_mx_fp():return None, scheme, layer_config, quant_block_listelif scheme.is_dynamic_afp8() and scheme.is_block_wfp8():@@ -163,7 +171,7 @@ class LLMCompressorFormat(OutputFormat):def pack_layer(self, layer_name, model, device=None, **kwargs):if self.backend is not None:return self.backend.pack_layer(layer_name, model, device=device, **kwargs)- if re.search(f"{BackendDataType.MX_FP.value}|{BackendDataType.NV_FP.value}", self.output_format):+ if re.search(f"{BackendDataType.MX_FP.value}|{BackendDataType.NV_FP.value}|nvfp4_v2", self.output_format):from auto_round.export.export_to_llmcompressor.export_to_fp import pack_layerreturn pack_layer(layer_name, model, device=device)@@ -199,7 +207,7 @@ class LLMCompressorFormat(OutputFormat):**kwargs,) -> torch.nn.Module:backend = self.get_backend_name()- if re.search(f"{BackendDataType.MX_FP.value}|{BackendDataType.NV_FP.value}", backend):+ if re.search(f"{BackendDataType.MX_FP.value}|{BackendDataType.NV_FP.value}|nvfp4_v2", backend):from auto_round.export.export_to_llmcompressor.export_to_fp import save_quantized_as_fpexport_func = save_quantized_as_fpdiff --git a/auto_round/export/formats/base.py b/auto_round/export/formats/base.pyindex 47c386d3b..618e3e0fa 100644--- a/auto_round/export/formats/base.py+++ b/auto_round/export/formats/base.py@@ -54,6 +54,7 @@ class BackendDataType(str, Enum):MXFP8 = "mxfp8"MXFP4 = "mxfp4"NVFP4 = "nvfp4"+ NVFP4_E5M3 = "nvfp4_v2"FP8 = "fp8"MX_FP = "mx_fp"NV_FP = "nv_fp"diff --git a/auto_round/inference/backend.py b/auto_round/inference/backend.pyindex ade2ca1ce..289e163de 100644--- a/auto_round/inference/backend.py+++ b/auto_round/inference/backend.py@@ -194,10 +194,25 @@ def fp8_static_scheme_checker(return config == FP8_STATIC+def cute_nvfp4_e5m3_checker(in_feature: int, out_feature: int, config: QuantizationScheme) -> bool:+ """Require CuTe DSL before selecting the optional NVFP4 E5M3 CUDA path."""+ del out_feature+ if not in_feature_checker_group_size(in_feature, 0, config):+ return False+ try:+ from auto_round_extension.cuda.cute_nvfp4_e5m3 import is_cute_dsl_available++ return torch.cuda.is_available() and is_cute_dsl_available() and torch.cuda.get_device_capability()[0] >= 8+ except (ImportError, RuntimeError):+ return False++GPTQ_FORMAT = ["auto_round:auto_gptq"] # zp+-1GPTQ_FORMAT_NO_ZP = ["auto_round", "auto_round:gptqmodel"]AWQ_FORMAT = ["auto_round:auto_awq"]LLM_COMPRESSOR_FORMAT = ["auto_round:llm_compressor"]+FAKE_FORMAT = ["auto_round:fake"]+NVFP4_E5M3_LLM_COMPRESSOR_FORMAT = ["auto_round:llm_compressor_nvfp4_e5m3"]WOQ_DEFAULT_ACT_BITS = [None, 16, 32]# CPU backends that target Intel/x86 (ark / auto_round_kernel) cannot@@ -333,6 +348,24 @@ BackendInfos["auto_round:torch_mxint4"] = BackendInfo(# NVFP4+BackendInfos["auto_round:fake"] = BackendInfo(+ device=["xpu", "cuda", "cpu"],+ packing_format=FAKE_FORMAT,+ sym=[True],+ compute_dtype=["float32", "float16", "bfloat16"],+ data_type=["nvfp4_v2"],+ group_size=[16],+ bits=[4],+ act_bits=[4],+ act_group_size=[16],+ act_sym=[True],+ act_data_type=["nvfp4_v2"],+ priority=0,+ checkers=[mxfp_nvfp_feature_checker],+ alias=["auto_round", "torch"],+ requirements=["auto-round>0.12.0"],+)+BackendInfos["auto_round:torch_nvfp4"] = BackendInfo(device=["xpu", "cuda", "cpu"],packing_format=LLM_COMPRESSOR_FORMAT,@@ -352,6 +385,44 @@ BackendInfos["auto_round:torch_nvfp4"] = BackendInfo(requirements=["auto-round>0.7.0"],)+BackendInfos["auto_round:torch_nvfp4_e5m3"] = BackendInfo(+ device=["xpu", "cuda", "cpu"],+ packing_format=NVFP4_E5M3_LLM_COMPRESSOR_FORMAT,+ sym=[True],+ compute_dtype=["float32", "float16", "bfloat16"],+ data_type=["nvfp4_v2"],+ group_size=[16],+ bits=[4],+ act_bits=[4],+ act_group_size=[16],+ act_sym=[True],+ act_data_type=["nvfp4_v2"],+ act_dynamic=[True],+ priority=3,+ checkers=[mxfp_nvfp_feature_checker],+ alias=["auto_round", "torch"],+ requirements=["auto-round>0.12.0"],+)++BackendInfos["auto_round:cute_nvfp4_e5m3"] = BackendInfo(+ device=["cuda"],+ packing_format=NVFP4_E5M3_LLM_COMPRESSOR_FORMAT,+ sym=[True],+ compute_dtype=["float32", "float16", "bfloat16"],+ data_type=["nvfp4_v2"],+ group_size=[16],+ bits=[4],+ act_bits=[4],+ act_group_size=[16],+ act_sym=[True],+ act_data_type=["nvfp4_v2"],+ act_dynamic=[True],+ priority=6,+ checkers=[cute_nvfp4_e5m3_checker],+ alias=["cute_nvfp4_e5m3"],+ requirements=["auto-round>0.12.0"],+)+BackendInfos["auto_round:tritonv2"] = BackendInfo(device=["cuda", "xpu"],data_type=["int"],@@ -779,8 +850,14 @@ def dynamic_import_inference_linear(backend, config, packing_format=None):return ar_qmodules.MXINT4QuantLinearif "torch_mxfp4" in backend:return ar_qmodules.MXFP4QuantLinear+ if backend == "auto_round:cute_nvfp4_e5m3":+ return ar_qmodules.CuteNVFP4E5M3QuantLinear+ if backend == "auto_round:torch_nvfp4_e5m3":+ return ar_qmodules.NVFP4E5M3QuantLinearif "torch_nvfp4" in backend:return ar_qmodules.NVFP4QuantLinear+ if "auto_round:fake" in backend:+ return ar_qmodules.FakeActQuantLinearif "auto_round_kernel" in backend or "ark" in backend:try:diff --git a/auto_round/inference/convert_model.py b/auto_round/inference/convert_model.pyindex 2a832d125..b3fc98de4 100644--- a/auto_round/inference/convert_model.py+++ b/auto_round/inference/convert_model.py@@ -13,6 +13,7 @@# limitations under the License.import osimport re+from types import SimpleNamespacefrom typing import Unionimport torch@@ -70,7 +71,7 @@ def skip_not_convert_modules(model, quantization_config, layer_names, layer_confif modules_to_not_convert:for layer_name in layer_names:if any([re.search(re.compile(n), layer_name) for n in modules_to_not_convert]):- layer_configs[layer_name] = {"bits": 16}+ layer_configs[layer_name] = {"bits": 16, "act_bits": 16}return layer_configs@@ -571,6 +572,9 @@ def _create_quant_layer(layer, layer_backend, config, in_features, out_features,or BackendDataType.MXFP4.value in layer_backendor BackendDataType.NVFP4.value in layer_backendor BackendDataType.MXINT4.value in layer_backend+ or layer_backend == "auto_round:torch_nvfp4_e5m3"+ or layer_backend == "auto_round:cute_nvfp4_e5m3"+ or layer_backend == "auto_round:fake"):return QuantLinear.from_original(config, layer)@@ -839,6 +843,30 @@ def convert_hf_model(model: nn.Module, target_device: str = "cpu") -> tuple[nn.Mif is_transformers_version_greater_or_equal_5():disable_moe_conversion_mapping(model)quantization_config = model.config.quantization_config+ config_format = (+ quantization_config.get("format")+ if isinstance(quantization_config, dict)+ else getattr(quantization_config, "format", None)+ )+ if config_format == "nvfp4-e5m3-pack-quantized":+ config_dict = quantization_config if isinstance(quantization_config, dict) else quantization_config.to_dict()+ ignored = config_dict.get("ignore") or []+ # TODO: For experimental purpose, will delete when it's not necessary.+ quantization_config = SimpleNamespace(+ quant_method="auto-round",+ packing_format="auto_round:llm_compressor_nvfp4_e5m3",+ bits=4,+ group_size=16,+ sym=True,+ data_type="nvfp4_v2",+ act_bits=4,+ act_group_size=16,+ act_sym=True,+ act_data_type="nvfp4_v2",+ act_dynamic=True,+ extra_config={name: {"bits": 16, "act_bits": 16} for name in ignored},+ )+ model.config.quantization_config = quantization_config# Check desc_act + static_groupsif getattr(quantization_config, "desc_act", False):@@ -879,6 +907,12 @@ def convert_hf_model(model: nn.Module, target_device: str = "cpu") -> tuple[nn.M# Replace layers with quantized versionslayer_configs = get_layer_config(model, quantization_config)used_backends = _replace_by_quant_layers(model, layer_configs, backend, target_device, packing_format)+ logger.info(+ "Inference backend selection: requested=%s, packing_format=%s, selected=%s",+ backend,+ packing_format,+ ", ".join(used_backends),+ )# Apply rotation hooks (hadamard, spinquant, quarot, etc.) via unified dispatch._has_rotation = getattr(quantization_config, "rotation_config", None) or getattr(diff --git a/auto_round/schemes.py b/auto_round/schemes.pyindex 31da6d95c..def0f79af 100644--- a/auto_round/schemes.py+++ b/auto_round/schemes.py@@ -706,6 +706,18 @@ NVFP4 = QuantizationScheme.from_dict(})+NVFP4_E5M3 = QuantizationScheme.from_dict(+ {+ "bits": 4,+ "group_size": 16,+ "data_type": "nvfp4_v2",+ "act_bits": 4,+ "act_data_type": "nvfp4_v2",+ "act_group_size": 16,+ "act_sym": True,+ }+)+FPW8A16 = QuantizationScheme.from_dict({"bits": 8,@@ -802,6 +814,7 @@ PRESET_SCHEMES = {"MXFP8": MXFP8,"MXFP8_RCEIL": MXFP8_RCEIL,"NVFP4": NVFP4,+ "NVFP4_E5M3": NVFP4_E5M3,"FPW8A16": FPW8A16,"W2A16G64": W2A16G64,"W2A16G32": W2A16G32,diff --git a/auto_round/utils/common.py b/auto_round/utils/common.pyindex c23aad8eb..8b68351c8 100644--- a/auto_round/utils/common.py+++ b/auto_round/utils/common.py@@ -353,12 +353,50 @@ def monkey_patch_transformers():_patch_tensor_get_dtype_for_prequantized_loading()_patch_default_rope_init()_patch_rotary_embedding_init_for_legacy_remote_code()+ _patch_nvfp4_e5m3_compressed_tensors_quantizer()if parsed_version >= version.parse("4.56.0"):_patch_classmethod_kwargs(transformers.AutoModelForCausalLM, "from_pretrained", torch_dtype="dtype")else:_patch_classmethod_kwargs(transformers.AutoModelForCausalLM, "from_pretrained", dtype="torch_dtype")+def _patch_nvfp4_e5m3_compressed_tensors_quantizer():+ """Route the global-scale-free E5M3 format through AutoRound qmodules."""+ try:+ from transformers.quantizers.quantizer_compressed_tensors import CompressedTensorsHfQuantizer+ except ImportError:+ return++ original_before = CompressedTensorsHfQuantizer._process_model_before_weight_loading+ if getattr(original_before, "_auto_round_nvfp4_e5m3_patch", False):+ return+ original_after = CompressedTensorsHfQuantizer._process_model_after_weight_loading++ def is_nvfp4_e5m3(self):+ compression_config = getattr(getattr(self, "compressor", None), "quantization_config", None)+ return getattr(compression_config, "format", None) == "nvfp4-e5m3-pack-quantized"++ def patched_before(self, model, **kwargs):+ if not is_nvfp4_e5m3(self):+ return original_before(self, model, **kwargs)+ from auto_round.inference.convert_model import convert_hf_model++ model.config.quantization_config = self.compressor.quantization_config+ target_device = "cuda" if torch.cuda.is_available() else "cpu"+ model, _ = convert_hf_model(model, target_device=target_device)+ self.run_compressed = True+ return model++ def patched_after(self, model, **kwargs):+ if is_nvfp4_e5m3(self):+ return model+ return original_after(self, model, **kwargs)++ patched_before._auto_round_nvfp4_e5m3_patch = True+ CompressedTensorsHfQuantizer._process_model_before_weight_loading = patched_before+ CompressedTensorsHfQuantizer._process_model_after_weight_loading = patched_after++@lru_cache(None)def monkey_patch():monkey_patch_transformers()diff --git a/auto_round/utils/model.py b/auto_round/utils/model.pyindex 617f59b54..907d3bc90 100644--- a/auto_round/utils/model.py+++ b/auto_round/utils/model.py@@ -2585,16 +2585,15 @@ def is_model_free_route(return Falseif fmt_first == "llm_compressor":- # llm_compressor output format is only supported for MXFP schemes.- from auto_round.compressors.utils import is_mx_fp+ from auto_round.compressors.model_free import _apply_scheme_overrides+ from auto_round.schemes import is_mx_fp as _is_mx_fptry:- from auto_round.compressors.model_free import _normalize_scheme-- scheme_obj = _normalize_scheme(scheme)- return common_conditions and is_mx_fp((scheme_obj.data_type or "").lower())- except (ValueError, TypeError):- return False+ scheme_obj = _apply_scheme_overrides(scheme, kwargs)+ scheme_is_mx_fp = _is_mx_fp(scheme_obj.data_type or "")+ except Exception:+ scheme_is_mx_fp = False+ return common_conditions and scheme_is_mx_fp and is_model_free_supported_scheme(scheme, kwargs)if fmt_first != "auto_round":return Falsereturn common_conditions and is_model_free_supported_scheme(scheme, kwargs)diff --git a/auto_round/utils/model_free_utils.py b/auto_round/utils/model_free_utils.pynew file mode 100644index 000000000..e489f0ca0--- /dev/null+++ b/auto_round/utils/model_free_utils.py@@ -0,0 +1,2710 @@+# Copyright (c) 2025 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++"""Utility helpers for model-free quantization.++This module hosts generic helpers used by model-free flow and keeps+model-special handlers at the end of the file.+"""++from __future__ import annotations++import copy+import json+import os+import re+import shutil+import warnings+from dataclasses import fields+from functools import lru_cache+from typing import Any, Callable, Optional, Union++import torch++from auto_round.compressors.utils import is_mx_fp, is_nv_fp+from auto_round.logger import logger+from auto_round.schemes import PRESET_SCHEMES, QuantizationScheme, preset_name_to_scheme+from auto_round.utils.common import to_standard_regex+from auto_round.utils.device import clear_memory, compile_func+from auto_round.utils.missing_tensors import quantize_weight_rtn, split_fused_expert_tensors++_NVFP4_E5M3_DATA_TYPE = "nvfp4_v2"+_BLOCK_NAME_TO_IGNORE = ("shared_expert_gate.", ".gate.", "embed", "conv")+_SUPPORTED_MXFP_BITS = (4, 8)+_SUPPORTED_INT_BITS = (2, 4, 8)+SUPPORTED_PRESET_SCHEMES = (+ "W2A16",+ "W2A16G32",+ "W2A16G64",+ "W4A16",+ "W4A16_MIXED",+ "W8A16",+ "MXFP4",+ "MXFP8",+ "NVFP4_E5M3",+ "BF16",+)+++# ---------------------------------------------------------------------------+# Generic Utility Helpers+# ---------------------------------------------------------------------------+++# Pattern Matching and Layer Scheme Resolution+# ---------------------------------------------------------------------------+++class _PatternMatcher:+ """Precompile ignore and layer-config patterns for shard processing."""++ __slots__ = (+ "_ignore_re",+ "_skip_re",+ "_layer_config",+ "_default_scheme",+ "_compiled_lc",+ "_ignore_cache",+ "_scheme_cache",+ )++ def __init__(+ self,+ ignore_patterns: list[str],+ layer_config: dict[str, dict],+ default_scheme: dict,+ ) -> None:+ self._default_scheme = default_scheme+ self._layer_config = layer_config+ self._ignore_re: re.Pattern | None = self._build_ignore_regex(ignore_patterns)+ self._skip_re = re.compile("|".join(re.escape(name) for name in _BLOCK_NAME_TO_IGNORE))++ self._compiled_lc: list[tuple[re.Pattern | None, str | None, dict]] = []+ for pattern, config in layer_config.items():+ try:+ self._compiled_lc.append((re.compile(to_standard_regex(pattern)), None, config))+ except re.error:+ self._compiled_lc.append((None, pattern, config))++ self._ignore_cache: dict[str, bool] = {}+ self._scheme_cache: dict[str, dict | None] = {}++ @staticmethod+ def _build_ignore_regex(patterns: list[str]) -> re.Pattern | None:+ if not patterns:+ return None+ normalized_patterns: list[str] = []+ for pattern in patterns:+ if pattern.endswith("."):+ normalized = to_standard_regex(pattern.rstrip(".")).removesuffix(".*")+ normalized_patterns.append(f"{normalized}(?:\\.|$)")+ else:+ normalized_patterns.append(to_standard_regex(pattern))+ return re.compile("|".join(normalized_patterns))++ def should_ignore(self, tensor_name: str) -> bool:+ cached = self._ignore_cache.get(tensor_name)+ if cached is not None:+ return cached+ layer_name = tensor_name.rsplit(".", 1)[0] if "." in tensor_name else tensor_name+ result = bool(self._ignore_re and self._ignore_re.search(layer_name))+ self._ignore_cache[tensor_name] = result+ return result++ def should_skip(self, tensor_name: str) -> bool:+ return bool(self._skip_re.search(tensor_name))++ def resolve_scheme(self, tensor_name: str) -> dict | None:+ if tensor_name not in self._scheme_cache:+ self._scheme_cache[tensor_name] = self._resolve_uncached(tensor_name)+ return self._scheme_cache[tensor_name]++ def _resolve_uncached(self, tensor_name: str) -> dict | None:+ layer_name = tensor_name.rsplit(".", 1)[0] if "." in tensor_name else tensor_name+ default = self._default_scheme+ if layer_name in self._layer_config:+ config = self._layer_config[layer_name]+ return None if config.get("bits", default.get("bits", 4)) >= 16 else {**default, **config}++ for compiled, plain, config in self._compiled_lc:+ if (compiled is not None and compiled.search(layer_name)) or (plain is not None and plain in layer_name):+ return None if config.get("bits", default.get("bits", 4)) >= 16 else {**default, **config}+ return default+++# ---------------------------------------------------------------------------+# Source Tensor Detection and Normalization+# ---------------------------------------------------------------------------+++def _collect_mxfp_source_entries(raw_tensors: dict[str, torch.Tensor]) -> list[tuple[str, str, str, int]]:+ """Collect MXFP source tensors present in a shard.++ Returns entries as ``(layer_name, weight_key, scale_key, bits)`` where+ ``bits`` is 8 for ``.weight`` (float8) and 4 for ``.weight_packed``.+ """+ entries: list[tuple[str, str, str, int]] = []+ for name, tensor in raw_tensors.items():+ if name.endswith(".weight") and tensor.dtype == torch.float8_e4m3fn:+ layer_name = name[: -len(".weight")]+ scale_key = f"{layer_name}.weight_scale"+ if scale_key in raw_tensors and raw_tensors[scale_key].dtype == torch.uint8:+ entries.append((layer_name, name, scale_key, 8))+ elif name.endswith(".weight_packed") and tensor.dtype in (torch.int8, torch.uint8):+ layer_name = name[: -len(".weight_packed")]+ if (+ f"{layer_name}.weight_global_scale" in raw_tensors+ or f"{layer_name}.input_global_scale" in raw_tensors+ or f"{layer_name}.weight_scale_2" in raw_tensors+ or f"{layer_name}.input_scale" in raw_tensors+ ):+ continue+ scale_key = f"{layer_name}.weight_scale"+ if scale_key in raw_tensors and raw_tensors[scale_key].dtype == torch.uint8:+ entries.append((layer_name, name, scale_key, 4))+ return entries+++def _normalize_nvfp4_source_tensors(+ raw_tensors: dict[str, torch.Tensor],+ shard_name: str | None = None,+) -> tuple[dict[str, torch.Tensor], list[str]]:+ """Normalize legacy NVFP4 source naming to llm-compressor naming."""++ def _reciprocal_global_scale(scale: torch.Tensor) -> torch.Tensor:+ return (1.0 / scale.float()).to(torch.float32).reshape([1])++ converted_layers: list[str] = []+ candidates: list[str] = []+ for name, tensor in list(raw_tensors.items()):+ if name.endswith(".weight"):+ layer_name = name[: -len(".weight")]+ elif name.endswith(".weight_packed"):+ layer_name = name[: -len(".weight_packed")]+ else:+ continue++ if tensor.dtype not in (torch.uint8, torch.int8):+ continue+ if f"{layer_name}.weight_scale" not in raw_tensors:+ continue++ has_legacy_global = f"{layer_name}.weight_scale_2" in raw_tensors or f"{layer_name}.input_scale" in raw_tensors+ has_new_global = (+ f"{layer_name}.weight_global_scale" in raw_tensors or f"{layer_name}.input_global_scale" in raw_tensors+ )+ has_new_packed = f"{layer_name}.weight_packed" in raw_tensors+ if has_legacy_global or has_new_global or has_new_packed:+ candidates.append(layer_name)++ if not candidates:+ return raw_tensors, converted_layers++ for layer_name in candidates:+ weight_key = f"{layer_name}.weight"+ weight_packed_key = f"{layer_name}.weight_packed"+ weight_scale_2_key = f"{layer_name}.weight_scale_2"+ input_scale_key = f"{layer_name}.input_scale"+ weight_global_scale_key = f"{layer_name}.weight_global_scale"+ input_global_scale_key = f"{layer_name}.input_global_scale"++ if weight_packed_key not in raw_tensors and weight_key in raw_tensors:+ raw_tensors[weight_packed_key] = raw_tensors.pop(weight_key).view(torch.uint8).contiguous()++ if weight_scale_2_key in raw_tensors and weight_global_scale_key not in raw_tensors:+ raw_tensors[weight_global_scale_key] = raw_tensors.pop(weight_scale_2_key)+ elif weight_scale_2_key in raw_tensors:+ raw_tensors.pop(weight_scale_2_key)++ if input_scale_key in raw_tensors and input_global_scale_key not in raw_tensors:+ raw_tensors[input_global_scale_key] = raw_tensors.pop(input_scale_key)+ elif input_scale_key in raw_tensors:+ raw_tensors.pop(input_scale_key)++ if weight_global_scale_key in raw_tensors:+ raw_tensors[weight_global_scale_key] = _reciprocal_global_scale(raw_tensors[weight_global_scale_key])+ if input_global_scale_key in raw_tensors:+ raw_tensors[input_global_scale_key] = _reciprocal_global_scale(raw_tensors[input_global_scale_key])++ converted_layers.append(layer_name)++ if converted_layers:+ shard_prefix = f"[{shard_name}] " if shard_name else ""+ logger.info(f"{shard_prefix}Normalized {len(converted_layers)} legacy NVFP4 layer(s) to llm-compressor naming.")+ return raw_tensors, converted_layers+++def _handle_nvfp4_source_tensors(+ raw_tensors: dict[str, torch.Tensor],+ matcher: Any,+) -> tuple[dict[str, torch.Tensor], dict[str, torch.Tensor], list[str]]:+ """Passthrough NVFP4 source tensors when target scheme for the layer is NVFP4."""+ passthrough_tensors: dict[str, torch.Tensor] = {}+ passthrough_layers: list[str] = []++ for name, tensor in list(raw_tensors.items()):+ if not name.endswith(".weight_packed") or tensor.dtype not in (torch.uint8, torch.int8):+ continue++ layer_name = name[: -len(".weight_packed")]+ scale_key = f"{layer_name}.weight_scale"+ if scale_key not in raw_tensors:+ continue++ scheme = matcher.resolve_scheme(f"{layer_name}.weight")+ if scheme is None:+ continue+ scheme_bits = scheme.get("bits")+ scheme_data_type = (scheme.get("data_type") or "").lower()+ if not (scheme_bits == 4 and (is_nv_fp(scheme_data_type) or scheme_data_type == _NVFP4_E5M3_DATA_TYPE)):+ continue++ keys_to_move = [name, scale_key]+ weight_global_scale_key = f"{layer_name}.weight_global_scale"+ input_global_scale_key = f"{layer_name}.input_global_scale"+ if weight_global_scale_key in raw_tensors:+ keys_to_move.append(weight_global_scale_key)+ if input_global_scale_key in raw_tensors:+ keys_to_move.append(input_global_scale_key)++ for key in keys_to_move:+ value = raw_tensors.pop(key).to("cpu")+ if key.endswith("_global_scale"):+ value = value.reshape([1])+ passthrough_tensors[key] = value+ passthrough_layers.append(layer_name)++ if passthrough_layers:+ logger.info(f"Handling NVFP4 source tensor(s): {len(passthrough_layers)} passthrough layer(s).")++ return raw_tensors, passthrough_tensors, passthrough_layers+++def _is_out_of_memory_error(exc: Exception) -> bool:+ if isinstance(exc, torch.OutOfMemoryError):+ return True+ message = str(exc).lower()+ return "out of memory" in message or "cuda error: out of memory" in message+++def _dequantize_with_device_fallback(+ *,+ dequant_device: str,+ shard_prefix: str,+ op_name: str,+ tensor_label: str,+ on_device: Callable[[], torch.Tensor],+ on_cpu: Callable[[], torch.Tensor],+) -> torch.Tensor:+ """Run dequantization on ``dequant_device`` and fall back to CPU on errors."""+ if dequant_device != "cpu":+ try:+ return on_device()+ except Exception as e:+ if _is_out_of_memory_error(e):+ logger.warning(+ f"{shard_prefix}{op_name} on {dequant_device} ran OOM for {tensor_label}: {e}. "+ "Clearing accelerator memory and falling back to CPU for this tensor."+ )+ clear_memory()+ else:+ logger.warning(+ f"{shard_prefix}{op_name} on {dequant_device} failed for {tensor_label}: {e}. "+ "Falling back to CPU for this tensor."+ )+ return on_cpu()+++def _normalize_scheme(scheme: Union[str, QuantizationScheme]) -> QuantizationScheme:+ """Convert *scheme* to a :class:`QuantizationScheme` instance.++ Raises ``ValueError`` for unknown preset names and ``TypeError`` for+ unsupported types.+ """+ if isinstance(scheme, str):+ scheme_name = scheme.upper()+ if scheme_name not in PRESET_SCHEMES:+ raise ValueError(f"Unknown scheme '{scheme}'. Available: {list(PRESET_SCHEMES.keys())}")+ return preset_name_to_scheme(scheme_name)+ if isinstance(scheme, QuantizationScheme):+ return scheme+ raise TypeError(f"Unsupported scheme type: {type(scheme)}")+++# ---------------------------------------------------------------------------+# Shard Tensor Processing+# ---------------------------------------------------------------------------+++def _is_eligible_weight(tensor_name: str, tensor: torch.Tensor) -> bool:+ """Check if a tensor is eligible for quantization (2D Linear weight)."""+ return tensor_name.endswith(".weight") and tensor.dim() == 2+++def _quantize_weight_mxfp(+ weight: torch.Tensor,+ layer_name: str,+ bits: int,+ group_size: int,+ data_type: str,+ device: str = "cpu",+ disable_opt_rtn: bool = False,+) -> dict[str, torch.Tensor]:+ """Quantize a 2D weight tensor to MXFP4 / MXFP8 and return packed outputs.++ Reuses :func:`auto_round.data_type.mxfp.quant_mx` to derive the per-block+ shared exponent (E8M0 scale), and :class:`auto_round.export.export_to_autoround.qlinear_fp.QuantLinear`+ to perform the same packing as :func:`auto_round.export.export_to_llmcompressor.export_to_fp.pack_layer`.++ Returns a dict with one of:+ * MXFP8: ``{layer_name+'.weight': float8_e4m3fn, layer_name+'.weight_scale': uint8}``+ * MXFP4: ``{layer_name+'.weight_packed': uint8, layer_name+'.weight_scale': uint8}``+ """+ import torch.nn as nn++ from auto_round.data_type.utils import get_quant_func+ from auto_round.export.export_to_autoround.qlinear_fp import QuantLinear++ if not is_mx_fp(data_type):+ data_type = "mx_fp4" if bits == 4 else "mx_fp8"++ out_features, in_features = weight.shape+ if in_features % group_size != 0:+ raise ValueError(+ f"in_features={in_features} for layer '{layer_name}' is not divisible "+ f"by MXFP group_size={group_size}; cannot pack."+ )++ weight_dev = weight.to(device)+ # Use get_quant_func (same as WrapperLinear) so that all registered MXFP+ # variants automatically get opt_rtn support via the QUANT_FUNC_WITH_DTYPE+ # registry (e.g. "opt_rtn_mx_fp4" -> quant_mx_opt_rtn, "mx_fp4" -> quant_mx).+ quant_func, _ = get_quant_func(data_type, bits, sym=True, disable_opt_rtn=disable_opt_rtn, iters=0)+ weight_dev, shared_exp, _ = quant_func(weight_dev, bits=bits, group_size=group_size, data_type=data_type)+ # Reshape to (out_features, n_groups) so the on-disk weight_scale matches+ # the llm-compressor convention (and QuantLinear's registered buffer shape).+ shared_exp = shared_exp.reshape(out_features, in_features // group_size)+ # Ensure shared_exp is a numeric float (not a storage-specific dtype like+ # float8) — QuantLinear.pack performs `2 ** scales` which dispatches to+ # torch.pow; some backends do not implement pow for float8 dtypes. Cast to+ # float32 here to avoid runtime errors like "pow_cuda not implemented for+ # 'Float8_e4m3fn'" while preserving numeric values.+ shared_exp = shared_exp.to(torch.float32)++ # Build a lightweight nn.Linear holding the original weight so we can+ # delegate packing to the existing QuantLinear.pack implementation.+ fake_linear = nn.Linear(in_features, out_features, bias=False)+ with torch.no_grad():+ fake_linear.weight = nn.Parameter(weight_dev, requires_grad=False)++ qlayer = QuantLinear(+ bits=bits,+ group_size=group_size,+ infeatures=in_features,+ outfeatures=out_features,+ bias=False,+ data_type="mx_fp4" if bits == 4 else "mx_fp8e4m3",+ sym=True,+ act_bits=bits,+ )+ qlayer.pack(fake_linear, shared_exp, device=device)++ if bits == 8:+ return {+ f"{layer_name}.weight": qlayer.weight.to("cpu"),+ f"{layer_name}.weight_scale": qlayer.weight_scale.to("cpu"),+ }+ return {+ f"{layer_name}.weight_packed": qlayer.weight_packed.to("cpu"),+ f"{layer_name}.weight_scale": qlayer.weight_scale.to("cpu"),+ }+++def _quantize_weight_nvfp4_e5m3(+ weight: torch.Tensor,+ layer_name: str,+ group_size: int = 16,+ device: str = "cpu",+) -> dict[str, torch.Tensor]:+ """Fake-quantize a 2D weight tensor to NVFP4 E5M3 and return its high-precision QDQ weight."""+ from auto_round.data_type.nvfp import nvfp4_v2++ out_features, in_features = weight.shape+ if group_size != 16:+ raise ValueError(f"NVFP4_E5M3 requires group_size=16, got {group_size} for layer '{layer_name}'.")+ if in_features % group_size != 0:+ raise ValueError(+ f"in_features={in_features} for layer '{layer_name}' is not divisible "+ f"by NVFP4_E5M3 group_size={group_size}; cannot quantize."+ )++ weight_dev = weight.to(device)+ qdq_weight, _, _ = nvfp4_v2(weight_dev, bits=4, group_size=group_size)+ return {f"{layer_name}.weight": qdq_weight.to(dtype=weight.dtype, device="cpu")}+++def _pack_weight_nvfp4_e5m3(+ weight: torch.Tensor,+ layer_name: str,+ group_size: int = 16,+ device: str = "cpu",+) -> dict[str, torch.Tensor]:+ """Pack FP4 E2M1 weights with unsigned E5M3 block scales."""+ from auto_round.data_type.nvfp import nvfp4_v2+ from auto_round.export.export_to_autoround.qlinear_fp import QuantLinear++ out_features, in_features = weight.shape+ if group_size != 16 or in_features % group_size != 0:+ raise ValueError(+ f"NVFP4_E5M3 requires in_features divisible by group_size=16, got {in_features} for '{layer_name}'."+ )+ weight_dev = weight.to(device)+ _, scale, _ = nvfp4_v2(weight_dev, bits=4, group_size=group_size)+ # nvfp4_v2 may return a flattened per-group scale layout (e.g. [N, 1]);+ # normalize to [out_features, in_features // group_size] before packing+ # so serialized .weight_scale keeps the expected 2D shape.+ scale = scale.reshape(out_features, in_features // group_size).to(torch.float32)+ linear = torch.nn.Linear(in_features, out_features, bias=False, device=device, dtype=weight.dtype)+ linear.weight = torch.nn.Parameter(weight_dev, requires_grad=False)+ qlayer = QuantLinear(+ 4, group_size, in_features, out_features, False, data_type="nvfp4_v2", act_bits=4, act_data_type="nvfp4_v2"+ )+ qlayer.pack(linear, scale, device=device)+ return {+ f"{layer_name}.weight_packed": qlayer.weight_packed.to("cpu"),+ f"{layer_name}.weight_scale": qlayer.weight_scale.to("cpu"),+ }+++def _quantize_single_tensor(+ tensor_name: str,+ tensor: torch.Tensor,+ matcher: "_PatternMatcher",+ device: str = "cpu",+ quantize_func: Callable = quantize_weight_rtn,+ disable_opt_rtn: bool = False,+) -> tuple[str, dict[str, torch.Tensor], str | None, str | None]:+ """Quantize one eligible weight tensor and return packed outputs.++ Returns:+ (layer_name, output_tensors_dict, quantized_layer_or_None, ignored_layer_or_None)+ """+ layer_name = tensor_name.rsplit(".", 1)[0]++ if not _is_eligible_weight(tensor_name, tensor):+ ignored_layer = layer_name if tensor_name.endswith(".weight") and tensor.dim() > 1 else None+ return layer_name, {tensor_name: tensor}, None, ignored_layer++ if matcher.should_ignore(tensor_name):+ logger.debug(f"Ignoring (user-specified): {layer_name}")+ return layer_name, {tensor_name: tensor}, None, layer_name++ if matcher.should_skip(tensor_name):+ logger.debug(f"Skipping (predefined): {layer_name}")+ return layer_name, {tensor_name: tensor}, None, layer_name++ scheme = matcher.resolve_scheme(tensor_name)+ if scheme is None:+ logger.debug(f"Keeping full precision: {layer_name}")+ return layer_name, {tensor_name: tensor}, None, layer_name++ bits = scheme["bits"]+ group_size = scheme["group_size"]+ sym = scheme.get("sym", True)+ data_type = (scheme.get("data_type") or "int").lower()++ if bits >= 16:+ return layer_name, {tensor_name: tensor}, None, layer_name++ # ---- MXFP path (MXFP4 / MXFP8) ----+ if is_mx_fp(data_type):+ try:+ out = _quantize_weight_mxfp(+ weight=tensor,+ layer_name=layer_name,+ bits=bits,+ group_size=group_size,+ data_type=data_type,+ device=device,+ disable_opt_rtn=disable_opt_rtn,+ )+ logger.debug(f"Quantized (MXFP): {layer_name} (bits={bits}, group_size={group_size})")+ return layer_name, out, layer_name, None+ except Exception as e:+ logger.warning(f"Failed to MXFP-quantize {layer_name}: {e}. Keeping original weight.")+ return layer_name, {tensor_name: tensor}, None, layer_name++ # ---- NVFP4 E5M3 fake-quantization path ----+ if data_type == _NVFP4_E5M3_DATA_TYPE:+ try:+ quantize_e5m3 = (+ _quantize_weight_nvfp4_e5m3 if scheme.get("_output_format") == "fake" else _pack_weight_nvfp4_e5m3+ )+ out = quantize_e5m3(+ weight=tensor,+ layer_name=layer_name,+ group_size=group_size,+ device=device,+ )+ logger.debug(f"Quantized (NVFP4_E5M3): {layer_name} (bits=4, group_size={group_size})")+ return layer_name, out, layer_name, None+ except Exception as e:+ logger.warning(f"Failed to NVFP4_E5M3-quantize {layer_name}: {e}. Keeping original weight.")+ return layer_name, {tensor_name: tensor}, None, layer_name++ # ---- Integer WOQ path ----+ # opt_rtn is always disabled for integer WOQ in model-free mode because+ # the scale search does not improve accuracy for INT quantization here.+ try:+ qweight, qzeros, scales = quantize_func(+ weight=tensor,+ bits=bits,+ group_size=group_size,+ sym=sym,+ device=device,+ disable_opt_rtn=True,+ )++ out: dict[str, torch.Tensor] = {+ f"{layer_name}.qweight": qweight,+ f"{layer_name}.qzeros": qzeros,+ f"{layer_name}.scales": scales,+ }++ logger.debug(f"Quantized: {layer_name} (bits={bits}, group_size={group_size}, sym={sym})")+ return layer_name, out, layer_name, None++ except Exception as e:+ logger.warning(f"Failed to quantize {layer_name}: {e}. Keeping original weight.")+ return layer_name, {tensor_name: tensor}, None, layer_name+++@lru_cache(maxsize=32)+def _load_weight_map_from_index(index_path: str) -> dict[str, str]:+ """Load weight_map from an index file with a small process-local cache."""+ with open(index_path) as f:+ index = json.load(f)+ weight_map = index.get("weight_map", {})+ return weight_map if isinstance(weight_map, dict) else {}+++def _build_cross_shard_pairs_from_weight_map(+ weight_map: dict[str, str],+) -> tuple[dict[str, dict[str, list[str]]], dict[str, set[str]]]:+ """Identify cross-shard FP8 (weight, weight_scale_inv) pairs from weight_map.++ A *cross-shard pair* exists when ``<layer>.weight`` and+ ``<layer>.weight_scale_inv`` reside in **different** shards. The shard+ that holds ``weight_scale_inv`` is the *donor*; the shard that holds+ ``weight`` is the *recipient*.++ Returns:+ recipient_to_donors:+ ``{recipient_shard: {donor_shard: [scale_inv_tensor_names]}}``+ donor_shard_tensors:+ ``{donor_shard: set(scale_inv tensor names it donates to other shards)}``+ """+ recipient_to_donors: dict[str, dict[str, list[str]]] = {}+ donor_shard_tensors: dict[str, set[str]] = {}++ for tensor_name, shard in weight_map.items():+ if not tensor_name.endswith(".weight_scale_inv"):+ continue+ # "layer.weight_scale_inv" -> "layer.weight"+ weight_name = tensor_name[: -len("_scale_inv")]+ weight_shard = weight_map.get(weight_name)+ if not weight_shard or weight_shard == shard:+ continue+ # `shard` (donor) donates `tensor_name` to `weight_shard` (recipient)+ recipient_to_donors.setdefault(weight_shard, {}).setdefault(shard, []).append(tensor_name)+ donor_shard_tensors.setdefault(shard, set()).add(tensor_name)++ return recipient_to_donors, donor_shard_tensors+++def _hydrate_missing_fp8_scales_from_index(+ raw_tensors: dict[str, torch.Tensor],+ shard_path: str,+ *,+ shard_name: str | None = None,+ index_dir: str | None = None,+ donor_shard_dir: str | None = None,+) -> dict[str, torch.Tensor]:+ """Populate missing ``.weight_scale_inv`` tensors from sibling shards.++ Some checkpoints shard FP8 weight and its corresponding scale tensor into+ different ``.safetensors`` files. Model-free processing is shard-local, so+ this helper hydrates missing ``<layer>.weight_scale_inv`` tensors by looking+ up ``weight_map`` in ``*.safetensors.index.json`` and loading only the+ needed tensors from referenced shards.++ Args:+ index_dir: Directory that contains ``*.safetensors.index.json``. When+ ``None`` (default) the directory of *shard_path* is used, which is+ correct for non-streaming mode. Streaming mode should pass+ ``work_dir`` here because downloaded shards live in a+ ``.cache/model_free_source_shards/`` sub-directory that does not+ contain the index file.+ donor_shard_dir: Directory where donor shard files can be found. When+ ``None`` (default) the same directory as *index_dir* is used.+ Streaming mode should pass the local shard cache directory so that+ already-downloaded donor shards are resolved correctly.+ """+ if not shard_path.endswith(".safetensors"):+ return raw_tensors++ weight_to_scale: dict[str, str] = {}+ for name, tensor in raw_tensors.items():+ if not name.endswith(".weight"):+ continue+ if tensor.dtype != torch.float8_e4m3fn:+ continue+ scale_inv_name = f"{name[: -len('.weight')]}.weight_scale_inv"+ if scale_inv_name not in raw_tensors:+ weight_to_scale[name] = scale_inv_name++ if not weight_to_scale:+ return raw_tensors++ shard_dir = os.path.dirname(shard_path)+ # Resolve the directory where index.json lives. In streaming mode the+ # shard lives in a .cache/ sub-directory that has no index file; callers+ # pass index_dir=work_dir to point at the correct location.+ idx_dir = index_dir if index_dir is not None else shard_dir+ # Donor shards are looked up in donor_shard_dir (defaults to idx_dir).+ donor_dir = donor_shard_dir if donor_shard_dir is not None else idx_dir++ index_path = os.path.join(idx_dir, "model.safetensors.index.json")+ if not os.path.exists(index_path):+ candidates = sorted(+ os.path.join(idx_dir, f) for f in os.listdir(idx_dir) if f.endswith(".safetensors.index.json")+ )+ if not candidates:+ return raw_tensors+ index_path = candidates[0]++ try:+ weight_map = _load_weight_map_from_index(index_path)+ except Exception:+ return raw_tensors++ current_shard = os.path.basename(shard_path)+ scales_by_shard: dict[str, list[str]] = {}+ for scale_name in weight_to_scale.values():+ target_shard = weight_map.get(scale_name)+ if not target_shard or target_shard == current_shard:+ continue+ scales_by_shard.setdefault(target_shard, []).append(scale_name)++ if not scales_by_shard:+ return raw_tensors++ from safetensors import safe_open++ hydrated = 0+ shard_prefix = f"[{shard_name}] " if shard_name else ""+ for target_shard, scale_names in scales_by_shard.items():+ target_path = os.path.join(donor_dir, target_shard)+ if not os.path.exists(target_path):+ logger.warning(+ f"{shard_prefix}Donor shard '{target_shard}' not found in '{donor_dir}' while hydrating "+ f"{len(scale_names)} FP8 scale_inv tensor(s); the affected weight(s) will remain in "+ f"float8_e4m3fn and may fail downstream quantization. This usually indicates a shard "+ f"scheduling/ordering issue (donor shard processed/downloaded after its recipient)."+ )+ continue+ try:+ with safe_open(target_path, framework="pt", device="cpu") as sf:+ for scale_name in scale_names:+ if scale_name in raw_tensors:+ continue+ try:+ raw_tensors[scale_name] = sf.get_tensor(scale_name)+ hydrated += 1+ except Exception:+ # Tensor may be absent in this shard; skip lazily.+ continue+ except Exception:+ continue++ if hydrated:+ logger.info(f"{shard_prefix}Hydrated {hydrated} FP8 scale tensor(s) from sibling shard(s) using index mapping.")++ return raw_tensors+++def _dequant_mxfp_tensors(+ raw_tensors: dict[str, torch.Tensor],+ device: str = "cpu",+ shard_name: str | None = None,+) -> dict[str, torch.Tensor]:+ """Dequantize llm-compressor MXFP8 / MXFP4 weight tensors to bfloat16.++ Detection is purely by *name* and *dtype*, reusing the dequant kernels in+ :mod:`auto_round_extension.vllm_ext`:++ * ``<layer>.weight`` (``float8_e4m3fn``) + ``<layer>.weight_scale`` → MXFP8,+ dequantized via :func:`~auto_round_extension.vllm_ext.mxfp8_qdq_utils.dequant_mx_fp8`.+ * ``<layer>.weight_packed`` (``uint8``) + ``<layer>.weight_scale`` → MXFP4,+ dequantized via :func:`~auto_round_extension.vllm_ext.mxfp4_qdq_utils.to_dtype`.++ The dequantized weight is written back under ``<layer>.weight`` and the+ scale (and any ``weight_packed``) tensor is removed, so the downstream RTN+ path can requantize the layer to the requested target scheme.+ """+ from auto_round_extension.vllm_ext.mxfp4_qdq_utils import to_dtype+ from auto_round_extension.vllm_ext.mxfp8_qdq_utils import dequant_mx_fp8++ # Tuple layout: (layer_name, weight_key, scale_key, bits)+ entries = _collect_mxfp_source_entries(raw_tensors)++ if not entries:+ return raw_tensors++ n_mxfp8 = sum(1 for _layer_name, _weight_key, _scale_key, bits in entries if bits == 8)+ n_mxfp4 = len(entries) - n_mxfp8+ dequant_device = str(device or "cpu")+ shard_prefix = f"[{shard_name}] " if shard_name else ""+ logger.info(+ f"{shard_prefix}Dequantizing MXFP tensor(s) to bfloat16 on {dequant_device}: "+ f"MXFP8={n_mxfp8}, MXFP4={n_mxfp4}, total={len(entries)}."+ )++ for layer_name, weight_key, scale_key, bits in entries:+ weight = raw_tensors.pop(weight_key)+ scale = raw_tensors.pop(scale_key).view(torch.uint8)+ if bits == 8:+ dq_weight = _dequantize_with_device_fallback(+ dequant_device=dequant_device,+ shard_prefix=shard_prefix,+ op_name="MXFP dequant",+ tensor_label=layer_name,+ on_device=lambda: dequant_mx_fp8(+ weight_fp8=weight.to(dequant_device, non_blocking=True),+ scale_e8m0=scale.to(dequant_device, non_blocking=True),+ block_size=32,+ target_dtype=torch.bfloat16,+ ).to("cpu"),+ on_cpu=lambda: dequant_mx_fp8(+ weight_fp8=weight,+ scale_e8m0=scale,+ block_size=32,+ target_dtype=torch.bfloat16,+ ),+ )+ else:+ dq_weight = _dequantize_with_device_fallback(+ dequant_device=dequant_device,+ shard_prefix=shard_prefix,+ op_name="MXFP dequant",+ tensor_label=layer_name,+ on_device=lambda: to_dtype(+ data_lp=weight.view(torch.uint8).contiguous().to(dequant_device, non_blocking=True),+ scale_e8m0=scale.to(dequant_device, non_blocking=True),+ elem_dtype="fp4_e2m1",+ block_size=32,+ target_dtype=torch.bfloat16,+ ).to("cpu"),+ on_cpu=lambda: to_dtype(+ data_lp=weight.view(torch.uint8).contiguous(),+ scale_e8m0=scale,+ elem_dtype="fp4_e2m1",+ block_size=32,+ target_dtype=torch.bfloat16,+ ),+ )+ raw_tensors[f"{layer_name}.weight"] = dq_weight++ return raw_tensors+++def _handle_mxfp_source_tensors(+ raw_tensors: dict[str, torch.Tensor],+ matcher: "_PatternMatcher",+ source_state: dict[str, int] | None = None,+ device: str = "cpu",+ shard_name: str | None = None,+) -> tuple[dict[str, torch.Tensor], dict[str, torch.Tensor], list[str]]:+ """Handle llm-compressor-style MXFP8/MXFP4 source tensors.++ Detects llm-compressor MXFP tensors purely by *name + dtype*:++ * ``<layer>.weight`` (``float8_e4m3fn``) + ``<layer>.weight_scale`` → MXFP8.+ * ``<layer>.weight_packed`` (``uint8``) + ``<layer>.weight_scale`` → MXFP4.++ For each detected layer the effective target scheme is resolved via *matcher*:++ * If the target is the **same MXFP format** (``data_type='mx_fp'``, matching+ ``bits``), the tensors are emitted directly as a passthrough — no+ dequantization is performed and the layer is recorded as already quantized.+ * Otherwise the tensors are dequantized to ``bfloat16`` via+ :func:`_dequant_mxfp_tensors` so the downstream RTN path can re-quantize+ them to the requested target scheme.++ Returns:+ ``(raw_tensors, passthrough_tensors, passthrough_layers)``.+ """+ entries = _collect_mxfp_source_entries(raw_tensors)+ if not entries:+ return raw_tensors, {}, []++ source_state = source_state or {}++ passthrough_tensors: dict[str, torch.Tensor] = {}+ passthrough_layers: list[str] = []+ n_dequant = 0++ for layer_name, weight_key, scale_key, bits in entries:+ scheme = matcher.resolve_scheme(f"{layer_name}.weight")+ is_ignored = matcher.should_ignore(f"{layer_name}.weight")+ target_is_same_mxfp = (+ scheme is not None and is_mx_fp((scheme.get("data_type") or "").lower()) and scheme.get("bits") == bits+ )+ if target_is_same_mxfp and not is_ignored:+ passthrough_tensors[weight_key] = raw_tensors.pop(weight_key).to("cpu")+ passthrough_tensors[scale_key] = raw_tensors.pop(scale_key).to("cpu")+ passthrough_layers.append(layer_name)+ else:+ n_dequant += 1++ if n_dequant:+ raw_tensors = _dequant_mxfp_tensors(raw_tensors, device=device, shard_name=shard_name)++ parts: list[str] = []+ if passthrough_layers:+ parts.append(f"{len(passthrough_layers)} passthrough")+ if n_dequant:+ parts.append(f"{n_dequant} dequantized to bfloat16")+ if source_state:+ parts.append(f"{len(source_state)} model_type-normalized")+ logger.info(f"Handling MXFP source tensor(s): {', '.join(parts)}.")++ return raw_tensors, passthrough_tensors, passthrough_layers+++def _dequant_fp8_tensors(+ raw_tensors: dict[str, torch.Tensor],+ block_size: list | None = None,+ device: str = "cpu",+ shard_name: str | None = None,+ shard_path: str | None = None,+ *,+ index_dir: str | None = None,+ donor_shard_dir: str | None = None,+) -> dict[str, torch.Tensor]:+ """Dequantize DeepSeek-V3-style FP8 weight tensors to bfloat16.++ Handles the **DeepSeek-V3 FP8** convention: weight dtype ``float8_e4m3fn``+ paired with a ``.weight_scale_inv`` tensor (per-block float32 scales, NOT+ E8M0). The weights are converted to ``bfloat16`` so downstream RTN+ quantization can proceed normally.++ MXFP sources are handled separately by+ :func:`preprocess_model_type_source_tensors` / :func:`_handle_mxfp_source_tensors`.+ """+ from auto_round.utils.weight_handler import _dequant_fp8_linear_weight++ if shard_path:+ raw_tensors = _hydrate_missing_fp8_scales_from_index(+ raw_tensors,+ shard_path,+ shard_name=shard_name,+ index_dir=index_dir,+ donor_shard_dir=donor_shard_dir,+ )++ quant_entries: list[tuple[str, str]] = []+ for name, tensor in raw_tensors.items():+ if not name.endswith(".weight"):+ continue+ if tensor.dtype != torch.float8_e4m3fn:+ continue+ # DeepSeek-V3 style: .weight_scale_inv (per-block float32 scales).+ scale_inv_name = f"{name[: -len('.weight')]}.weight_scale_inv"+ if scale_inv_name in raw_tensors:+ quant_entries.append((name, scale_inv_name))++ if not quant_entries:+ return raw_tensors++ # device has already been resolved by the caller; use it directly here.+ dequant_device = str(device or "cpu")+ shard_prefix = f"[{shard_name}] " if shard_name else ""++ logger.info(+ f"{shard_prefix}Dequantizing {len(quant_entries)} FP8 weight tensor(s) to bfloat16 on {dequant_device}."+ )++ for weight_name, scale_name in quant_entries:+ weight = raw_tensors[weight_name]+ scale = raw_tensors.pop(scale_name)++ # Dequantize on GPU for throughput, then move back to CPU to keep+ # per-shard memory usage bounded before per-layer quantization.+ raw_tensors[weight_name] = _dequantize_with_device_fallback(+ dequant_device=dequant_device,+ shard_prefix=shard_prefix,+ op_name="FP8 dequant",+ tensor_label=weight_name,+ on_device=lambda: _dequant_fp8_linear_weight(+ weight.to(dequant_device, non_blocking=True),+ scale.to(dequant_device, non_blocking=True),+ block_size=block_size,+ ).to("cpu"),+ on_cpu=lambda: _dequant_fp8_linear_weight(weight, scale, block_size=block_size),+ )++ return raw_tensors+++def _process_shard(+ shard_path: str,+ default_scheme: dict = None,+ layer_config: dict = None,+ ignore_patterns: list[str] = None,+ device: str = "cpu",+ *,+ shard_name: str | None = None,+ matcher: "_PatternMatcher | None" = None,+ fp8_block_size: list | None = None,+ model_type: str | None = None,+ source_quantization_config: dict | None = None,+ enable_torch_compile: bool = False,+ disable_opt_rtn: bool = False,+ index_dir: str | None = None,+ donor_shard_dir: str | None = None,+ donor_tensors_to_exclude: set[str] | None = None,+) -> tuple[dict[str, torch.Tensor], list[str], list[str]]:+ """Quantize eligible weights in a single safetensors shard.++ Returns:+ (output_tensors, quantized_layer_names, ignored_layer_names)++ ``ignored_layer_names`` is derived by comparing the set of input ``.weight``+ layer names (collected after fused-expert splitting) with the final set of+ quantized layer names. Any layer that had a ``.weight`` tensor in the input+ but was NOT quantized is reported as ignored — this correctly captures+ user-ignored layers, predefined-skipped layers, non-eligible weights, and+ any other pass-through case without separate per-tensor tracking.++ Args:+ index_dir: Directory containing the model's ``*.safetensors.index.json``.+ Used to resolve cross-shard FP8 scale_inv tensors. Defaults to the+ directory of *shard_path* (correct for non-streaming mode).+ donor_shard_dir: Directory where donor shards are cached. Defaults to+ *index_dir*. In streaming mode pass the local shard cache directory.+ donor_tensors_to_exclude: If provided, these ``weight_scale_inv`` tensor+ names are removed from the final output tensors. Used when this+ shard is a donor: its scale_inv tensors have already been consumed+ by the recipient shard and must not appear in the quantized output.+ """+ if matcher is None:+ matcher = _PatternMatcher(+ ignore_patterns if ignore_patterns is not None else [],+ layer_config if layer_config is not None else {},+ default_scheme if default_scheme is not None else {},+ )++ output_tensors: dict[str, torch.Tensor] = {}+ quantized_layers: list[str] = []+ quantize_func = compile_func(quantize_weight_rtn, device) if enable_torch_compile else quantize_weight_rtn++ if shard_path.endswith(".bin"):+ # PyTorch pickle checkpoint — load with weights_only where supported.+ try:+ raw_tensors = torch.load(shard_path, map_location="cpu", weights_only=True)+ except TypeError:+ # weights_only not available in older PyTorch versions+ raw_tensors = torch.load(shard_path, map_location="cpu") # nosec+ # Flatten nested state-dict wrappers if present.+ if not isinstance(raw_tensors, dict):+ raise ValueError(f"Expected a dict from {shard_path}, got {type(raw_tensors)}")+ else:+ from safetensors import safe_open++ with safe_open(shard_path, framework="pt", device="cpu") as f:+ raw_tensors = {name: f.get_tensor(name) for name in f.keys()}++ raw_tensors = split_fused_expert_tensors(raw_tensors)++ # Hydrate cross-shard FP8 weight_scale_inv tensors *before* any+ # preprocessing below. Otherwise a weight whose scale lives in a sibling+ # shard would miss the model_type-specific "expand-scale" passthrough+ # handling (step 1) simply because its scale wasn't loaded yet, and would+ # incorrectly fall through to the generic dequant-to-bf16 + RTN-requantize+ # path instead of being treated identically to same-shard weights.+ if shard_path:+ raw_tensors = _hydrate_missing_fp8_scales_from_index(+ raw_tensors,+ shard_path,+ shard_name=shard_name,+ index_dir=index_dir,+ donor_shard_dir=donor_shard_dir,+ )++ # Snapshot candidate weight layer names *before* any preprocessing. 1D+ # weights (for example LayerNorm) are not quantization targets, while 3D+ # weights remain tracked so unsupported layouts are visible as ignored.+ input_weight_layers: list[str] = list(+ dict.fromkeys(+ name.rsplit(".", 1)[0]+ for name, tensor in raw_tensors.items()+ if name.endswith(".weight") and tensor.dim() > 1+ )+ )++ # Preserve original tensors for predefined skipped layers so that already-+ # quantized weights (FP8, FP4-packed, etc.) are NOT dequantized.+ # User-specified ignore layers should still flow through the dequant path+ # so the saved model exports them in full precision.+ preserved_prefixes: set[str] = set()+ for tname in raw_tensors:+ if (+ tname.endswith(".weight") or tname.endswith(".weight_packed") or tname.endswith(".qweight")+ ) and matcher.should_skip(tname):+ preserved_prefixes.add(tname.rsplit(".", 1)[0])++ preserved_tensors: dict[str, torch.Tensor] = {}+ if preserved_prefixes:+ for key in list(raw_tensors.keys()):+ prefix = key.rsplit(".", 1)[0]+ if prefix in preserved_prefixes:+ preserved_tensors[key] = raw_tensors.pop(key)++ # 1) model-type-specific preprocessing (format conversion only)+ raw_tensors, source_state = preprocess_model_type_source_tensors(+ raw_tensors,+ model_type=model_type,+ quantization_config=source_quantization_config,+ shard_name=shard_name,+ )++ # 1.5) normalize legacy NVFP4 names to llm-compressor naming.+ raw_tensors, _converted_nvfp4_layers = _normalize_nvfp4_source_tensors(raw_tensors, shard_name=shard_name)++ # 1.5) model-type-specific low-precision dequantization (e.g. kimi_k25 INT4)+ raw_tensors = handle_model_type_low_precision_source_tensors(+ raw_tensors,+ model_type=model_type,+ source_quant_config=source_quantization_config,+ device=device,+ shard_name=shard_name,+ dequantize_with_device_fallback=_dequantize_with_device_fallback,+ )++ # 2) generic MXFP handling for both preprocessed and normal source models+ raw_tensors, passthrough_tensors, passthrough_layers = _handle_mxfp_source_tensors(+ raw_tensors,+ matcher,+ source_state=source_state,+ device=device,+ shard_name=shard_name,+ )+ output_tensors.update(passthrough_tensors)+ quantized_layers.extend(passthrough_layers)++ # 3) NVFP4 passthrough for layers already stored in packed format.+ raw_tensors, nvfp_passthrough_tensors, nvfp_passthrough_layers = _handle_nvfp4_source_tensors(+ raw_tensors,+ matcher,+ )+ output_tensors.update(nvfp_passthrough_tensors)+ quantized_layers.extend(nvfp_passthrough_layers)++ raw_tensors = _dequant_fp8_tensors(+ raw_tensors,+ block_size=fp8_block_size,+ device=device,+ shard_name=shard_name,+ shard_path=shard_path,+ index_dir=index_dir,+ donor_shard_dir=donor_shard_dir,+ )+ raw_tensors.update(preserved_tensors)++ for tensor_name in list(raw_tensors.keys()):+ tensor = raw_tensors.pop(tensor_name)+ _layer_name, out_dict, q_layer, _ig_layer = _quantize_single_tensor(+ tensor_name,+ tensor,+ matcher,+ device,+ quantize_func,+ disable_opt_rtn,+ )+ output_tensors.update(out_dict)+ if q_layer:+ quantized_layers.append(q_layer)++ # Remove scale_inv tensors that this shard donates to other shards.+ # These tensors have no corresponding weight in this shard; keeping them+ # would pollute the quantized output with raw FP8 scale metadata.+ if donor_tensors_to_exclude:+ removed = {t for t in donor_tensors_to_exclude if t in output_tensors}+ for t in removed:+ del output_tensors[t]+ if removed:+ shard_prefix = f"[{shard_name}] " if shard_name else ""+ logger.debug(+ f"{shard_prefix}Excluded {len(removed)} donated cross-shard scale_inv "+ f"tensor(s) from quantized shard output."+ )++ # Derive ignored layers by comparing input weight layers with quantized set.+ quantized_set = set(quantized_layers)+ ignored_layers: list[str] = [l for l in input_weight_layers if l not in quantized_set]++ return output_tensors, quantized_layers, ignored_layers+++# ---------------------------------------------------------------------------+# Model Source I/O and Shard Discovery+# ---------------------------------------------------------------------------+++def _get_model_cache_status(model_name_or_path: str) -> tuple[bool, str]:+ """Return cache decision and a short reason string.++ Cached means local dir, or HF cache contains config plus at least one+ weight entry file (index or single-file checkpoint).+ """+ if os.path.isdir(model_name_or_path):+ return True, "input is an existing local directory"++ try:+ from huggingface_hub import try_to_load_from_cache++ # ``config.json`` alone is not enough: many workflows prefetch only+ # config/tokenizer files, which would otherwise route to full+ # ``snapshot_download``.+ config_cached = isinstance(try_to_load_from_cache(model_name_or_path, "config.json"), str)+ if not config_cached:+ return False, "HF cache miss: config.json not found"++ weight_entry_candidates = (+ "model.safetensors.index.json",+ "pytorch_model.bin.index.json",+ "model.safetensors",+ "pytorch_model.bin",+ )+ hit_entries: list[str] = []+ for fname in weight_entry_candidates:+ if isinstance(try_to_load_from_cache(model_name_or_path, fname), str):+ hit_entries.append(fname)++ if hit_entries:+ return True, f"HF cache hit: config.json + {', '.join(hit_entries)}"+ return (+ False,+ "HF cache partial hit: config.json exists but no weight entry file found",+ )+ except Exception as e:+ return False, f"cache probe failed: {type(e).__name__}: {e}"+++def _is_model_cached(model_name_or_path: str) -> bool:+ """Return True if the model is already available locally or in HF cache."""+ cached, _ = _get_model_cache_status(model_name_or_path)+ return cached+++def _resolve_source_dir(model_name_or_path: str) -> str:+ """Resolve model_name_or_path to a local directory (download if needed)."""+ if os.path.isdir(model_name_or_path):+ return model_name_or_path+ from huggingface_hub import snapshot_download++ return snapshot_download(model_name_or_path)+++def _load_config(source_dir: str) -> dict:+ """Load config.json from model directory."""+ config_path = os.path.join(source_dir, "config.json")+ if not os.path.exists(config_path):+ raise FileNotFoundError(f"config.json not found in {source_dir}")+ with open(config_path) as f:+ return json.load(f)+++def _list_weight_shards(source_dir: str) -> list[str]:+ """Return list of weight shard filenames in order.++ Safetensors shards are preferred. When no safetensors files are+ found the function falls back to PyTorch ``.bin`` shards.++ Handles both standard naming (``model.safetensors``,+ ``pytorch_model.bin``) and custom prefixes such as+ ``diffusion_pytorch_model-XXXXX-of-XXXXX.safetensors`` by scanning+ all ``*.safetensors.index.json`` / ``*.bin.index.json`` index files+ in the directory when no standard index is found.+ """++ def _shards_from_index(index_path: str) -> list[str]:+ with open(index_path) as f:+ index = json.load(f)+ seen: set[str] = set()+ shards: list[str] = []+ for shard_file in index["weight_map"].values():+ if shard_file not in seen:+ seen.add(shard_file)+ shards.append(shard_file)+ return shards++ # --- safetensors: standard index ---+ st_index = os.path.join(source_dir, "model.safetensors.index.json")+ if os.path.exists(st_index):+ return _shards_from_index(st_index)++ # --- safetensors: custom-prefix index (e.g. diffusion_pytorch_model.safetensors.index.json) ---+ for fname in sorted(os.listdir(source_dir)):+ if fname.endswith(".safetensors.index.json"):+ return _shards_from_index(os.path.join(source_dir, fname))++ # --- safetensors: single file or index-less multi-file shards ---+ st_files = sorted(f for f in os.listdir(source_dir) if f.endswith(".safetensors"))+ if len(st_files) >= 1:+ return st_files++ # --- pytorch .bin: standard index ---+ bin_index = os.path.join(source_dir, "pytorch_model.bin.index.json")+ if os.path.exists(bin_index):+ return _shards_from_index(bin_index)++ # --- pytorch .bin: custom-prefix index ---+ for fname in sorted(os.listdir(source_dir)):+ if fname.endswith(".bin.index.json"):+ return _shards_from_index(os.path.join(source_dir, fname))++ # --- pytorch .bin: single file ---+ bin_single = os.path.join(source_dir, "pytorch_model.bin")+ if os.path.exists(bin_single):+ return ["pytorch_model.bin"]++ # --- pytorch .bin: any single .bin file ---+ bin_files = sorted(f for f in os.listdir(source_dir) if f.endswith(".bin"))+ if len(bin_files) >= 1:+ return bin_files+++def _is_weight_shard(fname: str) -> bool:+ """Return True if *fname* is a weight shard (safetensors or .bin).++ Excludes index files (``*.index.json``) so that they are copied to the+ output directory as normal metadata.+ """+ if fname.endswith(".index.json"):+ return False+ return fname.endswith(".safetensors") or fname.endswith(".bin")+++# Keep old name as an alias for backward compatibility.+_is_safetensors_shard = _is_weight_shard+++def _download_single_shard(+ model_name_or_path: str,+ shard_filename: str,+ local_dir: str,+) -> str:+ """Download a single safetensors shard file. Returns the local path."""+ os.makedirs(local_dir, exist_ok=True)+ local_path = os.path.join(local_dir, shard_filename)+ if os.path.exists(local_path):+ logger.info(f"Shard '{shard_filename}' already exists at '{local_path}', skipping download.")+ return local_path++ if os.path.isdir(model_name_or_path):+ src = os.path.join(model_name_or_path, shard_filename)+ if os.path.exists(src):+ shutil.copy2(src, local_path)+ return local_path+ raise FileNotFoundError(f"{shard_filename} not found in {model_name_or_path}")++ from huggingface_hub import hf_hub_download++ return hf_hub_download(+ repo_id=model_name_or_path,+ filename=shard_filename,+ local_dir=local_dir,+ )+++def _download_metadata_files(+ model_name_or_path: str,+ local_dir: str,+) -> str:+ """Download all non-safetensors files from a model repo. Returns local dir."""+ os.makedirs(local_dir, exist_ok=True)++ if os.path.isdir(model_name_or_path):+ for fname in os.listdir(model_name_or_path):+ if _is_weight_shard(fname):+ continue+ src = os.path.join(model_name_or_path, fname)+ dst = os.path.join(local_dir, fname)+ if os.path.isdir(src):+ if not os.path.exists(dst):+ shutil.copytree(src, dst)+ elif os.path.isfile(src) and not os.path.exists(dst):+ shutil.copy2(src, dst)+ return local_dir++ from huggingface_hub import snapshot_download++ snapshot_download(+ repo_id=model_name_or_path,+ local_dir=local_dir,+ ignore_patterns=["*.safetensors", "*.bin", "*.pth", "*.pt"],+ )+ return local_dir+++# ---------------------------------------------------------------------------+# ---------------------------------------------------------------------------+# Output Writers+# ---------------------------------------------------------------------------+++def _write_output_shard(+ output_dir: str,+ shard_name: str,+ tensors: dict[str, torch.Tensor],+ weight_map: dict[str, str],+):+ """Write a single output shard and update the weight_map."""+ from safetensors.torch import save_file++ shard_path = os.path.join(output_dir, shard_name)++ # Detect shared-storage tensors (e.g. tie_word_embeddings: wte ↔ lm_head).+ # safetensors refuses to serialise them as-is; clone the duplicates so each+ # tensor occupies its own memory region. The first occurrence keeps the+ # original storage; subsequent aliases are cloned.+ seen_data_ptrs: set[int] = set()+ deduped: dict[str, torch.Tensor] = {}+ for k, v in tensors.items():+ if not v.is_contiguous():+ v = v.contiguous()+ ptr = v.data_ptr()+ if ptr in seen_data_ptrs:+ v = v.clone()+ else:+ seen_data_ptrs.add(ptr)+ deduped[k] = v++ save_file(deduped, shard_path)+ for tensor_name in tensors:+ weight_map[tensor_name] = shard_name+++def _write_index_file(output_dir: str, weight_map: dict[str, str]):+ """Write model.safetensors.index.json (or rename single shard)."""+ if len(set(weight_map.values())) <= 1:+ shard_names = list(set(weight_map.values()))+ if shard_names and shard_names[0] != "model.safetensors":+ src = os.path.join(output_dir, shard_names[0])+ dst = os.path.join(output_dir, "model.safetensors")+ if os.path.exists(src):+ os.rename(src, dst)+ weight_map = {k: "model.safetensors" for k in weight_map}+ return++ index = {"metadata": {"total_size": 0}, "weight_map": weight_map}+ with open(os.path.join(output_dir, "model.safetensors.index.json"), "w") as f:+ json.dump(index, f, indent=2)+++# ---------------------------------------------------------------------------+# Quantization Configuration+# ---------------------------------------------------------------------------+++def _get_llm_compressor_metadata() -> dict[str, str]:+ """Return AutoRound provenance for model-free llm-compressor output."""+ # Keep metadata deterministic and installation-source agnostic.+ return {"provider": "auto-round"}+++def _build_nvfp4_e5m3_quantization_config(ignored_layers: list[str]) -> dict:+ """Build compressed-tensors metadata for NVFP4 E5M3 without global scales."""+ from auto_round.export.export_to_llmcompressor.config import initialize_nvfp4_e5m3_quantization++ qconfig = initialize_nvfp4_e5m3_quantization(ignore=ignored_layers)+ qconfig.update(_get_llm_compressor_metadata())+ return qconfig+++def _get_mxfp_group_scheme_and_format(+ group_bits: int,+ group_data_type: str,+ ignore: list[str],+):+ """Return ``(QuantizationScheme, format_str)`` for a single (bits, data_type) group.++ Handles MXFP (mx_fp), NVFP4 (nv_fp), and NVFP4_E5M3 (nvfp4_v2) groups.+ """+ from auto_round.export.export_to_llmcompressor.config import (+ initialize_nvfp4_e5m3_quantization,+ initialize_quantization,+ )++ if is_nv_fp(group_data_type):+ # Standard NVFP4 uses global scales and has a dedicated preset in+ # compressed-tensors. Reuse that preset so metadata matches tensor+ # layout when model-free does NVFP4 passthrough.+ tmp_qconfig = initialize_quantization(scheme="NVFP4", ignore=ignore)+ return tmp_qconfig.config_groups["group_0"], "nvfp4-pack-quantized"++ if group_data_type == _NVFP4_E5M3_DATA_TYPE:+ # NVFP4_E5M3 is not a compressed-tensors preset; use its dedicated builder.+ nvfp4_dict = initialize_nvfp4_e5m3_quantization(ignore=ignore)+ # Extract the QuantizationScheme object from the dict config.+ from compressed_tensors.quantization import QuantizationScheme # pylint: disable=E0401++ raw_scheme = nvfp4_dict["config_groups"]["group_0"]+ scheme_obj = QuantizationScheme.model_validate(raw_scheme)+ return scheme_obj, "nvfp4-e5m3-pack-quantized"+ else:+ scheme_name = "MXFP4" if group_bits == 4 else "MXFP8"+ fmt = "mxfp4-pack-quantized" if group_bits == 4 else "mxfp8-quantized"+ tmp_qconfig = initialize_quantization(scheme=scheme_name, ignore=ignore)+ return tmp_qconfig.config_groups["group_0"], fmt+++def _build_mxfp_quantization_config(+ default_scheme: dict,+ quantized_layers: list[str],+ ignored_layers: list[str],+ layer_config: dict | None = None,+) -> dict:+ """Build a compressed-tensors / llm-compressor style quantization_config+ dict for MXFP4 / MXFP8 model-free output, including mixed-precision cases.++ When *layer_config* contains layers that override the default bits or+ data_type (e.g. some layers are MXFP8 while the default is MXFP4, or+ some layers use NVFP4_E5M3 while the default is MXFP8), the function+ creates one ``config_group`` per distinct ``(bits, data_type)`` pair.+ Override groups list their layers explicitly; the default group uses+ ``targets=["Linear"]`` as a catch-all. The top-level ``"format"`` is+ set to ``"mixed-precision"`` when more than one group is produced.++ Mirrors the per-group format produced by+ :mod:`auto_round.export.export_to_llmcompressor.export_to_fp`.+ """+ from auto_round.export.export_to_llmcompressor.config import (+ initialize_quantization,+ )++ bits = default_scheme.get("bits", 4)+ default_data_type = (default_scheme.get("data_type") or "mx_fp").lower()+ is_fp_default = (bits or 0) >= 16 # BF16/FP16 full-precision default++ if not is_fp_default and bits not in _SUPPORTED_MXFP_BITS and default_data_type != _NVFP4_E5M3_DATA_TYPE:+ raise ValueError(f"Unsupported MXFP bits={bits} for model-free output.")++ # Default ignore list: any layer present in ignored_layers (deduped) that+ # was NOT quantized.+ ignore = list(dict.fromkeys(ignored_layers))+ quant_set = set(quantized_layers)+ ignore = [n for n in ignore if n not in quant_set]++ # Resolve each quantized layer's effective (bits, data_type) using layer_config overrides.+ # Key: (bits, data_type) to distinguish e.g. MXFP4 from NVFP4_E5M3 (both 4-bit).+ scheme_groups: dict[tuple[int, str], list[str]] = {} # (bits, data_type) -> [layer_names]+ if layer_config:+ temp_matcher = _PatternMatcher(+ ignore_patterns=[],+ layer_config=layer_config,+ default_scheme=default_scheme,+ )+ for layer in quantized_layers:+ scheme = temp_matcher.resolve_scheme(f"{layer}.weight")+ layer_bits = scheme.get("bits", bits) if scheme is not None else bits+ layer_dt = (+ (scheme.get("data_type") or default_data_type).lower() if scheme is not None else default_data_type+ )+ scheme_groups.setdefault((layer_bits, layer_dt), []).append(layer)+ else:+ if not is_fp_default:+ scheme_groups[(bits, default_data_type)] = list(quantized_layers)+ # else: BF16 default with no layer_config → no MXFP layers; scheme_groups stays {}++ if len(scheme_groups) <= 1:+ # Single scheme — use the actual (bits, data_type) from the group.+ if scheme_groups:+ actual_bits, actual_dt = next(iter(scheme_groups.keys()))+ else:+ actual_bits, actual_dt = bits, default_data_type+ if actual_dt != _NVFP4_E5M3_DATA_TYPE and actual_bits not in _SUPPORTED_MXFP_BITS:+ raise ValueError(f"Unsupported MXFP bits={actual_bits} for model-free output.")+ group_scheme, fmt = _get_mxfp_group_scheme_and_format(actual_bits, actual_dt, ignore)+ if actual_dt == _NVFP4_E5M3_DATA_TYPE:+ from auto_round.export.export_to_llmcompressor.config import initialize_nvfp4_e5m3_quantization++ qconfig = initialize_nvfp4_e5m3_quantization(ignore=ignore)+ if is_fp_default and scheme_groups:+ qconfig["config_groups"]["group_0"]["targets"] = list(quantized_layers)+ qconfig["format"] = fmt+ qconfig.update(_get_llm_compressor_metadata())+ return qconfig+ from auto_round.export.export_to_llmcompressor.config import initialize_quantization as _init_q++ scheme_name = "MXFP4" if actual_bits == 4 else "MXFP8"+ qconfig = _init_q(scheme=scheme_name, ignore=ignore)+ if is_fp_default and scheme_groups:+ targets = list(quantized_layers)+ qconfig.config_groups["group_0"].targets = targets+ qconfig = qconfig.to_dict()+ qconfig["format"] = fmt+ qconfig.update(_get_llm_compressor_metadata())+ return qconfig++ # Mixed precision: build one config_group per distinct (bits, data_type).+ # Override groups (non-default key) come first, default group last,+ # ordered by descending bit-width within each partition so that the+ # higher-precision group gets the lower group index.+ default_key = (bits, default_data_type)+ override_items = sorted(+ [(key, layers) for key, layers in scheme_groups.items() if key != default_key],+ key=lambda x: x[0][0],+ reverse=True,+ )+ default_item = (default_key, scheme_groups[default_key]) if default_key in scheme_groups else None+ ordered = override_items + ([default_item] if default_item else [])++ config_groups: dict = {}+ group_formats: dict[str, str] = {}+ for idx, ((group_bits, group_dt), layer_names) in enumerate(ordered):+ group_name = f"group_{idx}"+ is_default_group = (group_bits, group_dt) == default_key+ targets = ["Linear"] if is_default_group else layer_names+ group_scheme, fmt = _get_mxfp_group_scheme_and_format(group_bits, group_dt, ignore)+ group_scheme.targets = targets+ config_groups[group_name] = group_scheme+ group_formats[group_name] = fmt++ full_qconfig = initialize_quantization(scheme=None, config_groups=config_groups, ignore=ignore)+ full_dict = full_qconfig.to_dict()+ full_dict["format"] = "mixed-precision"+ for group_name, fmt in group_formats.items():+ full_dict["config_groups"][group_name]["format"] = fmt+ full_dict.update(_get_llm_compressor_metadata())+ return full_dict+++def _build_mxfp_autoround_quantization_config(+ default_scheme: dict,+ quantized_layers: list[str],+ ignored_layers: list[str],+ layer_config: dict | None = None,+ block_name_to_quantize: Optional[str] = None,+) -> dict:+ """Build an auto-round style quantization_config for MXFP4 / MXFP8.++ Unlike :func:`_build_mxfp_quantization_config` which produces a+ compressed-tensors / llm-compressor style config, this function produces+ an ``auto-round`` style config (``quant_method="auto-round"``,+ ``packing_format="auto_round:llm_compressor"``). The on-disk weight+ tensors are identical; only the ``quantization_config`` metadata differs.++ The generated config mirrors the layout of the regular AutoRound MXFP+ export, including activation quantization fields (``act_bits`` /+ ``act_data_type`` / …) from the scheme, ``enable_quanted_input``, and+ ``lm_head`` in ``extra_config`` when it was quantized.+ """+ from collections import Counter++ from auto_round.version import __version__++ bits = default_scheme.get("bits", 4)+ group_size = default_scheme.get("group_size", 32)+ is_fp_default = (bits or 0) >= 16++ # For BF16 default + MXFP layer_config overrides, derive the dominant+ # MXFP bit-width from the layers that were actually quantized.+ if is_fp_default and layer_config and quantized_layers:+ temp_matcher = _PatternMatcher(+ ignore_patterns=[],+ layer_config=layer_config,+ default_scheme=default_scheme,+ )+ mxfp_counter: Counter = Counter()+ for layer in quantized_layers:+ scheme = temp_matcher.resolve_scheme(f"{layer}.weight")+ if scheme is None:+ continue+ lb = scheme.get("bits")+ ldt = (scheme.get("data_type") or "").lower()+ if lb and lb < 16 and is_mx_fp(ldt):+ mxfp_counter[lb] += 1+ if mxfp_counter:+ bits, _ = mxfp_counter.most_common(1)[0]+ group_size = 32++ if (bits or 0) < 1 or bits not in _SUPPORTED_MXFP_BITS:+ bits = 4 # safe fallback++ qconfig: dict = {+ "quant_method": "auto-round",+ "packing_format": "auto_round:llm_compressor",+ "bits": bits,+ "group_size": group_size or 32,+ "sym": True, # MXFP is always symmetric+ "data_type": "mx_fp",+ "iters": 0,+ "model_free": True,+ "autoround_version": __version__,+ "enable_quanted_input": False,+ }+ if block_name_to_quantize:+ qconfig["block_name_to_quantize"] = block_name_to_quantize++ # Carry activation quantization fields from the scheme (e.g. MXFP4 has+ # act_bits=4, act_data_type="mx_fp", act_dynamic=True, act_group_size=32,+ # act_sym=True). Including these keeps the config consistent with the+ # output produced by the regular AutoRound MXFP export flow.+ for act_key in ("act_bits", "act_data_type", "act_dynamic", "act_group_size", "act_sym"):+ val = default_scheme.get(act_key)+ if val is not None:+ qconfig[act_key] = val++ scheme_keys = [f.name for f in fields(QuantizationScheme)]+ extra_config: dict = {}+ non_linear_ops = ["embed", "conv"]+ non_linear_re = re.compile("|".join(re.escape(op) for op in non_linear_ops))++ if layer_config:+ from auto_round.export.export_to_autoround.utils import check_neq_config++ expected_scheme = {key: qconfig.get(key) for key in scheme_keys}+ for layer_name, cfg in layer_config.items():+ if not isinstance(cfg, dict):+ continue+ neq_keys = check_neq_config(cfg, **expected_scheme)+ if neq_keys:+ extra_config[layer_name] = {key: cfg[key] for key in neq_keys if cfg.get(key) is not None}++ quantized_set = set(quantized_layers)+ unique_ignored = list(dict.fromkeys(ignored_layers))+ for layer_name in unique_ignored:+ if layer_name in quantized_set or layer_name in extra_config:+ continue+ if non_linear_re.search(layer_name):+ continue+ extra_config[layer_name] = {+ "bits": 16,+ "data_type": "float",+ "act_bits": 16,+ "act_data_type": "float",+ }++ # lm_head: when explicitly quantized, record its full scheme in extra_config+ # (mirrors the regular AutoRound MXFP export behavior).+ if "lm_head" in quantized_set and "lm_head" not in extra_config:+ lm_head_cfg = (layer_config or {}).get("lm_head", default_scheme)+ extra_config["lm_head"] = {k: lm_head_cfg.get(k) for k in scheme_keys if lm_head_cfg.get(k) is not None}++ if extra_config:+ qconfig["extra_config"] = extra_config++ return qconfig+++def _layer_config_has_mxfp(layer_config: dict | None) -> bool:+ """Return True if any layer_config entry requests MXFP quantization.++ Handles values that are plain strings (preset names), dicts (possibly+ with a ``"scheme"`` key or a ``"data_type"`` key), or+ :class:`QuantizationScheme` instances.+ """+ if not layer_config:+ return False+ for val in layer_config.values():+ if isinstance(val, str):+ try:+ s = _normalize_scheme(val.upper())+ if is_mx_fp((s.data_type or "").lower()):+ return True+ except Exception:+ pass+ elif isinstance(val, dict):+ # Direct data_type field+ if is_mx_fp((val.get("data_type") or "").lower()):+ return True+ # Nested 'scheme' key (e.g. {"scheme": "MXFP4"})+ scheme_val = val.get("scheme")+ if isinstance(scheme_val, str):+ try:+ s = _normalize_scheme(scheme_val.upper())+ if is_mx_fp((s.data_type or "").lower()):+ return True+ except Exception:+ pass+ elif isinstance(val, QuantizationScheme):+ if is_mx_fp((val.data_type or "").lower()):+ return True+ return False+++def _layer_config_has_nvfp4(layer_config: dict | None) -> bool:+ """Return True if any layer_config entry requests NVFP4 quantization.++ Detects both the standard NVFP4 (``data_type='nv_fp'``) and the+ global-scale-free variant NVFP4_E5M3 (``data_type='nvfp4_v2'``).++ Handles values that are plain strings (preset names), dicts (possibly+ with a ``"scheme"`` key or a ``"data_type"`` key), or+ :class:`QuantizationScheme` instances.+ """+ if not layer_config:+ return False+ for val in layer_config.values():+ if isinstance(val, str):+ try:+ s = _normalize_scheme(val.upper())+ dt = (s.data_type or "").lower()+ if dt == _NVFP4_E5M3_DATA_TYPE or is_nv_fp(dt):+ return True+ except Exception:+ pass+ elif isinstance(val, dict):+ dt = (val.get("data_type") or "").lower()+ if dt == _NVFP4_E5M3_DATA_TYPE or is_nv_fp(dt):+ return True+ scheme_val = val.get("scheme")+ if isinstance(scheme_val, str):+ try:+ s = _normalize_scheme(scheme_val.upper())+ dt = (s.data_type or "").lower()+ if dt == _NVFP4_E5M3_DATA_TYPE or is_nv_fp(dt):+ return True+ except Exception:+ pass+ elif isinstance(val, QuantizationScheme):+ dt = (val.data_type or "").lower()+ if dt == _NVFP4_E5M3_DATA_TYPE or is_nv_fp(dt):+ return True+ return False+++def _get_layer_config_nvfp4_dt(layer_config: dict | None) -> str | None:+ """Return the first NVFP4 data_type found in *layer_config*.++ Returns ``"nv_fp"`` for standard NVFP4, ``"nvfp4_v2"`` for NVFP4_E5M3,+ or ``None`` when no NVFP4 entry is present.+ """+ if not layer_config:+ return None+ for val in layer_config.values():+ if isinstance(val, str):+ try:+ s = _normalize_scheme(val.upper())+ dt = (s.data_type or "").lower()+ if dt == _NVFP4_E5M3_DATA_TYPE:+ return _NVFP4_E5M3_DATA_TYPE+ if is_nv_fp(dt):+ return "nv_fp"+ except Exception:+ pass+ elif isinstance(val, dict):+ dt = (val.get("data_type") or "").lower()+ if dt == _NVFP4_E5M3_DATA_TYPE:+ return _NVFP4_E5M3_DATA_TYPE+ if is_nv_fp(dt):+ return "nv_fp"+ scheme_val = val.get("scheme")+ if isinstance(scheme_val, str):+ try:+ s = _normalize_scheme(scheme_val.upper())+ dt = (s.data_type or "").lower()+ if dt == _NVFP4_E5M3_DATA_TYPE:+ return _NVFP4_E5M3_DATA_TYPE+ if is_nv_fp(dt):+ return "nv_fp"+ except Exception:+ pass+ elif isinstance(val, QuantizationScheme):+ dt = (val.data_type or "").lower()+ if dt == _NVFP4_E5M3_DATA_TYPE:+ return _NVFP4_E5M3_DATA_TYPE+ if is_nv_fp(dt):+ return "nv_fp"+ return None+++def _is_full_precision_default(scheme_input: Any) -> bool:+ """Return True when *scheme_input* represents a full-precision default (bits >= 16).++ Used to detect BF16/FP16 schemes where no weights are quantized by+ default but layer_config overrides are still applied.+ """+ try:+ s = _normalize_scheme(scheme_input)+ return (s.bits or 0) >= 16 and (s.act_bits or 16) >= 16+ except Exception:+ return False+++def _derive_dominant_int_scheme(+ quantized_layers: list[str],+ layer_config: dict,+ fallback: dict,+) -> dict | None:+ """Infer the dominant INT scheme from layers that were actually quantized.++ Used when the default scheme is full-precision (BF16/FP16) to produce a+ quantization_config where the most common INT scheme becomes the top-level+ default and the BF16 layers appear as ``bits=16`` exceptions in+ ``extra_config``. This matches the layout expected by AutoRound loaders+ (dominant quantized bits at the top, full-precision overrides below).++ Returns ``None`` when no INT layers were found in *quantized_layers*.+ """+ from collections import Counter++ temp_matcher = _PatternMatcher(+ ignore_patterns=[],+ layer_config=layer_config,+ default_scheme=fallback,+ )+ counter: "Counter[tuple]" = Counter()+ for layer in quantized_layers:+ scheme = temp_matcher.resolve_scheme(f"{layer}.weight")+ if scheme is None:+ continue+ bits = scheme.get("bits")+ if bits is None or bits >= 16:+ continue+ data_type = (scheme.get("data_type") or "int").lower()+ if is_mx_fp(data_type):+ continue # MXFP is handled by the dedicated path+ key = (bits, scheme.get("group_size") or fallback.get("group_size"), bool(scheme.get("sym", True)), data_type)+ counter[key] += 1++ if not counter:+ return None++ (bits, group_size, sym, data_type), _ = counter.most_common(1)[0]+ return {+ "bits": bits,+ "group_size": group_size,+ "sym": sym,+ "data_type": data_type,+ }+++def _build_quantization_config(+ default_scheme: dict,+ layer_config: dict,+ ignore_patterns: list[str],+ quantized_layers: list[str],+ ignored_layers: list[str],+ block_name_to_quantize: Optional[str] = None,+ format: str = "auto_round",+) -> dict:+ """Build a quantization_config dict compatible with auto-round format."""+ # MXFP (mx_fp) supports two output styles depending on *format*:+ # - "llm_compressor" → compressed-tensors / llm-compressor style config+ # (produced by _build_mxfp_quantization_config).+ # - "auto_round" / "auto_round:auto_gptq" → auto-round style config+ # (produced by _build_mxfp_autoround_quantization_config).+ # Also route to MXFP config when the default is full-precision (BF16/FP16)+ # but layer_config contains MXFP overrides.+ data_type = (default_scheme.get("data_type") or "int").lower()+ default_bits = default_scheme.get("bits", 4)+ is_fp_default = (default_bits or 0) >= 16 and not is_mx_fp(data_type)+ if data_type == _NVFP4_E5M3_DATA_TYPE and format == "llm_compressor":+ warnings.warn(+ "LLMC/llm-compressor does not currently support the NVFP4_E5M3 scheme. "+ "Please refer to docs/nvfp4_e5m3.md for the recommended export and usage path.",+ UserWarning,+ stacklevel=2,+ )+ return _build_nvfp4_e5m3_quantization_config(ignored_layers)+ # BF16/FP16 default with NVFP4 layer_config overrides and llm_compressor format:+ # build an NVFP4 config that targets only the explicitly quantized layers.+ if is_fp_default and _layer_config_has_nvfp4(layer_config) and format == "llm_compressor":+ nvfp4_dt = _get_layer_config_nvfp4_dt(layer_config)+ targets = list(dict.fromkeys(quantized_layers)) if quantized_layers else ["Linear"]+ if nvfp4_dt == _NVFP4_E5M3_DATA_TYPE:+ warnings.warn(+ "LLMC/llm-compressor does not currently support the NVFP4_E5M3 scheme. "+ "Please refer to docs/nvfp4_e5m3.md for the recommended export and usage path.",+ UserWarning,+ stacklevel=2,+ )+ # Global-scale-free variant (NVFP4_E5M3)+ from auto_round.export.export_to_llmcompressor.config import initialize_nvfp4_e5m3_quantization++ qconfig = initialize_nvfp4_e5m3_quantization(ignore=[])+ qconfig["config_groups"]["group_0"]["targets"] = targets+ qconfig["format"] = "nvfp4-e5m3-pack-quantized"+ else:+ # Standard NVFP4 (nv_fp, scale_dtype=float8_e4m3fn)+ from auto_round.export.export_to_llmcompressor.config import initialize_quantization++ qconfig_obj = initialize_quantization(scheme="NVFP4", ignore=[])+ qconfig = qconfig_obj.to_dict()+ qconfig["config_groups"]["group_0"]["targets"] = targets+ qconfig["format"] = "nvfp4-pack-quantized"+ qconfig.update(_get_llm_compressor_metadata())+ return qconfig+ if is_mx_fp(data_type) or (is_fp_default and _layer_config_has_mxfp(layer_config)):+ if format in ("auto_round", "auto_round:auto_gptq"):+ return _build_mxfp_autoround_quantization_config(+ default_scheme=default_scheme,+ quantized_layers=quantized_layers,+ ignored_layers=ignored_layers,+ layer_config=layer_config,+ block_name_to_quantize=block_name_to_quantize,+ )+ return _build_mxfp_quantization_config(+ default_scheme=default_scheme,+ quantized_layers=quantized_layers,+ ignored_layers=ignored_layers,+ layer_config=layer_config,+ )++ # When the default is full-precision (BF16/FP16) but INT layers were+ # quantized via layer_config, derive the dominant INT scheme and use it as+ # the top-level defaults so the output config reads as+ # "INT-quantized with BF16 exceptions" rather than the inverse.+ if is_fp_default and quantized_layers:+ dominant = _derive_dominant_int_scheme(quantized_layers, layer_config, default_scheme)+ if dominant is not None:+ default_scheme = dominant++ from auto_round.version import __version__++ scheme_keys = [f.name for f in fields(QuantizationScheme)]+ # vllm only support auto_round:auto_gptq, but transformers cannot load it correctly when sym=False.+ # So we keep auto_round for asymmetric quantization to maintain compatibility with both.+ if data_type == _NVFP4_E5M3_DATA_TYPE:+ packing_format = "auto_round:fake" if format == "fake" else "auto_round:llm_compressor_nvfp4_e5m3"+ else:+ packing_format = "auto_round:auto_gptq" if default_scheme.get("sym", True) else "auto_round"++ qconfig = {+ "quant_method": "auto-round",+ "packing_format": packing_format,+ "bits": default_scheme["bits"],+ "group_size": default_scheme["group_size"],+ "sym": default_scheme.get("sym", True),+ "data_type": default_scheme.get("data_type", "int"),+ "iters": 0,+ "model_free": True,+ "autoround_version": __version__,+ }++ if data_type == _NVFP4_E5M3_DATA_TYPE:+ for act_key in ("act_bits", "act_data_type", "act_group_size", "act_sym", "act_dynamic"):+ value = default_scheme.get(act_key)+ if value is not None:+ qconfig[act_key] = value++ if block_name_to_quantize:+ qconfig["block_name_to_quantize"] = block_name_to_quantize++ extra_config = {}+ for layer_name, cfg in layer_config.items():+ if cfg.get("bits", default_scheme["bits"]) >= 16:+ extra_config[layer_name] = {k: cfg.get(k) for k in scheme_keys if cfg.get(k) is not None}+ continue+ differs = False+ for key in ("bits", "group_size", "sym"):+ if cfg.get(key) is not None and cfg[key] != default_scheme.get(key):+ differs = True+ break+ if differs:+ extra_config[layer_name] = {k: cfg.get(k) for k in scheme_keys if cfg.get(k) is not None}++ # Filter out non-Linear ops (embed, conv) that don't need to be recorded in config.+ # Routing gates and other predefined patterns are still recorded.+ non_linear_ops = ["embed", "conv"]+ non_linear_re = re.compile("|".join(re.escape(op) for op in non_linear_ops))++ unique_ignored = list(dict.fromkeys(ignored_layers))+ for layer_name in unique_ignored:+ if layer_name not in extra_config:+ # Skip non-Linear ops (embed, conv) since they're not Linear layers+ if non_linear_re.search(layer_name):+ continue+ extra_config[layer_name] = {"bits": 16, "data_type": "float"}+ if data_type == _NVFP4_E5M3_DATA_TYPE:+ extra_config[layer_name].update({"act_bits": 16, "act_data_type": "float"})++ quantized_layer_set = set(quantized_layers)+ if "lm_head" in quantized_layer_set and "lm_head" not in extra_config:+ lm_head_cfg = layer_config.get("lm_head", default_scheme)+ extra_config["lm_head"] = {k: lm_head_cfg.get(k) for k in scheme_keys if lm_head_cfg.get(k) is not None}++ if extra_config:+ qconfig["extra_config"] = extra_config++ return qconfig+++# ---------------------------------------------------------------------------+# Scheme Validation and Override Resolution+# ---------------------------------------------------------------------------+++def _apply_scheme_overrides(+ scheme: Union[str, QuantizationScheme],+ scheme_overrides: Optional[dict] = None,+) -> QuantizationScheme:+ """Return the effective scheme after applying non-None overrides."""+ scheme_obj = copy.deepcopy(_normalize_scheme(scheme))+ if not scheme_overrides:+ return scheme_obj++ valid_fields = {field.name for field in fields(QuantizationScheme)}+ for key, value in scheme_overrides.items():+ if key in valid_fields and value is not None:+ setattr(scheme_obj, key, value)+ return scheme_obj+++def _validate_supported_scheme(+ scheme_obj: QuantizationScheme,+ scheme_input: Union[str, QuantizationScheme],+) -> None:+ """Raise ``ValueError`` if *scheme_obj* is not supported by model-free.++ Model-free supports:++ * Integer weight-only quantization (sym/asym), ``bits ∈ {2, 4, 8}``,+ packed in the ``auto_round:auto_gptq`` format.+ * MXFP weight quantization (``data_type='mx_fp'``), ``bits ∈ {4, 8}``,+ ``group_size=32``, packed in ``mxfp4-pack-quantized`` / ``mxfp8-quantized``+ format (compressed-tensors compatible).+ """+ data_type = (scheme_obj.data_type or "int").lower()+ bits = scheme_obj.bits+ act_bits = scheme_obj.act_bits if scheme_obj.act_bits is not None else 16++ # Full-precision (BF16/FP16) default: all layers stay in full precision+ # unless layer_config provides lower-bit overrides.+ if (bits or 0) >= 16 and act_bits >= 16:+ return++ # MXFP weight-only path: accept mx_fp data type with bits in {4, 8}.+ # Activation quantization for MXFP is dynamic at inference time, so the+ # weight-only RTN path here is independent of act_bits.+ if is_mx_fp(data_type):+ if scheme_obj.act_data_type not in (None, "mx_fp"):+ raise ValueError(+ "Model-free MXFP supports only act_data_type='mx_fp', " f"but got '{scheme_obj.act_data_type}'."+ )+ # Restrict to the two explicitly supported MXFP presets when a string+ # name is provided. Variants such as MXFP4_RCEIL / MXFP8_RCEIL use a+ # different activation format; silently mapping them to "MXFP4" /+ # "MXFP8" in the output config would misrepresent the requested scheme.+ if isinstance(scheme_input, str) and scheme_input.upper() not in ("MXFP4", "MXFP8"):+ raise ValueError(+ f"Model-free mode only supports MXFP preset names 'MXFP4' and 'MXFP8', "+ f"but got '{scheme_input}'. "+ f"Supported preset schemes: {list(SUPPORTED_PRESET_SCHEMES)}."+ )+ if bits is None or bits not in _SUPPORTED_MXFP_BITS:+ raise ValueError(+ f"Model-free mode supports MXFP bits in {_SUPPORTED_MXFP_BITS}, "+ f"but '{scheme_input}' requests bits={bits}. "+ f"Supported preset schemes: {list(SUPPORTED_PRESET_SCHEMES)}."+ )+ group_size = scheme_obj.group_size+ if group_size not in (None, 32):+ raise ValueError(+ f"Model-free mode supports MXFP only with group_size=32, "+ f"but '{scheme_input}' requests group_size={group_size}."+ )+ return++ if data_type == _NVFP4_E5M3_DATA_TYPE:+ if bits != 4 or scheme_obj.group_size != 16 or act_bits != 4:+ raise ValueError(+ f"Model-free NVFP4_E5M3 requires bits=4, group_size=16, and act_bits=4, "+ f"but '{scheme_input}' requests bits={bits}, group_size={scheme_obj.group_size}, "+ f"act_bits={act_bits}."+ )+ if (scheme_obj.act_data_type or "").lower() != _NVFP4_E5M3_DATA_TYPE or scheme_obj.act_group_size != 16:+ raise ValueError(+ f"Model-free NVFP4_E5M3 requires act_data_type='nvfp4_v2' and act_group_size=16, "+ f"but '{scheme_input}' requests act_data_type='{scheme_obj.act_data_type}', "+ f"act_group_size={scheme_obj.act_group_size}."+ )+ return++ if act_bits < 16:+ raise ValueError(+ f"Model-free mode only supports weight-only quantization (WOQ) schemes "+ f"where act_bits >= 16, but '{scheme_input}' has act_bits={act_bits}. "+ f"Supported preset schemes: {list(SUPPORTED_PRESET_SCHEMES)}."+ )++ if data_type != "int":+ raise ValueError(+ f"Model-free mode only supports integer weight quantization "+ f"(data_type='int') or MXFP (data_type='mx_fp'), but '{scheme_input}' "+ f"has data_type='{data_type}'. FP8 / NVFP / GGUF / BF16 schemes require "+ f"the standard AutoRound flow. Supported preset schemes: "+ f"{list(SUPPORTED_PRESET_SCHEMES)}."+ )++ if bits is None or bits not in _SUPPORTED_INT_BITS:+ raise ValueError(+ f"Model-free mode supports bits in {_SUPPORTED_INT_BITS}, "+ f"but '{scheme_input}' requests bits={bits}. "+ f"Supported preset schemes: {list(SUPPORTED_PRESET_SCHEMES)}."+ )+++def is_model_free_supported_scheme(+ scheme: Union[str, QuantizationScheme],+ scheme_overrides: Optional[dict] = None,+) -> bool:+ """Return True if *scheme* can be quantized via model-free mode.++ Useful for CLI auto-routing logic. Never raises.+ """+ try:+ scheme_obj = _apply_scheme_overrides(scheme, scheme_overrides)+ _validate_supported_scheme(scheme_obj, scheme)+ return True+ except (ValueError, TypeError):+ return False+++# ---------------------------------------------------------------------------+# AutoScheme support (two-phase: delta-loss selection + model-free packing)+# ---------------------------------------------------------------------------+++def _looks_like_auto_scheme(scheme: Any) -> bool:+ """Duck-typed check for an :class:`~auto_round.auto_scheme.AutoScheme`.++ Avoids importing ``AutoScheme`` at module scope (it pulls in exporter /+ compressor modules that would create an import cycle with this file).+ """+ return hasattr(scheme, "options") and hasattr(scheme, "avg_bits")+++def _validate_auto_scheme_options(auto_scheme: Any) -> str:+ """Validate AutoScheme options for model-free packing.++ Returns the single quantized data-type family shared by all options+ (``"int"`` or ``"mx_fp"``). ``BF16`` (or any >=16-bit no-op scheme) is+ allowed and treated as "keep layer in full precision" during conversion,+ so it does not contribute to the returned family.++ Raises ``ValueError`` when any non-BF16 option is unsupported or when INT+ and MXFP options are mixed (they use different packing formats and cannot+ be produced in one model-free run).+ """+ options = list(getattr(auto_scheme, "options", []) or [])+ if not options:+ raise ValueError("AutoScheme.options must be non-empty for model-free mode.")++ families: set[str] = set()+ unsupported: list[Any] = []+ for opt in options:+ # Preserve original string validation semantics so preset-name+ # restrictions (e.g. MXFP4/MXFP8 only) are enforced.+ if isinstance(opt, str):+ try:+ scheme_obj = _normalize_scheme(opt)+ except (ValueError, TypeError):+ scheme_obj = None+ elif isinstance(opt, QuantizationScheme):+ scheme_obj = opt+ else:+ scheme_obj = None++ # AutoScheme may include BF16/no-op options. In model-free flow these+ # layers become ignore_layers entries (full precision), so they are+ # allowed and excluded from family checks.+ if scheme_obj is not None:+ act_bits = scheme_obj.act_bits if scheme_obj.act_bits is not None else 16+ if (scheme_obj.bits or 0) >= 16 and act_bits >= 16:+ continue++ # GGUF k-quants carry super_bits and are not packable by the model-free+ # RTN kernel even though their data_type is nominally "int".+ if scheme_obj is None or getattr(scheme_obj, "super_bits", None) is not None:+ unsupported.append(opt)+ continue+ if not is_model_free_supported_scheme(opt):+ unsupported.append(opt)+ continue++ data_type = (scheme_obj.data_type or "int").lower()+ families.add("mx_fp" if is_mx_fp(data_type) else "int")++ if unsupported:+ raise ValueError(+ f"Model-free + AutoScheme received unsupported option(s): {unsupported}. "+ f"Model-free supports INT WOQ (bits in {_SUPPORTED_INT_BITS}) and MXFP "+ f"(bits in {_SUPPORTED_MXFP_BITS}); GGUF / NVFP4 / FP8 options are not "+ f"packable in model-free mode. Remove the unsupported options or pass "+ f"disable_model_free=True to use the regular flow."+ )+ if len(families) > 1:+ raise ValueError(+ "Model-free + AutoScheme cannot mix INT and MXFP options in a single run "+ f"(got families {sorted(families)}); INT and MXFP use different packing "+ "formats. Use a single data-type family, or pass disable_model_free=True."+ )+ return families.pop()+++def _convert_auto_scheme_layer_config(+ generated: dict[str, dict],+ preferred_base_scheme: Union[str, QuantizationScheme, None] = None,+) -> tuple[QuantizationScheme, dict[str, dict], list[str]]:+ """Convert an AutoScheme-generated ``layer_config`` into model-free inputs.++ Returns ``(base_scheme, per_layer_overrides, fp16_layers)`` where:++ * ``base_scheme`` is ``preferred_base_scheme`` when provided, matching the+ primary scheme selected by the regular AutoRound path. Otherwise the+ most common quantized scheme is used as a fallback.+ * ``per_layer_overrides`` maps every quantized layer name to its resolved+ :class:`QuantizationScheme` fields.+ * ``fp16_layers`` lists layers AutoScheme kept at >= 16 bits (added to the+ model-free ignore list so they stay in full precision).+ """+ from collections import Counter++ scheme_keys = {f.name for f in fields(QuantizationScheme)}+ per_layer: dict[str, dict] = {}+ fp16_layers: list[str] = []+ counter: "Counter[tuple]" = Counter()++ for name, cfg in generated.items():+ if not isinstance(cfg, dict):+ continue+ bits = cfg.get("bits")+ data_type_raw = (cfg.get("data_type") or "").strip()++ # Infer bits from compact dtype aliases that embed bit-width in the name,+ # e.g. "mxfp8" → 8, "MXFP4" → 4, "mx_fp4" → 4.+ if bits is None and data_type_raw:+ dt_lower = data_type_raw.lower()+ for prefix in ("mxfp", "mx_fp", "nvfp", "nv_fp"):+ tail = dt_lower[len(prefix) :]+ if dt_lower.startswith(prefix) and tail.isdigit():+ bits = int(tail)+ break++ if bits is None:+ continue++ clean = {k: cfg[k] for k in scheme_keys if cfg.get(k) is not None}+ clean["bits"] = bits # honour inferred value when not explicit++ # Normalise compact dtype aliases to canonical forms understood by+ # the quantization kernels ("mxfp8" / "MXFP4" → "mx_fp").+ if data_type_raw:+ dt_lower = data_type_raw.lower()+ if dt_lower.startswith("mxfp") or dt_lower.startswith("mx_fp"):+ clean["data_type"] = "mx_fp"+ elif dt_lower.startswith("nvfp") or dt_lower.startswith("nv_fp"):+ clean["data_type"] = "nv_fp"++ if bits >= 16:+ fp16_layers.append(name)+ continue+ data_type = (clean.get("data_type") or "int").lower()+ per_layer[name] = clean+ counter[(clean.get("bits"), clean.get("group_size"), bool(clean.get("sym", True)), data_type)] += 1++ if not counter:+ raise ValueError("AutoScheme did not assign any quantizable layers for model-free mode.")++ if preferred_base_scheme is not None:+ base_scheme = copy.deepcopy(_normalize_scheme(preferred_base_scheme))+ else:+ (base_bits, base_group_size, base_sym, base_dtype), _ = counter.most_common(1)[0]+ base_scheme = QuantizationScheme(+ bits=base_bits,+ group_size=base_group_size,+ sym=base_sym,+ data_type=base_dtype,+ )+ return base_scheme, per_layer, fp16_layers+++# ---------------------------------------------------------------------------+# Model-Special Handlers (Keep at End)+# ---------------------------------------------------------------------------+++def _read_int_like(value: Any) -> int | None:+ """Best-effort conversion for int-like config values."""+ if isinstance(value, bool) or value is None:+ return None+ if isinstance(value, int):+ return value+ if isinstance(value, float) and value.is_integer():+ return int(value)+ if isinstance(value, str):+ v = value.strip()+ if v.isdigit() or (v.startswith("-") and v[1:].isdigit()):+ return int(v)+ return None+++def _read_bool_like(value: Any) -> bool | None:+ """Best-effort conversion for bool-like config values."""+ if isinstance(value, bool):+ return value+ if isinstance(value, str):+ v = value.strip().lower()+ if v in {"true", "1", "yes"}:+ return True+ if v in {"false", "0", "no"}:+ return False+ return None+++def _resolve_kimi_k25_int4_params(source_quant_config: dict | None) -> tuple[int | None, bool]:+ """Resolve default (group_size, sym) for kimi_k25 INT4 source dequant."""+ group_size: int | None = None+ sym = True++ if not isinstance(source_quant_config, dict):+ return group_size, sym++ candidates = [source_quant_config]+ weights_cfg = source_quant_config.get("weights")+ if isinstance(weights_cfg, dict):+ candidates.append(weights_cfg)++ for cfg in candidates:+ if group_size is None:+ for key in ("group_size", "weight_group_size"):+ maybe = _read_int_like(cfg.get(key))+ if maybe is not None and maybe > 0:+ group_size = maybe+ break++ sym_val = _read_bool_like(cfg.get("sym"))+ if sym_val is not None:+ sym = sym_val++ zp_val = _read_bool_like(cfg.get("zero_point"))+ if zp_val is not None:+ sym = not zp_val++ return group_size, sym+++def _collect_kimi_k25_int4_source_entries(+ raw_tensors: dict[str, torch.Tensor],+) -> list[tuple[str, str, str, str | None]]:+ """Collect kimi_k25 INT4 source tensors."""+ entries: list[tuple[str, str, str, str | None]] = []+ for name, packed in raw_tensors.items():+ if not name.endswith(".weight_packed"):+ continue+ if packed.dtype not in (torch.uint8, torch.int8, torch.int32):+ continue++ layer_name = name[: -len(".weight_packed")]+ scale_key = f"{layer_name}.weight_scale"+ if scale_key not in raw_tensors:+ continue++ scale = raw_tensors[scale_key]+ if scale.dtype not in (torch.float16, torch.bfloat16, torch.float32, torch.float64):+ continue+ if packed.dim() != 2 or scale.dim() != 2:+ continue++ zp_key = None+ for candidate in (f"{layer_name}.weight_zero_point", f"{layer_name}.zero_point"):+ if candidate in raw_tensors:+ zp_key = candidate+ break++ entries.append((layer_name, name, scale_key, zp_key))++ return entries+++def _unpack_int4_weight_packed(packed: torch.Tensor) -> torch.Tensor:+ """Unpack packed int4 bytes/ints into int16 ``[out, in]`` nibbles."""+ if packed.dtype == torch.int32:+ packed_i32 = packed.view(torch.int32)+ rows, cols = packed_i32.shape+ nibble_list = [(packed_i32 >> (4 * i)) & 0xF for i in range(8)]+ out = torch.stack(nibble_list, dim=-1).reshape(rows, cols * 8).to(torch.int16)+ return out++ packed_u8 = packed.view(torch.uint8)+ lo = packed_u8 & 0x0F+ hi = (packed_u8 >> 4) & 0x0F++ out = torch.empty(+ (packed_u8.shape[0], packed_u8.shape[1] * 2),+ dtype=torch.int16,+ device=packed_u8.device,+ )+ out[:, 0::2] = lo.to(torch.int16)+ out[:, 1::2] = hi.to(torch.int16)+ return out+++def _dequant_kimi_k25_int4_tensors(+ raw_tensors: dict[str, torch.Tensor],+ source_quant_config: dict | None = None,+ device: str = "cpu",+ shard_name: str | None = None,+ dequantize_with_device_fallback: Callable[..., torch.Tensor] | None = None,+) -> dict[str, torch.Tensor]:+ """Dequantize kimi_k25-style INT4 packed weights to bfloat16 ``.weight``."""+ if dequantize_with_device_fallback is None:+ raise ValueError("dequantize_with_device_fallback callback is required")++ entries = _collect_kimi_k25_int4_source_entries(raw_tensors)+ if not entries:+ return raw_tensors++ default_group_size, default_sym = _resolve_kimi_k25_int4_params(source_quant_config)+ dequant_device = str(device or "cpu")+ shard_prefix = f"[{shard_name}] " if shard_name else ""+ logger.info(+ f"{shard_prefix}Dequantizing kimi_k25 INT4 tensor(s) to bfloat16 on {dequant_device}: total={len(entries)}."+ )++ for layer_name, packed_key, scale_key, zp_key in entries:+ packed = raw_tensors.pop(packed_key)+ scale = raw_tensors.pop(scale_key)+ zero = raw_tensors.pop(zp_key) if zp_key is not None else None+ raw_tensors.pop(f"{layer_name}.weight_shape", None)++ nibbles_per_element = 8 if packed.dtype == torch.int32 else 2+ in_features = packed.shape[1] * nibbles_per_element+ groups = scale.shape[1]+ inferred_group_size = (in_features // groups) if groups > 0 and in_features % groups == 0 else None+ group_size = inferred_group_size or default_group_size+ if group_size is None or group_size <= 0:+ raise ValueError(+ f"Cannot resolve group_size for INT4 tensor '{layer_name}': "+ f"packed_shape={tuple(packed.shape)}, scale_shape={tuple(scale.shape)}"+ )+ if in_features % group_size != 0:+ raise ValueError(+ f"Invalid group_size={group_size} for INT4 tensor '{layer_name}': "+ f"in_features={in_features} is not divisible by group_size."+ )++ if (+ inferred_group_size is not None+ and default_group_size is not None+ and inferred_group_size != default_group_size+ ):+ logger.warning(+ f"{shard_prefix}k25 INT4 group_size mismatch for {layer_name}: "+ f"config={default_group_size}, inferred={inferred_group_size}. "+ "Using inferred value from tensor shapes."+ )++ n_groups = in_features // group_size+ if scale.shape[1] != n_groups:+ raise ValueError(+ f"Scale shape mismatch for INT4 tensor '{layer_name}': "+ f"scale.shape={tuple(scale.shape)}, expected second dim {n_groups}."+ )++ def _dequant_impl(target_device: str) -> torch.Tensor:+ q = _unpack_int4_weight_packed(packed.to(target_device, non_blocking=True))+ sc = scale.to(target_device, non_blocking=True).to(torch.float32)++ if zero is not None:+ zp = zero.to(target_device, non_blocking=True)+ if zp.dim() == 2 and zp.shape == packed.shape and zp.dtype in (torch.int8, torch.uint8):+ zp = _unpack_int4_weight_packed(zp)+ elif zp.dim() == 2 and zp.shape[1] != n_groups:+ raise ValueError(f"Unsupported zero-point shape for INT4 tensor '{layer_name}': {tuple(zp.shape)}")+ if zp.dim() == 2 and zp.shape[1] == n_groups:+ zp = zp.to(torch.int16).repeat_interleave(group_size, dim=1)+ zp = zp[:, : q.shape[1]].to(torch.int16)+ q = q - zp+ else:+ if default_sym:+ q = torch.where(q >= 8, q - 16, q)+ else:+ q = q - 8++ sc = sc.repeat_interleave(group_size, dim=1)[:, : q.shape[1]]+ return (q.to(torch.float32) * sc).to(torch.bfloat16)++ dq_weight = dequantize_with_device_fallback(+ dequant_device=dequant_device,+ shard_prefix=shard_prefix,+ op_name="kimi_k25 INT4 dequant",+ tensor_label=layer_name,+ on_device=lambda: _dequant_impl(dequant_device).to("cpu"),+ on_cpu=lambda: _dequant_impl("cpu"),+ )+ raw_tensors[f"{layer_name}.weight"] = dq_weight++ return raw_tensors+++def _expand_e8m0_block_scale(+ scale: torch.Tensor,+ out_features: int,+ in_features: int,+ group_size: int = 32,+) -> torch.Tensor:+ """Expand a coarse 2D E8M0 block scale to per-group llm-compressor layout."""+ scale = scale.view(torch.uint8)+ if scale.dim() != 2:+ raise ValueError(f"Expected a 2D E8M0 block scale, got shape {tuple(scale.shape)}.")++ target_rows = out_features+ target_cols = in_features // group_size+ rows, cols = scale.shape++ if target_rows % rows == 0 and target_cols % cols == 0:+ if target_rows != rows:+ scale = scale.repeat_interleave(target_rows // rows, dim=0)+ if target_cols != cols:+ scale = scale.repeat_interleave(target_cols // cols, dim=1)+ return scale.contiguous()++ coarse_block = 128+ expected_rows = (out_features + coarse_block - 1) // coarse_block+ expected_cols = (in_features + coarse_block - 1) // coarse_block+ if rows == expected_rows and cols == expected_cols:+ if coarse_block % group_size != 0:+ raise ValueError(+ f"Cannot expand DeepSeek E8M0 block scale with group_size={group_size}; "+ f"{coarse_block} is not divisible by group_size."+ )+ groups_per_block_col = coarse_block // group_size+ scale = scale.repeat_interleave(coarse_block, dim=0)[:target_rows]+ scale = scale.repeat_interleave(groups_per_block_col, dim=1)[:, :target_cols]+ return scale.contiguous()++ raise ValueError(+ f"Cannot expand E8M0 block scale {tuple(scale.shape)} to "+ f"({target_rows}, {target_cols}); unsupported coarse/block layout."+ )+++def preprocess_model_type_source_tensors(+ raw_tensors: dict[str, torch.Tensor],+ model_type: str | None,+ group_size: int = 32,+ quantization_config: dict | None = None,+ shard_name: str | None = None,+) -> tuple[dict[str, torch.Tensor], dict[str, int]]:+ """Apply model-type-specific source tensor normalization."""+ model_type = (model_type or "").lower()+ quantization_config = quantization_config or {}+ is_deepseek_v4 = model_type == "deepseek_v4"+ is_deepseek_v32_ue8m0 = (+ model_type == "deepseek_v32"+ and quantization_config.get("quant_method") == "fp8"+ and str(quantization_config.get("fmt", "")).lower() == "e4m3"+ and str(quantization_config.get("scale_fmt", "")).lower() == "ue8m0"+ )+ if not is_deepseek_v4 and not is_deepseek_v32_ue8m0:+ return raw_tensors, {}++ entries: list[tuple[str, str, bool]] = []+ for name, tensor in raw_tensors.items():+ if not name.endswith(".weight"):+ continue+ layer_name = name[: -len(".weight")]+ scale_candidates = [f"{layer_name}.scale"]+ if is_deepseek_v32_ue8m0:+ scale_candidates.extend((f"{layer_name}.weight_scale", f"{layer_name}.weight_scale_inv"))+ scale_name = next((candidate for candidate in scale_candidates if candidate in raw_tensors), None)+ if scale_name is None:+ continue+ if tensor.dtype == torch.float8_e4m3fn:+ entries.append((name, scale_name, True))+ elif tensor.dtype in (torch.int8, torch.uint8):+ entries.append((name, scale_name, False))++ if not entries:+ return raw_tensors, {}++ source_state: dict[str, int] = {}+ n_fp8 = 0+ n_fp4 = 0+ for weight_name, scale_name, is_fp8 in entries:+ layer_name = weight_name[: -len(".weight")]+ weight = raw_tensors.pop(weight_name)+ scale = raw_tensors.pop(scale_name)++ if is_fp8:+ out_features, in_features = weight.shape+ weight_key = f"{layer_name}.weight"+ source_state[layer_name] = 8+ n_fp8 += 1++ if scale.dtype == torch.float32:+ shard_prefix = f"[{shard_name}] " if shard_name else ""+ logger.warning_once(+ f"[{model_type}] Scale tensor pattern has dtype float32 "+ f"with UE8M0 encoding (only the 8-bit exponent is significant). "+ f"Extracting uint8 E8M0 exponent bytes from fp32 representation."+ )+ scale = ((scale.view(torch.int32) >> 23) & 0xFF).to(torch.uint8)+ else:+ out_features = weight.shape[0]+ in_features = weight.shape[1] * 2+ weight = weight.view(torch.uint8).contiguous()+ weight_key = f"{layer_name}.weight_packed"+ source_state[layer_name] = 4+ n_fp4 += 1++ weight_scale = _expand_e8m0_block_scale(scale, out_features, in_features, group_size=group_size)+ raw_tensors[weight_key] = weight+ raw_tensors[f"{layer_name}.weight_scale"] = weight_scale++ shard_prefix = f"[{shard_name}] " if shard_name else ""+ logger.info(+ f"{shard_prefix}Applied model_type preprocessing for {model_type}: "+ f"{n_fp8} MXFP8 layer(s), {n_fp4} MXFP4 layer(s) converted to llm-compressor naming."+ )+ return raw_tensors, source_state+++def handle_model_type_low_precision_source_tensors(+ raw_tensors: dict[str, torch.Tensor],+ model_type: str | None,+ source_quant_config: dict | None = None,+ device: str = "cpu",+ shard_name: str | None = None,+ dequantize_with_device_fallback: Callable[..., torch.Tensor] | None = None,+) -> dict[str, torch.Tensor]:+ """Handle model-type-specific low-precision source tensors."""+ if dequantize_with_device_fallback is None:+ dequantize_with_device_fallback = _dequantize_with_device_fallback++ if (model_type or "").lower() == "kimi_k25":+ return _dequant_kimi_k25_int4_tensors(+ raw_tensors,+ source_quant_config=source_quant_config,+ device=device,+ shard_name=shard_name,+ dequantize_with_device_fallback=dequantize_with_device_fallback,+ )+ return raw_tensorsdiff --git a/auto_round_extension/cuda/cute_nvfp4_e5m3.py b/auto_round_extension/cuda/cute_nvfp4_e5m3.pynew file mode 100644index 000000000..b71ae4614--- /dev/null+++ b/auto_round_extension/cuda/cute_nvfp4_e5m3.py@@ -0,0 +1,270 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++"""Optional CuTe DSL dispatch for NVFP4 E5M3 activation QDQ."""++from functools import lru_cache+from importlib.util import find_spec+from typing import Optional++import torch++from auto_round.logger import logger++_SUPPORTED_DTYPES = (torch.float16, torch.bfloat16, torch.float32)+_GROUP_SIZE = 16+_THREADS_PER_BLOCK = 256+_FAILED_KERNEL_KEYS = set()+_FAILED_WEIGHT_DQ_KERNEL_KEYS = set()+++@lru_cache(maxsize=1)+def is_cute_dsl_available() -> bool:+ """Return whether the optional NVIDIA CuTe DSL package is installed."""+ return find_spec("cutlass") is not None+++def can_use_cute_nvfp4_v2_qdq(activation: torch.Tensor, group_size: int) -> bool:+ """Check whether an activation can use the CuTe QDQ kernel."""+ if not is_cute_dsl_available() or not activation.is_cuda:+ return False+ if group_size != _GROUP_SIZE or activation.dtype not in _SUPPORTED_DTYPES:+ return False+ if not activation.is_contiguous() or activation.shape[-1] % group_size:+ return False+ return torch.cuda.get_device_capability(activation.device)[0] >= 8+++def _make_qdq_kernel():+ import cutlass+ import cutlass.cute as cute+ from cutlass._mlir_helpers import math++ @cute.kernel+ def qdq_kernel(input_tensor: cute.Tensor, output_tensor: cute.Tensor):+ thread_idx, _, _ = cute.arch.thread_idx()+ block_idx, _, _ = cute.arch.block_idx()+ group_idx = block_idx * _THREADS_PER_BLOCK + thread_idx++ if group_idx < input_tensor.shape[0]:+ amax = cutlass.Float32(0.0)+ for column_idx in range(_GROUP_SIZE):+ value = cutlass.Float32(input_tensor[group_idx, column_idx])+ amax = cute.arch.fmax(amax, math.abs(value))++ # This reproduces the E5M3 scale domain used by nvfp4_v2 without+ # materializing scale or intermediate FP32 tensors in global memory.+ scale = amax / cutlass.Float32(6.0)+ if scale > cutlass.Float32(0.0):+ exponent = math.floor(math.log2(scale)) + cutlass.Float32(1.0)+ mantissa = scale / math.exp2(exponent)+ mantissa_bits = math.roundeven((mantissa - cutlass.Float32(0.5)) * cutlass.Float32(16.0))+ mantissa_bits = cute.arch.fmin(cute.arch.fmax(mantissa_bits, 0.0), 7.0)+ scale = (cutlass.Float32(1.0) + mantissa_bits / cutlass.Float32(8.0)) * math.exp2(+ exponent - cutlass.Float32(1.0)+ )++ for column_idx in range(_GROUP_SIZE):+ value = cutlass.Float32(input_tensor[group_idx, column_idx])+ scaled = value / scale if scale > cutlass.Float32(0.0) else cutlass.Float32(0.0)+ magnitude = math.abs(scaled)+ quantized = cutlass.Float32(0.0)+ if magnitude < cutlass.Float32(2.0):+ quantized = math.roundeven(magnitude * cutlass.Float32(2.0)) / cutlass.Float32(2.0)+ elif magnitude < cutlass.Float32(4.0):+ quantized = math.roundeven(magnitude)+ else:+ quantized = cutlass.Float32(2.0) * math.roundeven(magnitude / cutlass.Float32(2.0))+ quantized = cute.arch.fmin(cute.arch.fmax(quantized, 0.0), 6.0)+ if scaled < cutlass.Float32(0.0):+ quantized = -quantized+ output_tensor[group_idx, column_idx] = (quantized * scale).to(output_tensor.element_type)++ @cute.jit+ def launch_qdq(input_tensor: cute.Tensor, output_tensor: cute.Tensor):+ groups = input_tensor.shape[0]+ qdq_kernel(input_tensor, output_tensor).launch(+ grid=((groups + _THREADS_PER_BLOCK - 1) // _THREADS_PER_BLOCK, 1, 1),+ block=(_THREADS_PER_BLOCK, 1, 1),+ )++ return launch_qdq+++def _make_weight_dq_kernel():+ import cutlass+ import cutlass.cute as cute+ from cutlass._mlir_helpers import math++ @cute.kernel+ def weight_dq_kernel(weight_packed: cute.Tensor, weight_scale: cute.Tensor, output_tensor: cute.Tensor):+ thread_idx, _, _ = cute.arch.thread_idx()+ block_idx, _, _ = cute.arch.block_idx()+ group_idx = block_idx * _THREADS_PER_BLOCK + thread_idx++ if group_idx < output_tensor.shape[0]:+ scale_byte = cutlass.Float32(weight_scale[group_idx, 0])+ exponent = math.floor(scale_byte / cutlass.Float32(8.0))+ mantissa = scale_byte - exponent * cutlass.Float32(8.0)+ scale = (cutlass.Float32(1.0) + mantissa / cutlass.Float32(8.0)) * math.exp2(+ exponent - cutlass.Float32(15.0)+ )+ if exponent == cutlass.Float32(0.0):+ scale = mantissa * math.exp2(cutlass.Float32(-17.0))++ for column_idx in range(_GROUP_SIZE):+ packed = cutlass.Float32(weight_packed[group_idx, column_idx // 2])+ packed_high = math.floor(packed / cutlass.Float32(16.0))+ nibble = packed - packed_high * cutlass.Float32(16.0)+ if column_idx % 2:+ nibble = packed_high++ sign = cutlass.Float32(1.0)+ if nibble >= cutlass.Float32(8.0):+ sign = cutlass.Float32(-1.0)+ nibble = nibble - cutlass.Float32(8.0)++ value = cutlass.Float32(0.0)+ if nibble == cutlass.Float32(1.0):+ value = cutlass.Float32(0.5)+ elif nibble == cutlass.Float32(2.0):+ value = cutlass.Float32(1.0)+ elif nibble == cutlass.Float32(3.0):+ value = cutlass.Float32(1.5)+ elif nibble == cutlass.Float32(4.0):+ value = cutlass.Float32(2.0)+ elif nibble == cutlass.Float32(5.0):+ value = cutlass.Float32(3.0)+ elif nibble == cutlass.Float32(6.0):+ value = cutlass.Float32(4.0)+ elif nibble == cutlass.Float32(7.0):+ value = cutlass.Float32(6.0)++ output_tensor[group_idx, column_idx] = (sign * value * scale).to(output_tensor.element_type)++ @cute.jit+ def launch_weight_dq(weight_packed: cute.Tensor, weight_scale: cute.Tensor, output_tensor: cute.Tensor):+ groups = output_tensor.shape[0]+ weight_dq_kernel(weight_packed, weight_scale, output_tensor).launch(+ grid=((groups + _THREADS_PER_BLOCK - 1) // _THREADS_PER_BLOCK, 1, 1),+ block=(_THREADS_PER_BLOCK, 1, 1),+ )++ return launch_weight_dq+++@lru_cache(maxsize=None)+def _get_compiled_qdq_kernel(device_index: int, dtype: torch.dtype):+ import cutlass.cute as cute+ from cutlass.cute.runtime import from_dlpack++ input_tensor = torch.empty((1, _GROUP_SIZE), device=f"cuda:{device_index}", dtype=dtype)+ output_tensor = torch.empty_like(input_tensor)+ return cute.compile(+ _make_qdq_kernel(),+ from_dlpack(input_tensor).mark_layout_dynamic(),+ from_dlpack(output_tensor).mark_layout_dynamic(),+ )+++@lru_cache(maxsize=None)+def _get_compiled_weight_dq_kernel(device_index: int, dtype: torch.dtype):+ import cutlass.cute as cute+ from cutlass.cute.runtime import from_dlpack++ weight_packed = torch.empty((2, _GROUP_SIZE // 2), device=f"cuda:{device_index}", dtype=torch.uint8)+ weight_scale = torch.empty((2, 1), device=f"cuda:{device_index}", dtype=torch.uint8)+ output_tensor = torch.empty((2, _GROUP_SIZE), device=f"cuda:{device_index}", dtype=dtype)+ return cute.compile(+ _make_weight_dq_kernel(),+ from_dlpack(weight_packed).mark_layout_dynamic(),+ from_dlpack(weight_scale).mark_layout_dynamic(),+ from_dlpack(output_tensor).mark_layout_dynamic(),+ )+++def try_cute_nvfp4_v2_qdq(activation: torch.Tensor, group_size: int) -> Optional[torch.Tensor]:+ """Run a CuTe DSL group-size-16 FP4 QDQ kernel when eligible."""+ if not can_use_cute_nvfp4_v2_qdq(activation, group_size):+ return None+ if torch.cuda.is_current_stream_capturing():+ return None++ kernel_key = (activation.device.index, activation.dtype)+ if kernel_key in _FAILED_KERNEL_KEYS:+ return None++ try:+ groups = activation.numel() // group_size+ output = torch.empty_like(activation)+ compiled_qdq = _get_compiled_qdq_kernel(*kernel_key)+ compiled_qdq(activation.reshape(groups, group_size), output.reshape(groups, group_size))+ return output+ except Exception as error:+ _FAILED_KERNEL_KEYS.add(kernel_key)+ logger.warning_once("CuTe NVFP4 E5M3 QDQ failed; falling back to PyTorch reference: %s", error)+ return None+++def try_cute_nvfp4_e5m3_weight_dq(+ weight_packed: torch.Tensor, weight_scale: torch.Tensor, dtype: torch.dtype+) -> Optional[torch.Tensor]:+ """Dequantize packed FP4 E5M3 weights with CuTe."""+ if (+ not is_cute_dsl_available()+ or not weight_packed.is_cuda+ or weight_packed.dtype != torch.uint8+ or weight_scale.dtype != torch.uint8+ or weight_packed.ndim != 2+ or weight_scale.shape != (weight_packed.shape[0], weight_packed.shape[1] // (_GROUP_SIZE // 2))+ or dtype not in _SUPPORTED_DTYPES+ or torch.cuda.get_device_capability(weight_packed.device)[0] < 8+ or torch.cuda.is_current_stream_capturing()+ ):+ return None++ kernel_key = (weight_packed.device.index, dtype)+ if kernel_key in _FAILED_WEIGHT_DQ_KERNEL_KEYS:+ return None++ try:+ out_features = weight_packed.shape[0]+ in_features = weight_packed.shape[1] * 2+ groups = weight_scale.numel()+ packed_groups = weight_packed.reshape(groups, _GROUP_SIZE // 2)+ scale_groups = weight_scale.reshape(groups, 1)+ output = torch.empty((groups, _GROUP_SIZE), device=weight_packed.device, dtype=dtype)+ compiled_weight_dq = _get_compiled_weight_dq_kernel(*kernel_key)+ compiled_weight_dq(packed_groups, scale_groups, output)+ return output.reshape(out_features, in_features)+ except Exception as error:+ _FAILED_WEIGHT_DQ_KERNEL_KEYS.add(kernel_key)+ logger.warning_once("CuTe NVFP4 E5M3 weight DQ failed; falling back to PyTorch reference: %s", error)+ return None+++def try_cute_nvfp4_e5m3_linear(+ activation: torch.Tensor,+ weight_packed: torch.Tensor,+ weight_scale: torch.Tensor,+ bias: Optional[torch.Tensor],+) -> Optional[torch.Tensor]:+ """Reserved second-stage dispatch point for fused QDQ, unpack, and GEMM.++ Returning ``None`` keeps the reference Linear path active until the packed-weight+ mainloop has been validated against the existing E5M3 checkpoint format.+ """+ del activation, weight_packed, weight_scale, bias+ fused_output: Optional[torch.Tensor] = None+ return fused_outputdiff --git a/docs/environments.md b/docs/environments.mdindex 4e77d2ce0..47f36cb30 100644--- a/docs/environments.md+++ b/docs/environments.md@@ -171,6 +171,16 @@ export AR_AUTO_SCHEME_CACHE=/path/to/auto_scheme_cacheexport AR_ENABLE_AUTO_SCHEME_PARALLEL=0```+### AR_NVFP4_E5M3_CACHE_HP_WEIGHT+- **Description**: Controls whether `NVFP4E5M3QuantLinear` caches a dequantized high-precision weight after the first forward pass, instead of dequantizing the packed FP4 weight on every call.+- **Default**: `False` (equivalent to `"0"`)+- **Valid Values**: `"1"`, `"true"`, `"yes"`, `"on"` (case-insensitive) enable caching; any other value disables caching+- **Usage**: Enable this when repeated inference throughput matters more than memory footprint. The current implementation releases `weight_packed` and `weight_scale` after materializing the cached high-precision weight, so steady-state memory usage increases and the cache cannot be cleared back to packed storage.++```bash+export AR_NVFP4_E5M3_CACHE_HP_WEIGHT=1+```+### AR_DISK_STREAM_MODEL- **Description**: When enabled, `AutoRound(model=<path>, ...)` builds the model as a meta-device skeleton instead of fully materializing the checkpoint on CPU RAM up front, and streams each decoder block's real weights from the checkpoint's safetensors shards on demand -- materializing right before a block is used (calibration, tuning, or `AutoScheme` sensitivity scoring) and freeing it back to meta right after. This keeps peak CPU RAM roughly flat regardless of checkpoint size, instead of proportional to it. Non-block parameters (embeddings, `lm_head`, final norm) are still loaded up front, since they are typically small. Text-model AutoScheme scoring also supports combining this with parallel scoring, which is enabled by default; each worker streams its own block copy.- **Default**: `False`diff --git a/docs/environments_CN.md b/docs/environments_CN.mdindex 8684ccdcf..365b3b46d 100644--- a/docs/environments_CN.md+++ b/docs/environments_CN.md@@ -171,6 +171,16 @@ export AR_AUTO_SCHEME_CACHE=/path/to/auto_scheme_cacheexport AR_ENABLE_AUTO_SCHEME_PARALLEL=0```+### AR_NVFP4_E5M3_CACHE_HP_WEIGHT+- **描述**:控制 `NVFP4E5M3QuantLinear` 是否在首次前向后缓存解量化得到的高精度权重,而不是每次调用都从打包的 FP4 权重重新解量化。+- **默认值**:`False`(等价于 `"0"`)+- **有效值**:`"1"`、`"true"`、`"yes"`、`"on"`(不区分大小写)表示启用缓存;其他值表示禁用缓存+- **用途**:当重复推理吞吐比内存占用更重要时可启用。当前实现会在缓存高精度权重后释放 `weight_packed` 和 `weight_scale`,因此稳态内存占用会增大,且之后无法再切回打包存储。++```bash+export AR_NVFP4_E5M3_CACHE_HP_WEIGHT=1+```+### AR_DISK_STREAM_MODEL- **描述**:启用后,`AutoRound(model=<path>, ...)` 会将模型构建为 meta 设备骨架,而不是先把整个 checkpoint 完全加载到 CPU 内存;随后按需从 checkpoint 的 safetensors 分片中流式加载每个解码器块的真实权重——在该块被使用前(校准、调优或 `AutoScheme` 敏感度评分)才实体化,用完后立即释放回 meta。这样峰值 CPU 内存基本保持平稳,而不会随 checkpoint 大小成比例增长。非块参数(embedding、`lm_head`、最终归一化层)体积通常较小,仍会一次性加载。文本模型的 AutoScheme 评分也支持与默认启用的并行评分组合使用;每个 worker 会流式加载自己的 block 副本。- **默认值**:`False`diff --git a/docs/nvfp4_e5m3.md b/docs/nvfp4_e5m3.mdnew file mode 100644index 000000000..a88aeaa66--- /dev/null+++ b/docs/nvfp4_e5m3.md@@ -0,0 +1,139 @@+# NVFP4 E5M3 Quantization (Experimental)++> **Note**: This is an experimental feature. APIs, formats, and behaviours may+> change in future releases.++## Overview++NVFP4 E5M3 is a quantization scheme that stores weights in FP4 (E2M1) format+with **unsigned E5M3 block scales** (instead of the global-scale approach used+by standard NVFP4). Activations are quantized on-the-fly via a QDQ+(Quantize-Dequantize) pass.++Key properties:++| Property | Value |+|---|---|+| Preset name | `NVFP4_E5M3` |+| Weight dtype | FP4 E2M1 (`nvfp4_v2`) |+| Scale dtype | UE5M3 (per-block) |+| Group size | 16 |+| Activation bits | 4 (dynamic QDQ) |++---++> Important: `llm_compressor` export is not supported for the `NVFP4_E5M3` scheme in LLMC/llm-compressor for now. Please prefer the AutoRound format or the `fake` format for current workflows, and keep following the vLLM usage docs for the latest supported path.++## Supported Output Formats++| Format flag | Description |+|---|---|+| `auto_round` (default) | Packed `.weight_packed` + `.weight_scale` tensors; loadable with AutoRound inference |+| `llm_compressor` | `nvfp4-e5m3-pack-quantized` compressed-tensors layout |+| `fake` | Full-precision (BF16) `.weight` tensors with activation QDQ at runtime; no packing overhead |++---++## Supported Inference Backends++| Backend | Description |+|---|---|+| `auto_round:cute_nvfp4_e5m3` | CUDA CuTe DSL kernel (dequant weight + QDQ act). Requires `cutlass` package and SM80+. **Fastest**. |+| `auto_round:torch_nvfp4_e5m3` | Pure PyTorch dequant + QDQ act. Works on any CUDA device. |+| `auto_round:fake` | BF16 weight + activation QDQ only (no weight packing). For debugging / profiling. |++---++## Usage++### Standard AutoRound Flow++```python+from auto_round import AutoRound+from auto_round.schemes import NVFP4_E5M3++model, tokenizer = ... # load your model++autoround = AutoRound(model, tokenizer, scheme=NVFP4_E5M3)+autoround.quantize()++# Save as packed auto_round format (default)+autoround.save_quantized("output_dir")++# Or save as llm_compressor format+autoround.save_quantized("output_dir", format="llm_compressor")++# Or save as fake format (full-precision weights + QDQ at runtime)+autoround.save_quantized("output_dir", format="fake")+```++### Model-Free Quantization (no GPU training required)++```bash+# Pack a pre-quantized model (from HuggingFace or local dir)+auto-round \+ --model <model_id_or_path> \+ --scheme NVFP4_E5M3 \+ --model_free \+ --output_dir ./nvfp4_e5m3_model+```++```python+from auto_round.compressors.model_free import ModelFreeCompressor+from auto_round.schemes import NVFP4_E5M3++compressor = ModelFreeCompressor(+ model="<model_id_or_path>",+ scheme=NVFP4_E5M3,+ output_dir="./nvfp4_e5m3_model",+)+compressor.run()+```++### Loading a Quantized Model++```python+from transformers import AutoModelForCausalLM, AutoTokenizer+import auto_round # registers the AutoRound inference hooks++model = AutoModelForCausalLM.from_pretrained(+ "./nvfp4_e5m3_model",+ device_map="auto",+)+tokenizer = AutoTokenizer.from_pretrained("./nvfp4_e5m3_model")+```++---++## Environment Variables++### `AR_NVFP4_E5M3_CACHE_HP_WEIGHT`++Controls whether NVFP4 E5M3 `QuantLinear` caches a dequantized high-precision+(BF16) weight after the first forward pass instead of dequantizing on every+call.++| Value | Behaviour |+|---|---|+| `0` / `false` / `no` / `off` (default) | Dequantize on every forward pass (lower memory, higher compute) |+| `1` / `true` / `yes` / `on` | Cache the dequantized weight after first forward (lower compute, higher memory) |++When caching is enabled, the packed weight buffers are released after the+cache is materialized, trading lower runtime overhead for higher steady-state+memory usage.++```bash+export AR_NVFP4_E5M3_CACHE_HP_WEIGHT=1+```++---++## Accuracy Results (Llama-3.1-8B-Instruct, LAMBADA OpenAI)++| Configuration | Time | Accuracy |+|---|---|---|+| BF16 baseline | 1m 5s | 0.7213 |+| `auto_round` + `auto_round:cute_nvfp4_e5m3` | 1m 48s | 0.7196 |+| `auto_round` + `auto_round:torch_nvfp4_e5m3` | 4m 23s | 0.7204 |+| `llm_compressor` + `auto_round:cute_nvfp4_e5m3` | 1m 48s | 0.7196 |+| `fake` + `auto_round:fake` | 3m 26s | 0.7204 |diff --git a/docs/nvfp4_e5m3_CN.md b/docs/nvfp4_e5m3_CN.mdnew file mode 100644index 000000000..f9d5a76a0--- /dev/null+++ b/docs/nvfp4_e5m3_CN.md@@ -0,0 +1,131 @@+# NVFP4 E5M3 量化(实验性功能)++> **注意**:这是一个实验性功能,后续版本中 API、格式及行为可能发生变化。++## 概述++NVFP4 E5M3 是一种量化方案,以 FP4(E2M1)格式存储权重,并使用**无符号 E5M3 块级 scale**(区别于标准 NVFP4 使用全局 scale 的方式)。激活值通过 QDQ(量化-反量化)操作在推理时动态量化。++主要参数:++| 参数 | 值 |+|---|---|+| 预设名称 | `NVFP4_E5M3` |+| 权重数据类型 | FP4 E2M1(`nvfp4_v2`) |+| Scale 数据类型 | UE5M3(块级别) |+| 分组大小 | 16 |+| 激活量化位宽 | 4(动态 QDQ) |++---++> 重要:LLMC/llm-compressor 当前并不支持 `NVFP4_E5M3` 方案的导出。当前请优先使用 AutoRound 原生格式或 `fake` 格式,并持续关注 vLLM 使用文档中的最新支持路径。++## 支持的输出格式++| 格式标志 | 说明 |+|---|---|+| `auto_round`(默认) | 打包的 `.weight_packed` + `.weight_scale` 张量,可使用 AutoRound 推理加载 |+| `llm_compressor` | `nvfp4-e5m3-pack-quantized` compressed-tensors 布局 |+| `fake` | 全精度(BF16)`.weight` 张量,推理时仅做激活 QDQ,无打包开销 |++---++## 支持的推理后端++| 后端 | 说明 |+|---|---|+| `auto_round:cute_nvfp4_e5m3` | CUDA CuTe DSL 内核(权重反量化 + 激活 QDQ),需要 `cutlass` 包及 SM80+,**速度最快** |+| `auto_round:torch_nvfp4_e5m3` | 纯 PyTorch 反量化 + 激活 QDQ,适用于任意 CUDA 设备 |+| `auto_round:fake` | BF16 权重 + 仅激活 QDQ(无权重打包),用于调试 / 性能分析 |++---++## 使用方法++### 标准 AutoRound 流程++```python+from auto_round import AutoRound+from auto_round.schemes import NVFP4_E5M3++model, tokenizer = ... # 加载你的模型++autoround = AutoRound(model, tokenizer, scheme=NVFP4_E5M3)+autoround.quantize()++# 以 auto_round 打包格式保存(默认)+autoround.save_quantized("output_dir")++# 或以 llm_compressor 格式保存+autoround.save_quantized("output_dir", format="llm_compressor")++# 或以 fake 格式保存(全精度权重 + 推理时 QDQ)+autoround.save_quantized("output_dir", format="fake")+```++### 无模型量化(无需 GPU 训练)++```bash+# 对已有模型进行直接打包(来自 HuggingFace 或本地目录)+auto-round \+ --model <model_id_or_path> \+ --scheme NVFP4_E5M3 \+ --model_free \+ --output_dir ./nvfp4_e5m3_model+```++```python+from auto_round.compressors.model_free import ModelFreeCompressor+from auto_round.schemes import NVFP4_E5M3++compressor = ModelFreeCompressor(+ model="<model_id_or_path>",+ scheme=NVFP4_E5M3,+ output_dir="./nvfp4_e5m3_model",+)+compressor.run()+```++### 加载量化后的模型++```python+from transformers import AutoModelForCausalLM, AutoTokenizer+import auto_round # 注册 AutoRound 推理钩子++model = AutoModelForCausalLM.from_pretrained(+ "./nvfp4_e5m3_model",+ device_map="auto",+)+tokenizer = AutoTokenizer.from_pretrained("./nvfp4_e5m3_model")+```++---++## 环境变量++### `AR_NVFP4_E5M3_CACHE_HP_WEIGHT`++控制 NVFP4 E5M3 `QuantLinear` 是否在首次前向传播后缓存反量化的高精度(BF16)权重,而不是每次调用时重新反量化。++| 取值 | 行为 |+|---|---|+| `0` / `false` / `no` / `off`(默认) | 每次前向传播都进行反量化(内存更少,计算量更多) |+| `1` / `true` / `yes` / `on` | 首次前向后缓存反量化权重(计算量更少,内存占用更多) |++启用缓存后,打包权重缓冲区在缓存生成后会被释放,以较高的稳态内存换取较低的运行时开销。++```bash+export AR_NVFP4_E5M3_CACHE_HP_WEIGHT=1+```++---++## 精度结果(Llama-3.1-8B-Instruct,LAMBADA OpenAI)++| 配置 | 耗时 | 准确率 |+|---|---|---|+| BF16 基线 | 1m 5s | 0.7213 |+| `auto_round` + `auto_round:cute_nvfp4_e5m3` | 1m 48s | 0.7196 |+| `auto_round` + `auto_round:torch_nvfp4_e5m3` | 4m 23s | 0.7204 |+| `llm_compressor` + `auto_round:cute_nvfp4_e5m3` | 1m 48s | 0.7196 |+| `fake` + `auto_round:fake` | 3m 26s | 0.7204 |diff --git a/test/unit/test_cpu/core/test_format_decoupling.py b/test/unit/test_cpu/core/test_format_decoupling.pyindex b9dac1280..2dea8725d 100644--- a/test/unit/test_cpu/core/test_format_decoupling.py+++ b/test/unit/test_cpu/core/test_format_decoupling.py@@ -32,7 +32,9 @@ import torch.nn as nnfrom auto_round.algorithms.quantization.config import QuantizationConfigfrom auto_round.compressors.base import BaseCompressorfrom auto_round.compressors.config_resolution import FormatResolution, ResolvedScheme+from auto_round.experimental import qmodules as ar_qmodulesfrom auto_round.export.formats import resolve_formats+from auto_round.inference.backend import BackendInfos, dynamic_import_inference_linear, get_layer_backendfrom auto_round.schemes import QuantizationScheme, parse_scheme@@ -50,6 +52,7 @@ EXPECTED_FORMAT_SELECTION_BASELINE = {("FP8_STATIC", "llm_compressor"): ["llm_compressor:fp8_static"],("MXFP4", "auto_round"): ["auto_round:mx_fp"],("NVFP4", "auto_round"): ["auto_round:nv_fp"],+ ("NVFP4_E5M3", "auto_round"): ["auto_round:nvfp4_v2"],}@@ -67,6 +70,30 @@ def test_format_selection_baseline():assert results == EXPECTED_FORMAT_SELECTION_BASELINE+def test_nvfp4_e5m3_autoround_uses_llm_compressor_packing_and_torch_fallback():+ scheme = _resolved_scheme("NVFP4_E5M3")+ resolution = resolve_formats(+ ResolvedScheme.from_scheme(scheme), format="auto_round", model=nn.Sequential(nn.Linear(16, 16))+ )++ assert resolution.formats[0].get_backend_name() == "auto_round:nvfp4_v2"+ assert BackendInfos["auto_round:torch_nvfp4_e5m3"].packing_format == ["auto_round:llm_compressor_nvfp4_e5m3"]+ assert BackendInfos["auto_round:cute_nvfp4_e5m3"].priority > BackendInfos["auto_round:torch_nvfp4_e5m3"].priority+ assert dynamic_import_inference_linear("auto_round:torch_nvfp4_e5m3", scheme) is ar_qmodules.NVFP4E5M3QuantLinear+ assert dynamic_import_inference_linear("auto_round:cute_nvfp4_e5m3", scheme) is ar_qmodules.CuteNVFP4E5M3QuantLinear+ assert (+ get_layer_backend(+ "cpu",+ "auto",+ "auto_round:llm_compressor_nvfp4_e5m3",+ scheme,+ in_features=16,+ out_features=16,+ )+ == "auto_round:torch_nvfp4_e5m3"+ )++def test_gguf_correction_propagates_to_scheme_and_quantize_config(monkeypatch):# W4A16's plain int-woq fields don't match every field gguf_args_check enforces# for a GGUF target; after resolving "gguf:q4_k_m" the scheme-bearing objectsdiff --git a/test/unit/test_cpu/data_type/test_nvfp.py b/test/unit/test_cpu/data_type/test_nvfp.pyindex a231a30ec..444a14aad 100644--- a/test/unit/test_cpu/data_type/test_nvfp.py+++ b/test/unit/test_cpu/data_type/test_nvfp.py@@ -29,11 +29,11 @@ from auto_round.data_type.nvfp import (cast_to_ue5m3_ste,e5m3_to_float_tensor,float_to_e5m3_frexp,- fp4_v2,- fp4_v2_with_global_scale,get_reciprocal,nv_fp4,nv_fp4_with_static_gs,+ nvfp4_v2,+ nvfp4_v2_with_global_scale,ref_fp4_quant,ref_nvfp4_quant,search_nvfp4_scale,@@ -538,106 +538,106 @@ class TestRefFp4Quant:# ----------------------------------------------------------------------------# fp4_v2_with_global_scale+# nvfp4_v2_with_global_scale# ---------------------------------------------------------------------------class TestFp4V2WithGlobalScale:- """Test fp4_v2_with_global_scale."""+ """Test nvfp4_v2_with_global_scale."""def test_group_size_16(self):t = torch.randn(4, 32, dtype=torch.bfloat16)- q, s, z = fp4_v2_with_global_scale(t, group_size=16)+ q, s, z = nvfp4_v2_with_global_scale(t, group_size=16)assert q.shape == t.shapeassert z is Nonedef test_group_size_32(self):t = torch.randn(4, 64, dtype=torch.bfloat16)- q, s, z = fp4_v2_with_global_scale(t, group_size=32)+ q, s, z = nvfp4_v2_with_global_scale(t, group_size=32)assert q.shape == t.shapedef test_invalid_group_size(self):t = torch.randn(4, 32, dtype=torch.bfloat16)with pytest.raises(AssertionError):- fp4_v2_with_global_scale(t, group_size=64)+ nvfp4_v2_with_global_scale(t, group_size=64)def test_tensor_max_as_float(self):t = torch.randn(4, 32, dtype=torch.bfloat16)- q, s, z = fp4_v2_with_global_scale(t, group_size=16, tensor_max=2.0)+ q, s, z = nvfp4_v2_with_global_scale(t, group_size=16, tensor_max=2.0)assert q.shape == t.shapedef test_tensor_max_as_tensor(self):t = torch.randn(4, 32, dtype=torch.bfloat16)tm = torch.tensor(1.0, dtype=torch.float32)- q, s, z = fp4_v2_with_global_scale(t, group_size=16, tensor_max=tm)+ q, s, z = nvfp4_v2_with_global_scale(t, group_size=16, tensor_max=tm)assert q.shape == t.shapedef test_tensor_max_multi_element(self):t = torch.randn(4, 32, dtype=torch.bfloat16)tm = torch.tensor([1.0, 2.0])- q, s, z = fp4_v2_with_global_scale(t, group_size=16, tensor_max=tm)+ q, s, z = nvfp4_v2_with_global_scale(t, group_size=16, tensor_max=tm)assert q.shape == t.shapedef test_with_max_scale(self):t = torch.randn(4, 32, dtype=torch.bfloat16)- q, s, z = fp4_v2_with_global_scale(t, group_size=16, max_scale=1.5)+ q, s, z = nvfp4_v2_with_global_scale(t, group_size=16, max_scale=1.5)assert q.shape == t.shapedef test_float16_input(self):t = torch.randn(4, 32, dtype=torch.float16)- q, s, z = fp4_v2_with_global_scale(t, group_size=16)+ q, s, z = nvfp4_v2_with_global_scale(t, group_size=16)assert q.dtype == torch.float16def test_with_v(self):t = torch.randn(4, 32, dtype=torch.bfloat16)- q, s, z = fp4_v2_with_global_scale(t, group_size=16, v=0.5)+ q, s, z = nvfp4_v2_with_global_scale(t, group_size=16, v=0.5)assert q.shape == t.shape# ----------------------------------------------------------------------------# fp4_v2+# nvfp4_v2# ----------------------------------------------------------------------------class TestFp4V2:- """Test fp4_v2."""+class TestNvfp4V2:+ """Test nvfp4_v2."""def test_group_size_16(self):t = torch.randn(4, 32, dtype=torch.bfloat16)- q, s, z = fp4_v2(t, group_size=16)+ q, s, z = nvfp4_v2(t, group_size=16)assert q.shape == t.shapeassert z is Nonedef test_group_size_32(self):t = torch.randn(4, 64, dtype=torch.bfloat16)- q, s, z = fp4_v2(t, group_size=32)+ q, s, z = nvfp4_v2(t, group_size=32)assert q.shape == t.shapedef test_invalid_group_size(self):t = torch.randn(4, 32, dtype=torch.bfloat16)with pytest.raises(AssertionError):- fp4_v2(t, group_size=128)+ nvfp4_v2(t, group_size=128)def test_with_max_scale(self):t = torch.randn(4, 32, dtype=torch.bfloat16)- q, s, z = fp4_v2(t, group_size=16, max_scale=1.5)+ q, s, z = nvfp4_v2(t, group_size=16, max_scale=1.5)assert q.shape == t.shapedef test_with_v(self):t = torch.randn(4, 32, dtype=torch.bfloat16)- q, s, z = fp4_v2(t, group_size=16, v=0.5)+ q, s, z = nvfp4_v2(t, group_size=16, v=0.5)assert q.shape == t.shapedef test_float16_input(self):t = torch.randn(4, 32, dtype=torch.float16)- q, s, z = fp4_v2(t, group_size=16)+ q, s, z = nvfp4_v2(t, group_size=16)assert q.dtype == torch.float16def test_non_divisible_dim(self):"""Last dim not divisible by group_size -> padding path."""t = torch.randn(4, 50, dtype=torch.bfloat16)# group_size=32 divides 50 with padding to 64- q, s, z = fp4_v2(t, group_size=32)+ q, s, z = nvfp4_v2(t, group_size=32)assert q.shape == t.shape@@ -654,8 +654,8 @@ class TestQuantizationProperties:[lambda t: nv_fp4(t),lambda t: nv_fp4_with_static_gs(t),- lambda t: fp4_v2(t),- lambda t: fp4_v2_with_global_scale(t),+ lambda t: nvfp4_v2(t),+ lambda t: nvfp4_v2_with_global_scale(t),],)def test_outputs_are_finite(self, fn):@@ -670,8 +670,8 @@ class TestQuantizationProperties:[lambda t: nv_fp4(t),lambda t: nv_fp4_with_static_gs(t),- lambda t: fp4_v2(t),- lambda t: fp4_v2_with_global_scale(t),+ lambda t: nvfp4_v2(t),+ lambda t: nvfp4_v2_with_global_scale(t),],)def test_dtype_preserved(self, fn):diff --git a/test/unit/test_cpu/export/test_fake_format.py b/test/unit/test_cpu/export/test_fake_format.pynew file mode 100644index 000000000..10b064e4a--- /dev/null+++ b/test/unit/test_cpu/export/test_fake_format.py@@ -0,0 +1,188 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++import json+import os+from types import SimpleNamespace++import torch+from transformers import AutoModelForCausalLM, OPTConfig, OPTForCausalLM++from auto_round.experimental.qmodules.fake import FakeActQuantLinear+from auto_round.formats import FakeFormat+from auto_round.inference.convert_model import convert_hf_model+from auto_round.schemes import PRESET_SCHEMES+from auto_round.wrapper import WrapperWALayer+++class _WrappedLinear(WrapperWALayer):+ def __init__(self, linear):+ torch.nn.Module.__init__(self)+ self.orig_layer = linear+ self.register_buffer("act_max_scale", torch.ones(1))++ def forward(self, inputs):+ return self.orig_layer(inputs)+++class _SaveableModel(torch.nn.Module):+ def __init__(self):+ super().__init__()+ linear = torch.nn.Linear(4, 3)+ linear.register_parameter("act_max_scale", torch.nn.Parameter(torch.ones(1)))+ self.linear = _WrappedLinear(linear)+ self.config = SimpleNamespace()++ def save_pretrained(self, output_dir):+ os.makedirs(output_dir, exist_ok=True)+ torch.save(self.state_dict(), os.path.join(output_dir, "pytorch_model.bin"))+ with open(os.path.join(output_dir, "config.json"), "w") as config_file:+ json.dump({"quantization_config": self.config.quantization_config}, config_file)+++def test_fake_format_unwraps_quantized_layers_before_save(tmp_path):+ model = _SaveableModel()+ expected_weight = model.linear.orig_layer.weight.detach().clone()+ output_dir = str(tmp_path / "fake_model")++ FakeFormat("fake", PRESET_SCHEMES["NVFP4_E5M3"], SimpleNamespace(mllm=False)).save_quantized(+ output_dir=output_dir,+ model=model,+ inplace=False,+ serialization_dict={+ "bits": 4,+ "group_size": 16,+ "sym": True,+ "data_type": "nvfp4_v2",+ "act_bits": 4,+ "act_group_size": 16,+ "act_sym": True,+ "act_data_type": "nvfp4_v2",+ "supported_types": [torch.nn.Linear],+ },+ )++ state_dict = torch.load(os.path.join(output_dir, "pytorch_model.bin"), weights_only=True)+ assert set(state_dict) == {"linear.weight", "linear.bias"}+ assert torch.equal(state_dict["linear.weight"], expected_weight)+ assert hasattr(model.linear, "orig_layer")+ with open(os.path.join(output_dir, "config.json")) as config_file:+ quantization_config = json.load(config_file)["quantization_config"]+ assert "supported_types" not in quantization_config+ assert quantization_config["packing_format"] == "auto_round:fake"+++class _TinyLoadModel(torch.nn.Module):+ def __init__(self, quantization_config):+ super().__init__()+ self.block = torch.nn.Module()+ self.block.linear = torch.nn.Linear(16, 4)+ self.lm_head = torch.nn.Linear(16, 4)+ self.config = SimpleNamespace(quantization_config=quantization_config)+++def test_fake_config_replaces_linear_and_qdq_activation_on_load():+ quantization_config = SimpleNamespace(+ bits=4,+ group_size=16,+ sym=True,+ data_type="nvfp4_v2",+ act_bits=4,+ act_group_size=16,+ act_sym=True,+ act_data_type="nvfp4_v2",+ act_dynamic=True,+ quant_method="auto-round",+ packing_format="auto_round:fake",+ block_name_to_quantize="block",+ backend="auto",+ extra_config={},+ modules_to_not_convert=[],+ )+ model = _TinyLoadModel(quantization_config)+ original_weight = model.block.linear.weight.detach().clone()+ activation = torch.randn(2, 3, 16)++ model, used_backends = convert_hf_model(model, target_device="cpu")++ assert used_backends == ["auto_round:fake"]+ assert isinstance(model.block.linear, FakeActQuantLinear)+ assert torch.equal(model.block.linear.weight, original_weight)+ qdq_activation = model.block.linear.qdq_input(activation)+ assert not torch.equal(qdq_activation, activation)+ expected = torch.nn.functional.linear(qdq_activation, model.block.linear.weight, model.block.linear.bias)+ assert torch.equal(model.block.linear(activation), expected)+++def test_fake_config_keeps_modules_to_not_convert_in_full_precision():+ quantization_config = SimpleNamespace(+ bits=4,+ group_size=16,+ sym=True,+ data_type="nvfp4_v2",+ act_bits=4,+ act_group_size=16,+ act_sym=True,+ act_data_type="nvfp4_v2",+ act_dynamic=True,+ quant_method="auto-round",+ packing_format="auto_round:fake",+ block_name_to_quantize="",+ backend="auto",+ extra_config={},+ modules_to_not_convert=["lm_head"],+ )+ model = _TinyLoadModel(quantization_config)+ original_lm_head = model.lm_head++ model, used_backends = convert_hf_model(model, target_device="cpu")++ assert used_backends == ["auto_round:fake"]+ assert isinstance(model.block.linear, FakeActQuantLinear)+ assert model.lm_head is original_lm_head+++def test_transformers_load_replaces_fake_linear(tmp_path):+ config = OPTConfig(+ vocab_size=32,+ hidden_size=16,+ ffn_dim=32,+ num_hidden_layers=1,+ num_attention_heads=2,+ max_position_embeddings=32,+ word_embed_proj_dim=16,+ )+ config.quantization_config = {+ "bits": 4,+ "group_size": 16,+ "sym": True,+ "data_type": "nvfp4_v2",+ "act_bits": 4,+ "act_group_size": 16,+ "act_sym": True,+ "act_data_type": "nvfp4_v2",+ "act_dynamic": True,+ "quant_method": "auto-round",+ "packing_format": "auto_round:fake",+ "block_name_to_quantize": "model.decoder.layers",+ }+ model_dir = str(tmp_path / "fake_opt")+ OPTForCausalLM(config).save_pretrained(model_dir)++ loaded_model = AutoModelForCausalLM.from_pretrained(model_dir, device_map="cpu")++ q_proj = loaded_model.model.decoder.layers[0].self_attn.q_proj+ assert isinstance(q_proj, FakeActQuantLinear)+ activation = torch.randn(1, 2, 16)+ assert not torch.equal(q_proj.qdq_input(activation), activation)diff --git a/test/unit/test_cpu/quantization/test_model_free.py b/test/unit/test_cpu/quantization/test_model_free.pyindex 5e39a42cf..4e0d8e6e9 100644--- a/test/unit/test_cpu/quantization/test_model_free.py+++ b/test/unit/test_cpu/quantization/test_model_free.py@@ -52,6 +52,13 @@ from safetensors.torch import save_filefrom auto_round import AutoRoundfrom auto_round.compressors.model_free import (+ _ModelFreeCompressorCore,+ _process_single_shard_task,+ get_predefined_ignore_layers_from_config,+)+from auto_round.schemes import QuantizationScheme+from auto_round.utils.model import is_model_free_route+from auto_round.utils.model_free_utils import (_build_mxfp_autoround_quantization_config,_build_mxfp_quantization_config,_build_quantization_config,@@ -59,21 +66,23 @@ from auto_round.compressors.model_free import (_dequant_fp8_tensors,_dequant_mxfp_tensors,_expand_e8m0_block_scale,- _handle_model_type_low_precision_source_tensors,_handle_mxfp_source_tensors,_looks_like_auto_scheme,- _ModelFreeCompressorCore,_PatternMatcher,- _preprocess_model_type_source_tensors,_process_shard,- _process_single_shard_task,_quantize_weight_mxfp,+ _quantize_weight_nvfp4_e5m3,_validate_auto_scheme_options,- get_predefined_ignore_layers_from_config,+)+from auto_round.utils.model_free_utils import (+ handle_model_type_low_precision_source_tensors as _handle_model_type_low_precision_source_tensors,+)+from auto_round.utils.model_free_utils import (is_model_free_supported_scheme,)-from auto_round.schemes import QuantizationScheme-from auto_round.utils.model import is_model_free_route+from auto_round.utils.model_free_utils import (+ preprocess_model_type_source_tensors as _preprocess_model_type_source_tensors,+)def test_model_free_preserves_explicit_scheme_overrides():@@ -310,7 +319,7 @@ class TestProcessShard:shard_path = str(tmp_path / "shard.safetensors")save_file({"layer.fc1.weight": torch.randn(64, 128)}, shard_path)compile_mock = Mock(side_effect=lambda func, _device: func)- monkeypatch.setattr("auto_round.compressors.model_free.compile_func", compile_mock)+ monkeypatch.setattr("auto_round.utils.model_free_utils.compile_func", compile_mock)_process_shard(shard_path, _DEFAULT_SCHEME, {}, [], enable_torch_compile=enabled)@@ -323,6 +332,127 @@ class TestProcessShard:assert "layer.fc1" in quantizedassert "layer.fc1.qweight" in output and "layer.fc1.bias" in output++def test_nvfp4_e5m3_model_free_fake_quantization():+ weight = torch.randn(8, 32)+ output = _quantize_weight_nvfp4_e5m3(weight, "layer.fc", group_size=16)++ assert set(output) == {"layer.fc.weight"}+ assert output["layer.fc.weight"].shape == weight.shape+ assert output["layer.fc.weight"].dtype == weight.dtype+ assert not torch.equal(output["layer.fc.weight"], weight)+ assert is_model_free_supported_scheme("NVFP4_E5M3")+ assert not is_model_free_supported_scheme("UNVFP4")+ assert not is_model_free_supported_scheme("NVFP4+")+++def test_nvfp4_e5m3_model_free_end_to_end(tmp_path):+ tensors = {+ "model.layers.0.self_attn.q_proj.weight": torch.randn(32, 32),+ "lm_head.weight": torch.randn(64, 32),+ }+ model_dir = _make_model_dir(tmp_path, _LLAMA_CFG, tensors)+ output_dir = str(tmp_path / "output")+ os.makedirs(output_dir)+ with open(os.path.join(output_dir, "quantization_config.json"), "w") as f:+ json.dump({"stale": True}, f)++ compressor = _ModelFreeCompressorCore(model_name_or_path=model_dir, output_dir=output_dir, scheme="NVFP4_E5M3")+ compressor.run()++ output_keys = _read_output_keys(output_dir)+ assert "model.layers.0.self_attn.q_proj.weight" not in output_keys+ assert "model.layers.0.self_attn.q_proj.weight_packed" in output_keys+ assert "model.layers.0.self_attn.q_proj.weight_scale" in output_keys+ assert "lm_head.weight" in output_keys+ assert compressor.format == "auto_round"+ quantization_config = _read_qconfig(output_dir)+ assert quantization_config["packing_format"] == "auto_round:llm_compressor_nvfp4_e5m3"+ assert quantization_config["data_type"] == "nvfp4_v2"+ assert quantization_config["act_bits"] == 4+ assert quantization_config["act_data_type"] == "nvfp4_v2"+ assert quantization_config["act_group_size"] == 16+ assert quantization_config["act_sym"] is True+ assert quantization_config["extra_config"]["lm_head"] == {+ "bits": 16,+ "data_type": "float",+ "act_bits": 16,+ "act_data_type": "float",+ }+ assert os.path.exists(os.path.join(output_dir, "quantization_config.json"))+++def test_nvfp4_e5m3_model_free_llm_compressor(tmp_path):+ tensors = {+ "model.layers.0.self_attn.q_proj.weight": torch.randn(32, 32),+ "lm_head.weight": torch.randn(64, 32),+ }+ model_dir = _make_model_dir(tmp_path, _LLAMA_CFG, tensors)+ output_dir = str(tmp_path / "output")++ compressor = _ModelFreeCompressorCore(+ model_name_or_path=model_dir,+ output_dir=output_dir,+ scheme="NVFP4_E5M3",+ format="llm_compressor",+ )+ compressor.run()++ output_keys = _read_output_keys(output_dir)+ prefix = "model.layers.0.self_attn.q_proj"+ assert f"{prefix}.weight_packed" in output_keys+ assert f"{prefix}.weight_scale" in output_keys+ assert f"{prefix}.weight" not in output_keys+ assert f"{prefix}.weight_global_scale" not in output_keys+ assert f"{prefix}.input_global_scale" not in output_keys+ quantization_config = _read_qconfig(output_dir)+ group = quantization_config["config_groups"]["group_0"]+ assert quantization_config["format"] == "nvfp4-e5m3-pack-quantized"+ assert quantization_config["quant_method"] == "compressed-tensors"+ assert quantization_config["provider"] == "auto-round"+ assert group["weights"]["group_size"] == 16+ assert group["input_activations"]["dynamic"] == "local"+++def test_model_free_legacy_nvfp4_is_normalized_and_passthrough(tmp_path):+ prefix = "model.layers.0.mlp.down_proj"+ tensors = {+ f"{prefix}.weight": torch.randint(0, 256, (32, 64), dtype=torch.uint8),+ f"{prefix}.weight_scale": torch.randint(0, 256, (32, 4), dtype=torch.uint8),+ f"{prefix}.weight_scale_2": torch.tensor([2.0], dtype=torch.float32),+ f"{prefix}.input_scale": torch.tensor([4.0], dtype=torch.float32),+ "model.layers.0.self_attn.q_proj.weight": torch.randn(32, 32),+ }+ shard_path = str(tmp_path / "shard.safetensors")+ save_file(tensors, shard_path)++ layer_config = {+ prefix: {+ "bits": 4,+ "group_size": 16,+ "sym": True,+ "data_type": "nv_fp",+ }+ }+ output, quantized, ignored = _process_shard(shard_path, _DEFAULT_SCHEME, layer_config, [])++ # Legacy naming should be normalized to llm-compressor-style global-scale keys.+ assert f"{prefix}.weight_packed" in output+ assert f"{prefix}.weight_scale" in output+ assert f"{prefix}.weight_global_scale" in output+ assert f"{prefix}.input_global_scale" in output+ assert f"{prefix}.weight" not in output+ assert f"{prefix}.weight_scale_2" not in output+ assert f"{prefix}.input_scale" not in output+ assert torch.allclose(output[f"{prefix}.weight_global_scale"], torch.tensor([0.5], dtype=torch.float32))+ assert torch.allclose(output[f"{prefix}.input_global_scale"], torch.tensor([0.25], dtype=torch.float32))++ # The NVFP4 layer is treated as already quantized (passthrough) while+ # other Linear layers in the shard are still quantized by model-free RTN.+ assert prefix in quantized+ assert "model.layers.0.self_attn.q_proj" in quantized+ assert prefix not in ignored+def test_ignores_and_skips(self, tmp_path):shard_path = str(tmp_path / "shard.safetensors")save_file(@@ -412,24 +542,40 @@ class TestFP8Source:output, quantized, _ = _process_shard(shard_path, _DEFAULT_SCHEME, {}, [], device="cpu", fp8_block_size=None)assert "layer" in quantized and "layer.qweight" in output- def test_ignored_layer_preserves_original_fp8(self, tmp_path):- """Ignored layers keep their original quantized tensors (no dequant)."""- shard_path = str(tmp_path / "shard.safetensors")- w_fp8 = torch.randn(64, 128, dtype=torch.bfloat16).to(torch.float8_e4m3fn)- scale = torch.tensor(0.5)- save_file(- {"lm_head.weight": w_fp8, "lm_head.weight_scale_inv": scale, "layer.weight": torch.randn(64, 128)},- shard_path,- )- output, quantized, ignored = _process_shard(- shard_path, _DEFAULT_SCHEME, {}, ["lm_head"], device="cpu", fp8_block_size=None+ def test_dequant_fp8_hydrates_scale_from_sibling_shard(self, tmp_path):+ """When scale_inv is sharded separately, dequant should hydrate it via index."""+ shard_dir = tmp_path / "source"+ shard_dir.mkdir(parents=True, exist_ok=True)++ weight_name = "model.layers.0.mlp.experts.1.gate_proj.weight"+ scale_name = "model.layers.0.mlp.experts.1.gate_proj.weight_scale_inv"++ shard_a = shard_dir / "model-00001-of-00002.safetensors"+ shard_b = shard_dir / "model-00002-of-00002.safetensors"+ save_file({weight_name: torch.randn(2048, 7168, dtype=torch.bfloat16).to(torch.float8_e4m3fn)}, str(shard_a))+ save_file({scale_name: torch.ones(16, 56, dtype=torch.float32)}, str(shard_b))++ with open(shard_dir / "model.safetensors.index.json", "w") as f:+ json.dump(+ {+ "metadata": {"total_size": 0},+ "weight_map": {+ weight_name: shard_a.name,+ scale_name: shard_b.name,+ },+ },+ f,+ )++ output, quantized, _ = _process_shard(+ str(shard_a),+ _DEFAULT_SCHEME,+ {},+ [],+ fp8_block_size=[128, 128],)- # lm_head should be ignored and kept in original FP8 format- assert "lm_head" in ignored- assert output["lm_head.weight"].dtype == torch.float8_e4m3fn- assert "lm_head.weight_scale_inv" in output- # non-ignored layer should be quantized normally- assert "layer" in quantized+ assert "model.layers.0.mlp.experts.1.gate_proj" in quantized+ assert "model.layers.0.mlp.experts.1.gate_proj.qweight" in output# ===========================================================================diff --git a/test/unit/test_cpu/quantization/test_nvfp4_quant_linear.py b/test/unit/test_cpu/quantization/test_nvfp4_quant_linear.pyindex 860ef0ef0..82f403330 100644--- a/test/unit/test_cpu/quantization/test_nvfp4_quant_linear.py+++ b/test/unit/test_cpu/quantization/test_nvfp4_quant_linear.py@@ -1,10 +1,15 @@+from types import SimpleNamespace+from unittest.mock import patch+import pytestimport torch+from transformers.quantizers.auto import AutoHfQuantizer-from auto_round.data_type.nvfp import calculate_gparam+from auto_round.data_type.nvfp import calculate_gparam, nvfp4_v2from auto_round.data_type.utils import get_quant_funcfrom auto_round.experimental import qmodules as ar_qmodulesfrom auto_round.export.export_to_autoround.qlinear_fp import QuantLinear as _FPLinear+from auto_round.export.export_to_llmcompressor.config import initialize_nvfp4_e5m3_quantizationfrom auto_round.export.formats import BackendDataTypefrom auto_round.schemes import PRESET_SCHEMES@@ -35,6 +40,137 @@ def test_calculate_gparam_with_float8_input():assert torch.isfinite(global_scale)+def test_nvfp4_e5m3_compressed_tensors_loading_uses_no_global_scales():+ class TinyModel(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.config = SimpleNamespace(model_type="tiny")+ self.model = torch.nn.Module()+ self.model.layers = torch.nn.ModuleList([torch.nn.ModuleDict({"fc": torch.nn.Linear(16, 8)})])++ quantizer = AutoHfQuantizer.from_config(initialize_nvfp4_e5m3_quantization([]))+ model = quantizer._process_model_before_weight_loading(TinyModel())+ layer = model.model.layers[0]["fc"]++ assert isinstance(layer, ar_qmodules.NVFP4E5M3QuantLinear)+ assert set(layer.state_dict()) == {"bias", "weight_packed", "weight_scale"}+ assert quantizer._process_model_after_weight_loading(model) is model+++def test_nvfp4_e5m3_qdq_input_uses_reference_fallback_on_cpu():+ config = PRESET_SCHEMES["NVFP4_E5M3"]+ layer = ar_qmodules.NVFP4E5M3QuantLinear(16, 8, config, dtype=torch.float32)+ activation = torch.randn(2, 3, 16)++ expected, _, _ = nvfp4_v2(activation, bits=config.act_bits, group_size=config.act_group_size)++ assert torch.equal(layer.qdq_input(activation), expected)+++def test_nvfp4_e5m3_forward_does_not_cache_dequantized_weight_by_default():+ config = PRESET_SCHEMES["NVFP4_E5M3"]+ layer = ar_qmodules.NVFP4E5M3QuantLinear(16, 8, config, dtype=torch.float32)+ activation = torch.randn(2, 16)+ dequantized_weight = torch.randn(8, 16)++ with patch.object(layer, "dequant_weight_online", return_value=dequantized_weight) as dequant_weight_online:+ layer(activation)+ layer(activation)++ assert layer._cached_weight is None+ assert dequant_weight_online.call_count == 2+++def test_nvfp4_e5m3_forward_caches_dequantized_weight_when_enabled():+ config = PRESET_SCHEMES["NVFP4_E5M3"]+ layer = ar_qmodules.NVFP4E5M3QuantLinear(16, 8, config, dtype=torch.float32, cache_weight=True)+ activation = torch.randn(2, 16)+ dequantized_weight = torch.randn(8, 16)++ with patch.object(layer, "dequant_weight_online", return_value=dequantized_weight) as dequant_weight_online:+ layer(activation)+ layer(activation)++ dequant_weight_online.assert_called_once_with()+ assert layer.weight_packed is None+ assert layer.weight_scale is None+++def test_nvfp4_e5m3_cannot_clear_released_quantized_weight_buffers():+ config = PRESET_SCHEMES["NVFP4_E5M3"]+ layer = ar_qmodules.NVFP4E5M3QuantLinear(16, 8, config, dtype=torch.float32, cache_weight=True)++ layer(torch.randn(2, 16))++ with pytest.raises(RuntimeError, match="quantized weight buffers have been released"):+ layer.clear_weight_cache()+++def test_cute_nvfp4_e5m3_does_not_cache_dequantized_weight_by_default(monkeypatch):+ monkeypatch.delenv("AR_NVFP4_E5M3_CACHE_HP_WEIGHT", raising=False)+ config = PRESET_SCHEMES["NVFP4_E5M3"]+ layer = ar_qmodules.CuteNVFP4E5M3QuantLinear(16, 8, config, dtype=torch.float32)+ activation = torch.randn(2, 16)+ dequantized_weight = torch.randn(8, 16)++ with patch.object(layer, "dequant_weight_online", return_value=dequantized_weight) as dequant_weight_online:+ layer(activation)+ layer(activation)++ assert not layer.cache_weight+ assert dequant_weight_online.call_count == 2+++def test_nvfp4_e5m3_cache_weight_environment_override(monkeypatch):+ config = PRESET_SCHEMES["NVFP4_E5M3"]+ monkeypatch.setenv("AR_NVFP4_E5M3_CACHE_HP_WEIGHT", "1")+ assert ar_qmodules.NVFP4E5M3QuantLinear(16, 8, config, dtype=torch.float32).cache_weight+ assert ar_qmodules.CuteNVFP4E5M3QuantLinear(16, 8, config, dtype=torch.float32).cache_weight+ monkeypatch.setenv("AR_NVFP4_E5M3_CACHE_HP_WEIGHT", "0")+ assert not ar_qmodules.CuteNVFP4E5M3QuantLinear(16, 8, config, dtype=torch.float32).cache_weight+++def test_nvfp4_e5m3_torch_forward_does_not_call_cute():+ config = PRESET_SCHEMES["NVFP4_E5M3"]+ layer = ar_qmodules.NVFP4E5M3QuantLinear(16, 8, config, dtype=torch.float32)+ activation = torch.randn(2, 16)++ with patch(+ "auto_round.experimental.qmodules.nvfp4_e5m3.try_cute_nvfp4_e5m3_linear",+ ) as cute_linear:+ output = layer(activation)++ assert output.shape == (2, 8)+ cute_linear.assert_not_called()+++def test_nvfp4_e5m3_cute_forward_uses_fused_output_when_available():+ config = PRESET_SCHEMES["NVFP4_E5M3"]+ layer = ar_qmodules.CuteNVFP4E5M3QuantLinear(16, 8, config, dtype=torch.float32)+ activation = torch.randn(2, 16)+ fused_output = torch.randn(2, 8)++ with patch("auto_round.experimental.qmodules.nvfp4_e5m3.try_cute_nvfp4_e5m3_linear", return_value=fused_output):+ assert layer(activation) is fused_output+++def test_cute_nvfp4_e5m3_uses_cached_weight_when_enabled():+ config = PRESET_SCHEMES["NVFP4_E5M3"]+ layer = ar_qmodules.CuteNVFP4E5M3QuantLinear(16, 8, config, dtype=torch.float32, cache_weight=True)+ activation = torch.randn(2, 16)+ dequantized_weight = torch.randn(8, 16)++ with (+ patch.object(layer, "dequant_weight_online", return_value=dequantized_weight) as dequant_weight_online,+ patch("auto_round.experimental.qmodules.nvfp4_e5m3.try_cute_nvfp4_e5m3_linear") as cute_linear,+ ):+ layer(activation)+ layer(activation)++ dequant_weight_online.assert_called_once_with()+ cute_linear.assert_not_called()++@pytest.mark.parametrize("scheme", [BackendDataType.NVFP4.value])@torch.inference_mode()def test_nvfp4_quantlinear_from_original_and_forward(scheme):diff --git a/test/unit/test_cpu/schemes/test_auto_scheme.py b/test/unit/test_cpu/schemes/test_auto_scheme.pyindex e87093bde..43b90815d 100644--- a/test/unit/test_cpu/schemes/test_auto_scheme.py+++ b/test/unit/test_cpu/schemes/test_auto_scheme.py@@ -280,7 +280,7 @@ class TestAutoScheme:def teardown_class(self):shutil.rmtree("runs", ignore_errors=True)- @pytest.mark.timeout(60)+ @pytest.mark.timeout(70)def test_auto_scheme_export(self, tiny_opt_model_path):model_name = tiny_opt_model_pathint_save_dir = os.path.join(self.save_dir, "int")diff --git a/test/unit/test_cpu/utils/test_model_free_cross_shard.py b/test/unit/test_cpu/utils/test_model_free_cross_shard.pynew file mode 100644index 000000000..e2f0c4700--- /dev/null+++ b/test/unit/test_cpu/utils/test_model_free_cross_shard.py@@ -0,0 +1,270 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++"""Unit tests for cross-shard FP8 scale handling in model_free_utils."""++import json+import os+import tempfile++import pytest+import torch++from auto_round.utils.model_free_utils import (+ _build_cross_shard_pairs_from_weight_map,+ _hydrate_missing_fp8_scales_from_index,+)++# ---------------------------------------------------------------------------+# _build_cross_shard_pairs_from_weight_map+# ---------------------------------------------------------------------------+++class TestBuildCrossShardPairs:+ """Tests for _build_cross_shard_pairs_from_weight_map."""++ def _make_weight_map(self, entries: dict[str, str]) -> dict[str, str]:+ return dict(entries)++ def test_no_fp8_entries_returns_empty(self):+ weight_map = {+ "model.layer.weight": "shard-00001.safetensors",+ "model.layer.bias": "shard-00001.safetensors",+ }+ recipient_to_donors, donor_shard_tensors = _build_cross_shard_pairs_from_weight_map(weight_map)+ assert recipient_to_donors == {}+ assert donor_shard_tensors == {}++ def test_same_shard_scale_not_cross(self):+ """weight and weight_scale_inv in the same shard → not a cross-shard pair."""+ weight_map = {+ "model.layer.weight": "shard-00001.safetensors",+ "model.layer.weight_scale_inv": "shard-00001.safetensors",+ }+ recipient_to_donors, donor_shard_tensors = _build_cross_shard_pairs_from_weight_map(weight_map)+ assert recipient_to_donors == {}+ assert donor_shard_tensors == {}++ def test_cross_shard_single_pair(self):+ """weight in shard-1, weight_scale_inv in shard-2."""+ weight_map = {+ "model.layer.weight": "shard-00001.safetensors",+ "model.layer.weight_scale_inv": "shard-00002.safetensors",+ }+ recipient_to_donors, donor_shard_tensors = _build_cross_shard_pairs_from_weight_map(weight_map)++ assert "shard-00001.safetensors" in recipient_to_donors+ donor_map = recipient_to_donors["shard-00001.safetensors"]+ assert "shard-00002.safetensors" in donor_map+ assert "model.layer.weight_scale_inv" in donor_map["shard-00002.safetensors"]++ assert "shard-00002.safetensors" in donor_shard_tensors+ assert "model.layer.weight_scale_inv" in donor_shard_tensors["shard-00002.safetensors"]++ def test_cross_shard_multiple_layers_one_donor(self):+ """Multiple layers whose scale_inv all live in the same donor shard."""+ weight_map = {+ "model.layer0.weight": "shard-00001.safetensors",+ "model.layer0.weight_scale_inv": "shard-00002.safetensors",+ "model.layer1.weight": "shard-00001.safetensors",+ "model.layer1.weight_scale_inv": "shard-00002.safetensors",+ }+ recipient_to_donors, donor_shard_tensors = _build_cross_shard_pairs_from_weight_map(weight_map)++ donor_map = recipient_to_donors["shard-00001.safetensors"]+ scales = donor_map["shard-00002.safetensors"]+ assert "model.layer0.weight_scale_inv" in scales+ assert "model.layer1.weight_scale_inv" in scales+ assert len(donor_shard_tensors["shard-00002.safetensors"]) == 2++ def test_cross_shard_multiple_donors(self):+ """Recipient shard needs scales from two different donor shards."""+ weight_map = {+ "model.layerA.weight": "shard-00001.safetensors",+ "model.layerA.weight_scale_inv": "shard-00002.safetensors",+ "model.layerB.weight": "shard-00001.safetensors",+ "model.layerB.weight_scale_inv": "shard-00003.safetensors",+ }+ recipient_to_donors, donor_shard_tensors = _build_cross_shard_pairs_from_weight_map(weight_map)++ donor_map = recipient_to_donors["shard-00001.safetensors"]+ assert "shard-00002.safetensors" in donor_map+ assert "shard-00003.safetensors" in donor_map+ assert len(donor_shard_tensors) == 2++ def test_scale_inv_without_matching_weight_ignored(self):+ """weight_scale_inv present in weight_map but no corresponding .weight → ignored."""+ weight_map = {+ "model.layer.weight_scale_inv": "shard-00002.safetensors",+ # no "model.layer.weight" key at all+ }+ recipient_to_donors, donor_shard_tensors = _build_cross_shard_pairs_from_weight_map(weight_map)+ assert recipient_to_donors == {}+ assert donor_shard_tensors == {}++ def test_empty_weight_map(self):+ recipient_to_donors, donor_shard_tensors = _build_cross_shard_pairs_from_weight_map({})+ assert recipient_to_donors == {}+ assert donor_shard_tensors == {}+++# ---------------------------------------------------------------------------+# _hydrate_missing_fp8_scales_from_index+# ---------------------------------------------------------------------------+++def _write_fake_fp8_shard(path: str, tensors: dict[str, torch.Tensor]) -> None:+ """Save a dict of tensors as a safetensors file."""+ from safetensors.torch import save_file++ save_file(tensors, path)+++def _write_index_json(directory: str, weight_map: dict[str, str]) -> str:+ index_path = os.path.join(directory, "model.safetensors.index.json")+ with open(index_path, "w") as f:+ json.dump({"weight_map": weight_map}, f)+ return index_path+++class TestHydrateMissingFp8Scales:+ """Tests for _hydrate_missing_fp8_scales_from_index."""++ def test_non_safetensors_shard_returns_unchanged(self, tmp_path):+ raw = {"w": torch.zeros(4)}+ result = _hydrate_missing_fp8_scales_from_index(raw, str(tmp_path / "model.bin"))+ assert result is raw++ def test_no_fp8_weights_returns_unchanged(self, tmp_path):+ shard_path = str(tmp_path / "shard-00001.safetensors")+ # BF16 weight — not FP8, so nothing to hydrate+ raw = {"model.layer.weight": torch.zeros(4, dtype=torch.bfloat16)}+ result = _hydrate_missing_fp8_scales_from_index(raw, shard_path)+ assert result is raw++ def test_all_scales_present_returns_unchanged(self, tmp_path):+ shard_path = str(tmp_path / "shard-00001.safetensors")+ raw = {+ "model.layer.weight": torch.zeros(4, dtype=torch.float8_e4m3fn),+ "model.layer.weight_scale_inv": torch.ones(1),+ }+ result = _hydrate_missing_fp8_scales_from_index(raw, shard_path)+ assert "model.layer.weight_scale_inv" in result++ def test_cross_shard_hydration_local_mode(self, tmp_path):+ """Recipient shard gets scale_inv from donor shard in local (non-streaming) mode."""+ donor_name = "shard-00002.safetensors"+ recipient_name = "shard-00001.safetensors"+ scale_name = "model.layer.weight_scale_inv"++ # Write donor shard with scale_inv+ donor_path = tmp_path / donor_name+ _write_fake_fp8_shard(str(donor_path), {scale_name: torch.ones(1)})++ # Write index.json in the same directory+ weight_map = {+ "model.layer.weight": recipient_name,+ scale_name: donor_name,+ }+ _write_index_json(str(tmp_path), weight_map)++ # Recipient shard has FP8 weight but no scale_inv+ recipient_path = tmp_path / recipient_name+ raw = {"model.layer.weight": torch.zeros(4, dtype=torch.float8_e4m3fn)}++ result = _hydrate_missing_fp8_scales_from_index(raw, str(recipient_path))+ assert scale_name in result, "scale_inv should be hydrated from donor shard"+ assert result[scale_name].dtype == torch.float32 or result[scale_name].numel() == 1++ def test_cross_shard_hydration_streaming_mode(self, tmp_path):+ """In streaming mode, index.json lives in work_dir, shards in cache subdir."""+ work_dir = tmp_path / "work_dir"+ cache_dir = work_dir / ".cache" / "model_free_source_shards"+ work_dir.mkdir()+ cache_dir.mkdir(parents=True)++ donor_name = "shard-00002.safetensors"+ recipient_name = "shard-00001.safetensors"+ scale_name = "model.layer.weight_scale_inv"++ # Donor shard lives in cache subdir+ _write_fake_fp8_shard(str(cache_dir / donor_name), {scale_name: torch.ones(1)})++ # Index.json lives in work_dir (not in cache subdir)+ weight_map = {+ "model.layer.weight": recipient_name,+ scale_name: donor_name,+ }+ _write_index_json(str(work_dir), weight_map)++ # Recipient shard path is in cache dir+ recipient_path = cache_dir / recipient_name+ raw = {"model.layer.weight": torch.zeros(4, dtype=torch.float8_e4m3fn)}++ result = _hydrate_missing_fp8_scales_from_index(+ raw,+ str(recipient_path),+ index_dir=str(work_dir),+ donor_shard_dir=str(cache_dir),+ )+ assert scale_name in result, "streaming mode: scale_inv should be hydrated via index_dir/donor_shard_dir params"++ def test_missing_index_json_returns_unchanged(self, tmp_path):+ """If no index.json exists, raw_tensors is returned as-is (no crash)."""+ shard_path = str(tmp_path / "shard-00001.safetensors")+ raw = {"model.layer.weight": torch.zeros(4, dtype=torch.float8_e4m3fn)}+ result = _hydrate_missing_fp8_scales_from_index(raw, shard_path)+ assert result is raw++ def test_donor_shard_missing_on_disk_returns_unchanged(self, tmp_path):+ """Index references a donor shard that doesn't exist on disk → graceful skip."""+ scale_name = "model.layer.weight_scale_inv"+ recipient_name = "shard-00001.safetensors"+ donor_name = "shard-00002.safetensors" # not written++ weight_map = {+ "model.layer.weight": recipient_name,+ scale_name: donor_name,+ }+ _write_index_json(str(tmp_path), weight_map)++ raw = {"model.layer.weight": torch.zeros(4, dtype=torch.float8_e4m3fn)}+ result = _hydrate_missing_fp8_scales_from_index(raw, str(tmp_path / recipient_name))+ assert scale_name not in result # hydration skipped silently++ def test_multiple_layers_hydrated_from_single_donor(self, tmp_path):+ """Multiple missing scale_inv tensors are hydrated in a single donor open."""+ donor_name = "shard-00002.safetensors"+ recipient_name = "shard-00001.safetensors"+ scales = {+ "model.layerA.weight_scale_inv": torch.ones(1),+ "model.layerB.weight_scale_inv": torch.ones(1),+ }+ _write_fake_fp8_shard(str(tmp_path / donor_name), scales)++ weight_map = {+ "model.layerA.weight": recipient_name,+ "model.layerA.weight_scale_inv": donor_name,+ "model.layerB.weight": recipient_name,+ "model.layerB.weight_scale_inv": donor_name,+ }+ _write_index_json(str(tmp_path), weight_map)++ raw = {+ "model.layerA.weight": torch.zeros(4, dtype=torch.float8_e4m3fn),+ "model.layerB.weight": torch.zeros(4, dtype=torch.float8_e4m3fn),+ }+ result = _hydrate_missing_fp8_scales_from_index(raw, str(tmp_path / recipient_name))+ assert "model.layerA.weight_scale_inv" in result+ assert "model.layerB.weight_scale_inv" in resultdiff --git a/test/unit/test_cuda/quantization/test_nvfp4_e5m3_cute.py b/test/unit/test_cuda/quantization/test_nvfp4_e5m3_cute.pynew file mode 100644index 000000000..e6a9e2b89--- /dev/null+++ b/test/unit/test_cuda/quantization/test_nvfp4_e5m3_cute.py@@ -0,0 +1,66 @@+import pytest+import torch++from auto_round.data_type.nvfp import cast_to_fp4, e5m3_to_float_tensor, nvfp4_v2+from auto_round.experimental.qmodules.fp4_utils import unpack_fp4_from_uint8+from auto_round_extension.cuda.cute_nvfp4_e5m3 import (+ can_use_cute_nvfp4_v2_qdq,+ try_cute_nvfp4_e5m3_weight_dq,+ try_cute_nvfp4_v2_qdq,+)++[email protected](not torch.cuda.is_available(), reason="CUDA is required")+ ("dtype", "atol"),+ [(torch.float32, 1e-6), (torch.float16, 2e-3), (torch.bfloat16, 2e-2)],+)+def test_cute_nvfp4_v2_qdq_matches_reference(dtype, atol):+ if torch.cuda.get_device_capability()[0] < 8:+ pytest.skip("CuTe QDQ requires SM80 or newer")++ activation = torch.randn(32, 16, device="cuda", dtype=dtype)+ if not can_use_cute_nvfp4_v2_qdq(activation, 16):+ pytest.skip("CuTe DSL is not available")++ expected, _, _ = nvfp4_v2(activation.float(), bits=4, group_size=16)+ actual = try_cute_nvfp4_v2_qdq(activation, 16)++ assert actual is not None+ torch.testing.assert_close(actual, expected.to(dtype), rtol=0, atol=atol)++[email protected](not torch.cuda.is_available(), reason="CUDA is required")[email protected]("dtype", [torch.float32, torch.float16, torch.bfloat16])+def test_cute_nvfp4_e5m3_weight_dq_matches_reference_for_multiple_groups(dtype):+ if torch.cuda.get_device_capability()[0] < 8:+ pytest.skip("CuTe weight DQ requires SM80 or newer")++ torch.manual_seed(42)+ weight_packed = torch.randint(0, 256, (37, 64), device="cuda", dtype=torch.uint8)+ weight_scale = torch.randint(0, 120, (37, 8), device="cuda", dtype=torch.uint8)+ actual = try_cute_nvfp4_e5m3_weight_dq(weight_packed, weight_scale, dtype)+ if actual is None:+ pytest.skip("CuTe DSL is not available")++ unpacked = unpack_fp4_from_uint8(weight_packed, 37, 128, dtype=dtype).float()+ expected = (unpacked.reshape(-1, 16) * e5m3_to_float_tensor(weight_scale).reshape(-1, 1)).reshape(37, 128).to(dtype)+ torch.testing.assert_close(actual, expected, rtol=0, atol=0)+++def test_nvfp4_v2_uses_direct_scale_at_rounding_boundary():+ activation = torch.tensor([[1.25] + [1.0] * 15], dtype=torch.float32)++ actual, scale, _ = nvfp4_v2(activation, bits=4, group_size=16)+ expected = cast_to_fp4(activation / scale) * scale++ torch.testing.assert_close(actual, expected)+++def test_nvfp4_v2_direct_scale_handles_zero_group():+ activation = torch.zeros(1, 16, dtype=torch.float32)++ actual, scale, _ = nvfp4_v2(activation, bits=4, group_size=16)++ assert torch.equal(actual, activation)+ assert torch.equal(scale, torch.zeros_like(scale))