Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

feat: support LLMC/ModelOpt/Quark format NVFP4 -> MXFP4 conversion (#2492)

Xin He7222a20eaca4
diff --git a/auto_round/compressors/model_free.py b/auto_round/compressors/model_free.py
index 0e834d52c..e8b88c1b1 100644
--- a/auto_round/compressors/model_free.py
+++ b/auto_round/compressors/model_free.py
@@ -40,6 +40,9 @@ weights with ``fake`` format):
* Preset names: ``MXFP4``, ``MXFP8``.
* ``data_type="mx_fp"``, ``group_size=32``, ``bits in {4, 8}``.
+* NVFP4 source checkpoints (detected from the model name or the source quantization config) with
+ scheme ``MXFP4``: only the NVFP4 layers are converted to MXFP4 (plus any layer matched by
+ ``layer_config``); every other layer is kept as-is.
**NVFP4 E5M3** (``auto_round`` or ``fake`` format):
@@ -125,6 +128,7 @@ from auto_round.utils.device import clear_memory, memory_monitor
from auto_round.utils.device_manager import default_enable_torch_compile
from auto_round.utils.model_free_utils import (
_LM_HEAD_PATTERNS,
+ _NVFP_SOURCE_ONLY_KEY,
_apply_scheme_overrides,
_build_cross_shard_pairs_from_weight_map,
_build_quantization_config,
@@ -138,6 +142,7 @@ from auto_round.utils.model_free_utils import (
_layer_config_has_nvfp4,
_list_weight_shards,
_load_config,
+ _load_source_quant_config,
_load_weight_map_from_index,
_looks_like_auto_scheme,
_normalize_scheme,
@@ -148,6 +153,7 @@ from auto_round.utils.model_free_utils import (
_validate_supported_scheme,
_write_index_file,
_write_output_shard,
+ detect_model_dtypes,
handle_model_type_low_precision_source_tensors,
is_model_free_supported_scheme,
preprocess_model_type_source_tensors,
@@ -200,6 +206,7 @@ _QUANTIZATION_CONFIG_FILENAMES: tuple[str, ...] = (
"quantization_config.json",
"quantize_config.json",
"quant_config.json",
+ "hf_quant_config.json",
)
_RESUME_MANIFEST_NAME = "model_free_resume.json"
@@ -572,6 +579,8 @@ class _ModelFreeCompressorCore:
self.model_name_or_path = model_name_or_path
self.output_dir = output_dir
self.scheme_input = scheme
+ # Set by _detect_nvfp4_source_model(): convert only the NVFP4 source layers to MXFP4.
+ self.nvfp_source_only = False
self.layer_config_input = layer_config
self.ignore_layers_input = ignore_layers
self.format = format or "auto_round"
@@ -664,6 +673,33 @@ class _ModelFreeCompressorCore:
self.default_scheme = {k: v for k, v in ds.items() if v is not None}
self.default_scheme["_output_format"] = self.format
+ def _detect_nvfp4_source_model(self) -> None:
+ """With an MXFP4 scheme on an NVFP4 source, convert only the NVFP4 layers (plus layer_config matches)."""
+ data_type = (self.scheme_obj.data_type or "").lower()
+ if not (is_mx_fp(data_type) and self.scheme_obj.bits == 4):
+ return
+ reason = detect_model_dtypes(
+ self.model_name_or_path,
+ _load_source_quant_config(
+ self.work_dir if self.is_streaming else self.source_dir, self.source_quantization_config
+ ),
+ ).get("nvfp4")
+ if reason is None:
+ return
+ format_lower = self.format.lower().replace(" ", "").split(",")[0]
+ if format_lower not in ("llm_compressor", "auto_round:llm_compressor"):
+ raise ValueError(
+ f"Detected an NVFP4 source model ({reason}); converting its NVFP4 layers to MXFP4 "
+ f"currently only supports format 'llm_compressor', got '{self.format}'."
+ )
+ logger.warning(
+ f"Detected NVFP4 layers in the source model ({reason}). Only the NVFP4 layers will be quantized "
+ "to MXFP4; all other layers (BF16, FP8, ...) are kept as-is. To quantize additional layers, "
+ "set them explicitly via layer_config."
+ )
+ self.nvfp_source_only = True
+ self.default_scheme[_NVFP_SOURCE_ONLY_KEY] = True
+
def _parse_layer_config(self) -> None:
lc = copy.deepcopy(self.layer_config_input) if self.layer_config_input else {}
@@ -846,12 +882,7 @@ class _ModelFreeCompressorCore:
def _detect_fp8_source(self) -> None:
quant_config = self.source_quantization_config
- is_fp8 = (
- quant_config.get("quant_method") == "fp8"
- or quant_config.get("quantization_type") == "fp8"
- or quant_config.get("fmt", "").startswith("e4m3")
- )
- if is_fp8:
+ if "fp8" in detect_model_dtypes(self.model_name_or_path, quant_config):
self.fp8_block_size = quant_config.get("weight_block_size")
logger.info(
f"Detected FP8 source model (block_size={self.fp8_block_size}, "
@@ -1556,6 +1587,12 @@ class _ModelFreeCompressorCore:
block_name_to_quantize = ",".join(dict.fromkeys(block_prefixes)) or None
os.makedirs(self._quant_output_dir, exist_ok=True)
+ source_quant_config = None
+ if self.nvfp_source_only:
+ # Read before the stale source metadata files are removed below.
+ source_quant_config = _load_source_quant_config(
+ self.work_dir if self.is_streaming else self.source_dir, self.source_quantization_config
+ )
quantization_config = _build_quantization_config(
default_scheme=self.default_scheme,
layer_config=self.layer_config,
@@ -1564,6 +1601,7 @@ class _ModelFreeCompressorCore:
ignored_layers=self.all_ignored_layers,
block_name_to_quantize=block_name_to_quantize,
format=self.format,
+ source_quant_config=source_quant_config,
)
self._remove_stale_quantization_config_files()
@@ -1700,6 +1738,7 @@ class _ModelFreeCompressorCore:
# ---- source resolution ----
self._resolve_source()
+ self._detect_nvfp4_source_model()
self._check_conv1d_and_embedding()
self._apply_predefined_ignore_layers()
self._detect_fp8_source()
@@ -1742,6 +1781,7 @@ class _ModelFreeCompressorCore:
logger.info(
f"Model-free quantization: {self.model_name_or_path}\n"
f" Scheme: {self.scheme_obj}\n"
+ f" NVFP4 source layers only: {self.nvfp_source_only}\n"
f" Packing format: {packing_format}\n"
f" Optimized RTN: {'enabled' if opt_rtn_enabled else 'disabled'}\n"
f" Output: {self.output_dir}\n"
@@ -1769,6 +1809,12 @@ class _ModelFreeCompressorCore:
f"{', '.join(missing_shards)}"
)
+ if self.nvfp_source_only and not self.all_quantized_layers:
+ logger.warning(
+ f"No NVFP4 layers were found in the shards of '{self.model_name_or_path}'; "
+ "all layers were kept as-is."
+ )
+
# ---- write outputs ----
self._write_index()
self._write_config_files()
diff --git a/auto_round/export/export_to_llmcompressor/export.py b/auto_round/export/export_to_llmcompressor/export.py
index 12148253f..22af10d1e 100644
--- a/auto_round/export/export_to_llmcompressor/export.py
+++ b/auto_round/export/export_to_llmcompressor/export.py
@@ -19,6 +19,10 @@ from collections.abc import Callable, Mapping
import torch
+from auto_round.export.export_to_llmcompressor.utils import (
+ add_routed_experts_if_moe,
+ finalize_llmcompressor_config_names,
+)
from auto_round.export.utils import (
is_immediate_saving_mode,
save_config_artifact,
@@ -175,7 +179,7 @@ def _rewrite_config_group_targets(model, quantization_config_dict):
)
names = sig_to_names.get(sig)
if names:
- group["targets"] = sorted(names)
+ group["targets"] = add_routed_experts_if_moe(sorted(names), names)
else:
logger.warning(
"AutoRound llm_compressor export: no quantized module matched "
@@ -372,6 +376,12 @@ def save_quantized_as_llmcompressor(
# precision that makes groups indistinguishable to vLLM; rewrite targets with
# the concrete module names each group covers.
quantization_config_dict = _rewrite_config_group_targets(model, quantization_config_dict)
+ quantization_config_dict = finalize_llmcompressor_config_names(
+ quantization_config_dict,
+ model,
+ [],
+ [n for n, m in model.named_modules() if getattr(m, "quantization_scheme", None) is not None],
+ )
model.config.quantization_config = quantization_config_dict
if output_dir is None:
diff --git a/auto_round/export/export_to_llmcompressor/export_to_fp.py b/auto_round/export/export_to_llmcompressor/export_to_fp.py
index 344b8f212..bafa994b3 100644
--- a/auto_round/export/export_to_llmcompressor/export_to_fp.py
+++ b/auto_round/export/export_to_llmcompressor/export_to_fp.py
@@ -27,7 +27,11 @@ from tqdm import tqdm
from auto_round.compressors.utils import is_mx_fp, is_nv_fp
from auto_round.export.export_to_autoround.qlinear_fp import QuantLinear
from auto_round.export.export_to_llmcompressor.config import initialize_quantization
-from auto_round.export.export_to_llmcompressor.utils import generate_ignore_regex_list
+from auto_round.export.export_to_llmcompressor.utils import (
+ add_routed_experts_if_moe,
+ finalize_llmcompressor_config_names,
+ generate_ignore_regex_list,
+)
from auto_round.export.utils import (
filter_quantization_config,
is_immediate_saving_mode,
@@ -234,7 +238,9 @@ def _build_mixed_fp_quantization_config(
tmp_quantization_config = initialize_quantization(scheme=scheme)
tmp_quantization_config = tmp_quantization_config.config_groups["group_0"]
is_default = (lbits, ldata_type, lact_bits) == global_key
- tmp_quantization_config.targets = ["Linear"] if is_default else layer_names
+ tmp_quantization_config.targets = (
+ ["Linear"] if is_default else add_routed_experts_if_moe(list(layer_names), layer_names)
+ )
config_groups[group_name] = tmp_quantization_config
group_formats[group_name] = _get_group_format(lbits, ldata_type)
@@ -435,7 +441,10 @@ def save_quantized_as_fp(
f"Unsupported layer override data_type={layer_data_type}, bits={layer_bits}, act_bits={layer_act_bits}."
)
quantization_config = initialize_quantization(
- scheme=scheme, targets=targets, kv_cache_scheme=kv_cache_scheme, ignore=ignore
+ scheme=scheme,
+ targets=add_routed_experts_if_moe(list(targets), targets),
+ kv_cache_scheme=kv_cache_scheme,
+ ignore=ignore,
)
quantization_config.format = _get_group_format(layer_bits, layer_data_type)
quantization_config = quantization_config.to_dict()
@@ -466,6 +475,12 @@ def save_quantized_as_fp(
_configure_gaudi2_fp8_dtype(quantization_config)
if attention_config is not None:
quantization_config["attention_input_activations"] = attention_config
+ quantization_config = finalize_llmcompressor_config_names(
+ quantization_config,
+ model,
+ [],
+ [name for names in scheme_groups.values() for name in names],
+ )
if hasattr(model, "config"):
model.config.quantization_config = quantization_config
if output_dir is None:
diff --git a/auto_round/export/export_to_llmcompressor/export_to_static_fp.py b/auto_round/export/export_to_llmcompressor/export_to_static_fp.py
index 364d533fe..6b1c4fc44 100644
--- a/auto_round/export/export_to_llmcompressor/export_to_static_fp.py
+++ b/auto_round/export/export_to_llmcompressor/export_to_static_fp.py
@@ -26,6 +26,7 @@ from auto_round.data_type.utils import reshape_pad_tensor_by_group_size, revert_
from auto_round.experimental.utils import NVFP4_KV_DTYPE
from auto_round.export.export_to_autoround.export_to_fp8 import FP8QLinear
from auto_round.export.export_to_llmcompressor.config import check_compressed_tensors_supported
+from auto_round.export.export_to_llmcompressor.utils import finalize_llmcompressor_config_names
from auto_round.export.utils import is_immediate_saving_mode, save_model, save_pretrained_artifact
from auto_round.utils import (
SUPPORTED_LAYER_TYPES,
@@ -294,6 +295,9 @@ def save_quantized_as_static_fp(
quantization_config["config_groups"]["group_0"]["weights"]["group_size"] = group_size
if attention_config is not None:
quantization_config["attention_input_activations"] = attention_config
+ quantization_config = finalize_llmcompressor_config_names(
+ quantization_config, model, [], [n for n in layer_config if check_to_quantized(layer_config[n])]
+ )
if hasattr(model, "config"):
model.config.quantization_config = quantization_config
diff --git a/auto_round/export/export_to_llmcompressor/utils.py b/auto_round/export/export_to_llmcompressor/utils.py
index a6020c8f0..eaa175f3b 100644
--- a/auto_round/export/export_to_llmcompressor/utils.py
+++ b/auto_round/export/export_to_llmcompressor/utils.py
@@ -13,8 +13,151 @@
# limitations under the License.
+import re
+
+import torch
+
from auto_round.utils import matches_any_regex, to_standard_regex
+_EXPERT_SEGMENTS = ("experts", "shared_experts")
+
+
+# TODO: Remove when the issue is fixed
+# https://github.com/vllm-project/llm-compressor/issues/3069
+def add_routed_experts_if_moe(targets: list[str], layer_names: list[str]) -> list[str]:
+ """Append ``"RoutedExperts"`` for routed expert layouts that llm-compressor misses.
+
+ The generic ``"Linear"`` target does not cover all expert layouts. Keep the
+ safe canonical cases alone (``mlp.experts.*.gate_proj|up_proj|down_proj``),
+ but trigger for non-standard expert paths and for any routed
+ expert structure that lives outside the standard ``mlp.experts`` naming.
+ """
+ if "RoutedExperts" in targets:
+ return targets
+
+ def _projection_name(name: str) -> str:
+ # Eg. '...experts.0.gate_proj.weight' -> ['...', 'experts', '0', 'gate_proj', 'weight']
+ parts = name.split(".")
+ if len(parts) >= 2:
+ return parts[-2]
+ return parts[-1]
+
+ for name in layer_names:
+ lname = name.lower()
+ if "block_sparse_moe" in lname or ".moe." in lname or ".feed_forward." in lname:
+ return list(targets) + ["RoutedExperts"]
+
+ if ".experts." not in lname:
+ continue
+
+ projection = _projection_name(name)
+ if projection in {"gate_proj", "up_proj", "down_proj"}:
+ continue
+ return list(targets) + ["RoutedExperts"]
+
+ return targets
+
+
+def ignore_fused_experts_for_vllm(qconfig: dict, ignored_layers: list[str], quantized_layers: list[str]) -> dict:
+ """Add vLLM-matchable ignore entries for ignored fused expert modules when ``RoutedExperts`` is targeted.
+
+ vLLM resolves a FusedMoE scheme via ``<module>.0.{gate,up,down}_proj``, which fused layer names such as
+ ``<module>.w13_weight`` never match, so the global ``RoutedExperts`` target would quantize them.
+ """
+ groups = qconfig.get("config_groups") or {}
+ if not any("RoutedExperts" in (g.get("targets") or []) for g in groups.values()):
+ return qconfig
+
+ ignore = list(qconfig.get("ignore") or [])
+ for layer_name in dict.fromkeys(ignored_layers):
+ if layer_name.startswith("re:"):
+ continue
+ parts = layer_name.split(".")
+ if parts[-1] in _EXPERT_SEGMENTS:
+ module = layer_name
+ elif len(parts) >= 2 and parts[-2] in _EXPERT_SEGMENTS and not parts[-1].isdigit():
+ module = ".".join(parts[:-1])
+ else:
+ continue
+ # A partially quantized module cannot be described as ignored.
+ if any(q == module or q.startswith(f"{module}.") for q in quantized_layers):
+ continue
+ entry = f"re:^{re.escape(module)}\\.\\d+\\.(gate|up|down)_proj$"
+ if entry not in ignore:
+ ignore.append(entry)
+ qconfig["ignore"] = ignore
+ return qconfig
+
+
+def get_checkpoint_name_reverter(model: torch.nn.Module):
+ """Return ``fn(module_name) -> name`` mapping a loaded-model module name to the saved checkpoint namespace."""
+ from auto_round.utils.common import (
+ get_reverse_checkpoint_conversion_mapping,
+ get_reverse_weight_transforms,
+ revert_checkpoint_conversion_mapping,
+ revert_name_with_weight_transforms,
+ )
+
+ transforms = get_reverse_weight_transforms(model)
+ mapping = get_reverse_checkpoint_conversion_mapping(model) if transforms is None else None
+ if transforms is None and not mapping:
+ return lambda name: name
+
+ def _revert(name: str) -> str:
+ # Renamings are defined on parameter keys, so revert a full weight key.
+ key = f"{name}.weight"
+ if transforms is not None:
+ reverted = revert_name_with_weight_transforms(key, transforms)
+ else:
+ reverted = revert_checkpoint_conversion_mapping(key, mapping)
+ return reverted[: -len(".weight")] if reverted.endswith(".weight") else name
+
+ return _revert
+
+
+def revert_quantization_config_names(qconfig: dict, revert) -> dict:
+ """Rewrite module names in ``ignore`` and group ``targets`` with ``revert`` so they match the saved weights."""
+
+ def _revert_entry(entry):
+ if not isinstance(entry, str):
+ return entry
+ if not entry.startswith("re:"):
+ return revert(entry)
+ # Only rewrite regexes that wrap a single escaped module name, e.g. ``re:.*model\.layers\.0\.mlp\.gate.*``.
+ match = re.fullmatch(r"(re:(?:\.\*)?)(.*?)((?:\.\*)?)", entry)
+ plain = re.sub(r"\\(.)", r"\1", match.group(2))
+ if not plain or re.escape(plain) != match.group(2):
+ return entry
+ return f"{match.group(1)}{re.escape(revert(plain))}{match.group(3)}"
+
+ if qconfig.get("ignore"):
+ qconfig["ignore"] = list(dict.fromkeys(_revert_entry(e) for e in qconfig["ignore"]))
+ for group in (qconfig.get("config_groups") or {}).values():
+ if group.get("targets"):
+ group["targets"] = [_revert_entry(t) for t in group["targets"]]
+ return qconfig
+
+
+def finalize_llmcompressor_config_names(
+ qconfig: dict, model: torch.nn.Module, ignored_layers: list[str], quantized_layers: list[str]
+) -> dict:
+ """Revert config module names to the checkpoint namespace, then add vLLM ignore entries for fused experts."""
+ revert = get_checkpoint_name_reverter(model)
+ qconfig = revert_quantization_config_names(qconfig, revert)
+ ignored = list(qconfig.get("ignore") or []) + [
+ revert(n) for n in ignored_layers + collect_fused_expert_modules(model) if not n.startswith("re:")
+ ]
+ return ignore_fused_experts_for_vllm(qconfig, ignored, [revert(n) for n in quantized_layers])
+
+
+def collect_fused_expert_modules(model: torch.nn.Module) -> list[str]:
+ """Return expert modules still holding fused 3-D weights, i.e. never unfused into quantized Linear layers."""
+ return [
+ name
+ for name, module in model.named_modules()
+ if name.split(".")[-1] in _EXPERT_SEGMENTS and any(p.dim() == 3 for p in module.parameters(recurse=False))
+ ]
+
def generate_ignore_regex_list(regex_config: dict[str, dict], layer_config: dict[str, dict]) -> list[str]:
"""
diff --git a/auto_round/utils/model_free_utils.py b/auto_round/utils/model_free_utils.py
index 69731a074..f388c4ec9 100644
--- a/auto_round/utils/model_free_utils.py
+++ b/auto_round/utils/model_free_utils.py
@@ -34,6 +34,10 @@ from typing import Any
import torch
from auto_round.compressors.utils import is_mx_fp, is_nv_fp
+from auto_round.export.export_to_llmcompressor.utils import add_routed_experts_if_moe as _add_routed_experts_if_moe
+from auto_round.export.export_to_llmcompressor.utils import (
+ ignore_fused_experts_for_vllm as _ignore_fused_experts_for_vllm,
+)
from auto_round.logger import logger
from auto_round.schemes import PRESET_SCHEMES, QuantizationScheme, preset_name_to_scheme
from auto_round.utils.common import _normalize_tensor_name_for_warning, to_standard_regex
@@ -45,6 +49,8 @@ from auto_round.utils.path_safety import (
)
_NVFP4_E5M3_DATA_TYPE = "nvfp4_v2"
+# default_scheme flag restricting quantization to layers stored as NVFP4 in the source checkpoint.
+_NVFP_SOURCE_ONLY_KEY = "_nvfp_source_only"
_BLOCK_NAME_TO_IGNORE = ("shared_expert_gate.", ".gate.", "embed", "conv")
_SUPPORTED_MXFP_BITS = (4, 8)
_SUPPORTED_INT_BITS = (2, 4, 8)
@@ -459,6 +465,7 @@ class _PatternMatcher:
"""Precompile ignore and layer-config patterns for shard processing."""
__slots__ = (
+ "_allowed_layers",
"_compiled_lc",
"_default_scheme",
"_ignore_cache",
@@ -488,6 +495,16 @@ class _PatternMatcher:
self._ignore_cache: dict[str, bool] = {}
self._scheme_cache: dict[str, dict | None] = {}
+ self._allowed_layers: set[str] | None = None
+
+ @property
+ def nvfp_source_only(self) -> bool:
+ return bool(self._default_scheme.get(_NVFP_SOURCE_ONLY_KEY))
+
+ def restrict_to_layers(self, layers: set[str]) -> None:
+ """Treat every layer outside *layers* as ignored (unless explicitly listed in layer_config)."""
+ self._allowed_layers = set(layers)
+ self._ignore_cache.clear()
@staticmethod
def _build_ignore_regex(patterns: list[str]) -> re.Pattern | None:
@@ -511,10 +528,23 @@ class _PatternMatcher:
if layer_name in self._layer_config:
self._ignore_cache[tensor_name] = False
return False
+ if (
+ self._allowed_layers is not None
+ and layer_name not in self._allowed_layers
+ and not self._matches_layer_config(layer_name)
+ ):
+ self._ignore_cache[tensor_name] = True
+ return True
result = bool(self._ignore_re and self._ignore_re.search(layer_name))
self._ignore_cache[tensor_name] = result
return result
+ def _matches_layer_config(self, layer_name: str) -> bool:
+ return any(
+ (compiled is not None and compiled.search(layer_name)) or (plain is not None and plain in layer_name)
+ for compiled, plain, _ in self._compiled_lc
+ )
+
def should_skip(self, tensor_name: str) -> bool:
return bool(self._skip_re.search(tensor_name))
@@ -569,6 +599,34 @@ def _collect_mxfp_source_entries(raw_tensors: dict[str, torch.Tensor]) -> list[t
return entries
+def _collect_nvfp4_source_layers(raw_tensors: dict[str, torch.Tensor]) -> set[str]:
+ """Return names of layers stored as NVFP4 in a shard.
+
+ Expects llm-compressor naming (run after :func:`_normalize_nvfp4_source_tensors`):
+ ``<layer>.weight_packed`` + ``<layer>.weight_scale`` with a global scale or an FP8
+ block scale (MXFP4 uses uint8 E8M0 scales without a global scale). Inkling-style bare
+ tensors ``X`` with ``X.scale`` / ``X.scale2`` siblings are detected as layer ``X``.
+ """
+ layers: set[str] = set()
+ for name, tensor in raw_tensors.items():
+ if tensor.dtype not in (torch.uint8, torch.int8):
+ continue
+ if name.endswith(".weight_packed"):
+ layer_name = name[: -len(".weight_packed")]
+ scale = raw_tensors.get(f"{layer_name}.weight_scale")
+ if scale is None:
+ continue
+ if (
+ f"{layer_name}.weight_global_scale" in raw_tensors
+ or f"{layer_name}.input_global_scale" in raw_tensors
+ or scale.dtype == torch.float8_e4m3fn
+ ):
+ layers.add(layer_name)
+ elif f"{name}.scale" in raw_tensors and f"{name}.scale2" in raw_tensors:
+ layers.add(name)
+ return layers
+
+
def _normalize_nvfp4_source_tensors(
raw_tensors: dict[str, torch.Tensor],
shard_name: str | None = None,
@@ -597,8 +655,9 @@ def _normalize_nvfp4_source_tensors(
has_new_global = (
f"{layer_name}.weight_global_scale" in raw_tensors or f"{layer_name}.input_global_scale" in raw_tensors
)
- has_new_packed = f"{layer_name}.weight_packed" in raw_tensors
- if has_legacy_global or has_new_global or has_new_packed:
+ has_legacy_packed = name.endswith(".weight") and f"{layer_name}.weight_packed" not in raw_tensors
+ # Layers already in llm-compressor naming need no normalization.
+ if has_legacy_global or (has_legacy_packed and has_new_global):
candidates.append(layer_name)
if not candidates:
@@ -612,24 +671,25 @@ def _normalize_nvfp4_source_tensors(
weight_global_scale_key = f"{layer_name}.weight_global_scale"
input_global_scale_key = f"{layer_name}.input_global_scale"
+ # Quark NVFP4 names the ModelOpt-style input_scale "input_scale_2".
+ quark_input_scale = raw_tensors.pop(f"{layer_name}.input_scale_2", None)
+ if quark_input_scale is not None and input_scale_key not in raw_tensors:
+ raw_tensors[input_scale_key] = quark_input_scale
+
if weight_packed_key not in raw_tensors and weight_key in raw_tensors:
raw_tensors[weight_packed_key] = raw_tensors.pop(weight_key).view(torch.uint8).contiguous()
+ # ModelOpt scales (amax / (448 * 6)) are the reciprocal of llm-compressor global scales.
if weight_scale_2_key in raw_tensors and weight_global_scale_key not in raw_tensors:
- raw_tensors[weight_global_scale_key] = raw_tensors.pop(weight_scale_2_key)
+ raw_tensors[weight_global_scale_key] = _reciprocal_global_scale(raw_tensors.pop(weight_scale_2_key))
elif weight_scale_2_key in raw_tensors:
raw_tensors.pop(weight_scale_2_key)
if input_scale_key in raw_tensors and input_global_scale_key not in raw_tensors:
- raw_tensors[input_global_scale_key] = raw_tensors.pop(input_scale_key)
+ raw_tensors[input_global_scale_key] = _reciprocal_global_scale(raw_tensors.pop(input_scale_key))
elif input_scale_key in raw_tensors:
raw_tensors.pop(input_scale_key)
- if weight_global_scale_key in raw_tensors:
- raw_tensors[weight_global_scale_key] = _reciprocal_global_scale(raw_tensors[weight_global_scale_key])
- if input_global_scale_key in raw_tensors:
- raw_tensors[input_global_scale_key] = _reciprocal_global_scale(raw_tensors[input_global_scale_key])
-
converted_layers.append(layer_name)
if converted_layers:
@@ -638,10 +698,10 @@ def _normalize_nvfp4_source_tensors(
return raw_tensors, converted_layers
-# Auxiliary tensors emitted next to a ModelOpt / TensorRT-LLM NVFP4 packed
+# Auxiliary tensors emitted next to an Inkling (``inkling_mm_model``) NVFP4 packed
# weight. Once the packed weight has been dequantized back to high precision
# these carry no information and must not leak into the quantized output.
-_MODELOPT_NVFP4_AUX_SUFFIXES: tuple[str, ...] = (
+_INKLING_NVFP4_AUX_SUFFIXES: tuple[str, ...] = (
".scale",
".scale2",
".input_amax",
@@ -650,7 +710,7 @@ _MODELOPT_NVFP4_AUX_SUFFIXES: tuple[str, ...] = (
)
-def _dequant_modelopt_nvfp4_slice(
+def _dequant_inkling_nvfp4_slice(
packed: torch.Tensor,
scale: torch.Tensor,
global_scale: float,
@@ -660,7 +720,7 @@ def _dequant_modelopt_nvfp4_slice(
device: str,
target_dtype: torch.dtype,
) -> torch.Tensor:
- """Dequantize one 2-D ModelOpt NVFP4 slice back to *target_dtype* on CPU."""
+ """Dequantize one 2-D Inkling NVFP4 slice back to *target_dtype* on CPU."""
from auto_round.experimental.qmodules.fp4_utils import unpack_fp4_from_uint8
packed_dev = packed.to(device)
@@ -674,16 +734,18 @@ def _dequant_modelopt_nvfp4_slice(
return result
-def _dequant_modelopt_nvfp4_tensors(
+def _dequant_inkling_nvfp4_tensors(
raw_tensors: dict[str, torch.Tensor],
device: str = "cpu",
shard_name: str | None = None,
- target_dtype: torch.dtype = torch.bfloat16,
+ target_dtype: torch.dtype = torch.float32,
) -> dict[str, torch.Tensor]:
- """Dequantize ModelOpt / TensorRT-LLM NVFP4 source tensors to high precision.
+ """Dequantize Inkling-specific NVFP4 source tensors to high precision.
- ModelOpt checkpoints (``hf_quant_config.json`` with ``quant_algo="NVFP4"``)
- store a quantized weight as a *packed uint8* tensor ``X`` accompanied by:
+ Inkling (``model_type="inkling_mm_model"``) NVFP4 checkpoints use a special layout
+ that differs from the generic ModelOpt ``.weight`` / ``.weight_scale`` /
+ ``.weight_scale_2`` naming (handled by :func:`_normalize_nvfp4_source_tensors`):
+ a quantized weight is stored as a *packed uint8* tensor ``X`` accompanied by:
* ``X.scale`` – per-block (group_size=16) scale, ``float8_e4m3fn``
* ``X.scale2`` – global scale (scalar, or one per expert for fused MoE)
@@ -698,7 +760,7 @@ def _dequant_modelopt_nvfp4_tensors(
Both plain 2-D weights and fused 3-D stacked MoE expert weights
(``[num_experts, out, in]``) are supported. All auxiliary tensors listed in
- :data:`_MODELOPT_NVFP4_AUX_SUFFIXES` are dropped after dequantization.
+ :data:`_INKLING_NVFP4_AUX_SUFFIXES` are dropped after dequantization.
"""
entries: list[str] = []
for name, tensor in raw_tensors.items():
@@ -718,7 +780,7 @@ def _dequant_modelopt_nvfp4_tensors(
shard_prefix = f"[{shard_name}] " if shard_name else ""
logger.info(
- f"{shard_prefix}Dequantizing {len(entries)} ModelOpt NVFP4 source tensor(s) to {target_dtype} on {device}."
+ f"{shard_prefix}Dequantizing {len(entries)} Inkling NVFP4 source tensor(s) to {target_dtype} on {device}."
)
n_dropped = 0
@@ -734,7 +796,7 @@ def _dequant_modelopt_nvfp4_tensors(
try:
if packed.dim() == 2:
- dequantized = _dequant_modelopt_nvfp4_slice(
+ dequantized = _dequant_inkling_nvfp4_slice(
packed,
scale,
float(global_scale[0]),
@@ -752,7 +814,7 @@ def _dequant_modelopt_nvfp4_tensors(
# slices, so peak memory stays at one output tensor + one slice.
dequantized = torch.empty((num_experts, out_features, in_features), dtype=target_dtype)
for i in range(num_experts):
- dequantized[i] = _dequant_modelopt_nvfp4_slice(
+ dequantized[i] = _dequant_inkling_nvfp4_slice(
packed[i],
scale[i],
float(global_scale[i if global_scale.numel() > 1 else 0]),
@@ -764,7 +826,7 @@ def _dequant_modelopt_nvfp4_tensors(
)
except Exception as e:
logger.warning(
- f"{shard_prefix}Failed to dequantize ModelOpt NVFP4 tensor '{name}': {e}. "
+ f"{shard_prefix}Failed to dequantize Inkling NVFP4 tensor '{name}': {e}. "
f"Keeping the packed source tensor unchanged."
)
raw_tensors[name] = packed
@@ -776,7 +838,7 @@ def _dequant_modelopt_nvfp4_tensors(
del packed, scale, scale2, global_scale
# Drop the now-meaningless auxiliary metadata tensors.
- for suffix in _MODELOPT_NVFP4_AUX_SUFFIXES:
+ for suffix in _INKLING_NVFP4_AUX_SUFFIXES:
if raw_tensors.pop(f"{name}{suffix}", None) is not None:
n_dropped += 1
@@ -796,7 +858,7 @@ def _handle_nvfp4_source_tensors(
"""Passthrough NVFP4 source tensors when target scheme for the layer is NVFP4.
Layers whose target scheme is *not* NVFP4 (e.g. converting an NVFP4 donor
- checkpoint to MXFP4) are dequantized to bfloat16 via
+ checkpoint to MXFP4) are dequantized to float32 via
:func:`_dequant_nvfp4_tensors` instead of being left as raw
``weight_packed``/``weight_scale`` tensors — otherwise those companion
scale tensors would fall through to the generic RTN/MXFP quantization path
@@ -806,6 +868,8 @@ def _handle_nvfp4_source_tensors(
passthrough_tensors: dict[str, torch.Tensor] = {}
passthrough_layers: list[str] = []
dequant_layers: list[str] = []
+ # MXFP4 sources also use weight_packed + weight_scale; never treat them as NVFP4.
+ nvfp4_layers = _collect_nvfp4_source_layers(raw_tensors)
for name, tensor in list(raw_tensors.items()):
if not name.endswith(".weight_packed") or tensor.dtype not in (torch.uint8, torch.int8):
@@ -813,7 +877,7 @@ def _handle_nvfp4_source_tensors(
layer_name = name[: -len(".weight_packed")]
scale_key = f"{layer_name}.weight_scale"
- if scale_key not in raw_tensors:
+ if scale_key not in raw_tensors or layer_name not in nvfp4_layers:
continue
scheme = matcher.resolve_scheme(f"{layer_name}.weight")
@@ -854,7 +918,7 @@ def _dequant_nvfp4_tensors(
device: str = "cpu",
shard_name: str | None = None,
) -> dict[str, torch.Tensor]:
- """Dequantize NVFP4-packed weight tensors to bfloat16.
+ """Dequantize NVFP4-packed weight tensors to float32.
Reverses the NVFP4 packing (see :func:`auto_round.data_type.nvfp.nv_fp4`):
``real_weight = unpack(weight_packed) * weight_scale.float() * (1 / weight_global_scale)``
@@ -873,7 +937,7 @@ def _dequant_nvfp4_tensors(
group_size = 16
dequant_device = str(device or "cpu")
shard_prefix = f"[{shard_name}] " if shard_name else ""
- logger.info(f"{shard_prefix}Dequantizing {len(layer_names)} NVFP4 tensor(s) to bfloat16 on {dequant_device}.")
+ logger.info(f"{shard_prefix}Dequantizing {len(layer_names)} NVFP4 tensor(s) to float32 on {dequant_device}.")
for layer_name in layer_names:
packed_key = f"{layer_name}.weight_packed"
@@ -901,7 +965,7 @@ def _dequant_nvfp4_tensors(
dq = (unpacked.reshape(out_features, n_groups, group_size) * scale_f).reshape(out_features, in_features)
if global_scale is not None:
dq = dq * get_reciprocal(global_scale.to(torch.float32).reshape(()))
- return dq.to(torch.bfloat16)
+ return dq
dq_weight = _dequantize_with_device_fallback(
dequant_device=dequant_device,
@@ -1168,7 +1232,7 @@ def _quantize_weight_mxfp(
f"by MXFP group_size={group_size}; cannot pack."
)
- weight_dev = weight.to(device)
+ weight_dev = weight.to(device=device, dtype=torch.float32)
# Use get_quant_func (same as WrapperLinear) so that all registered MXFP
# variants automatically get opt_rtn support via the QUANT_FUNC_WITH_DTYPE
# registry (e.g. "opt_rtn_mx_fp4" -> quant_mx_opt_rtn, "mx_fp4" -> quant_mx).
@@ -1245,7 +1309,7 @@ def _quantize_weight_nvfp4_e5m3(
f"by NVFP4_E5M3 group_size={group_size}; cannot quantize."
)
- weight_dev = weight.to(device)
+ weight_dev = weight.to(device=device, dtype=torch.float32)
max_scale = 1.0
if not disable_opt_rtn:
grouped = reshape_pad_tensor_by_group_size(weight_dev, group_size)[0]
@@ -1294,7 +1358,7 @@ def _pack_weight_nvfp4_e5m3(
raise ValueError(
f"NVFP4_E5M3 requires in_features divisible by group_size=16, got {in_features} for '{layer_name}'."
)
- weight_dev = weight.to(device)
+ weight_dev = weight.to(device=device, dtype=torch.float32)
max_scale = 1.0
if not disable_opt_rtn:
grouped = reshape_pad_tensor_by_group_size(weight_dev, group_size)[0]
@@ -1307,7 +1371,7 @@ def _pack_weight_nvfp4_e5m3(
# Meta-device construction avoids allocating a full weight tensor on the
# accelerator (and zero-filled packed buffers) that are immediately replaced.
with torch.device("meta"):
- linear = torch.nn.Linear(in_features, out_features, bias=False, dtype=weight.dtype)
+ linear = torch.nn.Linear(in_features, out_features, bias=False, dtype=torch.float32)
linear.weight = torch.nn.Parameter(weight_dev, requires_grad=False)
with torch.device("meta"):
qlayer = QuantLinear(
@@ -1476,7 +1540,7 @@ def _build_cross_shard_pairs_from_weight_map(
Two source layouts are covered:
* **FP8**: ``<layer>.weight`` and ``<layer>.weight_scale_inv``.
- * **ModelOpt / TensorRT-LLM NVFP4**: a packed base tensor ``X`` and its
+ * **Inkling NVFP4** (``inkling_mm_model``): a packed base tensor ``X`` and its
``X.scale`` / ``X.scale2`` siblings.
A *cross-shard pair* exists when the two tensors reside in **different**
@@ -1502,7 +1566,7 @@ def _build_cross_shard_pairs_from_weight_map(
recipient_to_donors.setdefault(weight_shard, {}).setdefault(donor_shard, []).append(scale_name)
donor_shard_tensors.setdefault(donor_shard, set()).add(scale_name)
- nvfp4_bases = _modelopt_nvfp4_bases_from_weight_map(weight_map)
+ nvfp4_bases = _inkling_nvfp4_bases_from_weight_map(weight_map)
for tensor_name, shard in weight_map.items():
if tensor_name.endswith(".weight_scale_inv"):
@@ -1644,11 +1708,11 @@ def _find_index_path(idx_dir: str) -> str | None:
return candidates[0] if candidates else None
-def _modelopt_nvfp4_bases_from_weight_map(weight_map: dict[str, str]) -> set[str]:
- """Return base tensor names that use the ModelOpt / TensorRT-LLM NVFP4 layout.
+def _inkling_nvfp4_bases_from_weight_map(weight_map: dict[str, str]) -> set[str]:
+ """Return base tensor names that use the Inkling NVFP4 layout.
A base ``X`` qualifies when the checkpoint contains ``X``, ``X.scale`` and
- ``X.scale2`` (see :func:`_dequant_modelopt_nvfp4_tensors`).
+ ``X.scale2`` (see :func:`_dequant_inkling_nvfp4_tensors`).
"""
return {
name[: -len(".scale2")]
@@ -1705,12 +1769,12 @@ def dequant_mx_fp8(
) -> torch.Tensor:
"""Dequantize MXFP8 data using its per-block E8M0 scales."""
original_shape = weight_fp8.shape
- weight = weight_fp8.to(torch.bfloat16).reshape(-1, block_size)
+ weight = weight_fp8.to(torch.float32).reshape(-1, block_size)
scale = get_fp_scale(scale_e8m0).reshape(-1, 1)
return (weight * scale).reshape(original_shape).to(target_dtype)
-def _hydrate_and_clean_modelopt_nvfp4_aux(
+def _hydrate_and_clean_inkling_nvfp4_aux(
raw_tensors: dict[str, torch.Tensor],
shard_path: str,
*,
@@ -1718,12 +1782,12 @@ def _hydrate_and_clean_modelopt_nvfp4_aux(
index_dir: str | None = None,
donor_shard_dir: str | None = None,
) -> dict[str, torch.Tensor]:
- """Resolve ModelOpt NVFP4 ``.scale`` / ``.scale2`` tensors across shards.
+ """Resolve Inkling NVFP4 ``.scale`` / ``.scale2`` tensors across shards.
- ModelOpt checkpoints do **not** keep a packed weight and its scales in the
+ Inkling checkpoints do **not** keep a packed weight and its scales in the
same shard — e.g. ``layers.36...w13_weight`` can sit in shard 6 while its
``.scale`` is in shard 3 and its ``.scale2`` in shard 5. Since model-free
- processing is shard-local, :func:`_dequant_modelopt_nvfp4_tensors` would
+ processing is shard-local, :func:`_dequant_inkling_nvfp4_tensors` would
otherwise never see a complete triple and the packed weight would be left
untouched (and reported as an already-packed, non-quantizable tensor).
@@ -1753,7 +1817,7 @@ def _hydrate_and_clean_modelopt_nvfp4_aux(
except Exception:
return raw_tensors
- bases = _modelopt_nvfp4_bases_from_weight_map(weight_map)
+ bases = _inkling_nvfp4_bases_from_weight_map(weight_map)
if not bases:
return raw_tensors
@@ -1807,7 +1871,7 @@ def _hydrate_and_clean_modelopt_nvfp4_aux(
# --- 2) drop orphan auxiliary tensors owned by another shard's base ---
dropped = 0
for name in list(raw_tensors.keys()):
- for suffix in _MODELOPT_NVFP4_AUX_SUFFIXES:
+ for suffix in _INKLING_NVFP4_AUX_SUFFIXES:
if not name.endswith(suffix):
continue
base = name[: -len(suffix)]
@@ -1827,7 +1891,7 @@ def _dequant_mxfp_tensors(
device: str = "cpu",
shard_name: str | None = None,
) -> dict[str, torch.Tensor]:
- """Dequantize llm-compressor MXFP8 / MXFP4 weight tensors to bfloat16.
+ """Dequantize llm-compressor MXFP8 / MXFP4 weight tensors to float32.
Detection is purely by *name* and *dtype*:
@@ -1851,7 +1915,7 @@ def _dequant_mxfp_tensors(
dequant_device = str(device or "cpu")
shard_prefix = f"[{shard_name}] " if shard_name else ""
logger.info(
- f"{shard_prefix}Dequantizing MXFP tensor(s) to bfloat16 on {dequant_device}: "
+ f"{shard_prefix}Dequantizing MXFP tensor(s) to float32 on {dequant_device}: "
f"MXFP8={n_mxfp8}, MXFP4={n_mxfp4}, total={len(entries)}."
)
@@ -1868,13 +1932,13 @@ def _dequant_mxfp_tensors(
weight_fp8=weight.to(dequant_device, non_blocking=True),
scale_e8m0=scale.to(dequant_device, non_blocking=True),
block_size=32,
- target_dtype=torch.bfloat16,
+ target_dtype=torch.float32,
).to("cpu"),
on_cpu=lambda weight=weight, scale=scale: dequant_mx_fp8(
weight_fp8=weight,
scale_e8m0=scale,
block_size=32,
- target_dtype=torch.bfloat16,
+ target_dtype=torch.float32,
),
)
else:
@@ -1888,14 +1952,14 @@ def _dequant_mxfp_tensors(
scale_e8m0=scale.to(dequant_device, non_blocking=True),
elem_dtype="fp4_e2m1",
block_size=32,
- target_dtype=torch.bfloat16,
+ target_dtype=torch.float32,
).to("cpu"),
on_cpu=lambda weight=weight, scale=scale: dequant_mxfp4(
data_lp=weight.view(torch.uint8).contiguous(),
scale_e8m0=scale,
elem_dtype="fp4_e2m1",
block_size=32,
- target_dtype=torch.bfloat16,
+ target_dtype=torch.float32,
),
)
raw_tensors[f"{layer_name}.weight"] = dq_weight
@@ -1926,13 +1990,17 @@ def _handle_mxfp_source_tensors(
* If the target is the **same MXFP format** (``data_type='mx_fp'``, matching
``bits``), the tensors are emitted directly as a passthrough — no
dequantization is performed and the layer is recorded as already quantized.
- * Otherwise the tensors are dequantized to ``bfloat16`` via
+ * Otherwise the tensors are dequantized to ``float32`` via
:func:`_dequant_mxfp_tensors` so the downstream RTN path can re-quantize
them to the requested target scheme.
Returns:
``(raw_tensors, passthrough_tensors, passthrough_layers)``.
"""
+ # NVFP4 source -> MXFP4 keeps source MXFP layers untouched; they flow through as ignored tensors.
+ if getattr(matcher, "nvfp_source_only", False):
+ return raw_tensors, {}, []
+
entries = _collect_mxfp_source_entries(raw_tensors)
if not entries:
return raw_tensors, {}, []
@@ -1963,7 +2031,7 @@ def _handle_mxfp_source_tensors(
if passthrough_layers:
parts.append(f"{len(passthrough_layers)} passthrough")
if n_dequant:
- parts.append(f"{n_dequant} dequantized to bfloat16")
+ parts.append(f"{n_dequant} dequantized to float32")
if source_state:
parts.append(f"{len(source_state)} model_type-normalized")
logger.info(f"Handling MXFP source tensor(s): {', '.join(parts)}.")
@@ -1980,12 +2048,16 @@ def _dequant_fp8_tensors(
*,
index_dir: str | None = None,
donor_shard_dir: str | None = None,
+ hydrate_only: bool = False,
) -> dict[str, torch.Tensor]:
- """Dequantize DeepSeek-V3-style FP8 weight tensors to bfloat16.
+ """Dequantize DeepSeek-V3-style FP8 weight tensors to float32.
+
+ With ``hydrate_only`` the cross-shard ``weight_scale_inv`` tensors are hydrated but the
+ FP8 weights are kept as-is (NVFP4 source -> MXFP4).
Handles the **DeepSeek-V3 FP8** convention: weight dtype ``float8_e4m3fn``
paired with a ``.weight_scale_inv`` tensor (per-block float32 scales, NOT
- E8M0). The weights are converted to ``bfloat16`` so downstream RTN
+ E8M0). The weights are converted to ``float32`` so downstream RTN
quantization can proceed normally.
MXFP sources are handled separately by
@@ -2001,6 +2073,8 @@ def _dequant_fp8_tensors(
index_dir=index_dir,
donor_shard_dir=donor_shard_dir,
)
+ if hydrate_only:
+ return raw_tensors
quant_entries: list[tuple[str, str]] = []
for name, tensor in raw_tensors.items():
@@ -2020,9 +2094,7 @@ def _dequant_fp8_tensors(
dequant_device = str(device or "cpu")
shard_prefix = f"[{shard_name}] " if shard_name else ""
- logger.info(
- f"{shard_prefix}Dequantizing {len(quant_entries)} FP8 weight tensor(s) to bfloat16 on {dequant_device}."
- )
+ logger.info(f"{shard_prefix}Dequantizing {len(quant_entries)} FP8 weight tensor(s) to float32 on {dequant_device}.")
for weight_name, scale_name in quant_entries:
weight = raw_tensors[weight_name]
@@ -2039,8 +2111,11 @@ def _dequant_fp8_tensors(
weight.to(dequant_device, non_blocking=True),
scale.to(dequant_device, non_blocking=True),
block_size=block_size,
+ output_dtype=torch.float32,
).to("cpu"),
- on_cpu=lambda weight=weight, scale=scale: _dequant_fp8_linear_weight(weight, scale, block_size=block_size),
+ on_cpu=lambda weight=weight, scale=scale: _dequant_fp8_linear_weight(
+ weight, scale, block_size=block_size, output_dtype=torch.float32
+ ),
)
# ``weight`` still aliases the original FP8 storage after the dict slot
# was rebound; release it (and the scale) before the next iteration.
@@ -2114,6 +2189,8 @@ def _process_shard(
raw_tensors = {name: f.get_tensor(name) for name in f.keys()}
raw_tensors = split_fused_expert_tensors(raw_tensors, model_type=model_type)
+ # Dequantized sources are FP32 in-flight; those left high-precision are saved as BF16.
+ source_fp32_names = {name for name, t in raw_tensors.items() if t.dtype == torch.float32}
# Hydrate cross-shard FP8 weight_scale_inv tensors *before* any
# preprocessing below. Otherwise a weight whose scale lives in a sibling
@@ -2141,10 +2218,11 @@ def _process_shard(
(
_fused_expert_layer_name(name)
if _is_moe_fused_expert_weight(name, tensor)
- else (name.rsplit(".", 1)[0] if name.endswith(".weight") else name)
+ else (name.rsplit(".", 1)[0] if name.endswith((".weight", ".weight_packed")) else name)
)
for name, tensor in raw_tensors.items()
- if (name.endswith(".weight") and tensor.dim() > 1) or _is_moe_fused_expert_weight(name, tensor)
+ if (name.endswith((".weight", ".weight_packed")) and tensor.dim() > 1)
+ or _is_moe_fused_expert_weight(name, tensor)
)
)
@@ -2175,18 +2253,31 @@ def _process_shard(
# 1.5) normalize legacy NVFP4 names to llm-compressor naming.
raw_tensors, _converted_nvfp4_layers = _normalize_nvfp4_source_tensors(raw_tensors, shard_name=shard_name)
- # 1.6) ModelOpt / TensorRT-LLM NVFP4 sources store the packed weight under a
+ # 1.6) Inkling (inkling_mm_model) NVFP4 sources store the packed weight under a
# bare name (no ``.weight`` suffix) with ``.scale`` / ``.scale2`` siblings,
# which are frequently placed in *different* shards. Hydrate them from the
# index first, then dequantize and drop the stale metadata tensors.
- raw_tensors = _hydrate_and_clean_modelopt_nvfp4_aux(
+ raw_tensors = _hydrate_and_clean_inkling_nvfp4_aux(
raw_tensors,
shard_path,
shard_name=shard_name,
index_dir=index_dir,
donor_shard_dir=donor_shard_dir,
)
- raw_tensors = _dequant_modelopt_nvfp4_tensors(raw_tensors, device=device, shard_name=shard_name)
+ # Hydrated Inkling scales are source tensors; keep their dtype if they survive (failed dequant).
+ source_fp32_names.update(
+ n for n, t in raw_tensors.items() if t.dtype == torch.float32 and n.endswith((".scale", ".scale2"))
+ )
+
+ # NVFP4 source -> MXFP4: only NVFP4 source layers (plus layer_config matches) are quantized;
+ # all other layers are treated as ignored. Must run before NVFP4 dequantization.
+ if matcher.nvfp_source_only:
+ nvfp_layers = _collect_nvfp4_source_layers(raw_tensors)
+ matcher.restrict_to_layers(nvfp_layers)
+ shard_prefix = f"[{shard_name}] " if shard_name else ""
+ logger.info(f"{shard_prefix}Detected {len(nvfp_layers)} NVFP4 source layer(s) to convert.")
+
+ raw_tensors = _dequant_inkling_nvfp4_tensors(raw_tensors, device=device, shard_name=shard_name)
# 1.5) model-type-specific low-precision dequantization (e.g. kimi_k25 INT4)
raw_tensors = handle_model_type_low_precision_source_tensors(
@@ -2210,7 +2301,7 @@ def _process_shard(
quantized_layers.extend(passthrough_layers)
# 3) NVFP4 passthrough for layers already stored in packed format (or
- # dequant to bfloat16 when the target scheme requires re-quantization).
+ # dequant to float32 when the target scheme requires re-quantization).
raw_tensors, nvfp_passthrough_tensors, nvfp_passthrough_layers = _handle_nvfp4_source_tensors(
raw_tensors,
matcher,
@@ -2228,6 +2319,7 @@ def _process_shard(
shard_path=shard_path,
index_dir=index_dir,
donor_shard_dir=donor_shard_dir,
+ hydrate_only=matcher.nvfp_source_only,
)
raw_tensors.update(preserved_tensors)
# ``raw_tensors`` now owns these; drop the duplicate references.
@@ -2243,6 +2335,15 @@ def _process_shard(
quantize_func,
disable_opt_rtn,
)
+ # Cross-shard hydrated FP8 scale_inv tensors are FP32 sources, not dequantized weights.
+ if (
+ tensor.dtype == torch.float32
+ and tensor_name not in source_fp32_names
+ and not tensor_name.endswith(".weight_scale_inv")
+ ):
+ kept = out_dict.get(tensor_name)
+ if kept is not None and kept.dtype == torch.float32:
+ out_dict[tensor_name] = kept.to(torch.bfloat16)
output_tensors.update(out_dict)
if q_layer:
quantized_layers.append(q_layer)
@@ -2598,40 +2699,29 @@ def _build_nvfp4_e5m3_quantization_config(ignored_layers: list[str]) -> dict:
return qconfig
-# TODO: Remove when the issue is fixed
-# https://github.com/vllm-project/llm-compressor/issues/3069
-def _add_routed_experts_if_moe(targets: list[str], layer_names: list[str]) -> list[str]:
- """Append ``"RoutedExperts"`` for routed expert layouts that llm-compressor misses.
+def _is_mxfp_ct_group(group: dict) -> bool:
+ weights = group.get("weights") or {}
+ return (
+ weights.get("type") == "float"
+ and weights.get("num_bits") in _SUPPORTED_MXFP_BITS
+ and (weights.get("group_size") == 32)
+ )
- The generic ``"Linear"`` target does not cover all expert layouts. Keep the
- safe canonical cases alone (``mlp.experts.*.gate_proj|up_proj|down_proj``),
- but trigger for non-standard expert paths and for any routed
- expert structure that lives outside the standard ``mlp.experts`` naming.
- """
- if "RoutedExperts" in targets:
- return targets
-
- def _projection_name(name: str) -> str:
- # Eg. '...experts.0.gate_proj.weight' -> ['...', 'experts', '0', 'gate_proj', 'weight']
- parts = name.split(".")
- if len(parts) >= 2:
- return parts[-2]
- return parts[-1]
-
- for name in layer_names:
- lname = name.lower()
- if "block_sparse_moe" in lname or ".moe." in lname or ".feed_forward." in lname:
- return list(targets) + ["RoutedExperts"]
-
- if ".experts." not in lname:
- continue
- projection = _projection_name(name)
- if projection in {"gate_proj", "up_proj", "down_proj"}:
- continue
- return list(targets) + ["RoutedExperts"]
+def _is_catch_all_group(group: dict) -> bool:
+ # The default group may be ["Linear", "RoutedExperts"] after add_routed_experts_if_moe.
+ return "Linear" in (group.get("targets") or [])
- return targets
+
+def _explicit_mxfp_targets(qconfig: dict, quantized_layers: list[str]) -> dict:
+ """Replace the ``"Linear"`` catch-all of MXFP groups with the quantized layer names (+ ``RoutedExperts``)."""
+ groups = qconfig.get("config_groups") or {}
+ covered = {t for g in groups.values() if not _is_catch_all_group(g) for t in g.get("targets") or []}
+ layers = [name for name in dict.fromkeys(quantized_layers) if name not in covered]
+ for group in groups.values():
+ if _is_catch_all_group(group) and _is_mxfp_ct_group(group) and layers:
+ group["targets"] = _add_routed_experts_if_moe(list(layers), layers)
+ return qconfig
def _get_mxfp_group_scheme_and_format(
@@ -3110,8 +3200,13 @@ def _build_quantization_config(
ignored_layers: list[str],
block_name_to_quantize: str | None = None,
format: str = "auto_round",
+ source_quant_config: dict | None = None,
) -> dict:
- """Build a quantization_config dict compatible with auto-round format."""
+ """Build a quantization_config dict compatible with auto-round format.
+
+ ``source_quant_config`` (NVFP4 source -> MXFP4 only) is the source checkpoint's quantization config;
+ its NVFP4 groups are rewritten to MXFP4 and every other group (e.g. FP8) is kept as-is.
+ """
# MXFP (mx_fp) supports two output styles depending on *format*:
# - "llm_compressor" → compressed-tensors / llm-compressor style config
# (produced by _build_mxfp_quantization_config).
@@ -3159,7 +3254,7 @@ def _build_quantization_config(
qconfig["config_groups"]["group_0"]["targets"] = targets
qconfig["format"] = "nvfp4-pack-quantized"
qconfig.update(_get_llm_compressor_metadata())
- return qconfig
+ return _ignore_fused_experts_for_vllm(qconfig, ignored_layers, quantized_layers)
if is_mx_fp(data_type) or (is_fp_default and _layer_config_has_mxfp(layer_config)):
if format in ("auto_round", "auto_round:auto_gptq"):
return _build_mxfp_autoround_quantization_config(
@@ -3169,12 +3264,21 @@ def _build_quantization_config(
layer_config=layer_config,
block_name_to_quantize=block_name_to_quantize,
)
- return _build_mxfp_quantization_config(
+ qconfig = _build_mxfp_quantization_config(
default_scheme=default_scheme,
quantized_layers=quantized_layers,
ignored_layers=ignored_layers,
layer_config=layer_config,
)
+ source_ct = _source_config_to_ct(source_quant_config)
+ if source_ct:
+ qconfig = _rewrite_nvfp4_groups_as_mxfp(qconfig, source_ct, ignored_layers, quantized_layers)
+ # Only a compressed-tensors source vouches for its "Linear" catch-all; otherwise list the layers for vLLM.
+ if source_quant_config is not None and str(source_quant_config.get("quant_method") or "").lower() != (
+ "compressed-tensors"
+ ):
+ qconfig = _explicit_mxfp_targets(qconfig, quantized_layers)
+ return _ignore_fused_experts_for_vllm(qconfig, ignored_layers, quantized_layers)
# When the default is full-precision (BF16/FP16) but INT layers were
# quantized via layer_config, derive the dominant INT scheme and use it as
@@ -3261,6 +3365,349 @@ def _build_quantization_config(
return qconfig
+# ---------------------------------------------------------------------------
+# NVFP4 source -> MXFP4: rewrite the source quantization config (mixed-precision sources)
+# ---------------------------------------------------------------------------
+# Overall flow (scheme MXFP4 + model-free on a source detected as NVFP4, see _detect_nvfp4_source):
+# 1. Shard processing (_process_shard): layers stored as NVFP4 are detected from tensor names and
+# converted to MXFP4; layers matched by layer_config are quantized as configured; every other
+# layer (BF16, FP8, kv-cache scales, ...) is copied as-is.
+# 2. Config (_build_quantization_config, llm_compressor format only):
+# a. _load_source_quant_config: config.json quantization_config, else hf_quant_config.json & co.
+# b. _source_config_to_ct: compressed-tensors is used as-is; ModelOpt / Quark are converted to
+# compressed-tensors; anything else is warned about and skipped (plain MXFP4 config).
+# c. _rewrite_nvfp4_groups_as_mxfp: NVFP4 groups get the MXFP4 scheme (same targets); all other
+# groups, ignore and kv_cache_scheme are kept, so FP8 layers stay described as FP8.
+
+# Standalone quantization metadata files, read when config.json has no quantization_config.
+_SOURCE_QUANT_CONFIG_FILES: tuple[str, ...] = ("hf_quant_config.json", "quantization_config.json", "quant_config.json")
+_FP8_STATIC_TENSOR_ARGS: dict = {
+ "num_bits": 8,
+ "type": "float",
+ "symmetric": True,
+ "dynamic": False,
+ "strategy": "tensor",
+}
+_FP8_DYNAMIC_TOKEN_ARGS: dict = {
+ "num_bits": 8,
+ "type": "float",
+ "symmetric": True,
+ "dynamic": True,
+ "strategy": "token",
+}
+# Placeholder for NVFP4 groups converted from ModelOpt / Quark; they are replaced by MXFP4 anyway.
+_NVFP4_CT_GROUP: dict = {
+ "format": "nvfp4-pack-quantized",
+ "weights": {"num_bits": 4, "type": "float", "symmetric": True, "strategy": "tensor_group", "group_size": 16},
+}
+
+
+def _load_source_quant_config(source_dir: str | None, quantization_config: dict | None) -> dict:
+ """Return the source quantization config, falling back to standalone metadata files."""
+ if quantization_config:
+ return quantization_config
+ for fname in _SOURCE_QUANT_CONFIG_FILES:
+ path = os.path.join(source_dir, fname) if source_dir else ""
+ if not path or not os.path.isfile(path):
+ continue
+ try:
+ with open(path) as f:
+ data = json.load(f)
+ except (OSError, ValueError) as e:
+ logger.warning(f"Could not read source quantization metadata '{fname}': {e}; skipping it.")
+ continue
+ if not isinstance(data, dict) or not data:
+ continue
+ # ModelOpt hf_quant_config.json nests the settings under "quantization".
+ if isinstance(data.get("quantization"), dict):
+ return {**data["quantization"], "producer": data.get("producer"), "quant_method": "modelopt"}
+ return data
+ return {}
+
+
+def _glob_to_ct_target(pattern: str) -> str:
+ """Convert a ModelOpt / Quark fnmatch module pattern into a compressed-tensors target."""
+ from fnmatch import translate
+
+ return f"re:{translate(pattern)}" if any(c in pattern for c in "*?[") else pattern
+
+
+def _ct_target_matches(target: Any, layer_name: str) -> bool:
+ if not isinstance(target, str):
+ return False
+ if target.startswith("re:"):
+ try:
+ return re.match(target[3:], layer_name) is not None
+ except re.error:
+ return False
+ return target == layer_name
+
+
+def _lookup_ct_group(ct_config: dict, layer_name: str) -> dict | None:
+ """Return the compressed-tensors config group of *layer_name* (name match beats ``Linear`` class match)."""
+ if any(_ct_target_matches(t, layer_name) for t in ct_config.get("ignore") or []):
+ return None
+ class_match = None
+ for group in ct_config["config_groups"].values():
+ for target in group.get("targets") or []:
+ if _ct_target_matches(target, layer_name):
+ return group
+ if class_match is None and target == "Linear":
+ class_match = group
+ return class_match
+
+
+def _is_nvfp4_ct_group(group: dict) -> bool:
+ weights = group.get("weights") or {}
+ return weights.get("num_bits") == 4 and weights.get("type") == "float" and weights.get("group_size") == 16
+
+
+def _modelopt_algo_to_ct_group(algo: Any) -> dict | None:
+ algo = str(algo or "").upper()
+ if algo == "NVFP4":
+ return copy.deepcopy(_NVFP4_CT_GROUP)
+ if algo == "FP8":
+ weights, acts = {**_FP8_STATIC_TENSOR_ARGS}, {**_FP8_STATIC_TENSOR_ARGS}
+ elif algo == "FP8_PER_CHANNEL_PER_TOKEN":
+ weights, acts = {**_FP8_STATIC_TENSOR_ARGS, "strategy": "channel"}, {**_FP8_DYNAMIC_TOKEN_ARGS}
+ elif algo == "FP8_PB_WO":
+ weights, acts = {**_FP8_STATIC_TENSOR_ARGS, "strategy": "block", "block_structure": [128, 128]}, None
+ else:
+ return None
+ return {"format": "float-quantized", "weights": weights, "input_activations": acts}
+
+
+def _modelopt_to_ct_config(qc: dict) -> dict | None:
+ """Convert a ModelOpt quantization config (``quant_algo`` / ``quantized_layers``) to compressed-tensors."""
+ quantized_layers = qc.get("quantized_layers")
+ if isinstance(quantized_layers, dict) and quantized_layers:
+ targets_by_algo: dict[str, list[str]] = {}
+ for layer_name, entry in quantized_layers.items():
+ algo = str((entry or {}).get("quant_algo") or "").upper()
+ targets_by_algo.setdefault(algo, []).append(layer_name)
+ else:
+ targets_by_algo = {str(qc.get("quant_algo") or "").upper(): ["Linear"]}
+
+ config_groups: dict[str, dict] = {}
+ for algo, targets in targets_by_algo.items():
+ group = _modelopt_algo_to_ct_group(algo)
+ if group is None:
+ logger.warning(f"ModelOpt quant_algo '{algo}' is not supported; skipping the source quantization config.")
+ return None
+ config_groups[f"group_{len(config_groups)}"] = {**group, "targets": targets}
+
+ ct_config = {
+ "quant_method": "compressed-tensors",
+ "config_groups": config_groups,
+ # hf_quant_config.json uses "exclude_modules"; the config.json copy uses "ignore".
+ "ignore": [_glob_to_ct_target(p) for p in qc.get("exclude_modules") or qc.get("ignore") or []],
+ }
+ kv_scheme = qc.get("kv_cache_scheme")
+ if str(qc.get("kv_cache_quant_algo") or "").upper() == "FP8" or (
+ isinstance(kv_scheme, dict) and kv_scheme.get("num_bits") == 8 and kv_scheme.get("type") == "float"
+ ):
+ ct_config["kv_cache_scheme"] = {**_FP8_STATIC_TENSOR_ARGS}
+ return ct_config
+
+
+def _quark_fp8_args(spec: dict) -> dict | None:
+ dynamic = bool(spec.get("is_dynamic"))
+ qscheme = spec.get("qscheme")
+ if qscheme == "per_tensor":
+ strategy = {"strategy": "tensor"}
+ elif qscheme == "per_channel":
+ strategy = {"strategy": "token" if dynamic else "channel"}
+ elif qscheme == "per_group":
+ strategy = {"strategy": "group", "group_size": spec.get("group_size")}
+ else:
+ return None
+ return {"num_bits": 8, "type": "float", "symmetric": True, "dynamic": dynamic, **strategy}
+
+
+def _quark_layer_to_ct_group(layer_cfg: Any) -> tuple[bool, dict | None]:
+ """Return ``(supported, group)`` for a Quark layer config; ``group`` is None when unquantized."""
+ weight = layer_cfg.get("weight") if isinstance(layer_cfg, dict) else None
+ if not weight:
+ return True, None
+ if isinstance(weight, list):
+ # Quark NVFP4: [fp4 per_group(16), fp8 per_tensor global scale].
+ is_nvfp4 = len(weight) == 2 and weight[0].get("dtype") == "fp4" and weight[0].get("group_size") == 16
+ return (True, copy.deepcopy(_NVFP4_CT_GROUP)) if is_nvfp4 else (False, None)
+ weights = _quark_fp8_args(weight) if weight.get("dtype") == "fp8_e4m3" else None
+ if weights is None:
+ return False, None
+ group = {"format": "float-quantized", "weights": weights}
+ input_cfg = layer_cfg.get("input_tensors")
+ if input_cfg:
+ acts = (
+ _quark_fp8_args(input_cfg) if isinstance(input_cfg, dict) and input_cfg.get("dtype") == "fp8_e4m3" else None
+ )
+ if acts is None:
+ return False, None
+ group["input_activations"] = acts
+ return True, group
+
+
+def _quark_to_ct_config(qc: dict) -> dict | None:
+ """Convert a Quark quantization config to compressed-tensors."""
+ if qc.get("layer_type_quant_config"):
+ logger.warning("Quark 'layer_type_quant_config' is not supported; skipping the source quantization config.")
+ return None
+ entries = [(_glob_to_ct_target(p), cfg) for p, cfg in (qc.get("layer_quant_config") or {}).items()]
+ entries.append(("Linear", qc.get("global_quant_config")))
+
+ config_groups: dict[str, dict] = {}
+ ignore = [_glob_to_ct_target(p) for p in qc.get("exclude") or []]
+ for target, layer_cfg in entries:
+ supported, group = _quark_layer_to_ct_group(layer_cfg)
+ if not supported:
+ logger.warning(
+ f"Quark quantization config for '{target}' is not supported; skipping the source quantization config."
+ )
+ return None
+ if group is None:
+ if target != "Linear":
+ ignore.append(target)
+ continue
+ config_groups[f"group_{len(config_groups)}"] = {**group, "targets": [target]}
+
+ ct_config = {"quant_method": "compressed-tensors", "config_groups": config_groups, "ignore": ignore}
+ if (qc.get("export") or {}).get("kv_cache_group"):
+ ct_config["kv_cache_scheme"] = {**_FP8_STATIC_TENSOR_ARGS}
+ return ct_config
+
+
+def _source_config_to_ct(qc: dict | None) -> dict | None:
+ """Return the source quantization config in compressed-tensors form, or None if not understood."""
+ if not qc:
+ return None
+ method = str(qc.get("quant_method") or "").lower()
+ producer = qc.get("producer")
+ if method == "compressed-tensors" and isinstance(qc.get("config_groups"), dict):
+ return copy.deepcopy(qc)
+ if method == "modelopt" or (isinstance(producer, dict) and producer.get("name") == "modelopt"):
+ return _modelopt_to_ct_config(qc)
+ if method == "quark":
+ return _quark_to_ct_config(qc)
+ logger.warning(f"Source quantization config (quant_method='{method}') is not understood; skipping it.")
+ return None
+
+
+def _detect_nvfp4_source(model_name_or_path: str, source_quant_config: dict | None) -> str | None:
+ """Return why the source checkpoint looks NVFP4-quantized, or None.
+
+ Relies on the model name and the source quantization config (config.json / hf_quant_config.json & co.)
+ because the first shards may not contain any NVFP4 tensor.
+ """
+ if "nvfp4" in os.path.basename(str(model_name_or_path).rstrip("/\\")).lower():
+ return f"model name '{model_name_or_path}'"
+ qc = source_quant_config or {}
+ # Covers compressed-tensors "nvfp4-pack-quantized" and ModelOpt quant_algo "NVFP4".
+ if "nvfp4" in json.dumps(qc, default=str).lower():
+ return "source quantization config"
+ groups = qc.get("config_groups") if isinstance(qc.get("config_groups"), dict) else {}
+ if any(isinstance(g, dict) and _is_nvfp4_ct_group(g) for g in groups.values()):
+ return "source quantization config"
+ if str(qc.get("quant_method") or "").lower() == "quark":
+ layer_cfgs = [qc.get("global_quant_config"), *(qc.get("layer_quant_config") or {}).values()]
+ if any(_is_nvfp4_ct_group(_quark_layer_to_ct_group(cfg)[1] or {}) for cfg in layer_cfgs):
+ return "source quantization config"
+ return None
+
+
+def _detect_fp8_source(model_name_or_path: str, source_quant_config: dict | None) -> str | None:
+ """Return why the source checkpoint looks FP8-quantized (DeepSeek-V3 style), or None."""
+ qc = source_quant_config or {}
+ if (
+ qc.get("quant_method") == "fp8"
+ or qc.get("quantization_type") == "fp8"
+ or str(qc.get("fmt") or "").startswith("e4m3")
+ ):
+ return "source quantization config"
+ return None
+
+
+# Source dtype name -> detector(model_name_or_path, source_quant_config) returning a reason or None.
+_SOURCE_DTYPE_DETECTORS = {
+ "nvfp4": _detect_nvfp4_source,
+ "fp8": _detect_fp8_source,
+}
+
+
+def detect_model_dtypes(model_name_or_path: str, source_quant_config: dict | None) -> dict[str, str]:
+ """Return ``{source_dtype: reason}`` for every low-precision dtype detected in the source checkpoint."""
+ detected = {}
+ for dtype_name, detector in _SOURCE_DTYPE_DETECTORS.items():
+ reason = detector(model_name_or_path, source_quant_config)
+ if reason is not None:
+ detected[dtype_name] = reason
+ return detected
+
+
+def _rewrite_nvfp4_groups_as_mxfp(
+ mxfp_qconfig: dict, source_ct: dict, ignored_layers: list[str], quantized_layers: list[str] | None = None
+) -> dict:
+ """Replace the NVFP4 groups of *source_ct* with the MXFP4 scheme and keep every other group as-is.
+
+ Quantized layers outside the source NVFP4 groups (selected via layer_config) get extra groups.
+ """
+ mxfp_groups = mxfp_qconfig["config_groups"]
+ # The default (catch-all) group targets "Linear"; layer_config override groups list explicit layers.
+ default_name = next((n for n, g in mxfp_groups.items() if _is_catch_all_group(g)), next(iter(mxfp_groups)))
+ mxfp_format = mxfp_qconfig.get("format")
+
+ def _with_format(name: str) -> dict:
+ group = copy.deepcopy(mxfp_groups[name])
+ group["format"] = group.get("format") or mxfp_format
+ return group
+
+ source_format = source_ct.get("format")
+ config_groups: dict[str, dict] = {}
+ for name, group in source_ct["config_groups"].items():
+ if _is_nvfp4_ct_group(group):
+ group = {**_with_format(default_name), "targets": group.get("targets")}
+ elif not group.get("format") and source_format != "mixed-precision":
+ group = {**group, "format": source_format}
+ config_groups[name] = group
+ if not any(_is_nvfp4_ct_group(g) for g in source_ct["config_groups"].values()):
+ logger.warning("Source quantization config has no NVFP4 group; writing a plain MXFP4 config.")
+ return mxfp_qconfig
+
+ extras: dict[str, list[str]] = {}
+ for layer_name in dict.fromkeys(quantized_layers or []):
+ if _is_nvfp4_ct_group(_lookup_ct_group(source_ct, layer_name) or {}):
+ continue
+ owner = next(
+ (n for n, g in mxfp_groups.items() if n != default_name and layer_name in (g.get("targets") or [])),
+ default_name,
+ )
+ extras.setdefault(owner, []).append(layer_name)
+ for owner, layers in extras.items():
+ name = f"group_{len(config_groups)}"
+ while name in config_groups:
+ name += "_extra"
+ config_groups[name] = {**_with_format(owner), "targets": layers}
+ extra_layers = {layer for layers in extras.values() for layer in layers}
+
+ # NVFP4 source layers that were not converted (e.g. dequantized lm_head) are now unquantized.
+ ignore = [p for p in source_ct.get("ignore") or [] if p not in extra_layers]
+ for layer_name in dict.fromkeys(ignored_layers):
+ group = _lookup_ct_group(source_ct, layer_name)
+ if group is not None and _is_nvfp4_ct_group(group) and layer_name not in ignore:
+ ignore.append(layer_name)
+
+ formats = {group.get("format") for group in config_groups.values()}
+ qconfig = {
+ **mxfp_qconfig,
+ "config_groups": config_groups,
+ "ignore": ignore,
+ "format": formats.pop() if len(formats) == 1 else "mixed-precision",
+ }
+ if source_ct.get("kv_cache_scheme"):
+ qconfig["kv_cache_scheme"] = copy.deepcopy(source_ct["kv_cache_scheme"])
+ return qconfig
+
+
# ---------------------------------------------------------------------------
# Scheme Validation and Override Resolution
# ---------------------------------------------------------------------------
@@ -3679,7 +4126,7 @@ def _dequant_kimi_k25_int4_tensors(
shard_name: str | None = None,
dequantize_with_device_fallback: Callable[..., torch.Tensor] | None = None,
) -> dict[str, torch.Tensor]:
- """Dequantize kimi_k25-style INT4 packed weights to bfloat16 ``.weight``."""
+ """Dequantize kimi_k25-style INT4 packed weights to float32 ``.weight``."""
if dequantize_with_device_fallback is None:
raise ValueError("dequantize_with_device_fallback callback is required")
@@ -3691,7 +4138,7 @@ def _dequant_kimi_k25_int4_tensors(
dequant_device = str(device or "cpu")
shard_prefix = f"[{shard_name}] " if shard_name else ""
logger.info(
- f"{shard_prefix}Dequantizing kimi_k25 INT4 tensor(s) to bfloat16 on {dequant_device}: total={len(entries)}."
+ f"{shard_prefix}Dequantizing kimi_k25 INT4 tensor(s) to float32 on {dequant_device}: total={len(entries)}."
)
for layer_name, packed_key, scale_key, zp_key in entries:
@@ -3754,7 +4201,7 @@ def _dequant_kimi_k25_int4_tensors(
q = q - 8
sc = sc.repeat_interleave(group_size, dim=1)[:, : q.shape[1]]
- return (q.to(torch.float32) * sc).to(torch.bfloat16)
+ return q.to(torch.float32) * sc
dq_weight = dequantize_with_device_fallback(
dequant_device=dequant_device,
diff --git a/auto_round/utils/weight_handler.py b/auto_round/utils/weight_handler.py
index c14aad463..062f58229 100644
--- a/auto_round/utils/weight_handler.py
+++ b/auto_round/utils/weight_handler.py
@@ -483,10 +483,16 @@ def _pad_block_fp8_weight_naive(
@with_thread_limits()
def _dequant_fp8_linear_weight(
- weight: torch.Tensor, weight_scale: torch.Tensor, block_size: list | None = None, data_type: str | None = None
+ weight: torch.Tensor,
+ weight_scale: torch.Tensor,
+ block_size: list | None = None,
+ data_type: str | None = None,
+ output_dtype: torch.dtype = torch.bfloat16,
) -> torch.Tensor:
- """Core dequantization logic for block-wise FP8 weights."""
- dtype = torch.bfloat16
+ """Core dequantization logic for block-wise FP8 weights.
+
+ ``data_type`` is the source quantization data type; ``output_dtype`` is the torch dtype of the result.
+ """
if weight_scale is None:
return weight
@@ -500,7 +506,7 @@ def _dequant_fp8_linear_weight(
weight_scale = weight_scale.view(-1, 1)
elif weight_scale.numel() == weight.shape[-1]:
weight_scale = weight_scale.view(1, -1)
- return weight.to(dtype) * weight_scale.to(dtype)
+ return weight.to(output_dtype) * weight_scale.to(output_dtype)
assert len(block_size) == 2
@@ -514,14 +520,14 @@ def _dequant_fp8_linear_weight(
weight_scale_m, weight_scale_n = weight_scale.shape
weight_scale = weight_scale.view(weight_scale_m, 1, weight_scale_n, 1)
weight = weight.view(weight_scale_m, block_size_m, weight_scale_n, block_size_n)
- dequant_weight = weight.to(dtype) * weight_scale.to(dtype)
+ dequant_weight = weight.to(output_dtype) * weight_scale.to(output_dtype)
dequant_weight = dequant_weight.view(weight_scale_m * block_size_m, weight_scale_n * block_size_n)
keep_first_dim = False
elif weight_shape_len == 3:
fd, weight_scale_m, weight_scale_n = weight_scale.shape
weight_scale = weight_scale.view(fd, weight_scale_m, 1, weight_scale_n, 1)
weight = weight.view(fd, weight_scale_m, block_size_m, weight_scale_n, block_size_n)
- dequant_weight = weight.to(dtype) * weight_scale.to(dtype)
+ dequant_weight = weight.to(output_dtype) * weight_scale.to(output_dtype)
dequant_weight = dequant_weight.view(fd, weight_scale_m * block_size_m, weight_scale_n * block_size_n)
keep_first_dim = True
else:
diff --git a/docs/step_by_step.md b/docs/step_by_step.md
index 86349fda5..db708ac17 100755
--- a/docs/step_by_step.md
+++ b/docs/step_by_step.md
@@ -704,32 +704,21 @@ Model-free mode performs calibration-free WOQ quantization **without loading the
<details>
<summary>Model-free Parallelism Benchmarks (Rounded Minutes)</summary>
-Runtime normalization: all `mm:ss` values are rounded up to the next full minute. For example, `4:20 -> 5`, `15:45 -> 16`, `9:07 -> 10`, `7:29 -> 8`, `4:09 -> 5`.
-
-| Model | Device | Scheme | Parallelism | Peak Memory (G) | Runtime (min, rounded up) |
-|---|---|---|---:|---:|---:|
-| Qwen/Qwen3-Next-80B-A3B-Instruct | A100 | W4A16 | 1 | 2 | N/A |
-| Qwen/Qwen3-Next-80B-A3B-Instruct | A100 | W4A16 | 10 | 8 | 7 |
-| Qwen3-235B-A22B-Instruct-2507 | A100 | W4A16 | 1 | 2 | 17 |
-| Qwen3-235B-A22B-Instruct-2507 | A100 | W4A16 | 10 | 8 | 5 |
-| zai-org/GLM-5.2 | B200 | MXFP4-Mixed | 1 | 2 | 60 |
-| zai-org/GLM-5.2 | B200 | MXFP4-Mixed | 10 | 27 | 16 |
-| zai-org/GLM-5.2 | B200 | W4A16 | 1 | 3 | 30 |
-| zai-org/GLM-5.2 | B200 | W4A16 | 10 | 16 | 10 |
-| zai-org/GLM-5.2 | B200 | W4A16 | 20 | 32 | 8 |
-| MiniMaxAI/MiniMax-M2.7 (FP8) | B200 | W4A16 | 1 | 2 | 18 |
-| MiniMaxAI/MiniMax-M2.7 (FP8) | B200 | W4A16 | 10 | 10 | 5 |
-| deepseek-ai/DeepSeek-V4-Pro (MXFP) | B200 | W4A16 | 1 | 6 | 80 |
-| deepseek-ai/DeepSeek-V4-Pro (MXFP) | B200 | W4A16 | 10 | 50 | 13 |
-
-| Model | Scheme | Comparison | Runtime Change (min) | Speedup | Time Saved | Peak Memory Change |
-|---|---|---|---|---:|---:|---|
-| Qwen3-235B | W4A16 | Parallelism 1 -> 10 | 17 -> 5 | 3.40x | 70.6% | 2G -> 8G |
-| GLM-5.2 | MXFP4-Mixed | Parallelism 1 -> 10 | 60 -> 16 | 3.75x | 73.3% | 2G -> 27G |
-| GLM-5.2 | W4A16 | Parallelism 1 -> 10 | 30 -> 10 | 3.00x | 66.7% | 3G -> 16G |
-| GLM-5.2 | W4A16 | Parallelism 1 -> 20 | 30 -> 8 | 3.75x | 73.3% | 3G -> 32G |
-| MiniMax-M2.7 | W4A16 | Parallelism 1 -> 10 | 18 -> 5 | 3.60x | 72.2% | 2G -> 10G |
-| DeepSeek-V4-Pro | W4A16 | Parallelism 1 -> 10 | 80 -> 13 | 6.15x | 83.8% | 6G -> 50G |
+Model-free mode quantizes multiple weight shards in parallel worker processes. Set the parallelism with the `AR_MODEL_FREE_SHARD_PARALLELISM` environment variable (see the [Environment Variables Guide](./environments.md#ar_model_free_shard_parallelism)):
+- **Default (unset)**: `shard_count // 4`, min 1, max 10. For example, 8 shards use 2 workers and 40+ shards use 10 workers.
+- **Manual**: e.g. `export AR_MODEL_FREE_SHARD_PARALLELISM=20`. The effective value never exceeds the shard count.
+- Higher parallelism is usually faster but increases peak GPU (and host) memory; lower it when resources are tight.
+
+Runtimes are rounded up to the next full minute.
+
+| Model | Device | Scheme | Parallelism | Runtime (min) | Speedup | Peak Memory (G) |
+|---|---|---|---|---|---:|---|
+| Qwen3-Next-80B-A3B-Instruct | A100 | W4A16 | 1 -> 10 | N/A -> 7 | N/A | 2 -> 8 |
+| Qwen3-235B-A22B-Instruct-2507 | A100 | W4A16 | 1 -> 10 | 17 -> 5 | 3.40x | 2 -> 8 |
+| GLM-5.2 | B200 | MXFP4-Mixed | 1 -> 10 | 60 -> 16 | 3.75x | 2 -> 27 |
+| GLM-5.2 | B200 | W4A16 | 1 -> 10 / 20 | 30 -> 10 / 8 | 3.00x / 3.75x | 3 -> 16 / 32 |
+| MiniMax-M2.7 (FP8) | B200 | W4A16 | 1 -> 10 | 18 -> 5 | 3.60x | 2 -> 10 |
+| DeepSeek-V4-Pro (MXFP) | B200 | W4A16 | 1 -> 10 | 80 -> 13 | 6.15x | 6 -> 50 |
Key takeaway: model-free quantization usually gets about `3x-6x` runtime speedup with higher parallelism, while peak memory usage increases significantly.
@@ -766,6 +755,43 @@ You can also pass a custom `QuantizationScheme(bits=N, group_size=G, sym=True/Fa
Schemes that require special packing kernels (`W3A16`, `FPW8A16`, `BF16`, `MXINT4`, `NVFP4`, `FP8_BLOCK`, `FP8_STATIC`, `INT8_W8A8`, `GGUF:*`, ...) are **not** supported in model-free mode and will raise `ValueError`. Use the regular AutoRound flow for those.
+**NVFP4 source checkpoints → MXFP4**
+
+Goal: convert published NVFP4 checkpoints to MXFP4 to serve 4-bit LLMs with high throughput on Intel GPUs, which support MXFP4 but have no native NVFP4 acceleration path.
+
+Supported source tensor layouts:
+
+| Source layout | Tensors | Status |
+| --- | --- | --- |
+| compressed-tensors (NVFP4) | `.weight_packed` + `.weight_scale` + `.weight_global_scale` + `.input_global_scale` | Supported |
+| ModelOpt / Quark (NVFP4) | `.weight` (uint8) + `.weight_scale` (FP8) + `.weight_scale_2` + `.input_scale` | Supported |
+| `inkling_mm_model` type | `X` (uint8) + `X.scale` + `X.scale2` (may live in different shards) | Supported |
+
+Status of commonly requested checkpoints:
+
+| Model | Source format | Status | gsm8k strict-match (MXFP4) | gsm8k flexible-extract (MXFP4) |
+| --- | --- | --- | --- | --- |
+| `RedHatAI/Qwen3.8-27B-NVFP4` | compressed-tensors, NVFP4 + FP8 | Verified | 0.9636 ± 0.0052 | 0.9636 |
+| `nvidia/Qwen3.8-27B-NVFP4` | ModelOpt, NVFP4 + FP8 | Verified | 0.9644 ± 0.0051 | 0.9659 |
+| `RedHatAI/Qwen3.6-35B-A3B-NVFP4` | compressed-tensors, NVFP4 | Verified | 0.9575 ± 0.0056 | 0.9575 |
+| `amd/MiniMax-M2.7-NVFP4` | Quark, NVFP4 | Verified | 0.9568 ± 0.0056 | 0.9591 |
+| `nvidia/MiniMax-M2.7-NVFP4` | ModelOpt, NVFP4 | Verified | 0.9477 ± 0.0061 | 0.9522 |
+| `thinkingmachines/Inkling-Small-NVFP4` | Special format (`inkling_mm_model`), NVFP4 | Verified | 0.9477 ± 0.0061 | 0.9545 |
+
+gsm8k is evaluated with 5-shot on the converted MXFP4 models, with thinking disabled. These results only demonstrate that the converted models work correctly; the numbers are not guaranteed to be accurate.
+
+Running `--scheme MXFP4 --model_free` on an NVFP4 checkpoint converts **only** the NVFP4 layers to MXFP4. The source is detected as NVFP4 from the model name (containing `nvfp4`) or its quantization config (`config.json` / `hf_quant_config.json`), and a warning is printed:
+
+- **NVFP4 layers**: detected automatically in each shard and converted to MXFP4.
+- **Unquantized layers** (BF16, `lm_head`, router, ...): kept as-is and recorded as unquantized in `quantization_config`. To quantize some of them as well, list them in `--layer_config`, e.g. `--layer_config "{lm_head:mxfp4}"`.
+- **FP8 layers** (mixed-precision sources, e.g. FP8 attention + NVFP4 MLP): kept as-is. The source quantization config (compressed-tensors; ModelOpt and Quark configs are first converted to compressed-tensors) is reused with only its NVFP4 groups rewritten to MXFP4, so the output is a `mixed-precision` `quantization_config`.
+- Unrecognized source configs trigger a warning and are skipped.
+- Currently only `--format llm_compressor` is supported.
+
+```bash
+auto_round nvidia/MiniMax-M2.7-NVFP4 --model_free --scheme MXFP4 --format llm_compressor --output_dir ./mxfp4-model
+```
+
#### CLI Usage
```bash
diff --git a/docs/step_by_step_CN.md b/docs/step_by_step_CN.md
index 2d78a0df5..bd13f0a6b 100755
--- a/docs/step_by_step_CN.md
+++ b/docs/step_by_step_CN.md
@@ -682,32 +682,21 @@ ar.quantize_and_save(output_dir, format="auto_round")
<details>
<summary>Model-free 并行量化基准(分钟向上取整)</summary>
-时间归一化规则:所有 `mm:ss` 均向上取整到分钟。例如:`4:20 -> 5`、`15:45 -> 16`、`9:07 -> 10`、`7:29 -> 8`、`4:09 -> 5`。
-
-| 模型 | 设备 | 方案 | 并行度 | 峰值显存 (G) | 耗时(分钟,向上取整) |
-|---|---|---|---:|---:|---:|
-| Qwen/Qwen3-Next-80B-A3B-Instruct | A100 | W4A16 | 1 | 2 | N/A |
-| Qwen/Qwen3-Next-80B-A3B-Instruct | A100 | W4A16 | 10 | 8 | 7 |
-| Qwen3-235B-A22B-Instruct-2507 | A100 | W4A16 | 1 | 2 | 17 |
-| Qwen3-235B-A22B-Instruct-2507 | A100 | W4A16 | 10 | 8 | 5 |
-| zai-org/GLM-5.2 | B200 | MXFP4-Mixed | 1 | 2 | 60 |
-| zai-org/GLM-5.2 | B200 | MXFP4-Mixed | 10 | 27 | 16 |
-| zai-org/GLM-5.2 | B200 | W4A16 | 1 | 3 | 30 |
-| zai-org/GLM-5.2 | B200 | W4A16 | 10 | 16 | 10 |
-| zai-org/GLM-5.2 | B200 | W4A16 | 20 | 32 | 8 |
-| MiniMaxAI/MiniMax-M2.7 (FP8) | B200 | W4A16 | 1 | 2 | 18 |
-| MiniMaxAI/MiniMax-M2.7 (FP8) | B200 | W4A16 | 10 | 10 | 5 |
-| deepseek-ai/DeepSeek-V4-Pro (MXFP) | B200 | W4A16 | 1 | 6 | 80 |
-| deepseek-ai/DeepSeek-V4-Pro (MXFP) | B200 | W4A16 | 10 | 50 | 13 |
-
-| 模型 | 方案 | 对比 | 耗时变化(分钟) | 加速比 | 时间节省 | 峰值显存变化 |
-|---|---|---|---|---:|---:|---|
-| Qwen3-235B | W4A16 | 并行 1 -> 10 | 17 -> 5 | 3.40x | 70.6% | 2G -> 8G |
-| GLM-5.2 | MXFP4-Mixed | 并行 1 -> 10 | 60 -> 16 | 3.75x | 73.3% | 2G -> 27G |
-| GLM-5.2 | W4A16 | 并行 1 -> 10 | 30 -> 10 | 3.00x | 66.7% | 3G -> 16G |
-| GLM-5.2 | W4A16 | 并行 1 -> 20 | 30 -> 8 | 3.75x | 73.3% | 3G -> 32G |
-| MiniMax-M2.7 | W4A16 | 并行 1 -> 10 | 18 -> 5 | 3.60x | 72.2% | 2G -> 10G |
-| DeepSeek-V4-Pro | W4A16 | 并行 1 -> 10 | 80 -> 13 | 6.15x | 83.8% | 6G -> 50G |
+免模型模式会用多进程并行量化多个权重分片(shard),并行度通过环境变量 `AR_MODEL_FREE_SHARD_PARALLELISM` 设置,详见[环境变量指南](./environments_CN.md#ar_model_free_shard_parallelism):
+- **默认(未设置)**:`shard_count // 4`,最小 1、最大 10。例如:8 个 shard 对应 2 个 worker,40 个 shard 及以上对应 10 个 worker。
+- **手动设置**:例如 `export AR_MODEL_FREE_SHARD_PARALLELISM=20`,实际并行度不超过 shard 总数。
+- 并行度越高通常越快,但峰值显存(以及内存)会随之上升,资源受限时请调小。
+
+耗时均向上取整到分钟。
+
+| 模型 | 设备 | 方案 | 并行度 | 耗时(分钟) | 加速比 | 峰值显存 (G) |
+|---|---|---|---|---|---:|---|
+| Qwen3-Next-80B-A3B-Instruct | A100 | W4A16 | 1 -> 10 | N/A -> 7 | N/A | 2 -> 8 |
+| Qwen3-235B-A22B-Instruct-2507 | A100 | W4A16 | 1 -> 10 | 17 -> 5 | 3.40x | 2 -> 8 |
+| GLM-5.2 | B200 | MXFP4-Mixed | 1 -> 10 | 60 -> 16 | 3.75x | 2 -> 27 |
+| GLM-5.2 | B200 | W4A16 | 1 -> 10 / 20 | 30 -> 10 / 8 | 3.00x / 3.75x | 3 -> 16 / 32 |
+| MiniMax-M2.7 (FP8) | B200 | W4A16 | 1 -> 10 | 18 -> 5 | 3.60x | 2 -> 10 |
+| DeepSeek-V4-Pro (MXFP) | B200 | W4A16 | 1 -> 10 | 80 -> 13 | 6.15x | 6 -> 50 |
结论:在 model-free 量化中,提高并行度通常可带来约 `3x-6x` 的耗时加速,但峰值显存会明显上升。
@@ -744,6 +733,43 @@ ar.quantize_and_save(output_dir, format="auto_round")
需要特殊打包内核的 scheme(`W3A16`、`FPW8A16`、`BF16`、`MXINT4`、`NVFP4`、`FP8_BLOCK`、`FP8_STATIC`、`INT8_W8A8`、`GGUF:*` 等)**不被支持**,传入会抛 `ValueError`。这些请使用标准 AutoRound 流程。
+**NVFP4 源模型 → MXFP4**
+
+目标:将已发布的 NVFP4 checkpoint 转换为 MXFP4,以便在支持 MXFP4、但没有原生 NVFP4 加速路径的 Intel GPU 上,高吞吐地部署 4-bit 大语言模型。
+
+支持的源张量布局:
+
+| 源布局 | 张量 | 状态 |
+| --- | --- | --- |
+| compressed-tensors(NVFP4) | `.weight_packed` + `.weight_scale` + `.weight_global_scale` + `.input_global_scale` | 支持 |
+| ModelOpt / Quark(NVFP4) | `.weight`(uint8)+ `.weight_scale`(FP8)+ `.weight_scale_2` + `.input_scale` | 支持 |
+| `inkling_mm_model` 类型 | `X`(uint8)+ `X.scale` + `X.scale2`(可能位于不同分片) | 支持 |
+
+常见 checkpoint 的支持状态:
+
+| 模型 | 源格式 | 状态 | gsm8k strict-match(MXFP4) | gsm8k flexible-extract(MXFP4) |
+| --- | --- | --- | --- | --- |
+| `RedHatAI/Qwen3.8-27B-NVFP4` | compressed-tensors,NVFP4 + FP8 | 已验证 | 0.9636 ± 0.0052 | 0.9636 |
+| `nvidia/Qwen3.8-27B-NVFP4` | ModelOpt,NVFP4 + FP8 | 已验证 | 0.9644 ± 0.0051 | 0.9659 |
+| `RedHatAI/Qwen3.6-35B-A3B-NVFP4` | compressed-tensors,NVFP4 | 已验证 | 0.9575 ± 0.0056 | 0.9575 |
+| `amd/MiniMax-M2.7-NVFP4` | Quark,NVFP4 | 已验证 | 0.9568 ± 0.0056 | 0.9591 |
+| `nvidia/MiniMax-M2.7-NVFP4` | ModelOpt,NVFP4 | 已验证 | 0.9477 ± 0.0061 | 0.9522 |
+| `thinkingmachines/Inkling-Small-NVFP4` | 特殊格式(`inkling_mm_model`),NVFP4 | 已验证 | 0.9477 ± 0.0061 | 0.9545 |
+
+gsm8k 为转换后 MXFP4 模型的 5-shot 评测结果,未开启 thinking。结果仅用于证明模型可正常运行,数据不保证准确。
+
+对 NVFP4 checkpoint 使用 `--scheme MXFP4 --model_free` 时,**仅**将 NVFP4 层转换为 MXFP4。是否为 NVFP4 源模型由模型名(包含 `nvfp4`)或其量化配置(`config.json` / `hf_quant_config.json`)判断,检测到时会打印警告:
+
+- **NVFP4 层**:在每个分片中自动检测,并转换为 MXFP4。
+- **非量化层**(BF16、`lm_head`、router 等):保持原样,并在 `quantization_config` 中记录为未量化。如需同时量化其中部分层,请通过 `--layer_config` 额外指定,例如 `--layer_config "{lm_head:mxfp4}"`。
+- **FP8 层**(混合精度源模型,例如 FP8 attention + NVFP4 MLP):保持原样。直接复用源模型的量化配置(compressed-tensors;ModelOpt 与 Quark 配置会先转换为 compressed-tensors 格式),仅将其中的 NVFP4 分组改写为 MXFP4,输出为 `mixed-precision` 的 `quantization_config`。
+- 无法识别的源配置会给出警告并跳过。
+- 目前仅支持 `--format llm_compressor`。
+
+```bash
+auto_round nvidia/MiniMax-M2.7-NVFP4 --model_free --scheme MXFP4 --format llm_compressor --output_dir ./mxfp4-model
+```
+
#### 命令行用法
```bash
diff --git a/test/unit/test_cpu/export/test_llmc_signature.py b/test/unit/test_cpu/export/test_llmc_signature.py
index 35c6d2abd..a774da8d6 100644
--- a/test/unit/test_cpu/export/test_llmc_signature.py
+++ b/test/unit/test_cpu/export/test_llmc_signature.py
@@ -7,6 +7,11 @@ from auto_round.export.export_to_llmcompressor.export import (
_quant_args_signature,
_rewrite_config_group_targets,
)
+from auto_round.export.export_to_llmcompressor.utils import (
+ collect_fused_expert_modules,
+ ignore_fused_experts_for_vllm,
+ revert_quantization_config_names,
+)
class _FakeQuantizationArgs:
@@ -91,3 +96,67 @@ def test_rewrite_config_group_targets_preserves_uniform_targets():
config = {"config_groups": {"group_0": {"targets": ["Linear"]}}}
assert _rewrite_config_group_targets(torch.nn.Module(), config) == config
+
+
+def test_rewrite_config_group_targets_adds_routed_experts_for_moe():
+ first_args = {"num_bits": 4, "type": "int"}
+ second_args = {"num_bits": 8, "type": "int"}
+ model = torch.nn.Module()
+ model.moe = torch.nn.Module()
+ model.moe.experts = torch.nn.ModuleList([torch.nn.Module()])
+ model.moe.experts[0].w1 = torch.nn.Linear(1, 1)
+ model.moe.experts[0].w1.quantization_scheme = SimpleNamespace(weights=first_args, input_activations=None)
+ model.dense = torch.nn.Linear(1, 1)
+ model.dense.quantization_scheme = SimpleNamespace(weights=second_args, input_activations=None)
+ config = {
+ "config_groups": {
+ "group_0": {"weights": first_args, "input_activations": None, "targets": ["Linear"]},
+ "group_1": {"weights": second_args, "input_activations": None, "targets": ["Linear"]},
+ }
+ }
+
+ rewritten = _rewrite_config_group_targets(model, config)
+
+ assert rewritten["config_groups"]["group_0"]["targets"] == ["moe.experts.0.w1", "RoutedExperts"]
+ assert rewritten["config_groups"]["group_1"]["targets"] == ["dense"]
+
+
+def test_ignored_fused_experts_get_vllm_ignore_entry():
+ model = torch.nn.Module()
+ model.mlp = torch.nn.Module()
+ model.mlp.experts = torch.nn.Module()
+ model.mlp.experts.gate_up_proj = torch.nn.Parameter(torch.zeros(2, 4, 2))
+ model.mlp.shared_experts = torch.nn.Linear(2, 2)
+ fused = collect_fused_expert_modules(model)
+ assert fused == ["mlp.experts"]
+
+ config = {"config_groups": {"group_0": {"targets": ["dense", "RoutedExperts"]}}, "ignore": ["re:.*lm_head"]}
+ config = ignore_fused_experts_for_vllm(config, config["ignore"] + fused, ["dense"])
+
+ assert config["ignore"] == ["re:.*lm_head", r"re:^mlp\.experts\.\d+\.(gate|up|down)_proj$"]
+
+
+def test_revert_quantization_config_names_uses_checkpoint_namespace():
+ def revert(name):
+ return name.replace(".mlp.", ".block_sparse_moe.")
+
+ config = {
+ "config_groups": {"group_0": {"targets": ["Linear", "model.layers.0.mlp.experts.0.down_proj"]}},
+ "ignore": [
+ "model.layers.0.mlp.experts.0.gate_proj",
+ r"re:.*model\.layers\.0\.mlp\.gate.*",
+ r"re:.*mlp\.(gate|up).*",
+ ],
+ }
+
+ config = revert_quantization_config_names(config, revert)
+
+ assert config["config_groups"]["group_0"]["targets"] == [
+ "Linear",
+ "model.layers.0.block_sparse_moe.experts.0.down_proj",
+ ]
+ assert config["ignore"] == [
+ "model.layers.0.block_sparse_moe.experts.0.gate_proj",
+ r"re:.*model\.layers\.0\.block_sparse_moe\.gate.*",
+ r"re:.*mlp\.(gate|up).*",
+ ]
diff --git a/test/unit/test_cpu/quantization/test_model_free.py b/test/unit/test_cpu/quantization/test_model_free.py
index debd20bb3..01a849172 100644
--- a/test/unit/test_cpu/quantization/test_model_free.py
+++ b/test/unit/test_cpu/quantization/test_model_free.py
@@ -20,6 +20,7 @@ Backend-specific model-free behavior is covered by test_model_free_parity.py.
import json
import os
+import re
from unittest.mock import Mock
import pytest
@@ -36,6 +37,7 @@ def test_quantize_and_save_restores_temporary_state_on_failure(monkeypatch):
compressor.output_dir = "original-output"
compressor.format = "auto_round"
compressor.quantized = False
+ compressor.nvfp_source_only = False
def fail_run():
raise RuntimeError("save failed")
@@ -726,6 +728,20 @@ def test_model_free_legacy_nvfp4_is_normalized_and_passthrough(tmp_path):
assert "model.layers.0.self_attn.q_proj" in quantized
assert prefix not in ignored
+
+def test_model_free_llmc_nvfp4_is_not_renormalized(tmp_path):
+ prefix = "model.layers.0.mlp.down_proj"
+ tensors = _nvfp4_source_tensors(prefix)
+ tensors[f"{prefix}.weight_global_scale"] = torch.tensor([4.0], dtype=torch.float32)
+ shard_path = str(tmp_path / "shard.safetensors")
+ save_file(tensors, shard_path)
+
+ layer_config = {prefix: {"bits": 4, "group_size": 16, "sym": True, "data_type": "nv_fp"}}
+ output, _, _ = _process_shard(shard_path, _DEFAULT_SCHEME, layer_config, [])
+
+ # llm-compressor global scales are already in the expected convention.
+ assert torch.allclose(output[f"{prefix}.weight_global_scale"], torch.tensor([4.0], dtype=torch.float32))
+
def test_ignores_and_skips(self, tmp_path):
shard_path = str(tmp_path / "shard.safetensors")
save_file(
@@ -1261,6 +1277,577 @@ class TestModelFreeMXFP:
assert "lm_head.weight_packed" not in keys
+# ===========================================================================
+# NVFP4 source + MXFP4 scheme: convert only the NVFP4 layers of an NVFP4 source checkpoint
+# ===========================================================================
+
+_NV_ATTN = "model.layers.0.self_attn.q_proj"
+_NV_MLP = "model.layers.0.mlp.down_proj"
+_NV_MLP_UP = "model.layers.0.mlp.up_proj"
+
+
+def _nvfp4_source_tensors(prefix, out_features=32, in_features=64):
+ """compressed-tensors NVFP4 layer."""
+ return {
+ f"{prefix}.weight_packed": torch.randint(0, 256, (out_features, in_features // 2), dtype=torch.uint8),
+ f"{prefix}.weight_scale": torch.ones((out_features, in_features // 16)).to(torch.float8_e4m3fn),
+ f"{prefix}.weight_global_scale": torch.tensor([1.0], dtype=torch.float32),
+ f"{prefix}.input_global_scale": torch.tensor([1.0], dtype=torch.float32),
+ }
+
+
+def _modelopt_nvfp4_tensors(prefix, out_features=32, in_features=64, input_scale_key="input_scale"):
+ """ModelOpt NVFP4 layer (Quark uses ``input_scale_2`` instead of ``input_scale``)."""
+ return {
+ f"{prefix}.weight": torch.randint(0, 256, (out_features, in_features // 2), dtype=torch.uint8),
+ f"{prefix}.weight_scale": torch.ones((out_features, in_features // 16)).to(torch.float8_e4m3fn),
+ f"{prefix}.weight_scale_2": torch.tensor(1.0, dtype=torch.float32),
+ f"{prefix}.{input_scale_key}": torch.tensor(1.0, dtype=torch.float32),
+ }
+
+
+def _ct_fp8_tensors(prefix, out_features=64, in_features=64):
+ """compressed-tensors FP8 per-channel layer."""
+ return {
+ f"{prefix}.weight": torch.randn(out_features, in_features).to(torch.float8_e4m3fn),
+ f"{prefix}.weight_scale": torch.rand(out_features, 1, dtype=torch.float32),
+ }
+
+
+def _modelopt_fp8_tensors(prefix, out_features=64, in_features=64):
+ """ModelOpt FP8 per-tensor layer."""
+ return {
+ f"{prefix}.weight": torch.randn(out_features, in_features).to(torch.float8_e4m3fn),
+ f"{prefix}.weight_scale": torch.tensor(0.5, dtype=torch.float32),
+ f"{prefix}.input_scale": torch.tensor(0.25, dtype=torch.float32),
+ }
+
+
+# Trimmed from the config.json / hf_quant_config.json of the public checkpoints listed in the docs.
+# RedHatAI/Qwen3.6-35B-A3B-NVFP4 (compressed-tensors, NVFP4)
+_CT_NVFP4_QC = {
+ "quant_method": "compressed-tensors",
+ "format": "nvfp4-pack-quantized",
+ "quantization_status": "compressed",
+ "config_groups": {
+ "group_0": {
+ "format": "nvfp4-pack-quantized",
+ "targets": ["Linear"],
+ "weights": {
+ "num_bits": 4,
+ "type": "float",
+ "symmetric": True,
+ "strategy": "tensor_group",
+ "group_size": 16,
+ },
+ "input_activations": {
+ "num_bits": 4,
+ "type": "float",
+ "symmetric": True,
+ "strategy": "tensor_group",
+ "group_size": 16,
+ "dynamic": "local",
+ },
+ }
+ },
+ "ignore": ["lm_head", _NV_ATTN],
+ "kv_cache_scheme": None,
+}
+# RedHatAI/Qwen3.8-27B-NVFP4 (compressed-tensors, FP8 attention + NVFP4 MLP)
+_CT_FP8_GROUP = {
+ "format": "float-quantized",
+ "targets": [r"re:.*self_attn\.(q|k|v|o)_proj$"],
+ "weights": {"num_bits": 8, "type": "float", "symmetric": True, "strategy": "channel", "dynamic": False},
+ "input_activations": {"num_bits": 8, "type": "float", "symmetric": True, "strategy": "token", "dynamic": True},
+}
+_CT_KV_SCHEME = {"num_bits": 8, "type": "float", "symmetric": True, "strategy": "tensor", "dynamic": False}
+_CT_MIXED_QC = {
+ "quant_method": "compressed-tensors",
+ "format": "mixed-precision",
+ "config_groups": {
+ "group_0": _CT_FP8_GROUP,
+ "group_1": {
+ **_CT_NVFP4_QC["config_groups"]["group_0"],
+ "targets": [r"re:.*mlp\..*(gate|up|down)_proj$"],
+ },
+ },
+ "ignore": ["lm_head"],
+ "kv_cache_scheme": _CT_KV_SCHEME,
+}
+# nvidia/Qwen3.8-27B-NVFP4 hf_quant_config.json (ModelOpt, FP8 + NVFP4)
+_MODELOPT_MIXED_HF_QC = {
+ "producer": {"name": "modelopt", "version": "0.35.0"},
+ "quantization": {
+ "quant_algo": "MIXED_PRECISION",
+ "kv_cache_quant_algo": None,
+ "quantized_layers": {
+ _NV_ATTN: {"quant_algo": "FP8"},
+ _NV_MLP: {"quant_algo": "NVFP4", "group_size": 16},
+ },
+ "exclude_modules": ["lm_head", "mtp*"],
+ },
+}
+# nvidia/MiniMax-M2.7-NVFP4 config.json (ModelOpt, NVFP4 + FP8 kv cache)
+_MODELOPT_NVFP4_QC = {
+ "quant_method": "modelopt",
+ "quant_algo": "NVFP4",
+ "producer": {"name": "modelopt", "version": "0.37.0"},
+ "kv_cache_scheme": {"num_bits": 8, "type": "float", "dynamic": False},
+ "ignore": ["lm_head", "model.layers.0.self_attn*"],
+}
+# amd/MiniMax-M2.7-NVFP4 (Quark, NVFP4)
+_QUARK_FP4_SPEC = {"dtype": "fp4", "qscheme": "per_group", "group_size": 16, "is_dynamic": False}
+_QUARK_FP8_SCALE_SPEC = {"dtype": "fp8_e4m3", "qscheme": "per_tensor", "is_dynamic": False, "is_scale_quant": True}
+_QUARK_NVFP4_QC = {
+ "quant_method": "quark",
+ "global_quant_config": {
+ "weight": [_QUARK_FP4_SPEC, _QUARK_FP8_SCALE_SPEC],
+ "input_tensors": [{**_QUARK_FP4_SPEC, "is_dynamic": True}, _QUARK_FP8_SCALE_SPEC],
+ },
+ "layer_quant_config": {},
+ "layer_type_quant_config": {},
+ "exclude": [_NV_ATTN, "lm_head"],
+ "export": {"kv_cache_group": [], "pack_method": "reorder"},
+}
+
+
+def _cfg_with_qc(qc):
+ return {**_LLAMA_CFG, "quantization_config": qc}
+
+
+def _make_nvfp4_model_dir(tmp_path, config, tensors, **kwargs):
+ """Like ``_make_model_dir`` but the directory name marks the source as NVFP4."""
+ model_dir = _make_model_dir(tmp_path, config, tensors, **kwargs)
+ nvfp4_dir = os.path.join(os.path.dirname(model_dir), "Model-NVFP4")
+ os.rename(model_dir, nvfp4_dir)
+ return nvfp4_dir
+
+
+def _run_nvfp4_to_mxfp4(model_dir, output_dir, **kwargs):
+ kwargs.setdefault("format", "llm_compressor")
+ _ModelFreeCompressorCore(model_name_or_path=model_dir, output_dir=output_dir, scheme="MXFP4", **kwargs).run()
+ return _read_output_tensors(output_dir), _read_qconfig(output_dir)
+
+
+def _read_output_tensors(output_dir):
+ tensors = {}
+ for fname in os.listdir(output_dir):
+ if fname.endswith(".safetensors"):
+ with safe_open(os.path.join(output_dir, fname), framework="pt") as sf:
+ tensors.update({k: sf.get_tensor(k) for k in sf.keys()})
+ return tensors
+
+
+def _assert_mxfp4_layer(tensors, layer):
+ assert tensors[f"{layer}.weight_packed"].dtype == torch.uint8
+ assert tensors[f"{layer}.weight_scale"].dtype == torch.uint8
+ for stale in ("weight", "weight_global_scale", "input_global_scale", "weight_scale_2", "input_scale"):
+ assert f"{layer}.{stale}" not in tensors
+
+
+def _assert_unchanged(tensors, source, keys):
+ for key in keys:
+ assert tensors[key].dtype == source[key].dtype, key
+ assert torch.equal(tensors[key].reshape(-1).view(torch.uint8), source[key].reshape(-1).view(torch.uint8)), key
+
+
+def _assert_mxfp4_group(group, targets):
+ assert group["targets"] == targets
+ assert group["format"] == "mxfp4-pack-quantized"
+ assert group["weights"]["num_bits"] == 4
+ assert group["weights"]["type"] == "float"
+ assert group["weights"]["group_size"] == 32
+
+
+class TestNVFP4ToMXFP4:
+ """MXFP4 on an NVFP4 source: only NVFP4 source layers become MXFP4; everything else is kept byte-for-byte."""
+
+ def test_nvfp2mxfp_scheme_removed(self):
+ assert not is_model_free_supported_scheme("NVFP2MXFP")
+
+ @pytest.mark.parametrize(
+ "name, qc",
+ [
+ ("nvidia/MiniMax-M2.7-NVFP4", None),
+ ("/models/Qwen3-nvfp4/", None),
+ ("source_model", _CT_NVFP4_QC),
+ ("source_model", _CT_MIXED_QC),
+ ("source_model", _MODELOPT_NVFP4_QC),
+ ("source_model", {**_MODELOPT_MIXED_HF_QC["quantization"], "quant_method": "modelopt"}),
+ ("source_model", _QUARK_NVFP4_QC),
+ ],
+ ids=["hub_name", "local_name", "ct", "ct_mixed", "modelopt", "modelopt_mixed", "quark"],
+ )
+ def test_detect_nvfp4_source(self, name, qc):
+ from auto_round.utils.model_free_utils import detect_model_dtypes
+
+ assert "nvfp4" in detect_model_dtypes(name, qc)
+
+ @pytest.mark.parametrize(
+ "name, qc",
+ [
+ ("/data/nvfp4_models/Qwen3-8B", None),
+ ("Qwen/Qwen3-8B", {"quant_method": "fp8", "weight_block_size": [128, 128]}),
+ ("Qwen/Qwen3-8B", {**_CT_MIXED_QC, "config_groups": {"group_0": _CT_FP8_GROUP}}),
+ ],
+ ids=["parent_dir_only", "fp8", "ct_fp8"],
+ )
+ def test_detect_non_nvfp4_source(self, name, qc):
+ from auto_round.utils.model_free_utils import detect_model_dtypes
+
+ assert "nvfp4" not in detect_model_dtypes(name, qc)
+
+ @pytest.mark.parametrize(
+ "qc, expected",
+ [
+ ({"quant_method": "fp8", "weight_block_size": [128, 128]}, True),
+ ({"quantization_type": "fp8"}, True),
+ ({"fmt": "e4m3", "scale_fmt": "ue8m0"}, True),
+ ({"fmt": None}, False),
+ (_CT_NVFP4_QC, False),
+ ],
+ ids=["quant_method", "quantization_type", "fmt", "fmt_none", "ct_nvfp4"],
+ )
+ def test_detect_fp8_source(self, qc, expected):
+ from auto_round.utils.model_free_utils import detect_model_dtypes
+
+ assert ("fp8" in detect_model_dtypes("Qwen/Qwen3-8B", qc)) is expected
+
+ @require_compressed_tensors
+ def test_detection_warns(self, tmp_path, monkeypatch):
+ import auto_round.compressors.model_free as model_free_mod
+
+ warnings = []
+ monkeypatch.setattr(model_free_mod.logger, "warning", lambda msg, *a, **k: warnings.append(str(msg)))
+ model_dir = _make_nvfp4_model_dir(tmp_path, _LLAMA_CFG, _nvfp4_source_tensors(_NV_MLP))
+ _run_nvfp4_to_mxfp4(model_dir, str(tmp_path / "out"))
+ assert any("Detected NVFP4 layers" in w and "layer_config" in w for w in warnings)
+
+ @require_compressed_tensors
+ def test_detection_from_config_without_nvfp4_name(self, tmp_path):
+ source = {**_nvfp4_source_tensors(_NV_MLP), f"{_NV_ATTN}.weight": torch.randn(64, 64, dtype=torch.bfloat16)}
+ model_dir = _make_model_dir(tmp_path, _cfg_with_qc(_CT_NVFP4_QC), source)
+ tensors, _ = _run_nvfp4_to_mxfp4(model_dir, str(tmp_path / "out"))
+
+ _assert_mxfp4_layer(tensors, _NV_MLP)
+ _assert_unchanged(tensors, source, [f"{_NV_ATTN}.weight"])
+
+ @require_compressed_tensors
+ def test_non_nvfp4_source_quantizes_every_layer(self, tmp_path):
+ source = {f"{_NV_ATTN}.weight": torch.randn(64, 64, dtype=torch.bfloat16)}
+ model_dir = _make_model_dir(tmp_path, _LLAMA_CFG, source)
+ tensors, _ = _run_nvfp4_to_mxfp4(model_dir, str(tmp_path / "out"))
+
+ _assert_mxfp4_layer(tensors, _NV_ATTN)
+
+ @require_compressed_tensors
+ def test_layer_config_quantizes_extra_layers(self, tmp_path):
+ source = {
+ **_nvfp4_source_tensors(_NV_MLP),
+ f"{_NV_ATTN}.weight": torch.randn(64, 64, dtype=torch.bfloat16),
+ "lm_head.weight": torch.randn(128, 64, dtype=torch.bfloat16),
+ }
+ model_dir = _make_nvfp4_model_dir(tmp_path, _cfg_with_qc(_CT_NVFP4_QC), source)
+ tensors, qc = _run_nvfp4_to_mxfp4(model_dir, str(tmp_path / "out"), layer_config={"q_proj": "MXFP4"})
+
+ _assert_mxfp4_layer(tensors, _NV_MLP)
+ _assert_mxfp4_layer(tensors, _NV_ATTN)
+ _assert_unchanged(tensors, source, ["lm_head.weight"])
+ _assert_mxfp4_group(qc["config_groups"]["group_0"], ["Linear"])
+ _assert_mxfp4_group(qc["config_groups"]["group_1"], [_NV_ATTN])
+ assert qc["ignore"] == ["lm_head"]
+
+ # ------------------------------------------------------------------
+ # Tensor conversion per source layout
+ # ------------------------------------------------------------------
+
+ @require_compressed_tensors
+ def test_ct_nvfp4_only(self, tmp_path):
+ source = {
+ **_nvfp4_source_tensors(_NV_MLP),
+ f"{_NV_ATTN}.weight": torch.randn(64, 64, dtype=torch.bfloat16),
+ "lm_head.weight": torch.randn(128, 64, dtype=torch.bfloat16),
+ }
+ model_dir = _make_nvfp4_model_dir(tmp_path, _cfg_with_qc(_CT_NVFP4_QC), source)
+ tensors, qc = _run_nvfp4_to_mxfp4(model_dir, str(tmp_path / "out"))
+
+ _assert_mxfp4_layer(tensors, _NV_MLP)
+ _assert_unchanged(tensors, source, [f"{_NV_ATTN}.weight", "lm_head.weight"])
+ assert qc["quant_method"] == "compressed-tensors"
+ assert qc["format"] == "mxfp4-pack-quantized"
+ assert list(qc["config_groups"]) == ["group_0"]
+ _assert_mxfp4_group(qc["config_groups"]["group_0"], ["Linear"])
+ assert qc["ignore"] == ["lm_head", _NV_ATTN]
+
+ @require_compressed_tensors
+ def test_ct_fp8_and_nvfp4_mixed(self, tmp_path):
+ source = {
+ **_nvfp4_source_tensors(_NV_MLP),
+ **_nvfp4_source_tensors(_NV_MLP_UP),
+ **_ct_fp8_tensors(_NV_ATTN),
+ "model.layers.0.self_attn.k_scale": torch.tensor(0.1, dtype=torch.float32),
+ "lm_head.weight": torch.randn(128, 64, dtype=torch.bfloat16),
+ }
+ model_dir = _make_nvfp4_model_dir(tmp_path, _cfg_with_qc(_CT_MIXED_QC), source)
+ tensors, qc = _run_nvfp4_to_mxfp4(model_dir, str(tmp_path / "out"))
+
+ _assert_mxfp4_layer(tensors, _NV_MLP)
+ _assert_mxfp4_layer(tensors, _NV_MLP_UP)
+ _assert_unchanged(
+ tensors,
+ source,
+ [f"{_NV_ATTN}.weight", f"{_NV_ATTN}.weight_scale", "model.layers.0.self_attn.k_scale", "lm_head.weight"],
+ )
+ assert qc["format"] == "mixed-precision"
+ assert qc["config_groups"]["group_0"] == _CT_FP8_GROUP
+ _assert_mxfp4_group(qc["config_groups"]["group_1"], [r"re:.*mlp\..*(gate|up|down)_proj$"])
+ assert qc["ignore"] == ["lm_head"]
+ assert _NV_ATTN not in qc["ignore"]
+ assert qc["kv_cache_scheme"] == _CT_KV_SCHEME
+
+ @require_compressed_tensors
+ def test_modelopt_fp8_and_nvfp4_from_hf_quant_config(self, tmp_path):
+ source = {
+ **_modelopt_nvfp4_tensors(_NV_MLP),
+ **_modelopt_fp8_tensors(_NV_ATTN),
+ "lm_head.weight": torch.randn(128, 64, dtype=torch.bfloat16),
+ }
+ model_dir = _make_nvfp4_model_dir(tmp_path, _LLAMA_CFG, source)
+ with open(os.path.join(model_dir, "hf_quant_config.json"), "w") as f:
+ json.dump(_MODELOPT_MIXED_HF_QC, f)
+ output_dir = str(tmp_path / "out")
+ tensors, qc = _run_nvfp4_to_mxfp4(model_dir, output_dir)
+
+ _assert_mxfp4_layer(tensors, _NV_MLP)
+ _assert_unchanged(
+ tensors,
+ source,
+ [f"{_NV_ATTN}.weight", f"{_NV_ATTN}.weight_scale", f"{_NV_ATTN}.input_scale", "lm_head.weight"],
+ )
+ assert qc["quant_method"] == "compressed-tensors"
+ assert qc["format"] == "mixed-precision"
+ groups = list(qc["config_groups"].values())
+ fp8_group = next(g for g in groups if g["weights"]["num_bits"] == 8)
+ assert fp8_group["targets"] == [_NV_ATTN]
+ assert fp8_group["format"] == "float-quantized"
+ assert fp8_group["weights"]["strategy"] == "tensor" and fp8_group["weights"]["type"] == "float"
+ assert fp8_group["input_activations"]["strategy"] == "tensor"
+ assert fp8_group["input_activations"]["dynamic"] is False
+ _assert_mxfp4_group(next(g for g in groups if g["weights"]["num_bits"] == 4), [_NV_MLP])
+ assert "lm_head" in qc["ignore"]
+ assert any(t.startswith("re:") and re.match(t[3:], "mtp.layers.0.mlp") for t in qc["ignore"])
+ # The stale ModelOpt metadata file must not survive next to the rewritten config.
+ assert not os.path.exists(os.path.join(output_dir, "hf_quant_config.json"))
+
+ @require_compressed_tensors
+ def test_modelopt_nvfp4_with_kv_cache(self, tmp_path):
+ source = {
+ **_modelopt_nvfp4_tensors(_NV_MLP),
+ f"{_NV_ATTN}.weight": torch.randn(64, 64, dtype=torch.bfloat16),
+ "model.layers.0.self_attn.k_proj.k_scale": torch.tensor(0.1, dtype=torch.float32),
+ }
+ model_dir = _make_nvfp4_model_dir(tmp_path, _cfg_with_qc(_MODELOPT_NVFP4_QC), source)
+ tensors, qc = _run_nvfp4_to_mxfp4(model_dir, str(tmp_path / "out"))
+
+ _assert_mxfp4_layer(tensors, _NV_MLP)
+ _assert_unchanged(tensors, source, [f"{_NV_ATTN}.weight", "model.layers.0.self_attn.k_proj.k_scale"])
+ assert qc["format"] == "mxfp4-pack-quantized"
+ _assert_mxfp4_group(next(iter(qc["config_groups"].values())), [_NV_MLP])
+ assert qc["kv_cache_scheme"]["num_bits"] == 8 and qc["kv_cache_scheme"]["type"] == "float"
+ attn_ignore = [t for t in qc["ignore"] if t.startswith("re:")]
+ assert attn_ignore and re.match(attn_ignore[0][3:], _NV_ATTN)
+
+ @require_compressed_tensors
+ def test_quark_nvfp4(self, tmp_path):
+ source = {
+ **_modelopt_nvfp4_tensors(_NV_MLP, input_scale_key="input_scale_2"),
+ f"{_NV_ATTN}.weight": torch.randn(64, 64, dtype=torch.bfloat16),
+ }
+ model_dir = _make_nvfp4_model_dir(tmp_path, _cfg_with_qc(_QUARK_NVFP4_QC), source)
+ tensors, qc = _run_nvfp4_to_mxfp4(model_dir, str(tmp_path / "out"))
+
+ _assert_mxfp4_layer(tensors, _NV_MLP)
+ assert f"{_NV_MLP}.input_scale_2" not in tensors
+ _assert_unchanged(tensors, source, [f"{_NV_ATTN}.weight"])
+ assert qc["quant_method"] == "compressed-tensors"
+ assert qc["format"] == "mxfp4-pack-quantized"
+ _assert_mxfp4_group(next(iter(qc["config_groups"].values())), [_NV_MLP])
+ assert qc["ignore"] == [_NV_ATTN, "lm_head"]
+ assert "kv_cache_scheme" not in qc or qc["kv_cache_scheme"] is None
+
+ @require_compressed_tensors
+ def test_existing_mxfp_layers_kept(self, tmp_path):
+ mxfp4, mxfp8 = "model.layers.0.mlp.gate_proj", "model.layers.0.self_attn.o_proj"
+ source = {
+ **_nvfp4_source_tensors(_NV_MLP),
+ f"{mxfp4}.weight_packed": torch.randint(0, 256, (32, 32), dtype=torch.uint8),
+ f"{mxfp4}.weight_scale": torch.full((32, 2), 127, dtype=torch.uint8),
+ f"{mxfp8}.weight": torch.randn(64, 64).to(torch.float8_e4m3fn),
+ f"{mxfp8}.weight_scale": torch.full((64, 2), 127, dtype=torch.uint8),
+ }
+ model_dir = _make_nvfp4_model_dir(tmp_path, _LLAMA_CFG, source)
+ tensors, _ = _run_nvfp4_to_mxfp4(model_dir, str(tmp_path / "out"))
+
+ _assert_mxfp4_layer(tensors, _NV_MLP)
+ _assert_unchanged(
+ tensors,
+ source,
+ [f"{mxfp4}.weight_packed", f"{mxfp4}.weight_scale", f"{mxfp8}.weight", f"{mxfp8}.weight_scale"],
+ )
+
+ @require_compressed_tensors
+ def test_deepseek_fp8_kept_across_shards(self, tmp_path):
+ fp8 = "model.layers.0.self_attn.o_proj"
+ # _make_model_dir splits keys in half: the FP8 weight lands in shard 1, its scale_inv in shard 2.
+ source = {
+ **_nvfp4_source_tensors(_NV_MLP),
+ f"{fp8}.weight": torch.randn(128, 128).to(torch.float8_e4m3fn),
+ f"{fp8}.weight_scale_inv": torch.rand(1, 1, dtype=torch.float32),
+ "lm_head.weight": torch.randn(128, 64, dtype=torch.bfloat16),
+ f"{_NV_ATTN}.weight": torch.randn(64, 64, dtype=torch.bfloat16),
+ "model.norm.weight": torch.ones(64, dtype=torch.bfloat16),
+ "model.embed_tokens.weight": torch.randn(128, 64, dtype=torch.bfloat16),
+ }
+ model_dir = _make_nvfp4_model_dir(tmp_path, _LLAMA_CFG, source, multi_shard=True)
+ output_dir = str(tmp_path / "out")
+ tensors, _ = _run_nvfp4_to_mxfp4(model_dir, output_dir)
+
+ _assert_mxfp4_layer(tensors, _NV_MLP)
+ _assert_unchanged(tensors, source, [f"{fp8}.weight", f"{fp8}.weight_scale_inv", f"{_NV_ATTN}.weight"])
+ with open(os.path.join(output_dir, "model.safetensors.index.json")) as f:
+ weight_map = json.load(f)["weight_map"]
+ assert set(weight_map) == set(tensors)
+
+ @require_compressed_tensors
+ def test_mxfp4_matches_nvfp4_dequant(self, tmp_path):
+ from auto_round.utils.model_free_utils import _dequant_nvfp4_tensors
+
+ source = _nvfp4_source_tensors(_NV_MLP)
+ model_dir = _make_nvfp4_model_dir(tmp_path, _LLAMA_CFG, source)
+ tensors, _ = _run_nvfp4_to_mxfp4(model_dir, str(tmp_path / "out"), disable_opt_rtn=True)
+
+ ref = _dequant_nvfp4_tensors({k: v.clone() for k, v in source.items()}, [_NV_MLP])[f"{_NV_MLP}.weight"]
+ out = _dequant_mxfp_tensors(
+ {
+ f"{_NV_MLP}.weight_packed": tensors[f"{_NV_MLP}.weight_packed"],
+ f"{_NV_MLP}.weight_scale": tensors[f"{_NV_MLP}.weight_scale"],
+ }
+ )[f"{_NV_MLP}.weight"]
+ # With unit NVFP4 scales every value is an E2M1 grid point, so MXFP4 reproduces it exactly.
+ assert torch.equal(out.float(), ref.float())
+
+ # ------------------------------------------------------------------
+ # Edge cases
+ # ------------------------------------------------------------------
+
+ @require_compressed_tensors
+ def test_ignored_nvfp4_layer_is_dequantized_and_added_to_ignore(self, tmp_path):
+ source = {**_nvfp4_source_tensors(_NV_MLP), **_nvfp4_source_tensors(_NV_MLP_UP)}
+ model_dir = _make_nvfp4_model_dir(tmp_path, _cfg_with_qc(_CT_NVFP4_QC), source)
+ tensors, qc = _run_nvfp4_to_mxfp4(model_dir, str(tmp_path / "out"), ignore_layers="down_proj")
+
+ _assert_mxfp4_layer(tensors, _NV_MLP_UP)
+ assert tensors[f"{_NV_MLP}.weight"].dtype == torch.bfloat16
+ assert f"{_NV_MLP}.weight_packed" not in tensors
+ assert _NV_MLP in qc["ignore"]
+ assert _NV_MLP_UP not in qc["ignore"]
+
+ @require_compressed_tensors
+ def test_no_nvfp4_layers_keeps_everything(self, tmp_path):
+ source = {f"{_NV_ATTN}.weight": torch.randn(64, 64, dtype=torch.bfloat16), **_ct_fp8_tensors(_NV_MLP)}
+ model_dir = _make_nvfp4_model_dir(tmp_path, _LLAMA_CFG, source)
+ tensors, _ = _run_nvfp4_to_mxfp4(model_dir, str(tmp_path / "out"))
+
+ assert set(tensors) == set(source)
+ _assert_unchanged(tensors, source, list(source))
+
+ @require_compressed_tensors
+ def test_unknown_source_config_falls_back_to_plain_mxfp4(self, tmp_path):
+ source = {**_nvfp4_source_tensors(_NV_MLP), f"{_NV_ATTN}.weight": torch.randn(64, 64, dtype=torch.bfloat16)}
+ model_dir = _make_nvfp4_model_dir(tmp_path, _cfg_with_qc({"quant_method": "bitsandbytes"}), source)
+ tensors, qc = _run_nvfp4_to_mxfp4(model_dir, str(tmp_path / "out"))
+
+ _assert_mxfp4_layer(tensors, _NV_MLP)
+ assert qc["format"] == "mxfp4-pack-quantized"
+ assert next(iter(qc["config_groups"].values()))["targets"] == [_NV_MLP]
+ assert _NV_ATTN in qc["ignore"]
+
+ @require_compressed_tensors
+ def test_non_ct_moe_source_targets_routed_experts(self, tmp_path):
+ experts = [f"model.layers.0.block_sparse_moe.experts.{i}.w1" for i in range(2)]
+ source = {k: v for e in experts for k, v in _modelopt_nvfp4_tensors(e).items()}
+ source[f"{_NV_ATTN}.weight"] = torch.randn(64, 64, dtype=torch.bfloat16)
+ model_dir = _make_nvfp4_model_dir(tmp_path, _cfg_with_qc(_MODELOPT_NVFP4_QC), source)
+ tensors, qc = _run_nvfp4_to_mxfp4(model_dir, str(tmp_path / "out"))
+
+ for expert in experts:
+ _assert_mxfp4_layer(tensors, expert)
+ targets = next(iter(qc["config_groups"].values()))["targets"]
+ assert "Linear" not in targets
+ assert sorted(targets) == sorted(experts + ["RoutedExperts"])
+
+ @pytest.mark.parametrize(
+ "qc",
+ [
+ {**_QUARK_NVFP4_QC, "layer_type_quant_config": {"Linear": {"weight": _QUARK_FP4_SPEC}}},
+ {**_QUARK_NVFP4_QC, "global_quant_config": {"weight": {"dtype": "int4", "qscheme": "per_group"}}},
+ {**_MODELOPT_NVFP4_QC, "quant_algo": "W4A8_AWQ"},
+ {"quant_method": "gptq"},
+ ],
+ ids=["quark_layer_type", "quark_int4", "modelopt_awq", "gptq"],
+ )
+ def test_unsupported_source_config_is_skipped(self, qc):
+ from auto_round.utils.model_free_utils import _source_config_to_ct
+
+ assert _source_config_to_ct(qc) is None
+
+ def test_load_source_quant_config_prefers_config_json(self, tmp_path):
+ from auto_round.utils.model_free_utils import _load_source_quant_config
+
+ with open(tmp_path / "hf_quant_config.json", "w") as f:
+ json.dump(_MODELOPT_MIXED_HF_QC, f)
+ assert _load_source_quant_config(str(tmp_path), _CT_NVFP4_QC) is _CT_NVFP4_QC
+ loaded = _load_source_quant_config(str(tmp_path), {})
+ assert loaded["quant_method"] == "modelopt"
+ assert loaded["quantized_layers"] == _MODELOPT_MIXED_HF_QC["quantization"]["quantized_layers"]
+
+ # ------------------------------------------------------------------
+ # Entry points / format validation
+ # ------------------------------------------------------------------
+
+ @pytest.mark.parametrize("fmt", ["auto_round", "auto_round:auto_gptq", "fake"])
+ def test_core_rejects_non_llm_compressor_format(self, tmp_path, fmt):
+ model_dir = _make_nvfp4_model_dir(tmp_path, _LLAMA_CFG, _nvfp4_source_tensors(_NV_MLP))
+ with pytest.raises(ValueError, match="llm_compressor"):
+ _run_nvfp4_to_mxfp4(model_dir, str(tmp_path / "out"), format=fmt)
+
+ @require_compressed_tensors
+ def test_core_accepts_auto_round_llm_compressor_alias(self, tmp_path):
+ model_dir = _make_nvfp4_model_dir(tmp_path, _LLAMA_CFG, _nvfp4_source_tensors(_NV_MLP))
+ output_dir = str(tmp_path / "out")
+ _run_nvfp4_to_mxfp4(model_dir, output_dir, format="auto_round:llm_compressor")
+ _assert_mxfp4_layer(_read_output_tensors(output_dir), _NV_MLP)
+
+ @require_compressed_tensors
+ def test_autoround_entry(self, tmp_path):
+ source = {**_nvfp4_source_tensors(_NV_MLP), **_ct_fp8_tensors(_NV_ATTN)}
+ model_dir = _make_nvfp4_model_dir(tmp_path, _cfg_with_qc(_CT_MIXED_QC), source)
+ output_dir = str(tmp_path / "out")
+ AutoRound(model=model_dir, scheme="MXFP4", model_free=True, device_map="cpu").quantize_and_save(
+ output_dir, format="llm_compressor"
+ )
+ tensors = _read_output_tensors(output_dir)
+ _assert_mxfp4_layer(tensors, _NV_MLP)
+ _assert_unchanged(tensors, source, [f"{_NV_ATTN}.weight", f"{_NV_ATTN}.weight_scale"])
+ assert _read_qconfig(output_dir)["format"] == "mixed-precision"
+
+ def test_autoround_entry_rejects_non_llm_compressor_format(self, tmp_path):
+ model_dir = _make_nvfp4_model_dir(tmp_path, _LLAMA_CFG, _nvfp4_source_tensors(_NV_MLP))
+ ar = AutoRound(model=model_dir, scheme="MXFP4", model_free=True, device_map="cpu")
+ with pytest.raises(ValueError, match="llm_compressor"):
+ ar.quantize_and_save(str(tmp_path / "out"), format="auto_round")
+
+
# ===========================================================================
# deepseek_v4 MXFP-quantized source models
# ===========================================================================
diff --git a/test/unit/test_cpu/utils/test_model_free_utils.py b/test/unit/test_cpu/utils/test_model_free_utils.py
index 142b9a776..571829b61 100644
--- a/test/unit/test_cpu/utils/test_model_free_utils.py
+++ b/test/unit/test_cpu/utils/test_model_free_utils.py
@@ -36,14 +36,17 @@ from auto_round.utils.model_free_utils import (
_build_mxfp_autoround_quantization_config,
_build_mxfp_quantization_config,
_build_quantization_config,
+ _collect_nvfp4_source_layers,
_convert_auto_scheme_layer_config,
_dequant_fp8_tensors,
- _dequant_modelopt_nvfp4_tensors,
+ _dequant_inkling_nvfp4_tensors,
_dequant_mxfp_tensors,
_expand_e8m0_block_scale,
_handle_mxfp_source_tensors,
- _hydrate_and_clean_modelopt_nvfp4_aux,
+ _handle_nvfp4_source_tensors,
+ _hydrate_and_clean_inkling_nvfp4_aux,
_hydrate_missing_fp8_scales_from_index,
+ _ignore_fused_experts_for_vllm,
_looks_like_auto_scheme,
_PatternMatcher,
_process_shard,
@@ -104,7 +107,7 @@ def _read_output_keys(output_dir):
return keys
-def test_modelopt_nvfp4_failure_restores_scale2():
+def test_inkling_nvfp4_failure_restores_scale2():
"""A failed fused-NVFP4 decode must preserve its complete source triple."""
base = "model.layers.0.experts.w13_weight"
packed = torch.zeros((2, 4, 8), dtype=torch.uint8)
@@ -112,7 +115,7 @@ def test_modelopt_nvfp4_failure_restores_scale2():
scale2 = torch.ones(3, dtype=torch.float32)
tensors = {base: packed, f"{base}.scale": scale, f"{base}.scale2": scale2}
- result = _dequant_modelopt_nvfp4_tensors(tensors)
+ result = _dequant_inkling_nvfp4_tensors(tensors)
assert result[base] is packed
assert result[f"{base}.scale"] is scale
@@ -145,8 +148,8 @@ def test_fused_moe_weight_uses_canonical_name_for_matcher():
assert quantize.call_count == tensor.shape[0]
-def test_modelopt_nvfp4_hydrates_scales_from_separate_shards(tmp_path):
- """ModelOpt NVFP4 scales are hydrated from the index-selected donor shards."""
+def test_inkling_nvfp4_hydrates_scales_from_separate_shards(tmp_path):
+ """Inkling NVFP4 scales are hydrated from the index-selected donor shards."""
base = "model.layers.0.experts.w13_weight"
recipient_name = "model-00001-of-00003.safetensors"
scale_shard_name = "model-00002-of-00003.safetensors"
@@ -170,7 +173,7 @@ def test_modelopt_nvfp4_hydrates_scales_from_separate_shards(tmp_path):
file,
)
- result = _hydrate_and_clean_modelopt_nvfp4_aux(
+ result = _hydrate_and_clean_inkling_nvfp4_aux(
{base: torch.zeros((2, 4, 8), dtype=torch.uint8)},
str(tmp_path / recipient_name),
)
@@ -179,10 +182,10 @@ def test_modelopt_nvfp4_hydrates_scales_from_separate_shards(tmp_path):
assert torch.equal(result[scale2_name], scale2)
-def test_modelopt_nvfp4_dequantizes_fused_3d_weight():
- """Fused ModelOpt NVFP4 weights dequantize per expert and drop metadata."""
+def test_inkling_nvfp4_dequantizes_fused_3d_weight():
+ """Fused Inkling NVFP4 weights dequantize per expert and drop metadata."""
base = "model.layers.0.experts.w13_weight"
- result = _dequant_modelopt_nvfp4_tensors(
+ result = _dequant_inkling_nvfp4_tensors(
{
base: torch.zeros((2, 4, 8), dtype=torch.uint8),
f"{base}.scale": torch.ones((2, 4, 1), dtype=torch.float32),
@@ -191,7 +194,7 @@ def test_modelopt_nvfp4_dequantizes_fused_3d_weight():
)
assert set(result) == {base}
- assert result[base].dtype == torch.bfloat16
+ assert result[base].dtype == torch.float32
assert result[base].shape == (2, 4, 16)
@@ -516,6 +519,52 @@ class TestBuildQuantizationConfigNvfp4MoeLlmCompressor:
targets = _get_targets_from_qconfig(qconfig, "group_0")
assert "RoutedExperts" not in targets, f"Unexpected RoutedExperts in targets: {targets}"
+ def test_ignored_fused_experts_get_vllm_ignore_entry(self):
+ default_scheme = {"bits": 16, "data_type": "float", "group_size": -1}
+ layer_cfg = {".w1.": {"bits": 4, "data_type": "nv_fp", "group_size": 16}}
+ qconfig = _build_quantization_config(
+ default_scheme=default_scheme,
+ layer_config=layer_cfg,
+ ignore_patterns=[],
+ quantized_layers=_MOE_NV_QUANTIZED,
+ ignored_layers=["model.layers.2.mlp.experts.w13_weight", "model.layers.2.mlp.experts.w2_weight"],
+ format="llm_compressor",
+ )
+ assert qconfig["ignore"] == [r"re:^model\.layers\.2\.mlp\.experts\.\d+\.(gate|up|down)_proj$"]
+
+
+class TestIgnoreFusedExpertsForVllm:
+ _ENTRY = r"re:^model\.layers\.0\.mlp\.experts\.\d+\.(gate|up|down)_proj$"
+
+ def _qconfig(self, targets):
+ return {"config_groups": {"group_0": {"targets": targets}}, "ignore": ["lm_head"]}
+
+ def test_adds_entry_matching_vllm_probe_names(self):
+ import re
+
+ qconfig = _ignore_fused_experts_for_vllm(
+ self._qconfig(["Linear", "RoutedExperts"]),
+ ["model.layers.0.mlp.experts.w13_weight", "model.layers.0.mlp.experts.w2_weight"],
+ [],
+ )
+ assert qconfig["ignore"] == ["lm_head", self._ENTRY]
+ for proj in ("gate_proj", "up_proj", "down_proj"):
+ assert re.match(self._ENTRY[3:], f"model.layers.0.mlp.experts.0.{proj}")
+
+ def test_no_routed_experts_no_change(self):
+ qconfig = _ignore_fused_experts_for_vllm(
+ self._qconfig(["Linear"]), ["model.layers.0.mlp.experts.w13_weight"], []
+ )
+ assert qconfig["ignore"] == ["lm_head"]
+
+ def test_unfused_and_partially_quantized_modules_skipped(self):
+ qconfig = _ignore_fused_experts_for_vllm(
+ self._qconfig(["RoutedExperts"]),
+ ["model.layers.1.mlp.experts.0.gate_proj", "model.layers.0.mlp.experts.w13_weight"],
+ ["model.layers.0.mlp.experts.w2_weight"],
+ )
+ assert qconfig["ignore"] == ["lm_head"]
+
# ===========================================================================
# _PatternMatcher
@@ -825,6 +874,53 @@ def _make_deepseek_v4_mxfp8(out_f, in_f, block_h, block_w):
return weight_fp8, scale
+class TestNVFP4VsMXFP4SourceDetection:
+ """NVFP4 (FP8 E4M3 scale / global scale) and MXFP4 (uint8 E8M0 scale) share weight_packed + weight_scale."""
+
+ @staticmethod
+ def _mxfp4(prefix):
+ return {
+ f"{prefix}.weight_packed": torch.randint(0, 256, (32, 32), dtype=torch.uint8),
+ f"{prefix}.weight_scale": torch.full((32, 2), 127, dtype=torch.uint8),
+ }
+
+ @staticmethod
+ def _nvfp4(prefix, with_global=True):
+ tensors = {
+ f"{prefix}.weight_packed": torch.randint(0, 256, (32, 32), dtype=torch.uint8),
+ f"{prefix}.weight_scale": torch.ones((32, 4)).to(torch.float8_e4m3fn),
+ }
+ if with_global:
+ tensors[f"{prefix}.weight_global_scale"] = torch.tensor([1.0])
+ tensors[f"{prefix}.input_global_scale"] = torch.tensor([1.0])
+ return tensors
+
+ def test_collect_only_nvfp4_layers(self):
+ raw = {**self._mxfp4("mx"), **self._nvfp4("nv"), **self._nvfp4("nv_no_global", with_global=False)}
+ assert _collect_nvfp4_source_layers(raw) == {"nv", "nv_no_global"}
+
+ @pytest.mark.parametrize(
+ "default",
+ [
+ {"bits": 4, "group_size": 32, "data_type": "mx_fp"},
+ {"bits": 4, "group_size": 128, "sym": True, "data_type": "int"},
+ ],
+ ids=["mxfp4_target", "int4_target"],
+ )
+ def test_handle_nvfp4_skips_mxfp4_layers(self, default):
+ mx = self._mxfp4("mx")
+ raw = {**{k: v.clone() for k, v in mx.items()}, **self._nvfp4("nv")}
+ raw_out, passthrough, layers = _handle_nvfp4_source_tensors(raw, _matcher(default=default))
+
+ assert passthrough == {} and layers == []
+ for key, value in mx.items():
+ assert torch.equal(raw_out[key], value)
+ assert "mx.weight" not in raw_out
+ assert raw_out["nv.weight"].dtype == torch.float32
+ assert raw_out["nv.weight"].shape == (32, 64)
+ assert "nv.weight_packed" not in raw_out and "nv.weight_global_scale" not in raw_out
+
+
class TestHandleMXFPSourceTensors:
def test_passthrough_mxfp8_same_target(self):
weight_fp8 = torch.randn(64, 128, dtype=torch.bfloat16).to(torch.float8_e4m3fn)
@@ -857,8 +953,8 @@ class TestHandleMXFPSourceTensors:
matcher = _matcher(default={"bits": 4, "group_size": 128, "sym": True, "data_type": "int"})
raw_out, passthrough, layers = _handle_mxfp_source_tensors(raw, matcher)
assert layers == [] and passthrough == {}
- assert raw_out["layer.weight"].dtype == torch.bfloat16
- assert torch.allclose(raw_out["layer.weight"], weight_fp8.to(torch.bfloat16))
+ assert raw_out["layer.weight"].dtype == torch.float32
+ assert torch.allclose(raw_out["layer.weight"], weight_fp8.to(torch.float32))
assert "layer.weight_scale" not in raw_out
def test_mixed_passthrough_and_dequant(self):
@@ -877,7 +973,7 @@ class TestHandleMXFPSourceTensors:
assert "attn" in layers
assert passthrough["attn.weight"].dtype == torch.float8_e4m3fn
assert "mlp" not in layers
- assert raw_out["mlp.weight"].dtype == torch.bfloat16
+ assert raw_out["mlp.weight"].dtype == torch.float32
assert raw_out["mlp.weight"].shape == (64, 128)
def test_noop_without_mxfp_tensors(self):
@@ -915,9 +1011,9 @@ class TestHandleMXFPSourceTensors:
raw_out, state = _preprocess_model_type_source_tensors(raw, model_type="deepseek_v4")
raw_out, passthrough, layers = _handle_mxfp_source_tensors(raw_out, matcher, source_state=state)
assert layers == [] and passthrough == {}
- assert raw_out["layer.weight"].dtype == torch.bfloat16
+ assert raw_out["layer.weight"].dtype == torch.float32
assert raw_out["layer.weight"].shape == (64, 128)
- assert torch.allclose(raw_out["layer.weight"], weight_fp8.to(torch.bfloat16))
+ assert torch.allclose(raw_out["layer.weight"], weight_fp8.to(torch.float32))
assert "layer.scale" not in raw_out and "layer.weight_scale" not in raw_out
def test_convert_noop_without_quantized(self):
@@ -933,8 +1029,8 @@ class TestHandleMXFPSourceTensors:
weight_scale = torch.full((64, 4), 127, dtype=torch.uint8)
raw = {"layer.weight": weight_fp8.clone(), "layer.weight_scale": weight_scale.clone()}
out = _dequant_mxfp_tensors(raw)
- assert out["layer.weight"].dtype == torch.bfloat16
- assert torch.allclose(out["layer.weight"], weight_fp8.to(torch.bfloat16))
+ assert out["layer.weight"].dtype == torch.float32
+ assert torch.allclose(out["layer.weight"], weight_fp8.to(torch.float32))
assert "layer.weight_scale" not in out
@@ -1488,7 +1584,7 @@ class TestFP8Source:
w = torch.randn(64, 128, dtype=torch.bfloat16).to(torch.float8_e4m3fn)
raw = {"layer.weight": w, "layer.weight_scale_inv": torch.tensor(0.5), "layer.bias": torch.randn(64)}
result = _dequant_fp8_tensors(raw, block_size=None)
- assert result["layer.weight"].dtype == torch.bfloat16 and "layer.weight_scale_inv" not in result
+ assert result["layer.weight"].dtype == torch.float32 and "layer.weight_scale_inv" not in result
def test_no_fp8_noop(self):
raw = {"layer.weight": torch.randn(64, 128)}
@@ -1559,7 +1655,7 @@ class TestKimiK25Int4Source:
device="cpu",
)
- expected = ((nibbles - 8).float() * 0.5).to(torch.bfloat16)
+ expected = (nibbles - 8).float() * 0.5
assert torch.equal(out["layer.weight"], expected)
def test_kimi_k25_int4_dequant_helper(self):
@@ -1574,7 +1670,7 @@ class TestKimiK25Int4Source:
device="cpu",
)
assert "layer.weight" in out
- assert out["layer.weight"].dtype == torch.bfloat16
+ assert out["layer.weight"].dtype == torch.float32
assert out["layer.weight"].shape == (128, 128)
assert "layer.weight_packed" not in out
assert "layer.weight_scale" not in out
@@ -1618,3 +1714,19 @@ class TestKimiK25Int4Source:
# Re-quantized MXFP4 scales are uint8 E8M0 (source INT4 scales were fp16).
assert found_scale_dtype == torch.uint8
assert found_packed_shape == (128, 64)
+
+
+def test_explicit_mxfp_targets_replaces_linear_with_routed_experts_catch_all():
+ from auto_round.utils.model_free_utils import _explicit_mxfp_targets
+
+ weights = {"type": "float", "num_bits": 4, "group_size": 32}
+ qconfig = {
+ "config_groups": {
+ "group_0": {"targets": ["model.layers.0.mlp.fc1"], "weights": weights},
+ "group_1": {"targets": ["Linear", "RoutedExperts"], "weights": weights},
+ }
+ }
+ layers = ["model.layers.0.mlp.fc1", "model.layers.0.self_attn.q_proj"]
+ out = _explicit_mxfp_targets(qconfig, layers)
+ assert out["config_groups"]["group_0"]["targets"] == ["model.layers.0.mlp.fc1"]
+ assert out["config_groups"]["group_1"]["targets"] == ["model.layers.0.self_attn.q_proj"]