PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
feat: support LLMC/ModelOpt/Quark format NVFP4 -> MXFP4 conversion (#2492)
Xin He7222a20eaca4
diff --git a/auto_round/compressors/model_free.py b/auto_round/compressors/model_free.pyindex 0e834d52c..e8b88c1b1 100644--- a/auto_round/compressors/model_free.py+++ b/auto_round/compressors/model_free.py@@ -40,6 +40,9 @@ weights with ``fake`` format):* Preset names: ``MXFP4``, ``MXFP8``.* ``data_type="mx_fp"``, ``group_size=32``, ``bits in {4, 8}``.+* NVFP4 source checkpoints (detected from the model name or the source quantization config) with+ scheme ``MXFP4``: only the NVFP4 layers are converted to MXFP4 (plus any layer matched by+ ``layer_config``); every other layer is kept as-is.**NVFP4 E5M3** (``auto_round`` or ``fake`` format):@@ -125,6 +128,7 @@ from auto_round.utils.device import clear_memory, memory_monitorfrom auto_round.utils.device_manager import default_enable_torch_compilefrom auto_round.utils.model_free_utils import (_LM_HEAD_PATTERNS,+ _NVFP_SOURCE_ONLY_KEY,_apply_scheme_overrides,_build_cross_shard_pairs_from_weight_map,_build_quantization_config,@@ -138,6 +142,7 @@ from auto_round.utils.model_free_utils import (_layer_config_has_nvfp4,_list_weight_shards,_load_config,+ _load_source_quant_config,_load_weight_map_from_index,_looks_like_auto_scheme,_normalize_scheme,@@ -148,6 +153,7 @@ from auto_round.utils.model_free_utils import (_validate_supported_scheme,_write_index_file,_write_output_shard,+ detect_model_dtypes,handle_model_type_low_precision_source_tensors,is_model_free_supported_scheme,preprocess_model_type_source_tensors,@@ -200,6 +206,7 @@ _QUANTIZATION_CONFIG_FILENAMES: tuple[str, ...] = ("quantization_config.json","quantize_config.json","quant_config.json",+ "hf_quant_config.json",)_RESUME_MANIFEST_NAME = "model_free_resume.json"@@ -572,6 +579,8 @@ class _ModelFreeCompressorCore:self.model_name_or_path = model_name_or_pathself.output_dir = output_dirself.scheme_input = scheme+ # Set by _detect_nvfp4_source_model(): convert only the NVFP4 source layers to MXFP4.+ self.nvfp_source_only = Falseself.layer_config_input = layer_configself.ignore_layers_input = ignore_layersself.format = format or "auto_round"@@ -664,6 +673,33 @@ class _ModelFreeCompressorCore:self.default_scheme = {k: v for k, v in ds.items() if v is not None}self.default_scheme["_output_format"] = self.format+ def _detect_nvfp4_source_model(self) -> None:+ """With an MXFP4 scheme on an NVFP4 source, convert only the NVFP4 layers (plus layer_config matches)."""+ data_type = (self.scheme_obj.data_type or "").lower()+ if not (is_mx_fp(data_type) and self.scheme_obj.bits == 4):+ return+ reason = detect_model_dtypes(+ self.model_name_or_path,+ _load_source_quant_config(+ self.work_dir if self.is_streaming else self.source_dir, self.source_quantization_config+ ),+ ).get("nvfp4")+ if reason is None:+ return+ format_lower = self.format.lower().replace(" ", "").split(",")[0]+ if format_lower not in ("llm_compressor", "auto_round:llm_compressor"):+ raise ValueError(+ f"Detected an NVFP4 source model ({reason}); converting its NVFP4 layers to MXFP4 "+ f"currently only supports format 'llm_compressor', got '{self.format}'."+ )+ logger.warning(+ f"Detected NVFP4 layers in the source model ({reason}). Only the NVFP4 layers will be quantized "+ "to MXFP4; all other layers (BF16, FP8, ...) are kept as-is. To quantize additional layers, "+ "set them explicitly via layer_config."+ )+ self.nvfp_source_only = True+ self.default_scheme[_NVFP_SOURCE_ONLY_KEY] = True+def _parse_layer_config(self) -> None:lc = copy.deepcopy(self.layer_config_input) if self.layer_config_input else {}@@ -846,12 +882,7 @@ class _ModelFreeCompressorCore:def _detect_fp8_source(self) -> None:quant_config = self.source_quantization_config- is_fp8 = (- quant_config.get("quant_method") == "fp8"- or quant_config.get("quantization_type") == "fp8"- or quant_config.get("fmt", "").startswith("e4m3")- )- if is_fp8:+ if "fp8" in detect_model_dtypes(self.model_name_or_path, quant_config):self.fp8_block_size = quant_config.get("weight_block_size")logger.info(f"Detected FP8 source model (block_size={self.fp8_block_size}, "@@ -1556,6 +1587,12 @@ class _ModelFreeCompressorCore:block_name_to_quantize = ",".join(dict.fromkeys(block_prefixes)) or Noneos.makedirs(self._quant_output_dir, exist_ok=True)+ source_quant_config = None+ if self.nvfp_source_only:+ # Read before the stale source metadata files are removed below.+ source_quant_config = _load_source_quant_config(+ self.work_dir if self.is_streaming else self.source_dir, self.source_quantization_config+ )quantization_config = _build_quantization_config(default_scheme=self.default_scheme,layer_config=self.layer_config,@@ -1564,6 +1601,7 @@ class _ModelFreeCompressorCore:ignored_layers=self.all_ignored_layers,block_name_to_quantize=block_name_to_quantize,format=self.format,+ source_quant_config=source_quant_config,)self._remove_stale_quantization_config_files()@@ -1700,6 +1738,7 @@ class _ModelFreeCompressorCore:# ---- source resolution ----self._resolve_source()+ self._detect_nvfp4_source_model()self._check_conv1d_and_embedding()self._apply_predefined_ignore_layers()self._detect_fp8_source()@@ -1742,6 +1781,7 @@ class _ModelFreeCompressorCore:logger.info(f"Model-free quantization: {self.model_name_or_path}\n"f" Scheme: {self.scheme_obj}\n"+ f" NVFP4 source layers only: {self.nvfp_source_only}\n"f" Packing format: {packing_format}\n"f" Optimized RTN: {'enabled' if opt_rtn_enabled else 'disabled'}\n"f" Output: {self.output_dir}\n"@@ -1769,6 +1809,12 @@ class _ModelFreeCompressorCore:f"{', '.join(missing_shards)}")+ if self.nvfp_source_only and not self.all_quantized_layers:+ logger.warning(+ f"No NVFP4 layers were found in the shards of '{self.model_name_or_path}'; "+ "all layers were kept as-is."+ )+# ---- write outputs ----self._write_index()self._write_config_files()diff --git a/auto_round/export/export_to_llmcompressor/export.py b/auto_round/export/export_to_llmcompressor/export.pyindex 12148253f..22af10d1e 100644--- a/auto_round/export/export_to_llmcompressor/export.py+++ b/auto_round/export/export_to_llmcompressor/export.py@@ -19,6 +19,10 @@ from collections.abc import Callable, Mappingimport torch+from auto_round.export.export_to_llmcompressor.utils import (+ add_routed_experts_if_moe,+ finalize_llmcompressor_config_names,+)from auto_round.export.utils import (is_immediate_saving_mode,save_config_artifact,@@ -175,7 +179,7 @@ def _rewrite_config_group_targets(model, quantization_config_dict):)names = sig_to_names.get(sig)if names:- group["targets"] = sorted(names)+ group["targets"] = add_routed_experts_if_moe(sorted(names), names)else:logger.warning("AutoRound llm_compressor export: no quantized module matched "@@ -372,6 +376,12 @@ def save_quantized_as_llmcompressor(# precision that makes groups indistinguishable to vLLM; rewrite targets with# the concrete module names each group covers.quantization_config_dict = _rewrite_config_group_targets(model, quantization_config_dict)+ quantization_config_dict = finalize_llmcompressor_config_names(+ quantization_config_dict,+ model,+ [],+ [n for n, m in model.named_modules() if getattr(m, "quantization_scheme", None) is not None],+ )model.config.quantization_config = quantization_config_dictif output_dir is None:diff --git a/auto_round/export/export_to_llmcompressor/export_to_fp.py b/auto_round/export/export_to_llmcompressor/export_to_fp.pyindex 344b8f212..bafa994b3 100644--- a/auto_round/export/export_to_llmcompressor/export_to_fp.py+++ b/auto_round/export/export_to_llmcompressor/export_to_fp.py@@ -27,7 +27,11 @@ from tqdm import tqdmfrom auto_round.compressors.utils import is_mx_fp, is_nv_fpfrom auto_round.export.export_to_autoround.qlinear_fp import QuantLinearfrom auto_round.export.export_to_llmcompressor.config import initialize_quantization-from auto_round.export.export_to_llmcompressor.utils import generate_ignore_regex_list+from auto_round.export.export_to_llmcompressor.utils import (+ add_routed_experts_if_moe,+ finalize_llmcompressor_config_names,+ generate_ignore_regex_list,+)from auto_round.export.utils import (filter_quantization_config,is_immediate_saving_mode,@@ -234,7 +238,9 @@ def _build_mixed_fp_quantization_config(tmp_quantization_config = initialize_quantization(scheme=scheme)tmp_quantization_config = tmp_quantization_config.config_groups["group_0"]is_default = (lbits, ldata_type, lact_bits) == global_key- tmp_quantization_config.targets = ["Linear"] if is_default else layer_names+ tmp_quantization_config.targets = (+ ["Linear"] if is_default else add_routed_experts_if_moe(list(layer_names), layer_names)+ )config_groups[group_name] = tmp_quantization_configgroup_formats[group_name] = _get_group_format(lbits, ldata_type)@@ -435,7 +441,10 @@ def save_quantized_as_fp(f"Unsupported layer override data_type={layer_data_type}, bits={layer_bits}, act_bits={layer_act_bits}.")quantization_config = initialize_quantization(- scheme=scheme, targets=targets, kv_cache_scheme=kv_cache_scheme, ignore=ignore+ scheme=scheme,+ targets=add_routed_experts_if_moe(list(targets), targets),+ kv_cache_scheme=kv_cache_scheme,+ ignore=ignore,)quantization_config.format = _get_group_format(layer_bits, layer_data_type)quantization_config = quantization_config.to_dict()@@ -466,6 +475,12 @@ def save_quantized_as_fp(_configure_gaudi2_fp8_dtype(quantization_config)if attention_config is not None:quantization_config["attention_input_activations"] = attention_config+ quantization_config = finalize_llmcompressor_config_names(+ quantization_config,+ model,+ [],+ [name for names in scheme_groups.values() for name in names],+ )if hasattr(model, "config"):model.config.quantization_config = quantization_configif output_dir is None:diff --git a/auto_round/export/export_to_llmcompressor/export_to_static_fp.py b/auto_round/export/export_to_llmcompressor/export_to_static_fp.pyindex 364d533fe..6b1c4fc44 100644--- a/auto_round/export/export_to_llmcompressor/export_to_static_fp.py+++ b/auto_round/export/export_to_llmcompressor/export_to_static_fp.py@@ -26,6 +26,7 @@ from auto_round.data_type.utils import reshape_pad_tensor_by_group_size, revert_from auto_round.experimental.utils import NVFP4_KV_DTYPEfrom auto_round.export.export_to_autoround.export_to_fp8 import FP8QLinearfrom auto_round.export.export_to_llmcompressor.config import check_compressed_tensors_supported+from auto_round.export.export_to_llmcompressor.utils import finalize_llmcompressor_config_namesfrom auto_round.export.utils import is_immediate_saving_mode, save_model, save_pretrained_artifactfrom auto_round.utils import (SUPPORTED_LAYER_TYPES,@@ -294,6 +295,9 @@ def save_quantized_as_static_fp(quantization_config["config_groups"]["group_0"]["weights"]["group_size"] = group_sizeif attention_config is not None:quantization_config["attention_input_activations"] = attention_config+ quantization_config = finalize_llmcompressor_config_names(+ quantization_config, model, [], [n for n in layer_config if check_to_quantized(layer_config[n])]+ )if hasattr(model, "config"):model.config.quantization_config = quantization_configdiff --git a/auto_round/export/export_to_llmcompressor/utils.py b/auto_round/export/export_to_llmcompressor/utils.pyindex a6020c8f0..eaa175f3b 100644--- a/auto_round/export/export_to_llmcompressor/utils.py+++ b/auto_round/export/export_to_llmcompressor/utils.py@@ -13,8 +13,151 @@# limitations under the License.+import re++import torch+from auto_round.utils import matches_any_regex, to_standard_regex+_EXPERT_SEGMENTS = ("experts", "shared_experts")+++# TODO: Remove when the issue is fixed+# https://github.com/vllm-project/llm-compressor/issues/3069+def add_routed_experts_if_moe(targets: list[str], layer_names: list[str]) -> list[str]:+ """Append ``"RoutedExperts"`` for routed expert layouts that llm-compressor misses.++ The generic ``"Linear"`` target does not cover all expert layouts. Keep the+ safe canonical cases alone (``mlp.experts.*.gate_proj|up_proj|down_proj``),+ but trigger for non-standard expert paths and for any routed+ expert structure that lives outside the standard ``mlp.experts`` naming.+ """+ if "RoutedExperts" in targets:+ return targets++ def _projection_name(name: str) -> str:+ # Eg. '...experts.0.gate_proj.weight' -> ['...', 'experts', '0', 'gate_proj', 'weight']+ parts = name.split(".")+ if len(parts) >= 2:+ return parts[-2]+ return parts[-1]++ for name in layer_names:+ lname = name.lower()+ if "block_sparse_moe" in lname or ".moe." in lname or ".feed_forward." in lname:+ return list(targets) + ["RoutedExperts"]++ if ".experts." not in lname:+ continue++ projection = _projection_name(name)+ if projection in {"gate_proj", "up_proj", "down_proj"}:+ continue+ return list(targets) + ["RoutedExperts"]++ return targets+++def ignore_fused_experts_for_vllm(qconfig: dict, ignored_layers: list[str], quantized_layers: list[str]) -> dict:+ """Add vLLM-matchable ignore entries for ignored fused expert modules when ``RoutedExperts`` is targeted.++ vLLM resolves a FusedMoE scheme via ``<module>.0.{gate,up,down}_proj``, which fused layer names such as+ ``<module>.w13_weight`` never match, so the global ``RoutedExperts`` target would quantize them.+ """+ groups = qconfig.get("config_groups") or {}+ if not any("RoutedExperts" in (g.get("targets") or []) for g in groups.values()):+ return qconfig++ ignore = list(qconfig.get("ignore") or [])+ for layer_name in dict.fromkeys(ignored_layers):+ if layer_name.startswith("re:"):+ continue+ parts = layer_name.split(".")+ if parts[-1] in _EXPERT_SEGMENTS:+ module = layer_name+ elif len(parts) >= 2 and parts[-2] in _EXPERT_SEGMENTS and not parts[-1].isdigit():+ module = ".".join(parts[:-1])+ else:+ continue+ # A partially quantized module cannot be described as ignored.+ if any(q == module or q.startswith(f"{module}.") for q in quantized_layers):+ continue+ entry = f"re:^{re.escape(module)}\\.\\d+\\.(gate|up|down)_proj$"+ if entry not in ignore:+ ignore.append(entry)+ qconfig["ignore"] = ignore+ return qconfig+++def get_checkpoint_name_reverter(model: torch.nn.Module):+ """Return ``fn(module_name) -> name`` mapping a loaded-model module name to the saved checkpoint namespace."""+ from auto_round.utils.common import (+ get_reverse_checkpoint_conversion_mapping,+ get_reverse_weight_transforms,+ revert_checkpoint_conversion_mapping,+ revert_name_with_weight_transforms,+ )++ transforms = get_reverse_weight_transforms(model)+ mapping = get_reverse_checkpoint_conversion_mapping(model) if transforms is None else None+ if transforms is None and not mapping:+ return lambda name: name++ def _revert(name: str) -> str:+ # Renamings are defined on parameter keys, so revert a full weight key.+ key = f"{name}.weight"+ if transforms is not None:+ reverted = revert_name_with_weight_transforms(key, transforms)+ else:+ reverted = revert_checkpoint_conversion_mapping(key, mapping)+ return reverted[: -len(".weight")] if reverted.endswith(".weight") else name++ return _revert+++def revert_quantization_config_names(qconfig: dict, revert) -> dict:+ """Rewrite module names in ``ignore`` and group ``targets`` with ``revert`` so they match the saved weights."""++ def _revert_entry(entry):+ if not isinstance(entry, str):+ return entry+ if not entry.startswith("re:"):+ return revert(entry)+ # Only rewrite regexes that wrap a single escaped module name, e.g. ``re:.*model\.layers\.0\.mlp\.gate.*``.+ match = re.fullmatch(r"(re:(?:\.\*)?)(.*?)((?:\.\*)?)", entry)+ plain = re.sub(r"\\(.)", r"\1", match.group(2))+ if not plain or re.escape(plain) != match.group(2):+ return entry+ return f"{match.group(1)}{re.escape(revert(plain))}{match.group(3)}"++ if qconfig.get("ignore"):+ qconfig["ignore"] = list(dict.fromkeys(_revert_entry(e) for e in qconfig["ignore"]))+ for group in (qconfig.get("config_groups") or {}).values():+ if group.get("targets"):+ group["targets"] = [_revert_entry(t) for t in group["targets"]]+ return qconfig+++def finalize_llmcompressor_config_names(+ qconfig: dict, model: torch.nn.Module, ignored_layers: list[str], quantized_layers: list[str]+) -> dict:+ """Revert config module names to the checkpoint namespace, then add vLLM ignore entries for fused experts."""+ revert = get_checkpoint_name_reverter(model)+ qconfig = revert_quantization_config_names(qconfig, revert)+ ignored = list(qconfig.get("ignore") or []) + [+ revert(n) for n in ignored_layers + collect_fused_expert_modules(model) if not n.startswith("re:")+ ]+ return ignore_fused_experts_for_vllm(qconfig, ignored, [revert(n) for n in quantized_layers])+++def collect_fused_expert_modules(model: torch.nn.Module) -> list[str]:+ """Return expert modules still holding fused 3-D weights, i.e. never unfused into quantized Linear layers."""+ return [+ name+ for name, module in model.named_modules()+ if name.split(".")[-1] in _EXPERT_SEGMENTS and any(p.dim() == 3 for p in module.parameters(recurse=False))+ ]+def generate_ignore_regex_list(regex_config: dict[str, dict], layer_config: dict[str, dict]) -> list[str]:"""diff --git a/auto_round/utils/model_free_utils.py b/auto_round/utils/model_free_utils.pyindex 69731a074..f388c4ec9 100644--- a/auto_round/utils/model_free_utils.py+++ b/auto_round/utils/model_free_utils.py@@ -34,6 +34,10 @@ from typing import Anyimport torchfrom auto_round.compressors.utils import is_mx_fp, is_nv_fp+from auto_round.export.export_to_llmcompressor.utils import add_routed_experts_if_moe as _add_routed_experts_if_moe+from auto_round.export.export_to_llmcompressor.utils import (+ ignore_fused_experts_for_vllm as _ignore_fused_experts_for_vllm,+)from auto_round.logger import loggerfrom auto_round.schemes import PRESET_SCHEMES, QuantizationScheme, preset_name_to_schemefrom auto_round.utils.common import _normalize_tensor_name_for_warning, to_standard_regex@@ -45,6 +49,8 @@ from auto_round.utils.path_safety import ()_NVFP4_E5M3_DATA_TYPE = "nvfp4_v2"+# default_scheme flag restricting quantization to layers stored as NVFP4 in the source checkpoint.+_NVFP_SOURCE_ONLY_KEY = "_nvfp_source_only"_BLOCK_NAME_TO_IGNORE = ("shared_expert_gate.", ".gate.", "embed", "conv")_SUPPORTED_MXFP_BITS = (4, 8)_SUPPORTED_INT_BITS = (2, 4, 8)@@ -459,6 +465,7 @@ class _PatternMatcher:"""Precompile ignore and layer-config patterns for shard processing."""__slots__ = (+ "_allowed_layers","_compiled_lc","_default_scheme","_ignore_cache",@@ -488,6 +495,16 @@ class _PatternMatcher:self._ignore_cache: dict[str, bool] = {}self._scheme_cache: dict[str, dict | None] = {}+ self._allowed_layers: set[str] | None = None++ @property+ def nvfp_source_only(self) -> bool:+ return bool(self._default_scheme.get(_NVFP_SOURCE_ONLY_KEY))++ def restrict_to_layers(self, layers: set[str]) -> None:+ """Treat every layer outside *layers* as ignored (unless explicitly listed in layer_config)."""+ self._allowed_layers = set(layers)+ self._ignore_cache.clear()@staticmethoddef _build_ignore_regex(patterns: list[str]) -> re.Pattern | None:@@ -511,10 +528,23 @@ class _PatternMatcher:if layer_name in self._layer_config:self._ignore_cache[tensor_name] = Falsereturn False+ if (+ self._allowed_layers is not None+ and layer_name not in self._allowed_layers+ and not self._matches_layer_config(layer_name)+ ):+ self._ignore_cache[tensor_name] = True+ return Trueresult = bool(self._ignore_re and self._ignore_re.search(layer_name))self._ignore_cache[tensor_name] = resultreturn result+ def _matches_layer_config(self, layer_name: str) -> bool:+ return any(+ (compiled is not None and compiled.search(layer_name)) or (plain is not None and plain in layer_name)+ for compiled, plain, _ in self._compiled_lc+ )+def should_skip(self, tensor_name: str) -> bool:return bool(self._skip_re.search(tensor_name))@@ -569,6 +599,34 @@ def _collect_mxfp_source_entries(raw_tensors: dict[str, torch.Tensor]) -> list[treturn entries+def _collect_nvfp4_source_layers(raw_tensors: dict[str, torch.Tensor]) -> set[str]:+ """Return names of layers stored as NVFP4 in a shard.++ Expects llm-compressor naming (run after :func:`_normalize_nvfp4_source_tensors`):+ ``<layer>.weight_packed`` + ``<layer>.weight_scale`` with a global scale or an FP8+ block scale (MXFP4 uses uint8 E8M0 scales without a global scale). Inkling-style bare+ tensors ``X`` with ``X.scale`` / ``X.scale2`` siblings are detected as layer ``X``.+ """+ layers: set[str] = set()+ for name, tensor in raw_tensors.items():+ if tensor.dtype not in (torch.uint8, torch.int8):+ continue+ if name.endswith(".weight_packed"):+ layer_name = name[: -len(".weight_packed")]+ scale = raw_tensors.get(f"{layer_name}.weight_scale")+ if scale is None:+ continue+ if (+ f"{layer_name}.weight_global_scale" in raw_tensors+ or f"{layer_name}.input_global_scale" in raw_tensors+ or scale.dtype == torch.float8_e4m3fn+ ):+ layers.add(layer_name)+ elif f"{name}.scale" in raw_tensors and f"{name}.scale2" in raw_tensors:+ layers.add(name)+ return layers++def _normalize_nvfp4_source_tensors(raw_tensors: dict[str, torch.Tensor],shard_name: str | None = None,@@ -597,8 +655,9 @@ def _normalize_nvfp4_source_tensors(has_new_global = (f"{layer_name}.weight_global_scale" in raw_tensors or f"{layer_name}.input_global_scale" in raw_tensors)- has_new_packed = f"{layer_name}.weight_packed" in raw_tensors- if has_legacy_global or has_new_global or has_new_packed:+ has_legacy_packed = name.endswith(".weight") and f"{layer_name}.weight_packed" not in raw_tensors+ # Layers already in llm-compressor naming need no normalization.+ if has_legacy_global or (has_legacy_packed and has_new_global):candidates.append(layer_name)if not candidates:@@ -612,24 +671,25 @@ def _normalize_nvfp4_source_tensors(weight_global_scale_key = f"{layer_name}.weight_global_scale"input_global_scale_key = f"{layer_name}.input_global_scale"+ # Quark NVFP4 names the ModelOpt-style input_scale "input_scale_2".+ quark_input_scale = raw_tensors.pop(f"{layer_name}.input_scale_2", None)+ if quark_input_scale is not None and input_scale_key not in raw_tensors:+ raw_tensors[input_scale_key] = quark_input_scale+if weight_packed_key not in raw_tensors and weight_key in raw_tensors:raw_tensors[weight_packed_key] = raw_tensors.pop(weight_key).view(torch.uint8).contiguous()+ # ModelOpt scales (amax / (448 * 6)) are the reciprocal of llm-compressor global scales.if weight_scale_2_key in raw_tensors and weight_global_scale_key not in raw_tensors:- raw_tensors[weight_global_scale_key] = raw_tensors.pop(weight_scale_2_key)+ raw_tensors[weight_global_scale_key] = _reciprocal_global_scale(raw_tensors.pop(weight_scale_2_key))elif weight_scale_2_key in raw_tensors:raw_tensors.pop(weight_scale_2_key)if input_scale_key in raw_tensors and input_global_scale_key not in raw_tensors:- raw_tensors[input_global_scale_key] = raw_tensors.pop(input_scale_key)+ raw_tensors[input_global_scale_key] = _reciprocal_global_scale(raw_tensors.pop(input_scale_key))elif input_scale_key in raw_tensors:raw_tensors.pop(input_scale_key)- if weight_global_scale_key in raw_tensors:- raw_tensors[weight_global_scale_key] = _reciprocal_global_scale(raw_tensors[weight_global_scale_key])- if input_global_scale_key in raw_tensors:- raw_tensors[input_global_scale_key] = _reciprocal_global_scale(raw_tensors[input_global_scale_key])-converted_layers.append(layer_name)if converted_layers:@@ -638,10 +698,10 @@ def _normalize_nvfp4_source_tensors(return raw_tensors, converted_layers-# Auxiliary tensors emitted next to a ModelOpt / TensorRT-LLM NVFP4 packed+# Auxiliary tensors emitted next to an Inkling (``inkling_mm_model``) NVFP4 packed# weight. Once the packed weight has been dequantized back to high precision# these carry no information and must not leak into the quantized output.-_MODELOPT_NVFP4_AUX_SUFFIXES: tuple[str, ...] = (+_INKLING_NVFP4_AUX_SUFFIXES: tuple[str, ...] = (".scale",".scale2",".input_amax",@@ -650,7 +710,7 @@ _MODELOPT_NVFP4_AUX_SUFFIXES: tuple[str, ...] = ()-def _dequant_modelopt_nvfp4_slice(+def _dequant_inkling_nvfp4_slice(packed: torch.Tensor,scale: torch.Tensor,global_scale: float,@@ -660,7 +720,7 @@ def _dequant_modelopt_nvfp4_slice(device: str,target_dtype: torch.dtype,) -> torch.Tensor:- """Dequantize one 2-D ModelOpt NVFP4 slice back to *target_dtype* on CPU."""+ """Dequantize one 2-D Inkling NVFP4 slice back to *target_dtype* on CPU."""from auto_round.experimental.qmodules.fp4_utils import unpack_fp4_from_uint8packed_dev = packed.to(device)@@ -674,16 +734,18 @@ def _dequant_modelopt_nvfp4_slice(return result-def _dequant_modelopt_nvfp4_tensors(+def _dequant_inkling_nvfp4_tensors(raw_tensors: dict[str, torch.Tensor],device: str = "cpu",shard_name: str | None = None,- target_dtype: torch.dtype = torch.bfloat16,+ target_dtype: torch.dtype = torch.float32,) -> dict[str, torch.Tensor]:- """Dequantize ModelOpt / TensorRT-LLM NVFP4 source tensors to high precision.+ """Dequantize Inkling-specific NVFP4 source tensors to high precision.- ModelOpt checkpoints (``hf_quant_config.json`` with ``quant_algo="NVFP4"``)- store a quantized weight as a *packed uint8* tensor ``X`` accompanied by:+ Inkling (``model_type="inkling_mm_model"``) NVFP4 checkpoints use a special layout+ that differs from the generic ModelOpt ``.weight`` / ``.weight_scale`` /+ ``.weight_scale_2`` naming (handled by :func:`_normalize_nvfp4_source_tensors`):+ a quantized weight is stored as a *packed uint8* tensor ``X`` accompanied by:* ``X.scale`` – per-block (group_size=16) scale, ``float8_e4m3fn``* ``X.scale2`` – global scale (scalar, or one per expert for fused MoE)@@ -698,7 +760,7 @@ def _dequant_modelopt_nvfp4_tensors(Both plain 2-D weights and fused 3-D stacked MoE expert weights(``[num_experts, out, in]``) are supported. All auxiliary tensors listed in- :data:`_MODELOPT_NVFP4_AUX_SUFFIXES` are dropped after dequantization.+ :data:`_INKLING_NVFP4_AUX_SUFFIXES` are dropped after dequantization."""entries: list[str] = []for name, tensor in raw_tensors.items():@@ -718,7 +780,7 @@ def _dequant_modelopt_nvfp4_tensors(shard_prefix = f"[{shard_name}] " if shard_name else ""logger.info(- f"{shard_prefix}Dequantizing {len(entries)} ModelOpt NVFP4 source tensor(s) to {target_dtype} on {device}."+ f"{shard_prefix}Dequantizing {len(entries)} Inkling NVFP4 source tensor(s) to {target_dtype} on {device}.")n_dropped = 0@@ -734,7 +796,7 @@ def _dequant_modelopt_nvfp4_tensors(try:if packed.dim() == 2:- dequantized = _dequant_modelopt_nvfp4_slice(+ dequantized = _dequant_inkling_nvfp4_slice(packed,scale,float(global_scale[0]),@@ -752,7 +814,7 @@ def _dequant_modelopt_nvfp4_tensors(# slices, so peak memory stays at one output tensor + one slice.dequantized = torch.empty((num_experts, out_features, in_features), dtype=target_dtype)for i in range(num_experts):- dequantized[i] = _dequant_modelopt_nvfp4_slice(+ dequantized[i] = _dequant_inkling_nvfp4_slice(packed[i],scale[i],float(global_scale[i if global_scale.numel() > 1 else 0]),@@ -764,7 +826,7 @@ def _dequant_modelopt_nvfp4_tensors()except Exception as e:logger.warning(- f"{shard_prefix}Failed to dequantize ModelOpt NVFP4 tensor '{name}': {e}. "+ f"{shard_prefix}Failed to dequantize Inkling NVFP4 tensor '{name}': {e}. "f"Keeping the packed source tensor unchanged.")raw_tensors[name] = packed@@ -776,7 +838,7 @@ def _dequant_modelopt_nvfp4_tensors(del packed, scale, scale2, global_scale# Drop the now-meaningless auxiliary metadata tensors.- for suffix in _MODELOPT_NVFP4_AUX_SUFFIXES:+ for suffix in _INKLING_NVFP4_AUX_SUFFIXES:if raw_tensors.pop(f"{name}{suffix}", None) is not None:n_dropped += 1@@ -796,7 +858,7 @@ def _handle_nvfp4_source_tensors("""Passthrough NVFP4 source tensors when target scheme for the layer is NVFP4.Layers whose target scheme is *not* NVFP4 (e.g. converting an NVFP4 donor- checkpoint to MXFP4) are dequantized to bfloat16 via+ checkpoint to MXFP4) are dequantized to float32 via:func:`_dequant_nvfp4_tensors` instead of being left as raw``weight_packed``/``weight_scale`` tensors — otherwise those companionscale tensors would fall through to the generic RTN/MXFP quantization path@@ -806,6 +868,8 @@ def _handle_nvfp4_source_tensors(passthrough_tensors: dict[str, torch.Tensor] = {}passthrough_layers: list[str] = []dequant_layers: list[str] = []+ # MXFP4 sources also use weight_packed + weight_scale; never treat them as NVFP4.+ nvfp4_layers = _collect_nvfp4_source_layers(raw_tensors)for name, tensor in list(raw_tensors.items()):if not name.endswith(".weight_packed") or tensor.dtype not in (torch.uint8, torch.int8):@@ -813,7 +877,7 @@ def _handle_nvfp4_source_tensors(layer_name = name[: -len(".weight_packed")]scale_key = f"{layer_name}.weight_scale"- if scale_key not in raw_tensors:+ if scale_key not in raw_tensors or layer_name not in nvfp4_layers:continuescheme = matcher.resolve_scheme(f"{layer_name}.weight")@@ -854,7 +918,7 @@ def _dequant_nvfp4_tensors(device: str = "cpu",shard_name: str | None = None,) -> dict[str, torch.Tensor]:- """Dequantize NVFP4-packed weight tensors to bfloat16.+ """Dequantize NVFP4-packed weight tensors to float32.Reverses the NVFP4 packing (see :func:`auto_round.data_type.nvfp.nv_fp4`):``real_weight = unpack(weight_packed) * weight_scale.float() * (1 / weight_global_scale)``@@ -873,7 +937,7 @@ def _dequant_nvfp4_tensors(group_size = 16dequant_device = str(device or "cpu")shard_prefix = f"[{shard_name}] " if shard_name else ""- logger.info(f"{shard_prefix}Dequantizing {len(layer_names)} NVFP4 tensor(s) to bfloat16 on {dequant_device}.")+ logger.info(f"{shard_prefix}Dequantizing {len(layer_names)} NVFP4 tensor(s) to float32 on {dequant_device}.")for layer_name in layer_names:packed_key = f"{layer_name}.weight_packed"@@ -901,7 +965,7 @@ def _dequant_nvfp4_tensors(dq = (unpacked.reshape(out_features, n_groups, group_size) * scale_f).reshape(out_features, in_features)if global_scale is not None:dq = dq * get_reciprocal(global_scale.to(torch.float32).reshape(()))- return dq.to(torch.bfloat16)+ return dqdq_weight = _dequantize_with_device_fallback(dequant_device=dequant_device,@@ -1168,7 +1232,7 @@ def _quantize_weight_mxfp(f"by MXFP group_size={group_size}; cannot pack.")- weight_dev = weight.to(device)+ weight_dev = weight.to(device=device, dtype=torch.float32)# Use get_quant_func (same as WrapperLinear) so that all registered MXFP# variants automatically get opt_rtn support via the QUANT_FUNC_WITH_DTYPE# registry (e.g. "opt_rtn_mx_fp4" -> quant_mx_opt_rtn, "mx_fp4" -> quant_mx).@@ -1245,7 +1309,7 @@ def _quantize_weight_nvfp4_e5m3(f"by NVFP4_E5M3 group_size={group_size}; cannot quantize.")- weight_dev = weight.to(device)+ weight_dev = weight.to(device=device, dtype=torch.float32)max_scale = 1.0if not disable_opt_rtn:grouped = reshape_pad_tensor_by_group_size(weight_dev, group_size)[0]@@ -1294,7 +1358,7 @@ def _pack_weight_nvfp4_e5m3(raise ValueError(f"NVFP4_E5M3 requires in_features divisible by group_size=16, got {in_features} for '{layer_name}'.")- weight_dev = weight.to(device)+ weight_dev = weight.to(device=device, dtype=torch.float32)max_scale = 1.0if not disable_opt_rtn:grouped = reshape_pad_tensor_by_group_size(weight_dev, group_size)[0]@@ -1307,7 +1371,7 @@ def _pack_weight_nvfp4_e5m3(# Meta-device construction avoids allocating a full weight tensor on the# accelerator (and zero-filled packed buffers) that are immediately replaced.with torch.device("meta"):- linear = torch.nn.Linear(in_features, out_features, bias=False, dtype=weight.dtype)+ linear = torch.nn.Linear(in_features, out_features, bias=False, dtype=torch.float32)linear.weight = torch.nn.Parameter(weight_dev, requires_grad=False)with torch.device("meta"):qlayer = QuantLinear(@@ -1476,7 +1540,7 @@ def _build_cross_shard_pairs_from_weight_map(Two source layouts are covered:* **FP8**: ``<layer>.weight`` and ``<layer>.weight_scale_inv``.- * **ModelOpt / TensorRT-LLM NVFP4**: a packed base tensor ``X`` and its+ * **Inkling NVFP4** (``inkling_mm_model``): a packed base tensor ``X`` and its``X.scale`` / ``X.scale2`` siblings.A *cross-shard pair* exists when the two tensors reside in **different**@@ -1502,7 +1566,7 @@ def _build_cross_shard_pairs_from_weight_map(recipient_to_donors.setdefault(weight_shard, {}).setdefault(donor_shard, []).append(scale_name)donor_shard_tensors.setdefault(donor_shard, set()).add(scale_name)- nvfp4_bases = _modelopt_nvfp4_bases_from_weight_map(weight_map)+ nvfp4_bases = _inkling_nvfp4_bases_from_weight_map(weight_map)for tensor_name, shard in weight_map.items():if tensor_name.endswith(".weight_scale_inv"):@@ -1644,11 +1708,11 @@ def _find_index_path(idx_dir: str) -> str | None:return candidates[0] if candidates else None-def _modelopt_nvfp4_bases_from_weight_map(weight_map: dict[str, str]) -> set[str]:- """Return base tensor names that use the ModelOpt / TensorRT-LLM NVFP4 layout.+def _inkling_nvfp4_bases_from_weight_map(weight_map: dict[str, str]) -> set[str]:+ """Return base tensor names that use the Inkling NVFP4 layout.A base ``X`` qualifies when the checkpoint contains ``X``, ``X.scale`` and- ``X.scale2`` (see :func:`_dequant_modelopt_nvfp4_tensors`).+ ``X.scale2`` (see :func:`_dequant_inkling_nvfp4_tensors`)."""return {name[: -len(".scale2")]@@ -1705,12 +1769,12 @@ def dequant_mx_fp8() -> torch.Tensor:"""Dequantize MXFP8 data using its per-block E8M0 scales."""original_shape = weight_fp8.shape- weight = weight_fp8.to(torch.bfloat16).reshape(-1, block_size)+ weight = weight_fp8.to(torch.float32).reshape(-1, block_size)scale = get_fp_scale(scale_e8m0).reshape(-1, 1)return (weight * scale).reshape(original_shape).to(target_dtype)-def _hydrate_and_clean_modelopt_nvfp4_aux(+def _hydrate_and_clean_inkling_nvfp4_aux(raw_tensors: dict[str, torch.Tensor],shard_path: str,*,@@ -1718,12 +1782,12 @@ def _hydrate_and_clean_modelopt_nvfp4_aux(index_dir: str | None = None,donor_shard_dir: str | None = None,) -> dict[str, torch.Tensor]:- """Resolve ModelOpt NVFP4 ``.scale`` / ``.scale2`` tensors across shards.+ """Resolve Inkling NVFP4 ``.scale`` / ``.scale2`` tensors across shards.- ModelOpt checkpoints do **not** keep a packed weight and its scales in the+ Inkling checkpoints do **not** keep a packed weight and its scales in thesame shard — e.g. ``layers.36...w13_weight`` can sit in shard 6 while its``.scale`` is in shard 3 and its ``.scale2`` in shard 5. Since model-free- processing is shard-local, :func:`_dequant_modelopt_nvfp4_tensors` would+ processing is shard-local, :func:`_dequant_inkling_nvfp4_tensors` wouldotherwise never see a complete triple and the packed weight would be leftuntouched (and reported as an already-packed, non-quantizable tensor).@@ -1753,7 +1817,7 @@ def _hydrate_and_clean_modelopt_nvfp4_aux(except Exception:return raw_tensors- bases = _modelopt_nvfp4_bases_from_weight_map(weight_map)+ bases = _inkling_nvfp4_bases_from_weight_map(weight_map)if not bases:return raw_tensors@@ -1807,7 +1871,7 @@ def _hydrate_and_clean_modelopt_nvfp4_aux(# --- 2) drop orphan auxiliary tensors owned by another shard's base ---dropped = 0for name in list(raw_tensors.keys()):- for suffix in _MODELOPT_NVFP4_AUX_SUFFIXES:+ for suffix in _INKLING_NVFP4_AUX_SUFFIXES:if not name.endswith(suffix):continuebase = name[: -len(suffix)]@@ -1827,7 +1891,7 @@ def _dequant_mxfp_tensors(device: str = "cpu",shard_name: str | None = None,) -> dict[str, torch.Tensor]:- """Dequantize llm-compressor MXFP8 / MXFP4 weight tensors to bfloat16.+ """Dequantize llm-compressor MXFP8 / MXFP4 weight tensors to float32.Detection is purely by *name* and *dtype*:@@ -1851,7 +1915,7 @@ def _dequant_mxfp_tensors(dequant_device = str(device or "cpu")shard_prefix = f"[{shard_name}] " if shard_name else ""logger.info(- f"{shard_prefix}Dequantizing MXFP tensor(s) to bfloat16 on {dequant_device}: "+ f"{shard_prefix}Dequantizing MXFP tensor(s) to float32 on {dequant_device}: "f"MXFP8={n_mxfp8}, MXFP4={n_mxfp4}, total={len(entries)}.")@@ -1868,13 +1932,13 @@ def _dequant_mxfp_tensors(weight_fp8=weight.to(dequant_device, non_blocking=True),scale_e8m0=scale.to(dequant_device, non_blocking=True),block_size=32,- target_dtype=torch.bfloat16,+ target_dtype=torch.float32,).to("cpu"),on_cpu=lambda weight=weight, scale=scale: dequant_mx_fp8(weight_fp8=weight,scale_e8m0=scale,block_size=32,- target_dtype=torch.bfloat16,+ target_dtype=torch.float32,),)else:@@ -1888,14 +1952,14 @@ def _dequant_mxfp_tensors(scale_e8m0=scale.to(dequant_device, non_blocking=True),elem_dtype="fp4_e2m1",block_size=32,- target_dtype=torch.bfloat16,+ target_dtype=torch.float32,).to("cpu"),on_cpu=lambda weight=weight, scale=scale: dequant_mxfp4(data_lp=weight.view(torch.uint8).contiguous(),scale_e8m0=scale,elem_dtype="fp4_e2m1",block_size=32,- target_dtype=torch.bfloat16,+ target_dtype=torch.float32,),)raw_tensors[f"{layer_name}.weight"] = dq_weight@@ -1926,13 +1990,17 @@ def _handle_mxfp_source_tensors(* If the target is the **same MXFP format** (``data_type='mx_fp'``, matching``bits``), the tensors are emitted directly as a passthrough — nodequantization is performed and the layer is recorded as already quantized.- * Otherwise the tensors are dequantized to ``bfloat16`` via+ * Otherwise the tensors are dequantized to ``float32`` via:func:`_dequant_mxfp_tensors` so the downstream RTN path can re-quantizethem to the requested target scheme.Returns:``(raw_tensors, passthrough_tensors, passthrough_layers)``."""+ # NVFP4 source -> MXFP4 keeps source MXFP layers untouched; they flow through as ignored tensors.+ if getattr(matcher, "nvfp_source_only", False):+ return raw_tensors, {}, []+entries = _collect_mxfp_source_entries(raw_tensors)if not entries:return raw_tensors, {}, []@@ -1963,7 +2031,7 @@ def _handle_mxfp_source_tensors(if passthrough_layers:parts.append(f"{len(passthrough_layers)} passthrough")if n_dequant:- parts.append(f"{n_dequant} dequantized to bfloat16")+ parts.append(f"{n_dequant} dequantized to float32")if source_state:parts.append(f"{len(source_state)} model_type-normalized")logger.info(f"Handling MXFP source tensor(s): {', '.join(parts)}.")@@ -1980,12 +2048,16 @@ def _dequant_fp8_tensors(*,index_dir: str | None = None,donor_shard_dir: str | None = None,+ hydrate_only: bool = False,) -> dict[str, torch.Tensor]:- """Dequantize DeepSeek-V3-style FP8 weight tensors to bfloat16.+ """Dequantize DeepSeek-V3-style FP8 weight tensors to float32.++ With ``hydrate_only`` the cross-shard ``weight_scale_inv`` tensors are hydrated but the+ FP8 weights are kept as-is (NVFP4 source -> MXFP4).Handles the **DeepSeek-V3 FP8** convention: weight dtype ``float8_e4m3fn``paired with a ``.weight_scale_inv`` tensor (per-block float32 scales, NOT- E8M0). The weights are converted to ``bfloat16`` so downstream RTN+ E8M0). The weights are converted to ``float32`` so downstream RTNquantization can proceed normally.MXFP sources are handled separately by@@ -2001,6 +2073,8 @@ def _dequant_fp8_tensors(index_dir=index_dir,donor_shard_dir=donor_shard_dir,)+ if hydrate_only:+ return raw_tensorsquant_entries: list[tuple[str, str]] = []for name, tensor in raw_tensors.items():@@ -2020,9 +2094,7 @@ def _dequant_fp8_tensors(dequant_device = str(device or "cpu")shard_prefix = f"[{shard_name}] " if shard_name else ""- logger.info(- f"{shard_prefix}Dequantizing {len(quant_entries)} FP8 weight tensor(s) to bfloat16 on {dequant_device}."- )+ logger.info(f"{shard_prefix}Dequantizing {len(quant_entries)} FP8 weight tensor(s) to float32 on {dequant_device}.")for weight_name, scale_name in quant_entries:weight = raw_tensors[weight_name]@@ -2039,8 +2111,11 @@ def _dequant_fp8_tensors(weight.to(dequant_device, non_blocking=True),scale.to(dequant_device, non_blocking=True),block_size=block_size,+ output_dtype=torch.float32,).to("cpu"),- on_cpu=lambda weight=weight, scale=scale: _dequant_fp8_linear_weight(weight, scale, block_size=block_size),+ on_cpu=lambda weight=weight, scale=scale: _dequant_fp8_linear_weight(+ weight, scale, block_size=block_size, output_dtype=torch.float32+ ),)# ``weight`` still aliases the original FP8 storage after the dict slot# was rebound; release it (and the scale) before the next iteration.@@ -2114,6 +2189,8 @@ def _process_shard(raw_tensors = {name: f.get_tensor(name) for name in f.keys()}raw_tensors = split_fused_expert_tensors(raw_tensors, model_type=model_type)+ # Dequantized sources are FP32 in-flight; those left high-precision are saved as BF16.+ source_fp32_names = {name for name, t in raw_tensors.items() if t.dtype == torch.float32}# Hydrate cross-shard FP8 weight_scale_inv tensors *before* any# preprocessing below. Otherwise a weight whose scale lives in a sibling@@ -2141,10 +2218,11 @@ def _process_shard((_fused_expert_layer_name(name)if _is_moe_fused_expert_weight(name, tensor)- else (name.rsplit(".", 1)[0] if name.endswith(".weight") else name)+ else (name.rsplit(".", 1)[0] if name.endswith((".weight", ".weight_packed")) else name))for name, tensor in raw_tensors.items()- if (name.endswith(".weight") and tensor.dim() > 1) or _is_moe_fused_expert_weight(name, tensor)+ if (name.endswith((".weight", ".weight_packed")) and tensor.dim() > 1)+ or _is_moe_fused_expert_weight(name, tensor)))@@ -2175,18 +2253,31 @@ def _process_shard(# 1.5) normalize legacy NVFP4 names to llm-compressor naming.raw_tensors, _converted_nvfp4_layers = _normalize_nvfp4_source_tensors(raw_tensors, shard_name=shard_name)- # 1.6) ModelOpt / TensorRT-LLM NVFP4 sources store the packed weight under a+ # 1.6) Inkling (inkling_mm_model) NVFP4 sources store the packed weight under a# bare name (no ``.weight`` suffix) with ``.scale`` / ``.scale2`` siblings,# which are frequently placed in *different* shards. Hydrate them from the# index first, then dequantize and drop the stale metadata tensors.- raw_tensors = _hydrate_and_clean_modelopt_nvfp4_aux(+ raw_tensors = _hydrate_and_clean_inkling_nvfp4_aux(raw_tensors,shard_path,shard_name=shard_name,index_dir=index_dir,donor_shard_dir=donor_shard_dir,)- raw_tensors = _dequant_modelopt_nvfp4_tensors(raw_tensors, device=device, shard_name=shard_name)+ # Hydrated Inkling scales are source tensors; keep their dtype if they survive (failed dequant).+ source_fp32_names.update(+ n for n, t in raw_tensors.items() if t.dtype == torch.float32 and n.endswith((".scale", ".scale2"))+ )++ # NVFP4 source -> MXFP4: only NVFP4 source layers (plus layer_config matches) are quantized;+ # all other layers are treated as ignored. Must run before NVFP4 dequantization.+ if matcher.nvfp_source_only:+ nvfp_layers = _collect_nvfp4_source_layers(raw_tensors)+ matcher.restrict_to_layers(nvfp_layers)+ shard_prefix = f"[{shard_name}] " if shard_name else ""+ logger.info(f"{shard_prefix}Detected {len(nvfp_layers)} NVFP4 source layer(s) to convert.")++ raw_tensors = _dequant_inkling_nvfp4_tensors(raw_tensors, device=device, shard_name=shard_name)# 1.5) model-type-specific low-precision dequantization (e.g. kimi_k25 INT4)raw_tensors = handle_model_type_low_precision_source_tensors(@@ -2210,7 +2301,7 @@ def _process_shard(quantized_layers.extend(passthrough_layers)# 3) NVFP4 passthrough for layers already stored in packed format (or- # dequant to bfloat16 when the target scheme requires re-quantization).+ # dequant to float32 when the target scheme requires re-quantization).raw_tensors, nvfp_passthrough_tensors, nvfp_passthrough_layers = _handle_nvfp4_source_tensors(raw_tensors,matcher,@@ -2228,6 +2319,7 @@ def _process_shard(shard_path=shard_path,index_dir=index_dir,donor_shard_dir=donor_shard_dir,+ hydrate_only=matcher.nvfp_source_only,)raw_tensors.update(preserved_tensors)# ``raw_tensors`` now owns these; drop the duplicate references.@@ -2243,6 +2335,15 @@ def _process_shard(quantize_func,disable_opt_rtn,)+ # Cross-shard hydrated FP8 scale_inv tensors are FP32 sources, not dequantized weights.+ if (+ tensor.dtype == torch.float32+ and tensor_name not in source_fp32_names+ and not tensor_name.endswith(".weight_scale_inv")+ ):+ kept = out_dict.get(tensor_name)+ if kept is not None and kept.dtype == torch.float32:+ out_dict[tensor_name] = kept.to(torch.bfloat16)output_tensors.update(out_dict)if q_layer:quantized_layers.append(q_layer)@@ -2598,40 +2699,29 @@ def _build_nvfp4_e5m3_quantization_config(ignored_layers: list[str]) -> dict:return qconfig-# TODO: Remove when the issue is fixed-# https://github.com/vllm-project/llm-compressor/issues/3069-def _add_routed_experts_if_moe(targets: list[str], layer_names: list[str]) -> list[str]:- """Append ``"RoutedExperts"`` for routed expert layouts that llm-compressor misses.+def _is_mxfp_ct_group(group: dict) -> bool:+ weights = group.get("weights") or {}+ return (+ weights.get("type") == "float"+ and weights.get("num_bits") in _SUPPORTED_MXFP_BITS+ and (weights.get("group_size") == 32)+ )- The generic ``"Linear"`` target does not cover all expert layouts. Keep the- safe canonical cases alone (``mlp.experts.*.gate_proj|up_proj|down_proj``),- but trigger for non-standard expert paths and for any routed- expert structure that lives outside the standard ``mlp.experts`` naming.- """- if "RoutedExperts" in targets:- return targets-- def _projection_name(name: str) -> str:- # Eg. '...experts.0.gate_proj.weight' -> ['...', 'experts', '0', 'gate_proj', 'weight']- parts = name.split(".")- if len(parts) >= 2:- return parts[-2]- return parts[-1]-- for name in layer_names:- lname = name.lower()- if "block_sparse_moe" in lname or ".moe." in lname or ".feed_forward." in lname:- return list(targets) + ["RoutedExperts"]-- if ".experts." not in lname:- continue- projection = _projection_name(name)- if projection in {"gate_proj", "up_proj", "down_proj"}:- continue- return list(targets) + ["RoutedExperts"]+def _is_catch_all_group(group: dict) -> bool:+ # The default group may be ["Linear", "RoutedExperts"] after add_routed_experts_if_moe.+ return "Linear" in (group.get("targets") or [])- return targets++def _explicit_mxfp_targets(qconfig: dict, quantized_layers: list[str]) -> dict:+ """Replace the ``"Linear"`` catch-all of MXFP groups with the quantized layer names (+ ``RoutedExperts``)."""+ groups = qconfig.get("config_groups") or {}+ covered = {t for g in groups.values() if not _is_catch_all_group(g) for t in g.get("targets") or []}+ layers = [name for name in dict.fromkeys(quantized_layers) if name not in covered]+ for group in groups.values():+ if _is_catch_all_group(group) and _is_mxfp_ct_group(group) and layers:+ group["targets"] = _add_routed_experts_if_moe(list(layers), layers)+ return qconfigdef _get_mxfp_group_scheme_and_format(@@ -3110,8 +3200,13 @@ def _build_quantization_config(ignored_layers: list[str],block_name_to_quantize: str | None = None,format: str = "auto_round",+ source_quant_config: dict | None = None,) -> dict:- """Build a quantization_config dict compatible with auto-round format."""+ """Build a quantization_config dict compatible with auto-round format.++ ``source_quant_config`` (NVFP4 source -> MXFP4 only) is the source checkpoint's quantization config;+ its NVFP4 groups are rewritten to MXFP4 and every other group (e.g. FP8) is kept as-is.+ """# MXFP (mx_fp) supports two output styles depending on *format*:# - "llm_compressor" → compressed-tensors / llm-compressor style config# (produced by _build_mxfp_quantization_config).@@ -3159,7 +3254,7 @@ def _build_quantization_config(qconfig["config_groups"]["group_0"]["targets"] = targetsqconfig["format"] = "nvfp4-pack-quantized"qconfig.update(_get_llm_compressor_metadata())- return qconfig+ return _ignore_fused_experts_for_vllm(qconfig, ignored_layers, quantized_layers)if is_mx_fp(data_type) or (is_fp_default and _layer_config_has_mxfp(layer_config)):if format in ("auto_round", "auto_round:auto_gptq"):return _build_mxfp_autoround_quantization_config(@@ -3169,12 +3264,21 @@ def _build_quantization_config(layer_config=layer_config,block_name_to_quantize=block_name_to_quantize,)- return _build_mxfp_quantization_config(+ qconfig = _build_mxfp_quantization_config(default_scheme=default_scheme,quantized_layers=quantized_layers,ignored_layers=ignored_layers,layer_config=layer_config,)+ source_ct = _source_config_to_ct(source_quant_config)+ if source_ct:+ qconfig = _rewrite_nvfp4_groups_as_mxfp(qconfig, source_ct, ignored_layers, quantized_layers)+ # Only a compressed-tensors source vouches for its "Linear" catch-all; otherwise list the layers for vLLM.+ if source_quant_config is not None and str(source_quant_config.get("quant_method") or "").lower() != (+ "compressed-tensors"+ ):+ qconfig = _explicit_mxfp_targets(qconfig, quantized_layers)+ return _ignore_fused_experts_for_vllm(qconfig, ignored_layers, quantized_layers)# When the default is full-precision (BF16/FP16) but INT layers were# quantized via layer_config, derive the dominant INT scheme and use it as@@ -3261,6 +3365,349 @@ def _build_quantization_config(return qconfig+# ---------------------------------------------------------------------------+# NVFP4 source -> MXFP4: rewrite the source quantization config (mixed-precision sources)+# ---------------------------------------------------------------------------+# Overall flow (scheme MXFP4 + model-free on a source detected as NVFP4, see _detect_nvfp4_source):+# 1. Shard processing (_process_shard): layers stored as NVFP4 are detected from tensor names and+# converted to MXFP4; layers matched by layer_config are quantized as configured; every other+# layer (BF16, FP8, kv-cache scales, ...) is copied as-is.+# 2. Config (_build_quantization_config, llm_compressor format only):+# a. _load_source_quant_config: config.json quantization_config, else hf_quant_config.json & co.+# b. _source_config_to_ct: compressed-tensors is used as-is; ModelOpt / Quark are converted to+# compressed-tensors; anything else is warned about and skipped (plain MXFP4 config).+# c. _rewrite_nvfp4_groups_as_mxfp: NVFP4 groups get the MXFP4 scheme (same targets); all other+# groups, ignore and kv_cache_scheme are kept, so FP8 layers stay described as FP8.++# Standalone quantization metadata files, read when config.json has no quantization_config.+_SOURCE_QUANT_CONFIG_FILES: tuple[str, ...] = ("hf_quant_config.json", "quantization_config.json", "quant_config.json")+_FP8_STATIC_TENSOR_ARGS: dict = {+ "num_bits": 8,+ "type": "float",+ "symmetric": True,+ "dynamic": False,+ "strategy": "tensor",+}+_FP8_DYNAMIC_TOKEN_ARGS: dict = {+ "num_bits": 8,+ "type": "float",+ "symmetric": True,+ "dynamic": True,+ "strategy": "token",+}+# Placeholder for NVFP4 groups converted from ModelOpt / Quark; they are replaced by MXFP4 anyway.+_NVFP4_CT_GROUP: dict = {+ "format": "nvfp4-pack-quantized",+ "weights": {"num_bits": 4, "type": "float", "symmetric": True, "strategy": "tensor_group", "group_size": 16},+}+++def _load_source_quant_config(source_dir: str | None, quantization_config: dict | None) -> dict:+ """Return the source quantization config, falling back to standalone metadata files."""+ if quantization_config:+ return quantization_config+ for fname in _SOURCE_QUANT_CONFIG_FILES:+ path = os.path.join(source_dir, fname) if source_dir else ""+ if not path or not os.path.isfile(path):+ continue+ try:+ with open(path) as f:+ data = json.load(f)+ except (OSError, ValueError) as e:+ logger.warning(f"Could not read source quantization metadata '{fname}': {e}; skipping it.")+ continue+ if not isinstance(data, dict) or not data:+ continue+ # ModelOpt hf_quant_config.json nests the settings under "quantization".+ if isinstance(data.get("quantization"), dict):+ return {**data["quantization"], "producer": data.get("producer"), "quant_method": "modelopt"}+ return data+ return {}+++def _glob_to_ct_target(pattern: str) -> str:+ """Convert a ModelOpt / Quark fnmatch module pattern into a compressed-tensors target."""+ from fnmatch import translate++ return f"re:{translate(pattern)}" if any(c in pattern for c in "*?[") else pattern+++def _ct_target_matches(target: Any, layer_name: str) -> bool:+ if not isinstance(target, str):+ return False+ if target.startswith("re:"):+ try:+ return re.match(target[3:], layer_name) is not None+ except re.error:+ return False+ return target == layer_name+++def _lookup_ct_group(ct_config: dict, layer_name: str) -> dict | None:+ """Return the compressed-tensors config group of *layer_name* (name match beats ``Linear`` class match)."""+ if any(_ct_target_matches(t, layer_name) for t in ct_config.get("ignore") or []):+ return None+ class_match = None+ for group in ct_config["config_groups"].values():+ for target in group.get("targets") or []:+ if _ct_target_matches(target, layer_name):+ return group+ if class_match is None and target == "Linear":+ class_match = group+ return class_match+++def _is_nvfp4_ct_group(group: dict) -> bool:+ weights = group.get("weights") or {}+ return weights.get("num_bits") == 4 and weights.get("type") == "float" and weights.get("group_size") == 16+++def _modelopt_algo_to_ct_group(algo: Any) -> dict | None:+ algo = str(algo or "").upper()+ if algo == "NVFP4":+ return copy.deepcopy(_NVFP4_CT_GROUP)+ if algo == "FP8":+ weights, acts = {**_FP8_STATIC_TENSOR_ARGS}, {**_FP8_STATIC_TENSOR_ARGS}+ elif algo == "FP8_PER_CHANNEL_PER_TOKEN":+ weights, acts = {**_FP8_STATIC_TENSOR_ARGS, "strategy": "channel"}, {**_FP8_DYNAMIC_TOKEN_ARGS}+ elif algo == "FP8_PB_WO":+ weights, acts = {**_FP8_STATIC_TENSOR_ARGS, "strategy": "block", "block_structure": [128, 128]}, None+ else:+ return None+ return {"format": "float-quantized", "weights": weights, "input_activations": acts}+++def _modelopt_to_ct_config(qc: dict) -> dict | None:+ """Convert a ModelOpt quantization config (``quant_algo`` / ``quantized_layers``) to compressed-tensors."""+ quantized_layers = qc.get("quantized_layers")+ if isinstance(quantized_layers, dict) and quantized_layers:+ targets_by_algo: dict[str, list[str]] = {}+ for layer_name, entry in quantized_layers.items():+ algo = str((entry or {}).get("quant_algo") or "").upper()+ targets_by_algo.setdefault(algo, []).append(layer_name)+ else:+ targets_by_algo = {str(qc.get("quant_algo") or "").upper(): ["Linear"]}++ config_groups: dict[str, dict] = {}+ for algo, targets in targets_by_algo.items():+ group = _modelopt_algo_to_ct_group(algo)+ if group is None:+ logger.warning(f"ModelOpt quant_algo '{algo}' is not supported; skipping the source quantization config.")+ return None+ config_groups[f"group_{len(config_groups)}"] = {**group, "targets": targets}++ ct_config = {+ "quant_method": "compressed-tensors",+ "config_groups": config_groups,+ # hf_quant_config.json uses "exclude_modules"; the config.json copy uses "ignore".+ "ignore": [_glob_to_ct_target(p) for p in qc.get("exclude_modules") or qc.get("ignore") or []],+ }+ kv_scheme = qc.get("kv_cache_scheme")+ if str(qc.get("kv_cache_quant_algo") or "").upper() == "FP8" or (+ isinstance(kv_scheme, dict) and kv_scheme.get("num_bits") == 8 and kv_scheme.get("type") == "float"+ ):+ ct_config["kv_cache_scheme"] = {**_FP8_STATIC_TENSOR_ARGS}+ return ct_config+++def _quark_fp8_args(spec: dict) -> dict | None:+ dynamic = bool(spec.get("is_dynamic"))+ qscheme = spec.get("qscheme")+ if qscheme == "per_tensor":+ strategy = {"strategy": "tensor"}+ elif qscheme == "per_channel":+ strategy = {"strategy": "token" if dynamic else "channel"}+ elif qscheme == "per_group":+ strategy = {"strategy": "group", "group_size": spec.get("group_size")}+ else:+ return None+ return {"num_bits": 8, "type": "float", "symmetric": True, "dynamic": dynamic, **strategy}+++def _quark_layer_to_ct_group(layer_cfg: Any) -> tuple[bool, dict | None]:+ """Return ``(supported, group)`` for a Quark layer config; ``group`` is None when unquantized."""+ weight = layer_cfg.get("weight") if isinstance(layer_cfg, dict) else None+ if not weight:+ return True, None+ if isinstance(weight, list):+ # Quark NVFP4: [fp4 per_group(16), fp8 per_tensor global scale].+ is_nvfp4 = len(weight) == 2 and weight[0].get("dtype") == "fp4" and weight[0].get("group_size") == 16+ return (True, copy.deepcopy(_NVFP4_CT_GROUP)) if is_nvfp4 else (False, None)+ weights = _quark_fp8_args(weight) if weight.get("dtype") == "fp8_e4m3" else None+ if weights is None:+ return False, None+ group = {"format": "float-quantized", "weights": weights}+ input_cfg = layer_cfg.get("input_tensors")+ if input_cfg:+ acts = (+ _quark_fp8_args(input_cfg) if isinstance(input_cfg, dict) and input_cfg.get("dtype") == "fp8_e4m3" else None+ )+ if acts is None:+ return False, None+ group["input_activations"] = acts+ return True, group+++def _quark_to_ct_config(qc: dict) -> dict | None:+ """Convert a Quark quantization config to compressed-tensors."""+ if qc.get("layer_type_quant_config"):+ logger.warning("Quark 'layer_type_quant_config' is not supported; skipping the source quantization config.")+ return None+ entries = [(_glob_to_ct_target(p), cfg) for p, cfg in (qc.get("layer_quant_config") or {}).items()]+ entries.append(("Linear", qc.get("global_quant_config")))++ config_groups: dict[str, dict] = {}+ ignore = [_glob_to_ct_target(p) for p in qc.get("exclude") or []]+ for target, layer_cfg in entries:+ supported, group = _quark_layer_to_ct_group(layer_cfg)+ if not supported:+ logger.warning(+ f"Quark quantization config for '{target}' is not supported; skipping the source quantization config."+ )+ return None+ if group is None:+ if target != "Linear":+ ignore.append(target)+ continue+ config_groups[f"group_{len(config_groups)}"] = {**group, "targets": [target]}++ ct_config = {"quant_method": "compressed-tensors", "config_groups": config_groups, "ignore": ignore}+ if (qc.get("export") or {}).get("kv_cache_group"):+ ct_config["kv_cache_scheme"] = {**_FP8_STATIC_TENSOR_ARGS}+ return ct_config+++def _source_config_to_ct(qc: dict | None) -> dict | None:+ """Return the source quantization config in compressed-tensors form, or None if not understood."""+ if not qc:+ return None+ method = str(qc.get("quant_method") or "").lower()+ producer = qc.get("producer")+ if method == "compressed-tensors" and isinstance(qc.get("config_groups"), dict):+ return copy.deepcopy(qc)+ if method == "modelopt" or (isinstance(producer, dict) and producer.get("name") == "modelopt"):+ return _modelopt_to_ct_config(qc)+ if method == "quark":+ return _quark_to_ct_config(qc)+ logger.warning(f"Source quantization config (quant_method='{method}') is not understood; skipping it.")+ return None+++def _detect_nvfp4_source(model_name_or_path: str, source_quant_config: dict | None) -> str | None:+ """Return why the source checkpoint looks NVFP4-quantized, or None.++ Relies on the model name and the source quantization config (config.json / hf_quant_config.json & co.)+ because the first shards may not contain any NVFP4 tensor.+ """+ if "nvfp4" in os.path.basename(str(model_name_or_path).rstrip("/\\")).lower():+ return f"model name '{model_name_or_path}'"+ qc = source_quant_config or {}+ # Covers compressed-tensors "nvfp4-pack-quantized" and ModelOpt quant_algo "NVFP4".+ if "nvfp4" in json.dumps(qc, default=str).lower():+ return "source quantization config"+ groups = qc.get("config_groups") if isinstance(qc.get("config_groups"), dict) else {}+ if any(isinstance(g, dict) and _is_nvfp4_ct_group(g) for g in groups.values()):+ return "source quantization config"+ if str(qc.get("quant_method") or "").lower() == "quark":+ layer_cfgs = [qc.get("global_quant_config"), *(qc.get("layer_quant_config") or {}).values()]+ if any(_is_nvfp4_ct_group(_quark_layer_to_ct_group(cfg)[1] or {}) for cfg in layer_cfgs):+ return "source quantization config"+ return None+++def _detect_fp8_source(model_name_or_path: str, source_quant_config: dict | None) -> str | None:+ """Return why the source checkpoint looks FP8-quantized (DeepSeek-V3 style), or None."""+ qc = source_quant_config or {}+ if (+ qc.get("quant_method") == "fp8"+ or qc.get("quantization_type") == "fp8"+ or str(qc.get("fmt") or "").startswith("e4m3")+ ):+ return "source quantization config"+ return None+++# Source dtype name -> detector(model_name_or_path, source_quant_config) returning a reason or None.+_SOURCE_DTYPE_DETECTORS = {+ "nvfp4": _detect_nvfp4_source,+ "fp8": _detect_fp8_source,+}+++def detect_model_dtypes(model_name_or_path: str, source_quant_config: dict | None) -> dict[str, str]:+ """Return ``{source_dtype: reason}`` for every low-precision dtype detected in the source checkpoint."""+ detected = {}+ for dtype_name, detector in _SOURCE_DTYPE_DETECTORS.items():+ reason = detector(model_name_or_path, source_quant_config)+ if reason is not None:+ detected[dtype_name] = reason+ return detected+++def _rewrite_nvfp4_groups_as_mxfp(+ mxfp_qconfig: dict, source_ct: dict, ignored_layers: list[str], quantized_layers: list[str] | None = None+) -> dict:+ """Replace the NVFP4 groups of *source_ct* with the MXFP4 scheme and keep every other group as-is.++ Quantized layers outside the source NVFP4 groups (selected via layer_config) get extra groups.+ """+ mxfp_groups = mxfp_qconfig["config_groups"]+ # The default (catch-all) group targets "Linear"; layer_config override groups list explicit layers.+ default_name = next((n for n, g in mxfp_groups.items() if _is_catch_all_group(g)), next(iter(mxfp_groups)))+ mxfp_format = mxfp_qconfig.get("format")++ def _with_format(name: str) -> dict:+ group = copy.deepcopy(mxfp_groups[name])+ group["format"] = group.get("format") or mxfp_format+ return group++ source_format = source_ct.get("format")+ config_groups: dict[str, dict] = {}+ for name, group in source_ct["config_groups"].items():+ if _is_nvfp4_ct_group(group):+ group = {**_with_format(default_name), "targets": group.get("targets")}+ elif not group.get("format") and source_format != "mixed-precision":+ group = {**group, "format": source_format}+ config_groups[name] = group+ if not any(_is_nvfp4_ct_group(g) for g in source_ct["config_groups"].values()):+ logger.warning("Source quantization config has no NVFP4 group; writing a plain MXFP4 config.")+ return mxfp_qconfig++ extras: dict[str, list[str]] = {}+ for layer_name in dict.fromkeys(quantized_layers or []):+ if _is_nvfp4_ct_group(_lookup_ct_group(source_ct, layer_name) or {}):+ continue+ owner = next(+ (n for n, g in mxfp_groups.items() if n != default_name and layer_name in (g.get("targets") or [])),+ default_name,+ )+ extras.setdefault(owner, []).append(layer_name)+ for owner, layers in extras.items():+ name = f"group_{len(config_groups)}"+ while name in config_groups:+ name += "_extra"+ config_groups[name] = {**_with_format(owner), "targets": layers}+ extra_layers = {layer for layers in extras.values() for layer in layers}++ # NVFP4 source layers that were not converted (e.g. dequantized lm_head) are now unquantized.+ ignore = [p for p in source_ct.get("ignore") or [] if p not in extra_layers]+ for layer_name in dict.fromkeys(ignored_layers):+ group = _lookup_ct_group(source_ct, layer_name)+ if group is not None and _is_nvfp4_ct_group(group) and layer_name not in ignore:+ ignore.append(layer_name)++ formats = {group.get("format") for group in config_groups.values()}+ qconfig = {+ **mxfp_qconfig,+ "config_groups": config_groups,+ "ignore": ignore,+ "format": formats.pop() if len(formats) == 1 else "mixed-precision",+ }+ if source_ct.get("kv_cache_scheme"):+ qconfig["kv_cache_scheme"] = copy.deepcopy(source_ct["kv_cache_scheme"])+ return qconfig++# ---------------------------------------------------------------------------# Scheme Validation and Override Resolution# ---------------------------------------------------------------------------@@ -3679,7 +4126,7 @@ def _dequant_kimi_k25_int4_tensors(shard_name: str | None = None,dequantize_with_device_fallback: Callable[..., torch.Tensor] | None = None,) -> dict[str, torch.Tensor]:- """Dequantize kimi_k25-style INT4 packed weights to bfloat16 ``.weight``."""+ """Dequantize kimi_k25-style INT4 packed weights to float32 ``.weight``."""if dequantize_with_device_fallback is None:raise ValueError("dequantize_with_device_fallback callback is required")@@ -3691,7 +4138,7 @@ def _dequant_kimi_k25_int4_tensors(dequant_device = str(device or "cpu")shard_prefix = f"[{shard_name}] " if shard_name else ""logger.info(- f"{shard_prefix}Dequantizing kimi_k25 INT4 tensor(s) to bfloat16 on {dequant_device}: total={len(entries)}."+ f"{shard_prefix}Dequantizing kimi_k25 INT4 tensor(s) to float32 on {dequant_device}: total={len(entries)}.")for layer_name, packed_key, scale_key, zp_key in entries:@@ -3754,7 +4201,7 @@ def _dequant_kimi_k25_int4_tensors(q = q - 8sc = sc.repeat_interleave(group_size, dim=1)[:, : q.shape[1]]- return (q.to(torch.float32) * sc).to(torch.bfloat16)+ return q.to(torch.float32) * scdq_weight = dequantize_with_device_fallback(dequant_device=dequant_device,diff --git a/auto_round/utils/weight_handler.py b/auto_round/utils/weight_handler.pyindex c14aad463..062f58229 100644--- a/auto_round/utils/weight_handler.py+++ b/auto_round/utils/weight_handler.py@@ -483,10 +483,16 @@ def _pad_block_fp8_weight_naive(@with_thread_limits()def _dequant_fp8_linear_weight(- weight: torch.Tensor, weight_scale: torch.Tensor, block_size: list | None = None, data_type: str | None = None+ weight: torch.Tensor,+ weight_scale: torch.Tensor,+ block_size: list | None = None,+ data_type: str | None = None,+ output_dtype: torch.dtype = torch.bfloat16,) -> torch.Tensor:- """Core dequantization logic for block-wise FP8 weights."""- dtype = torch.bfloat16+ """Core dequantization logic for block-wise FP8 weights.++ ``data_type`` is the source quantization data type; ``output_dtype`` is the torch dtype of the result.+ """if weight_scale is None:return weight@@ -500,7 +506,7 @@ def _dequant_fp8_linear_weight(weight_scale = weight_scale.view(-1, 1)elif weight_scale.numel() == weight.shape[-1]:weight_scale = weight_scale.view(1, -1)- return weight.to(dtype) * weight_scale.to(dtype)+ return weight.to(output_dtype) * weight_scale.to(output_dtype)assert len(block_size) == 2@@ -514,14 +520,14 @@ def _dequant_fp8_linear_weight(weight_scale_m, weight_scale_n = weight_scale.shapeweight_scale = weight_scale.view(weight_scale_m, 1, weight_scale_n, 1)weight = weight.view(weight_scale_m, block_size_m, weight_scale_n, block_size_n)- dequant_weight = weight.to(dtype) * weight_scale.to(dtype)+ dequant_weight = weight.to(output_dtype) * weight_scale.to(output_dtype)dequant_weight = dequant_weight.view(weight_scale_m * block_size_m, weight_scale_n * block_size_n)keep_first_dim = Falseelif weight_shape_len == 3:fd, weight_scale_m, weight_scale_n = weight_scale.shapeweight_scale = weight_scale.view(fd, weight_scale_m, 1, weight_scale_n, 1)weight = weight.view(fd, weight_scale_m, block_size_m, weight_scale_n, block_size_n)- dequant_weight = weight.to(dtype) * weight_scale.to(dtype)+ dequant_weight = weight.to(output_dtype) * weight_scale.to(output_dtype)dequant_weight = dequant_weight.view(fd, weight_scale_m * block_size_m, weight_scale_n * block_size_n)keep_first_dim = Trueelse:diff --git a/docs/step_by_step.md b/docs/step_by_step.mdindex 86349fda5..db708ac17 100755--- a/docs/step_by_step.md+++ b/docs/step_by_step.md@@ -704,32 +704,21 @@ Model-free mode performs calibration-free WOQ quantization **without loading the<details><summary>Model-free Parallelism Benchmarks (Rounded Minutes)</summary>-Runtime normalization: all `mm:ss` values are rounded up to the next full minute. For example, `4:20 -> 5`, `15:45 -> 16`, `9:07 -> 10`, `7:29 -> 8`, `4:09 -> 5`.--| Model | Device | Scheme | Parallelism | Peak Memory (G) | Runtime (min, rounded up) |-|---|---|---|---:|---:|---:|-| Qwen/Qwen3-Next-80B-A3B-Instruct | A100 | W4A16 | 1 | 2 | N/A |-| Qwen/Qwen3-Next-80B-A3B-Instruct | A100 | W4A16 | 10 | 8 | 7 |-| Qwen3-235B-A22B-Instruct-2507 | A100 | W4A16 | 1 | 2 | 17 |-| Qwen3-235B-A22B-Instruct-2507 | A100 | W4A16 | 10 | 8 | 5 |-| zai-org/GLM-5.2 | B200 | MXFP4-Mixed | 1 | 2 | 60 |-| zai-org/GLM-5.2 | B200 | MXFP4-Mixed | 10 | 27 | 16 |-| zai-org/GLM-5.2 | B200 | W4A16 | 1 | 3 | 30 |-| zai-org/GLM-5.2 | B200 | W4A16 | 10 | 16 | 10 |-| zai-org/GLM-5.2 | B200 | W4A16 | 20 | 32 | 8 |-| MiniMaxAI/MiniMax-M2.7 (FP8) | B200 | W4A16 | 1 | 2 | 18 |-| MiniMaxAI/MiniMax-M2.7 (FP8) | B200 | W4A16 | 10 | 10 | 5 |-| deepseek-ai/DeepSeek-V4-Pro (MXFP) | B200 | W4A16 | 1 | 6 | 80 |-| deepseek-ai/DeepSeek-V4-Pro (MXFP) | B200 | W4A16 | 10 | 50 | 13 |--| Model | Scheme | Comparison | Runtime Change (min) | Speedup | Time Saved | Peak Memory Change |-|---|---|---|---|---:|---:|---|-| Qwen3-235B | W4A16 | Parallelism 1 -> 10 | 17 -> 5 | 3.40x | 70.6% | 2G -> 8G |-| GLM-5.2 | MXFP4-Mixed | Parallelism 1 -> 10 | 60 -> 16 | 3.75x | 73.3% | 2G -> 27G |-| GLM-5.2 | W4A16 | Parallelism 1 -> 10 | 30 -> 10 | 3.00x | 66.7% | 3G -> 16G |-| GLM-5.2 | W4A16 | Parallelism 1 -> 20 | 30 -> 8 | 3.75x | 73.3% | 3G -> 32G |-| MiniMax-M2.7 | W4A16 | Parallelism 1 -> 10 | 18 -> 5 | 3.60x | 72.2% | 2G -> 10G |-| DeepSeek-V4-Pro | W4A16 | Parallelism 1 -> 10 | 80 -> 13 | 6.15x | 83.8% | 6G -> 50G |+Model-free mode quantizes multiple weight shards in parallel worker processes. Set the parallelism with the `AR_MODEL_FREE_SHARD_PARALLELISM` environment variable (see the [Environment Variables Guide](./environments.md#ar_model_free_shard_parallelism)):+- **Default (unset)**: `shard_count // 4`, min 1, max 10. For example, 8 shards use 2 workers and 40+ shards use 10 workers.+- **Manual**: e.g. `export AR_MODEL_FREE_SHARD_PARALLELISM=20`. The effective value never exceeds the shard count.+- Higher parallelism is usually faster but increases peak GPU (and host) memory; lower it when resources are tight.++Runtimes are rounded up to the next full minute.++| Model | Device | Scheme | Parallelism | Runtime (min) | Speedup | Peak Memory (G) |+|---|---|---|---|---|---:|---|+| Qwen3-Next-80B-A3B-Instruct | A100 | W4A16 | 1 -> 10 | N/A -> 7 | N/A | 2 -> 8 |+| Qwen3-235B-A22B-Instruct-2507 | A100 | W4A16 | 1 -> 10 | 17 -> 5 | 3.40x | 2 -> 8 |+| GLM-5.2 | B200 | MXFP4-Mixed | 1 -> 10 | 60 -> 16 | 3.75x | 2 -> 27 |+| GLM-5.2 | B200 | W4A16 | 1 -> 10 / 20 | 30 -> 10 / 8 | 3.00x / 3.75x | 3 -> 16 / 32 |+| MiniMax-M2.7 (FP8) | B200 | W4A16 | 1 -> 10 | 18 -> 5 | 3.60x | 2 -> 10 |+| DeepSeek-V4-Pro (MXFP) | B200 | W4A16 | 1 -> 10 | 80 -> 13 | 6.15x | 6 -> 50 |Key takeaway: model-free quantization usually gets about `3x-6x` runtime speedup with higher parallelism, while peak memory usage increases significantly.@@ -766,6 +755,43 @@ You can also pass a custom `QuantizationScheme(bits=N, group_size=G, sym=True/FaSchemes that require special packing kernels (`W3A16`, `FPW8A16`, `BF16`, `MXINT4`, `NVFP4`, `FP8_BLOCK`, `FP8_STATIC`, `INT8_W8A8`, `GGUF:*`, ...) are **not** supported in model-free mode and will raise `ValueError`. Use the regular AutoRound flow for those.+**NVFP4 source checkpoints → MXFP4**++Goal: convert published NVFP4 checkpoints to MXFP4 to serve 4-bit LLMs with high throughput on Intel GPUs, which support MXFP4 but have no native NVFP4 acceleration path.++Supported source tensor layouts:++| Source layout | Tensors | Status |+| --- | --- | --- |+| compressed-tensors (NVFP4) | `.weight_packed` + `.weight_scale` + `.weight_global_scale` + `.input_global_scale` | Supported |+| ModelOpt / Quark (NVFP4) | `.weight` (uint8) + `.weight_scale` (FP8) + `.weight_scale_2` + `.input_scale` | Supported |+| `inkling_mm_model` type | `X` (uint8) + `X.scale` + `X.scale2` (may live in different shards) | Supported |++Status of commonly requested checkpoints:++| Model | Source format | Status | gsm8k strict-match (MXFP4) | gsm8k flexible-extract (MXFP4) |+| --- | --- | --- | --- | --- |+| `RedHatAI/Qwen3.8-27B-NVFP4` | compressed-tensors, NVFP4 + FP8 | Verified | 0.9636 ± 0.0052 | 0.9636 |+| `nvidia/Qwen3.8-27B-NVFP4` | ModelOpt, NVFP4 + FP8 | Verified | 0.9644 ± 0.0051 | 0.9659 |+| `RedHatAI/Qwen3.6-35B-A3B-NVFP4` | compressed-tensors, NVFP4 | Verified | 0.9575 ± 0.0056 | 0.9575 |+| `amd/MiniMax-M2.7-NVFP4` | Quark, NVFP4 | Verified | 0.9568 ± 0.0056 | 0.9591 |+| `nvidia/MiniMax-M2.7-NVFP4` | ModelOpt, NVFP4 | Verified | 0.9477 ± 0.0061 | 0.9522 |+| `thinkingmachines/Inkling-Small-NVFP4` | Special format (`inkling_mm_model`), NVFP4 | Verified | 0.9477 ± 0.0061 | 0.9545 |++gsm8k is evaluated with 5-shot on the converted MXFP4 models, with thinking disabled. These results only demonstrate that the converted models work correctly; the numbers are not guaranteed to be accurate.++Running `--scheme MXFP4 --model_free` on an NVFP4 checkpoint converts **only** the NVFP4 layers to MXFP4. The source is detected as NVFP4 from the model name (containing `nvfp4`) or its quantization config (`config.json` / `hf_quant_config.json`), and a warning is printed:++- **NVFP4 layers**: detected automatically in each shard and converted to MXFP4.+- **Unquantized layers** (BF16, `lm_head`, router, ...): kept as-is and recorded as unquantized in `quantization_config`. To quantize some of them as well, list them in `--layer_config`, e.g. `--layer_config "{lm_head:mxfp4}"`.+- **FP8 layers** (mixed-precision sources, e.g. FP8 attention + NVFP4 MLP): kept as-is. The source quantization config (compressed-tensors; ModelOpt and Quark configs are first converted to compressed-tensors) is reused with only its NVFP4 groups rewritten to MXFP4, so the output is a `mixed-precision` `quantization_config`.+- Unrecognized source configs trigger a warning and are skipped.+- Currently only `--format llm_compressor` is supported.++```bash+auto_round nvidia/MiniMax-M2.7-NVFP4 --model_free --scheme MXFP4 --format llm_compressor --output_dir ./mxfp4-model+```+#### CLI Usage```bashdiff --git a/docs/step_by_step_CN.md b/docs/step_by_step_CN.mdindex 2d78a0df5..bd13f0a6b 100755--- a/docs/step_by_step_CN.md+++ b/docs/step_by_step_CN.md@@ -682,32 +682,21 @@ ar.quantize_and_save(output_dir, format="auto_round")<details><summary>Model-free 并行量化基准(分钟向上取整)</summary>-时间归一化规则:所有 `mm:ss` 均向上取整到分钟。例如:`4:20 -> 5`、`15:45 -> 16`、`9:07 -> 10`、`7:29 -> 8`、`4:09 -> 5`。--| 模型 | 设备 | 方案 | 并行度 | 峰值显存 (G) | 耗时(分钟,向上取整) |-|---|---|---|---:|---:|---:|-| Qwen/Qwen3-Next-80B-A3B-Instruct | A100 | W4A16 | 1 | 2 | N/A |-| Qwen/Qwen3-Next-80B-A3B-Instruct | A100 | W4A16 | 10 | 8 | 7 |-| Qwen3-235B-A22B-Instruct-2507 | A100 | W4A16 | 1 | 2 | 17 |-| Qwen3-235B-A22B-Instruct-2507 | A100 | W4A16 | 10 | 8 | 5 |-| zai-org/GLM-5.2 | B200 | MXFP4-Mixed | 1 | 2 | 60 |-| zai-org/GLM-5.2 | B200 | MXFP4-Mixed | 10 | 27 | 16 |-| zai-org/GLM-5.2 | B200 | W4A16 | 1 | 3 | 30 |-| zai-org/GLM-5.2 | B200 | W4A16 | 10 | 16 | 10 |-| zai-org/GLM-5.2 | B200 | W4A16 | 20 | 32 | 8 |-| MiniMaxAI/MiniMax-M2.7 (FP8) | B200 | W4A16 | 1 | 2 | 18 |-| MiniMaxAI/MiniMax-M2.7 (FP8) | B200 | W4A16 | 10 | 10 | 5 |-| deepseek-ai/DeepSeek-V4-Pro (MXFP) | B200 | W4A16 | 1 | 6 | 80 |-| deepseek-ai/DeepSeek-V4-Pro (MXFP) | B200 | W4A16 | 10 | 50 | 13 |--| 模型 | 方案 | 对比 | 耗时变化(分钟) | 加速比 | 时间节省 | 峰值显存变化 |-|---|---|---|---|---:|---:|---|-| Qwen3-235B | W4A16 | 并行 1 -> 10 | 17 -> 5 | 3.40x | 70.6% | 2G -> 8G |-| GLM-5.2 | MXFP4-Mixed | 并行 1 -> 10 | 60 -> 16 | 3.75x | 73.3% | 2G -> 27G |-| GLM-5.2 | W4A16 | 并行 1 -> 10 | 30 -> 10 | 3.00x | 66.7% | 3G -> 16G |-| GLM-5.2 | W4A16 | 并行 1 -> 20 | 30 -> 8 | 3.75x | 73.3% | 3G -> 32G |-| MiniMax-M2.7 | W4A16 | 并行 1 -> 10 | 18 -> 5 | 3.60x | 72.2% | 2G -> 10G |-| DeepSeek-V4-Pro | W4A16 | 并行 1 -> 10 | 80 -> 13 | 6.15x | 83.8% | 6G -> 50G |+免模型模式会用多进程并行量化多个权重分片(shard),并行度通过环境变量 `AR_MODEL_FREE_SHARD_PARALLELISM` 设置,详见[环境变量指南](./environments_CN.md#ar_model_free_shard_parallelism):+- **默认(未设置)**:`shard_count // 4`,最小 1、最大 10。例如:8 个 shard 对应 2 个 worker,40 个 shard 及以上对应 10 个 worker。+- **手动设置**:例如 `export AR_MODEL_FREE_SHARD_PARALLELISM=20`,实际并行度不超过 shard 总数。+- 并行度越高通常越快,但峰值显存(以及内存)会随之上升,资源受限时请调小。++耗时均向上取整到分钟。++| 模型 | 设备 | 方案 | 并行度 | 耗时(分钟) | 加速比 | 峰值显存 (G) |+|---|---|---|---|---|---:|---|+| Qwen3-Next-80B-A3B-Instruct | A100 | W4A16 | 1 -> 10 | N/A -> 7 | N/A | 2 -> 8 |+| Qwen3-235B-A22B-Instruct-2507 | A100 | W4A16 | 1 -> 10 | 17 -> 5 | 3.40x | 2 -> 8 |+| GLM-5.2 | B200 | MXFP4-Mixed | 1 -> 10 | 60 -> 16 | 3.75x | 2 -> 27 |+| GLM-5.2 | B200 | W4A16 | 1 -> 10 / 20 | 30 -> 10 / 8 | 3.00x / 3.75x | 3 -> 16 / 32 |+| MiniMax-M2.7 (FP8) | B200 | W4A16 | 1 -> 10 | 18 -> 5 | 3.60x | 2 -> 10 |+| DeepSeek-V4-Pro (MXFP) | B200 | W4A16 | 1 -> 10 | 80 -> 13 | 6.15x | 6 -> 50 |结论:在 model-free 量化中,提高并行度通常可带来约 `3x-6x` 的耗时加速,但峰值显存会明显上升。@@ -744,6 +733,43 @@ ar.quantize_and_save(output_dir, format="auto_round")需要特殊打包内核的 scheme(`W3A16`、`FPW8A16`、`BF16`、`MXINT4`、`NVFP4`、`FP8_BLOCK`、`FP8_STATIC`、`INT8_W8A8`、`GGUF:*` 等)**不被支持**,传入会抛 `ValueError`。这些请使用标准 AutoRound 流程。+**NVFP4 源模型 → MXFP4**++目标:将已发布的 NVFP4 checkpoint 转换为 MXFP4,以便在支持 MXFP4、但没有原生 NVFP4 加速路径的 Intel GPU 上,高吞吐地部署 4-bit 大语言模型。++支持的源张量布局:++| 源布局 | 张量 | 状态 |+| --- | --- | --- |+| compressed-tensors(NVFP4) | `.weight_packed` + `.weight_scale` + `.weight_global_scale` + `.input_global_scale` | 支持 |+| ModelOpt / Quark(NVFP4) | `.weight`(uint8)+ `.weight_scale`(FP8)+ `.weight_scale_2` + `.input_scale` | 支持 |+| `inkling_mm_model` 类型 | `X`(uint8)+ `X.scale` + `X.scale2`(可能位于不同分片) | 支持 |++常见 checkpoint 的支持状态:++| 模型 | 源格式 | 状态 | gsm8k strict-match(MXFP4) | gsm8k flexible-extract(MXFP4) |+| --- | --- | --- | --- | --- |+| `RedHatAI/Qwen3.8-27B-NVFP4` | compressed-tensors,NVFP4 + FP8 | 已验证 | 0.9636 ± 0.0052 | 0.9636 |+| `nvidia/Qwen3.8-27B-NVFP4` | ModelOpt,NVFP4 + FP8 | 已验证 | 0.9644 ± 0.0051 | 0.9659 |+| `RedHatAI/Qwen3.6-35B-A3B-NVFP4` | compressed-tensors,NVFP4 | 已验证 | 0.9575 ± 0.0056 | 0.9575 |+| `amd/MiniMax-M2.7-NVFP4` | Quark,NVFP4 | 已验证 | 0.9568 ± 0.0056 | 0.9591 |+| `nvidia/MiniMax-M2.7-NVFP4` | ModelOpt,NVFP4 | 已验证 | 0.9477 ± 0.0061 | 0.9522 |+| `thinkingmachines/Inkling-Small-NVFP4` | 特殊格式(`inkling_mm_model`),NVFP4 | 已验证 | 0.9477 ± 0.0061 | 0.9545 |++gsm8k 为转换后 MXFP4 模型的 5-shot 评测结果,未开启 thinking。结果仅用于证明模型可正常运行,数据不保证准确。++对 NVFP4 checkpoint 使用 `--scheme MXFP4 --model_free` 时,**仅**将 NVFP4 层转换为 MXFP4。是否为 NVFP4 源模型由模型名(包含 `nvfp4`)或其量化配置(`config.json` / `hf_quant_config.json`)判断,检测到时会打印警告:++- **NVFP4 层**:在每个分片中自动检测,并转换为 MXFP4。+- **非量化层**(BF16、`lm_head`、router 等):保持原样,并在 `quantization_config` 中记录为未量化。如需同时量化其中部分层,请通过 `--layer_config` 额外指定,例如 `--layer_config "{lm_head:mxfp4}"`。+- **FP8 层**(混合精度源模型,例如 FP8 attention + NVFP4 MLP):保持原样。直接复用源模型的量化配置(compressed-tensors;ModelOpt 与 Quark 配置会先转换为 compressed-tensors 格式),仅将其中的 NVFP4 分组改写为 MXFP4,输出为 `mixed-precision` 的 `quantization_config`。+- 无法识别的源配置会给出警告并跳过。+- 目前仅支持 `--format llm_compressor`。++```bash+auto_round nvidia/MiniMax-M2.7-NVFP4 --model_free --scheme MXFP4 --format llm_compressor --output_dir ./mxfp4-model+```+#### 命令行用法```bashdiff --git a/test/unit/test_cpu/export/test_llmc_signature.py b/test/unit/test_cpu/export/test_llmc_signature.pyindex 35c6d2abd..a774da8d6 100644--- a/test/unit/test_cpu/export/test_llmc_signature.py+++ b/test/unit/test_cpu/export/test_llmc_signature.py@@ -7,6 +7,11 @@ from auto_round.export.export_to_llmcompressor.export import (_quant_args_signature,_rewrite_config_group_targets,)+from auto_round.export.export_to_llmcompressor.utils import (+ collect_fused_expert_modules,+ ignore_fused_experts_for_vllm,+ revert_quantization_config_names,+)class _FakeQuantizationArgs:@@ -91,3 +96,67 @@ def test_rewrite_config_group_targets_preserves_uniform_targets():config = {"config_groups": {"group_0": {"targets": ["Linear"]}}}assert _rewrite_config_group_targets(torch.nn.Module(), config) == config+++def test_rewrite_config_group_targets_adds_routed_experts_for_moe():+ first_args = {"num_bits": 4, "type": "int"}+ second_args = {"num_bits": 8, "type": "int"}+ model = torch.nn.Module()+ model.moe = torch.nn.Module()+ model.moe.experts = torch.nn.ModuleList([torch.nn.Module()])+ model.moe.experts[0].w1 = torch.nn.Linear(1, 1)+ model.moe.experts[0].w1.quantization_scheme = SimpleNamespace(weights=first_args, input_activations=None)+ model.dense = torch.nn.Linear(1, 1)+ model.dense.quantization_scheme = SimpleNamespace(weights=second_args, input_activations=None)+ config = {+ "config_groups": {+ "group_0": {"weights": first_args, "input_activations": None, "targets": ["Linear"]},+ "group_1": {"weights": second_args, "input_activations": None, "targets": ["Linear"]},+ }+ }++ rewritten = _rewrite_config_group_targets(model, config)++ assert rewritten["config_groups"]["group_0"]["targets"] == ["moe.experts.0.w1", "RoutedExperts"]+ assert rewritten["config_groups"]["group_1"]["targets"] == ["dense"]+++def test_ignored_fused_experts_get_vllm_ignore_entry():+ model = torch.nn.Module()+ model.mlp = torch.nn.Module()+ model.mlp.experts = torch.nn.Module()+ model.mlp.experts.gate_up_proj = torch.nn.Parameter(torch.zeros(2, 4, 2))+ model.mlp.shared_experts = torch.nn.Linear(2, 2)+ fused = collect_fused_expert_modules(model)+ assert fused == ["mlp.experts"]++ config = {"config_groups": {"group_0": {"targets": ["dense", "RoutedExperts"]}}, "ignore": ["re:.*lm_head"]}+ config = ignore_fused_experts_for_vllm(config, config["ignore"] + fused, ["dense"])++ assert config["ignore"] == ["re:.*lm_head", r"re:^mlp\.experts\.\d+\.(gate|up|down)_proj$"]+++def test_revert_quantization_config_names_uses_checkpoint_namespace():+ def revert(name):+ return name.replace(".mlp.", ".block_sparse_moe.")++ config = {+ "config_groups": {"group_0": {"targets": ["Linear", "model.layers.0.mlp.experts.0.down_proj"]}},+ "ignore": [+ "model.layers.0.mlp.experts.0.gate_proj",+ r"re:.*model\.layers\.0\.mlp\.gate.*",+ r"re:.*mlp\.(gate|up).*",+ ],+ }++ config = revert_quantization_config_names(config, revert)++ assert config["config_groups"]["group_0"]["targets"] == [+ "Linear",+ "model.layers.0.block_sparse_moe.experts.0.down_proj",+ ]+ assert config["ignore"] == [+ "model.layers.0.block_sparse_moe.experts.0.gate_proj",+ r"re:.*model\.layers\.0\.block_sparse_moe\.gate.*",+ r"re:.*mlp\.(gate|up).*",+ ]diff --git a/test/unit/test_cpu/quantization/test_model_free.py b/test/unit/test_cpu/quantization/test_model_free.pyindex debd20bb3..01a849172 100644--- a/test/unit/test_cpu/quantization/test_model_free.py+++ b/test/unit/test_cpu/quantization/test_model_free.py@@ -20,6 +20,7 @@ Backend-specific model-free behavior is covered by test_model_free_parity.py.import jsonimport os+import refrom unittest.mock import Mockimport pytest@@ -36,6 +37,7 @@ def test_quantize_and_save_restores_temporary_state_on_failure(monkeypatch):compressor.output_dir = "original-output"compressor.format = "auto_round"compressor.quantized = False+ compressor.nvfp_source_only = Falsedef fail_run():raise RuntimeError("save failed")@@ -726,6 +728,20 @@ def test_model_free_legacy_nvfp4_is_normalized_and_passthrough(tmp_path):assert "model.layers.0.self_attn.q_proj" in quantizedassert prefix not in ignored++def test_model_free_llmc_nvfp4_is_not_renormalized(tmp_path):+ prefix = "model.layers.0.mlp.down_proj"+ tensors = _nvfp4_source_tensors(prefix)+ tensors[f"{prefix}.weight_global_scale"] = torch.tensor([4.0], dtype=torch.float32)+ shard_path = str(tmp_path / "shard.safetensors")+ save_file(tensors, shard_path)++ layer_config = {prefix: {"bits": 4, "group_size": 16, "sym": True, "data_type": "nv_fp"}}+ output, _, _ = _process_shard(shard_path, _DEFAULT_SCHEME, layer_config, [])++ # llm-compressor global scales are already in the expected convention.+ assert torch.allclose(output[f"{prefix}.weight_global_scale"], torch.tensor([4.0], dtype=torch.float32))+def test_ignores_and_skips(self, tmp_path):shard_path = str(tmp_path / "shard.safetensors")save_file(@@ -1261,6 +1277,577 @@ class TestModelFreeMXFP:assert "lm_head.weight_packed" not in keys+# ===========================================================================+# NVFP4 source + MXFP4 scheme: convert only the NVFP4 layers of an NVFP4 source checkpoint+# ===========================================================================++_NV_ATTN = "model.layers.0.self_attn.q_proj"+_NV_MLP = "model.layers.0.mlp.down_proj"+_NV_MLP_UP = "model.layers.0.mlp.up_proj"+++def _nvfp4_source_tensors(prefix, out_features=32, in_features=64):+ """compressed-tensors NVFP4 layer."""+ return {+ f"{prefix}.weight_packed": torch.randint(0, 256, (out_features, in_features // 2), dtype=torch.uint8),+ f"{prefix}.weight_scale": torch.ones((out_features, in_features // 16)).to(torch.float8_e4m3fn),+ f"{prefix}.weight_global_scale": torch.tensor([1.0], dtype=torch.float32),+ f"{prefix}.input_global_scale": torch.tensor([1.0], dtype=torch.float32),+ }+++def _modelopt_nvfp4_tensors(prefix, out_features=32, in_features=64, input_scale_key="input_scale"):+ """ModelOpt NVFP4 layer (Quark uses ``input_scale_2`` instead of ``input_scale``)."""+ return {+ f"{prefix}.weight": torch.randint(0, 256, (out_features, in_features // 2), dtype=torch.uint8),+ f"{prefix}.weight_scale": torch.ones((out_features, in_features // 16)).to(torch.float8_e4m3fn),+ f"{prefix}.weight_scale_2": torch.tensor(1.0, dtype=torch.float32),+ f"{prefix}.{input_scale_key}": torch.tensor(1.0, dtype=torch.float32),+ }+++def _ct_fp8_tensors(prefix, out_features=64, in_features=64):+ """compressed-tensors FP8 per-channel layer."""+ return {+ f"{prefix}.weight": torch.randn(out_features, in_features).to(torch.float8_e4m3fn),+ f"{prefix}.weight_scale": torch.rand(out_features, 1, dtype=torch.float32),+ }+++def _modelopt_fp8_tensors(prefix, out_features=64, in_features=64):+ """ModelOpt FP8 per-tensor layer."""+ return {+ f"{prefix}.weight": torch.randn(out_features, in_features).to(torch.float8_e4m3fn),+ f"{prefix}.weight_scale": torch.tensor(0.5, dtype=torch.float32),+ f"{prefix}.input_scale": torch.tensor(0.25, dtype=torch.float32),+ }+++# Trimmed from the config.json / hf_quant_config.json of the public checkpoints listed in the docs.+# RedHatAI/Qwen3.6-35B-A3B-NVFP4 (compressed-tensors, NVFP4)+_CT_NVFP4_QC = {+ "quant_method": "compressed-tensors",+ "format": "nvfp4-pack-quantized",+ "quantization_status": "compressed",+ "config_groups": {+ "group_0": {+ "format": "nvfp4-pack-quantized",+ "targets": ["Linear"],+ "weights": {+ "num_bits": 4,+ "type": "float",+ "symmetric": True,+ "strategy": "tensor_group",+ "group_size": 16,+ },+ "input_activations": {+ "num_bits": 4,+ "type": "float",+ "symmetric": True,+ "strategy": "tensor_group",+ "group_size": 16,+ "dynamic": "local",+ },+ }+ },+ "ignore": ["lm_head", _NV_ATTN],+ "kv_cache_scheme": None,+}+# RedHatAI/Qwen3.8-27B-NVFP4 (compressed-tensors, FP8 attention + NVFP4 MLP)+_CT_FP8_GROUP = {+ "format": "float-quantized",+ "targets": [r"re:.*self_attn\.(q|k|v|o)_proj$"],+ "weights": {"num_bits": 8, "type": "float", "symmetric": True, "strategy": "channel", "dynamic": False},+ "input_activations": {"num_bits": 8, "type": "float", "symmetric": True, "strategy": "token", "dynamic": True},+}+_CT_KV_SCHEME = {"num_bits": 8, "type": "float", "symmetric": True, "strategy": "tensor", "dynamic": False}+_CT_MIXED_QC = {+ "quant_method": "compressed-tensors",+ "format": "mixed-precision",+ "config_groups": {+ "group_0": _CT_FP8_GROUP,+ "group_1": {+ **_CT_NVFP4_QC["config_groups"]["group_0"],+ "targets": [r"re:.*mlp\..*(gate|up|down)_proj$"],+ },+ },+ "ignore": ["lm_head"],+ "kv_cache_scheme": _CT_KV_SCHEME,+}+# nvidia/Qwen3.8-27B-NVFP4 hf_quant_config.json (ModelOpt, FP8 + NVFP4)+_MODELOPT_MIXED_HF_QC = {+ "producer": {"name": "modelopt", "version": "0.35.0"},+ "quantization": {+ "quant_algo": "MIXED_PRECISION",+ "kv_cache_quant_algo": None,+ "quantized_layers": {+ _NV_ATTN: {"quant_algo": "FP8"},+ _NV_MLP: {"quant_algo": "NVFP4", "group_size": 16},+ },+ "exclude_modules": ["lm_head", "mtp*"],+ },+}+# nvidia/MiniMax-M2.7-NVFP4 config.json (ModelOpt, NVFP4 + FP8 kv cache)+_MODELOPT_NVFP4_QC = {+ "quant_method": "modelopt",+ "quant_algo": "NVFP4",+ "producer": {"name": "modelopt", "version": "0.37.0"},+ "kv_cache_scheme": {"num_bits": 8, "type": "float", "dynamic": False},+ "ignore": ["lm_head", "model.layers.0.self_attn*"],+}+# amd/MiniMax-M2.7-NVFP4 (Quark, NVFP4)+_QUARK_FP4_SPEC = {"dtype": "fp4", "qscheme": "per_group", "group_size": 16, "is_dynamic": False}+_QUARK_FP8_SCALE_SPEC = {"dtype": "fp8_e4m3", "qscheme": "per_tensor", "is_dynamic": False, "is_scale_quant": True}+_QUARK_NVFP4_QC = {+ "quant_method": "quark",+ "global_quant_config": {+ "weight": [_QUARK_FP4_SPEC, _QUARK_FP8_SCALE_SPEC],+ "input_tensors": [{**_QUARK_FP4_SPEC, "is_dynamic": True}, _QUARK_FP8_SCALE_SPEC],+ },+ "layer_quant_config": {},+ "layer_type_quant_config": {},+ "exclude": [_NV_ATTN, "lm_head"],+ "export": {"kv_cache_group": [], "pack_method": "reorder"},+}+++def _cfg_with_qc(qc):+ return {**_LLAMA_CFG, "quantization_config": qc}+++def _make_nvfp4_model_dir(tmp_path, config, tensors, **kwargs):+ """Like ``_make_model_dir`` but the directory name marks the source as NVFP4."""+ model_dir = _make_model_dir(tmp_path, config, tensors, **kwargs)+ nvfp4_dir = os.path.join(os.path.dirname(model_dir), "Model-NVFP4")+ os.rename(model_dir, nvfp4_dir)+ return nvfp4_dir+++def _run_nvfp4_to_mxfp4(model_dir, output_dir, **kwargs):+ kwargs.setdefault("format", "llm_compressor")+ _ModelFreeCompressorCore(model_name_or_path=model_dir, output_dir=output_dir, scheme="MXFP4", **kwargs).run()+ return _read_output_tensors(output_dir), _read_qconfig(output_dir)+++def _read_output_tensors(output_dir):+ tensors = {}+ for fname in os.listdir(output_dir):+ if fname.endswith(".safetensors"):+ with safe_open(os.path.join(output_dir, fname), framework="pt") as sf:+ tensors.update({k: sf.get_tensor(k) for k in sf.keys()})+ return tensors+++def _assert_mxfp4_layer(tensors, layer):+ assert tensors[f"{layer}.weight_packed"].dtype == torch.uint8+ assert tensors[f"{layer}.weight_scale"].dtype == torch.uint8+ for stale in ("weight", "weight_global_scale", "input_global_scale", "weight_scale_2", "input_scale"):+ assert f"{layer}.{stale}" not in tensors+++def _assert_unchanged(tensors, source, keys):+ for key in keys:+ assert tensors[key].dtype == source[key].dtype, key+ assert torch.equal(tensors[key].reshape(-1).view(torch.uint8), source[key].reshape(-1).view(torch.uint8)), key+++def _assert_mxfp4_group(group, targets):+ assert group["targets"] == targets+ assert group["format"] == "mxfp4-pack-quantized"+ assert group["weights"]["num_bits"] == 4+ assert group["weights"]["type"] == "float"+ assert group["weights"]["group_size"] == 32+++class TestNVFP4ToMXFP4:+ """MXFP4 on an NVFP4 source: only NVFP4 source layers become MXFP4; everything else is kept byte-for-byte."""++ def test_nvfp2mxfp_scheme_removed(self):+ assert not is_model_free_supported_scheme("NVFP2MXFP")++ @pytest.mark.parametrize(+ "name, qc",+ [+ ("nvidia/MiniMax-M2.7-NVFP4", None),+ ("/models/Qwen3-nvfp4/", None),+ ("source_model", _CT_NVFP4_QC),+ ("source_model", _CT_MIXED_QC),+ ("source_model", _MODELOPT_NVFP4_QC),+ ("source_model", {**_MODELOPT_MIXED_HF_QC["quantization"], "quant_method": "modelopt"}),+ ("source_model", _QUARK_NVFP4_QC),+ ],+ ids=["hub_name", "local_name", "ct", "ct_mixed", "modelopt", "modelopt_mixed", "quark"],+ )+ def test_detect_nvfp4_source(self, name, qc):+ from auto_round.utils.model_free_utils import detect_model_dtypes++ assert "nvfp4" in detect_model_dtypes(name, qc)++ @pytest.mark.parametrize(+ "name, qc",+ [+ ("/data/nvfp4_models/Qwen3-8B", None),+ ("Qwen/Qwen3-8B", {"quant_method": "fp8", "weight_block_size": [128, 128]}),+ ("Qwen/Qwen3-8B", {**_CT_MIXED_QC, "config_groups": {"group_0": _CT_FP8_GROUP}}),+ ],+ ids=["parent_dir_only", "fp8", "ct_fp8"],+ )+ def test_detect_non_nvfp4_source(self, name, qc):+ from auto_round.utils.model_free_utils import detect_model_dtypes++ assert "nvfp4" not in detect_model_dtypes(name, qc)++ @pytest.mark.parametrize(+ "qc, expected",+ [+ ({"quant_method": "fp8", "weight_block_size": [128, 128]}, True),+ ({"quantization_type": "fp8"}, True),+ ({"fmt": "e4m3", "scale_fmt": "ue8m0"}, True),+ ({"fmt": None}, False),+ (_CT_NVFP4_QC, False),+ ],+ ids=["quant_method", "quantization_type", "fmt", "fmt_none", "ct_nvfp4"],+ )+ def test_detect_fp8_source(self, qc, expected):+ from auto_round.utils.model_free_utils import detect_model_dtypes++ assert ("fp8" in detect_model_dtypes("Qwen/Qwen3-8B", qc)) is expected++ @require_compressed_tensors+ def test_detection_warns(self, tmp_path, monkeypatch):+ import auto_round.compressors.model_free as model_free_mod++ warnings = []+ monkeypatch.setattr(model_free_mod.logger, "warning", lambda msg, *a, **k: warnings.append(str(msg)))+ model_dir = _make_nvfp4_model_dir(tmp_path, _LLAMA_CFG, _nvfp4_source_tensors(_NV_MLP))+ _run_nvfp4_to_mxfp4(model_dir, str(tmp_path / "out"))+ assert any("Detected NVFP4 layers" in w and "layer_config" in w for w in warnings)++ @require_compressed_tensors+ def test_detection_from_config_without_nvfp4_name(self, tmp_path):+ source = {**_nvfp4_source_tensors(_NV_MLP), f"{_NV_ATTN}.weight": torch.randn(64, 64, dtype=torch.bfloat16)}+ model_dir = _make_model_dir(tmp_path, _cfg_with_qc(_CT_NVFP4_QC), source)+ tensors, _ = _run_nvfp4_to_mxfp4(model_dir, str(tmp_path / "out"))++ _assert_mxfp4_layer(tensors, _NV_MLP)+ _assert_unchanged(tensors, source, [f"{_NV_ATTN}.weight"])++ @require_compressed_tensors+ def test_non_nvfp4_source_quantizes_every_layer(self, tmp_path):+ source = {f"{_NV_ATTN}.weight": torch.randn(64, 64, dtype=torch.bfloat16)}+ model_dir = _make_model_dir(tmp_path, _LLAMA_CFG, source)+ tensors, _ = _run_nvfp4_to_mxfp4(model_dir, str(tmp_path / "out"))++ _assert_mxfp4_layer(tensors, _NV_ATTN)++ @require_compressed_tensors+ def test_layer_config_quantizes_extra_layers(self, tmp_path):+ source = {+ **_nvfp4_source_tensors(_NV_MLP),+ f"{_NV_ATTN}.weight": torch.randn(64, 64, dtype=torch.bfloat16),+ "lm_head.weight": torch.randn(128, 64, dtype=torch.bfloat16),+ }+ model_dir = _make_nvfp4_model_dir(tmp_path, _cfg_with_qc(_CT_NVFP4_QC), source)+ tensors, qc = _run_nvfp4_to_mxfp4(model_dir, str(tmp_path / "out"), layer_config={"q_proj": "MXFP4"})++ _assert_mxfp4_layer(tensors, _NV_MLP)+ _assert_mxfp4_layer(tensors, _NV_ATTN)+ _assert_unchanged(tensors, source, ["lm_head.weight"])+ _assert_mxfp4_group(qc["config_groups"]["group_0"], ["Linear"])+ _assert_mxfp4_group(qc["config_groups"]["group_1"], [_NV_ATTN])+ assert qc["ignore"] == ["lm_head"]++ # ------------------------------------------------------------------+ # Tensor conversion per source layout+ # ------------------------------------------------------------------++ @require_compressed_tensors+ def test_ct_nvfp4_only(self, tmp_path):+ source = {+ **_nvfp4_source_tensors(_NV_MLP),+ f"{_NV_ATTN}.weight": torch.randn(64, 64, dtype=torch.bfloat16),+ "lm_head.weight": torch.randn(128, 64, dtype=torch.bfloat16),+ }+ model_dir = _make_nvfp4_model_dir(tmp_path, _cfg_with_qc(_CT_NVFP4_QC), source)+ tensors, qc = _run_nvfp4_to_mxfp4(model_dir, str(tmp_path / "out"))++ _assert_mxfp4_layer(tensors, _NV_MLP)+ _assert_unchanged(tensors, source, [f"{_NV_ATTN}.weight", "lm_head.weight"])+ assert qc["quant_method"] == "compressed-tensors"+ assert qc["format"] == "mxfp4-pack-quantized"+ assert list(qc["config_groups"]) == ["group_0"]+ _assert_mxfp4_group(qc["config_groups"]["group_0"], ["Linear"])+ assert qc["ignore"] == ["lm_head", _NV_ATTN]++ @require_compressed_tensors+ def test_ct_fp8_and_nvfp4_mixed(self, tmp_path):+ source = {+ **_nvfp4_source_tensors(_NV_MLP),+ **_nvfp4_source_tensors(_NV_MLP_UP),+ **_ct_fp8_tensors(_NV_ATTN),+ "model.layers.0.self_attn.k_scale": torch.tensor(0.1, dtype=torch.float32),+ "lm_head.weight": torch.randn(128, 64, dtype=torch.bfloat16),+ }+ model_dir = _make_nvfp4_model_dir(tmp_path, _cfg_with_qc(_CT_MIXED_QC), source)+ tensors, qc = _run_nvfp4_to_mxfp4(model_dir, str(tmp_path / "out"))++ _assert_mxfp4_layer(tensors, _NV_MLP)+ _assert_mxfp4_layer(tensors, _NV_MLP_UP)+ _assert_unchanged(+ tensors,+ source,+ [f"{_NV_ATTN}.weight", f"{_NV_ATTN}.weight_scale", "model.layers.0.self_attn.k_scale", "lm_head.weight"],+ )+ assert qc["format"] == "mixed-precision"+ assert qc["config_groups"]["group_0"] == _CT_FP8_GROUP+ _assert_mxfp4_group(qc["config_groups"]["group_1"], [r"re:.*mlp\..*(gate|up|down)_proj$"])+ assert qc["ignore"] == ["lm_head"]+ assert _NV_ATTN not in qc["ignore"]+ assert qc["kv_cache_scheme"] == _CT_KV_SCHEME++ @require_compressed_tensors+ def test_modelopt_fp8_and_nvfp4_from_hf_quant_config(self, tmp_path):+ source = {+ **_modelopt_nvfp4_tensors(_NV_MLP),+ **_modelopt_fp8_tensors(_NV_ATTN),+ "lm_head.weight": torch.randn(128, 64, dtype=torch.bfloat16),+ }+ model_dir = _make_nvfp4_model_dir(tmp_path, _LLAMA_CFG, source)+ with open(os.path.join(model_dir, "hf_quant_config.json"), "w") as f:+ json.dump(_MODELOPT_MIXED_HF_QC, f)+ output_dir = str(tmp_path / "out")+ tensors, qc = _run_nvfp4_to_mxfp4(model_dir, output_dir)++ _assert_mxfp4_layer(tensors, _NV_MLP)+ _assert_unchanged(+ tensors,+ source,+ [f"{_NV_ATTN}.weight", f"{_NV_ATTN}.weight_scale", f"{_NV_ATTN}.input_scale", "lm_head.weight"],+ )+ assert qc["quant_method"] == "compressed-tensors"+ assert qc["format"] == "mixed-precision"+ groups = list(qc["config_groups"].values())+ fp8_group = next(g for g in groups if g["weights"]["num_bits"] == 8)+ assert fp8_group["targets"] == [_NV_ATTN]+ assert fp8_group["format"] == "float-quantized"+ assert fp8_group["weights"]["strategy"] == "tensor" and fp8_group["weights"]["type"] == "float"+ assert fp8_group["input_activations"]["strategy"] == "tensor"+ assert fp8_group["input_activations"]["dynamic"] is False+ _assert_mxfp4_group(next(g for g in groups if g["weights"]["num_bits"] == 4), [_NV_MLP])+ assert "lm_head" in qc["ignore"]+ assert any(t.startswith("re:") and re.match(t[3:], "mtp.layers.0.mlp") for t in qc["ignore"])+ # The stale ModelOpt metadata file must not survive next to the rewritten config.+ assert not os.path.exists(os.path.join(output_dir, "hf_quant_config.json"))++ @require_compressed_tensors+ def test_modelopt_nvfp4_with_kv_cache(self, tmp_path):+ source = {+ **_modelopt_nvfp4_tensors(_NV_MLP),+ f"{_NV_ATTN}.weight": torch.randn(64, 64, dtype=torch.bfloat16),+ "model.layers.0.self_attn.k_proj.k_scale": torch.tensor(0.1, dtype=torch.float32),+ }+ model_dir = _make_nvfp4_model_dir(tmp_path, _cfg_with_qc(_MODELOPT_NVFP4_QC), source)+ tensors, qc = _run_nvfp4_to_mxfp4(model_dir, str(tmp_path / "out"))++ _assert_mxfp4_layer(tensors, _NV_MLP)+ _assert_unchanged(tensors, source, [f"{_NV_ATTN}.weight", "model.layers.0.self_attn.k_proj.k_scale"])+ assert qc["format"] == "mxfp4-pack-quantized"+ _assert_mxfp4_group(next(iter(qc["config_groups"].values())), [_NV_MLP])+ assert qc["kv_cache_scheme"]["num_bits"] == 8 and qc["kv_cache_scheme"]["type"] == "float"+ attn_ignore = [t for t in qc["ignore"] if t.startswith("re:")]+ assert attn_ignore and re.match(attn_ignore[0][3:], _NV_ATTN)++ @require_compressed_tensors+ def test_quark_nvfp4(self, tmp_path):+ source = {+ **_modelopt_nvfp4_tensors(_NV_MLP, input_scale_key="input_scale_2"),+ f"{_NV_ATTN}.weight": torch.randn(64, 64, dtype=torch.bfloat16),+ }+ model_dir = _make_nvfp4_model_dir(tmp_path, _cfg_with_qc(_QUARK_NVFP4_QC), source)+ tensors, qc = _run_nvfp4_to_mxfp4(model_dir, str(tmp_path / "out"))++ _assert_mxfp4_layer(tensors, _NV_MLP)+ assert f"{_NV_MLP}.input_scale_2" not in tensors+ _assert_unchanged(tensors, source, [f"{_NV_ATTN}.weight"])+ assert qc["quant_method"] == "compressed-tensors"+ assert qc["format"] == "mxfp4-pack-quantized"+ _assert_mxfp4_group(next(iter(qc["config_groups"].values())), [_NV_MLP])+ assert qc["ignore"] == [_NV_ATTN, "lm_head"]+ assert "kv_cache_scheme" not in qc or qc["kv_cache_scheme"] is None++ @require_compressed_tensors+ def test_existing_mxfp_layers_kept(self, tmp_path):+ mxfp4, mxfp8 = "model.layers.0.mlp.gate_proj", "model.layers.0.self_attn.o_proj"+ source = {+ **_nvfp4_source_tensors(_NV_MLP),+ f"{mxfp4}.weight_packed": torch.randint(0, 256, (32, 32), dtype=torch.uint8),+ f"{mxfp4}.weight_scale": torch.full((32, 2), 127, dtype=torch.uint8),+ f"{mxfp8}.weight": torch.randn(64, 64).to(torch.float8_e4m3fn),+ f"{mxfp8}.weight_scale": torch.full((64, 2), 127, dtype=torch.uint8),+ }+ model_dir = _make_nvfp4_model_dir(tmp_path, _LLAMA_CFG, source)+ tensors, _ = _run_nvfp4_to_mxfp4(model_dir, str(tmp_path / "out"))++ _assert_mxfp4_layer(tensors, _NV_MLP)+ _assert_unchanged(+ tensors,+ source,+ [f"{mxfp4}.weight_packed", f"{mxfp4}.weight_scale", f"{mxfp8}.weight", f"{mxfp8}.weight_scale"],+ )++ @require_compressed_tensors+ def test_deepseek_fp8_kept_across_shards(self, tmp_path):+ fp8 = "model.layers.0.self_attn.o_proj"+ # _make_model_dir splits keys in half: the FP8 weight lands in shard 1, its scale_inv in shard 2.+ source = {+ **_nvfp4_source_tensors(_NV_MLP),+ f"{fp8}.weight": torch.randn(128, 128).to(torch.float8_e4m3fn),+ f"{fp8}.weight_scale_inv": torch.rand(1, 1, dtype=torch.float32),+ "lm_head.weight": torch.randn(128, 64, dtype=torch.bfloat16),+ f"{_NV_ATTN}.weight": torch.randn(64, 64, dtype=torch.bfloat16),+ "model.norm.weight": torch.ones(64, dtype=torch.bfloat16),+ "model.embed_tokens.weight": torch.randn(128, 64, dtype=torch.bfloat16),+ }+ model_dir = _make_nvfp4_model_dir(tmp_path, _LLAMA_CFG, source, multi_shard=True)+ output_dir = str(tmp_path / "out")+ tensors, _ = _run_nvfp4_to_mxfp4(model_dir, output_dir)++ _assert_mxfp4_layer(tensors, _NV_MLP)+ _assert_unchanged(tensors, source, [f"{fp8}.weight", f"{fp8}.weight_scale_inv", f"{_NV_ATTN}.weight"])+ with open(os.path.join(output_dir, "model.safetensors.index.json")) as f:+ weight_map = json.load(f)["weight_map"]+ assert set(weight_map) == set(tensors)++ @require_compressed_tensors+ def test_mxfp4_matches_nvfp4_dequant(self, tmp_path):+ from auto_round.utils.model_free_utils import _dequant_nvfp4_tensors++ source = _nvfp4_source_tensors(_NV_MLP)+ model_dir = _make_nvfp4_model_dir(tmp_path, _LLAMA_CFG, source)+ tensors, _ = _run_nvfp4_to_mxfp4(model_dir, str(tmp_path / "out"), disable_opt_rtn=True)++ ref = _dequant_nvfp4_tensors({k: v.clone() for k, v in source.items()}, [_NV_MLP])[f"{_NV_MLP}.weight"]+ out = _dequant_mxfp_tensors(+ {+ f"{_NV_MLP}.weight_packed": tensors[f"{_NV_MLP}.weight_packed"],+ f"{_NV_MLP}.weight_scale": tensors[f"{_NV_MLP}.weight_scale"],+ }+ )[f"{_NV_MLP}.weight"]+ # With unit NVFP4 scales every value is an E2M1 grid point, so MXFP4 reproduces it exactly.+ assert torch.equal(out.float(), ref.float())++ # ------------------------------------------------------------------+ # Edge cases+ # ------------------------------------------------------------------++ @require_compressed_tensors+ def test_ignored_nvfp4_layer_is_dequantized_and_added_to_ignore(self, tmp_path):+ source = {**_nvfp4_source_tensors(_NV_MLP), **_nvfp4_source_tensors(_NV_MLP_UP)}+ model_dir = _make_nvfp4_model_dir(tmp_path, _cfg_with_qc(_CT_NVFP4_QC), source)+ tensors, qc = _run_nvfp4_to_mxfp4(model_dir, str(tmp_path / "out"), ignore_layers="down_proj")++ _assert_mxfp4_layer(tensors, _NV_MLP_UP)+ assert tensors[f"{_NV_MLP}.weight"].dtype == torch.bfloat16+ assert f"{_NV_MLP}.weight_packed" not in tensors+ assert _NV_MLP in qc["ignore"]+ assert _NV_MLP_UP not in qc["ignore"]++ @require_compressed_tensors+ def test_no_nvfp4_layers_keeps_everything(self, tmp_path):+ source = {f"{_NV_ATTN}.weight": torch.randn(64, 64, dtype=torch.bfloat16), **_ct_fp8_tensors(_NV_MLP)}+ model_dir = _make_nvfp4_model_dir(tmp_path, _LLAMA_CFG, source)+ tensors, _ = _run_nvfp4_to_mxfp4(model_dir, str(tmp_path / "out"))++ assert set(tensors) == set(source)+ _assert_unchanged(tensors, source, list(source))++ @require_compressed_tensors+ def test_unknown_source_config_falls_back_to_plain_mxfp4(self, tmp_path):+ source = {**_nvfp4_source_tensors(_NV_MLP), f"{_NV_ATTN}.weight": torch.randn(64, 64, dtype=torch.bfloat16)}+ model_dir = _make_nvfp4_model_dir(tmp_path, _cfg_with_qc({"quant_method": "bitsandbytes"}), source)+ tensors, qc = _run_nvfp4_to_mxfp4(model_dir, str(tmp_path / "out"))++ _assert_mxfp4_layer(tensors, _NV_MLP)+ assert qc["format"] == "mxfp4-pack-quantized"+ assert next(iter(qc["config_groups"].values()))["targets"] == [_NV_MLP]+ assert _NV_ATTN in qc["ignore"]++ @require_compressed_tensors+ def test_non_ct_moe_source_targets_routed_experts(self, tmp_path):+ experts = [f"model.layers.0.block_sparse_moe.experts.{i}.w1" for i in range(2)]+ source = {k: v for e in experts for k, v in _modelopt_nvfp4_tensors(e).items()}+ source[f"{_NV_ATTN}.weight"] = torch.randn(64, 64, dtype=torch.bfloat16)+ model_dir = _make_nvfp4_model_dir(tmp_path, _cfg_with_qc(_MODELOPT_NVFP4_QC), source)+ tensors, qc = _run_nvfp4_to_mxfp4(model_dir, str(tmp_path / "out"))++ for expert in experts:+ _assert_mxfp4_layer(tensors, expert)+ targets = next(iter(qc["config_groups"].values()))["targets"]+ assert "Linear" not in targets+ assert sorted(targets) == sorted(experts + ["RoutedExperts"])++ @pytest.mark.parametrize(+ "qc",+ [+ {**_QUARK_NVFP4_QC, "layer_type_quant_config": {"Linear": {"weight": _QUARK_FP4_SPEC}}},+ {**_QUARK_NVFP4_QC, "global_quant_config": {"weight": {"dtype": "int4", "qscheme": "per_group"}}},+ {**_MODELOPT_NVFP4_QC, "quant_algo": "W4A8_AWQ"},+ {"quant_method": "gptq"},+ ],+ ids=["quark_layer_type", "quark_int4", "modelopt_awq", "gptq"],+ )+ def test_unsupported_source_config_is_skipped(self, qc):+ from auto_round.utils.model_free_utils import _source_config_to_ct++ assert _source_config_to_ct(qc) is None++ def test_load_source_quant_config_prefers_config_json(self, tmp_path):+ from auto_round.utils.model_free_utils import _load_source_quant_config++ with open(tmp_path / "hf_quant_config.json", "w") as f:+ json.dump(_MODELOPT_MIXED_HF_QC, f)+ assert _load_source_quant_config(str(tmp_path), _CT_NVFP4_QC) is _CT_NVFP4_QC+ loaded = _load_source_quant_config(str(tmp_path), {})+ assert loaded["quant_method"] == "modelopt"+ assert loaded["quantized_layers"] == _MODELOPT_MIXED_HF_QC["quantization"]["quantized_layers"]++ # ------------------------------------------------------------------+ # Entry points / format validation+ # ------------------------------------------------------------------++ @pytest.mark.parametrize("fmt", ["auto_round", "auto_round:auto_gptq", "fake"])+ def test_core_rejects_non_llm_compressor_format(self, tmp_path, fmt):+ model_dir = _make_nvfp4_model_dir(tmp_path, _LLAMA_CFG, _nvfp4_source_tensors(_NV_MLP))+ with pytest.raises(ValueError, match="llm_compressor"):+ _run_nvfp4_to_mxfp4(model_dir, str(tmp_path / "out"), format=fmt)++ @require_compressed_tensors+ def test_core_accepts_auto_round_llm_compressor_alias(self, tmp_path):+ model_dir = _make_nvfp4_model_dir(tmp_path, _LLAMA_CFG, _nvfp4_source_tensors(_NV_MLP))+ output_dir = str(tmp_path / "out")+ _run_nvfp4_to_mxfp4(model_dir, output_dir, format="auto_round:llm_compressor")+ _assert_mxfp4_layer(_read_output_tensors(output_dir), _NV_MLP)++ @require_compressed_tensors+ def test_autoround_entry(self, tmp_path):+ source = {**_nvfp4_source_tensors(_NV_MLP), **_ct_fp8_tensors(_NV_ATTN)}+ model_dir = _make_nvfp4_model_dir(tmp_path, _cfg_with_qc(_CT_MIXED_QC), source)+ output_dir = str(tmp_path / "out")+ AutoRound(model=model_dir, scheme="MXFP4", model_free=True, device_map="cpu").quantize_and_save(+ output_dir, format="llm_compressor"+ )+ tensors = _read_output_tensors(output_dir)+ _assert_mxfp4_layer(tensors, _NV_MLP)+ _assert_unchanged(tensors, source, [f"{_NV_ATTN}.weight", f"{_NV_ATTN}.weight_scale"])+ assert _read_qconfig(output_dir)["format"] == "mixed-precision"++ def test_autoround_entry_rejects_non_llm_compressor_format(self, tmp_path):+ model_dir = _make_nvfp4_model_dir(tmp_path, _LLAMA_CFG, _nvfp4_source_tensors(_NV_MLP))+ ar = AutoRound(model=model_dir, scheme="MXFP4", model_free=True, device_map="cpu")+ with pytest.raises(ValueError, match="llm_compressor"):+ ar.quantize_and_save(str(tmp_path / "out"), format="auto_round")++# ===========================================================================# deepseek_v4 MXFP-quantized source models# ===========================================================================diff --git a/test/unit/test_cpu/utils/test_model_free_utils.py b/test/unit/test_cpu/utils/test_model_free_utils.pyindex 142b9a776..571829b61 100644--- a/test/unit/test_cpu/utils/test_model_free_utils.py+++ b/test/unit/test_cpu/utils/test_model_free_utils.py@@ -36,14 +36,17 @@ from auto_round.utils.model_free_utils import (_build_mxfp_autoround_quantization_config,_build_mxfp_quantization_config,_build_quantization_config,+ _collect_nvfp4_source_layers,_convert_auto_scheme_layer_config,_dequant_fp8_tensors,- _dequant_modelopt_nvfp4_tensors,+ _dequant_inkling_nvfp4_tensors,_dequant_mxfp_tensors,_expand_e8m0_block_scale,_handle_mxfp_source_tensors,- _hydrate_and_clean_modelopt_nvfp4_aux,+ _handle_nvfp4_source_tensors,+ _hydrate_and_clean_inkling_nvfp4_aux,_hydrate_missing_fp8_scales_from_index,+ _ignore_fused_experts_for_vllm,_looks_like_auto_scheme,_PatternMatcher,_process_shard,@@ -104,7 +107,7 @@ def _read_output_keys(output_dir):return keys-def test_modelopt_nvfp4_failure_restores_scale2():+def test_inkling_nvfp4_failure_restores_scale2():"""A failed fused-NVFP4 decode must preserve its complete source triple."""base = "model.layers.0.experts.w13_weight"packed = torch.zeros((2, 4, 8), dtype=torch.uint8)@@ -112,7 +115,7 @@ def test_modelopt_nvfp4_failure_restores_scale2():scale2 = torch.ones(3, dtype=torch.float32)tensors = {base: packed, f"{base}.scale": scale, f"{base}.scale2": scale2}- result = _dequant_modelopt_nvfp4_tensors(tensors)+ result = _dequant_inkling_nvfp4_tensors(tensors)assert result[base] is packedassert result[f"{base}.scale"] is scale@@ -145,8 +148,8 @@ def test_fused_moe_weight_uses_canonical_name_for_matcher():assert quantize.call_count == tensor.shape[0]-def test_modelopt_nvfp4_hydrates_scales_from_separate_shards(tmp_path):- """ModelOpt NVFP4 scales are hydrated from the index-selected donor shards."""+def test_inkling_nvfp4_hydrates_scales_from_separate_shards(tmp_path):+ """Inkling NVFP4 scales are hydrated from the index-selected donor shards."""base = "model.layers.0.experts.w13_weight"recipient_name = "model-00001-of-00003.safetensors"scale_shard_name = "model-00002-of-00003.safetensors"@@ -170,7 +173,7 @@ def test_modelopt_nvfp4_hydrates_scales_from_separate_shards(tmp_path):file,)- result = _hydrate_and_clean_modelopt_nvfp4_aux(+ result = _hydrate_and_clean_inkling_nvfp4_aux({base: torch.zeros((2, 4, 8), dtype=torch.uint8)},str(tmp_path / recipient_name),)@@ -179,10 +182,10 @@ def test_modelopt_nvfp4_hydrates_scales_from_separate_shards(tmp_path):assert torch.equal(result[scale2_name], scale2)-def test_modelopt_nvfp4_dequantizes_fused_3d_weight():- """Fused ModelOpt NVFP4 weights dequantize per expert and drop metadata."""+def test_inkling_nvfp4_dequantizes_fused_3d_weight():+ """Fused Inkling NVFP4 weights dequantize per expert and drop metadata."""base = "model.layers.0.experts.w13_weight"- result = _dequant_modelopt_nvfp4_tensors(+ result = _dequant_inkling_nvfp4_tensors({base: torch.zeros((2, 4, 8), dtype=torch.uint8),f"{base}.scale": torch.ones((2, 4, 1), dtype=torch.float32),@@ -191,7 +194,7 @@ def test_modelopt_nvfp4_dequantizes_fused_3d_weight():)assert set(result) == {base}- assert result[base].dtype == torch.bfloat16+ assert result[base].dtype == torch.float32assert result[base].shape == (2, 4, 16)@@ -516,6 +519,52 @@ class TestBuildQuantizationConfigNvfp4MoeLlmCompressor:targets = _get_targets_from_qconfig(qconfig, "group_0")assert "RoutedExperts" not in targets, f"Unexpected RoutedExperts in targets: {targets}"+ def test_ignored_fused_experts_get_vllm_ignore_entry(self):+ default_scheme = {"bits": 16, "data_type": "float", "group_size": -1}+ layer_cfg = {".w1.": {"bits": 4, "data_type": "nv_fp", "group_size": 16}}+ qconfig = _build_quantization_config(+ default_scheme=default_scheme,+ layer_config=layer_cfg,+ ignore_patterns=[],+ quantized_layers=_MOE_NV_QUANTIZED,+ ignored_layers=["model.layers.2.mlp.experts.w13_weight", "model.layers.2.mlp.experts.w2_weight"],+ format="llm_compressor",+ )+ assert qconfig["ignore"] == [r"re:^model\.layers\.2\.mlp\.experts\.\d+\.(gate|up|down)_proj$"]+++class TestIgnoreFusedExpertsForVllm:+ _ENTRY = r"re:^model\.layers\.0\.mlp\.experts\.\d+\.(gate|up|down)_proj$"++ def _qconfig(self, targets):+ return {"config_groups": {"group_0": {"targets": targets}}, "ignore": ["lm_head"]}++ def test_adds_entry_matching_vllm_probe_names(self):+ import re++ qconfig = _ignore_fused_experts_for_vllm(+ self._qconfig(["Linear", "RoutedExperts"]),+ ["model.layers.0.mlp.experts.w13_weight", "model.layers.0.mlp.experts.w2_weight"],+ [],+ )+ assert qconfig["ignore"] == ["lm_head", self._ENTRY]+ for proj in ("gate_proj", "up_proj", "down_proj"):+ assert re.match(self._ENTRY[3:], f"model.layers.0.mlp.experts.0.{proj}")++ def test_no_routed_experts_no_change(self):+ qconfig = _ignore_fused_experts_for_vllm(+ self._qconfig(["Linear"]), ["model.layers.0.mlp.experts.w13_weight"], []+ )+ assert qconfig["ignore"] == ["lm_head"]++ def test_unfused_and_partially_quantized_modules_skipped(self):+ qconfig = _ignore_fused_experts_for_vllm(+ self._qconfig(["RoutedExperts"]),+ ["model.layers.1.mlp.experts.0.gate_proj", "model.layers.0.mlp.experts.w13_weight"],+ ["model.layers.0.mlp.experts.w2_weight"],+ )+ assert qconfig["ignore"] == ["lm_head"]+# ===========================================================================# _PatternMatcher@@ -825,6 +874,53 @@ def _make_deepseek_v4_mxfp8(out_f, in_f, block_h, block_w):return weight_fp8, scale+class TestNVFP4VsMXFP4SourceDetection:+ """NVFP4 (FP8 E4M3 scale / global scale) and MXFP4 (uint8 E8M0 scale) share weight_packed + weight_scale."""++ @staticmethod+ def _mxfp4(prefix):+ return {+ f"{prefix}.weight_packed": torch.randint(0, 256, (32, 32), dtype=torch.uint8),+ f"{prefix}.weight_scale": torch.full((32, 2), 127, dtype=torch.uint8),+ }++ @staticmethod+ def _nvfp4(prefix, with_global=True):+ tensors = {+ f"{prefix}.weight_packed": torch.randint(0, 256, (32, 32), dtype=torch.uint8),+ f"{prefix}.weight_scale": torch.ones((32, 4)).to(torch.float8_e4m3fn),+ }+ if with_global:+ tensors[f"{prefix}.weight_global_scale"] = torch.tensor([1.0])+ tensors[f"{prefix}.input_global_scale"] = torch.tensor([1.0])+ return tensors++ def test_collect_only_nvfp4_layers(self):+ raw = {**self._mxfp4("mx"), **self._nvfp4("nv"), **self._nvfp4("nv_no_global", with_global=False)}+ assert _collect_nvfp4_source_layers(raw) == {"nv", "nv_no_global"}++ @pytest.mark.parametrize(+ "default",+ [+ {"bits": 4, "group_size": 32, "data_type": "mx_fp"},+ {"bits": 4, "group_size": 128, "sym": True, "data_type": "int"},+ ],+ ids=["mxfp4_target", "int4_target"],+ )+ def test_handle_nvfp4_skips_mxfp4_layers(self, default):+ mx = self._mxfp4("mx")+ raw = {**{k: v.clone() for k, v in mx.items()}, **self._nvfp4("nv")}+ raw_out, passthrough, layers = _handle_nvfp4_source_tensors(raw, _matcher(default=default))++ assert passthrough == {} and layers == []+ for key, value in mx.items():+ assert torch.equal(raw_out[key], value)+ assert "mx.weight" not in raw_out+ assert raw_out["nv.weight"].dtype == torch.float32+ assert raw_out["nv.weight"].shape == (32, 64)+ assert "nv.weight_packed" not in raw_out and "nv.weight_global_scale" not in raw_out++class TestHandleMXFPSourceTensors:def test_passthrough_mxfp8_same_target(self):weight_fp8 = torch.randn(64, 128, dtype=torch.bfloat16).to(torch.float8_e4m3fn)@@ -857,8 +953,8 @@ class TestHandleMXFPSourceTensors:matcher = _matcher(default={"bits": 4, "group_size": 128, "sym": True, "data_type": "int"})raw_out, passthrough, layers = _handle_mxfp_source_tensors(raw, matcher)assert layers == [] and passthrough == {}- assert raw_out["layer.weight"].dtype == torch.bfloat16- assert torch.allclose(raw_out["layer.weight"], weight_fp8.to(torch.bfloat16))+ assert raw_out["layer.weight"].dtype == torch.float32+ assert torch.allclose(raw_out["layer.weight"], weight_fp8.to(torch.float32))assert "layer.weight_scale" not in raw_outdef test_mixed_passthrough_and_dequant(self):@@ -877,7 +973,7 @@ class TestHandleMXFPSourceTensors:assert "attn" in layersassert passthrough["attn.weight"].dtype == torch.float8_e4m3fnassert "mlp" not in layers- assert raw_out["mlp.weight"].dtype == torch.bfloat16+ assert raw_out["mlp.weight"].dtype == torch.float32assert raw_out["mlp.weight"].shape == (64, 128)def test_noop_without_mxfp_tensors(self):@@ -915,9 +1011,9 @@ class TestHandleMXFPSourceTensors:raw_out, state = _preprocess_model_type_source_tensors(raw, model_type="deepseek_v4")raw_out, passthrough, layers = _handle_mxfp_source_tensors(raw_out, matcher, source_state=state)assert layers == [] and passthrough == {}- assert raw_out["layer.weight"].dtype == torch.bfloat16+ assert raw_out["layer.weight"].dtype == torch.float32assert raw_out["layer.weight"].shape == (64, 128)- assert torch.allclose(raw_out["layer.weight"], weight_fp8.to(torch.bfloat16))+ assert torch.allclose(raw_out["layer.weight"], weight_fp8.to(torch.float32))assert "layer.scale" not in raw_out and "layer.weight_scale" not in raw_outdef test_convert_noop_without_quantized(self):@@ -933,8 +1029,8 @@ class TestHandleMXFPSourceTensors:weight_scale = torch.full((64, 4), 127, dtype=torch.uint8)raw = {"layer.weight": weight_fp8.clone(), "layer.weight_scale": weight_scale.clone()}out = _dequant_mxfp_tensors(raw)- assert out["layer.weight"].dtype == torch.bfloat16- assert torch.allclose(out["layer.weight"], weight_fp8.to(torch.bfloat16))+ assert out["layer.weight"].dtype == torch.float32+ assert torch.allclose(out["layer.weight"], weight_fp8.to(torch.float32))assert "layer.weight_scale" not in out@@ -1488,7 +1584,7 @@ class TestFP8Source:w = torch.randn(64, 128, dtype=torch.bfloat16).to(torch.float8_e4m3fn)raw = {"layer.weight": w, "layer.weight_scale_inv": torch.tensor(0.5), "layer.bias": torch.randn(64)}result = _dequant_fp8_tensors(raw, block_size=None)- assert result["layer.weight"].dtype == torch.bfloat16 and "layer.weight_scale_inv" not in result+ assert result["layer.weight"].dtype == torch.float32 and "layer.weight_scale_inv" not in resultdef test_no_fp8_noop(self):raw = {"layer.weight": torch.randn(64, 128)}@@ -1559,7 +1655,7 @@ class TestKimiK25Int4Source:device="cpu",)- expected = ((nibbles - 8).float() * 0.5).to(torch.bfloat16)+ expected = (nibbles - 8).float() * 0.5assert torch.equal(out["layer.weight"], expected)def test_kimi_k25_int4_dequant_helper(self):@@ -1574,7 +1670,7 @@ class TestKimiK25Int4Source:device="cpu",)assert "layer.weight" in out- assert out["layer.weight"].dtype == torch.bfloat16+ assert out["layer.weight"].dtype == torch.float32assert out["layer.weight"].shape == (128, 128)assert "layer.weight_packed" not in outassert "layer.weight_scale" not in out@@ -1618,3 +1714,19 @@ class TestKimiK25Int4Source:# Re-quantized MXFP4 scales are uint8 E8M0 (source INT4 scales were fp16).assert found_scale_dtype == torch.uint8assert found_packed_shape == (128, 64)+++def test_explicit_mxfp_targets_replaces_linear_with_routed_experts_catch_all():+ from auto_round.utils.model_free_utils import _explicit_mxfp_targets++ weights = {"type": "float", "num_bits": 4, "group_size": 32}+ qconfig = {+ "config_groups": {+ "group_0": {"targets": ["model.layers.0.mlp.fc1"], "weights": weights},+ "group_1": {"targets": ["Linear", "RoutedExperts"], "weights": weights},+ }+ }+ layers = ["model.layers.0.mlp.fc1", "model.layers.0.self_attn.q_proj"]+ out = _explicit_mxfp_targets(qconfig, layers)+ assert out["config_groups"]["group_0"]["targets"] == ["model.layers.0.mlp.fc1"]+ assert out["config_groups"]["group_1"]["targets"] == ["model.layers.0.self_attn.q_proj"]