PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
support model_type == "inkling_mm_model" (#2214)
Xin Hede801f9f500d
diff --git a/auto_round/compressors/model_free.py b/auto_round/compressors/model_free.pyindex 3cc5175ea..f54765477 100644--- a/auto_round/compressors/model_free.py+++ b/auto_round/compressors/model_free.py@@ -420,6 +420,11 @@ def _quantize_local_shard_task(local_weight_map,)tensor_names = list(local_weight_map.keys())+ # Drop the packed tensors *before* reclaiming memory; otherwise the whole+ # quantized shard (hundreds of MB) is still reachable and clear_memory()+ # cannot return it to the allocator/OS.+ output_tensors.clear()+ del output_tensorsclear_memory()if cleanup_source_shard:@@ -850,7 +855,7 @@ class _ModelFreeCompressorCore:if self.cross_shard_deps:n_pairs = sum(len(t) for donors in self.cross_shard_deps.values() for t in donors.values())logger.info(- f"Cross-shard FP8 dependencies: {n_pairs} scale_inv tensor(s), "+ f"Cross-shard scale dependencies: {n_pairs} scale tensor(s), "f"{len(self.cross_shard_deps)} recipient shard(s), "f"{len(self.donor_shard_tensors)} donor shard(s).")diff --git a/auto_round/utils/missing_tensors.py b/auto_round/utils/missing_tensors.pyindex 8af39c970..17e0ca2e5 100644--- a/auto_round/utils/missing_tensors.py+++ b/auto_round/utils/missing_tensors.py@@ -63,6 +63,13 @@ _FUSED_EXPERT_PROJ_PATTERNS: dict[str, list[str]] = {_AUTOROUND_ISSUE_URL = "https://github.com/intel/auto-round/issues"_WARNING_INDEX_PLACEHOLDER = "<idx>"+# model_type values whose native inference-engine loader (e.g. vLLM's custom+# MoE weight loader) expects the *fused*, per-layer stacked 3-D expert+# tensors (``experts.w13_weight`` / ``experts.w2_weight``) to be preserved+# as-is (quantized in place) rather than unfused into per-expert 2-D+# tensors. Splitting these would break loading for such architectures.+_KEEP_FUSED_EXPERT_MODEL_TYPES: frozenset[str] = frozenset({"inkling_mm_model"})+def _normalize_tensor_name_for_warning(name: str, numeric_replacement: str = _WARNING_INDEX_PLACEHOLDER) -> str:"""Normalize tensor names for warning_once deduplication.@@ -84,6 +91,7 @@ def _normalize_tensor_name_for_warning(name: str, numeric_replacement: str = _WAdef split_fused_expert_tensors(tensors_dict: dict[str, torch.Tensor],+ model_type: str | None = None,) -> dict[str, torch.Tensor]:"""Split 3-D fused expert tensors into per-expert 2-D tensors.@@ -110,12 +118,23 @@ def split_fused_expert_tensors(Non-3-D or non-expert tensors pass through unchanged.+ Some architectures (see ``_KEEP_FUSED_EXPERT_MODEL_TYPES``) ship a custom+ inference-engine loader that consumes the fused 3-D stacked tensors+ directly (quantizing/packing them in place without unfusing per expert).+ For those ``model_type`` values, this function is a no-op passthrough.+Args:tensors_dict: Mapping of tensor names to tensors.+ model_type: Source model's ``model_type`` (from ``config.json``), used+ to skip splitting for architectures whose native loader expects+ fused stacked expert tensors.Returns:New dict with fused expert tensors replaced by per-expert 2-D tensors."""+ if (model_type or "").lower() in _KEEP_FUSED_EXPERT_MODEL_TYPES:+ return dict(tensors_dict)+result: dict[str, torch.Tensor] = {}split_count = 0diff --git a/auto_round/utils/model_free_utils.py b/auto_round/utils/model_free_utils.pyindex b9e7d9811..c3d9bc3fe 100644--- a/auto_round/utils/model_free_utils.py+++ b/auto_round/utils/model_free_utils.py@@ -255,6 +255,155 @@ def _normalize_nvfp4_source_tensors(return raw_tensors, converted_layers+# Auxiliary tensors emitted next to a ModelOpt / TensorRT-LLM NVFP4 packed+# weight. Once the packed weight has been dequantized back to high precision+# these carry no information and must not leak into the quantized output.+_MODELOPT_NVFP4_AUX_SUFFIXES: tuple[str, ...] = (+ ".scale",+ ".scale2",+ ".input_amax",+ ".weight_amax",+ ".original_shape",+)+++def _dequant_modelopt_nvfp4_slice(+ packed: torch.Tensor,+ scale: torch.Tensor,+ global_scale: float,+ out_features: int,+ in_features: int,+ group_size: int,+ device: str,+ target_dtype: torch.dtype,+) -> torch.Tensor:+ """Dequantize one 2-D ModelOpt NVFP4 slice back to *target_dtype* on CPU."""+ from auto_round.experimental.qmodules.fp4_utils import unpack_fp4_from_uint8++ packed_dev = packed.to(device)+ unpacked = unpack_fp4_from_uint8(packed_dev, out_features, in_features, dtype=torch.float32)+ n_groups = in_features // group_size+ scale_dev = scale.to(device).to(torch.float32).reshape(out_features, n_groups, 1)+ weight = unpacked.reshape(out_features, n_groups, group_size) * scale_dev+ weight = weight.reshape(out_features, in_features) * global_scale+ result = weight.to(target_dtype).to("cpu")+ del packed_dev, unpacked, scale_dev, weight+ return result+++def _dequant_modelopt_nvfp4_tensors(+ raw_tensors: dict[str, torch.Tensor],+ device: str = "cpu",+ shard_name: str | None = None,+ target_dtype: torch.dtype = torch.bfloat16,+) -> dict[str, torch.Tensor]:+ """Dequantize ModelOpt / TensorRT-LLM NVFP4 source tensors to high precision.++ ModelOpt checkpoints (``hf_quant_config.json`` with ``quant_algo="NVFP4"``)+ store a quantized weight as a *packed uint8* tensor ``X`` accompanied by:++ * ``X.scale`` – per-block (group_size=16) scale, ``float8_e4m3fn``+ * ``X.scale2`` – global scale (scalar, or one per expert for fused MoE)+ * ``X.input_amax`` / ``X.original_shape`` – activation / bookkeeping metadata++ Note the base tensor has **no** ``.weight`` suffix (e.g.+ ``...mlp.experts.w13_weight``), so neither the generic FP8 dequant path nor+ :func:`_normalize_nvfp4_source_tensors` (which keys off ``.weight`` /+ ``.weight_packed`` + ``.weight_scale``) recognises it. Without this pass the+ packed uint8 bytes were quantized *as if they were real weights*, producing+ silently corrupted output with half the expected ``in_features``.++ Both plain 2-D weights and fused 3-D stacked MoE expert weights+ (``[num_experts, out, in]``) are supported. All auxiliary tensors listed in+ :data:`_MODELOPT_NVFP4_AUX_SUFFIXES` are dropped after dequantization.+ """+ entries: list[str] = []+ for name, tensor in raw_tensors.items():+ if tensor.dtype != torch.uint8 or tensor.dim() not in (2, 3):+ continue+ scale = raw_tensors.get(f"{name}.scale")+ if scale is None or f"{name}.scale2" not in raw_tensors:+ continue+ if scale.dtype not in (torch.float8_e4m3fn, torch.float8_e5m2, torch.float32, torch.bfloat16, torch.float16):+ continue+ if scale.dim() != tensor.dim():+ continue+ entries.append(name)++ if not entries:+ return raw_tensors++ shard_prefix = f"[{shard_name}] " if shard_name else ""+ logger.info(+ f"{shard_prefix}Dequantizing {len(entries)} ModelOpt NVFP4 source tensor(s) to {target_dtype} on {device}."+ )++ n_dropped = 0+ for name in entries:+ packed = raw_tensors.pop(name)+ scale = raw_tensors.pop(f"{name}.scale")+ scale2 = raw_tensors.pop(f"{name}.scale2")+ global_scale = scale2.float().flatten()++ out_features = packed.shape[-2]+ in_features = packed.shape[-1] * 2+ group_size = in_features // scale.shape[-1]++ try:+ if packed.dim() == 2:+ dequantized = _dequant_modelopt_nvfp4_slice(+ packed,+ scale,+ float(global_scale[0]),+ out_features,+ in_features,+ group_size,+ device,+ target_dtype,+ )+ else:+ num_experts = packed.shape[0]+ if global_scale.numel() not in (1, num_experts):+ raise ValueError(f"scale2 has {global_scale.numel()} element(s); expected 1 or {num_experts}.")+ # Fill a pre-allocated buffer instead of stacking a list of+ # slices, so peak memory stays at one output tensor + one slice.+ dequantized = torch.empty((num_experts, out_features, in_features), dtype=target_dtype)+ for i in range(num_experts):+ dequantized[i] = _dequant_modelopt_nvfp4_slice(+ packed[i],+ scale[i],+ float(global_scale[i if global_scale.numel() > 1 else 0]),+ out_features,+ in_features,+ group_size,+ device,+ target_dtype,+ )+ except Exception as e:+ logger.warning(+ f"{shard_prefix}Failed to dequantize ModelOpt NVFP4 tensor '{name}': {e}. "+ f"Keeping the packed source tensor unchanged."+ )+ raw_tensors[name] = packed+ raw_tensors[f"{name}.scale"] = scale+ raw_tensors[f"{name}.scale2"] = scale2+ continue++ raw_tensors[name] = dequantized+ del packed, scale, scale2, global_scale++ # Drop the now-meaningless auxiliary metadata tensors.+ for suffix in _MODELOPT_NVFP4_AUX_SUFFIXES:+ if raw_tensors.pop(f"{name}{suffix}", None) is not None:+ n_dropped += 1++ if n_dropped:+ logger.info(f"{shard_prefix}Dropped {n_dropped} stale NVFP4 metadata tensor(s) after dequantization.")++ clear_memory()+ return raw_tensors++def _handle_nvfp4_source_tensors(raw_tensors: dict[str, torch.Tensor],matcher: Any,@@ -319,20 +468,30 @@ def _dequantize_with_device_fallback() -> torch.Tensor:"""Run dequantization on ``dequant_device`` and fall back to CPU on errors."""if dequant_device != "cpu":+ failure: tuple[bool, str] | None = Nonetry:return on_device()except Exception as e:- if _is_out_of_memory_error(e):- logger.warning(- f"{shard_prefix}{op_name} on {dequant_device} ran OOM for {tensor_label}: {e}. "- "Clearing accelerator memory and falling back to CPU for this tensor."- )- clear_memory()- else:- logger.warning(- f"{shard_prefix}{op_name} on {dequant_device} failed for {tensor_label}: {e}. "- "Falling back to CPU for this tensor."- )+ # Only keep plain data here. Holding the exception object alive+ # keeps its traceback (and therefore every device tensor still+ # referenced by the failed frames) reachable, which would make the+ # clear_memory() below a no-op for exactly the memory we need back.+ failure = (_is_out_of_memory_error(e), str(e))++ # Outside the ``except`` block the exception and its traceback have+ # already been released, so reclaiming memory is now effective.+ is_oom, message = failure+ if is_oom:+ logger.warning(+ f"{shard_prefix}{op_name} on {dequant_device} ran OOM for {tensor_label}: {message}. "+ "Clearing accelerator memory and falling back to CPU for this tensor."+ )+ clear_memory()+ else:+ logger.warning(+ f"{shard_prefix}{op_name} on {dequant_device} failed for {tensor_label}: {message}. "+ "Falling back to CPU for this tensor."+ )return on_cpu()@@ -358,8 +517,151 @@ def _normalize_scheme(scheme: Union[str, QuantizationScheme]) -> QuantizationSchdef _is_eligible_weight(tensor_name: str, tensor: torch.Tensor) -> bool:- """Check if a tensor is eligible for quantization (2D Linear weight)."""- return tensor_name.endswith(".weight") and tensor.dim() == 2+ """Check if a tensor is eligible for quantization (2D high-precision Linear weight).++ ``is_floating_point`` is required: an already-packed weight (uint8/int8 FP4+ nibbles, int32 GPTQ words, ...) that slipped past the source-format handlers+ must never be fed to the quantizer, which would reinterpret the packed bytes+ as real values and silently corrupt the layer.+ """+ return tensor_name.endswith(".weight") and tensor.dim() == 2 and tensor.is_floating_point()+++def _is_moe_fused_expert_weight(tensor_name: str, tensor: torch.Tensor) -> bool:+ """Check if *tensor* is a 3-D fused per-layer stacked MoE expert weight+ (e.g. ``experts.w13_weight`` / ``experts.w2_weight``, shape+ ``[num_experts, out, in]``).++ Such tensors are produced when :func:`split_fused_expert_tensors+ <auto_round.utils.missing_tensors.split_fused_expert_tensors>` skips+ unfusing for the source model's ``model_type`` (see+ ``_KEEP_FUSED_EXPERT_MODEL_TYPES``) because the target inference engine's+ loader expects the fused layout preserved rather than split per expert.++ Matches ``<prefix>.experts.<proj>_weight`` as well as the nested tensor+ layouts ``<prefix>.experts.<proj>_weight.weight`` (dequantized) and+ ``<prefix>.experts.<proj>_weight.weight_packed`` (MXFP4 source).+ In some model families (e.g. Inkling) shared experts appear under the+ path segment ``shared_experts`` (e.g. ``...shared_experts.shared_w13_weight``).+ Treat these fused 3-D shared expert tensors the same as ordinary fused+ expert tensors so they can be considered for in-place quantization.+ Auxiliary 3-D tensors such as ``.scale`` / ``.weight_scale`` are still+ excluded because they are not weight tensors.+ """+ if tensor.dim() != 3:+ return False+ parts = tensor_name.split(".")+ # Accept fused experts stored under either "...experts..." or+ # "...shared_experts..." path segments.+ if "experts" not in parts and "shared_experts" not in parts:+ return False+ last = parts[-1]+ if last in ("weight", "weight_packed"):+ return True+ return len(parts) >= 2 and parts[-2] in ("experts", "shared_experts") and last.endswith("_weight")+++def _fused_expert_layer_name(tensor_name: str) -> str:+ """Strip the tensor-kind suffix from a fused MoE expert tensor name.++ ``experts.w13_weight`` stays as-is, while ``experts.w13_weight.weight`` and+ ``experts.w13_weight.weight_packed`` collapse to ``experts.w13_weight`` so+ quantized / ignored layer bookkeeping uses one consistent layer name.+ """+ for suffix in (".weight_packed", ".weight"):+ if tensor_name.endswith(suffix):+ return tensor_name[: -len(suffix)]+ return tensor_name+++def _quantize_moe_fused_expert_weight(+ tensor_name: str,+ tensor: torch.Tensor,+ matcher: "_PatternMatcher",+ device: str = "cpu",+ disable_opt_rtn: bool = False,+) -> tuple[str, dict[str, torch.Tensor], str | None, str | None]:+ """Quantize a 3-D fused per-layer stacked MoE expert weight in place.++ Each expert's 2-D slice (``tensor[i]``) is quantized independently via+ :func:`_quantize_weight_mxfp` and the packed outputs are re-stacked along+ a new leading dimension, preserving the tensor's original fused layout+ (e.g. ``experts.w13_weight.weight_packed`` stays 3-D instead of being+ unfused into per-expert 2-D tensors). RTN/MXFP quantization groups values+ along the last (``in_features``) dimension on a per-row basis, so this is+ numerically equivalent to quantizing whatever 2-D weight each row+ originally came from -- any gate/up row interleaving within a slice does+ not affect correctness.++ Only the MXFP4/MXFP8 path is currently supported for this layout; other+ schemes fall back to keeping the original (unquantized) weight.++ Returns:+ (layer_name, output_tensors_dict, quantized_layer_or_None, ignored_layer_or_None)+ """+ layer_name = _fused_expert_layer_name(tensor_name)+ matcher_name = f"{layer_name}.weight"++ if not tensor.is_floating_point():+ # An already-packed source tensor (e.g. NVFP4/MXFP4 uint8 nibbles) that+ # was not consumed by a source-format handler. Quantizing it would+ # reinterpret packed bytes as weights and silently corrupt the layer.+ logger.warning_once(+ f"Fused 3-D MoE tensor '{tensor_name}' has non-floating dtype {tensor.dtype}; "+ "it appears to be already packed and is kept unchanged instead of being quantized."+ )+ return layer_name, {tensor_name: tensor}, None, layer_name++ if matcher.should_ignore(matcher_name) or matcher.should_skip(matcher_name):+ return layer_name, {tensor_name: tensor}, None, layer_name++ scheme = matcher.resolve_scheme(matcher_name)+ if scheme is None:+ return layer_name, {tensor_name: tensor}, None, layer_name++ bits = scheme["bits"]+ if bits >= 16:+ return layer_name, {tensor_name: tensor}, None, layer_name++ data_type = (scheme.get("data_type") or "int").lower()+ group_size = scheme["group_size"]++ if not is_mx_fp(data_type):+ logger.warning_once(+ f"3-D fused MoE weight '{tensor_name}' (shape={list(tensor.shape)}) is only "+ "supported for MXFP schemes in model_free mode; keeping original weight."+ )+ return layer_name, {tensor_name: tensor}, None, layer_name++ try:+ packed_parts: dict[str, list[torch.Tensor]] = {}+ for i in range(tensor.shape[0]):+ slice_out = _quantize_weight_mxfp(+ weight=tensor[i],+ layer_name=f"{layer_name}.{i}",+ bits=bits,+ group_size=group_size,+ data_type=data_type,+ device=device,+ disable_opt_rtn=disable_opt_rtn,+ )+ prefix = f"{layer_name}.{i}"+ for key, value in slice_out.items():+ suffix = key[len(prefix) :] # e.g. ".weight_packed" / ".weight_scale"+ packed_parts.setdefault(suffix, []).append(value)++ out = {f"{layer_name}{suffix}": torch.stack(values, dim=0) for suffix, values in packed_parts.items()}+ # The per-expert slices have been copied into the stacked tensors; free+ # them immediately instead of holding a second full-size copy.+ packed_parts.clear()+ logger.debug(+ f"Quantized (MXFP, fused 3-D): {layer_name} "+ f"(bits={bits}, group_size={group_size}, num_experts={tensor.shape[0]})"+ )+ return layer_name, out, layer_name, None+ except Exception as e:+ logger.warning(f"Failed to MXFP-quantize fused 3-D MoE weight {layer_name}: {e}. Keeping original weight.")+ return layer_name, {tensor_name: tensor}, None, layer_namedef _quantize_weight_mxfp(@@ -414,31 +716,43 @@ def _quantize_weight_mxfp(# Build a lightweight nn.Linear holding the original weight so we can# delegate packing to the existing QuantLinear.pack implementation.- fake_linear = nn.Linear(in_features, out_features, bias=False)+ # Materialize it on the meta device so no (out_features x in_features)+ # buffer is allocated/initialized just to be immediately overwritten.+ with torch.device("meta"):+ fake_linear = nn.Linear(in_features, out_features, bias=False)with torch.no_grad():fake_linear.weight = nn.Parameter(weight_dev, requires_grad=False)- qlayer = QuantLinear(- bits=bits,- group_size=group_size,- infeatures=in_features,- outfeatures=out_features,- bias=False,- data_type="mx_fp4" if bits == 4 else "mx_fp8e4m3",- sym=True,- act_bits=bits,- )+ # QuantLinear registers zero-filled packed weight/scale buffers that pack()+ # replaces wholesale; allocate them on meta to avoid the throwaway copy.+ with torch.device("meta"):+ qlayer = QuantLinear(+ bits=bits,+ group_size=group_size,+ infeatures=in_features,+ outfeatures=out_features,+ bias=False,+ data_type="mx_fp4" if bits == 4 else "mx_fp8e4m3",+ sym=True,+ act_bits=bits,+ )qlayer.pack(fake_linear, shared_exp, device=device)+ # Drop every intermediate before returning so the (potentially large)+ # high-precision weight and device-side buffers are freed as soon as the+ # packed CPU copies exist.if bits == 8:- return {+ out = {f"{layer_name}.weight": qlayer.weight.to("cpu"),f"{layer_name}.weight_scale": qlayer.weight_scale.to("cpu"),}- return {- f"{layer_name}.weight_packed": qlayer.weight_packed.to("cpu"),- f"{layer_name}.weight_scale": qlayer.weight_scale.to("cpu"),- }+ else:+ out = {+ f"{layer_name}.weight_packed": qlayer.weight_packed.to("cpu"),+ f"{layer_name}.weight_scale": qlayer.weight_scale.to("cpu"),+ }+ del qlayer, fake_linear, weight_dev, shared_exp+ return outdef _quantize_weight_nvfp4_e5m3(@@ -485,16 +799,22 @@ def _pack_weight_nvfp4_e5m3(# normalize to [out_features, in_features // group_size] before packing# so serialized .weight_scale keeps the expected 2D shape.scale = scale.reshape(out_features, in_features // group_size).to(torch.float32)- linear = torch.nn.Linear(in_features, out_features, bias=False, device=device, dtype=weight.dtype)+ # Meta-device construction avoids allocating a full weight tensor on the+ # accelerator (and zero-filled packed buffers) that are immediately replaced.+ with torch.device("meta"):+ linear = torch.nn.Linear(in_features, out_features, bias=False, dtype=weight.dtype)linear.weight = torch.nn.Parameter(weight_dev, requires_grad=False)- qlayer = QuantLinear(- 4, group_size, in_features, out_features, False, data_type="nvfp4_v2", act_bits=4, act_data_type="nvfp4_v2"- )+ with torch.device("meta"):+ qlayer = QuantLinear(+ 4, group_size, in_features, out_features, False, data_type="nvfp4_v2", act_bits=4, act_data_type="nvfp4_v2"+ )qlayer.pack(linear, scale, device=device)- return {+ out = {f"{layer_name}.weight_packed": qlayer.weight_packed.to("cpu"),f"{layer_name}.weight_scale": qlayer.weight_scale.to("cpu"),}+ del qlayer, linear, weight_dev, scale+ return outdef _declared_int_packing(sym: bool) -> str:@@ -525,6 +845,9 @@ def _quantize_single_tensor(Returns:(layer_name, output_tensors_dict, quantized_layer_or_None, ignored_layer_or_None)"""+ if _is_moe_fused_expert_weight(tensor_name, tensor):+ return _quantize_moe_fused_expert_weight(tensor_name, tensor, matcher, device, disable_opt_rtn)+layer_name = tensor_name.rsplit(".", 1)[0]if not _is_eligible_weight(tensor_name, tensor):@@ -630,33 +953,50 @@ def _load_weight_map_from_index(index_path: str) -> dict[str, str]:def _build_cross_shard_pairs_from_weight_map(weight_map: dict[str, str],) -> tuple[dict[str, dict[str, list[str]]], dict[str, set[str]]]:- """Identify cross-shard FP8 (weight, weight_scale_inv) pairs from weight_map.+ """Identify cross-shard (weight, scale-metadata) pairs from weight_map.++ Two source layouts are covered:- A *cross-shard pair* exists when ``<layer>.weight`` and- ``<layer>.weight_scale_inv`` reside in **different** shards. The shard- that holds ``weight_scale_inv`` is the *donor*; the shard that holds- ``weight`` is the *recipient*.+ * **FP8**: ``<layer>.weight`` and ``<layer>.weight_scale_inv``.+ * **ModelOpt / TensorRT-LLM NVFP4**: a packed base tensor ``X`` and its+ ``X.scale`` / ``X.scale2`` siblings.++ A *cross-shard pair* exists when the two tensors reside in **different**+ shards. The shard holding the scale metadata is the *donor*; the shard+ holding the weight is the *recipient*. Recording these lets the shard+ scheduler order donors before recipients (critical in streaming mode, where+ a donor shard must be downloaded before the recipient can hydrate from it).Returns:recipient_to_donors:- ``{recipient_shard: {donor_shard: [scale_inv_tensor_names]}}``+ ``{recipient_shard: {donor_shard: [scale_tensor_names]}}``donor_shard_tensors:- ``{donor_shard: set(scale_inv tensor names it donates to other shards)}``+ ``{donor_shard: set(scale tensor names it donates to other shards)}``"""recipient_to_donors: dict[str, dict[str, list[str]]] = {}donor_shard_tensors: dict[str, set[str]] = {}- for tensor_name, shard in weight_map.items():- if not tensor_name.endswith(".weight_scale_inv"):- continue- # "layer.weight_scale_inv" -> "layer.weight"- weight_name = tensor_name[: -len("_scale_inv")]+ def _record(scale_name: str, donor_shard: str, weight_name: str) -> None:weight_shard = weight_map.get(weight_name)- if not weight_shard or weight_shard == shard:+ if not weight_shard or weight_shard == donor_shard:+ return+ # `donor_shard` donates `scale_name` to `weight_shard` (recipient)+ recipient_to_donors.setdefault(weight_shard, {}).setdefault(donor_shard, []).append(scale_name)+ donor_shard_tensors.setdefault(donor_shard, set()).add(scale_name)++ nvfp4_bases = _modelopt_nvfp4_bases_from_weight_map(weight_map)++ for tensor_name, shard in weight_map.items():+ if tensor_name.endswith(".weight_scale_inv"):+ # "layer.weight_scale_inv" -> "layer.weight"+ _record(tensor_name, shard, tensor_name[: -len("_scale_inv")])continue- # `shard` (donor) donates `tensor_name` to `weight_shard` (recipient)- recipient_to_donors.setdefault(weight_shard, {}).setdefault(shard, []).append(tensor_name)- donor_shard_tensors.setdefault(shard, set()).add(tensor_name)+ for suffix in (".scale2", ".scale"):+ if tensor_name.endswith(suffix):+ base = tensor_name[: -len(suffix)]+ if base in nvfp4_bases:+ _record(tensor_name, shard, base)+ breakreturn recipient_to_donors, donor_shard_tensors@@ -772,6 +1112,147 @@ def _hydrate_missing_fp8_scales_from_index(return raw_tensors+def _find_index_path(idx_dir: str) -> str | None:+ """Return the path of the ``*.safetensors.index.json`` file in *idx_dir*."""+ index_path = os.path.join(idx_dir, "model.safetensors.index.json")+ if os.path.exists(index_path):+ return index_path+ try:+ candidates = sorted(+ os.path.join(idx_dir, f) for f in os.listdir(idx_dir) if f.endswith(".safetensors.index.json")+ )+ except OSError:+ return None+ return candidates[0] if candidates else None+++def _modelopt_nvfp4_bases_from_weight_map(weight_map: dict[str, str]) -> set[str]:+ """Return base tensor names that use the ModelOpt / TensorRT-LLM NVFP4 layout.++ A base ``X`` qualifies when the checkpoint contains ``X``, ``X.scale`` and+ ``X.scale2`` (see :func:`_dequant_modelopt_nvfp4_tensors`).+ """+ return {+ name[: -len(".scale2")]+ for name in weight_map+ if name.endswith(".scale2")+ and name[: -len(".scale2")] in weight_map+ and f"{name[: -len('.scale2')]}.scale" in weight_map+ }+++def _hydrate_and_clean_modelopt_nvfp4_aux(+ raw_tensors: dict[str, torch.Tensor],+ shard_path: str,+ *,+ shard_name: str | None = None,+ index_dir: str | None = None,+ donor_shard_dir: str | None = None,+) -> dict[str, torch.Tensor]:+ """Resolve ModelOpt NVFP4 ``.scale`` / ``.scale2`` tensors across shards.++ ModelOpt checkpoints do **not** keep a packed weight and its scales in the+ same shard — e.g. ``layers.36...w13_weight`` can sit in shard 6 while its+ ``.scale`` is in shard 3 and its ``.scale2`` in shard 5. Since model-free+ processing is shard-local, :func:`_dequant_modelopt_nvfp4_tensors` would+ otherwise never see a complete triple and the packed weight would be left+ untouched (and reported as an already-packed, non-quantizable tensor).++ This helper does two things, both driven by ``*.safetensors.index.json``:++ 1. **Hydrate** the missing ``.scale`` / ``.scale2`` tensors for every NVFP4+ base weight present in this shard, reading only those tensors from the+ sibling shards that hold them.+ 2. **Drop** orphan auxiliary tensors (``.scale``, ``.scale2``,+ ``.input_amax``, ``.original_shape``) whose base weight lives in another+ shard, so this shard's output is not polluted with stale metadata that+ the consuming shard has already accounted for.+ """+ if not shard_path.endswith(".safetensors"):+ return raw_tensors++ shard_dir = os.path.dirname(shard_path)+ idx_dir = index_dir if index_dir is not None else shard_dir+ donor_dir = donor_shard_dir if donor_shard_dir is not None else idx_dir++ index_path = _find_index_path(idx_dir)+ if index_path is None:+ return raw_tensors++ try:+ weight_map = _load_weight_map_from_index(index_path)+ except Exception:+ return raw_tensors++ bases = _modelopt_nvfp4_bases_from_weight_map(weight_map)+ if not bases:+ return raw_tensors++ shard_prefix = f"[{shard_name}] " if shard_name else ""++ # --- 1) hydrate missing .scale / .scale2 for bases present in this shard ---+ current_shard = os.path.basename(shard_path)+ wanted_by_shard: dict[str, list[str]] = {}+ for base in bases:+ if base not in raw_tensors:+ continue+ for suffix in (".scale", ".scale2"):+ aux_name = f"{base}{suffix}"+ if aux_name in raw_tensors:+ continue+ target_shard = weight_map.get(aux_name)+ if not target_shard or target_shard == current_shard:+ continue+ wanted_by_shard.setdefault(target_shard, []).append(aux_name)++ if wanted_by_shard:+ from safetensors import safe_open++ hydrated = 0+ for target_shard, aux_names in wanted_by_shard.items():+ target_path = os.path.join(donor_dir, target_shard)+ if not os.path.exists(target_path):+ logger.warning(+ f"{shard_prefix}Donor shard '{target_shard}' not found in '{donor_dir}' while hydrating "+ f"{len(aux_names)} NVFP4 scale tensor(s); the affected weight(s) will stay packed and "+ f"will not be quantized."+ )+ continue+ try:+ with safe_open(target_path, framework="pt", device="cpu") as sf:+ for aux_name in aux_names:+ try:+ raw_tensors[aux_name] = sf.get_tensor(aux_name)+ hydrated += 1+ except Exception:+ continue+ except Exception as e:+ logger.warning(f"{shard_prefix}Could not read donor shard '{target_shard}': {e}.")+ continue++ if hydrated:+ logger.info(+ f"{shard_prefix}Hydrated {hydrated} NVFP4 scale tensor(s) from sibling shard(s) using index mapping."+ )++ # --- 2) drop orphan auxiliary tensors owned by another shard's base ---+ dropped = 0+ for name in list(raw_tensors.keys()):+ for suffix in _MODELOPT_NVFP4_AUX_SUFFIXES:+ if not name.endswith(suffix):+ continue+ base = name[: -len(suffix)]+ if base in bases and base not in raw_tensors:+ del raw_tensors[name]+ dropped += 1+ break++ if dropped:+ logger.info(f"{shard_prefix}Dropped {dropped} orphan NVFP4 metadata tensor(s) owned by other shard(s).")++ return raw_tensors++def _dequant_mxfp_tensors(raw_tensors: dict[str, torch.Tensor],device: str = "cpu",@@ -818,13 +1299,13 @@ def _dequant_mxfp_tensors(shard_prefix=shard_prefix,op_name="MXFP dequant",tensor_label=layer_name,- on_device=lambda: dequant_mx_fp8(+ on_device=lambda weight=weight, scale=scale: dequant_mx_fp8(weight_fp8=weight.to(dequant_device, non_blocking=True),scale_e8m0=scale.to(dequant_device, non_blocking=True),block_size=32,target_dtype=torch.bfloat16,).to("cpu"),- on_cpu=lambda: dequant_mx_fp8(+ on_cpu=lambda weight=weight, scale=scale: dequant_mx_fp8(weight_fp8=weight,scale_e8m0=scale,block_size=32,@@ -837,14 +1318,14 @@ def _dequant_mxfp_tensors(shard_prefix=shard_prefix,op_name="MXFP dequant",tensor_label=layer_name,- on_device=lambda: to_dtype(+ on_device=lambda weight=weight, scale=scale: to_dtype(data_lp=weight.view(torch.uint8).contiguous().to(dequant_device, non_blocking=True),scale_e8m0=scale.to(dequant_device, non_blocking=True),elem_dtype="fp4_e2m1",block_size=32,target_dtype=torch.bfloat16,).to("cpu"),- on_cpu=lambda: to_dtype(+ on_cpu=lambda weight=weight, scale=scale: to_dtype(data_lp=weight.view(torch.uint8).contiguous(),scale_e8m0=scale,elem_dtype="fp4_e2m1",@@ -853,6 +1334,10 @@ def _dequant_mxfp_tensors(),)raw_tensors[f"{layer_name}.weight"] = dq_weight+ # Drop the loop-local aliases so the source (packed weight + scale) and+ # the previous iteration's dequantized tensor are not kept alive while+ # the next, potentially much larger, layer is being processed.+ del weight, scale, dq_weightreturn raw_tensors@@ -985,13 +1470,16 @@ def _dequant_fp8_tensors(shard_prefix=shard_prefix,op_name="FP8 dequant",tensor_label=weight_name,- on_device=lambda: _dequant_fp8_linear_weight(+ on_device=lambda weight=weight, scale=scale: _dequant_fp8_linear_weight(weight.to(dequant_device, non_blocking=True),scale.to(dequant_device, non_blocking=True),block_size=block_size,).to("cpu"),- on_cpu=lambda: _dequant_fp8_linear_weight(weight, scale, block_size=block_size),+ on_cpu=lambda weight=weight, scale=scale: _dequant_fp8_linear_weight(weight, scale, block_size=block_size),)+ # ``weight`` still aliases the original FP8 storage after the dict slot+ # was rebound; release it (and the scale) before the next iteration.+ del weight, scalereturn raw_tensors@@ -1064,7 +1552,7 @@ def _process_shard(with safe_open(shard_path, framework="pt", device="cpu") as f:raw_tensors = {name: f.get_tensor(name) for name in f.keys()}- raw_tensors = split_fused_expert_tensors(raw_tensors)+ raw_tensors = split_fused_expert_tensors(raw_tensors, model_type=model_type)# Hydrate cross-shard FP8 weight_scale_inv tensors *before* any# preprocessing below. Otherwise a weight whose scale lives in a sibling@@ -1084,11 +1572,18 @@ def _process_shard(# Snapshot candidate weight layer names *before* any preprocessing. 1D# weights (for example LayerNorm) are not quantization targets, while 3D# weights remain tracked so unsupported layouts are visible as ignored.+ # Fused 3-D MoE expert weights (e.g. ``experts.w13_weight``, kept fused+ # for model_type architectures like Inkling) have no ``.weight`` suffix+ # of their own, so they are tracked separately here.input_weight_layers: list[str] = list(dict.fromkeys(- name.rsplit(".", 1)[0]+ (+ _fused_expert_layer_name(name)+ if _is_moe_fused_expert_weight(name, tensor)+ else (name.rsplit(".", 1)[0] if name.endswith(".weight") else name)+ )for name, tensor in raw_tensors.items()- if name.endswith(".weight") and tensor.dim() > 1+ if (name.endswith(".weight") and tensor.dim() > 1) or _is_moe_fused_expert_weight(name, tensor)))@@ -1121,6 +1616,19 @@ def _process_shard(# 1.5) normalize legacy NVFP4 names to llm-compressor naming.raw_tensors, _converted_nvfp4_layers = _normalize_nvfp4_source_tensors(raw_tensors, shard_name=shard_name)+ # 1.6) ModelOpt / TensorRT-LLM NVFP4 sources store the packed weight under a+ # bare name (no ``.weight`` suffix) with ``.scale`` / ``.scale2`` siblings,+ # which are frequently placed in *different* shards. Hydrate them from the+ # index first, then dequantize and drop the stale metadata tensors.+ raw_tensors = _hydrate_and_clean_modelopt_nvfp4_aux(+ raw_tensors,+ shard_path,+ shard_name=shard_name,+ index_dir=index_dir,+ donor_shard_dir=donor_shard_dir,+ )+ raw_tensors = _dequant_modelopt_nvfp4_tensors(raw_tensors, device=device, shard_name=shard_name)+# 1.5) model-type-specific low-precision dequantization (e.g. kimi_k25 INT4)raw_tensors = handle_model_type_low_precision_source_tensors(raw_tensors,@@ -1160,6 +1668,8 @@ def _process_shard(donor_shard_dir=donor_shard_dir,)raw_tensors.update(preserved_tensors)+ # ``raw_tensors`` now owns these; drop the duplicate references.+ preserved_tensors.clear()for tensor_name in list(raw_tensors.keys()):tensor = raw_tensors.pop(tensor_name)@@ -1174,6 +1684,10 @@ def _process_shard(output_tensors.update(out_dict)if q_layer:quantized_layers.append(q_layer)+ # Release the (possibly high-precision) source tensor and the packing+ # result before loading the next one, so peak RSS stays at ~one layer+ # instead of two.+ del tensor, out_dict# Remove scale_inv tensors that this shard donates to other shards.# These tensors have no corresponding weight in this shard; keeping them@@ -1201,11 +1715,34 @@ def _process_shard(# ---------------------------------------------------------------------------+def _index_shards_cache_status(model_name_or_path: str, index_path: str) -> tuple[bool, int, int]:+ """Return ``(all_cached, n_cached, n_total)`` for shards listed in an index file."""+ from huggingface_hub import try_to_load_from_cache++ try:+ shard_files = sorted(set(_load_weight_map_from_index(index_path).values()))+ except Exception:+ return False, 0, 0++ if not shard_files:+ return False, 0, 0++ n_cached = sum(1 for f in shard_files if isinstance(try_to_load_from_cache(model_name_or_path, f), str))+ return n_cached == len(shard_files), n_cached, len(shard_files)++def _get_model_cache_status(model_name_or_path: str) -> tuple[bool, str]:"""Return cache decision and a short reason string.- Cached means local dir, or HF cache contains config plus at least one- weight entry file (index or single-file checkpoint).+ Cached means a local directory, or an HF cache that holds ``config.json``+ **plus the actual weight data**.++ A cached ``*.index.json`` is deliberately *not* treated as a weight hit on+ its own: many workflows (and this module's own streaming path) prefetch+ only metadata. Trusting the index would route such models to a full+ ``snapshot_download``, downloading the entire checkpoint up-front instead+ of streaming and quantizing shard-by-shard. So when an index is present+ its referenced shards are verified individually."""if os.path.isdir(model_name_or_path):return True, "input is an existing local directory"@@ -1220,19 +1757,24 @@ def _get_model_cache_status(model_name_or_path: str) -> tuple[bool, str]:if not config_cached:return False, "HF cache miss: config.json not found"- weight_entry_candidates = (- "model.safetensors.index.json",- "pytorch_model.bin.index.json",- "model.safetensors",- "pytorch_model.bin",- )- hit_entries: list[str] = []- for fname in weight_entry_candidates:+ # Single-file checkpoints: the file itself is the weight data.+ for fname in ("model.safetensors", "pytorch_model.bin"):if isinstance(try_to_load_from_cache(model_name_or_path, fname), str):- hit_entries.append(fname)+ return True, f"HF cache hit: config.json + {fname}"++ # Sharded checkpoints: an index is only a hit if every shard is cached.+ for index_name in ("model.safetensors.index.json", "pytorch_model.bin.index.json"):+ index_path = try_to_load_from_cache(model_name_or_path, index_name)+ if not isinstance(index_path, str):+ continue+ all_cached, n_cached, n_total = _index_shards_cache_status(model_name_or_path, index_path)+ if all_cached:+ return True, f"HF cache hit: config.json + {index_name} + all {n_total} shard(s)"+ return (+ False,+ f"HF cache partial hit: {index_name} present but only {n_cached}/{n_total} shard(s) cached",+ )- if hit_entries:- return True, f"HF cache hit: config.json + {', '.join(hit_entries)}"return (False,"HF cache partial hit: config.json exists but no weight entry file found",@@ -1476,24 +2018,36 @@ def _build_nvfp4_e5m3_quantization_config(ignored_layers: list[str]) -> dict:# TODO: Remove when the issue is fixed# https://github.com/vllm-project/llm-compressor/issues/3069def _add_routed_experts_if_moe(targets: list[str], layer_names: list[str]) -> list[str]:- """Append ``"RoutedExperts"`` to *targets* when *layer_names* indicates a MoE model.-- LLM-Compressor has a known bug where MoE expert projections (e.g. ``w1``/- ``w3`` or layers inside ``block_sparse_moe``) are not matched by the generic- ``"Linear"`` target and require the explicit ``"RoutedExperts"`` target to be- present in the config group. This helper detects that situation and injects- the extra target so quantization is applied correctly.-- A model is considered MoE if any layer name contains:+ """Append ``"RoutedExperts"`` for routed expert layouts that llm-compressor misses.- * ``.w1`` or ``.w3`` (typical Mixtral / DeepSeek gating projections), or- * ``block_sparse_moe`` (Mixtral-style naming).+ The generic ``"Linear"`` target does not cover all expert layouts. Keep the+ safe canonical cases alone (``mlp.experts.*.gate_proj|up_proj|down_proj``),+ but trigger for non-standard expert paths and for any routed+ expert structure that lives outside the standard ``mlp.experts`` naming."""if "RoutedExperts" in targets:return targets- moe_re = re.compile(r"\.w[13](?:\.|$)|block_sparse_moe")- if any(moe_re.search(name) for name in layer_names):++ def _projection_name(name: str) -> str:+ # Eg. '...experts.0.gate_proj.weight' -> ['...', 'experts', '0', 'gate_proj', 'weight']+ parts = name.split(".")+ if len(parts) >= 2:+ return parts[-2]+ return parts[-1]++ for name in layer_names:+ lname = name.lower()+ if "block_sparse_moe" in lname or ".moe." in lname or ".feed_forward." in lname:+ return list(targets) + ["RoutedExperts"]++ if ".experts." not in lname:+ continue++ projection = _projection_name(name)+ if projection in {"gate_proj", "up_proj", "down_proj"}:+ continuereturn list(targets) + ["RoutedExperts"]+return targetsdiff --git a/test/unit/test_cpu/utils/test_model_free_utils.py b/test/unit/test_cpu/utils/test_model_free_utils.pyindex 5cbb2bfc9..6d4cf5577 100644--- a/test/unit/test_cpu/utils/test_model_free_utils.py+++ b/test/unit/test_cpu/utils/test_model_free_utils.py@@ -17,7 +17,7 @@import jsonimport osimport tempfile-from unittest.mock import Mock+from unittest.mock import Mock, patchimport pytestimport torch@@ -38,13 +38,16 @@ from auto_round.utils.model_free_utils import (_build_quantization_config,_convert_auto_scheme_layer_config,_dequant_fp8_tensors,+ _dequant_modelopt_nvfp4_tensors,_dequant_mxfp_tensors,_expand_e8m0_block_scale,_handle_mxfp_source_tensors,+ _hydrate_and_clean_modelopt_nvfp4_aux,_hydrate_missing_fp8_scales_from_index,_looks_like_auto_scheme,_PatternMatcher,_process_shard,+ _quantize_single_tensor,_quantize_weight_mxfp,_validate_auto_scheme_options,)@@ -98,6 +101,97 @@ def _read_output_keys(output_dir):return keys+def test_modelopt_nvfp4_failure_restores_scale2():+ """A failed fused-NVFP4 decode must preserve its complete source triple."""+ base = "model.layers.0.experts.w13_weight"+ packed = torch.zeros((2, 4, 8), dtype=torch.uint8)+ scale = torch.ones((2, 4, 1), dtype=torch.float32)+ scale2 = torch.ones(3, dtype=torch.float32)+ tensors = {base: packed, f"{base}.scale": scale, f"{base}.scale2": scale2}++ result = _dequant_modelopt_nvfp4_tensors(tensors)++ assert result[base] is packed+ assert result[f"{base}.scale"] is scale+ assert result[f"{base}.scale2"] is scale2+++def test_fused_moe_weight_uses_canonical_name_for_matcher():+ """Explicit fused-layer config overrides a matching ignore pattern."""+ base = "model.layers.0.experts.w13_weight"+ matcher = _matcher(+ ignore=["experts"],+ layer_config={base: {"bits": 4, "data_type": "mx_fp", "group_size": 16}},+ default={"bits": 16, "data_type": "float", "group_size": -1},+ )+ tensor = torch.ones((2, 4, 16), dtype=torch.bfloat16)++ def fake_quantize(weight, layer_name, **_):+ return {+ f"{layer_name}.weight_packed": torch.zeros((4, 8), dtype=torch.uint8),+ f"{layer_name}.weight_scale": torch.ones((4, 1), dtype=torch.uint8),+ }++ with patch("auto_round.utils.model_free_utils._quantize_weight_mxfp", side_effect=fake_quantize) as quantize:+ layer_name, outputs, quantized_layer, ignored_layer = _quantize_single_tensor(base, tensor, matcher)++ assert layer_name == base+ assert quantized_layer == base+ assert ignored_layer is None+ assert set(outputs) == {f"{base}.weight_packed", f"{base}.weight_scale"}+ assert quantize.call_count == tensor.shape[0]+++def test_modelopt_nvfp4_hydrates_scales_from_separate_shards(tmp_path):+ """ModelOpt NVFP4 scales are hydrated from the index-selected donor shards."""+ base = "model.layers.0.experts.w13_weight"+ recipient_name = "model-00001-of-00003.safetensors"+ scale_shard_name = "model-00002-of-00003.safetensors"+ scale2_shard_name = "model-00003-of-00003.safetensors"+ scale_name = f"{base}.scale"+ scale2_name = f"{base}.scale2"+ scale = torch.ones((2, 4, 1), dtype=torch.float32)+ scale2 = torch.ones(2, dtype=torch.float32)+ save_file({scale_name: scale}, str(tmp_path / scale_shard_name))+ save_file({scale2_name: scale2}, str(tmp_path / scale2_shard_name))+ with open(tmp_path / "model.safetensors.index.json", "w") as file:+ json.dump(+ {+ "metadata": {"total_size": 0},+ "weight_map": {+ base: recipient_name,+ scale_name: scale_shard_name,+ scale2_name: scale2_shard_name,+ },+ },+ file,+ )++ result = _hydrate_and_clean_modelopt_nvfp4_aux(+ {base: torch.zeros((2, 4, 8), dtype=torch.uint8)},+ str(tmp_path / recipient_name),+ )++ assert torch.equal(result[scale_name], scale)+ assert torch.equal(result[scale2_name], scale2)+++def test_modelopt_nvfp4_dequantizes_fused_3d_weight():+ """Fused ModelOpt NVFP4 weights dequantize per expert and drop metadata."""+ base = "model.layers.0.experts.w13_weight"+ result = _dequant_modelopt_nvfp4_tensors(+ {+ base: torch.zeros((2, 4, 8), dtype=torch.uint8),+ f"{base}.scale": torch.ones((2, 4, 1), dtype=torch.float32),+ f"{base}.scale2": torch.ones(2, dtype=torch.float32),+ }+ )++ assert set(result) == {base}+ assert result[base].dtype == torch.bfloat16+ assert result[base].shape == (2, 4, 16)++# ---------------------------------------------------------------------------# Shared helpers# ---------------------------------------------------------------------------@@ -139,18 +233,23 @@ class TestAddRoutedExpertsIfMoe:result = _add_routed_experts_if_moe(targets, layers)assert "RoutedExperts" in result- def test_non_moe_no_change(self):+ def test_expert_path_not_under_mlp_triggers(self):targets = ["Linear"]- layers = ["model.layers.0.mlp.down_proj.weight", "model.layers.0.mlp.up_proj.weight"]+ layers = ["model.layers.0.moe.experts.0.gate_proj.weight"]result = _add_routed_experts_if_moe(targets, layers)- assert result == ["Linear"]- assert "RoutedExperts" not in result+ assert "RoutedExperts" in result++ def test_generic_projection_name_triggers(self):+ targets = ["Linear"]+ layers = ["model.layers.0.mlp.experts.0.proj.weight"]+ result = _add_routed_experts_if_moe(targets, layers)+ assert "RoutedExperts" in result- def test_w2_alone_does_not_trigger(self):- """w2 (down-projection) alone should not trigger RoutedExperts."""+ def test_non_moe_no_change(self):targets = ["Linear"]- layers = ["model.layers.0.mlp.experts.0.w2.weight"]+ layers = ["model.layers.0.mlp.down_proj.weight", "model.layers.0.mlp.up_proj.weight"]result = _add_routed_experts_if_moe(targets, layers)+ assert result == ["Linear"]assert "RoutedExperts" not in resultdef test_already_present_not_duplicated(self):