Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

188 KB
Raw Blame
# Copyright (c) 2025 Intel Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Utility helpers for model-free quantization.

This module hosts generic helpers used by model-free flow and keeps
model-special handlers at the end of the file.
"""

from __future__ import annotations

import copy
import json
import os
import re
import shutil
import warnings
from collections.abc import Callable
from dataclasses import fields
from functools import lru_cache
from typing import Any

import torch

from auto_round.compressors.utils import is_mx_fp, is_nv_fp
from auto_round.export.export_to_llmcompressor.utils import add_routed_experts_if_moe as _add_routed_experts_if_moe
from auto_round.export.export_to_llmcompressor.utils import (
    ignore_fused_experts_for_vllm as _ignore_fused_experts_for_vllm,
)
from auto_round.logger import logger
from auto_round.schemes import PRESET_SCHEMES, QuantizationScheme, preset_name_to_scheme
from auto_round.utils.common import _normalize_tensor_name_for_warning, to_standard_regex
from auto_round.utils.device import clear_memory, compile_func
from auto_round.utils.path_safety import (
    resolve_within_directory,
    sanitize_shard_name,
    validate_weight_map,
)

_NVFP4_E5M3_DATA_TYPE = "nvfp4_v2"
_NVFP4_E5M3_GS_DATA_TYPE = "nvfp4_v2_gs"
_NVFP4_E5M3_GS_ACT_DATA_TYPE = "nvfp4_v2_with_static_gs"
_NVFP4_E5M3_FAMILY_DATA_TYPES = (_NVFP4_E5M3_DATA_TYPE, _NVFP4_E5M3_GS_DATA_TYPE)
# default_scheme flag restricting quantization to layers stored as NVFP4 in the source checkpoint.
_NVFP_SOURCE_ONLY_KEY = "_nvfp_source_only"
_BLOCK_NAME_TO_IGNORE = ("shared_expert_gate.", ".gate.", "embed", "conv")
_SUPPORTED_MXFP_BITS = (4, 8)
_SUPPORTED_INT_BITS = (2, 4, 8)
# Known lm_head layer name variants across model families:
#   "lm_head"  – most models (LLaMA, Mistral, Qwen, …)
#   "head"     – DeepSeek v4
#   "embed_out" – Pythia / Dolly
#   "output"   – some InternLM variants
_LM_HEAD_PATTERNS: tuple[str, ...] = ("lm_head", "head", "embed_out", "output")
SUPPORTED_PRESET_SCHEMES = (
    "W2A16",
    "W2A16G32",
    "W2A16G64",
    "W4A16",
    "W4A16_MIXED",
    "W8A16",
    "MXFP4",
    "MXFP8",
    "NVFP4_E5M3",
    "NVFP4_E5M3_GS",
    "BF16",
)


# ---------------------------------------------------------------------------
# Fused Expert Tensor Splitting (shared with auto_round.utils.missing_tensors)
# ---------------------------------------------------------------------------
# Maps a fused projection name to its constituent split names.
# When a 3D tensor ``*.experts.<fused_name>`` (shape [num_experts, ...])
# is encountered, it is split along dimension 0 of the per-expert 2-D
# slice to produce one tensor per split name per expert.
_FUSED_EXPERT_PROJ_PATTERNS: dict[str, list[str]] = {
    "gate_up_proj": ["gate_proj", "up_proj"],
    "w13": ["w1", "w3"],
}

# Native vLLM/compressed-tensors FusedMoE parameter naming encodes the
# projection *and* its companion scale directly in the tensor name, with no
# dot separator (e.g. ``experts.w13_weight``, ``experts.w2_weight_scale``,
# ``experts.w2_weight_scale_2``, ``experts.w13_input_scale``), unlike the
# dotted ``experts.<proj>.weight`` convention used elsewhere. Order matters
# only for readability here: none of these suffixes is itself a suffix of
# another, so matching is unambiguous.
_MOE_NATIVE_SUFFIXES: list[tuple[str, str]] = [
    ("_weight_scale_2", "weight_scale_2"),
    ("_weight_scale", "weight_scale"),
    ("_weight", "weight"),
    ("_input_scale", "input_scale"),
]

_AUTOROUND_ISSUE_URL = "https://github.com/intel/auto-round/issues"
_WARNING_INDEX_PLACEHOLDER = "<idx>"

# model_type values whose native inference-engine loader (e.g. vLLM's custom
# MoE weight loader) expects the *fused*, per-layer stacked 3-D expert
# tensors (``experts.w13_weight`` / ``experts.w2_weight``) to be preserved
# as-is (quantized in place) rather than unfused into per-expert 2-D
# tensors. Splitting these would break loading for such architectures.
_KEEP_FUSED_EXPERT_MODEL_TYPES: frozenset[str] = frozenset({"inkling_mm_model"})


# _normalize_tensor_name_for_warning moved to auto_round.utils.common to
# avoid circular imports; keep compatibility by importing it from there.


def _parse_fused_proj_token(token: str) -> tuple[str, str]:
    """Split a raw expert-projection token into ``(base_proj, out_suffix)``.

    Handles both the plain dotted convention (token is just the projection
    name, e.g. ``"down_proj"`` / ``"gate_up_proj"`` -> ``out_suffix="weight"``)
    and the native FusedMoE convention where the companion tensor kind is
    encoded in the token itself (e.g. ``"w2_weight_scale"`` ->
    ``base_proj="w2"``, ``out_suffix="weight_scale"``).
    """
    for raw_suffix, out_suffix in _MOE_NATIVE_SUFFIXES:
        if token.endswith(raw_suffix) and len(token) > len(raw_suffix):
            return token[: -len(raw_suffix)], out_suffix
    return token, "weight"


def split_fused_expert_tensors(
    tensors_dict: dict[str, torch.Tensor],
    model_type: str | None = None,
) -> dict[str, torch.Tensor]:
    """Split 3-D fused expert tensors into per-expert 2-D tensors.

    Many MoE checkpoints store expert weights as fused 3-D parameters
    under either ``*.experts.<proj_name>`` or ``*.moe.<proj_name>``
    (for example ``gate_up_proj`` with shape
    ``[num_experts, 2*intermediate, hidden]``, or ``down_proj`` with shape
    ``[num_experts, out, in]``).  After unfusing, each expert gets its own
    2-D weight tensor, e.g.
    ``experts.0.gate_proj.weight [intermediate, hidden]``.

    Splitting rules:

        * **gate_up_proj** ``[N, 2*inter, hidden]`` →
      ``experts.{i}.gate_proj.weight`` + ``experts.{i}.up_proj.weight``
        * **Other** stacked projections (e.g. ``down_proj``) ``[N, out, in]`` →
            ``experts.{i}.<proj>.weight``

        For ``*.moe.<proj_name>`` tensors, outputs use
        ``*.moe.experts.{i}.<proj>.weight`` so they align with the sequential
        expert module layout used by quantized MoE replacements.

    Native vLLM/compressed-tensors FusedMoE naming (e.g. ``experts.w13_weight``,
    ``experts.w2_weight_scale``, ``experts.w13_weight_scale_2``,
    ``experts.w2_input_scale``) is also recognized: the companion tensor kind
    is decoded from the token itself (see :func:`_parse_fused_proj_token`) so
    weight/weight_scale/weight_scale_2/input_scale tensors for the same
    projection are split consistently and end up under the same
    ``experts.{i}.<proj>`` prefix, instead of scale tensors being mistaken for
    independent weights.

    Non-3-D tensors are only split when recognized as one of these native
    scale companions; other non-3-D or non-expert tensors pass through
    unchanged.

    Some architectures (see ``_KEEP_FUSED_EXPERT_MODEL_TYPES``) ship a custom
    inference-engine loader that consumes the fused 3-D stacked tensors
    directly (quantizing/packing them in place without unfusing per expert).
    For those ``model_type`` values, this function is a no-op passthrough.

    Args:
        tensors_dict: Mapping of tensor names to tensors.
        model_type: Source model's ``model_type`` (from ``config.json``), used
            to skip splitting for architectures whose native loader expects
            fused stacked expert tensors.

    Returns:
        New dict with fused expert tensors replaced by per-expert 2-D tensors.
    """
    if (model_type or "").lower() in _KEEP_FUSED_EXPERT_MODEL_TYPES:
        return dict(tensors_dict)

    result: dict[str, torch.Tensor] = {}
    split_count = 0

    for tensor_name, tensor in tensors_dict.items():
        if tensor.dim() not in (1, 2, 3):
            result[tensor_name] = tensor
            continue

        warning_tensor_name = _normalize_tensor_name_for_warning(tensor_name)

        # Strip optional .weight suffix (dotted convention) for pattern matching
        stripped = tensor_name.removesuffix(".weight")  # len(".weight") == 7

        # Expect: <prefix>.experts.<proj_name>
        dot_idx = stripped.rfind(".")
        if dot_idx < 0:
            result[tensor_name] = tensor
            continue

        parent = stripped[:dot_idx]
        raw_token = stripped[dot_idx + 1 :]
        proj_name, out_suffix = _parse_fused_proj_token(raw_token)

        parent_last = parent.split(".")[-1]
        is_experts_parent = parent.endswith("experts") or parent_last == "experts"
        is_moe_parent = parent.endswith("moe") or parent_last == "moe"

        # The immediate parent must be "experts" or "moe"
        if not is_experts_parent and not is_moe_parent:
            if tensor.dim() == 3:
                logger.warning_once(
                    "Found 3-D tensor '%s' while splitting expert tensors; "
                    "it will be kept unchanged. If this is an MoE/expert weight that should be split/quantized, "
                    "please open an issue at %s.",
                    warning_tensor_name,
                    _AUTOROUND_ISSUE_URL,
                )
            result[tensor_name] = tensor
            continue

        # 1-D/2-D tensors under an "experts"/"moe" parent are only touched here
        # when they are a recognized native FusedMoE scale companion (e.g.
        # ``w13_weight_scale_2`` [num_experts, 2] or ``w2_weight_scale_2``
        # [num_experts]). Anything else (e.g. a plain 1-D bias/norm) passes
        # through unchanged.
        if tensor.dim() != 3 and out_suffix == "weight":
            result[tensor_name] = tensor
            continue

        target_prefix = parent if is_experts_parent else f"{parent}.experts"
        num_experts = tensor.shape[0]
        split_names = _FUSED_EXPERT_PROJ_PATTERNS.get(proj_name, [proj_name])

        if tensor.dim() == 3:
            if len(split_names) > 1:
                logger.warning_once(
                    f"Splitting fused expert tensor '{warning_tensor_name}' "
                    f"(shape={list(tensor.shape)}, num_experts={num_experts}) "
                    f"into {split_names}"
                )
            else:
                logger.warning_once(
                    f"Splitting stacked expert tensor '{warning_tensor_name}' "
                    f"(shape={list(tensor.shape)}, num_experts={num_experts})"
                )
            for i in range(num_experts):
                expert_2d = tensor[i]  # [fused_out, in_features] or [out, in]
                chunks = expert_2d.chunk(len(split_names), dim=0)
                for split_name, chunk in zip(split_names, chunks):
                    out_key = f"{target_prefix}.{i}.{split_name}.{out_suffix}"
                    result[out_key] = chunk.contiguous()
        else:
            # Native per-expert scale companion (e.g. weight_scale_2 /
            # input_scale), shape [num_experts] or [num_experts, len(split_names)].
            logger.warning_once(
                f"Splitting fused expert scale tensor '{warning_tensor_name}' "
                f"(shape={list(tensor.shape)}, num_experts={num_experts}) "
                f"into {split_names}"
            )
            flat = tensor.reshape(num_experts, -1)
            n_splits = len(split_names)
            for i in range(num_experts):
                row = flat[i]
                if row.numel() == n_splits:
                    per_split_vals = [row[j].reshape(1) for j in range(n_splits)]
                else:
                    # Single value shared across all splits (e.g. a scalar
                    # global scale not broken out per gate/up projection).
                    shared = row.reshape(-1)[:1]
                    per_split_vals = [shared for _ in range(n_splits)]
                for split_name, val in zip(split_names, per_split_vals):
                    out_key = f"{target_prefix}.{i}.{split_name}.{out_suffix}"
                    result[out_key] = val.contiguous()

        split_count += 1

    return result


def quantize_weight_rtn(
    weight: torch.Tensor,
    bits: int,
    group_size: int,
    sym: bool = True,
    device: torch.device | None = None,
    disable_opt_rtn: bool = True,
    *,
    packing: str,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
    """Quantize a 2-D weight tensor and pack into auto_gptq format.

    Parameters
    ----------
    weight : Tensor [out_features, in_features]
    bits   : target bit-width (e.g. 4, 8)
    group_size : quantization group size along in_features
    sym    : use symmetric quantisation
    device : compute device (cuda / cpu). Results are always returned on CPU.
    disable_opt_rtn : when False and sym=True, use the optimised-RTN scale
        search (``quant_tensor_opt_rtn_sym``) which evaluates three E8M0
        candidates per group and picks the best MSE.  Defaults to True
        (plain RTN) to preserve backward-compatible behaviour.
    packing : the packing_format the artifact will declare (required); the
        zero-point convention follows the inference backend registry.  GPTQ_FORMAT
        entries ("auto_round:auto_gptq", qlinear_torch_zp) store zp - 1 per
        nibble and unpack with +1; GPTQ_FORMAT_NO_ZP entries ("auto_round",
        "auto_round:gptqmodel", qlinear_torch) store the zero point directly.
        Any other value raises.

    Returns
    -------
    qweight : [in_features // pack_factor, out_features]  int32
    qzeros  : [num_groups,  out_features // pack_factor]   int32
    scales  : [num_groups,  out_features]                   float16
    """
    assert weight.dim() == 2, f"Expected 2-D weight, got {weight.dim()}-D"
    from auto_round.inference.backend import GPTQ_FORMAT, GPTQ_FORMAT_NO_ZP

    if packing in GPTQ_FORMAT:
        zp_minus_one = True
    elif packing in GPTQ_FORMAT_NO_ZP:
        zp_minus_one = False
    else:
        raise ValueError(
            f"unsupported packing format '{packing}': zero-point conventions are defined only for "
            f"{GPTQ_FORMAT} (zp - 1, unpacked with +1) and {GPTQ_FORMAT_NO_ZP} (zp stored directly)"
        )
    out_features, in_features = weight.shape
    if device is None:
        device = weight.device
    # Single-step transfer + cast avoids an intermediate BF16 copy on CUDA
    # (``weight.to(device).float()`` would briefly allocate both BF16 and
    # float32 buffers on the target device).
    weight = weight.to(device=device, dtype=torch.float32)

    # --- pad in_features to multiple of group_size ---
    if in_features % group_size != 0:
        pad = group_size - (in_features % group_size)
        weight = torch.nn.functional.pad(weight, (0, pad))
        in_features = weight.shape[1]

    num_groups = in_features // group_size
    pack_factor = 32 // bits  # values per int32

    # --- pad out_features to multiple of pack_factor (needed for qzeros) ---
    out_pad = 0
    if out_features % pack_factor != 0:
        out_pad = pack_factor - (out_features % pack_factor)
        weight = torch.nn.functional.pad(weight, (0, 0, 0, out_pad))
    padded_out = weight.shape[0]

    from auto_round.data_type.utils import get_quant_func, reshape_pad_tensor_by_group_size

    # Use get_quant_func (same as WrapperLinear) so all data types and opt_rtn
    # variants are resolved via the QUANT_FUNC_WITH_DTYPE registry uniformly.
    quant_func, _ = get_quant_func("int", bits, sym=sym, disable_opt_rtn=disable_opt_rtn, iters=0)
    # quant_func returns (qdq_result, scale, zp_or_maxq)
    _, scale, zp_val = quant_func(weight, bits=bits, group_size=group_size)

    if sym:
        maxq = 1 << (bits - 1)  # e.g. 8 for 4-bit
        zero_point = maxq  # unsigned offset for packing

        # scale shape: [padded_out * num_groups, 1]
        # Reshape weight for group-wise quantization: [padded_out * num_groups, group_size]
        w_grouped, _, _ = reshape_pad_tensor_by_group_size(weight, group_size)
        w_grouped = w_grouped.to(device=device, dtype=torch.float32)
        del weight

        # Compute integer values for packing
        q = (w_grouped / scale).round_().clamp_(-maxq, maxq - 1)
        del w_grouped
        q += zero_point  # shift to unsigned [0, 2*maxq - 1]
        q = q.to(torch.int32)

        # scale → [num_groups, padded_out] (float16)
        scales_out = scale.squeeze(-1).reshape(padded_out, num_groups).t().contiguous().to(torch.float16)
        del scale

        zp = torch.full((num_groups, padded_out), zero_point, dtype=torch.int32, device=device)
    else:
        # Asymmetric quantization
        max_int = (1 << bits) - 1

        # scale shape: [padded_out * num_groups, 1], zp_val shape: [padded_out * num_groups, 1]

        # Reshape weight for group-wise quantization
        w_grouped, _, _ = reshape_pad_tensor_by_group_size(weight, group_size)
        w_grouped = w_grouped.to(device=device, dtype=torch.float32)
        del weight

        # Compute integer values for packing
        q = (w_grouped / scale).round_()
        del w_grouped
        q += zp_val
        q.clamp_(0, max_int)
        q = q.to(torch.int32)

        # scale → [num_groups, padded_out] (float16)
        scales_out = scale.squeeze(-1).reshape(padded_out, num_groups).t().contiguous().to(torch.float16)
        del scale

        # zp → [num_groups, padded_out]
        zp = zp_val.squeeze(-1).reshape(padded_out, num_groups).t().contiguous().to(torch.int32)
        del zp_val

    # q → [in_features, padded_out]
    q = q.reshape(padded_out, in_features).t().contiguous()

    # ---- Pack qweight: [in_features // pack_factor, padded_out] ----
    # Vectorised: reshape → broadcast shift → int64 sum (≡ bitwise-OR for
    # non-overlapping bit lanes) avoids a Python loop per bit-lane.
    _shifts = torch.arange(pack_factor, dtype=torch.int64, device=device) * bits
    q_packed = q.reshape(in_features // pack_factor, pack_factor, padded_out).to(torch.int64)
    del q
    qweight = (q_packed << _shifts[None, :, None]).sum(dim=1).to(torch.int32)
    del q_packed

    # ---- Pack qzeros: [num_groups, padded_out // pack_factor] ----
    # The zero-point convention is a property of the declared packing format
    # (see the backend registry): the auto_round:auto_gptq format
    # (qlinear_torch_zp) adds +1 to zeros after unpacking, so we subtract 1
    # before packing to compensate. The (zp-1) packing cannot represent
    # zp=0: a -1 left-shifts into a negative word and corrupts every nibble
    # sharing it. Clamp the stored value so only that zero-point degrades
    # (decodes as +1). Formats in GPTQ_FORMAT_NO_ZP (plain qlinear_torch)
    # read the nibble as the zero point directly, so it is stored as-is
    # (symmetric packing stores the offset-binary constant the same way:
    # 7 for the gptq family, 8 for the direct family).
    if zp_minus_one:
        zp = (zp - 1).clamp_(0, (1 << bits) - 1)
    else:
        zp = zp.clamp_(0, (1 << bits) - 1)
    zp_packed = zp.reshape(num_groups, padded_out // pack_factor, pack_factor).to(torch.int64)
    del zp
    qzeros = (zp_packed << _shifts[None, None, :]).sum(dim=2).to(torch.int32)
    del zp_packed, _shifts

    # Remove output padding from qweight / scales (qzeros stays in pack units)
    if out_pad > 0:
        qweight = qweight[:, :out_features]
        scales_out = scales_out[:, :out_features]

    # Always return CPU tensors (safetensors requires CPU)
    return qweight.cpu(), qzeros.cpu(), scales_out.cpu()


# ---------------------------------------------------------------------------
# Generic Utility Helpers
# ---------------------------------------------------------------------------


# Pattern Matching and Layer Scheme Resolution
# ---------------------------------------------------------------------------


class _PatternMatcher:
    """Precompile ignore and layer-config patterns for shard processing."""

    __slots__ = (
        "_allowed_layers",
        "_compiled_lc",
        "_default_scheme",
        "_ignore_cache",
        "_ignore_re",
        "_layer_config",
        "_scheme_cache",
        "_skip_re",
    )

    def __init__(
        self,
        ignore_patterns: list[str],
        layer_config: dict[str, dict],
        default_scheme: dict,
    ) -> None:
        self._default_scheme = default_scheme
        self._layer_config = layer_config
        self._ignore_re: re.Pattern | None = self._build_ignore_regex(ignore_patterns)
        self._skip_re = re.compile("|".join(re.escape(name) for name in _BLOCK_NAME_TO_IGNORE))

        self._compiled_lc: list[tuple[re.Pattern | None, str | None, dict]] = []
        for pattern, config in layer_config.items():
            try:
                self._compiled_lc.append((re.compile(to_standard_regex(pattern)), None, config))
            except re.error:
                self._compiled_lc.append((None, pattern, config))

        self._ignore_cache: dict[str, bool] = {}
        self._scheme_cache: dict[str, dict | None] = {}
        self._allowed_layers: set[str] | None = None

    @property
    def nvfp_source_only(self) -> bool:
        return bool(self._default_scheme.get(_NVFP_SOURCE_ONLY_KEY))

    def restrict_to_layers(self, layers: set[str]) -> None:
        """Treat every layer outside *layers* as ignored (unless explicitly listed in layer_config)."""
        self._allowed_layers = set(layers)
        self._ignore_cache.clear()

    @staticmethod
    def _build_ignore_regex(patterns: list[str]) -> re.Pattern | None:
        if not patterns:
            return None
        normalized_patterns: list[str] = []
        for pattern in patterns:
            if pattern.endswith("."):
                normalized = to_standard_regex(pattern.rstrip(".")).removesuffix(".*")
                normalized_patterns.append(f"{normalized}(?:\\.|$)")
            else:
                normalized_patterns.append(to_standard_regex(pattern))
        return re.compile("|".join(normalized_patterns))

    def should_ignore(self, tensor_name: str) -> bool:
        cached = self._ignore_cache.get(tensor_name)
        if cached is not None:
            return cached
        layer_name = tensor_name.rsplit(".", 1)[0] if "." in tensor_name else tensor_name
        # Explicit layer_config entries take priority: never ignore a layer the user has explicitly configured.
        if layer_name in self._layer_config:
            self._ignore_cache[tensor_name] = False
            return False
        if (
            self._allowed_layers is not None
            and layer_name not in self._allowed_layers
            and not self._matches_layer_config(layer_name)
        ):
            self._ignore_cache[tensor_name] = True
            return True
        result = bool(self._ignore_re and self._ignore_re.search(layer_name))
        self._ignore_cache[tensor_name] = result
        return result

    def _matches_layer_config(self, layer_name: str) -> bool:
        return any(
            (compiled is not None and compiled.search(layer_name)) or (plain is not None and plain in layer_name)
            for compiled, plain, _ in self._compiled_lc
        )

    def should_skip(self, tensor_name: str) -> bool:
        return bool(self._skip_re.search(tensor_name))

    def resolve_scheme(self, tensor_name: str) -> dict | None:
        if tensor_name not in self._scheme_cache:
            self._scheme_cache[tensor_name] = self._resolve_uncached(tensor_name)
        return self._scheme_cache[tensor_name]

    def _resolve_uncached(self, tensor_name: str) -> dict | None:
        layer_name = tensor_name.rsplit(".", 1)[0] if "." in tensor_name else tensor_name
        default = self._default_scheme
        if layer_name in self._layer_config:
            config = self._layer_config[layer_name]
            return None if config.get("bits", default.get("bits", 4)) >= 16 else {**default, **config}

        for compiled, plain, config in self._compiled_lc:
            if (compiled is not None and compiled.search(layer_name)) or (plain is not None and plain in layer_name):
                return None if config.get("bits", default.get("bits", 4)) >= 16 else {**default, **config}
        return default


# ---------------------------------------------------------------------------
# Source Tensor Detection and Normalization
# ---------------------------------------------------------------------------


def _collect_mxfp_source_entries(raw_tensors: dict[str, torch.Tensor]) -> list[tuple[str, str, str, int]]:
    """Collect MXFP source tensors present in a shard.

    Returns entries as ``(layer_name, weight_key, scale_key, bits)`` where
    ``bits`` is 8 for ``.weight`` (float8) and 4 for ``.weight_packed``.
    """
    entries: list[tuple[str, str, str, int]] = []
    for name, tensor in raw_tensors.items():
        if name.endswith(".weight") and tensor.dtype == torch.float8_e4m3fn:
            layer_name = name[: -len(".weight")]
            scale_key = f"{layer_name}.weight_scale"
            if scale_key in raw_tensors and raw_tensors[scale_key].dtype == torch.uint8:
                entries.append((layer_name, name, scale_key, 8))
        elif name.endswith(".weight_packed") and tensor.dtype in (torch.int8, torch.uint8):
            layer_name = name[: -len(".weight_packed")]
            if (
                f"{layer_name}.weight_global_scale" in raw_tensors
                or f"{layer_name}.input_global_scale" in raw_tensors
                or f"{layer_name}.weight_scale_2" in raw_tensors
                or f"{layer_name}.input_scale" in raw_tensors
            ):
                continue
            scale_key = f"{layer_name}.weight_scale"
            if scale_key in raw_tensors and raw_tensors[scale_key].dtype == torch.uint8:
                entries.append((layer_name, name, scale_key, 4))
    return entries


def _collect_nvfp4_source_layers(raw_tensors: dict[str, torch.Tensor]) -> set[str]:
    """Return names of layers stored as NVFP4 in a shard.

    Expects llm-compressor naming (run after :func:`_normalize_nvfp4_source_tensors`):
    ``<layer>.weight_packed`` + ``<layer>.weight_scale`` with a global scale or an FP8
    block scale (MXFP4 uses uint8 E8M0 scales without a global scale). Inkling-style bare
    tensors ``X`` with ``X.scale`` / ``X.scale2`` siblings are detected as layer ``X``.
    """
    layers: set[str] = set()
    for name, tensor in raw_tensors.items():
        if tensor.dtype not in (torch.uint8, torch.int8):
            continue
        if name.endswith(".weight_packed"):
            layer_name = name[: -len(".weight_packed")]
            scale = raw_tensors.get(f"{layer_name}.weight_scale")
            if scale is None:
                continue
            if (
                f"{layer_name}.weight_global_scale" in raw_tensors
                or f"{layer_name}.input_global_scale" in raw_tensors
                or scale.dtype == torch.float8_e4m3fn
            ):
                layers.add(layer_name)
        elif f"{name}.scale" in raw_tensors and f"{name}.scale2" in raw_tensors:
            layers.add(name)
    return layers


def _normalize_nvfp4_source_tensors(
    raw_tensors: dict[str, torch.Tensor],
    shard_name: str | None = None,
) -> tuple[dict[str, torch.Tensor], list[str]]:
    """Normalize legacy NVFP4 source naming to llm-compressor naming."""

    def _reciprocal_global_scale(scale: torch.Tensor) -> torch.Tensor:
        return (1.0 / scale.float()).to(torch.float32).reshape([1])

    converted_layers: list[str] = []
    candidates: list[str] = []
    for name, tensor in list(raw_tensors.items()):
        if name.endswith(".weight"):
            layer_name = name[: -len(".weight")]
        elif name.endswith(".weight_packed"):
            layer_name = name[: -len(".weight_packed")]
        else:
            continue

        if tensor.dtype not in (torch.uint8, torch.int8):
            continue
        if f"{layer_name}.weight_scale" not in raw_tensors:
            continue

        has_legacy_global = f"{layer_name}.weight_scale_2" in raw_tensors or f"{layer_name}.input_scale" in raw_tensors
        has_new_global = (
            f"{layer_name}.weight_global_scale" in raw_tensors or f"{layer_name}.input_global_scale" in raw_tensors
        )
        has_legacy_packed = name.endswith(".weight") and f"{layer_name}.weight_packed" not in raw_tensors
        # Layers already in llm-compressor naming need no normalization.
        if has_legacy_global or (has_legacy_packed and has_new_global):
            candidates.append(layer_name)

    if not candidates:
        return raw_tensors, converted_layers

    for layer_name in candidates:
        weight_key = f"{layer_name}.weight"
        weight_packed_key = f"{layer_name}.weight_packed"
        weight_scale_2_key = f"{layer_name}.weight_scale_2"
        input_scale_key = f"{layer_name}.input_scale"
        weight_global_scale_key = f"{layer_name}.weight_global_scale"
        input_global_scale_key = f"{layer_name}.input_global_scale"

        # Quark NVFP4 names the ModelOpt-style input_scale "input_scale_2".
        quark_input_scale = raw_tensors.pop(f"{layer_name}.input_scale_2", None)
        if quark_input_scale is not None and input_scale_key not in raw_tensors:
            raw_tensors[input_scale_key] = quark_input_scale

        if weight_packed_key not in raw_tensors and weight_key in raw_tensors:
            raw_tensors[weight_packed_key] = raw_tensors.pop(weight_key).view(torch.uint8).contiguous()

        # ModelOpt scales (amax / (448 * 6)) are the reciprocal of llm-compressor global scales.
        if weight_scale_2_key in raw_tensors and weight_global_scale_key not in raw_tensors:
            raw_tensors[weight_global_scale_key] = _reciprocal_global_scale(raw_tensors.pop(weight_scale_2_key))
        elif weight_scale_2_key in raw_tensors:
            raw_tensors.pop(weight_scale_2_key)

        if input_scale_key in raw_tensors and input_global_scale_key not in raw_tensors:
            raw_tensors[input_global_scale_key] = _reciprocal_global_scale(raw_tensors.pop(input_scale_key))
        elif input_scale_key in raw_tensors:
            raw_tensors.pop(input_scale_key)

        converted_layers.append(layer_name)

    if converted_layers:
        shard_prefix = f"[{shard_name}] " if shard_name else ""
        logger.info(f"{shard_prefix}Normalized {len(converted_layers)} legacy NVFP4 layer(s) to llm-compressor naming.")
    return raw_tensors, converted_layers


# Auxiliary tensors emitted next to an Inkling (``inkling_mm_model``) NVFP4 packed
# weight.  Once the packed weight has been dequantized back to high precision
# these carry no information and must not leak into the quantized output.
_INKLING_NVFP4_AUX_SUFFIXES: tuple[str, ...] = (
    ".scale",
    ".scale2",
    ".input_amax",
    ".weight_amax",
    ".original_shape",
)


def _dequant_inkling_nvfp4_slice(
    packed: torch.Tensor,
    scale: torch.Tensor,
    global_scale: float,
    out_features: int,
    in_features: int,
    group_size: int,
    device: str,
    target_dtype: torch.dtype,
) -> torch.Tensor:
    """Dequantize one 2-D Inkling NVFP4 slice back to *target_dtype* on CPU."""
    from auto_round.experimental.qmodules.fp4_utils import unpack_fp4_from_uint8

    packed_dev = packed.to(device)
    unpacked = unpack_fp4_from_uint8(packed_dev, out_features, in_features, dtype=torch.float32)
    n_groups = in_features // group_size
    scale_dev = scale.to(device).to(torch.float32).reshape(out_features, n_groups, 1)
    weight = unpacked.reshape(out_features, n_groups, group_size) * scale_dev
    weight = weight.reshape(out_features, in_features) * global_scale
    result = weight.to(target_dtype).to("cpu")
    del packed_dev, unpacked, scale_dev, weight
    return result


def _dequant_inkling_nvfp4_tensors(
    raw_tensors: dict[str, torch.Tensor],
    device: str = "cpu",
    shard_name: str | None = None,
    target_dtype: torch.dtype = torch.float32,
) -> dict[str, torch.Tensor]:
    """Dequantize Inkling-specific NVFP4 source tensors to high precision.

    Inkling (``model_type="inkling_mm_model"``) NVFP4 checkpoints use a special layout
    that differs from the generic ModelOpt ``.weight`` / ``.weight_scale`` /
    ``.weight_scale_2`` naming (handled by :func:`_normalize_nvfp4_source_tensors`):
    a quantized weight is stored as a *packed uint8* tensor ``X`` accompanied by:

    * ``X.scale``  – per-block (group_size=16) scale, ``float8_e4m3fn``
    * ``X.scale2`` – global scale (scalar, or one per expert for fused MoE)
    * ``X.input_amax`` / ``X.original_shape`` – activation / bookkeeping metadata

    Note the base tensor has **no** ``.weight`` suffix (e.g.
    ``...mlp.experts.w13_weight``), so neither the generic FP8 dequant path nor
    :func:`_normalize_nvfp4_source_tensors` (which keys off ``.weight`` /
    ``.weight_packed`` + ``.weight_scale``) recognises it.  Without this pass the
    packed uint8 bytes were quantized *as if they were real weights*, producing
    silently corrupted output with half the expected ``in_features``.

    Both plain 2-D weights and fused 3-D stacked MoE expert weights
    (``[num_experts, out, in]``) are supported.  All auxiliary tensors listed in
    :data:`_INKLING_NVFP4_AUX_SUFFIXES` are dropped after dequantization.
    """
    entries: list[str] = []
    for name, tensor in raw_tensors.items():
        if tensor.dtype != torch.uint8 or tensor.dim() not in (2, 3):
            continue
        scale = raw_tensors.get(f"{name}.scale")
        if scale is None or f"{name}.scale2" not in raw_tensors:
            continue
        if scale.dtype not in (torch.float8_e4m3fn, torch.float8_e5m2, torch.float32, torch.bfloat16, torch.float16):
            continue
        if scale.dim() != tensor.dim():
            continue
        entries.append(name)

    if not entries:
        return raw_tensors

    shard_prefix = f"[{shard_name}] " if shard_name else ""
    logger.info(
        f"{shard_prefix}Dequantizing {len(entries)} Inkling NVFP4 source tensor(s) to {target_dtype} on {device}."
    )

    n_dropped = 0
    for name in entries:
        packed = raw_tensors.pop(name)
        scale = raw_tensors.pop(f"{name}.scale")
        scale2 = raw_tensors.pop(f"{name}.scale2")
        global_scale = scale2.float().flatten()

        out_features = packed.shape[-2]
        in_features = packed.shape[-1] * 2
        group_size = in_features // scale.shape[-1]

        try:
            if packed.dim() == 2:
                dequantized = _dequant_inkling_nvfp4_slice(
                    packed,
                    scale,
                    float(global_scale[0]),
                    out_features,
                    in_features,
                    group_size,
                    device,
                    target_dtype,
                )
            else:
                num_experts = packed.shape[0]
                if global_scale.numel() not in (1, num_experts):
                    raise ValueError(f"scale2 has {global_scale.numel()} element(s); expected 1 or {num_experts}.")
                # Fill a pre-allocated buffer instead of stacking a list of
                # slices, so peak memory stays at one output tensor + one slice.
                dequantized = torch.empty((num_experts, out_features, in_features), dtype=target_dtype)
                for i in range(num_experts):
                    dequantized[i] = _dequant_inkling_nvfp4_slice(
                        packed[i],
                        scale[i],
                        float(global_scale[i if global_scale.numel() > 1 else 0]),
                        out_features,
                        in_features,
                        group_size,
                        device,
                        target_dtype,
                    )
        except Exception as e:
            logger.warning(
                f"{shard_prefix}Failed to dequantize Inkling NVFP4 tensor '{name}': {e}. "
                f"Keeping the packed source tensor unchanged."
            )
            raw_tensors[name] = packed
            raw_tensors[f"{name}.scale"] = scale
            raw_tensors[f"{name}.scale2"] = scale2
            continue

        raw_tensors[name] = dequantized
        del packed, scale, scale2, global_scale

        # Drop the now-meaningless auxiliary metadata tensors.
        for suffix in _INKLING_NVFP4_AUX_SUFFIXES:
            if raw_tensors.pop(f"{name}{suffix}", None) is not None:
                n_dropped += 1

    if n_dropped:
        logger.info(f"{shard_prefix}Dropped {n_dropped} stale NVFP4 metadata tensor(s) after dequantization.")

    clear_memory()
    return raw_tensors


def _handle_nvfp4_source_tensors(
    raw_tensors: dict[str, torch.Tensor],
    matcher: Any,
    device: str = "cpu",
    shard_name: str | None = None,
) -> tuple[dict[str, torch.Tensor], dict[str, torch.Tensor], list[str]]:
    """Passthrough NVFP4 source tensors when target scheme for the layer is NVFP4.

    Layers whose target scheme is *not* NVFP4 (e.g. converting an NVFP4 donor
    checkpoint to MXFP4) are dequantized to float32 via
    :func:`_dequant_nvfp4_tensors` instead of being left as raw
    ``weight_packed``/``weight_scale`` tensors — otherwise those companion
    scale tensors would fall through to the generic RTN/MXFP quantization path
    and be mistaken for ordinary weights, corrupting both the weight and its
    (unrelated) scale.
    """
    passthrough_tensors: dict[str, torch.Tensor] = {}
    passthrough_layers: list[str] = []
    dequant_layers: list[str] = []
    # MXFP4 sources also use weight_packed + weight_scale; never treat them as NVFP4.
    nvfp4_layers = _collect_nvfp4_source_layers(raw_tensors)

    for name, tensor in list(raw_tensors.items()):
        if not name.endswith(".weight_packed") or tensor.dtype not in (torch.uint8, torch.int8):
            continue

        layer_name = name[: -len(".weight_packed")]
        scale_key = f"{layer_name}.weight_scale"
        if scale_key not in raw_tensors or layer_name not in nvfp4_layers:
            continue

        scheme = matcher.resolve_scheme(f"{layer_name}.weight")
        if scheme is None:
            continue
        scheme_bits = scheme.get("bits")
        scheme_data_type = (scheme.get("data_type") or "").lower()
        if scheme_bits == 4 and (is_nv_fp(scheme_data_type) or scheme_data_type == _NVFP4_E5M3_DATA_TYPE):
            keys_to_move = [name, scale_key]
            weight_global_scale_key = f"{layer_name}.weight_global_scale"
            input_global_scale_key = f"{layer_name}.input_global_scale"
            if weight_global_scale_key in raw_tensors:
                keys_to_move.append(weight_global_scale_key)
            if input_global_scale_key in raw_tensors:
                keys_to_move.append(input_global_scale_key)

            for key in keys_to_move:
                value = raw_tensors.pop(key).to("cpu")
                if key.endswith("_global_scale"):
                    value = value.reshape([1])
                passthrough_tensors[key] = value
            passthrough_layers.append(layer_name)
        else:
            dequant_layers.append(layer_name)

    if passthrough_layers:
        logger.info(f"Handling NVFP4 source tensor(s): {len(passthrough_layers)} passthrough layer(s).")

    if dequant_layers:
        raw_tensors = _dequant_nvfp4_tensors(raw_tensors, dequant_layers, device=device, shard_name=shard_name)

    return raw_tensors, passthrough_tensors, passthrough_layers


def _dequant_nvfp4_tensors(
    raw_tensors: dict[str, torch.Tensor],
    layer_names: list[str],
    device: str = "cpu",
    shard_name: str | None = None,
) -> dict[str, torch.Tensor]:
    """Dequantize NVFP4-packed weight tensors to float32.

    Reverses the NVFP4 packing (see :func:`auto_round.data_type.nvfp.nv_fp4`):
    ``real_weight = unpack(weight_packed) * weight_scale.float() * (1 / weight_global_scale)``
    where ``weight_scale`` is the per-block (group_size=16) FP8 E4M3 scale and
    ``weight_global_scale`` is the per-tensor FP32 global scale (already
    reciprocal-normalized by :func:`_normalize_nvfp4_source_tensors`).

    The dequantized weight is written back under ``<layer>.weight`` (removing
    ``weight_packed``/``weight_scale``/``weight_global_scale``/
    ``input_global_scale``) so the downstream RTN/MXFP path can requantize it
    to the requested target scheme.
    """
    from auto_round.data_type.nvfp import get_reciprocal
    from auto_round.experimental.qmodules.fp4_utils import unpack_fp4_from_uint8

    group_size = 16
    dequant_device = str(device or "cpu")
    shard_prefix = f"[{shard_name}] " if shard_name else ""
    logger.info(f"{shard_prefix}Dequantizing {len(layer_names)} NVFP4 tensor(s) to float32 on {dequant_device}.")

    for layer_name in layer_names:
        packed_key = f"{layer_name}.weight_packed"
        scale_key = f"{layer_name}.weight_scale"
        if packed_key not in raw_tensors or scale_key not in raw_tensors:
            continue
        packed = raw_tensors.pop(packed_key)
        packed = packed.contiguous().view(torch.uint8)
        block_scale = raw_tensors.pop(scale_key)
        global_scale = raw_tensors.pop(f"{layer_name}.weight_global_scale", None)
        # Input scale has no meaning once the weight is dequantized back to a
        # plain high-precision tensor; drop it.
        raw_tensors.pop(f"{layer_name}.input_global_scale", None)

        def _do_dequant(
            packed: torch.Tensor = packed,
            block_scale: torch.Tensor = block_scale,
            global_scale: torch.Tensor | None = global_scale,
        ) -> torch.Tensor:
            out_features, half_in_features = packed.shape
            in_features = half_in_features * 2
            unpacked = unpack_fp4_from_uint8(packed, out_features, in_features, dtype=torch.float32)
            n_groups = in_features // group_size
            scale_f = block_scale.to(torch.float32).reshape(out_features, n_groups, 1)
            dq = (unpacked.reshape(out_features, n_groups, group_size) * scale_f).reshape(out_features, in_features)
            if global_scale is not None:
                dq = dq * get_reciprocal(global_scale.to(torch.float32).reshape(()))
            return dq

        dq_weight = _dequantize_with_device_fallback(
            dequant_device=dequant_device,
            shard_prefix=shard_prefix,
            op_name="NVFP4 dequant",
            tensor_label=layer_name,
            on_device=lambda: _do_dequant(
                packed=packed.to(dequant_device, non_blocking=True),
                block_scale=block_scale.to(dequant_device, non_blocking=True),
                global_scale=(global_scale.to(dequant_device, non_blocking=True) if global_scale is not None else None),
            ).to("cpu"),
            on_cpu=_do_dequant,
        )
        raw_tensors[f"{layer_name}.weight"] = dq_weight

    return raw_tensors


def _is_out_of_memory_error(exc: Exception) -> bool:
    if isinstance(exc, torch.OutOfMemoryError):
        return True
    message = str(exc).lower()
    return "out of memory" in message or "cuda error: out of memory" in message


def _dequantize_with_device_fallback(
    *,
    dequant_device: str,
    shard_prefix: str,
    op_name: str,
    tensor_label: str,
    on_device: Callable[[], torch.Tensor],
    on_cpu: Callable[[], torch.Tensor],
) -> torch.Tensor:
    """Run dequantization on ``dequant_device`` and fall back to CPU on errors."""
    if dequant_device != "cpu":
        failure: tuple[bool, str] | None = None
        try:
            return on_device()
        except Exception as e:
            # Only keep plain data here.  Holding the exception object alive
            # keeps its traceback (and therefore every device tensor still
            # referenced by the failed frames) reachable, which would make the
            # clear_memory() below a no-op for exactly the memory we need back.
            failure = (_is_out_of_memory_error(e), str(e))

        # Outside the ``except`` block the exception and its traceback have
        # already been released, so reclaiming memory is now effective.
        is_oom, message = failure
        if is_oom:
            logger.warning(
                f"{shard_prefix}{op_name} on {dequant_device} ran OOM for {tensor_label}: {message}. "
                "Clearing accelerator memory and falling back to CPU for this tensor."
            )
            clear_memory()
        else:
            logger.warning(
                f"{shard_prefix}{op_name} on {dequant_device} failed for {tensor_label}: {message}. "
                "Falling back to CPU for this tensor."
            )
    return on_cpu()


def _normalize_scheme(scheme: str | QuantizationScheme) -> QuantizationScheme:
    """Convert *scheme* to a :class:`QuantizationScheme` instance.

    Raises ``ValueError`` for unknown preset names and ``TypeError`` for
    unsupported types.
    """
    if isinstance(scheme, str):
        scheme_name = scheme.upper()
        if scheme_name not in PRESET_SCHEMES:
            raise ValueError(f"Unknown scheme '{scheme}'. Available: {list(PRESET_SCHEMES.keys())}")
        return preset_name_to_scheme(scheme_name)
    if isinstance(scheme, QuantizationScheme):
        return scheme
    raise TypeError(f"Unsupported scheme type: {type(scheme)}")


# ---------------------------------------------------------------------------
# Shard Tensor Processing
# ---------------------------------------------------------------------------


def _is_eligible_weight(tensor_name: str, tensor: torch.Tensor) -> bool:
    """Check if a tensor is eligible for quantization (2D high-precision Linear weight).

    ``is_floating_point`` is required: an already-packed weight (uint8/int8 FP4
    nibbles, int32 GPTQ words, ...) that slipped past the source-format handlers
    must never be fed to the quantizer, which would reinterpret the packed bytes
    as real values and silently corrupt the layer.
    """
    return tensor_name.endswith(".weight") and tensor.dim() == 2 and tensor.is_floating_point()


def _is_moe_fused_expert_weight(tensor_name: str, tensor: torch.Tensor) -> bool:
    """Check if *tensor* is a 3-D fused per-layer stacked MoE expert weight
    (e.g. ``experts.w13_weight`` / ``experts.w2_weight``, shape
    ``[num_experts, out, in]``).

    Such tensors are produced when :func:`split_fused_expert_tensors
    <auto_round.utils.missing_tensors.split_fused_expert_tensors>` skips
    unfusing for the source model's ``model_type`` (see
    ``_KEEP_FUSED_EXPERT_MODEL_TYPES``) because the target inference engine's
    loader expects the fused layout preserved rather than split per expert.

    Matches ``<prefix>.experts.<proj>_weight`` as well as the nested tensor
    layouts ``<prefix>.experts.<proj>_weight.weight`` (dequantized) and
    ``<prefix>.experts.<proj>_weight.weight_packed`` (MXFP4 source).
    In some model families (e.g. Inkling) shared experts appear under the
    path segment ``shared_experts`` (e.g. ``...shared_experts.shared_w13_weight``).
    Treat these fused 3-D shared expert tensors the same as ordinary fused
    expert tensors so they can be considered for in-place quantization.
    Auxiliary 3-D tensors such as ``.scale`` / ``.weight_scale`` are still
    excluded because they are not weight tensors.
    """
    if tensor.dim() != 3:
        return False
    parts = tensor_name.split(".")
    # Accept fused experts stored under either "...experts..." or
    # "...shared_experts..." path segments.
    if "experts" not in parts and "shared_experts" not in parts:
        return False
    last = parts[-1]
    if last in ("weight", "weight_packed"):
        return True
    return len(parts) >= 2 and parts[-2] in ("experts", "shared_experts") and last.endswith("_weight")


def _fused_expert_layer_name(tensor_name: str) -> str:
    """Strip the tensor-kind suffix from a fused MoE expert tensor name.

    ``experts.w13_weight`` stays as-is, while ``experts.w13_weight.weight`` and
    ``experts.w13_weight.weight_packed`` collapse to ``experts.w13_weight`` so
    quantized / ignored layer bookkeeping uses one consistent layer name.
    """
    for suffix in (".weight_packed", ".weight"):
        if tensor_name.endswith(suffix):
            return tensor_name[: -len(suffix)]
    return tensor_name


def _quantize_moe_fused_expert_weight(
    tensor_name: str,
    tensor: torch.Tensor,
    matcher: _PatternMatcher,
    device: str = "cpu",
    disable_opt_rtn: bool = False,
) -> tuple[str, dict[str, torch.Tensor], str | None, str | None]:
    """Quantize a 3-D fused per-layer stacked MoE expert weight in place.

    Each expert's 2-D slice (``tensor[i]``) is quantized independently via
    :func:`_quantize_weight_mxfp` and the packed outputs are re-stacked along
    a new leading dimension, preserving the tensor's original fused layout
    (e.g. ``experts.w13_weight.weight_packed`` stays 3-D instead of being
    unfused into per-expert 2-D tensors). RTN/MXFP quantization groups values
    along the last (``in_features``) dimension on a per-row basis, so this is
    numerically equivalent to quantizing whatever 2-D weight each row
    originally came from -- any gate/up row interleaving within a slice does
    not affect correctness.

    Only the MXFP4/MXFP8 path is currently supported for this layout; other
    schemes fall back to keeping the original (unquantized) weight.

    Returns:
        (layer_name, output_tensors_dict, quantized_layer_or_None, ignored_layer_or_None)
    """
    layer_name = _fused_expert_layer_name(tensor_name)
    matcher_name = f"{layer_name}.weight"

    if not tensor.is_floating_point():
        # An already-packed source tensor (e.g. NVFP4/MXFP4 uint8 nibbles) that
        # was not consumed by a source-format handler. Quantizing it would
        # reinterpret packed bytes as weights and silently corrupt the layer.
        logger.warning_once(
            f"Fused 3-D MoE tensor '{tensor_name}' has non-floating dtype {tensor.dtype}; "
            "it appears to be already packed and is kept unchanged instead of being quantized."
        )
        return layer_name, {tensor_name: tensor}, None, layer_name

    if matcher.should_ignore(matcher_name) or matcher.should_skip(matcher_name):
        return layer_name, {tensor_name: tensor}, None, layer_name

    scheme = matcher.resolve_scheme(matcher_name)
    if scheme is None:
        return layer_name, {tensor_name: tensor}, None, layer_name

    bits = scheme["bits"]
    if bits >= 16:
        return layer_name, {tensor_name: tensor}, None, layer_name

    data_type = (scheme.get("data_type") or "int").lower()
    group_size = scheme["group_size"]

    if not is_mx_fp(data_type):
        logger.warning_once(
            f"3-D fused MoE weight '{tensor_name}' (shape={list(tensor.shape)}) is only "
            "supported for MXFP schemes in model_free mode; keeping original weight."
        )
        return layer_name, {tensor_name: tensor}, None, layer_name

    try:
        packed_parts: dict[str, list[torch.Tensor]] = {}
        for i in range(tensor.shape[0]):
            slice_out = _quantize_weight_mxfp(
                weight=tensor[i],
                layer_name=f"{layer_name}.{i}",
                bits=bits,
                group_size=group_size,
                data_type=data_type,
                device=device,
                disable_opt_rtn=disable_opt_rtn,
            )
            prefix = f"{layer_name}.{i}"
            for key, value in slice_out.items():
                suffix = key[len(prefix) :]  # e.g. ".weight_packed" / ".weight_scale"
                packed_parts.setdefault(suffix, []).append(value)

        out = {f"{layer_name}{suffix}": torch.stack(values, dim=0) for suffix, values in packed_parts.items()}
        # The per-expert slices have been copied into the stacked tensors; free
        # them immediately instead of holding a second full-size copy.
        packed_parts.clear()
        logger.debug(
            f"Quantized (MXFP, fused 3-D): {layer_name} "
            f"(bits={bits}, group_size={group_size}, num_experts={tensor.shape[0]})"
        )
        return layer_name, out, layer_name, None
    except Exception as e:
        logger.warning(f"Failed to MXFP-quantize fused 3-D MoE weight {layer_name}: {e}. Keeping original weight.")
        return layer_name, {tensor_name: tensor}, None, layer_name


def _quantize_weight_mxfp(
    weight: torch.Tensor,
    layer_name: str,
    bits: int,
    group_size: int,
    data_type: str,
    device: str = "cpu",
    disable_opt_rtn: bool = False,
) -> dict[str, torch.Tensor]:
    """Quantize a 2D weight tensor to MXFP4 / MXFP8 and return packed outputs.

    Reuses :func:`auto_round.data_type.mxfp.quant_mx` to derive the per-block
    shared exponent (E8M0 scale), and :class:`auto_round.export.export_to_autoround.qlinear_fp.QuantLinear`
    to perform the same packing as :func:`auto_round.export.export_to_llmcompressor.export_to_fp.pack_layer`.

    Returns a dict with one of:
      * MXFP8: ``{layer_name+'.weight': float8_e4m3fn, layer_name+'.weight_scale': uint8}``
      * MXFP4: ``{layer_name+'.weight_packed': uint8, layer_name+'.weight_scale': uint8}``
    """
    import torch.nn as nn

    from auto_round.data_type.utils import get_quant_func
    from auto_round.export.export_to_autoround.qlinear_fp import QuantLinear

    if not is_mx_fp(data_type):
        data_type = "mx_fp4" if bits == 4 else "mx_fp8"

    out_features, in_features = weight.shape
    if in_features % group_size != 0:
        raise ValueError(
            f"in_features={in_features} for layer '{layer_name}' is not divisible "
            f"by MXFP group_size={group_size}; cannot pack."
        )

    weight_dev = weight.to(device=device, dtype=torch.float32)
    # Use get_quant_func (same as WrapperLinear) so that all registered MXFP
    # variants automatically get opt_rtn support via the QUANT_FUNC_WITH_DTYPE
    # registry (e.g. "opt_rtn_mx_fp4" -> quant_mx_opt_rtn, "mx_fp4" -> quant_mx).
    quant_func, _ = get_quant_func(data_type, bits, sym=True, disable_opt_rtn=disable_opt_rtn, iters=0)
    weight_dev, shared_exp, _ = quant_func(weight_dev, bits=bits, group_size=group_size, data_type=data_type)
    # Reshape to (out_features, n_groups) so the on-disk weight_scale matches
    # the llm-compressor convention (and QuantLinear's registered buffer shape).
    shared_exp = shared_exp.reshape(out_features, in_features // group_size)
    # Ensure shared_exp is a numeric float (not a storage-specific dtype like
    # float8) — QuantLinear.pack performs `2 ** scales` which dispatches to
    # torch.pow; some backends do not implement pow for float8 dtypes. Cast to
    # float32 here to avoid runtime errors like "pow_cuda not implemented for
    # 'Float8_e4m3fn'" while preserving numeric values.
    shared_exp = shared_exp.to(torch.float32)

    # Build a lightweight nn.Linear holding the original weight so we can
    # delegate packing to the existing QuantLinear.pack implementation.
    # Materialize it on the meta device so no (out_features x in_features)
    # buffer is allocated/initialized just to be immediately overwritten.
    with torch.device("meta"):
        fake_linear = nn.Linear(in_features, out_features, bias=False)
    with torch.no_grad():
        fake_linear.weight = nn.Parameter(weight_dev, requires_grad=False)

    # QuantLinear registers zero-filled packed weight/scale buffers that pack()
    # replaces wholesale; allocate them on meta to avoid the throwaway copy.
    with torch.device("meta"):
        qlayer = QuantLinear(
            bits=bits,
            group_size=group_size,
            infeatures=in_features,
            outfeatures=out_features,
            bias=False,
            data_type="mx_fp4" if bits == 4 else "mx_fp8e4m3",
            sym=True,
            act_bits=bits,
        )
    qlayer.pack(fake_linear, shared_exp, device=device)

    # Drop every intermediate before returning so the (potentially large)
    # high-precision weight and device-side buffers are freed as soon as the
    # packed CPU copies exist.
    if bits == 8:
        out = {
            f"{layer_name}.weight": qlayer.weight.to("cpu"),
            f"{layer_name}.weight_scale": qlayer.weight_scale.to("cpu"),
        }
    else:
        out = {
            f"{layer_name}.weight_packed": qlayer.weight_packed.to("cpu"),
            f"{layer_name}.weight_scale": qlayer.weight_scale.to("cpu"),
        }
    del qlayer, fake_linear, weight_dev, shared_exp
    return out


def _quantize_weight_nvfp4_e5m3(
    weight: torch.Tensor,
    layer_name: str,
    group_size: int = 16,
    device: str = "cpu",
    disable_opt_rtn: bool = False,
) -> dict[str, torch.Tensor]:
    """Fake-quantize a 2D weight tensor to NVFP4 E5M3 and return its high-precision QDQ weight."""
    from auto_round.data_type.nvfp import nvfp4_v2, search_nvfp4_v2_scale
    from auto_round.data_type.utils import reshape_pad_tensor_by_group_size

    out_features, in_features = weight.shape
    if group_size != 16:
        raise ValueError(f"NVFP4_E5M3 requires group_size=16, got {group_size} for layer '{layer_name}'.")
    if in_features % group_size != 0:
        raise ValueError(
            f"in_features={in_features} for layer '{layer_name}' is not divisible "
            f"by NVFP4_E5M3 group_size={group_size}; cannot quantize."
        )

    weight_dev = weight.to(device=device, dtype=torch.float32)
    max_scale = 1.0
    if not disable_opt_rtn:
        grouped = reshape_pad_tensor_by_group_size(weight_dev, group_size)[0]
        max_scale = search_nvfp4_v2_scale(grouped)
    qdq_weight, _, _ = nvfp4_v2(weight_dev, bits=4, group_size=group_size, max_scale=max_scale)
    return {f"{layer_name}.weight": qdq_weight.to(dtype=weight.dtype, device="cpu")}


def _qdq_weight_nvfp4_e5m3_gs(
    weight: torch.Tensor, layer_name: str, group_size: int, device: str, disable_opt_rtn: bool
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
    """Return (fp32 weight, qdq weight, encoded-domain block scale, weight global scale) for NVFP4_E5M3_GS."""
    from auto_round.data_type.nvfp import calculate_gparam_e5m3, nvfp4_v2_gs, opt_rtn_nvfp4_v2_gs

    in_features = weight.shape[1]
    if group_size != 16 or in_features % group_size != 0:
        raise ValueError(
            f"NVFP4_E5M3_GS requires in_features divisible by group_size=16, got group_size={group_size}, "
            f"in_features={in_features} for '{layer_name}'."
        )
    weight_dev = weight.to(device=device, dtype=torch.float32)
    global_scale = calculate_gparam_e5m3(weight_dev, group_size).reshape(1)
    quant_func = nvfp4_v2_gs if disable_opt_rtn else opt_rtn_nvfp4_v2_gs
    qdq_weight, scale, _ = quant_func(weight_dev, bits=4, group_size=group_size, global_scale=global_scale)
    return weight_dev, qdq_weight, scale, global_scale


def _quantize_weight_nvfp4_e5m3_gs(
    weight: torch.Tensor,
    layer_name: str,
    group_size: int = 16,
    device: str = "cpu",
    disable_opt_rtn: bool = False,
) -> dict[str, torch.Tensor]:
    """Fake-quantize a 2D weight tensor to NVFP4_E5M3_GS; model-free uses a fixed input global scale of 1."""
    _, qdq_weight, _, _ = _qdq_weight_nvfp4_e5m3_gs(weight, layer_name, group_size, device, disable_opt_rtn)
    return {
        f"{layer_name}.weight": qdq_weight.to(dtype=weight.dtype, device="cpu"),
        f"{layer_name}.input_global_scale": torch.ones(1, dtype=torch.float32),
    }


def _pack_weight_nvfp4_e5m3_gs(
    weight: torch.Tensor,
    layer_name: str,
    group_size: int = 16,
    device: str = "cpu",
    disable_opt_rtn: bool = False,
) -> dict[str, torch.Tensor]:
    """Pack FP4 E2M1 weights with UE5M3 block scales and a per-tensor weight global scale."""
    from auto_round.export.export_to_autoround.qlinear_fp import QuantLinear

    out_features, in_features = weight.shape
    weight_dev, _, scale, global_scale = _qdq_weight_nvfp4_e5m3_gs(
        weight, layer_name, group_size, device, disable_opt_rtn
    )
    scale = scale.reshape(out_features, in_features // group_size).to(torch.float32)
    with torch.device("meta"):
        linear = torch.nn.Linear(in_features, out_features, bias=False, dtype=torch.float32)
    linear.weight = torch.nn.Parameter(weight_dev, requires_grad=False)
    with torch.device("meta"):
        qlayer = QuantLinear(
            4,
            group_size,
            in_features,
            out_features,
            False,
            data_type=_NVFP4_E5M3_GS_DATA_TYPE,
            act_bits=4,
            act_data_type=_NVFP4_E5M3_GS_ACT_DATA_TYPE,
        )
    qlayer.pack(
        linear,
        scale,
        global_scale=global_scale,
        input_global_scale=torch.ones(1, dtype=torch.float32),
        device=device,
    )
    out = {
        f"{layer_name}.weight_packed": qlayer.weight_packed.to("cpu"),
        f"{layer_name}.weight_scale": qlayer.weight_scale.to("cpu"),
        f"{layer_name}.weight_global_scale": qlayer.weight_global_scale.to("cpu"),
        f"{layer_name}.input_global_scale": qlayer.input_global_scale.to("cpu"),
    }
    del qlayer, linear, weight_dev, scale
    return out


def _quantize_weight_int_fake(
    weight: torch.Tensor,
    layer_name: str,
    bits: int,
    group_size: int,
    sym: bool,
    device: str = "cpu",
    disable_opt_rtn: bool = True,
) -> dict[str, torch.Tensor]:
    """Fake-quantize an INT WOQ weight tensor and return its QDQ weight."""
    from auto_round.data_type.utils import get_quant_func

    original_dtype = weight.dtype
    original_in_features = weight.shape[1]
    weight_dev = weight.to(device=device, dtype=torch.float32)
    if original_in_features % group_size:
        weight_dev = torch.nn.functional.pad(weight_dev, (0, group_size - original_in_features % group_size))

    quant_func, _ = get_quant_func("int", bits, sym=sym, disable_opt_rtn=disable_opt_rtn, iters=0)
    qdq_weight, _, _ = quant_func(weight_dev, bits=bits, group_size=group_size)
    return {f"{layer_name}.weight": qdq_weight[:, :original_in_features].to(dtype=original_dtype, device="cpu")}


def _pack_weight_nvfp4_e5m3(
    weight: torch.Tensor,
    layer_name: str,
    group_size: int = 16,
    device: str = "cpu",
    disable_opt_rtn: bool = False,
) -> dict[str, torch.Tensor]:
    """Pack FP4 E2M1 weights with unsigned E5M3 block scales."""
    from auto_round.data_type.nvfp import nvfp4_v2, search_nvfp4_v2_scale
    from auto_round.data_type.utils import reshape_pad_tensor_by_group_size
    from auto_round.export.export_to_autoround.qlinear_fp import QuantLinear

    out_features, in_features = weight.shape
    if group_size != 16 or in_features % group_size != 0:
        raise ValueError(
            f"NVFP4_E5M3 requires in_features divisible by group_size=16, got {in_features} for '{layer_name}'."
        )
    weight_dev = weight.to(device=device, dtype=torch.float32)
    max_scale = 1.0
    if not disable_opt_rtn:
        grouped = reshape_pad_tensor_by_group_size(weight_dev, group_size)[0]
        max_scale = search_nvfp4_v2_scale(grouped)
    _, scale, _ = nvfp4_v2(weight_dev, bits=4, group_size=group_size, max_scale=max_scale)
    # nvfp4_v2 may return a flattened per-group scale layout (e.g. [N, 1]);
    # normalize to [out_features, in_features // group_size] before packing
    # so serialized .weight_scale keeps the expected 2D shape.
    scale = scale.reshape(out_features, in_features // group_size).to(torch.float32)
    # Meta-device construction avoids allocating a full weight tensor on the
    # accelerator (and zero-filled packed buffers) that are immediately replaced.
    with torch.device("meta"):
        linear = torch.nn.Linear(in_features, out_features, bias=False, dtype=torch.float32)
    linear.weight = torch.nn.Parameter(weight_dev, requires_grad=False)
    with torch.device("meta"):
        qlayer = QuantLinear(
            4, group_size, in_features, out_features, False, data_type="nvfp4_v2", act_bits=4, act_data_type="nvfp4_v2"
        )
    qlayer.pack(linear, scale, device=device)
    out = {
        f"{layer_name}.weight_packed": qlayer.weight_packed.to("cpu"),
        f"{layer_name}.weight_scale": qlayer.weight_scale.to("cpu"),
    }
    del qlayer, linear, weight_dev, scale
    return out


def _declared_int_packing(sym: bool) -> str:
    """Packing format model-free int artifacts declare, by symmetry.

    Single source shared by the packing call (``_quantize_single_tensor``) and
    the config builder (``_build_quantization_config``) so the written bytes and
    the declared packing_format can never disagree: symmetric int artifacts are
    declared ``auto_round:auto_gptq`` (qlinear_torch_zp, zero points packed as
    zp - 1), asymmetric int artifacts are declared ``auto_round`` (plain
    qlinear_torch, zero points packed directly).
    """
    from auto_round.inference.backend import GPTQ_FORMAT, GPTQ_FORMAT_NO_ZP

    return GPTQ_FORMAT[0] if sym else GPTQ_FORMAT_NO_ZP[0]


def _quantize_single_tensor(
    tensor_name: str,
    tensor: torch.Tensor,
    matcher: _PatternMatcher,
    device: str = "cpu",
    quantize_func: Callable = quantize_weight_rtn,
    disable_opt_rtn: bool = False,
) -> tuple[str, dict[str, torch.Tensor], str | None, str | None]:
    """Quantize one eligible weight tensor and return packed outputs.

    Returns:
        (layer_name, output_tensors_dict, quantized_layer_or_None, ignored_layer_or_None)
    """
    if _is_moe_fused_expert_weight(tensor_name, tensor):
        return _quantize_moe_fused_expert_weight(tensor_name, tensor, matcher, device, disable_opt_rtn)

    layer_name = tensor_name.rsplit(".", 1)[0]

    if not _is_eligible_weight(tensor_name, tensor):
        ignored_layer = layer_name if tensor_name.endswith(".weight") and tensor.dim() > 1 else None
        return layer_name, {tensor_name: tensor}, None, ignored_layer

    if matcher.should_ignore(tensor_name):
        logger.debug(f"Ignoring (user-specified): {layer_name}")
        return layer_name, {tensor_name: tensor}, None, layer_name

    if matcher.should_skip(tensor_name):
        logger.debug(f"Skipping (predefined): {layer_name}")
        return layer_name, {tensor_name: tensor}, None, layer_name

    scheme = matcher.resolve_scheme(tensor_name)
    if scheme is None:
        logger.debug(f"Keeping full precision: {layer_name}")
        return layer_name, {tensor_name: tensor}, None, layer_name

    bits = scheme["bits"]
    group_size = scheme["group_size"]
    sym = scheme.get("sym", True)
    data_type = (scheme.get("data_type") or "int").lower()

    if bits >= 16:
        return layer_name, {tensor_name: tensor}, None, layer_name

    # ---- MXFP path (MXFP4 / MXFP8) ----
    if is_mx_fp(data_type):
        try:
            out = _quantize_weight_mxfp(
                weight=tensor,
                layer_name=layer_name,
                bits=bits,
                group_size=group_size,
                data_type=data_type,
                device=device,
                disable_opt_rtn=disable_opt_rtn,
            )
            logger.debug(f"Quantized (MXFP): {layer_name} (bits={bits}, group_size={group_size})")
            return layer_name, out, layer_name, None
        except Exception as e:
            logger.warning(f"Failed to MXFP-quantize {layer_name}: {e}. Keeping original weight.")
            return layer_name, {tensor_name: tensor}, None, layer_name

    # ---- NVFP4 E5M3 fake-quantization path ----
    if data_type == _NVFP4_E5M3_DATA_TYPE:
        try:
            quantize_e5m3 = (
                _quantize_weight_nvfp4_e5m3 if scheme.get("_output_format") == "fake" else _pack_weight_nvfp4_e5m3
            )
            out = quantize_e5m3(
                weight=tensor,
                layer_name=layer_name,
                group_size=group_size,
                device=device,
                disable_opt_rtn=disable_opt_rtn,
            )
            logger.debug(f"Quantized (NVFP4_E5M3): {layer_name} (bits=4, group_size={group_size})")
            return layer_name, out, layer_name, None
        except Exception as e:
            logger.warning(f"Failed to NVFP4_E5M3-quantize {layer_name}: {e}. Keeping original weight.")
            return layer_name, {tensor_name: tensor}, None, layer_name

    if data_type == _NVFP4_E5M3_GS_DATA_TYPE:
        try:
            quantize_e5m3_gs = (
                _quantize_weight_nvfp4_e5m3_gs if scheme.get("_output_format") == "fake" else _pack_weight_nvfp4_e5m3_gs
            )
            out = quantize_e5m3_gs(
                weight=tensor,
                layer_name=layer_name,
                group_size=group_size,
                device=device,
                disable_opt_rtn=disable_opt_rtn,
            )
            logger.debug(f"Quantized (NVFP4_E5M3_GS): {layer_name} (bits=4, group_size={group_size})")
            return layer_name, out, layer_name, None
        except Exception as e:
            logger.warning(f"Failed to NVFP4_E5M3_GS-quantize {layer_name}: {e}. Keeping original weight.")
            return layer_name, {tensor_name: tensor}, None, layer_name

    # ---- Integer WOQ path ----
    # opt_rtn is always disabled for integer WOQ in model-free mode because
    # the scale search does not improve accuracy for INT quantization here.
    try:
        if scheme.get("_output_format") == "fake":
            out = _quantize_weight_int_fake(
                weight=tensor,
                layer_name=layer_name,
                bits=bits,
                group_size=group_size,
                sym=sym,
                device=device,
            )
            logger.debug(f"Fake-quantized: {layer_name} (bits={bits}, group_size={group_size}, sym={sym})")
            return layer_name, out, layer_name, None

        qweight, qzeros, scales = quantize_func(
            weight=tensor,
            bits=bits,
            group_size=group_size,
            sym=sym,
            device=device,
            disable_opt_rtn=True,
            # The packing convention must match the declared packing_format:
            # see _declared_int_packing (single source for bytes and config).
            packing=_declared_int_packing(sym),
        )

        out: dict[str, torch.Tensor] = {
            f"{layer_name}.qweight": qweight,
            f"{layer_name}.qzeros": qzeros,
            f"{layer_name}.scales": scales,
        }

        logger.debug(f"Quantized: {layer_name} (bits={bits}, group_size={group_size}, sym={sym})")
        return layer_name, out, layer_name, None

    except Exception as e:
        logger.warning(f"Failed to quantize {layer_name}: {e}. Keeping original weight.")
        return layer_name, {tensor_name: tensor}, None, layer_name


@lru_cache(maxsize=32)
def _load_weight_map_from_index(index_path: str) -> dict[str, str]:
    """Load weight_map from an index file with a small process-local cache."""
    with open(index_path) as f:
        index = json.load(f)
    weight_map = index.get("weight_map", {})
    return weight_map if isinstance(weight_map, dict) else {}


def _build_cross_shard_pairs_from_weight_map(
    weight_map: dict[str, str],
) -> tuple[dict[str, dict[str, list[str]]], dict[str, set[str]]]:
    """Identify cross-shard (weight, scale-metadata) pairs from weight_map.

    Two source layouts are covered:

    * **FP8**: ``<layer>.weight`` and ``<layer>.weight_scale_inv``.
    * **Inkling NVFP4** (``inkling_mm_model``): a packed base tensor ``X`` and its
      ``X.scale`` / ``X.scale2`` siblings.

    A *cross-shard pair* exists when the two tensors reside in **different**
    shards.  The shard holding the scale metadata is the *donor*; the shard
    holding the weight is the *recipient*.  Recording these lets the shard
    scheduler order donors before recipients (critical in streaming mode, where
    a donor shard must be downloaded before the recipient can hydrate from it).

    Returns:
        recipient_to_donors:
            ``{recipient_shard: {donor_shard: [scale_tensor_names]}}``
        donor_shard_tensors:
            ``{donor_shard: set(scale tensor names it donates to other shards)}``
    """
    recipient_to_donors: dict[str, dict[str, list[str]]] = {}
    donor_shard_tensors: dict[str, set[str]] = {}

    def _record(scale_name: str, donor_shard: str, weight_name: str) -> None:
        weight_shard = weight_map.get(weight_name)
        if not weight_shard or weight_shard == donor_shard:
            return
        # `donor_shard` donates `scale_name` to `weight_shard` (recipient)
        recipient_to_donors.setdefault(weight_shard, {}).setdefault(donor_shard, []).append(scale_name)
        donor_shard_tensors.setdefault(donor_shard, set()).add(scale_name)

    nvfp4_bases = _inkling_nvfp4_bases_from_weight_map(weight_map)

    for tensor_name, shard in weight_map.items():
        if tensor_name.endswith(".weight_scale_inv"):
            # "layer.weight_scale_inv" -> "layer.weight"
            _record(tensor_name, shard, tensor_name[: -len("_scale_inv")])
            continue
        for suffix in (".scale2", ".scale"):
            if tensor_name.endswith(suffix):
                base = tensor_name[: -len(suffix)]
                if base in nvfp4_bases:
                    _record(tensor_name, shard, base)
                break

    return recipient_to_donors, donor_shard_tensors


def _hydrate_missing_fp8_scales_from_index(
    raw_tensors: dict[str, torch.Tensor],
    shard_path: str,
    *,
    shard_name: str | None = None,
    index_dir: str | None = None,
    donor_shard_dir: str | None = None,
) -> dict[str, torch.Tensor]:
    """Populate missing ``.weight_scale_inv`` tensors from sibling shards.

    Some checkpoints shard FP8 weight and its corresponding scale tensor into
    different ``.safetensors`` files. Model-free processing is shard-local, so
    this helper hydrates missing ``<layer>.weight_scale_inv`` tensors by looking
    up ``weight_map`` in ``*.safetensors.index.json`` and loading only the
    needed tensors from referenced shards.

    Args:
        index_dir: Directory that contains ``*.safetensors.index.json``.  When
            ``None`` (default) the directory of *shard_path* is used, which is
            correct for non-streaming mode.  Streaming mode should pass
            ``work_dir`` here because downloaded shards live in a
            ``.cache/model_free_source_shards/`` sub-directory that does not
            contain the index file.
        donor_shard_dir: Directory where donor shard files can be found.  When
            ``None`` (default) the same directory as *index_dir* is used.
            Streaming mode should pass the local shard cache directory so that
            already-downloaded donor shards are resolved correctly.
    """
    if not shard_path.endswith(".safetensors"):
        return raw_tensors

    weight_to_scale: dict[str, str] = {}
    for name, tensor in raw_tensors.items():
        if not name.endswith(".weight"):
            continue
        if tensor.dtype != torch.float8_e4m3fn:
            continue
        scale_inv_name = f"{name[: -len('.weight')]}.weight_scale_inv"
        if scale_inv_name not in raw_tensors:
            weight_to_scale[name] = scale_inv_name

    if not weight_to_scale:
        return raw_tensors

    shard_dir = os.path.dirname(shard_path)
    # Resolve the directory where index.json lives.  In streaming mode the
    # shard lives in a .cache/ sub-directory that has no index file; callers
    # pass index_dir=work_dir to point at the correct location.
    idx_dir = index_dir if index_dir is not None else shard_dir
    # Donor shards are looked up in donor_shard_dir (defaults to idx_dir).
    donor_dir = donor_shard_dir if donor_shard_dir is not None else idx_dir

    index_path = os.path.join(idx_dir, "model.safetensors.index.json")
    if not os.path.exists(index_path):
        candidates = sorted(
            os.path.join(idx_dir, f) for f in os.listdir(idx_dir) if f.endswith(".safetensors.index.json")
        )
        if not candidates:
            return raw_tensors
        index_path = candidates[0]

    try:
        weight_map = _load_weight_map_from_index(index_path)
    except Exception:
        return raw_tensors

    current_shard = os.path.basename(shard_path)
    scales_by_shard: dict[str, list[str]] = {}
    for scale_name in weight_to_scale.values():
        target_shard = weight_map.get(scale_name)
        if not target_shard or target_shard == current_shard:
            continue
        scales_by_shard.setdefault(target_shard, []).append(scale_name)

    if not scales_by_shard:
        return raw_tensors

    from safetensors import safe_open

    hydrated = 0
    shard_prefix = f"[{shard_name}] " if shard_name else ""
    for target_shard, scale_names in scales_by_shard.items():
        target_path = str(resolve_within_directory(donor_dir, target_shard, origin="weight_map"))
        if not os.path.exists(target_path):
            logger.warning(
                f"{shard_prefix}Donor shard '{target_shard}' not found in '{donor_dir}' while hydrating "
                f"{len(scale_names)} FP8 scale_inv tensor(s); the affected weight(s) will remain in "
                f"float8_e4m3fn and may fail downstream quantization. This usually indicates a shard "
                f"scheduling/ordering issue (donor shard processed/downloaded after its recipient)."
            )
            continue
        try:
            with safe_open(target_path, framework="pt", device="cpu") as sf:
                for scale_name in scale_names:
                    if scale_name in raw_tensors:
                        continue
                    try:
                        raw_tensors[scale_name] = sf.get_tensor(scale_name)
                        hydrated += 1
                    except Exception:
                        # Tensor may be absent in this shard; skip lazily.
                        continue
        except Exception:
            continue

    if hydrated:
        logger.info(f"{shard_prefix}Hydrated {hydrated} FP8 scale tensor(s) from sibling shard(s) using index mapping.")

    return raw_tensors


def _find_index_path(idx_dir: str) -> str | None:
    """Return the path of the ``*.safetensors.index.json`` file in *idx_dir*."""
    index_path = os.path.join(idx_dir, "model.safetensors.index.json")
    if os.path.exists(index_path):
        return index_path
    try:
        candidates = sorted(
            os.path.join(idx_dir, f) for f in os.listdir(idx_dir) if f.endswith(".safetensors.index.json")
        )
    except OSError:
        return None
    return candidates[0] if candidates else None


def _inkling_nvfp4_bases_from_weight_map(weight_map: dict[str, str]) -> set[str]:
    """Return base tensor names that use the Inkling NVFP4 layout.

    A base ``X`` qualifies when the checkpoint contains ``X``, ``X.scale`` and
    ``X.scale2`` (see :func:`_dequant_inkling_nvfp4_tensors`).
    """
    return {
        name[: -len(".scale2")]
        for name in weight_map
        if name.endswith(".scale2")
        and name[: -len(".scale2")] in weight_map
        and f"{name[: -len('.scale2')]}.scale" in weight_map
    }


def get_fp_scale(scale_e8m0: torch.Tensor) -> torch.Tensor:
    """Convert biased E8M0 exponents to floating-point scales."""
    exponent = scale_e8m0.view(torch.uint8).to(torch.int16) - 127
    return torch.pow(2.0, exponent.to(torch.float32))


def dequant_mxfp4(
    data_lp: torch.Tensor,
    scale_e8m0: torch.Tensor,
    elem_dtype: str,
    block_size: int,
    target_dtype: torch.dtype,
    scale_dtype: torch.dtype | None = None,
    return_scale: bool = False,
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
    """Dequantize packed MXFP4 data to ``target_dtype``."""
    from auto_round.experimental.qmodules.fp4_utils import unpack_fp4_from_uint8

    original_shape = data_lp.shape
    last_dim = original_shape[-1]
    data_lp = data_lp.reshape(-1, last_dim)
    result_shape = original_shape[:-1] + (last_dim * 2,)
    assert data_lp.is_contiguous(), f"Data must be contiguous, got {data_lp.stride()}"
    assert elem_dtype == "fp4_e2m1", f"Expected 'fp4_e2m1', got {elem_dtype}"

    rows, half_columns = data_lp.shape
    data_hp = unpack_fp4_from_uint8(data_lp, rows, half_columns * 2, dtype=target_dtype)
    data_hp = data_hp.reshape(-1, block_size)

    if scale_dtype is None:
        scale_dtype = target_dtype
    scale = get_fp_scale(scale_e8m0).reshape(-1, 1).to(scale_dtype)
    if return_scale:
        return data_hp.reshape(result_shape), scale

    return (data_hp * scale).reshape(result_shape)


def dequant_mx_fp8(
    weight_fp8: torch.Tensor,
    scale_e8m0: torch.Tensor,
    block_size: int,
    target_dtype: torch.dtype,
) -> torch.Tensor:
    """Dequantize MXFP8 data using its per-block E8M0 scales."""
    original_shape = weight_fp8.shape
    weight = weight_fp8.to(torch.float32).reshape(-1, block_size)
    scale = get_fp_scale(scale_e8m0).reshape(-1, 1)
    return (weight * scale).reshape(original_shape).to(target_dtype)


def _hydrate_and_clean_inkling_nvfp4_aux(
    raw_tensors: dict[str, torch.Tensor],
    shard_path: str,
    *,
    shard_name: str | None = None,
    index_dir: str | None = None,
    donor_shard_dir: str | None = None,
) -> dict[str, torch.Tensor]:
    """Resolve Inkling NVFP4 ``.scale`` / ``.scale2`` tensors across shards.

    Inkling checkpoints do **not** keep a packed weight and its scales in the
    same shard — e.g. ``layers.36...w13_weight`` can sit in shard 6 while its
    ``.scale`` is in shard 3 and its ``.scale2`` in shard 5.  Since model-free
    processing is shard-local, :func:`_dequant_inkling_nvfp4_tensors` would
    otherwise never see a complete triple and the packed weight would be left
    untouched (and reported as an already-packed, non-quantizable tensor).

    This helper does two things, both driven by ``*.safetensors.index.json``:

    1. **Hydrate** the missing ``.scale`` / ``.scale2`` tensors for every NVFP4
       base weight present in this shard, reading only those tensors from the
       sibling shards that hold them.
    2. **Drop** orphan auxiliary tensors (``.scale``, ``.scale2``,
       ``.input_amax``, ``.original_shape``) whose base weight lives in another
       shard, so this shard's output is not polluted with stale metadata that
       the consuming shard has already accounted for.
    """
    if not shard_path.endswith(".safetensors"):
        return raw_tensors

    shard_dir = os.path.dirname(shard_path)
    idx_dir = index_dir if index_dir is not None else shard_dir
    donor_dir = donor_shard_dir if donor_shard_dir is not None else idx_dir

    index_path = _find_index_path(idx_dir)
    if index_path is None:
        return raw_tensors

    try:
        weight_map = _load_weight_map_from_index(index_path)
    except Exception:
        return raw_tensors

    bases = _inkling_nvfp4_bases_from_weight_map(weight_map)
    if not bases:
        return raw_tensors

    shard_prefix = f"[{shard_name}] " if shard_name else ""

    # --- 1) hydrate missing .scale / .scale2 for bases present in this shard ---
    current_shard = os.path.basename(shard_path)
    wanted_by_shard: dict[str, list[str]] = {}
    for base in bases:
        if base not in raw_tensors:
            continue
        for suffix in (".scale", ".scale2"):
            aux_name = f"{base}{suffix}"
            if aux_name in raw_tensors:
                continue
            target_shard = weight_map.get(aux_name)
            if not target_shard or target_shard == current_shard:
                continue
            wanted_by_shard.setdefault(target_shard, []).append(aux_name)

    if wanted_by_shard:
        from safetensors import safe_open

        hydrated = 0
        for target_shard, aux_names in wanted_by_shard.items():
            target_path = str(resolve_within_directory(donor_dir, target_shard, origin="weight_map"))
            if not os.path.exists(target_path):
                logger.warning(
                    f"{shard_prefix}Donor shard '{target_shard}' not found in '{donor_dir}' while hydrating "
                    f"{len(aux_names)} NVFP4 scale tensor(s); the affected weight(s) will stay packed and "
                    f"will not be quantized."
                )
                continue
            try:
                with safe_open(target_path, framework="pt", device="cpu") as sf:
                    for aux_name in aux_names:
                        try:
                            raw_tensors[aux_name] = sf.get_tensor(aux_name)
                            hydrated += 1
                        except Exception:
                            continue
            except Exception as e:
                logger.warning(f"{shard_prefix}Could not read donor shard '{target_shard}': {e}.")
                continue

        if hydrated:
            logger.info(
                f"{shard_prefix}Hydrated {hydrated} NVFP4 scale tensor(s) from sibling shard(s) using index mapping."
            )

    # --- 2) drop orphan auxiliary tensors owned by another shard's base ---
    dropped = 0
    for name in list(raw_tensors.keys()):
        for suffix in _INKLING_NVFP4_AUX_SUFFIXES:
            if not name.endswith(suffix):
                continue
            base = name[: -len(suffix)]
            if base in bases and base not in raw_tensors:
                del raw_tensors[name]
                dropped += 1
            break

    if dropped:
        logger.info(f"{shard_prefix}Dropped {dropped} orphan NVFP4 metadata tensor(s) owned by other shard(s).")

    return raw_tensors


def _dequant_mxfp_tensors(
    raw_tensors: dict[str, torch.Tensor],
    device: str = "cpu",
    shard_name: str | None = None,
) -> dict[str, torch.Tensor]:
    """Dequantize llm-compressor MXFP8 / MXFP4 weight tensors to float32.

    Detection is purely by *name* and *dtype*:

    * ``<layer>.weight`` (``float8_e4m3fn``) + ``<layer>.weight_scale`` → MXFP8,
    dequantized via :func:`dequant_mx_fp8`.
    * ``<layer>.weight_packed`` (``uint8``) + ``<layer>.weight_scale`` → MXFP4,
    dequantized via :func:`dequant_mxfp4`.

    The dequantized weight is written back under ``<layer>.weight`` and the
    scale (and any ``weight_packed``) tensor is removed, so the downstream RTN
    path can requantize the layer to the requested target scheme.
    """
    # Tuple layout: (layer_name, weight_key, scale_key, bits)
    entries = _collect_mxfp_source_entries(raw_tensors)

    if not entries:
        return raw_tensors

    n_mxfp8 = sum(1 for _layer_name, _weight_key, _scale_key, bits in entries if bits == 8)
    n_mxfp4 = len(entries) - n_mxfp8
    dequant_device = str(device or "cpu")
    shard_prefix = f"[{shard_name}] " if shard_name else ""
    logger.info(
        f"{shard_prefix}Dequantizing MXFP tensor(s) to float32 on {dequant_device}: "
        f"MXFP8={n_mxfp8}, MXFP4={n_mxfp4}, total={len(entries)}."
    )

    for layer_name, weight_key, scale_key, bits in entries:
        weight = raw_tensors.pop(weight_key)
        scale = raw_tensors.pop(scale_key).view(torch.uint8)
        if bits == 8:
            dq_weight = _dequantize_with_device_fallback(
                dequant_device=dequant_device,
                shard_prefix=shard_prefix,
                op_name="MXFP dequant",
                tensor_label=layer_name,
                on_device=lambda weight=weight, scale=scale: dequant_mx_fp8(
                    weight_fp8=weight.to(dequant_device, non_blocking=True),
                    scale_e8m0=scale.to(dequant_device, non_blocking=True),
                    block_size=32,
                    target_dtype=torch.float32,
                ).to("cpu"),
                on_cpu=lambda weight=weight, scale=scale: dequant_mx_fp8(
                    weight_fp8=weight,
                    scale_e8m0=scale,
                    block_size=32,
                    target_dtype=torch.float32,
                ),
            )
        else:
            dq_weight = _dequantize_with_device_fallback(
                dequant_device=dequant_device,
                shard_prefix=shard_prefix,
                op_name="MXFP dequant",
                tensor_label=layer_name,
                on_device=lambda weight=weight, scale=scale: dequant_mxfp4(
                    data_lp=weight.view(torch.uint8).contiguous().to(dequant_device, non_blocking=True),
                    scale_e8m0=scale.to(dequant_device, non_blocking=True),
                    elem_dtype="fp4_e2m1",
                    block_size=32,
                    target_dtype=torch.float32,
                ).to("cpu"),
                on_cpu=lambda weight=weight, scale=scale: dequant_mxfp4(
                    data_lp=weight.view(torch.uint8).contiguous(),
                    scale_e8m0=scale,
                    elem_dtype="fp4_e2m1",
                    block_size=32,
                    target_dtype=torch.float32,
                ),
            )
        raw_tensors[f"{layer_name}.weight"] = dq_weight
        # Drop the loop-local aliases so the source (packed weight + scale) and
        # the previous iteration's dequantized tensor are not kept alive while
        # the next, potentially much larger, layer is being processed.
        del weight, scale, dq_weight

    return raw_tensors


def _handle_mxfp_source_tensors(
    raw_tensors: dict[str, torch.Tensor],
    matcher: _PatternMatcher,
    source_state: dict[str, int] | None = None,
    device: str = "cpu",
    shard_name: str | None = None,
) -> tuple[dict[str, torch.Tensor], dict[str, torch.Tensor], list[str]]:
    """Handle llm-compressor-style MXFP8/MXFP4 source tensors.

    Detects llm-compressor MXFP tensors purely by *name + dtype*:

    * ``<layer>.weight`` (``float8_e4m3fn``) + ``<layer>.weight_scale`` → MXFP8.
    * ``<layer>.weight_packed`` (``uint8``) + ``<layer>.weight_scale`` → MXFP4.

    For each detected layer the effective target scheme is resolved via *matcher*:

    * If the target is the **same MXFP format** (``data_type='mx_fp'``, matching
      ``bits``), the tensors are emitted directly as a passthrough — no
      dequantization is performed and the layer is recorded as already quantized.
    * Otherwise the tensors are dequantized to ``float32`` via
      :func:`_dequant_mxfp_tensors` so the downstream RTN path can re-quantize
      them to the requested target scheme.

    Returns:
        ``(raw_tensors, passthrough_tensors, passthrough_layers)``.
    """
    # NVFP4 source -> MXFP4 keeps source MXFP layers untouched; they flow through as ignored tensors.
    if getattr(matcher, "nvfp_source_only", False):
        return raw_tensors, {}, []

    entries = _collect_mxfp_source_entries(raw_tensors)
    if not entries:
        return raw_tensors, {}, []

    source_state = source_state or {}

    passthrough_tensors: dict[str, torch.Tensor] = {}
    passthrough_layers: list[str] = []
    n_dequant = 0

    for layer_name, weight_key, scale_key, bits in entries:
        scheme = matcher.resolve_scheme(f"{layer_name}.weight")
        is_ignored = matcher.should_ignore(f"{layer_name}.weight")
        target_is_same_mxfp = (
            scheme is not None and is_mx_fp((scheme.get("data_type") or "").lower()) and scheme.get("bits") == bits
        )
        if target_is_same_mxfp and not is_ignored:
            passthrough_tensors[weight_key] = raw_tensors.pop(weight_key).to("cpu")
            passthrough_tensors[scale_key] = raw_tensors.pop(scale_key).to("cpu")
            passthrough_layers.append(layer_name)
        else:
            n_dequant += 1

    if n_dequant:
        raw_tensors = _dequant_mxfp_tensors(raw_tensors, device=device, shard_name=shard_name)

    parts: list[str] = []
    if passthrough_layers:
        parts.append(f"{len(passthrough_layers)} passthrough")
    if n_dequant:
        parts.append(f"{n_dequant} dequantized to float32")
    if source_state:
        parts.append(f"{len(source_state)} model_type-normalized")
    logger.info(f"Handling MXFP source tensor(s): {', '.join(parts)}.")

    return raw_tensors, passthrough_tensors, passthrough_layers


def _dequant_fp8_tensors(
    raw_tensors: dict[str, torch.Tensor],
    block_size: list | None = None,
    device: str = "cpu",
    shard_name: str | None = None,
    shard_path: str | None = None,
    *,
    index_dir: str | None = None,
    donor_shard_dir: str | None = None,
    hydrate_only: bool = False,
) -> dict[str, torch.Tensor]:
    """Dequantize DeepSeek-V3-style FP8 weight tensors to float32.

    With ``hydrate_only`` the cross-shard ``weight_scale_inv`` tensors are hydrated but the
    FP8 weights are kept as-is (NVFP4 source -> MXFP4).

    Handles the **DeepSeek-V3 FP8** convention: weight dtype ``float8_e4m3fn``
    paired with a ``.weight_scale_inv`` tensor (per-block float32 scales, NOT
    E8M0).  The weights are converted to ``float32`` so downstream RTN
    quantization can proceed normally.

    MXFP sources are handled separately by
    :func:`preprocess_model_type_source_tensors` / :func:`_handle_mxfp_source_tensors`.
    """
    from auto_round.utils.weight_handler import _dequant_fp8_linear_weight

    if shard_path:
        raw_tensors = _hydrate_missing_fp8_scales_from_index(
            raw_tensors,
            shard_path,
            shard_name=shard_name,
            index_dir=index_dir,
            donor_shard_dir=donor_shard_dir,
        )
    if hydrate_only:
        return raw_tensors

    quant_entries: list[tuple[str, str]] = []
    for name, tensor in raw_tensors.items():
        if not name.endswith(".weight"):
            continue
        if tensor.dtype != torch.float8_e4m3fn:
            continue
        # DeepSeek-V3 style: .weight_scale_inv (per-block float32 scales).
        scale_inv_name = f"{name[: -len('.weight')]}.weight_scale_inv"
        if scale_inv_name in raw_tensors:
            quant_entries.append((name, scale_inv_name))

    if not quant_entries:
        return raw_tensors

    # device has already been resolved by the caller; use it directly here.
    dequant_device = str(device or "cpu")
    shard_prefix = f"[{shard_name}] " if shard_name else ""

    logger.info(f"{shard_prefix}Dequantizing {len(quant_entries)} FP8 weight tensor(s) to float32 on {dequant_device}.")

    for weight_name, scale_name in quant_entries:
        weight = raw_tensors[weight_name]
        scale = raw_tensors.pop(scale_name)

        # Dequantize on GPU for throughput, then move back to CPU to keep
        # per-shard memory usage bounded before per-layer quantization.
        raw_tensors[weight_name] = _dequantize_with_device_fallback(
            dequant_device=dequant_device,
            shard_prefix=shard_prefix,
            op_name="FP8 dequant",
            tensor_label=weight_name,
            on_device=lambda weight=weight, scale=scale: _dequant_fp8_linear_weight(
                weight.to(dequant_device, non_blocking=True),
                scale.to(dequant_device, non_blocking=True),
                block_size=block_size,
                output_dtype=torch.float32,
            ).to("cpu"),
            on_cpu=lambda weight=weight, scale=scale: _dequant_fp8_linear_weight(
                weight, scale, block_size=block_size, output_dtype=torch.float32
            ),
        )
        # ``weight`` still aliases the original FP8 storage after the dict slot
        # was rebound; release it (and the scale) before the next iteration.
        del weight, scale

    return raw_tensors


def _process_shard(
    shard_path: str,
    default_scheme: dict | None = None,
    layer_config: dict | None = None,
    ignore_patterns: list[str] | None = None,
    device: str = "cpu",
    *,
    shard_name: str | None = None,
    matcher: _PatternMatcher | None = None,
    fp8_block_size: list | None = None,
    model_type: str | None = None,
    source_quantization_config: dict | None = None,
    enable_torch_compile: bool = False,
    disable_opt_rtn: bool = False,
    index_dir: str | None = None,
    donor_shard_dir: str | None = None,
    donor_tensors_to_exclude: set[str] | None = None,
) -> tuple[dict[str, torch.Tensor], list[str], list[str]]:
    """Quantize eligible weights in a single safetensors shard.

    Returns:
        (output_tensors, quantized_layer_names, ignored_layer_names)

    ``ignored_layer_names`` is derived by comparing the set of input ``.weight``
    layer names (collected after fused-expert splitting) with the final set of
    quantized layer names.  Any layer that had a ``.weight`` tensor in the input
    but was NOT quantized is reported as ignored — this correctly captures
    user-ignored layers, predefined-skipped layers, non-eligible weights, and
    any other pass-through case without separate per-tensor tracking.

    Args:
        index_dir: Directory containing the model's ``*.safetensors.index.json``.
            Used to resolve cross-shard FP8 scale_inv tensors.  Defaults to the
            directory of *shard_path* (correct for non-streaming mode).
        donor_shard_dir: Directory where donor shards are cached.  Defaults to
            *index_dir*.  In streaming mode pass the local shard cache directory.
        donor_tensors_to_exclude: If provided, these ``weight_scale_inv`` tensor
            names are removed from the final output tensors.  Used when this
            shard is a donor: its scale_inv tensors have already been consumed
            by the recipient shard and must not appear in the quantized output.
    """
    if matcher is None:
        matcher = _PatternMatcher(
            ignore_patterns if ignore_patterns is not None else [],
            layer_config if layer_config is not None else {},
            default_scheme if default_scheme is not None else {},
        )

    output_tensors: dict[str, torch.Tensor] = {}
    quantized_layers: list[str] = []
    quantize_func = compile_func(quantize_weight_rtn, device) if enable_torch_compile else quantize_weight_rtn

    if shard_path.endswith(".bin"):
        # No unrestricted fallback: this pickle comes from an untrusted artifact.
        raw_tensors = torch.load(shard_path, map_location="cpu", weights_only=True)
        # Flatten nested state-dict wrappers if present.
        if not isinstance(raw_tensors, dict):
            raise ValueError(f"Expected a dict from {shard_path}, got {type(raw_tensors)}")
    else:
        from safetensors import safe_open

        with safe_open(shard_path, framework="pt", device="cpu") as f:
            raw_tensors = {name: f.get_tensor(name) for name in f.keys()}

    raw_tensors = split_fused_expert_tensors(raw_tensors, model_type=model_type)
    # Dequantized sources are FP32 in-flight; those left high-precision are saved as BF16.
    source_fp32_names = {name for name, t in raw_tensors.items() if t.dtype == torch.float32}

    # Hydrate cross-shard FP8 weight_scale_inv tensors *before* any
    # preprocessing below. Otherwise a weight whose scale lives in a sibling
    # shard would miss the model_type-specific "expand-scale" passthrough
    # handling (step 1) simply because its scale wasn't loaded yet, and would
    # incorrectly fall through to the generic dequant-to-bf16 + RTN-requantize
    # path instead of being treated identically to same-shard weights.
    if shard_path:
        raw_tensors = _hydrate_missing_fp8_scales_from_index(
            raw_tensors,
            shard_path,
            shard_name=shard_name,
            index_dir=index_dir,
            donor_shard_dir=donor_shard_dir,
        )

    # Snapshot candidate weight layer names *before* any preprocessing. 1D
    # weights (for example LayerNorm) are not quantization targets, while 3D
    # weights remain tracked so unsupported layouts are visible as ignored.
    # Fused 3-D MoE expert weights (e.g. ``experts.w13_weight``, kept fused
    # for model_type architectures like Inkling) have no ``.weight`` suffix
    # of their own, so they are tracked separately here.
    input_weight_layers: list[str] = list(
        dict.fromkeys(
            (
                _fused_expert_layer_name(name)
                if _is_moe_fused_expert_weight(name, tensor)
                else (name.rsplit(".", 1)[0] if name.endswith((".weight", ".weight_packed")) else name)
            )
            for name, tensor in raw_tensors.items()
            if (name.endswith((".weight", ".weight_packed")) and tensor.dim() > 1)
            or _is_moe_fused_expert_weight(name, tensor)
        )
    )

    # Preserve original tensors for predefined skipped layers so that already-
    # quantized weights (FP8, FP4-packed, etc.) are NOT dequantized.
    # User-specified ignore layers should still flow through the dequant path
    # so the saved model exports them in full precision.
    preserved_prefixes: set[str] = set()
    for tname in raw_tensors:
        if tname.endswith((".weight", ".weight_packed", ".qweight")) and matcher.should_skip(tname):
            preserved_prefixes.add(tname.rsplit(".", 1)[0])

    preserved_tensors: dict[str, torch.Tensor] = {}
    if preserved_prefixes:
        for key in list(raw_tensors.keys()):
            prefix = key.rsplit(".", 1)[0]
            if prefix in preserved_prefixes:
                preserved_tensors[key] = raw_tensors.pop(key)

    # 1) model-type-specific preprocessing (format conversion only)
    raw_tensors, source_state = preprocess_model_type_source_tensors(
        raw_tensors,
        model_type=model_type,
        quantization_config=source_quantization_config,
        shard_name=shard_name,
    )

    # 1.5) normalize legacy NVFP4 names to llm-compressor naming.
    raw_tensors, _converted_nvfp4_layers = _normalize_nvfp4_source_tensors(raw_tensors, shard_name=shard_name)

    # 1.6) Inkling (inkling_mm_model) NVFP4 sources store the packed weight under a
    # bare name (no ``.weight`` suffix) with ``.scale`` / ``.scale2`` siblings,
    # which are frequently placed in *different* shards. Hydrate them from the
    # index first, then dequantize and drop the stale metadata tensors.
    raw_tensors = _hydrate_and_clean_inkling_nvfp4_aux(
        raw_tensors,
        shard_path,
        shard_name=shard_name,
        index_dir=index_dir,
        donor_shard_dir=donor_shard_dir,
    )
    # Hydrated Inkling scales are source tensors; keep their dtype if they survive (failed dequant).
    source_fp32_names.update(
        n for n, t in raw_tensors.items() if t.dtype == torch.float32 and n.endswith((".scale", ".scale2"))
    )

    # NVFP4 source -> MXFP4: only NVFP4 source layers (plus layer_config matches) are quantized;
    # all other layers are treated as ignored. Must run before NVFP4 dequantization.
    if matcher.nvfp_source_only:
        nvfp_layers = _collect_nvfp4_source_layers(raw_tensors)
        matcher.restrict_to_layers(nvfp_layers)
        shard_prefix = f"[{shard_name}] " if shard_name else ""
        logger.info(f"{shard_prefix}Detected {len(nvfp_layers)} NVFP4 source layer(s) to convert.")

    raw_tensors = _dequant_inkling_nvfp4_tensors(raw_tensors, device=device, shard_name=shard_name)

    # 1.5) model-type-specific low-precision dequantization (e.g. kimi_k25 INT4)
    raw_tensors = handle_model_type_low_precision_source_tensors(
        raw_tensors,
        model_type=model_type,
        source_quant_config=source_quantization_config,
        device=device,
        shard_name=shard_name,
        dequantize_with_device_fallback=_dequantize_with_device_fallback,
    )

    # 2) generic MXFP handling for both preprocessed and normal source models
    raw_tensors, passthrough_tensors, passthrough_layers = _handle_mxfp_source_tensors(
        raw_tensors,
        matcher,
        source_state=source_state,
        device=device,
        shard_name=shard_name,
    )
    output_tensors.update(passthrough_tensors)
    quantized_layers.extend(passthrough_layers)

    # 3) NVFP4 passthrough for layers already stored in packed format (or
    #    dequant to float32 when the target scheme requires re-quantization).
    raw_tensors, nvfp_passthrough_tensors, nvfp_passthrough_layers = _handle_nvfp4_source_tensors(
        raw_tensors,
        matcher,
        device=device,
        shard_name=shard_name,
    )
    output_tensors.update(nvfp_passthrough_tensors)
    quantized_layers.extend(nvfp_passthrough_layers)

    raw_tensors = _dequant_fp8_tensors(
        raw_tensors,
        block_size=fp8_block_size,
        device=device,
        shard_name=shard_name,
        shard_path=shard_path,
        index_dir=index_dir,
        donor_shard_dir=donor_shard_dir,
        hydrate_only=matcher.nvfp_source_only,
    )
    raw_tensors.update(preserved_tensors)
    # ``raw_tensors`` now owns these; drop the duplicate references.
    preserved_tensors.clear()

    for tensor_name in list(raw_tensors.keys()):
        tensor = raw_tensors.pop(tensor_name)
        _layer_name, out_dict, q_layer, _ig_layer = _quantize_single_tensor(
            tensor_name,
            tensor,
            matcher,
            device,
            quantize_func,
            disable_opt_rtn,
        )
        # Cross-shard hydrated FP8 scale_inv tensors are FP32 sources, not dequantized weights.
        if (
            tensor.dtype == torch.float32
            and tensor_name not in source_fp32_names
            and not tensor_name.endswith(".weight_scale_inv")
        ):
            kept = out_dict.get(tensor_name)
            if kept is not None and kept.dtype == torch.float32:
                out_dict[tensor_name] = kept.to(torch.bfloat16)
        output_tensors.update(out_dict)
        if q_layer:
            quantized_layers.append(q_layer)
        # Release the (possibly high-precision) source tensor and the packing
        # result before loading the next one, so peak RSS stays at ~one layer
        # instead of two.
        del tensor, out_dict

    # Remove scale_inv tensors that this shard donates to other shards.
    # These tensors have no corresponding weight in this shard; keeping them
    # would pollute the quantized output with raw FP8 scale metadata.
    if donor_tensors_to_exclude:
        removed = {t for t in donor_tensors_to_exclude if t in output_tensors}
        for t in removed:
            del output_tensors[t]
        if removed:
            shard_prefix = f"[{shard_name}] " if shard_name else ""
            logger.debug(
                f"{shard_prefix}Excluded {len(removed)} donated cross-shard scale_inv "
                f"tensor(s) from quantized shard output."
            )

    # Derive ignored layers by comparing input weight layers with quantized set.
    quantized_set = set(quantized_layers)
    ignored_layers: list[str] = [l for l in input_weight_layers if l not in quantized_set]

    return output_tensors, quantized_layers, ignored_layers


# ---------------------------------------------------------------------------
# Model Source I/O and Shard Discovery
# ---------------------------------------------------------------------------


def _index_shards_cache_status(model_name_or_path: str, index_path: str) -> tuple[bool, int, int]:
    """Return ``(all_cached, n_cached, n_total)`` for shards listed in an index file."""
    from huggingface_hub import try_to_load_from_cache

    try:
        shard_files = sorted(set(_load_weight_map_from_index(index_path).values()))
    except Exception:
        return False, 0, 0

    if not shard_files:
        return False, 0, 0

    n_cached = sum(1 for f in shard_files if isinstance(try_to_load_from_cache(model_name_or_path, f), str))
    return n_cached == len(shard_files), n_cached, len(shard_files)


def _get_model_cache_status(model_name_or_path: str) -> tuple[bool, str]:
    """Return cache decision and a short reason string.

    Cached means a local directory, or an HF cache that holds ``config.json``
    **plus the actual weight data**.

    A cached ``*.index.json`` is deliberately *not* treated as a weight hit on
    its own: many workflows (and this module's own streaming path) prefetch
    only metadata.  Trusting the index would route such models to a full
    ``snapshot_download``, downloading the entire checkpoint up-front instead
    of streaming and quantizing shard-by-shard.  So when an index is present
    its referenced shards are verified individually.
    """
    if os.path.isdir(model_name_or_path):
        return True, "input is an existing local directory"

    try:
        from huggingface_hub import try_to_load_from_cache

        # ``config.json`` alone is not enough: many workflows prefetch only
        # config/tokenizer files, which would otherwise route to full
        # ``snapshot_download``.
        config_cached = isinstance(try_to_load_from_cache(model_name_or_path, "config.json"), str)
        if not config_cached:
            return False, "HF cache miss: config.json not found"

        # Single-file checkpoints: the file itself is the weight data.
        for fname in ("model.safetensors", "pytorch_model.bin"):
            if isinstance(try_to_load_from_cache(model_name_or_path, fname), str):
                return True, f"HF cache hit: config.json + {fname}"

        # Sharded checkpoints: an index is only a hit if every shard is cached.
        for index_name in ("model.safetensors.index.json", "pytorch_model.bin.index.json"):
            index_path = try_to_load_from_cache(model_name_or_path, index_name)
            if not isinstance(index_path, str):
                continue
            all_cached, n_cached, n_total = _index_shards_cache_status(model_name_or_path, index_path)
            if all_cached:
                return True, f"HF cache hit: config.json + {index_name} + all {n_total} shard(s)"
            return (
                False,
                f"HF cache partial hit: {index_name} present but only {n_cached}/{n_total} shard(s) cached",
            )

        return (
            False,
            "HF cache partial hit: config.json exists but no weight entry file found",
        )
    except Exception as e:
        return False, f"cache probe failed: {type(e).__name__}: {e}"


def _is_model_cached(model_name_or_path: str) -> bool:
    """Return True if the model is already available locally or in HF cache."""
    cached, _ = _get_model_cache_status(model_name_or_path)
    return cached


def _resolve_source_dir(model_name_or_path: str) -> str:
    """Resolve model_name_or_path to a local directory (download if needed)."""
    if os.path.isdir(model_name_or_path):
        return model_name_or_path
    from huggingface_hub import snapshot_download

    return snapshot_download(model_name_or_path)


def _load_config(source_dir: str) -> dict:
    """Load config.json from model directory."""
    config_path = os.path.join(source_dir, "config.json")
    if not os.path.exists(config_path):
        raise FileNotFoundError(f"config.json not found in {source_dir}")
    with open(config_path) as f:
        return json.load(f)


def _list_weight_shards(source_dir: str) -> list[str]:
    """Return list of weight shard filenames in order.

    Safetensors shards are preferred.  When no safetensors files are
    found the function falls back to PyTorch ``.bin`` shards.

    Handles both standard naming (``model.safetensors``,
    ``pytorch_model.bin``) and custom prefixes such as
    ``diffusion_pytorch_model-XXXXX-of-XXXXX.safetensors`` by scanning
    all ``*.safetensors.index.json`` / ``*.bin.index.json`` index files
    in the directory when no standard index is found.
    """

    def _shards_from_index(index_path: str) -> list[str]:
        with open(index_path) as f:
            index = json.load(f)
        # Shard names are declared by the checkpoint's own index and are later
        # joined onto source_dir / a shard cache dir, so contain them here.
        weight_map = validate_weight_map(index["weight_map"], source_dir, index_path=index_path)
        seen: set[str] = set()
        shards: list[str] = []
        for shard_file in weight_map.values():
            if shard_file not in seen:
                seen.add(shard_file)
                shards.append(shard_file)
        return shards

    # --- safetensors: standard index ---
    st_index = os.path.join(source_dir, "model.safetensors.index.json")
    if os.path.exists(st_index):
        return _shards_from_index(st_index)

    # --- safetensors: custom-prefix index (e.g. diffusion_pytorch_model.safetensors.index.json) ---
    for fname in sorted(os.listdir(source_dir)):
        if fname.endswith(".safetensors.index.json"):
            return _shards_from_index(os.path.join(source_dir, fname))

    # --- safetensors: single file or index-less multi-file shards ---
    st_files = sorted(f for f in os.listdir(source_dir) if f.endswith(".safetensors"))
    if len(st_files) >= 1:
        return st_files

    # --- pytorch .bin: standard index ---
    bin_index = os.path.join(source_dir, "pytorch_model.bin.index.json")
    if os.path.exists(bin_index):
        return _shards_from_index(bin_index)

    # --- pytorch .bin: custom-prefix index ---
    for fname in sorted(os.listdir(source_dir)):
        if fname.endswith(".bin.index.json"):
            return _shards_from_index(os.path.join(source_dir, fname))

    # --- pytorch .bin: single file ---
    bin_single = os.path.join(source_dir, "pytorch_model.bin")
    if os.path.exists(bin_single):
        return ["pytorch_model.bin"]

    # --- pytorch .bin: any single .bin file ---
    bin_files = sorted(f for f in os.listdir(source_dir) if f.endswith(".bin"))
    if len(bin_files) >= 1:
        return bin_files
    return []


def _list_remote_weight_shards(model_name_or_path: str, subfolder: str | None = None) -> list[str]:
    """Return remote weight filenames for streaming repos without a weight index."""
    from huggingface_hub import list_repo_files

    repo_files = list_repo_files(model_name_or_path)
    if subfolder:
        prefix = subfolder.rstrip("/") + "/"
        repo_files = [name for name in repo_files if name.startswith(prefix)]
    safetensors_files = sorted(
        name for name in repo_files if name.endswith(".safetensors") and not name.endswith(".index.json")
    )
    if safetensors_files:
        return safetensors_files

    return sorted(name for name in repo_files if name.endswith(".bin") and not name.endswith(".index.json"))


def _is_weight_shard(fname: str) -> bool:
    """Return True if *fname* is a weight shard (safetensors or .bin).

    Excludes index files (``*.index.json``) so that they are copied to the
    output directory as normal metadata.
    """
    if fname.endswith(".index.json"):
        return False
    return fname.endswith((".safetensors", ".bin"))


# Keep old name as an alias for backward compatibility.
_is_safetensors_shard = _is_weight_shard


def _download_single_shard(
    model_name_or_path: str,
    shard_filename: str,
    local_dir: str,
) -> str:
    """Download a single safetensors shard file. Returns the local path."""
    os.makedirs(local_dir, exist_ok=True)
    local_path = os.path.join(local_dir, sanitize_shard_name(shard_filename, origin="shard list"))
    if os.path.exists(local_path):
        logger.info(f"Shard '{shard_filename}' already exists at '{local_path}', skipping download.")
        return local_path

    if os.path.isdir(model_name_or_path):
        src = str(resolve_within_directory(model_name_or_path, shard_filename, origin="shard list"))
        if os.path.exists(src):
            shutil.copy2(src, local_path)
            return local_path
        raise FileNotFoundError(f"{shard_filename} not found in {model_name_or_path}")

    from huggingface_hub import hf_hub_download

    return hf_hub_download(
        repo_id=model_name_or_path,
        filename=shard_filename,
        local_dir=local_dir,
    )


def _download_metadata_files(
    model_name_or_path: str,
    local_dir: str,
) -> str:
    """Download all non-safetensors files from a model repo. Returns local dir."""
    os.makedirs(local_dir, exist_ok=True)

    if os.path.isdir(model_name_or_path):
        for fname in os.listdir(model_name_or_path):
            if _is_weight_shard(fname):
                continue
            src = os.path.join(model_name_or_path, fname)
            dst = os.path.join(local_dir, fname)
            if os.path.isdir(src):
                if not os.path.exists(dst):
                    shutil.copytree(src, dst)
            elif os.path.isfile(src) and not os.path.exists(dst):
                shutil.copy2(src, dst)
        return local_dir

    from huggingface_hub import snapshot_download

    snapshot_download(
        repo_id=model_name_or_path,
        local_dir=local_dir,
        ignore_patterns=["*.safetensors", "*.bin", "*.pth", "*.pt"],
    )
    return local_dir


# ---------------------------------------------------------------------------
# ---------------------------------------------------------------------------
# Output Writers
# ---------------------------------------------------------------------------


def _write_output_shard(
    output_dir: str,
    shard_name: str,
    tensors: dict[str, torch.Tensor],
    weight_map: dict[str, str],
):
    """Write a single output shard and update the weight_map."""
    from safetensors.torch import save_file

    shard_path = os.path.join(output_dir, shard_name)

    # Detect shared-storage tensors (e.g. tie_word_embeddings: wte ↔ lm_head).
    # safetensors refuses to serialise them as-is; clone the duplicates so each
    # tensor occupies its own memory region.  The first occurrence keeps the
    # original storage; subsequent aliases are cloned.
    seen_data_ptrs: set[int] = set()
    deduped: dict[str, torch.Tensor] = {}
    for k, v in tensors.items():
        if not v.is_contiguous():
            v = v.contiguous()
        ptr = v.data_ptr()
        if ptr in seen_data_ptrs:
            v = v.clone()
        else:
            seen_data_ptrs.add(ptr)
        deduped[k] = v

    save_file(deduped, shard_path)
    for tensor_name in tensors:
        weight_map[tensor_name] = shard_name


def _write_index_file(output_dir: str, weight_map: dict[str, str]):
    """Write model.safetensors.index.json (or rename single shard)."""
    if len(set(weight_map.values())) <= 1:
        shard_names = list(set(weight_map.values()))
        if shard_names and shard_names[0] != "model.safetensors":
            src = os.path.join(output_dir, shard_names[0])
            dst = os.path.join(output_dir, "model.safetensors")
            if os.path.exists(src):
                os.rename(src, dst)
            weight_map = {k: "model.safetensors" for k in weight_map}
        return

    index = {"metadata": {"total_size": 0}, "weight_map": weight_map}
    with open(os.path.join(output_dir, "model.safetensors.index.json"), "w") as f:
        json.dump(index, f, indent=2)


# ---------------------------------------------------------------------------
# Quantization Configuration
# ---------------------------------------------------------------------------


def _get_llm_compressor_metadata() -> dict[str, str]:
    """Return AutoRound provenance for model-free llm-compressor output."""
    # Keep metadata deterministic and installation-source agnostic.
    return {"provider": "auto-round"}


def _build_nvfp4_e5m3_quantization_config(ignored_layers: list[str], with_global_scale: bool = False) -> dict:
    """Build compressed-tensors metadata for NVFP4 E5M3 (optionally with per-tensor global scales)."""
    from auto_round.export.export_to_llmcompressor.config import initialize_nvfp4_e5m3_quantization

    qconfig = initialize_nvfp4_e5m3_quantization(ignore=ignored_layers, with_global_scale=with_global_scale)
    qconfig.update(_get_llm_compressor_metadata())
    return qconfig


def _is_mxfp_ct_group(group: dict) -> bool:
    weights = group.get("weights") or {}
    return (
        weights.get("type") == "float"
        and weights.get("num_bits") in _SUPPORTED_MXFP_BITS
        and (weights.get("group_size") == 32)
    )


def _is_catch_all_group(group: dict) -> bool:
    # The default group may be ["Linear", "RoutedExperts"] after add_routed_experts_if_moe.
    return "Linear" in (group.get("targets") or [])


def _explicit_mxfp_targets(qconfig: dict, quantized_layers: list[str]) -> dict:
    """Replace the ``"Linear"`` catch-all of MXFP groups with the quantized layer names (+ ``RoutedExperts``)."""
    groups = qconfig.get("config_groups") or {}
    covered = {t for g in groups.values() if not _is_catch_all_group(g) for t in g.get("targets") or []}
    layers = [name for name in dict.fromkeys(quantized_layers) if name not in covered]
    for group in groups.values():
        if _is_catch_all_group(group) and _is_mxfp_ct_group(group) and layers:
            group["targets"] = _add_routed_experts_if_moe(list(layers), layers)
    return qconfig


def _get_mxfp_group_scheme_and_format(
    group_bits: int,
    group_data_type: str,
    ignore: list[str],
):
    """Return ``(QuantizationScheme, format_str)`` for a single (bits, data_type) group.

    Handles MXFP (mx_fp), NVFP4 (nv_fp), and NVFP4_E5M3 (nvfp4_v2) groups.
    """
    from auto_round.export.export_to_llmcompressor.config import (
        initialize_nvfp4_e5m3_quantization,
        initialize_quantization,
    )

    if is_nv_fp(group_data_type):
        # Standard NVFP4 uses global scales and has a dedicated preset in
        # compressed-tensors. Reuse that preset so metadata matches tensor
        # layout when model-free does NVFP4 passthrough.
        tmp_qconfig = initialize_quantization(scheme="NVFP4", ignore=ignore)
        return tmp_qconfig.config_groups["group_0"], "nvfp4-pack-quantized"

    if group_data_type in _NVFP4_E5M3_FAMILY_DATA_TYPES:
        # NVFP4_E5M3(_GS) is not a compressed-tensors preset; use its dedicated builder.
        with_global_scale = group_data_type == _NVFP4_E5M3_GS_DATA_TYPE
        nvfp4_dict = initialize_nvfp4_e5m3_quantization(ignore=ignore, with_global_scale=with_global_scale)
        # Extract the QuantizationScheme object from the dict config.
        from compressed_tensors.quantization import QuantizationScheme  # pylint: disable=E0401

        raw_scheme = nvfp4_dict["config_groups"]["group_0"]
        scheme_obj = QuantizationScheme.model_validate(raw_scheme)
        return scheme_obj, nvfp4_dict["format"]
    else:
        scheme_name = "MXFP4" if group_bits == 4 else "MXFP8"
        fmt = "mxfp4-pack-quantized" if group_bits == 4 else "mxfp8-quantized"
        tmp_qconfig = initialize_quantization(scheme=scheme_name, ignore=ignore)
        return tmp_qconfig.config_groups["group_0"], fmt


def _build_mxfp_quantization_config(
    default_scheme: dict,
    quantized_layers: list[str],
    ignored_layers: list[str],
    layer_config: dict | None = None,
) -> dict:
    """Build a compressed-tensors / llm-compressor style quantization_config
    dict for MXFP4 / MXFP8 model-free output, including mixed-precision cases.

    When *layer_config* contains layers that override the default bits or
    data_type (e.g. some layers are MXFP8 while the default is MXFP4, or
    some layers use NVFP4_E5M3 while the default is MXFP8), the function
    creates one ``config_group`` per distinct ``(bits, data_type)`` pair.
    Override groups list their layers explicitly; the default group uses
    ``targets=["Linear"]`` as a catch-all.  The top-level ``"format"`` is
    set to ``"mixed-precision"`` when more than one group is produced.

    Mirrors the per-group format produced by
    :mod:`auto_round.export.export_to_llmcompressor.export_to_fp`.
    """
    from auto_round.export.export_to_llmcompressor.config import (
        initialize_quantization,
    )

    bits = default_scheme.get("bits", 4)
    default_data_type = (default_scheme.get("data_type") or "mx_fp").lower()
    is_fp_default = (bits or 0) >= 16  # BF16/FP16 full-precision default

    if (
        not is_fp_default
        and bits not in _SUPPORTED_MXFP_BITS
        and default_data_type not in _NVFP4_E5M3_FAMILY_DATA_TYPES
    ):
        raise ValueError(f"Unsupported MXFP bits={bits} for model-free output.")

    # Default ignore list: any layer present in ignored_layers (deduped) that
    # was NOT quantized.
    ignore = list(dict.fromkeys(ignored_layers))
    quant_set = set(quantized_layers)
    ignore = [n for n in ignore if n not in quant_set]

    # Resolve each quantized layer's effective (bits, data_type) using layer_config overrides.
    # Key: (bits, data_type) to distinguish e.g. MXFP4 from NVFP4_E5M3 (both 4-bit).
    scheme_groups: dict[tuple[int, str], list[str]] = {}  # (bits, data_type) -> [layer_names]
    if layer_config:
        temp_matcher = _PatternMatcher(
            ignore_patterns=[],
            layer_config=layer_config,
            default_scheme=default_scheme,
        )
        for layer in quantized_layers:
            scheme = temp_matcher.resolve_scheme(f"{layer}.weight")
            layer_bits = scheme.get("bits", bits) if scheme is not None else bits
            layer_dt = (
                (scheme.get("data_type") or default_data_type).lower() if scheme is not None else default_data_type
            )
            scheme_groups.setdefault((layer_bits, layer_dt), []).append(layer)
    else:
        if not is_fp_default:
            scheme_groups[(bits, default_data_type)] = list(quantized_layers)
        # else: BF16 default with no layer_config → no MXFP layers; scheme_groups stays {}

    if len(scheme_groups) <= 1:
        # Single scheme — use the actual (bits, data_type) from the group.
        if scheme_groups:
            actual_bits, actual_dt = next(iter(scheme_groups.keys()))
        else:
            actual_bits, actual_dt = bits, default_data_type
        if actual_dt not in _NVFP4_E5M3_FAMILY_DATA_TYPES and actual_bits not in _SUPPORTED_MXFP_BITS:
            raise ValueError(f"Unsupported MXFP bits={actual_bits} for model-free output.")
        group_scheme, fmt = _get_mxfp_group_scheme_and_format(actual_bits, actual_dt, ignore)
        if actual_dt in _NVFP4_E5M3_FAMILY_DATA_TYPES:
            from auto_round.export.export_to_llmcompressor.config import initialize_nvfp4_e5m3_quantization

            qconfig = initialize_nvfp4_e5m3_quantization(
                ignore=ignore, with_global_scale=actual_dt == _NVFP4_E5M3_GS_DATA_TYPE
            )
            if is_fp_default and scheme_groups:
                targets = _add_routed_experts_if_moe(list(quantized_layers), quantized_layers)
                qconfig["config_groups"]["group_0"]["targets"] = targets
            qconfig["format"] = fmt
            qconfig.update(_get_llm_compressor_metadata())
            return qconfig
        from auto_round.export.export_to_llmcompressor.config import initialize_quantization as _init_q

        scheme_name = "MXFP4" if actual_bits == 4 else "MXFP8"
        qconfig = _init_q(scheme=scheme_name, ignore=ignore)
        if is_fp_default and scheme_groups:
            targets = _add_routed_experts_if_moe(list(quantized_layers), quantized_layers)
            qconfig.config_groups["group_0"].targets = targets
        qconfig = qconfig.to_dict()
        qconfig["format"] = fmt
        qconfig.update(_get_llm_compressor_metadata())
        return qconfig

    # Mixed precision: build one config_group per distinct (bits, data_type).
    # Override groups (non-default key) come first, default group last,
    # ordered by descending bit-width within each partition so that the
    # higher-precision group gets the lower group index.
    default_key = (bits, default_data_type)
    override_items = sorted(
        [(key, layers) for key, layers in scheme_groups.items() if key != default_key],
        key=lambda x: x[0][0],
        reverse=True,
    )
    default_item = (default_key, scheme_groups[default_key]) if default_key in scheme_groups else None
    ordered = override_items + ([default_item] if default_item else [])

    config_groups: dict = {}
    group_formats: dict[str, str] = {}
    for idx, ((group_bits, group_dt), layer_names) in enumerate(ordered):
        group_name = f"group_{idx}"
        is_default_group = (group_bits, group_dt) == default_key
        targets = ["Linear"] if is_default_group else layer_names
        targets = _add_routed_experts_if_moe(targets, layer_names)
        group_scheme, fmt = _get_mxfp_group_scheme_and_format(group_bits, group_dt, ignore)
        group_scheme.targets = targets
        config_groups[group_name] = group_scheme
        group_formats[group_name] = fmt

    full_qconfig = initialize_quantization(scheme=None, config_groups=config_groups, ignore=ignore)
    full_dict = full_qconfig.to_dict()
    full_dict["format"] = "mixed-precision"
    for group_name, fmt in group_formats.items():
        full_dict["config_groups"][group_name]["format"] = fmt
    full_dict.update(_get_llm_compressor_metadata())
    return full_dict


def _build_mxfp_autoround_quantization_config(
    default_scheme: dict,
    quantized_layers: list[str],
    ignored_layers: list[str],
    layer_config: dict | None = None,
    block_name_to_quantize: str | None = None,
) -> dict:
    """Build an auto-round style quantization_config for MXFP4 / MXFP8.

    Unlike :func:`_build_mxfp_quantization_config` which produces a
    compressed-tensors / llm-compressor style config, this function produces
    an ``auto-round`` style config (``quant_method="auto-round"``,
    ``packing_format="auto_round:llm_compressor"``).  The on-disk weight
    tensors are identical; only the ``quantization_config`` metadata differs.

    The generated config mirrors the layout of the regular AutoRound MXFP
    export, including activation quantization fields (``act_bits`` /
    ``act_data_type`` / …) from the scheme, ``enable_quanted_input``, and
    ``lm_head`` in ``extra_config`` when it was quantized.
    """
    from collections import Counter

    from auto_round.version import __version__

    bits = default_scheme.get("bits", 4)
    group_size = default_scheme.get("group_size", 32)
    is_fp_default = (bits or 0) >= 16

    # For BF16 default + MXFP layer_config overrides, derive the dominant
    # MXFP bit-width from the layers that were actually quantized.
    if is_fp_default and layer_config and quantized_layers:
        temp_matcher = _PatternMatcher(
            ignore_patterns=[],
            layer_config=layer_config,
            default_scheme=default_scheme,
        )
        mxfp_counter: Counter = Counter()
        for layer in quantized_layers:
            scheme = temp_matcher.resolve_scheme(f"{layer}.weight")
            if scheme is None:
                continue
            lb = scheme.get("bits")
            ldt = (scheme.get("data_type") or "").lower()
            if lb and lb < 16 and is_mx_fp(ldt):
                mxfp_counter[lb] += 1
        if mxfp_counter:
            bits, _ = mxfp_counter.most_common(1)[0]
            group_size = 32

    if (bits or 0) < 1 or bits not in _SUPPORTED_MXFP_BITS:
        bits = 4  # safe fallback

    qconfig: dict = {
        "quant_method": "auto-round",
        "packing_format": "auto_round:llm_compressor",
        "bits": bits,
        "group_size": group_size or 32,
        "sym": True,  # MXFP is always symmetric
        "data_type": "mx_fp",
        "iters": 0,
        "model_free": True,
        "autoround_version": __version__,
        "enable_quanted_input": False,
    }
    if block_name_to_quantize:
        qconfig["block_name_to_quantize"] = block_name_to_quantize

    # Carry activation quantization fields from the scheme (e.g. MXFP4 has
    # act_bits=4, act_data_type="mx_fp", act_dynamic=True, act_group_size=32,
    # act_sym=True).  Including these keeps the config consistent with the
    # output produced by the regular AutoRound MXFP export flow.
    for act_key in ("act_bits", "act_data_type", "act_dynamic", "act_group_size", "act_sym"):
        val = default_scheme.get(act_key)
        if val is not None:
            qconfig[act_key] = val

    scheme_keys = [f.name for f in fields(QuantizationScheme)]
    extra_config: dict = {}
    non_linear_ops = ["embed", "conv"]
    non_linear_re = re.compile("|".join(re.escape(op) for op in non_linear_ops))

    if layer_config:
        from auto_round.export.export_to_autoround.utils import check_neq_config

        expected_scheme = {key: qconfig.get(key) for key in scheme_keys}
        for layer_name, cfg in layer_config.items():
            if not isinstance(cfg, dict):
                continue
            neq_keys = check_neq_config(cfg, **expected_scheme)
            if neq_keys:
                extra_config[layer_name] = {key: cfg[key] for key in neq_keys if cfg.get(key) is not None}

    quantized_set = set(quantized_layers)
    unique_ignored = list(dict.fromkeys(ignored_layers))
    for layer_name in unique_ignored:
        if layer_name in quantized_set or layer_name in extra_config:
            continue
        if non_linear_re.search(layer_name):
            continue
        extra_config[layer_name] = {
            "bits": 16,
            "data_type": "float",
            "act_bits": 16,
            "act_data_type": "float",
        }

    # lm_head variants: when explicitly quantized, record each variant's full scheme in extra_config
    # (mirrors the regular AutoRound MXFP export behavior).
    for lm_head_name in _LM_HEAD_PATTERNS:
        if lm_head_name in quantized_set and lm_head_name not in extra_config:
            lm_head_cfg = (layer_config or {}).get(lm_head_name, default_scheme)
            extra_config[lm_head_name] = {k: lm_head_cfg.get(k) for k in scheme_keys if lm_head_cfg.get(k) is not None}

    if extra_config:
        qconfig["extra_config"] = extra_config

    return qconfig


def _layer_config_has_mxfp(layer_config: dict | None) -> bool:
    """Return True if any layer_config entry requests MXFP quantization.

    Handles values that are plain strings (preset names), dicts (possibly
    with a ``"scheme"`` key or a ``"data_type"`` key), or
    :class:`QuantizationScheme` instances.
    """
    if not layer_config:
        return False
    for val in layer_config.values():
        if isinstance(val, str):
            try:
                s = _normalize_scheme(val.upper())
                if is_mx_fp((s.data_type or "").lower()):
                    return True
            except Exception:
                pass
        elif isinstance(val, dict):
            # Direct data_type field
            if is_mx_fp((val.get("data_type") or "").lower()):
                return True
            # Nested 'scheme' key (e.g. {"scheme": "MXFP4"})
            scheme_val = val.get("scheme")
            if isinstance(scheme_val, str):
                try:
                    s = _normalize_scheme(scheme_val.upper())
                    if is_mx_fp((s.data_type or "").lower()):
                        return True
                except Exception:
                    pass
        elif isinstance(val, QuantizationScheme):
            if is_mx_fp((val.data_type or "").lower()):
                return True
    return False


def _layer_config_has_nvfp4(layer_config: dict | None) -> bool:
    """Return True if any layer_config entry requests NVFP4 quantization.

    Detects both the standard NVFP4 (``data_type='nv_fp'``) and the
    global-scale-free variant NVFP4_E5M3 (``data_type='nvfp4_v2'``).

    Handles values that are plain strings (preset names), dicts (possibly
    with a ``"scheme"`` key or a ``"data_type"`` key), or
    :class:`QuantizationScheme` instances.
    """
    return _get_layer_config_nvfp4_dt(layer_config) is not None


def _get_layer_config_nvfp4_dt(layer_config: dict | None) -> str | None:
    """Return the first NVFP4 data_type found in *layer_config*.

    Returns ``"nv_fp"`` for standard NVFP4, ``"nvfp4_v2"`` for NVFP4_E5M3,
    ``"nvfp4_v2_gs"`` for NVFP4_E5M3_GS, or ``None`` when no NVFP4 entry is present.
    """

    def _classify(dt: str) -> str | None:
        if dt in _NVFP4_E5M3_FAMILY_DATA_TYPES:
            return dt
        if is_nv_fp(dt):
            return "nv_fp"
        return None

    def _scheme_dt(name: str) -> str:
        try:
            return (_normalize_scheme(name.upper()).data_type or "").lower()
        except Exception:
            return ""

    if not layer_config:
        return None
    for val in layer_config.values():
        candidates = []
        if isinstance(val, str):
            candidates.append(_scheme_dt(val))
        elif isinstance(val, dict):
            candidates.append((val.get("data_type") or "").lower())
            if isinstance(val.get("scheme"), str):
                candidates.append(_scheme_dt(val["scheme"]))
        elif isinstance(val, QuantizationScheme):
            candidates.append((val.data_type or "").lower())
        for dt in candidates:
            result = _classify(dt)
            if result is not None:
                return result
    return None


def _is_full_precision_default(scheme_input: Any) -> bool:
    """Return True when *scheme_input* represents a full-precision default (bits >= 16).

    Used to detect BF16/FP16 schemes where no weights are quantized by
    default but layer_config overrides are still applied.
    """
    try:
        s = _normalize_scheme(scheme_input)
        return (s.bits or 0) >= 16 and (s.act_bits or 16) >= 16
    except Exception:
        return False


def _derive_dominant_int_scheme(
    quantized_layers: list[str],
    layer_config: dict,
    fallback: dict,
) -> dict | None:
    """Infer the dominant INT scheme from layers that were actually quantized.

    Used when the default scheme is full-precision (BF16/FP16) to produce a
    quantization_config where the most common INT scheme becomes the top-level
    default and the BF16 layers appear as ``bits=16`` exceptions in
    ``extra_config``.  This matches the layout expected by AutoRound loaders
    (dominant quantized bits at the top, full-precision overrides below).

    Returns ``None`` when no INT layers were found in *quantized_layers*.
    """
    from collections import Counter

    temp_matcher = _PatternMatcher(
        ignore_patterns=[],
        layer_config=layer_config,
        default_scheme=fallback,
    )
    counter: Counter[tuple] = Counter()
    for layer in quantized_layers:
        scheme = temp_matcher.resolve_scheme(f"{layer}.weight")
        if scheme is None:
            continue
        bits = scheme.get("bits")
        if bits is None or bits >= 16:
            continue
        data_type = (scheme.get("data_type") or "int").lower()
        if is_mx_fp(data_type):
            continue  # MXFP is handled by the dedicated path
        key = (bits, scheme.get("group_size") or fallback.get("group_size"), bool(scheme.get("sym", True)), data_type)
        counter[key] += 1

    if not counter:
        return None

    (bits, group_size, sym, data_type), _ = counter.most_common(1)[0]
    dominant = {
        "bits": bits,
        "group_size": group_size,
        "sym": sym,
        "data_type": data_type,
    }
    if data_type in _NVFP4_E5M3_FAMILY_DATA_TYPES:
        for act_key in ("act_bits", "act_data_type", "act_group_size", "act_sym", "act_dynamic"):
            value = scheme.get(act_key)
            if value is not None:
                dominant[act_key] = value
    return dominant


def _build_quantization_config(
    default_scheme: dict,
    layer_config: dict,
    ignore_patterns: list[str],
    quantized_layers: list[str],
    ignored_layers: list[str],
    block_name_to_quantize: str | None = None,
    format: str = "auto_round",
    source_quant_config: dict | None = None,
) -> dict:
    """Build a quantization_config dict compatible with auto-round format.

    ``source_quant_config`` (NVFP4 source -> MXFP4 only) is the source checkpoint's quantization config;
    its NVFP4 groups are rewritten to MXFP4 and every other group (e.g. FP8) is kept as-is.
    """
    # MXFP (mx_fp) supports two output styles depending on *format*:
    #   - "llm_compressor" → compressed-tensors / llm-compressor style config
    #     (produced by _build_mxfp_quantization_config).
    #   - "auto_round" / "auto_round:auto_gptq" → auto-round style config
    #     (produced by _build_mxfp_autoround_quantization_config).
    # Also route to MXFP config when the default is full-precision (BF16/FP16)
    # but layer_config contains MXFP overrides.
    data_type = (default_scheme.get("data_type") or "int").lower()
    default_bits = default_scheme.get("bits", 4)
    is_fp_default = (default_bits or 0) >= 16 and not is_mx_fp(data_type)
    if data_type in _NVFP4_E5M3_FAMILY_DATA_TYPES and format == "llm_compressor":
        warnings.warn(
            "LLMC/llm-compressor does not currently support the NVFP4_E5M3 scheme. "
            "Please refer to docs/nvfp4_e5m3.md for the recommended export and usage path.",
            UserWarning,
            stacklevel=2,
        )
        return _build_nvfp4_e5m3_quantization_config(
            ignored_layers, with_global_scale=data_type == _NVFP4_E5M3_GS_DATA_TYPE
        )
    # BF16/FP16 default with NVFP4 layer_config overrides and llm_compressor format:
    # build an NVFP4 config that targets only the explicitly quantized layers.
    if is_fp_default and _layer_config_has_nvfp4(layer_config) and format == "llm_compressor":
        nvfp4_dt = _get_layer_config_nvfp4_dt(layer_config)
        targets = list(dict.fromkeys(quantized_layers)) if quantized_layers else ["Linear"]
        if quantized_layers:
            targets = _add_routed_experts_if_moe(targets, quantized_layers)
        if nvfp4_dt in _NVFP4_E5M3_FAMILY_DATA_TYPES:
            warnings.warn(
                "LLMC/llm-compressor does not currently support the NVFP4_E5M3 scheme. "
                "Please refer to docs/nvfp4_e5m3.md for the recommended export and usage path.",
                UserWarning,
                stacklevel=2,
            )
            from auto_round.export.export_to_llmcompressor.config import initialize_nvfp4_e5m3_quantization

            qconfig = initialize_nvfp4_e5m3_quantization(
                ignore=[], with_global_scale=nvfp4_dt == _NVFP4_E5M3_GS_DATA_TYPE
            )
            qconfig["config_groups"]["group_0"]["targets"] = targets
        else:
            # Standard NVFP4 (nv_fp, scale_dtype=float8_e4m3fn)
            from auto_round.export.export_to_llmcompressor.config import initialize_quantization

            qconfig_obj = initialize_quantization(scheme="NVFP4", ignore=[])
            qconfig = qconfig_obj.to_dict()
            qconfig["config_groups"]["group_0"]["targets"] = targets
            qconfig["format"] = "nvfp4-pack-quantized"
        qconfig.update(_get_llm_compressor_metadata())
        return _ignore_fused_experts_for_vllm(qconfig, ignored_layers, quantized_layers)
    if is_mx_fp(data_type) or (is_fp_default and _layer_config_has_mxfp(layer_config)):
        if format in ("auto_round", "auto_round:auto_gptq"):
            return _build_mxfp_autoround_quantization_config(
                default_scheme=default_scheme,
                quantized_layers=quantized_layers,
                ignored_layers=ignored_layers,
                layer_config=layer_config,
                block_name_to_quantize=block_name_to_quantize,
            )
        qconfig = _build_mxfp_quantization_config(
            default_scheme=default_scheme,
            quantized_layers=quantized_layers,
            ignored_layers=ignored_layers,
            layer_config=layer_config,
        )
        source_ct = _source_config_to_ct(source_quant_config)
        if source_ct:
            qconfig = _rewrite_nvfp4_groups_as_mxfp(qconfig, source_ct, ignored_layers, quantized_layers)
        # Only a compressed-tensors source vouches for its "Linear" catch-all; otherwise list the layers for vLLM.
        if source_quant_config is not None and str(source_quant_config.get("quant_method") or "").lower() != (
            "compressed-tensors"
        ):
            qconfig = _explicit_mxfp_targets(qconfig, quantized_layers)
        return _ignore_fused_experts_for_vllm(qconfig, ignored_layers, quantized_layers)

    # When the default is full-precision (BF16/FP16) but INT layers were
    # quantized via layer_config, derive the dominant INT scheme and use it as
    # the top-level defaults so the output config reads as
    # "INT-quantized with BF16 exceptions" rather than the inverse.
    if is_fp_default and quantized_layers:
        dominant = _derive_dominant_int_scheme(quantized_layers, layer_config, default_scheme)
        if dominant is not None:
            default_scheme = dominant
            data_type = (default_scheme.get("data_type") or "int").lower()

    from auto_round.version import __version__

    scheme_keys = [f.name for f in fields(QuantizationScheme)]
    # The packing must match this declaration: "auto_round" bytes use the
    # qlinear_torch zero-point convention (nibble stores zp directly), while
    # "auto_round:auto_gptq" bytes use qlinear_torch_zp (nibble stores zp - 1,
    # unpacked with +1). _declared_int_packing keeps both sides in sync.
    # vllm only support auto_round:auto_gptq, but transformers cannot load it correctly when sym=False.
    # So we keep auto_round for asymmetric quantization to maintain compatibility with both.
    if data_type == _NVFP4_E5M3_DATA_TYPE:
        packing_format = "auto_round:fake" if format == "fake" else "auto_round:llm_compressor_nvfp4_e5m3"
    elif data_type == _NVFP4_E5M3_GS_DATA_TYPE:
        packing_format = "auto_round:fake" if format == "fake" else "auto_round:llm_compressor_nvfp4_e5m3_gs"
    else:
        packing_format = _declared_int_packing(default_scheme.get("sym", True))

    qconfig = {
        "quant_method": "auto-round",
        "packing_format": packing_format,
        "bits": default_scheme["bits"],
        "group_size": default_scheme["group_size"],
        "sym": default_scheme.get("sym", True),
        "data_type": default_scheme.get("data_type", "int"),
        "iters": 0,
        "model_free": True,
        "autoround_version": __version__,
    }

    if data_type in _NVFP4_E5M3_FAMILY_DATA_TYPES:
        for act_key in ("act_bits", "act_data_type", "act_group_size", "act_sym", "act_dynamic"):
            value = default_scheme.get(act_key)
            if value is not None:
                qconfig[act_key] = value

    if block_name_to_quantize:
        qconfig["block_name_to_quantize"] = block_name_to_quantize

    extra_config = {}
    for layer_name, cfg in layer_config.items():
        if cfg.get("bits", default_scheme["bits"]) >= 16:
            extra_config[layer_name] = {k: cfg.get(k) for k in scheme_keys if cfg.get(k) is not None}
            continue
        differs = False
        for key in ("bits", "group_size", "sym"):
            if cfg.get(key) is not None and cfg[key] != default_scheme.get(key):
                differs = True
                break
        if differs:
            extra_config[layer_name] = {k: cfg.get(k) for k in scheme_keys if cfg.get(k) is not None}

    # Filter out non-Linear ops (embed, conv) that don't need to be recorded in config.
    # Routing gates and other predefined patterns are still recorded.
    non_linear_ops = ["embed", "conv"]
    non_linear_re = re.compile("|".join(re.escape(op) for op in non_linear_ops))

    unique_ignored = list(dict.fromkeys(ignored_layers))
    for layer_name in unique_ignored:
        if layer_name not in extra_config:
            # Skip non-Linear ops (embed, conv) since they're not Linear layers
            if non_linear_re.search(layer_name):
                continue
            extra_config[layer_name] = {"bits": 16, "data_type": "float"}
            if data_type in _NVFP4_E5M3_FAMILY_DATA_TYPES:
                extra_config[layer_name].update({"act_bits": 16, "act_data_type": "float"})

    quantized_layer_set = set(quantized_layers)
    for lm_head_name in _LM_HEAD_PATTERNS:
        if lm_head_name in quantized_layer_set and lm_head_name not in extra_config:
            lm_head_cfg = layer_config.get(lm_head_name, default_scheme)
            extra_config[lm_head_name] = {k: lm_head_cfg.get(k) for k in scheme_keys if lm_head_cfg.get(k) is not None}

    if extra_config:
        qconfig["extra_config"] = extra_config

    return qconfig


# ---------------------------------------------------------------------------
# NVFP4 source -> MXFP4: rewrite the source quantization config (mixed-precision sources)
# ---------------------------------------------------------------------------
# Overall flow (scheme MXFP4 + model-free on a source detected as NVFP4, see _detect_nvfp4_source):
#   1. Shard processing (_process_shard): layers stored as NVFP4 are detected from tensor names and
#      converted to MXFP4; layers matched by layer_config are quantized as configured; every other
#      layer (BF16, FP8, kv-cache scales, ...) is copied as-is.
#   2. Config (_build_quantization_config, llm_compressor format only):
#      a. _load_source_quant_config: config.json quantization_config, else hf_quant_config.json & co.
#      b. _source_config_to_ct: compressed-tensors is used as-is; ModelOpt / Quark are converted to
#         compressed-tensors; anything else is warned about and skipped (plain MXFP4 config).
#      c. _rewrite_nvfp4_groups_as_mxfp: NVFP4 groups get the MXFP4 scheme (same targets); all other
#         groups, ignore and kv_cache_scheme are kept, so FP8 layers stay described as FP8.

# Standalone quantization metadata files, read when config.json has no quantization_config.
_SOURCE_QUANT_CONFIG_FILES: tuple[str, ...] = ("hf_quant_config.json", "quantization_config.json", "quant_config.json")
_FP8_STATIC_TENSOR_ARGS: dict = {
    "num_bits": 8,
    "type": "float",
    "symmetric": True,
    "dynamic": False,
    "strategy": "tensor",
}
_FP8_DYNAMIC_TOKEN_ARGS: dict = {
    "num_bits": 8,
    "type": "float",
    "symmetric": True,
    "dynamic": True,
    "strategy": "token",
}
# Placeholder for NVFP4 groups converted from ModelOpt / Quark; they are replaced by MXFP4 anyway.
_NVFP4_CT_GROUP: dict = {
    "format": "nvfp4-pack-quantized",
    "weights": {"num_bits": 4, "type": "float", "symmetric": True, "strategy": "tensor_group", "group_size": 16},
}


def _load_source_quant_config(source_dir: str | None, quantization_config: dict | None) -> dict:
    """Return the source quantization config, falling back to standalone metadata files."""
    if quantization_config:
        return quantization_config
    for fname in _SOURCE_QUANT_CONFIG_FILES:
        path = os.path.join(source_dir, fname) if source_dir else ""
        if not path or not os.path.isfile(path):
            continue
        try:
            with open(path) as f:
                data = json.load(f)
        except (OSError, ValueError) as e:
            logger.warning(f"Could not read source quantization metadata '{fname}': {e}; skipping it.")
            continue
        if not isinstance(data, dict) or not data:
            continue
        # ModelOpt hf_quant_config.json nests the settings under "quantization".
        if isinstance(data.get("quantization"), dict):
            return {**data["quantization"], "producer": data.get("producer"), "quant_method": "modelopt"}
        return data
    return {}


def _glob_to_ct_target(pattern: str) -> str:
    """Convert a ModelOpt / Quark fnmatch module pattern into a compressed-tensors target."""
    from fnmatch import translate

    return f"re:{translate(pattern)}" if any(c in pattern for c in "*?[") else pattern


def _ct_target_matches(target: Any, layer_name: str) -> bool:
    if not isinstance(target, str):
        return False
    if target.startswith("re:"):
        try:
            return re.match(target[3:], layer_name) is not None
        except re.error:
            return False
    return target == layer_name


def _lookup_ct_group(ct_config: dict, layer_name: str) -> dict | None:
    """Return the compressed-tensors config group of *layer_name* (name match beats ``Linear`` class match)."""
    if any(_ct_target_matches(t, layer_name) for t in ct_config.get("ignore") or []):
        return None
    class_match = None
    for group in ct_config["config_groups"].values():
        for target in group.get("targets") or []:
            if _ct_target_matches(target, layer_name):
                return group
            if class_match is None and target == "Linear":
                class_match = group
    return class_match


def _is_nvfp4_ct_group(group: dict) -> bool:
    weights = group.get("weights") or {}
    return weights.get("num_bits") == 4 and weights.get("type") == "float" and weights.get("group_size") == 16


def _modelopt_algo_to_ct_group(algo: Any) -> dict | None:
    algo = str(algo or "").upper()
    if algo == "NVFP4":
        return copy.deepcopy(_NVFP4_CT_GROUP)
    if algo == "FP8":
        weights, acts = {**_FP8_STATIC_TENSOR_ARGS}, {**_FP8_STATIC_TENSOR_ARGS}
    elif algo == "FP8_PER_CHANNEL_PER_TOKEN":
        weights, acts = {**_FP8_STATIC_TENSOR_ARGS, "strategy": "channel"}, {**_FP8_DYNAMIC_TOKEN_ARGS}
    elif algo == "FP8_PB_WO":
        weights, acts = {**_FP8_STATIC_TENSOR_ARGS, "strategy": "block", "block_structure": [128, 128]}, None
    else:
        return None
    return {"format": "float-quantized", "weights": weights, "input_activations": acts}


def _modelopt_to_ct_config(qc: dict) -> dict | None:
    """Convert a ModelOpt quantization config (``quant_algo`` / ``quantized_layers``) to compressed-tensors."""
    quantized_layers = qc.get("quantized_layers")
    if isinstance(quantized_layers, dict) and quantized_layers:
        targets_by_algo: dict[str, list[str]] = {}
        for layer_name, entry in quantized_layers.items():
            algo = str((entry or {}).get("quant_algo") or "").upper()
            targets_by_algo.setdefault(algo, []).append(layer_name)
    else:
        targets_by_algo = {str(qc.get("quant_algo") or "").upper(): ["Linear"]}

    config_groups: dict[str, dict] = {}
    for algo, targets in targets_by_algo.items():
        group = _modelopt_algo_to_ct_group(algo)
        if group is None:
            logger.warning(f"ModelOpt quant_algo '{algo}' is not supported; skipping the source quantization config.")
            return None
        config_groups[f"group_{len(config_groups)}"] = {**group, "targets": targets}

    ct_config = {
        "quant_method": "compressed-tensors",
        "config_groups": config_groups,
        # hf_quant_config.json uses "exclude_modules"; the config.json copy uses "ignore".
        "ignore": [_glob_to_ct_target(p) for p in qc.get("exclude_modules") or qc.get("ignore") or []],
    }
    kv_scheme = qc.get("kv_cache_scheme")
    if str(qc.get("kv_cache_quant_algo") or "").upper() == "FP8" or (
        isinstance(kv_scheme, dict) and kv_scheme.get("num_bits") == 8 and kv_scheme.get("type") == "float"
    ):
        ct_config["kv_cache_scheme"] = {**_FP8_STATIC_TENSOR_ARGS}
    return ct_config


def _quark_fp8_args(spec: dict) -> dict | None:
    dynamic = bool(spec.get("is_dynamic"))
    qscheme = spec.get("qscheme")
    if qscheme == "per_tensor":
        strategy = {"strategy": "tensor"}
    elif qscheme == "per_channel":
        strategy = {"strategy": "token" if dynamic else "channel"}
    elif qscheme == "per_group":
        strategy = {"strategy": "group", "group_size": spec.get("group_size")}
    else:
        return None
    return {"num_bits": 8, "type": "float", "symmetric": True, "dynamic": dynamic, **strategy}


def _quark_layer_to_ct_group(layer_cfg: Any) -> tuple[bool, dict | None]:
    """Return ``(supported, group)`` for a Quark layer config; ``group`` is None when unquantized."""
    weight = layer_cfg.get("weight") if isinstance(layer_cfg, dict) else None
    if not weight:
        return True, None
    if isinstance(weight, list):
        # Quark NVFP4: [fp4 per_group(16), fp8 per_tensor global scale].
        is_nvfp4 = len(weight) == 2 and weight[0].get("dtype") == "fp4" and weight[0].get("group_size") == 16
        return (True, copy.deepcopy(_NVFP4_CT_GROUP)) if is_nvfp4 else (False, None)
    weights = _quark_fp8_args(weight) if weight.get("dtype") == "fp8_e4m3" else None
    if weights is None:
        return False, None
    group = {"format": "float-quantized", "weights": weights}
    input_cfg = layer_cfg.get("input_tensors")
    if input_cfg:
        acts = (
            _quark_fp8_args(input_cfg) if isinstance(input_cfg, dict) and input_cfg.get("dtype") == "fp8_e4m3" else None
        )
        if acts is None:
            return False, None
        group["input_activations"] = acts
    return True, group


def _quark_to_ct_config(qc: dict) -> dict | None:
    """Convert a Quark quantization config to compressed-tensors."""
    if qc.get("layer_type_quant_config"):
        logger.warning("Quark 'layer_type_quant_config' is not supported; skipping the source quantization config.")
        return None
    entries = [(_glob_to_ct_target(p), cfg) for p, cfg in (qc.get("layer_quant_config") or {}).items()]
    entries.append(("Linear", qc.get("global_quant_config")))

    config_groups: dict[str, dict] = {}
    ignore = [_glob_to_ct_target(p) for p in qc.get("exclude") or []]
    for target, layer_cfg in entries:
        supported, group = _quark_layer_to_ct_group(layer_cfg)
        if not supported:
            logger.warning(
                f"Quark quantization config for '{target}' is not supported; skipping the source quantization config."
            )
            return None
        if group is None:
            if target != "Linear":
                ignore.append(target)
            continue
        config_groups[f"group_{len(config_groups)}"] = {**group, "targets": [target]}

    ct_config = {"quant_method": "compressed-tensors", "config_groups": config_groups, "ignore": ignore}
    if (qc.get("export") or {}).get("kv_cache_group"):
        ct_config["kv_cache_scheme"] = {**_FP8_STATIC_TENSOR_ARGS}
    return ct_config


def _source_config_to_ct(qc: dict | None) -> dict | None:
    """Return the source quantization config in compressed-tensors form, or None if not understood."""
    if not qc:
        return None
    method = str(qc.get("quant_method") or "").lower()
    producer = qc.get("producer")
    if method == "compressed-tensors" and isinstance(qc.get("config_groups"), dict):
        return copy.deepcopy(qc)
    if method == "modelopt" or (isinstance(producer, dict) and producer.get("name") == "modelopt"):
        return _modelopt_to_ct_config(qc)
    if method == "quark":
        return _quark_to_ct_config(qc)
    logger.warning(f"Source quantization config (quant_method='{method}') is not understood; skipping it.")
    return None


def _detect_nvfp4_source(model_name_or_path: str, source_quant_config: dict | None) -> str | None:
    """Return why the source checkpoint looks NVFP4-quantized, or None.

    Relies on the model name and the source quantization config (config.json / hf_quant_config.json & co.)
    because the first shards may not contain any NVFP4 tensor.
    """
    if "nvfp4" in os.path.basename(str(model_name_or_path).rstrip("/\\")).lower():
        return f"model name '{model_name_or_path}'"
    qc = source_quant_config or {}
    # Covers compressed-tensors "nvfp4-pack-quantized" and ModelOpt quant_algo "NVFP4".
    if "nvfp4" in json.dumps(qc, default=str).lower():
        return "source quantization config"
    groups = qc.get("config_groups") if isinstance(qc.get("config_groups"), dict) else {}
    if any(isinstance(g, dict) and _is_nvfp4_ct_group(g) for g in groups.values()):
        return "source quantization config"
    if str(qc.get("quant_method") or "").lower() == "quark":
        layer_cfgs = [qc.get("global_quant_config"), *(qc.get("layer_quant_config") or {}).values()]
        if any(_is_nvfp4_ct_group(_quark_layer_to_ct_group(cfg)[1] or {}) for cfg in layer_cfgs):
            return "source quantization config"
    return None


def _detect_fp8_source(model_name_or_path: str, source_quant_config: dict | None) -> str | None:
    """Return why the source checkpoint looks FP8-quantized (DeepSeek-V3 style), or None."""
    qc = source_quant_config or {}
    if (
        qc.get("quant_method") == "fp8"
        or qc.get("quantization_type") == "fp8"
        or str(qc.get("fmt") or "").startswith("e4m3")
    ):
        return "source quantization config"
    return None


# Source dtype name -> detector(model_name_or_path, source_quant_config) returning a reason or None.
_SOURCE_DTYPE_DETECTORS = {
    "nvfp4": _detect_nvfp4_source,
    "fp8": _detect_fp8_source,
}


def detect_model_dtypes(model_name_or_path: str, source_quant_config: dict | None) -> dict[str, str]:
    """Return ``{source_dtype: reason}`` for every low-precision dtype detected in the source checkpoint."""
    detected = {}
    for dtype_name, detector in _SOURCE_DTYPE_DETECTORS.items():
        reason = detector(model_name_or_path, source_quant_config)
        if reason is not None:
            detected[dtype_name] = reason
    return detected


def _rewrite_nvfp4_groups_as_mxfp(
    mxfp_qconfig: dict, source_ct: dict, ignored_layers: list[str], quantized_layers: list[str] | None = None
) -> dict:
    """Replace the NVFP4 groups of *source_ct* with the MXFP4 scheme and keep every other group as-is.

    Quantized layers outside the source NVFP4 groups (selected via layer_config) get extra groups.
    """
    mxfp_groups = mxfp_qconfig["config_groups"]
    # The default (catch-all) group targets "Linear"; layer_config override groups list explicit layers.
    default_name = next((n for n, g in mxfp_groups.items() if _is_catch_all_group(g)), next(iter(mxfp_groups)))
    mxfp_format = mxfp_qconfig.get("format")

    def _with_format(name: str) -> dict:
        group = copy.deepcopy(mxfp_groups[name])
        group["format"] = group.get("format") or mxfp_format
        return group

    source_format = source_ct.get("format")
    config_groups: dict[str, dict] = {}
    for name, group in source_ct["config_groups"].items():
        if _is_nvfp4_ct_group(group):
            group = {**_with_format(default_name), "targets": group.get("targets")}
        elif not group.get("format") and source_format != "mixed-precision":
            group = {**group, "format": source_format}
        config_groups[name] = group
    if not any(_is_nvfp4_ct_group(g) for g in source_ct["config_groups"].values()):
        logger.warning("Source quantization config has no NVFP4 group; writing a plain MXFP4 config.")
        return mxfp_qconfig

    extras: dict[str, list[str]] = {}
    for layer_name in dict.fromkeys(quantized_layers or []):
        if _is_nvfp4_ct_group(_lookup_ct_group(source_ct, layer_name) or {}):
            continue
        owner = next(
            (n for n, g in mxfp_groups.items() if n != default_name and layer_name in (g.get("targets") or [])),
            default_name,
        )
        extras.setdefault(owner, []).append(layer_name)
    for owner, layers in extras.items():
        name = f"group_{len(config_groups)}"
        while name in config_groups:
            name += "_extra"
        config_groups[name] = {**_with_format(owner), "targets": layers}
    extra_layers = {layer for layers in extras.values() for layer in layers}

    # NVFP4 source layers that were not converted (e.g. dequantized lm_head) are now unquantized.
    ignore = [p for p in source_ct.get("ignore") or [] if p not in extra_layers]
    for layer_name in dict.fromkeys(ignored_layers):
        group = _lookup_ct_group(source_ct, layer_name)
        if group is not None and _is_nvfp4_ct_group(group) and layer_name not in ignore:
            ignore.append(layer_name)

    formats = {group.get("format") for group in config_groups.values()}
    qconfig = {
        **mxfp_qconfig,
        "config_groups": config_groups,
        "ignore": ignore,
        "format": formats.pop() if len(formats) == 1 else "mixed-precision",
    }
    if source_ct.get("kv_cache_scheme"):
        qconfig["kv_cache_scheme"] = copy.deepcopy(source_ct["kv_cache_scheme"])
    return qconfig


# ---------------------------------------------------------------------------
# Scheme Validation and Override Resolution
# ---------------------------------------------------------------------------


def _apply_scheme_overrides(
    scheme: str | QuantizationScheme,
    scheme_overrides: dict | None = None,
) -> QuantizationScheme:
    """Return the effective scheme after applying non-None overrides."""
    scheme_obj = copy.deepcopy(_normalize_scheme(scheme))
    if not scheme_overrides:
        return scheme_obj

    valid_fields = {field.name for field in fields(QuantizationScheme)}
    for key, value in scheme_overrides.items():
        if key in valid_fields and value is not None:
            setattr(scheme_obj, key, value)
    return scheme_obj


def _validate_supported_scheme(
    scheme_obj: QuantizationScheme,
    scheme_input: str | QuantizationScheme,
) -> None:
    """Raise ``ValueError`` if *scheme_obj* is not supported by model-free.

    Model-free supports:

    * Integer weight-only quantization (sym/asym), ``bits ∈ {2, 4, 8}``,
      packed in the ``auto_round:auto_gptq`` format.
    * MXFP weight quantization (``data_type='mx_fp'``), ``bits ∈ {4, 8}``,
      ``group_size=32``, packed in ``mxfp4-pack-quantized`` / ``mxfp8-quantized``
      format (compressed-tensors compatible).
    """
    data_type = (scheme_obj.data_type or "int").lower()
    bits = scheme_obj.bits
    act_bits = scheme_obj.act_bits if scheme_obj.act_bits is not None else 16

    # Full-precision (BF16/FP16) default: all layers stay in full precision
    # unless layer_config provides lower-bit overrides.
    if (bits or 0) >= 16 and act_bits >= 16:
        return

    # MXFP weight-only path: accept mx_fp data type with bits in {4, 8}.
    # Activation quantization for MXFP is dynamic at inference time, so the
    # weight-only RTN path here is independent of act_bits.
    if is_mx_fp(data_type):
        if scheme_obj.act_data_type not in (None, "mx_fp"):
            raise ValueError(
                "Model-free MXFP supports only act_data_type='mx_fp', " f"but got '{scheme_obj.act_data_type}'."
            )
        # Restrict to the two explicitly supported MXFP presets when a string
        # name is provided.  Variants such as MXFP4_RCEIL / MXFP8_RCEIL use a
        # different activation format; silently mapping them to "MXFP4" /
        # "MXFP8" in the output config would misrepresent the requested scheme.
        if isinstance(scheme_input, str) and scheme_input.upper() not in ("MXFP4", "MXFP8"):
            raise ValueError(
                f"Model-free mode only supports MXFP preset names 'MXFP4' and 'MXFP8', "
                f"but got '{scheme_input}'. "
                f"Supported preset schemes: {list(SUPPORTED_PRESET_SCHEMES)}."
            )
        if bits is None or bits not in _SUPPORTED_MXFP_BITS:
            raise ValueError(
                f"Model-free mode supports MXFP bits in {_SUPPORTED_MXFP_BITS}, "
                f"but '{scheme_input}' requests bits={bits}. "
                f"Supported preset schemes: {list(SUPPORTED_PRESET_SCHEMES)}."
            )
        group_size = scheme_obj.group_size
        if group_size not in (None, 32):
            raise ValueError(
                f"Model-free mode supports MXFP only with group_size=32, "
                f"but '{scheme_input}' requests group_size={group_size}."
            )
        return

    if data_type == _NVFP4_E5M3_DATA_TYPE:
        if bits != 4 or scheme_obj.group_size != 16 or act_bits != 4:
            raise ValueError(
                f"Model-free NVFP4_E5M3 requires bits=4, group_size=16, and act_bits=4, "
                f"but '{scheme_input}' requests bits={bits}, group_size={scheme_obj.group_size}, "
                f"act_bits={act_bits}."
            )
        if (scheme_obj.act_data_type or "").lower() != _NVFP4_E5M3_DATA_TYPE or scheme_obj.act_group_size != 16:
            raise ValueError(
                f"Model-free NVFP4_E5M3 requires act_data_type='nvfp4_v2' and act_group_size=16, "
                f"but '{scheme_input}' requests act_data_type='{scheme_obj.act_data_type}', "
                f"act_group_size={scheme_obj.act_group_size}."
            )
        return

    if data_type == _NVFP4_E5M3_GS_DATA_TYPE:
        if bits != 4 or scheme_obj.group_size != 16 or act_bits != 4:
            raise ValueError(
                f"Model-free NVFP4_E5M3_GS requires bits=4, group_size=16, and act_bits=4, "
                f"but '{scheme_input}' requests bits={bits}, group_size={scheme_obj.group_size}, "
                f"act_bits={act_bits}."
            )
        act_data_type = (scheme_obj.act_data_type or "").lower()
        if act_data_type != _NVFP4_E5M3_GS_ACT_DATA_TYPE or scheme_obj.act_group_size != 16:
            raise ValueError(
                f"Model-free NVFP4_E5M3_GS requires act_data_type='{_NVFP4_E5M3_GS_ACT_DATA_TYPE}' and "
                f"act_group_size=16, but '{scheme_input}' requests act_data_type='{scheme_obj.act_data_type}', "
                f"act_group_size={scheme_obj.act_group_size}."
            )
        return

    if act_bits < 16:
        raise ValueError(
            f"Model-free mode only supports weight-only quantization (WOQ) schemes "
            f"where act_bits >= 16, but '{scheme_input}' has act_bits={act_bits}. "
            f"Supported preset schemes: {list(SUPPORTED_PRESET_SCHEMES)}."
        )

    if data_type != "int":
        raise ValueError(
            f"Model-free mode only supports integer weight quantization "
            f"(data_type='int') or MXFP (data_type='mx_fp'), but '{scheme_input}' "
            f"has data_type='{data_type}'. FP8 / NVFP / GGUF / BF16 schemes require "
            f"the standard AutoRound flow.  Supported preset schemes: "
            f"{list(SUPPORTED_PRESET_SCHEMES)}."
        )

    if bits is None or bits not in _SUPPORTED_INT_BITS:
        raise ValueError(
            f"Model-free mode supports bits in {_SUPPORTED_INT_BITS}, "
            f"but '{scheme_input}' requests bits={bits}. "
            f"Supported preset schemes: {list(SUPPORTED_PRESET_SCHEMES)}."
        )


def is_model_free_supported_scheme(
    scheme: str | QuantizationScheme,
    scheme_overrides: dict | None = None,
) -> bool:
    """Return True if *scheme* can be quantized via model-free mode.

    Useful for CLI auto-routing logic.  Never raises.
    """
    try:
        scheme_obj = _apply_scheme_overrides(scheme, scheme_overrides)
        _validate_supported_scheme(scheme_obj, scheme)
        return True
    except (ValueError, TypeError):
        return False


# ---------------------------------------------------------------------------
# AutoScheme support (two-phase: delta-loss selection + model-free packing)
# ---------------------------------------------------------------------------


def _looks_like_auto_scheme(scheme: Any) -> bool:
    """Duck-typed check for an :class:`~auto_round.auto_scheme.AutoScheme`.

    Avoids importing ``AutoScheme`` at module scope (it pulls in exporter /
    compressor modules that would create an import cycle with this file).
    """
    return hasattr(scheme, "options") and hasattr(scheme, "avg_bits")


def _validate_auto_scheme_options(auto_scheme: Any) -> str:
    """Validate AutoScheme options for model-free packing.

    Returns the single quantized data-type family shared by all options
    (``"int"`` or ``"mx_fp"``). ``BF16`` (or any >=16-bit no-op scheme) is
    allowed and treated as "keep layer in full precision" during conversion,
    so it does not contribute to the returned family.

    Raises ``ValueError`` when any non-BF16 option is unsupported or when INT
    and MXFP options are mixed (they use different packing formats and cannot
    be produced in one model-free run).
    """
    options = list(getattr(auto_scheme, "options", []) or [])
    if not options:
        raise ValueError("AutoScheme.options must be non-empty for model-free mode.")

    families: set[str] = set()
    unsupported: list[Any] = []
    for opt in options:
        # Preserve original string validation semantics so preset-name
        # restrictions (e.g. MXFP4/MXFP8 only) are enforced.
        if isinstance(opt, str):
            try:
                scheme_obj = _normalize_scheme(opt)
            except (ValueError, TypeError):
                scheme_obj = None
        elif isinstance(opt, QuantizationScheme):
            scheme_obj = opt
        else:
            scheme_obj = None

        # AutoScheme may include BF16/no-op options. In model-free flow these
        # layers become ignore_layers entries (full precision), so they are
        # allowed and excluded from family checks.
        if scheme_obj is not None:
            act_bits = scheme_obj.act_bits if scheme_obj.act_bits is not None else 16
            if (scheme_obj.bits or 0) >= 16 and act_bits >= 16:
                continue

        # GGUF k-quants carry super_bits and are not packable by the model-free
        # RTN kernel even though their data_type is nominally "int".
        if scheme_obj is None or getattr(scheme_obj, "super_bits", None) is not None:
            unsupported.append(opt)
            continue
        if not is_model_free_supported_scheme(opt):
            unsupported.append(opt)
            continue

        data_type = (scheme_obj.data_type or "int").lower()
        families.add("mx_fp" if is_mx_fp(data_type) else "int")

    if unsupported:
        raise ValueError(
            f"Model-free + AutoScheme received unsupported option(s): {unsupported}. "
            f"Model-free supports INT WOQ (bits in {_SUPPORTED_INT_BITS}) and MXFP "
            f"(bits in {_SUPPORTED_MXFP_BITS}); GGUF / NVFP4 / FP8 options are not "
            f"packable in model-free mode. Remove the unsupported options or pass "
            f"disable_model_free=True to use the regular flow."
        )
    if len(families) > 1:
        raise ValueError(
            "Model-free + AutoScheme cannot mix INT and MXFP options in a single run "
            f"(got families {sorted(families)}); INT and MXFP use different packing "
            "formats. Use a single data-type family, or pass disable_model_free=True."
        )
    return families.pop()


def _convert_auto_scheme_layer_config(
    generated: dict[str, dict],
    preferred_base_scheme: str | QuantizationScheme | None = None,
) -> tuple[QuantizationScheme, dict[str, dict], list[str]]:
    """Convert an AutoScheme-generated ``layer_config`` into model-free inputs.

    Returns ``(base_scheme, per_layer_overrides, fp16_layers)`` where:

        * ``base_scheme`` is ``preferred_base_scheme`` when provided, matching the
            primary scheme selected by the regular AutoRound path. Otherwise the
            most common quantized scheme is used as a fallback.
    * ``per_layer_overrides`` maps every quantized layer name to its resolved
      :class:`QuantizationScheme` fields.
    * ``fp16_layers`` lists layers AutoScheme kept at >= 16 bits (added to the
      model-free ignore list so they stay in full precision).
    """
    from collections import Counter

    scheme_keys = {f.name for f in fields(QuantizationScheme)}
    per_layer: dict[str, dict] = {}
    fp16_layers: list[str] = []
    counter: Counter[tuple] = Counter()

    for name, cfg in generated.items():
        if not isinstance(cfg, dict):
            continue
        bits = cfg.get("bits")
        data_type_raw = (cfg.get("data_type") or "").strip()

        # Infer bits from compact dtype aliases that embed bit-width in the name,
        # e.g. "mxfp8" → 8, "MXFP4" → 4, "mx_fp4" → 4.
        if bits is None and data_type_raw:
            dt_lower = data_type_raw.lower()
            for prefix in ("mxfp", "mx_fp", "nvfp", "nv_fp"):
                tail = dt_lower[len(prefix) :]
                if dt_lower.startswith(prefix) and tail.isdigit():
                    bits = int(tail)
                    break

        if bits is None:
            continue

        clean = {k: cfg[k] for k in scheme_keys if cfg.get(k) is not None}
        clean["bits"] = bits  # honour inferred value when not explicit

        # Normalise compact dtype aliases to canonical forms understood by
        # the quantization kernels ("mxfp8" / "MXFP4" → "mx_fp").
        if data_type_raw:
            dt_lower = data_type_raw.lower()
            if dt_lower.startswith(("mxfp", "mx_fp")):
                clean["data_type"] = "mx_fp"
            elif dt_lower.startswith(("nvfp", "nv_fp")):
                clean["data_type"] = "nv_fp"

        if bits >= 16:
            fp16_layers.append(name)
            continue
        data_type = (clean.get("data_type") or "int").lower()
        per_layer[name] = clean
        counter[(clean.get("bits"), clean.get("group_size"), bool(clean.get("sym", True)), data_type)] += 1

    if not counter:
        raise ValueError("AutoScheme did not assign any quantizable layers for model-free mode.")

    if preferred_base_scheme is not None:
        base_scheme = copy.deepcopy(_normalize_scheme(preferred_base_scheme))
    else:
        (base_bits, base_group_size, base_sym, base_dtype), _ = counter.most_common(1)[0]
        base_scheme = QuantizationScheme(
            bits=base_bits,
            group_size=base_group_size,
            sym=base_sym,
            data_type=base_dtype,
        )
    return base_scheme, per_layer, fp16_layers


# ---------------------------------------------------------------------------
# Model-Special Handlers (Keep at End)
# ---------------------------------------------------------------------------


def _read_int_like(value: Any) -> int | None:
    """Best-effort conversion for int-like config values."""
    if isinstance(value, bool) or value is None:
        return None
    if isinstance(value, int):
        return value
    if isinstance(value, float) and value.is_integer():
        return int(value)
    if isinstance(value, str):
        v = value.strip()
        if v.isdigit() or (v.startswith("-") and v[1:].isdigit()):
            return int(v)
    return None


def _read_bool_like(value: Any) -> bool | None:
    """Best-effort conversion for bool-like config values."""
    if isinstance(value, bool):
        return value
    if isinstance(value, str):
        v = value.strip().lower()
        if v in {"true", "1", "yes"}:
            return True
        if v in {"false", "0", "no"}:
            return False
    return None


def _resolve_kimi_k25_int4_params(source_quant_config: dict | None) -> tuple[int | None, bool]:
    """Resolve default (group_size, sym) for kimi_k25 INT4 source dequant."""
    group_size: int | None = None
    sym = True

    if not isinstance(source_quant_config, dict):
        return group_size, sym

    candidates = [source_quant_config]
    weights_cfg = source_quant_config.get("weights")
    if isinstance(weights_cfg, dict):
        candidates.append(weights_cfg)

    for cfg in candidates:
        if group_size is None:
            for key in ("group_size", "weight_group_size"):
                maybe = _read_int_like(cfg.get(key))
                if maybe is not None and maybe > 0:
                    group_size = maybe
                    break

        sym_val = _read_bool_like(cfg.get("sym"))
        if sym_val is not None:
            sym = sym_val

        zp_val = _read_bool_like(cfg.get("zero_point"))
        if zp_val is not None:
            sym = not zp_val

    return group_size, sym


def _collect_kimi_k25_int4_source_entries(
    raw_tensors: dict[str, torch.Tensor],
) -> list[tuple[str, str, str, str | None]]:
    """Collect kimi_k25 INT4 source tensors."""
    entries: list[tuple[str, str, str, str | None]] = []
    for name, packed in raw_tensors.items():
        if not name.endswith(".weight_packed"):
            continue
        if packed.dtype not in (torch.uint8, torch.int8, torch.int32):
            continue

        layer_name = name[: -len(".weight_packed")]
        scale_key = f"{layer_name}.weight_scale"
        if scale_key not in raw_tensors:
            continue

        scale = raw_tensors[scale_key]
        if scale.dtype not in (torch.float16, torch.bfloat16, torch.float32, torch.float64):
            continue
        if packed.dim() != 2 or scale.dim() != 2:
            continue

        zp_key = None
        for candidate in (f"{layer_name}.weight_zero_point", f"{layer_name}.zero_point"):
            if candidate in raw_tensors:
                zp_key = candidate
                break

        entries.append((layer_name, name, scale_key, zp_key))

    return entries


def _unpack_int4_weight_packed(packed: torch.Tensor) -> torch.Tensor:
    """Unpack packed int4 bytes/ints into int16 ``[out, in]`` nibbles."""
    if packed.dtype == torch.int32:
        packed_i32 = packed.view(torch.int32)
        rows, cols = packed_i32.shape
        nibble_list = [(packed_i32 >> (4 * i)) & 0xF for i in range(8)]
        out = torch.stack(nibble_list, dim=-1).reshape(rows, cols * 8).to(torch.int16)
        return out

    packed_u8 = packed.view(torch.uint8)
    lo = packed_u8 & 0x0F
    hi = (packed_u8 >> 4) & 0x0F

    out = torch.empty(
        (packed_u8.shape[0], packed_u8.shape[1] * 2),
        dtype=torch.int16,
        device=packed_u8.device,
    )
    out[:, 0::2] = lo.to(torch.int16)
    out[:, 1::2] = hi.to(torch.int16)
    return out


def _dequant_kimi_k25_int4_tensors(
    raw_tensors: dict[str, torch.Tensor],
    source_quant_config: dict | None = None,
    device: str = "cpu",
    shard_name: str | None = None,
    dequantize_with_device_fallback: Callable[..., torch.Tensor] | None = None,
) -> dict[str, torch.Tensor]:
    """Dequantize kimi_k25-style INT4 packed weights to float32 ``.weight``."""
    if dequantize_with_device_fallback is None:
        raise ValueError("dequantize_with_device_fallback callback is required")

    entries = _collect_kimi_k25_int4_source_entries(raw_tensors)
    if not entries:
        return raw_tensors

    default_group_size, _ = _resolve_kimi_k25_int4_params(source_quant_config)
    dequant_device = str(device or "cpu")
    shard_prefix = f"[{shard_name}] " if shard_name else ""
    logger.info(
        f"{shard_prefix}Dequantizing kimi_k25 INT4 tensor(s) to float32 on {dequant_device}: total={len(entries)}."
    )

    for layer_name, packed_key, scale_key, zp_key in entries:
        packed = raw_tensors.pop(packed_key)
        scale = raw_tensors.pop(scale_key)
        zero = raw_tensors.pop(zp_key) if zp_key is not None else None
        raw_tensors.pop(f"{layer_name}.weight_shape", None)

        nibbles_per_element = 8 if packed.dtype == torch.int32 else 2
        in_features = packed.shape[1] * nibbles_per_element
        groups = scale.shape[1]
        inferred_group_size = (in_features // groups) if groups > 0 and in_features % groups == 0 else None
        group_size = inferred_group_size or default_group_size
        if group_size is None or group_size <= 0:
            raise ValueError(
                f"Cannot resolve group_size for INT4 tensor '{layer_name}': "
                f"packed_shape={tuple(packed.shape)}, scale_shape={tuple(scale.shape)}"
            )
        if in_features % group_size != 0:
            raise ValueError(
                f"Invalid group_size={group_size} for INT4 tensor '{layer_name}': "
                f"in_features={in_features} is not divisible by group_size."
            )

        if (
            inferred_group_size is not None
            and default_group_size is not None
            and inferred_group_size != default_group_size
        ):
            logger.warning(
                f"{shard_prefix}k25 INT4 group_size mismatch for {layer_name}: "
                f"config={default_group_size}, inferred={inferred_group_size}. "
                "Using inferred value from tensor shapes."
            )

        n_groups = in_features // group_size
        if scale.shape[1] != n_groups:
            raise ValueError(
                f"Scale shape mismatch for INT4 tensor '{layer_name}': "
                f"scale.shape={tuple(scale.shape)}, expected second dim {n_groups}."
            )

        def _dequant_impl(target_device: str) -> torch.Tensor:
            q = _unpack_int4_weight_packed(packed.to(target_device, non_blocking=True))
            sc = scale.to(target_device, non_blocking=True).to(torch.float32)

            if zero is not None:
                zp = zero.to(target_device, non_blocking=True)
                if zp.dim() == 2 and zp.shape == packed.shape and zp.dtype in (torch.int8, torch.uint8):
                    zp = _unpack_int4_weight_packed(zp)
                elif zp.dim() == 2 and zp.shape[1] != n_groups:
                    raise ValueError(f"Unsupported zero-point shape for INT4 tensor '{layer_name}': {tuple(zp.shape)}")
                if zp.dim() == 2 and zp.shape[1] == n_groups:
                    zp = zp.to(torch.int16).repeat_interleave(group_size, dim=1)
                zp = zp[:, : q.shape[1]].to(torch.int16)
                q = q - zp
            else:
                # Kimi RAWINT4 stores signed values with a fixed bias of 8:
                # logical_q = nibble - 8, not 4-bit two's-complement.
                q = q - 8

            sc = sc.repeat_interleave(group_size, dim=1)[:, : q.shape[1]]
            return q.to(torch.float32) * sc

        dq_weight = dequantize_with_device_fallback(
            dequant_device=dequant_device,
            shard_prefix=shard_prefix,
            op_name="kimi_k25 INT4 dequant",
            tensor_label=layer_name,
            on_device=lambda: _dequant_impl(dequant_device).to("cpu"),
            on_cpu=lambda: _dequant_impl("cpu"),
        )
        raw_tensors[f"{layer_name}.weight"] = dq_weight

    return raw_tensors


def _expand_e8m0_block_scale(
    scale: torch.Tensor,
    out_features: int,
    in_features: int,
    group_size: int = 32,
) -> torch.Tensor:
    """Expand a coarse 2D E8M0 block scale to per-group llm-compressor layout."""
    scale = scale.view(torch.uint8)
    if scale.dim() != 2:
        raise ValueError(f"Expected a 2D E8M0 block scale, got shape {tuple(scale.shape)}.")

    target_rows = out_features
    target_cols = in_features // group_size
    rows, cols = scale.shape

    if target_rows % rows == 0 and target_cols % cols == 0:
        if target_rows != rows:
            scale = scale.repeat_interleave(target_rows // rows, dim=0)
        if target_cols != cols:
            scale = scale.repeat_interleave(target_cols // cols, dim=1)
        return scale.contiguous()

    coarse_block = 128
    expected_rows = (out_features + coarse_block - 1) // coarse_block
    expected_cols = (in_features + coarse_block - 1) // coarse_block
    if rows == expected_rows and cols == expected_cols:
        if coarse_block % group_size != 0:
            raise ValueError(
                f"Cannot expand DeepSeek E8M0 block scale with group_size={group_size}; "
                f"{coarse_block} is not divisible by group_size."
            )
        groups_per_block_col = coarse_block // group_size
        scale = scale.repeat_interleave(coarse_block, dim=0)[:target_rows]
        scale = scale.repeat_interleave(groups_per_block_col, dim=1)[:, :target_cols]
        return scale.contiguous()

    raise ValueError(
        f"Cannot expand E8M0 block scale {tuple(scale.shape)} to "
        f"({target_rows}, {target_cols}); unsupported coarse/block layout."
    )


def preprocess_model_type_source_tensors(
    raw_tensors: dict[str, torch.Tensor],
    model_type: str | None,
    group_size: int = 32,
    quantization_config: dict | None = None,
    shard_name: str | None = None,
) -> tuple[dict[str, torch.Tensor], dict[str, int]]:
    """Apply model-type-specific source tensor normalization."""
    model_type = (model_type or "").lower()
    quantization_config = quantization_config or {}
    is_deepseek_v4 = model_type == "deepseek_v4"
    is_deepseek_v32_ue8m0 = (
        model_type == "deepseek_v32"
        and quantization_config.get("quant_method") == "fp8"
        and str(quantization_config.get("fmt", "")).lower() == "e4m3"
        and str(quantization_config.get("scale_fmt", "")).lower() == "ue8m0"
    )
    if not is_deepseek_v4 and not is_deepseek_v32_ue8m0:
        return raw_tensors, {}

    entries: list[tuple[str, str, bool]] = []
    for name, tensor in raw_tensors.items():
        if not name.endswith(".weight"):
            continue
        layer_name = name[: -len(".weight")]
        scale_candidates = [f"{layer_name}.scale"]
        if is_deepseek_v32_ue8m0:
            scale_candidates.extend((f"{layer_name}.weight_scale", f"{layer_name}.weight_scale_inv"))
        scale_name = next((candidate for candidate in scale_candidates if candidate in raw_tensors), None)
        if scale_name is None:
            continue
        if tensor.dtype == torch.float8_e4m3fn:
            entries.append((name, scale_name, True))
        elif tensor.dtype in (torch.int8, torch.uint8):
            entries.append((name, scale_name, False))

    if not entries:
        return raw_tensors, {}

    source_state: dict[str, int] = {}
    n_fp8 = 0
    n_fp4 = 0
    for weight_name, scale_name, is_fp8 in entries:
        layer_name = weight_name[: -len(".weight")]
        weight = raw_tensors.pop(weight_name)
        scale = raw_tensors.pop(scale_name)

        if is_fp8:
            out_features, in_features = weight.shape
            weight_key = f"{layer_name}.weight"
            source_state[layer_name] = 8
            n_fp8 += 1

            if scale.dtype == torch.float32:
                shard_prefix = f"[{shard_name}] " if shard_name else ""
                logger.warning_once(
                    f"[{model_type}] Scale tensor pattern has dtype float32 "
                    f"with UE8M0 encoding (only the 8-bit exponent is significant). "
                    f"Extracting uint8 E8M0 exponent bytes from fp32 representation."
                )
                scale = ((scale.view(torch.int32) >> 23) & 0xFF).to(torch.uint8)
        else:
            out_features = weight.shape[0]
            in_features = weight.shape[1] * 2
            weight = weight.view(torch.uint8).contiguous()
            weight_key = f"{layer_name}.weight_packed"
            source_state[layer_name] = 4
            n_fp4 += 1

        weight_scale = _expand_e8m0_block_scale(scale, out_features, in_features, group_size=group_size)
        raw_tensors[weight_key] = weight
        raw_tensors[f"{layer_name}.weight_scale"] = weight_scale

    shard_prefix = f"[{shard_name}] " if shard_name else ""
    logger.info(
        f"{shard_prefix}Applied model_type preprocessing for {model_type}: "
        f"{n_fp8} MXFP8 layer(s), {n_fp4} MXFP4 layer(s) converted to llm-compressor naming."
    )
    return raw_tensors, source_state


def handle_model_type_low_precision_source_tensors(
    raw_tensors: dict[str, torch.Tensor],
    model_type: str | None,
    source_quant_config: dict | None = None,
    device: str = "cpu",
    shard_name: str | None = None,
    dequantize_with_device_fallback: Callable[..., torch.Tensor] | None = None,
) -> dict[str, torch.Tensor]:
    """Handle model-type-specific low-precision source tensors."""
    if dequantize_with_device_fallback is None:
        dequantize_with_device_fallback = _dequantize_with_device_fallback

    if (model_type or "").lower() == "kimi_k25":
        return _dequant_kimi_k25_int4_tensors(
            raw_tensors,
            source_quant_config=source_quant_config,
            device=device,
            shard_name=shard_name,
            dequantize_with_device_fallback=dequantize_with_device_fallback,
        )
    return raw_tensors