PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
refactor: reorganize tensor utility functions and enhance split_fused_expert_tensors for special cases (#2320)
Xin Hec18bf770e603
diff --git a/auto_round/compressors/shard_writer.py b/auto_round/compressors/shard_writer.pyindex 6ac0c8b06..3aaf72058 100644--- a/auto_round/compressors/shard_writer.py+++ b/auto_round/compressors/shard_writer.py@@ -238,7 +238,7 @@ class ShardWriter:List of (key, 2D tensor) pairs, or None if not a fused expert param."""from auto_round.modeling.fused_moe.replace_modules import MOE_SKIP_PREFIXES- from auto_round.utils.missing_tensors import split_fused_expert_tensors+ from auto_round.utils.model_free_utils import split_fused_expert_tensorsparts = name.rsplit(".", 1)if len(parts) != 2:diff --git a/auto_round/utils/__init__.py b/auto_round/utils/__init__.pyindex 4e8b9c71d..67987b294 100644--- a/auto_round/utils/__init__.py+++ b/auto_round/utils/__init__.py@@ -21,7 +21,13 @@ from auto_round.utils.weight_handler import (detect_weight_type,is_quantized_input_module,)-from auto_round.utils.missing_tensors import copy_missing_tensors_from_source+++def copy_missing_tensors_from_source(*args, **kwargs):+ from auto_round.utils.missing_tensors import copy_missing_tensors_from_source as _copy_missing_tensors_from_source++ return _copy_missing_tensors_from_source(*args, **kwargs)+import transformersfrom packaging.version import Versiondiff --git a/auto_round/utils/missing_tensors.py b/auto_round/utils/missing_tensors.pyindex 17e0ca2e5..457b608ce 100644--- a/auto_round/utils/missing_tensors.py+++ b/auto_round/utils/missing_tensors.py@@ -39,172 +39,18 @@import jsonimport os-import re-from typing import Optional, Tupleimport torchfrom auto_round.logger import loggerfrom auto_round.utils.common import compress_layer_names+from auto_round.utils.model_free_utils import (+ _normalize_tensor_name_for_warning,+ quantize_weight_rtn,+ split_fused_expert_tensors,+)from auto_round.utils.weight_handler import _dequant_fp8_linear_weight-# ------------------------------------------------------------------ #-# Fused expert projection patterns #-# ------------------------------------------------------------------ #-# Maps a fused projection name to its constituent split names.-# When a 3D tensor ``*.experts.<fused_name>`` (shape [num_experts, ...])-# is encountered, it is split along dimension 0 of the per-expert 2-D-# slice to produce one tensor per split name per expert.-_FUSED_EXPERT_PROJ_PATTERNS: dict[str, list[str]] = {- "gate_up_proj": ["gate_proj", "up_proj"],- "w13": ["w1", "w3"],-}--_AUTOROUND_ISSUE_URL = "https://github.com/intel/auto-round/issues"-_WARNING_INDEX_PLACEHOLDER = "<idx>"--# model_type values whose native inference-engine loader (e.g. vLLM's custom-# MoE weight loader) expects the *fused*, per-layer stacked 3-D expert-# tensors (``experts.w13_weight`` / ``experts.w2_weight``) to be preserved-# as-is (quantized in place) rather than unfused into per-expert 2-D-# tensors. Splitting these would break loading for such architectures.-_KEEP_FUSED_EXPERT_MODEL_TYPES: frozenset[str] = frozenset({"inkling_mm_model"})---def _normalize_tensor_name_for_warning(name: str, numeric_replacement: str = _WARNING_INDEX_PLACEHOLDER) -> str:- """Normalize tensor names for warning_once deduplication.-- Replace standalone numeric path segments (e.g. ``layers.12.experts.3``)- and bracket indices (e.g. ``layers[12]``) with a fixed placeholder- (``<idx>`` by default) so warning keys are stable across different- layer/expert ids.- """- parts = name.split(".")- normalized_parts = []- for part in parts:- if part.isdigit():- normalized_parts.append(numeric_replacement)- continue- normalized_parts.append(re.sub(r"\[(\d+)\]", f"[{numeric_replacement}]", part))- return ".".join(normalized_parts)---def split_fused_expert_tensors(- tensors_dict: dict[str, torch.Tensor],- model_type: str | None = None,-) -> dict[str, torch.Tensor]:- """Split 3-D fused expert tensors into per-expert 2-D tensors.-- Many MoE checkpoints store expert weights as fused 3-D parameters- under either ``*.experts.<proj_name>`` or ``*.moe.<proj_name>``- (for example ``gate_up_proj`` with shape- ``[num_experts, 2*intermediate, hidden]``, or ``down_proj`` with shape- ``[num_experts, out, in]``). After unfusing, each expert gets its own- 2-D weight tensor, e.g.- ``experts.0.gate_proj.weight [intermediate, hidden]``.-- Splitting rules:-- * **gate_up_proj** ``[N, 2*inter, hidden]`` →- ``experts.{i}.gate_proj.weight`` + ``experts.{i}.up_proj.weight``- * **up_gate_proj** ``[N, 2*inter, hidden]`` →- ``experts.{i}.up_proj.weight`` + ``experts.{i}.gate_proj.weight``- * **Other** stacked projections (e.g. ``down_proj``) ``[N, out, in]`` →- ``experts.{i}.<proj>.weight``-- For ``*.moe.<proj_name>`` tensors, outputs use- ``*.moe.experts.{i}.<proj>.weight`` so they align with the sequential- expert module layout used by quantized MoE replacements.-- Non-3-D or non-expert tensors pass through unchanged.-- Some architectures (see ``_KEEP_FUSED_EXPERT_MODEL_TYPES``) ship a custom- inference-engine loader that consumes the fused 3-D stacked tensors- directly (quantizing/packing them in place without unfusing per expert).- For those ``model_type`` values, this function is a no-op passthrough.-- Args:- tensors_dict: Mapping of tensor names to tensors.- model_type: Source model's ``model_type`` (from ``config.json``), used- to skip splitting for architectures whose native loader expects- fused stacked expert tensors.-- Returns:- New dict with fused expert tensors replaced by per-expert 2-D tensors.- """- if (model_type or "").lower() in _KEEP_FUSED_EXPERT_MODEL_TYPES:- return dict(tensors_dict)-- result: dict[str, torch.Tensor] = {}- split_count = 0-- for tensor_name, tensor in tensors_dict.items():- if tensor.dim() != 3:- result[tensor_name] = tensor- continue-- warning_tensor_name = _normalize_tensor_name_for_warning(tensor_name)-- # Strip optional .weight suffix for pattern matching- stripped = tensor_name- stripped = stripped.removesuffix(".weight") # len(".weight") == 7-- # Expect: <prefix>.experts.<proj_name>- dot_idx = stripped.rfind(".")- if dot_idx < 0:- result[tensor_name] = tensor- continue-- parent = stripped[:dot_idx]- proj_name = stripped[dot_idx + 1 :]-- parent_last = parent.split(".")[-1]- is_experts_parent = parent.endswith("experts") or parent_last == "experts"- is_moe_parent = parent.endswith("moe") or parent_last == "moe"-- # The immediate parent must be "experts" or "moe"- if not is_experts_parent and not is_moe_parent:- logger.warning_once(- "Found 3-D tensor '%s' while splitting expert tensors; "- "it will be kept unchanged. If this is an MoE/expert weight that should be split/quantized, "- "please open an issue at %s.",- warning_tensor_name,- _AUTOROUND_ISSUE_URL,- )- result[tensor_name] = tensor- continue-- target_prefix = parent if is_experts_parent else f"{parent}.experts"-- num_experts = tensor.shape[0]-- if proj_name in _FUSED_EXPERT_PROJ_PATTERNS:- split_names = _FUSED_EXPERT_PROJ_PATTERNS[proj_name]- logger.warning_once(- f"Splitting fused expert tensor '{warning_tensor_name}' "- f"(shape={list(tensor.shape)}, num_experts={num_experts}) "- f"into {split_names}"- )- for i in range(num_experts):- expert_2d = tensor[i] # [fused_out, in_features]- chunks = expert_2d.chunk(len(split_names), dim=0)- for split_name, chunk in zip(split_names, chunks):- out_key = f"{target_prefix}.{i}.{split_name}.weight"- result[out_key] = chunk.contiguous()- else:- logger.warning_once(- f"Splitting stacked expert tensor '{warning_tensor_name}' "- f"(shape={list(tensor.shape)}, num_experts={num_experts})"- )- for i in range(num_experts):- expert_2d = tensor[i] # [out, in]- out_key = f"{target_prefix}.{i}.{proj_name}.weight"- result[out_key] = expert_2d.contiguous()-- split_count += 1-- return result-def copy_missing_tensors_from_source(source_dir: str,@@ -534,174 +380,6 @@ def copy_missing_tensors_from_source()-def quantize_weight_rtn(- weight: torch.Tensor,- bits: int,- group_size: int,- sym: bool = True,- device: Optional[torch.device] = None,- disable_opt_rtn: bool = True,- *,- packing: str,-) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:- """Quantize a 2-D weight tensor and pack into auto_gptq format.-- Parameters- ----------- weight : Tensor [out_features, in_features]- bits : target bit-width (e.g. 4, 8)- group_size : quantization group size along in_features- sym : use symmetric quantisation- device : compute device (cuda / cpu). Results are always returned on CPU.- disable_opt_rtn : when False and sym=True, use the optimised-RTN scale- search (``quant_tensor_opt_rtn_sym``) which evaluates three E8M0- candidates per group and picks the best MSE. Defaults to True- (plain RTN) to preserve backward-compatible behaviour.- packing : the packing_format the artifact will declare (required); the- zero-point convention follows the inference backend registry. GPTQ_FORMAT- entries ("auto_round:auto_gptq", qlinear_torch_zp) store zp - 1 per- nibble and unpack with +1; GPTQ_FORMAT_NO_ZP entries ("auto_round",- "auto_round:gptqmodel", qlinear_torch) store the zero point directly.- Any other value raises.-- Returns- -------- qweight : [in_features // pack_factor, out_features] int32- qzeros : [num_groups, out_features // pack_factor] int32- scales : [num_groups, out_features] float16- """- assert weight.dim() == 2, f"Expected 2-D weight, got {weight.dim()}-D"- from auto_round.inference.backend import GPTQ_FORMAT, GPTQ_FORMAT_NO_ZP-- if packing in GPTQ_FORMAT:- zp_minus_one = True- elif packing in GPTQ_FORMAT_NO_ZP:- zp_minus_one = False- else:- raise ValueError(- f"unsupported packing format '{packing}': zero-point conventions are defined only for "- f"{GPTQ_FORMAT} (zp - 1, unpacked with +1) and {GPTQ_FORMAT_NO_ZP} (zp stored directly)"- )- out_features, in_features = weight.shape- if device is None:- device = weight.device- # Single-step transfer + cast avoids an intermediate BF16 copy on CUDA- # (``weight.to(device).float()`` would briefly allocate both BF16 and- # float32 buffers on the target device).- weight = weight.to(device=device, dtype=torch.float32)-- # --- pad in_features to multiple of group_size ---- if in_features % group_size != 0:- pad = group_size - (in_features % group_size)- weight = torch.nn.functional.pad(weight, (0, pad))- in_features = weight.shape[1]-- num_groups = in_features // group_size- pack_factor = 32 // bits # values per int32-- # --- pad out_features to multiple of pack_factor (needed for qzeros) ---- out_pad = 0- if out_features % pack_factor != 0:- out_pad = pack_factor - (out_features % pack_factor)- weight = torch.nn.functional.pad(weight, (0, 0, 0, out_pad))- padded_out = weight.shape[0]-- from auto_round.data_type.utils import get_quant_func, reshape_pad_tensor_by_group_size-- # Use get_quant_func (same as WrapperLinear) so all data types and opt_rtn- # variants are resolved via the QUANT_FUNC_WITH_DTYPE registry uniformly.- quant_func, _ = get_quant_func("int", bits, sym=sym, disable_opt_rtn=disable_opt_rtn, iters=0)- # quant_func returns (qdq_result, scale, zp_or_maxq)- _, scale, zp_val = quant_func(weight, bits=bits, group_size=group_size)-- if sym:- maxq = 1 << (bits - 1) # e.g. 8 for 4-bit- zero_point = maxq # unsigned offset for packing-- # scale shape: [padded_out * num_groups, 1]- # Reshape weight for group-wise quantization: [padded_out * num_groups, group_size]- w_grouped, _, _ = reshape_pad_tensor_by_group_size(weight, group_size)- w_grouped = w_grouped.to(device=device, dtype=torch.float32)- del weight-- # Compute integer values for packing- q = (w_grouped / scale).round_().clamp_(-maxq, maxq - 1)- del w_grouped- q += zero_point # shift to unsigned [0, 2*maxq - 1]- q = q.to(torch.int32)-- # scale → [num_groups, padded_out] (float16)- scales_out = scale.squeeze(-1).reshape(padded_out, num_groups).t().contiguous().to(torch.float16)- del scale-- zp = torch.full((num_groups, padded_out), zero_point, dtype=torch.int32, device=device)- else:- # Asymmetric quantization- max_int = (1 << bits) - 1-- # scale shape: [padded_out * num_groups, 1], zp_val shape: [padded_out * num_groups, 1]-- # Reshape weight for group-wise quantization- w_grouped, _, _ = reshape_pad_tensor_by_group_size(weight, group_size)- w_grouped = w_grouped.to(device=device, dtype=torch.float32)- del weight-- # Compute integer values for packing- q = (w_grouped / scale).round_()- del w_grouped- q += zp_val- q.clamp_(0, max_int)- q = q.to(torch.int32)-- # scale → [num_groups, padded_out] (float16)- scales_out = scale.squeeze(-1).reshape(padded_out, num_groups).t().contiguous().to(torch.float16)- del scale-- # zp → [num_groups, padded_out]- zp = zp_val.squeeze(-1).reshape(padded_out, num_groups).t().contiguous().to(torch.int32)- del zp_val-- # q → [in_features, padded_out]- q = q.reshape(padded_out, in_features).t().contiguous()-- # ---- Pack qweight: [in_features // pack_factor, padded_out] ----- # Vectorised: reshape → broadcast shift → int64 sum (≡ bitwise-OR for- # non-overlapping bit lanes) avoids a Python loop per bit-lane.- _shifts = torch.arange(pack_factor, dtype=torch.int64, device=device) * bits- q_packed = q.reshape(in_features // pack_factor, pack_factor, padded_out).to(torch.int64)- del q- qweight = (q_packed << _shifts[None, :, None]).sum(dim=1).to(torch.int32)- del q_packed-- # ---- Pack qzeros: [num_groups, padded_out // pack_factor] ----- # The zero-point convention is a property of the declared packing format- # (see the backend registry): the auto_round:auto_gptq format- # (qlinear_torch_zp) adds +1 to zeros after unpacking, so we subtract 1- # before packing to compensate. The (zp-1) packing cannot represent- # zp=0: a -1 left-shifts into a negative word and corrupts every nibble- # sharing it. Clamp the stored value so only that zero-point degrades- # (decodes as +1). Formats in GPTQ_FORMAT_NO_ZP (plain qlinear_torch)- # read the nibble as the zero point directly, so it is stored as-is- # (symmetric packing stores the offset-binary constant the same way:- # 7 for the gptq family, 8 for the direct family).- if zp_minus_one:- zp = (zp - 1).clamp_(0, (1 << bits) - 1)- else:- zp = zp.clamp_(0, (1 << bits) - 1)- zp_packed = zp.reshape(num_groups, padded_out // pack_factor, pack_factor).to(torch.int64)- del zp- qzeros = (zp_packed << _shifts[None, None, :]).sum(dim=2).to(torch.int32)- del zp_packed, _shifts-- # Remove output padding from qweight / scales (qzeros stays in pack units)- if out_pad > 0:- qweight = qweight[:, :out_features]- scales_out = scales_out[:, :out_features]-- # Always return CPU tensors (safetensors requires CPU)- return qweight.cpu(), qzeros.cpu(), scales_out.cpu()--def _woq_quantize_missing_tensors(target_dir: str, missing_tensors_dict: dict) -> dict:"""Apply WOQ (Weight-Only Quantization) to missing Linear weight tensors.diff --git a/auto_round/utils/model_free_utils.py b/auto_round/utils/model_free_utils.pyindex c3d9bc3fe..6cf82a2da 100644--- a/auto_round/utils/model_free_utils.py+++ b/auto_round/utils/model_free_utils.py@@ -37,7 +37,6 @@ from auto_round.logger import loggerfrom auto_round.schemes import PRESET_SCHEMES, QuantizationScheme, preset_name_to_schemefrom auto_round.utils.common import to_standard_regexfrom auto_round.utils.device import clear_memory, compile_func-from auto_round.utils.missing_tensors import quantize_weight_rtn, split_fused_expert_tensors_NVFP4_E5M3_DATA_TYPE = "nvfp4_v2"_BLOCK_NAME_TO_IGNORE = ("shared_expert_gate.", ".gate.", "embed", "conv")@@ -63,6 +62,398 @@ SUPPORTED_PRESET_SCHEMES = ()+# ---------------------------------------------------------------------------+# Fused Expert Tensor Splitting (shared with auto_round.utils.missing_tensors)+# ---------------------------------------------------------------------------+# Maps a fused projection name to its constituent split names.+# When a 3D tensor ``*.experts.<fused_name>`` (shape [num_experts, ...])+# is encountered, it is split along dimension 0 of the per-expert 2-D+# slice to produce one tensor per split name per expert.+_FUSED_EXPERT_PROJ_PATTERNS: dict[str, list[str]] = {+ "gate_up_proj": ["gate_proj", "up_proj"],+ "w13": ["w1", "w3"],+}++# Native vLLM/compressed-tensors FusedMoE parameter naming encodes the+# projection *and* its companion scale directly in the tensor name, with no+# dot separator (e.g. ``experts.w13_weight``, ``experts.w2_weight_scale``,+# ``experts.w2_weight_scale_2``, ``experts.w13_input_scale``), unlike the+# dotted ``experts.<proj>.weight`` convention used elsewhere. Order matters+# only for readability here: none of these suffixes is itself a suffix of+# another, so matching is unambiguous.+_MOE_NATIVE_SUFFIXES: list[tuple[str, str]] = [+ ("_weight_scale_2", "weight_scale_2"),+ ("_weight_scale", "weight_scale"),+ ("_weight", "weight"),+ ("_input_scale", "input_scale"),+]++_AUTOROUND_ISSUE_URL = "https://github.com/intel/auto-round/issues"+_WARNING_INDEX_PLACEHOLDER = "<idx>"++# model_type values whose native inference-engine loader (e.g. vLLM's custom+# MoE weight loader) expects the *fused*, per-layer stacked 3-D expert+# tensors (``experts.w13_weight`` / ``experts.w2_weight``) to be preserved+# as-is (quantized in place) rather than unfused into per-expert 2-D+# tensors. Splitting these would break loading for such architectures.+_KEEP_FUSED_EXPERT_MODEL_TYPES: frozenset[str] = frozenset({"inkling_mm_model"})+++def _normalize_tensor_name_for_warning(name: str, numeric_replacement: str = _WARNING_INDEX_PLACEHOLDER) -> str:+ """Normalize tensor names for warning_once deduplication.++ Replace standalone numeric path segments (e.g. ``layers.12.experts.3``)+ and bracket indices (e.g. ``layers[12]``) with a fixed placeholder+ (``<idx>`` by default) so warning keys are stable across different+ layer/expert ids.+ """+ parts = name.split(".")+ normalized_parts = []+ for part in parts:+ if part.isdigit():+ normalized_parts.append(numeric_replacement)+ continue+ normalized_parts.append(re.sub(r"\[(\d+)\]", f"[{numeric_replacement}]", part))+ return ".".join(normalized_parts)+++def _parse_fused_proj_token(token: str) -> tuple[str, str]:+ """Split a raw expert-projection token into ``(base_proj, out_suffix)``.++ Handles both the plain dotted convention (token is just the projection+ name, e.g. ``"down_proj"`` / ``"gate_up_proj"`` -> ``out_suffix="weight"``)+ and the native FusedMoE convention where the companion tensor kind is+ encoded in the token itself (e.g. ``"w2_weight_scale"`` ->+ ``base_proj="w2"``, ``out_suffix="weight_scale"``).+ """+ for raw_suffix, out_suffix in _MOE_NATIVE_SUFFIXES:+ if token.endswith(raw_suffix) and len(token) > len(raw_suffix):+ return token[: -len(raw_suffix)], out_suffix+ return token, "weight"+++def split_fused_expert_tensors(+ tensors_dict: dict[str, torch.Tensor],+ model_type: str | None = None,+) -> dict[str, torch.Tensor]:+ """Split 3-D fused expert tensors into per-expert 2-D tensors.++ Many MoE checkpoints store expert weights as fused 3-D parameters+ under either ``*.experts.<proj_name>`` or ``*.moe.<proj_name>``+ (for example ``gate_up_proj`` with shape+ ``[num_experts, 2*intermediate, hidden]``, or ``down_proj`` with shape+ ``[num_experts, out, in]``). After unfusing, each expert gets its own+ 2-D weight tensor, e.g.+ ``experts.0.gate_proj.weight [intermediate, hidden]``.++ Splitting rules:++ * **gate_up_proj** ``[N, 2*inter, hidden]`` →+ ``experts.{i}.gate_proj.weight`` + ``experts.{i}.up_proj.weight``+ * **Other** stacked projections (e.g. ``down_proj``) ``[N, out, in]`` →+ ``experts.{i}.<proj>.weight``++ For ``*.moe.<proj_name>`` tensors, outputs use+ ``*.moe.experts.{i}.<proj>.weight`` so they align with the sequential+ expert module layout used by quantized MoE replacements.++ Native vLLM/compressed-tensors FusedMoE naming (e.g. ``experts.w13_weight``,+ ``experts.w2_weight_scale``, ``experts.w13_weight_scale_2``,+ ``experts.w2_input_scale``) is also recognized: the companion tensor kind+ is decoded from the token itself (see :func:`_parse_fused_proj_token`) so+ weight/weight_scale/weight_scale_2/input_scale tensors for the same+ projection are split consistently and end up under the same+ ``experts.{i}.<proj>`` prefix, instead of scale tensors being mistaken for+ independent weights.++ Non-3-D tensors are only split when recognized as one of these native+ scale companions; other non-3-D or non-expert tensors pass through+ unchanged.++ Some architectures (see ``_KEEP_FUSED_EXPERT_MODEL_TYPES``) ship a custom+ inference-engine loader that consumes the fused 3-D stacked tensors+ directly (quantizing/packing them in place without unfusing per expert).+ For those ``model_type`` values, this function is a no-op passthrough.++ Args:+ tensors_dict: Mapping of tensor names to tensors.+ model_type: Source model's ``model_type`` (from ``config.json``), used+ to skip splitting for architectures whose native loader expects+ fused stacked expert tensors.++ Returns:+ New dict with fused expert tensors replaced by per-expert 2-D tensors.+ """+ if (model_type or "").lower() in _KEEP_FUSED_EXPERT_MODEL_TYPES:+ return dict(tensors_dict)++ result: dict[str, torch.Tensor] = {}+ split_count = 0++ for tensor_name, tensor in tensors_dict.items():+ if tensor.dim() not in (1, 2, 3):+ result[tensor_name] = tensor+ continue++ warning_tensor_name = _normalize_tensor_name_for_warning(tensor_name)++ # Strip optional .weight suffix (dotted convention) for pattern matching+ stripped = tensor_name.removesuffix(".weight") # len(".weight") == 7++ # Expect: <prefix>.experts.<proj_name>+ dot_idx = stripped.rfind(".")+ if dot_idx < 0:+ result[tensor_name] = tensor+ continue++ parent = stripped[:dot_idx]+ raw_token = stripped[dot_idx + 1 :]+ proj_name, out_suffix = _parse_fused_proj_token(raw_token)++ parent_last = parent.split(".")[-1]+ is_experts_parent = parent.endswith("experts") or parent_last == "experts"+ is_moe_parent = parent.endswith("moe") or parent_last == "moe"++ # The immediate parent must be "experts" or "moe"+ if not is_experts_parent and not is_moe_parent:+ if tensor.dim() == 3:+ logger.warning_once(+ "Found 3-D tensor '%s' while splitting expert tensors; "+ "it will be kept unchanged. If this is an MoE/expert weight that should be split/quantized, "+ "please open an issue at %s.",+ warning_tensor_name,+ _AUTOROUND_ISSUE_URL,+ )+ result[tensor_name] = tensor+ continue++ # 1-D/2-D tensors under an "experts"/"moe" parent are only touched here+ # when they are a recognized native FusedMoE scale companion (e.g.+ # ``w13_weight_scale_2`` [num_experts, 2] or ``w2_weight_scale_2``+ # [num_experts]). Anything else (e.g. a plain 1-D bias/norm) passes+ # through unchanged.+ if tensor.dim() != 3 and out_suffix == "weight":+ result[tensor_name] = tensor+ continue++ target_prefix = parent if is_experts_parent else f"{parent}.experts"+ num_experts = tensor.shape[0]+ split_names = _FUSED_EXPERT_PROJ_PATTERNS.get(proj_name, [proj_name])++ if tensor.dim() == 3:+ if len(split_names) > 1:+ logger.warning_once(+ f"Splitting fused expert tensor '{warning_tensor_name}' "+ f"(shape={list(tensor.shape)}, num_experts={num_experts}) "+ f"into {split_names}"+ )+ else:+ logger.warning_once(+ f"Splitting stacked expert tensor '{warning_tensor_name}' "+ f"(shape={list(tensor.shape)}, num_experts={num_experts})"+ )+ for i in range(num_experts):+ expert_2d = tensor[i] # [fused_out, in_features] or [out, in]+ chunks = expert_2d.chunk(len(split_names), dim=0)+ for split_name, chunk in zip(split_names, chunks):+ out_key = f"{target_prefix}.{i}.{split_name}.{out_suffix}"+ result[out_key] = chunk.contiguous()+ else:+ # Native per-expert scale companion (e.g. weight_scale_2 /+ # input_scale), shape [num_experts] or [num_experts, len(split_names)].+ logger.warning_once(+ f"Splitting fused expert scale tensor '{warning_tensor_name}' "+ f"(shape={list(tensor.shape)}, num_experts={num_experts}) "+ f"into {split_names}"+ )+ flat = tensor.reshape(num_experts, -1)+ n_splits = len(split_names)+ for i in range(num_experts):+ row = flat[i]+ if row.numel() == n_splits:+ per_split_vals = [row[j].reshape(1) for j in range(n_splits)]+ else:+ # Single value shared across all splits (e.g. a scalar+ # global scale not broken out per gate/up projection).+ shared = row.reshape(-1)[:1]+ per_split_vals = [shared for _ in range(n_splits)]+ for split_name, val in zip(split_names, per_split_vals):+ out_key = f"{target_prefix}.{i}.{split_name}.{out_suffix}"+ result[out_key] = val.contiguous()++ split_count += 1++ return result+++def quantize_weight_rtn(+ weight: torch.Tensor,+ bits: int,+ group_size: int,+ sym: bool = True,+ device: Optional[torch.device] = None,+ disable_opt_rtn: bool = True,+ *,+ packing: str,+) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:+ """Quantize a 2-D weight tensor and pack into auto_gptq format.++ Parameters+ ----------+ weight : Tensor [out_features, in_features]+ bits : target bit-width (e.g. 4, 8)+ group_size : quantization group size along in_features+ sym : use symmetric quantisation+ device : compute device (cuda / cpu). Results are always returned on CPU.+ disable_opt_rtn : when False and sym=True, use the optimised-RTN scale+ search (``quant_tensor_opt_rtn_sym``) which evaluates three E8M0+ candidates per group and picks the best MSE. Defaults to True+ (plain RTN) to preserve backward-compatible behaviour.+ packing : the packing_format the artifact will declare (required); the+ zero-point convention follows the inference backend registry. GPTQ_FORMAT+ entries ("auto_round:auto_gptq", qlinear_torch_zp) store zp - 1 per+ nibble and unpack with +1; GPTQ_FORMAT_NO_ZP entries ("auto_round",+ "auto_round:gptqmodel", qlinear_torch) store the zero point directly.+ Any other value raises.++ Returns+ -------+ qweight : [in_features // pack_factor, out_features] int32+ qzeros : [num_groups, out_features // pack_factor] int32+ scales : [num_groups, out_features] float16+ """+ assert weight.dim() == 2, f"Expected 2-D weight, got {weight.dim()}-D"+ from auto_round.inference.backend import GPTQ_FORMAT, GPTQ_FORMAT_NO_ZP++ if packing in GPTQ_FORMAT:+ zp_minus_one = True+ elif packing in GPTQ_FORMAT_NO_ZP:+ zp_minus_one = False+ else:+ raise ValueError(+ f"unsupported packing format '{packing}': zero-point conventions are defined only for "+ f"{GPTQ_FORMAT} (zp - 1, unpacked with +1) and {GPTQ_FORMAT_NO_ZP} (zp stored directly)"+ )+ out_features, in_features = weight.shape+ if device is None:+ device = weight.device+ # Single-step transfer + cast avoids an intermediate BF16 copy on CUDA+ # (``weight.to(device).float()`` would briefly allocate both BF16 and+ # float32 buffers on the target device).+ weight = weight.to(device=device, dtype=torch.float32)++ # --- pad in_features to multiple of group_size ---+ if in_features % group_size != 0:+ pad = group_size - (in_features % group_size)+ weight = torch.nn.functional.pad(weight, (0, pad))+ in_features = weight.shape[1]++ num_groups = in_features // group_size+ pack_factor = 32 // bits # values per int32++ # --- pad out_features to multiple of pack_factor (needed for qzeros) ---+ out_pad = 0+ if out_features % pack_factor != 0:+ out_pad = pack_factor - (out_features % pack_factor)+ weight = torch.nn.functional.pad(weight, (0, 0, 0, out_pad))+ padded_out = weight.shape[0]++ from auto_round.data_type.utils import get_quant_func, reshape_pad_tensor_by_group_size++ # Use get_quant_func (same as WrapperLinear) so all data types and opt_rtn+ # variants are resolved via the QUANT_FUNC_WITH_DTYPE registry uniformly.+ quant_func, _ = get_quant_func("int", bits, sym=sym, disable_opt_rtn=disable_opt_rtn, iters=0)+ # quant_func returns (qdq_result, scale, zp_or_maxq)+ _, scale, zp_val = quant_func(weight, bits=bits, group_size=group_size)++ if sym:+ maxq = 1 << (bits - 1) # e.g. 8 for 4-bit+ zero_point = maxq # unsigned offset for packing++ # scale shape: [padded_out * num_groups, 1]+ # Reshape weight for group-wise quantization: [padded_out * num_groups, group_size]+ w_grouped, _, _ = reshape_pad_tensor_by_group_size(weight, group_size)+ w_grouped = w_grouped.to(device=device, dtype=torch.float32)+ del weight++ # Compute integer values for packing+ q = (w_grouped / scale).round_().clamp_(-maxq, maxq - 1)+ del w_grouped+ q += zero_point # shift to unsigned [0, 2*maxq - 1]+ q = q.to(torch.int32)++ # scale → [num_groups, padded_out] (float16)+ scales_out = scale.squeeze(-1).reshape(padded_out, num_groups).t().contiguous().to(torch.float16)+ del scale++ zp = torch.full((num_groups, padded_out), zero_point, dtype=torch.int32, device=device)+ else:+ # Asymmetric quantization+ max_int = (1 << bits) - 1++ # scale shape: [padded_out * num_groups, 1], zp_val shape: [padded_out * num_groups, 1]++ # Reshape weight for group-wise quantization+ w_grouped, _, _ = reshape_pad_tensor_by_group_size(weight, group_size)+ w_grouped = w_grouped.to(device=device, dtype=torch.float32)+ del weight++ # Compute integer values for packing+ q = (w_grouped / scale).round_()+ del w_grouped+ q += zp_val+ q.clamp_(0, max_int)+ q = q.to(torch.int32)++ # scale → [num_groups, padded_out] (float16)+ scales_out = scale.squeeze(-1).reshape(padded_out, num_groups).t().contiguous().to(torch.float16)+ del scale++ # zp → [num_groups, padded_out]+ zp = zp_val.squeeze(-1).reshape(padded_out, num_groups).t().contiguous().to(torch.int32)+ del zp_val++ # q → [in_features, padded_out]+ q = q.reshape(padded_out, in_features).t().contiguous()++ # ---- Pack qweight: [in_features // pack_factor, padded_out] ----+ # Vectorised: reshape → broadcast shift → int64 sum (≡ bitwise-OR for+ # non-overlapping bit lanes) avoids a Python loop per bit-lane.+ _shifts = torch.arange(pack_factor, dtype=torch.int64, device=device) * bits+ q_packed = q.reshape(in_features // pack_factor, pack_factor, padded_out).to(torch.int64)+ del q+ qweight = (q_packed << _shifts[None, :, None]).sum(dim=1).to(torch.int32)+ del q_packed++ # ---- Pack qzeros: [num_groups, padded_out // pack_factor] ----+ # The zero-point convention is a property of the declared packing format+ # (see the backend registry): the auto_round:auto_gptq format+ # (qlinear_torch_zp) adds +1 to zeros after unpacking, so we subtract 1+ # before packing to compensate. The (zp-1) packing cannot represent+ # zp=0: a -1 left-shifts into a negative word and corrupts every nibble+ # sharing it. Clamp the stored value so only that zero-point degrades+ # (decodes as +1). Formats in GPTQ_FORMAT_NO_ZP (plain qlinear_torch)+ # read the nibble as the zero point directly, so it is stored as-is+ # (symmetric packing stores the offset-binary constant the same way:+ # 7 for the gptq family, 8 for the direct family).+ if zp_minus_one:+ zp = (zp - 1).clamp_(0, (1 << bits) - 1)+ else:+ zp = zp.clamp_(0, (1 << bits) - 1)+ zp_packed = zp.reshape(num_groups, padded_out // pack_factor, pack_factor).to(torch.int64)+ del zp+ qzeros = (zp_packed << _shifts[None, None, :]).sum(dim=2).to(torch.int32)+ del zp_packed, _shifts++ # Remove output padding from qweight / scales (qzeros stays in pack units)+ if out_pad > 0:+ qweight = qweight[:, :out_features]+ scales_out = scales_out[:, :out_features]++ # Always return CPU tensors (safetensors requires CPU)+ return qweight.cpu(), qzeros.cpu(), scales_out.cpu()++# ---------------------------------------------------------------------------# Generic Utility Helpers# ---------------------------------------------------------------------------@@ -407,10 +798,22 @@ def _dequant_modelopt_nvfp4_tensors(def _handle_nvfp4_source_tensors(raw_tensors: dict[str, torch.Tensor],matcher: Any,+ device: str = "cpu",+ shard_name: str | None = None,) -> tuple[dict[str, torch.Tensor], dict[str, torch.Tensor], list[str]]:- """Passthrough NVFP4 source tensors when target scheme for the layer is NVFP4."""+ """Passthrough NVFP4 source tensors when target scheme for the layer is NVFP4.++ Layers whose target scheme is *not* NVFP4 (e.g. converting an NVFP4 donor+ checkpoint to MXFP4) are dequantized to bfloat16 via+ :func:`_dequant_nvfp4_tensors` instead of being left as raw+ ``weight_packed``/``weight_scale`` tensors — otherwise those companion+ scale tensors would fall through to the generic RTN/MXFP quantization path+ and be mistaken for ordinary weights, corrupting both the weight and its+ (unrelated) scale.+ """passthrough_tensors: dict[str, torch.Tensor] = {}passthrough_layers: list[str] = []+ dequant_layers: list[str] = []for name, tensor in list(raw_tensors.items()):if not name.endswith(".weight_packed") or tensor.dtype not in (torch.uint8, torch.int8):@@ -426,30 +829,105 @@ def _handle_nvfp4_source_tensors(continuescheme_bits = scheme.get("bits")scheme_data_type = (scheme.get("data_type") or "").lower()- if not (scheme_bits == 4 and (is_nv_fp(scheme_data_type) or scheme_data_type == _NVFP4_E5M3_DATA_TYPE)):- continue-- keys_to_move = [name, scale_key]- weight_global_scale_key = f"{layer_name}.weight_global_scale"- input_global_scale_key = f"{layer_name}.input_global_scale"- if weight_global_scale_key in raw_tensors:- keys_to_move.append(weight_global_scale_key)- if input_global_scale_key in raw_tensors:- keys_to_move.append(input_global_scale_key)-- for key in keys_to_move:- value = raw_tensors.pop(key).to("cpu")- if key.endswith("_global_scale"):- value = value.reshape([1])- passthrough_tensors[key] = value- passthrough_layers.append(layer_name)+ if scheme_bits == 4 and (is_nv_fp(scheme_data_type) or scheme_data_type == _NVFP4_E5M3_DATA_TYPE):+ keys_to_move = [name, scale_key]+ weight_global_scale_key = f"{layer_name}.weight_global_scale"+ input_global_scale_key = f"{layer_name}.input_global_scale"+ if weight_global_scale_key in raw_tensors:+ keys_to_move.append(weight_global_scale_key)+ if input_global_scale_key in raw_tensors:+ keys_to_move.append(input_global_scale_key)++ for key in keys_to_move:+ value = raw_tensors.pop(key).to("cpu")+ if key.endswith("_global_scale"):+ value = value.reshape([1])+ passthrough_tensors[key] = value+ passthrough_layers.append(layer_name)+ else:+ dequant_layers.append(layer_name)if passthrough_layers:logger.info(f"Handling NVFP4 source tensor(s): {len(passthrough_layers)} passthrough layer(s).")+ if dequant_layers:+ raw_tensors = _dequant_nvfp4_tensors(raw_tensors, dequant_layers, device=device, shard_name=shard_name)+return raw_tensors, passthrough_tensors, passthrough_layers+def _dequant_nvfp4_tensors(+ raw_tensors: dict[str, torch.Tensor],+ layer_names: list[str],+ device: str = "cpu",+ shard_name: str | None = None,+) -> dict[str, torch.Tensor]:+ """Dequantize NVFP4-packed weight tensors to bfloat16.++ Reverses the NVFP4 packing (see :func:`auto_round.data_type.nvfp.nv_fp4`):+ ``real_weight = unpack(weight_packed) * weight_scale.float() * (1 / weight_global_scale)``+ where ``weight_scale`` is the per-block (group_size=16) FP8 E4M3 scale and+ ``weight_global_scale`` is the per-tensor FP32 global scale (already+ reciprocal-normalized by :func:`_normalize_nvfp4_source_tensors`).++ The dequantized weight is written back under ``<layer>.weight`` (removing+ ``weight_packed``/``weight_scale``/``weight_global_scale``/+ ``input_global_scale``) so the downstream RTN/MXFP path can requantize it+ to the requested target scheme.+ """+ from auto_round.data_type.nvfp import get_reciprocal+ from auto_round.experimental.qmodules.fp4_utils import unpack_fp4_from_uint8++ group_size = 16+ dequant_device = str(device or "cpu")+ shard_prefix = f"[{shard_name}] " if shard_name else ""+ logger.info(f"{shard_prefix}Dequantizing {len(layer_names)} NVFP4 tensor(s) to bfloat16 on {dequant_device}.")++ for layer_name in layer_names:+ packed_key = f"{layer_name}.weight_packed"+ scale_key = f"{layer_name}.weight_scale"+ if packed_key not in raw_tensors or scale_key not in raw_tensors:+ continue+ packed = raw_tensors.pop(packed_key)+ packed = packed.contiguous().view(torch.uint8)+ block_scale = raw_tensors.pop(scale_key)+ global_scale = raw_tensors.pop(f"{layer_name}.weight_global_scale", None)+ # Input scale has no meaning once the weight is dequantized back to a+ # plain high-precision tensor; drop it.+ raw_tensors.pop(f"{layer_name}.input_global_scale", None)++ def _do_dequant(+ packed: torch.Tensor = packed,+ block_scale: torch.Tensor = block_scale,+ global_scale: torch.Tensor | None = global_scale,+ ) -> torch.Tensor:+ out_features, half_in_features = packed.shape+ in_features = half_in_features * 2+ unpacked = unpack_fp4_from_uint8(packed, out_features, in_features, dtype=torch.float32)+ n_groups = in_features // group_size+ scale_f = block_scale.to(torch.float32).reshape(out_features, n_groups, 1)+ dq = (unpacked.reshape(out_features, n_groups, group_size) * scale_f).reshape(out_features, in_features)+ if global_scale is not None:+ dq = dq * get_reciprocal(global_scale.to(torch.float32).reshape(()))+ return dq.to(torch.bfloat16)++ dq_weight = _dequantize_with_device_fallback(+ dequant_device=dequant_device,+ shard_prefix=shard_prefix,+ op_name="NVFP4 dequant",+ tensor_label=layer_name,+ on_device=lambda: _do_dequant(+ packed=packed.to(dequant_device, non_blocking=True),+ block_scale=block_scale.to(dequant_device, non_blocking=True),+ global_scale=(global_scale.to(dequant_device, non_blocking=True) if global_scale is not None else None),+ ).to("cpu"),+ on_cpu=_do_dequant,+ )+ raw_tensors[f"{layer_name}.weight"] = dq_weight++ return raw_tensors++def _is_out_of_memory_error(exc: Exception) -> bool:if isinstance(exc, torch.OutOfMemoryError):return True@@ -1650,10 +2128,13 @@ def _process_shard(output_tensors.update(passthrough_tensors)quantized_layers.extend(passthrough_layers)- # 3) NVFP4 passthrough for layers already stored in packed format.+ # 3) NVFP4 passthrough for layers already stored in packed format (or+ # dequant to bfloat16 when the target scheme requires re-quantization).raw_tensors, nvfp_passthrough_tensors, nvfp_passthrough_layers = _handle_nvfp4_source_tensors(raw_tensors,matcher,+ device=device,+ shard_name=shard_name,)output_tensors.update(nvfp_passthrough_tensors)quantized_layers.extend(nvfp_passthrough_layers)diff --git a/auto_round/utils/offload.py b/auto_round/utils/offload.pyindex 6ae7fc84a..fe8cdc9fb 100644--- a/auto_round/utils/offload.py+++ b/auto_round/utils/offload.py@@ -101,7 +101,7 @@ def _maybe_split_fused_expert_keys(state_dict: dict, module: torch.nn.Module) ->continue # original fused module still in the tree; assign as-isto_split[key] = state_dict.pop(key)if to_split:- from auto_round.utils.missing_tensors import split_fused_expert_tensors+ from auto_round.utils.model_free_utils import split_fused_expert_tensorsstate_dict.update(split_fused_expert_tensors(to_split))return state_dictdiff --git a/test/unit/common/utils/test_missing_tensors.py b/test/unit/common/utils/test_missing_tensors.pyindex 5bcaac255..a1397f27a 100644--- a/test/unit/common/utils/test_missing_tensors.py+++ b/test/unit/common/utils/test_missing_tensors.py@@ -200,6 +200,15 @@ class TestSplitFusedExpertTensors:assert result[key].shape == (O, I)assert torch.equal(result[key], stacked[i])+ def test_keep_fused_expert_model_type_passthrough(self):+ fused = torch.randn(2, 32, 16)+ tensors = {"model.layers.0.mlp.experts.w13_weight": fused}++ result = split_fused_expert_tensors(tensors, model_type="inkling_mm_model")++ assert set(result.keys()) == set(tensors.keys())+ assert torch.equal(result["model.layers.0.mlp.experts.w13_weight"], fused)+def test_mixed_fused_and_normal(self):N, I, H = 2, 32, 16tensors = {diff --git a/test/unit/test_cpu/export/test_qlinear_pack_clamp.py b/test/unit/test_cpu/export/test_qlinear_pack_clamp.pyindex 29aeda6e6..8d9e0819e 100644--- a/test/unit/test_cpu/export/test_qlinear_pack_clamp.py+++ b/test/unit/test_cpu/export/test_qlinear_pack_clamp.py@@ -194,7 +194,7 @@ def test_missing_tensors_gptq_pack_clamps_zero_zero_point():"""quantize_weight_rtn packs zp with the same (zp-1) convention; anall-positive weight yields zp=0 everywhere, which must clamp instead ofsmearing -1 across the packed zero-point words."""- from auto_round.utils.missing_tensors import quantize_weight_rtn+ from auto_round.utils.model_free_utils import quantize_weight_rtntorch.manual_seed(0)bits, group = 4, 32