Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

support MTP module WOQ quantization and step3p5 model quantization (#1526)

Xin He42fdb06903e1
diff --git a/auto_round/__main__.py b/auto_round/__main__.py
index 8ee9ed7f6..2254aafa8 100644
--- a/auto_round/__main__.py
+++ b/auto_round/__main__.py
@@ -12,7 +12,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import argparse
+import json
import os
+import re
import sys
from auto_round.auto_scheme import AutoScheme
@@ -24,6 +26,7 @@ from auto_round.utils import (
clear_memory,
get_device_and_parallelism,
get_model_dtype,
+ parse_layer_config_arg,
)
RECIPES = {
@@ -312,6 +315,15 @@ class BasicArgumentParser(argparse.ArgumentParser):
scheme.add_argument(
"--disable_act_dynamic", action="store_true", help="Use static instead of dynamic activation quantization. "
)
+ scheme.add_argument(
+ "--layer_config",
+ default=None,
+ type=str,
+ help="Per-layer quantization config for missing tensors (e.g., MTP layers) as a JSON string. "
+ "Keys are name prefixes, values are config dicts with optional bits/group_size/sym. "
+ 'Example: "{mtp:{bits:8,data_type:int},mtp.fc:{bits:16}}". '
+ "These settings are saved to extra_config and override the global quantization config.",
+ )
scheme.add_argument(
"--shared_layers",
type=str,
@@ -655,6 +667,9 @@ def tune(args):
extra_config.diffusion_config = diffusion_config
layer_config = {}
+ if args.layer_config:
+ layer_config = parse_layer_config_arg(args.layer_config)
+ args.layer_config = layer_config
if args.avg_bits is not None:
if args.options is None:
diff --git a/auto_round/compressors/shard_writer.py b/auto_round/compressors/shard_writer.py
index 990c5c387..b6dd329e3 100644
--- a/auto_round/compressors/shard_writer.py
+++ b/auto_round/compressors/shard_writer.py
@@ -55,6 +55,10 @@ class ShardWriter:
self.global_weight_map = {}
self.shard_counter = 0
+ # Persistent set of all parameter names already flushed to a shard file.
+ # Maintained incrementally in _flush_shard to avoid O(N^2) rebuilds in _add_tensor.
+ self._all_saved = set()
+
# Stats
self.total_param_elems = 0
self.total_param_size_bytes = 0
@@ -99,6 +103,11 @@ class ShardWriter:
if isinstance(tensor, torch.Tensor) and tensor.device.type == "meta":
self.skipped_meta_tensors.append(name)
return
+
+ # Guard against duplicate saving of the same parameter
+ if name in self._all_saved or name in self.current_shard_tensors:
+ return
+
t_size = tensor.nbytes
self.total_param_elems += tensor.numel()
self.total_param_size_bytes += t_size
@@ -135,6 +144,7 @@ class ShardWriter:
saved_params = list(self.current_shard_tensors.keys())
self.shard_meta.append({"tmp_file": tmp_name, "params": saved_params})
+ self._all_saved.update(saved_params)
# Offload logic: move modules to meta device once all params are saved
self._offload_to_meta(saved_params)
@@ -144,18 +154,15 @@ class ShardWriter:
def _offload_to_meta(self, saved_params):
"""Attempts to move fully saved modules to the 'meta' device to free RAM."""
- # Using a set for faster lookup of all saved parameters
- all_saved = {p for meta in self.shard_meta for p in meta["params"]}
-
for param_full_name in saved_params:
module_path = param_full_name.rsplit(".", 1)[0]
module = get_module(self.model, module_path)
- # Check if all parameters of this module are now in 'all_saved'
+ # Check if all parameters of this module are now in '_all_saved'
if (
module is not None
and isinstance(module, torch.nn.Module)
- and all(f"{module_path}.{k}" in all_saved for k in module.state_dict().keys())
+ and all(f"{module_path}.{k}" in self._all_saved for k in module.state_dict().keys())
):
module.to("meta")
@@ -164,11 +171,10 @@ class ShardWriter:
# 1. Capture remaining weights not yet saved
full_sd = self.model.state_dict()
tie_word_embeddings = getattr(getattr(self.model, "config", None), "tie_word_embeddings", True)
- all_saved_names = {p for meta in self.shard_meta for p in meta["params"]}
finalize_skipped_meta_tensors = []
for pname, tensor in full_sd.items():
- if pname in all_saved_names:
+ if pname in self._all_saved:
continue
if tensor.device.type == "meta":
continue
diff --git a/auto_round/compressors/utils.py b/auto_round/compressors/utils.py
index 73ed63206..1aef5bd7c 100644
--- a/auto_round/compressors/utils.py
+++ b/auto_round/compressors/utils.py
@@ -12,6 +12,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import copy
+import json
+import os
import random
import re
import sys
@@ -208,6 +210,63 @@ def infer_bits_by_data_type(data_type: str):
return None
+def _get_safetensor_layer_names_not_in_model(model, all_module_names: list) -> list:
+ """Collect layer names from safetensor files that are not loaded into the model.
+
+ Some tensors (e.g. MTP layers) exist in the original checkpoint but are not
+ instantiated by ``transformers``. This function discovers them so that regex
+ patterns in ``layer_config`` can still match them.
+
+ Returns:
+ List of layer names (the path without the ``.weight`` suffix) for weight
+ tensors present in the safetensor files but absent from *all_module_names*.
+ """
+ name_or_path = None
+ if hasattr(model, "config") and hasattr(model.config, "name_or_path"):
+ name_or_path = model.config.name_or_path
+ if not name_or_path:
+ return []
+
+ if not os.path.isdir(name_or_path):
+ try:
+ from auto_round.utils.model import download_hf_model
+
+ name_or_path = download_hf_model(name_or_path)
+ except Exception as e:
+ logger.debug(f"Could not resolve source model path to check for missing tensors: {e}")
+ return []
+
+ try:
+ from safetensors import safe_open
+ except ImportError:
+ return []
+
+ # Build tensor-name list from the safetensors index or single file
+ source_index_file = os.path.join(name_or_path, "model.safetensors.index.json")
+ source_single_file = os.path.join(name_or_path, "model.safetensors")
+
+ tensor_names: list = []
+ if os.path.exists(source_index_file):
+ with open(source_index_file) as f:
+ src_index = json.load(f)
+ tensor_names = list(src_index["weight_map"].keys())
+ elif os.path.exists(source_single_file):
+ with safe_open(source_single_file, framework="pt", device="cpu") as f:
+ tensor_names = list(f.keys())
+ else:
+ return []
+
+ module_name_set = set(all_module_names)
+ extra_layer_names = []
+ for tensor_name in tensor_names:
+ if not tensor_name.endswith(".weight"):
+ continue
+ layer_name = tensor_name[: -len(".weight")]
+ if layer_name not in module_name_set:
+ extra_layer_names.append(layer_name)
+ return extra_layer_names
+
+
def set_layer_config(
model: torch.nn.Module,
layer_config: dict[str, Union[str, dict, "QuantizationScheme"]],
@@ -235,6 +294,10 @@ def set_layer_config(
"""Assign scheme values as attributes to matched modules."""
for layer_name, scheme in layer_config.items():
module = get_module(model, layer_name)
+ if module is None:
+ # Layer exists in safetensor files but is not loaded into the model
+ # (e.g. MTP layers that transformers does not instantiate). Skip.
+ continue
for attr, value in scheme.items():
setattr(module, attr, value)
@@ -337,6 +400,10 @@ def set_layer_config(
if isinstance(m, embedding_types) or m.__class__.__name__.endswith("Embedding"):
embedding_layer_names.append(n)
+ # Also include layer names from safetensor files not loaded into the model
+ # (e.g. MTP layers that transformers does not instantiate).
+ safetensor_only_names = _get_safetensor_layer_names_not_in_model(model, all_module_names)
+
# 6. expand regex configs
regex_config = {}
for name in list(layer_config.keys()):
@@ -346,12 +413,14 @@ def set_layer_config(
m = get_module(model, name)
if len(list(m.children())) == 0 and type(m) not in supported_types:
layer_config.pop(name)
- logger.warning(f"{name} is not supported in current scheme, ignoring its setting in `layer_config`")
+ logger.debug(f"{name} is not supported in current scheme, ignoring its setting in `layer_config`")
continue
regex = re.compile(name)
matched = [ln for ln in all_supported_layer_names if regex.search(ln)]
- if not matched:
+ safetensor_only_matched = [ln for ln in safetensor_only_names if regex.search(ln)]
+ # skip it for mtp layers not loaded in transformers
+ if not matched and not safetensor_only_matched:
raise ValueError(f"Invalid '{name}' in layer_config, no match found.")
val = layer_config.pop(name)
regex_config[name] = val # keep regex config
@@ -898,6 +967,7 @@ def get_fp_layer_names(model: torch.nn.Module, ignore_layers: str):
for name in all_layer_names:
if fp_layer in name:
not_to_quantized_layers.append(name)
+ not_to_quantized_layers.extend(ignore_layers) # keep regex name for later use
logger.trace(f"not_to_quantized_layers: {not_to_quantized_layers}")
return not_to_quantized_layers
diff --git a/auto_round/export/utils.py b/auto_round/export/utils.py
index 42ae86d5f..df8ebd37a 100644
--- a/auto_round/export/utils.py
+++ b/auto_round/export/utils.py
@@ -16,7 +16,12 @@ import os
import torch.nn as nn
-from auto_round.utils import copy_python_files_from_model_cache, logger, unsupported_meta_device
+from auto_round.utils import (
+ copy_missing_tensors_from_source,
+ copy_python_files_from_model_cache,
+ logger,
+ unsupported_meta_device,
+)
def save_model(
@@ -57,6 +62,14 @@ def save_model(
else:
model.save_pretrained(save_dir, max_shard_size=max_shard_size, safe_serialization=safe_serialization)
+ try:
+ copy_missing_tensors_from_source(
+ source_dir=model.config._name_or_path,
+ target_dir=save_dir,
+ )
+ except Exception as e:
+ logger.warning("Skipping copy of missing tensors from source checkpoint due to error: %s", e)
+
config_path = os.path.join(save_dir, "config.json")
if dtype is not None and dtype != model.dtype and os.path.exists(os.path.join(save_dir, "config.json")):
with open(config_path, "r") as file:
diff --git a/auto_round/modeling/fused_moe/replace_modules.py b/auto_round/modeling/fused_moe/replace_modules.py
index 1566ccbc2..f18cc2151 100644
--- a/auto_round/modeling/fused_moe/replace_modules.py
+++ b/auto_round/modeling/fused_moe/replace_modules.py
@@ -37,6 +37,8 @@ BUILTIN_MODULES = {
# supports transformers >= 5.0.0
"qwen3_5_moe": LazyImport("auto_round.modeling.fused_moe.qwen3_5_moe"),
"qwen3_5_moe_text": LazyImport("auto_round.modeling.fused_moe.qwen3_5_moe"),
+ # Step 3.5 MoE: splits fused MoELinear into per-expert nn.Linear
+ "step3p5": LazyImport("auto_round.modeling.fused_moe.step3_5_moe"),
}
@@ -99,7 +101,7 @@ def is_custom_model(model: torch.nn.Module) -> bool:
def _log_first_moe_block(model: torch.nn.Module, label: str) -> None:
"""Log the first experts module found in the model for debugging."""
for name, module in model.named_modules():
- if name.endswith(".experts"):
+ if name.endswith(".experts") or name.endswith(".moe"):
logger.info(f"Experts ({label}) [{name}] ({module.__class__.__name__}):\n{module}")
return True
return False
diff --git a/auto_round/modeling/fused_moe/step3_5_moe.py b/auto_round/modeling/fused_moe/step3_5_moe.py
new file mode 100755
index 000000000..5f622f1e4
--- /dev/null
+++ b/auto_round/modeling/fused_moe/step3_5_moe.py
@@ -0,0 +1,168 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Note: adapted from: https://huggingface.co/stepfun-ai/Step-3.5-Flash/blob/main/modeling_step3p5.py
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from transformers.activations import ACT2FN
+
+from auto_round.modeling.fused_moe.replace_modules import ReplacementModuleBase
+from auto_round.modeling.fused_moe.utils import _update_parameter
+from auto_round.utils import clear_memory, unsupported_meta_device
+
+
+class Step3p5ExpertMLP(nn.Module):
+ """Single expert MLP with gate_proj, up_proj, down_proj as nn.Linear."""
+
+ def __init__(self, hidden_size, intermediate_size, limit=None):
+ super().__init__()
+ self.gate_proj = nn.Linear(hidden_size, intermediate_size, bias=False)
+ self.up_proj = nn.Linear(hidden_size, intermediate_size, bias=False)
+ self.down_proj = nn.Linear(intermediate_size, hidden_size, bias=False)
+ self.act_fn = ACT2FN["silu"]
+ self.limit = limit
+
+ def forward(self, x):
+ up = self.up_proj(x)
+ gate = self.act_fn(self.gate_proj(x))
+ if self.limit is not None:
+ gate = gate.clamp(min=None, max=self.limit)
+ up = up.clamp(min=-self.limit, max=self.limit)
+ return self.down_proj(gate * up)
+
+
+class SequentialStep3p5MoeExperts(torch.nn.ModuleList):
+ """Converts fused MoELinear weights into individual Step3p5ExpertMLP modules."""
+
+ def __init__(self, original):
+ super().__init__()
+ self.num_experts = original.up_proj.num_experts
+ hidden_size = original.up_proj.in_features
+ intermediate_size = original.up_proj.out_features
+ limit = original.limit
+
+ with torch.device("meta"):
+ super().__init__([Step3p5ExpertMLP(hidden_size, intermediate_size, limit) for _ in range(self.num_experts)])
+
+ def _materialize_weights(self, original) -> None:
+ """Split fused MoELinear weights into individual expert nn.Linear weights.
+
+ Args:
+ original: The original Step3p5MoEMLP module containing fused MoELinear
+ modules (gate_proj, up_proj, down_proj).
+ """
+ if not unsupported_meta_device(original.up_proj):
+ for i in range(self.num_experts):
+ _update_parameter(self[i].gate_proj, "weight", original.gate_proj.weight[i].contiguous())
+ _update_parameter(self[i].up_proj, "weight", original.up_proj.weight[i].contiguous())
+ _update_parameter(self[i].down_proj, "weight", original.down_proj.weight[i].contiguous())
+ original.gate_proj.to_empty(device="meta")
+ original.up_proj.to_empty(device="meta")
+ original.down_proj.to_empty(device="meta")
+ clear_memory()
+
+
+class LinearStep3p5MoEMLP(ReplacementModuleBase):
+ """Replacement for Step3p5MoEMLP that splits fused MoELinear into
+ individual nn.Linear per expert for quantization support."""
+
+ def __init__(self, original, config=None):
+ super().__init__(original)
+ self.num_experts = original.num_experts
+ self.top_k = original.top_k
+ self.hidden_size = original.hidden_size
+ self.moe_intermediate_size = original.moe_intermediate_size
+
+ self.gate = original.gate
+ self.need_fp32_gate = original.need_fp32_gate
+ self.routed_scaling_factor = original.routed_scaling_factor
+
+ # routing function
+ self.use_moe_router_bias = original.use_moe_router_bias
+ if self.use_moe_router_bias:
+ self.router_bias = original.router_bias
+ self.custom_routing_function = self.router_bias_func
+ else:
+ self.custom_routing_function = original.custom_routing_function
+
+ # split fused experts into sequential
+ with torch.device("meta"):
+ self.experts = SequentialStep3p5MoeExperts(original)
+
+ @classmethod
+ def original_module_class(cls) -> str:
+ """Return the class name of the module this replaces."""
+ return "Step3p5MoEMLP"
+
+ def _materialize_weights(self) -> None:
+ original = self._get_original_module()
+ self.experts._materialize_weights(original)
+ clear_memory()
+
+ def router_bias_func(self, gating_output: torch.Tensor, topk: int, renormalize: bool):
+ gate_prob = torch.sigmoid(gating_output.float())
+ gate_prob_with_bias = gate_prob + self.router_bias.unsqueeze(0)
+ _, indices = torch.topk(gate_prob_with_bias, k=topk, dim=1)
+ topk_prob = torch.gather(gate_prob, 1, indices)
+ expert_topk_weight = topk_prob
+ if renormalize:
+ expert_topk_weight = expert_topk_weight / (torch.sum(expert_topk_weight, dim=-1, keepdim=True) + 1e-20)
+ return expert_topk_weight, indices
+
+ def forward(self, hidden_states):
+ """Forward pass with sequential expert computation."""
+ batch_size, sequence_length, hidden_dim = hidden_states.shape
+ hidden_states = hidden_states.view(-1, hidden_dim)
+ if self.need_fp32_gate:
+ router_logits = torch.matmul(hidden_states.to(torch.float32), self.gate.weight.t().to(torch.float32))
+ else:
+ router_logits = self.gate(hidden_states)
+
+ if self.custom_routing_function:
+ routing_weights, selected_experts = self.custom_routing_function(
+ router_logits, self.top_k, renormalize=True
+ )
+ else:
+ routing_weights = F.softmax(router_logits, dim=1, dtype=torch.float)
+ routing_weights, selected_experts = torch.topk(routing_weights, self.top_k, dim=-1)
+
+ routing_weights = routing_weights * self.routed_scaling_factor
+
+ final_hidden_states = torch.zeros(
+ (batch_size * sequence_length, hidden_dim), dtype=hidden_states.dtype, device=hidden_states.device
+ )
+
+ expert_mask = torch.nn.functional.one_hot(selected_experts, num_classes=self.num_experts).permute(2, 1, 0)
+
+ for expert_idx in range(self.num_experts):
+ idx, top_x = torch.where(expert_mask[expert_idx])
+
+ current_state = hidden_states[None, top_x].reshape(-1, hidden_dim)
+ current_hidden_states = self.experts[expert_idx](current_state) * routing_weights[top_x, idx, None]
+
+ final_hidden_states.index_add_(0, top_x, current_hidden_states.to(hidden_states.dtype))
+ final_hidden_states = final_hidden_states.reshape(batch_size, sequence_length, hidden_dim)
+ return final_hidden_states
+
+ @classmethod
+ def from_original(
+ cls,
+ original,
+ config=None,
+ **kwargs,
+ ):
+ """Create an instance from the original module."""
+ return cls(original, config)
diff --git a/auto_round/special_model_handler.py b/auto_round/special_model_handler.py
index 9da31a1e3..fa0000bce 100644
--- a/auto_round/special_model_handler.py
+++ b/auto_round/special_model_handler.py
@@ -256,6 +256,21 @@ register_ignore_layers(
)
+# step3p5
+register_ignore_layers(
+ matchers=[
+ ModelTypeMatcher(r"step3p5", mode="full"),
+ ],
+ ignore_layers=[
+ "g_proj", # shape issue [96, 4096], 96 is not divisible by 64
+ "moe.gate",
+ "eh_proj", # MTP layer
+ "shared_head", # MTP layer
+ "layers.45", # MTP layer, requiring g_idx in vLLM, skip it
+ ],
+)
+
+
def get_predefined_ignore_layers(model: torch.nn.Module) -> list[str]:
layers = []
for rule in _PRE_DEFINED_IGNORE_LAYERS:
diff --git a/auto_round/utils/__init__.py b/auto_round/utils/__init__.py
index 32d976702..0d4d5325b 100644
--- a/auto_round/utils/__init__.py
+++ b/auto_round/utils/__init__.py
@@ -16,9 +16,11 @@ from auto_round.utils.device import *
from auto_round.utils.common import *
from auto_round.utils.model import *
from auto_round.utils.weight_handler import (
+ convert_module_to_hp_if_necessary,
detect_weight_type,
is_quantized_input_module,
)
+from auto_round.utils.missing_tensors import copy_missing_tensors_from_source
import transformers
from packaging.version import Version
diff --git a/auto_round/utils/common.py b/auto_round/utils/common.py
index a0dc89e8f..decd5540f 100644
--- a/auto_round/utils/common.py
+++ b/auto_round/utils/common.py
@@ -469,3 +469,42 @@ def is_transformers_version_greater_or_equal_4():
from packaging import version
return version.parse(transformers.__version__) >= version.parse("4.0.0")
+
+
+def parse_layer_config_arg(s: str) -> dict:
+ """Parse --layer_config with unquoted keys/values.
+
+ Delimiters are ``{``, ``}``, ``,``, ``:``. Each non-delimiter token is
+ auto-typed: numeric strings become ``int``, everything else stays ``str``.
+
+ Example::
+
+ {mtp:{bits:8},mtp.fc:{bits:16,data_type:fp}}
+ """
+ tokens = re.findall(r"[{}:,]|[^\s{}:,]+", s)
+ pos = [0]
+
+ def _val():
+ tok = tokens[pos[0]]
+ if tok == "{":
+ return _dict()
+ pos[0] += 1
+ try:
+ return int(tok)
+ except ValueError:
+ return tok
+
+ def _dict():
+ pos[0] += 1 # consume '{'
+ result = {}
+ while tokens[pos[0]] != "}":
+ key = tokens[pos[0]]
+ pos[0] += 1 # key
+ pos[0] += 1 # consume ':'
+ result[key] = _val()
+ if tokens[pos[0]] == ",":
+ pos[0] += 1 # consume ','
+ pos[0] += 1 # consume '}'
+ return result
+
+ return _dict()
diff --git a/auto_round/utils/missing_tensors.py b/auto_round/utils/missing_tensors.py
new file mode 100644
index 000000000..68c8f8fa3
--- /dev/null
+++ b/auto_round/utils/missing_tensors.py
@@ -0,0 +1,706 @@
+# Copyright (c) 2025 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Background
+# ----------
+# When ``transformers`` loads a model, it only instantiates the sub-modules
+# declared in the model class. Some architectures include auxiliary
+# parameters — most notably **MTP (Multi-Token Prediction)** layers — that are
+# stored in the original checkpoint but are *not* part of the default module
+# graph. As a result, ``model.save_pretrained`` silently omits those tensors
+# from the quantized output.
+#
+# This module provides :func:`copy_missing_tensors_from_source`, which detects
+# and copies such tensors from the original checkpoint into the saved output
+# directory. Detection works by comparing the **parent layer names**
+# (``tensor_name.rsplit(".", 1)[0]``) present in the source checkpoint against
+# those present in the saved output: any source tensor whose parent layer has
+# no tensors at all in the saved output (and is itself absent) is considered
+# "missing". This requires no model/config loading.
+#
+# Before writing, the function optionally:
+# - **dequantizes FP8 weights** to BF16 (when a matching ``weight_scale_inv``
+# tensor is present in the source checkpoint), and
+# - **re-quantizes with RTN** into the packed WOQ format used by the rest of
+# the saved model (when ``quant_method == "auto-round"`` and
+# ``packing_format == "auto_round:auto_gptq"`` are detected in
+# ``config.json``).
+
+import json
+import os
+from typing import Optional, Tuple
+
+import torch
+
+from auto_round.logger import logger
+from auto_round.utils.weight_handler import _dequant_fp8_linear_weight
+
+
+def copy_missing_tensors_from_source(
+ source_dir: str,
+ target_dir: str,
+) -> None:
+ """Copy tensors from the source checkpoint that are absent from the saved output.
+
+ Some parameters (e.g., MTP layers) are not loaded by ``transformers`` and
+ therefore not written by ``model.save_pretrained``. This function copies
+ those tensors from the original checkpoint into the quantized output dir.
+
+ Detection works by comparing **parent layer names**
+ (``tensor_name.rsplit(".", 1)[0]``) between the source checkpoint and the
+ saved output. A source tensor is considered "missing" when:
+ 1. its name is **not** already present in the saved output, **and**
+ 2. its parent layer name is **not** among the parent layer names of any
+ saved tensor (i.e. the layer was entirely skipped by
+ ``save_pretrained``).
+
+ No model or config loading is required.
+
+ FP8 handling:
+ Missing weight tensors in FP8 dtype that have a corresponding
+ ``weight_scale_inv`` tensor are dequantized to BF16 before saving.
+
+ WOQ quantization:
+ If the saved ``config.json`` contains a ``quantization_config`` with
+ ``quant_method == "auto-round"`` and
+ ``packing_format == "auto_round:auto_gptq"``, missing 2-D weight tensors
+ will be quantized with RTN and packed into qweight/qzeros/scales format.
+
+ Args:
+ source_dir: Path to the original (pre-quantization) checkpoint directory
+ or a HuggingFace repo id.
+ target_dir: Directory to which the quantized model was saved. Must
+ contain a ``config.json``.
+ """
+
+ try:
+ from safetensors import safe_open
+ from safetensors.torch import save_file
+ except ImportError:
+ logger.warning("safetensors not available, skipping copy of missing tensors from source checkpoint")
+ return
+
+ # ------------------------------------------------------------------ #
+ # Resolve source directory #
+ # ------------------------------------------------------------------ #
+ config_path = os.path.join(target_dir, "config.json")
+ if not os.path.exists(config_path):
+ return
+
+ if not os.path.isdir(source_dir):
+ try:
+ from auto_round.utils.model import download_hf_model
+
+ source_dir = download_hf_model(source_dir)
+ except Exception as e:
+ logger.debug(f"Could not resolve source model path to check for missing tensors: {e}")
+ return
+
+ if not source_dir or not os.path.isdir(source_dir):
+ return
+
+ # ------------------------------------------------------------------ #
+ # Build a mapping: tensor_name -> source shard file path #
+ # ------------------------------------------------------------------ #
+ source_index_file = os.path.join(source_dir, "model.safetensors.index.json")
+ source_single_file = os.path.join(source_dir, "model.safetensors")
+
+ source_tensor_to_file: dict = {}
+ if os.path.exists(source_index_file):
+ with open(source_index_file) as f:
+ src_index = json.load(f)
+ for tensor_name, shard_file in src_index["weight_map"].items():
+ source_tensor_to_file[tensor_name] = os.path.join(source_dir, shard_file)
+ elif os.path.exists(source_single_file):
+ with safe_open(source_single_file, framework="pt", device="cpu") as f:
+ for key in f.keys():
+ source_tensor_to_file[key] = source_single_file
+ else:
+ return
+
+ # ------------------------------------------------------------------ #
+ # Collect tensor names already present in the saved output #
+ # ------------------------------------------------------------------ #
+ saved_tensor_names: set = set()
+ saved_index_file = os.path.join(target_dir, "model.safetensors.index.json")
+ saved_single_file = os.path.join(target_dir, "model.safetensors")
+
+ if os.path.exists(saved_index_file):
+ with open(saved_index_file) as f:
+ saved_idx = json.load(f)
+ saved_tensor_names = set(saved_idx["weight_map"].keys())
+ elif os.path.exists(saved_single_file):
+ with safe_open(saved_single_file, framework="pt", device="cpu") as f:
+ saved_tensor_names = set(f.keys())
+ else:
+ return
+
+ # ------------------------------------------------------------------ #
+ # Identify missing tensors via block-prefix statistics #
+ # ------------------------------------------------------------------ #
+ # A source tensor is considered "missing" when:
+ # 1. its name is absent from the saved output, AND
+ # 2. its parent layer name is NOT among the parent layers of any saved
+ # tensor (catches layers whose suffix changed, e.g. .weight →
+ # .qweight after quantization), AND
+ # 3. its "block prefix" — the path up to and including the first
+ # numeric segment, e.g. ``model.layers.0`` — is NOT present in the
+ # set of block prefixes collected from the saved output.
+
+ saved_parent_layers: set = {name.rsplit(".", 1)[0] for name in saved_tensor_names}
+
+ def _first_numeric_prefix(tensor_name: str) -> str | None:
+ """Return the path up to and including the first numeric segment, or None.
+
+ e.g. ``model.layers.0.mlp.gate.qweight`` → ``model.layers.0``
+ ``model.embed_tokens.weight`` → ``None``
+ """
+ parts = tensor_name.split(".")
+ for i, p in enumerate(parts):
+ if p.isdigit():
+ return ".".join(parts[: i + 1])
+ return None
+
+ # Collect block prefixes from the saved output: path up to the first
+ # numeric segment in each saved tensor name (e.g. "model.layers.0",
+ # "mlp.experts.0").
+ saved_block_prefix: set = set()
+ for tensor_name in saved_tensor_names:
+ bp = _first_numeric_prefix(tensor_name)
+ if bp is not None:
+ saved_block_prefix.add(bp)
+
+ def _is_truly_missing(name: str) -> bool:
+ # For Qwen/Qwen3-0.6B-FP8, lm_head is tied but still in source_dir → not missing
+ if name == "lm_head.weight":
+ return False
+ if name in saved_tensor_names:
+ return False
+ parent = name.rsplit(".", 1)[0]
+ if parent in saved_parent_layers:
+ return False
+ src_block = _first_numeric_prefix(name)
+ if src_block is not None:
+ # Source tensor belongs to a numbered block; if that exact block
+ # appears in the saved output it was processed (possibly renamed
+ # or split) rather than omitted.
+ return src_block not in saved_block_prefix
+ return True
+
+ missing_tensor_names: list = [name for name in source_tensor_to_file if _is_truly_missing(name)]
+
+ if not missing_tensor_names:
+ return
+
+ logger.debug(f"Missing tensors detected: {missing_tensor_names}")
+
+ logger.info(
+ f"Found {len(missing_tensor_names)} tensor(s) in the source checkpoint that are "
+ f"absent from the saved output (e.g., MTP parameters). Copying them now..."
+ )
+
+ # ------------------------------------------------------------------ #
+ # Load missing tensors from source shards #
+ # ------------------------------------------------------------------ #
+ shard_to_missing: dict = {}
+ for tensor_name in missing_tensor_names:
+ shard_file = source_tensor_to_file[tensor_name]
+ shard_to_missing.setdefault(shard_file, []).append(tensor_name)
+
+ missing_tensors_dict: dict = {}
+ for shard_file, tensor_names in shard_to_missing.items():
+ with safe_open(shard_file, framework="pt", device="cpu") as f:
+ for tensor_name in tensor_names:
+ missing_tensors_dict[tensor_name] = f.get_tensor(tensor_name)
+
+ # ------------------------------------------------------------------ #
+ # FP8 dequantization: if a weight is FP8 and its scale_inv is present #
+ # ------------------------------------------------------------------ #
+
+ fp8_dtypes = {"torch.float8_e4m3fn", "torch.float8_e5m2", "torch.float8_e4m3fnuz", "torch.float8_e5m2fnuz"}
+
+ dequantized_keys: list = []
+ keys_to_remove: list = []
+ for tensor_name, tensor in list(missing_tensors_dict.items()):
+ if str(tensor.dtype) not in fp8_dtypes:
+ continue
+ if not tensor_name.endswith(".weight"):
+ continue
+
+ # Look for the corresponding scale_inv tensor
+ base = tensor_name[: -len(".weight")]
+ scale_inv_name = base + ".weight_scale_inv"
+ scale_name = base + ".weight_scale"
+
+ weight_scale = missing_tensors_dict.get(scale_inv_name)
+ if weight_scale is None:
+ weight_scale = missing_tensors_dict.get(scale_name)
+
+ if weight_scale is None:
+ continue
+
+ # Try to determine block_size from config or fall back to per-tensor
+ block_size = None
+ if weight_scale.dim() >= 2 and weight_scale.shape != tensor.shape:
+ block_m = tensor.shape[-2] // weight_scale.shape[-2] if weight_scale.shape[-2] > 0 else 1
+ block_n = tensor.shape[-1] // weight_scale.shape[-1] if weight_scale.shape[-1] > 0 else 1
+ if block_m > 1 or block_n > 1:
+ block_size = [block_m, block_n]
+
+ dq_weight = _dequant_fp8_linear_weight(tensor, weight_scale, block_size)
+ missing_tensors_dict[tensor_name] = dq_weight.to(torch.bfloat16)
+ dequantized_keys.append(tensor_name)
+
+ for s_name in [scale_inv_name, scale_name]:
+ if s_name in missing_tensors_dict:
+ keys_to_remove.append(s_name)
+
+ for k in keys_to_remove:
+ missing_tensors_dict.pop(k, None)
+
+ if dequantized_keys:
+ logger.info(
+ f"Dequantized {len(dequantized_keys)} FP8 weight(s) to BF16 before saving: "
+ f"{dequantized_keys[:5]}{'...' if len(dequantized_keys) > 5 else ''}"
+ )
+
+ if not missing_tensors_dict:
+ return
+
+ # ------------------------------------------------------------------ #
+ # WOQ quantization: quantize missing Linear weights if model is WOQ #
+ # ------------------------------------------------------------------ #
+ missing_tensors_dict = _woq_quantize_missing_tensors(target_dir, missing_tensors_dict)
+
+ if not missing_tensors_dict:
+ return
+
+ # ------------------------------------------------------------------ #
+ # Write missing tensors to a new shard and update the index #
+ # ------------------------------------------------------------------ #
+ is_sharded = os.path.exists(saved_index_file)
+
+ new_shard_name = "model_extra_tensors.safetensors"
+ new_shard_path = os.path.join(target_dir, new_shard_name)
+ missing_tensors_dict = {k: v.contiguous() for k, v in missing_tensors_dict.items()}
+ save_file(missing_tensors_dict, new_shard_path)
+
+ if is_sharded:
+ with open(saved_index_file) as f:
+ saved_index = json.load(f)
+ for tensor_name in missing_tensors_dict:
+ saved_index["weight_map"][tensor_name] = new_shard_name
+ with open(saved_index_file, "w") as f:
+ json.dump(saved_index, f, indent=2)
+ elif os.path.exists(saved_single_file):
+ weight_map: dict = {}
+ with safe_open(saved_single_file, framework="pt", device="cpu") as f:
+ for key in f.keys():
+ weight_map[key] = "model.safetensors"
+ for tensor_name in missing_tensors_dict:
+ weight_map[tensor_name] = new_shard_name
+ new_index = {"metadata": {"total_size": 0}, "weight_map": weight_map}
+ with open(saved_index_file, "w") as f:
+ json.dump(new_index, f, indent=2)
+
+ logger.info(
+ f"Successfully wrote {len(missing_tensors_dict)} missing tensor(s) to " f"'{new_shard_name}' in {target_dir}."
+ )
+
+
+def quantize_weight_rtn(
+ weight: torch.Tensor,
+ bits: int,
+ group_size: int,
+ sym: bool = True,
+ device: Optional[torch.device] = None,
+) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
+ """Quantize a 2-D weight tensor and pack into auto_gptq format.
+
+ Parameters
+ ----------
+ weight : Tensor [out_features, in_features]
+ bits : target bit-width (e.g. 4, 8)
+ group_size : quantization group size along in_features
+ sym : use symmetric quantisation
+ device : compute device (cuda / cpu). Results are always returned on CPU.
+
+ Returns
+ -------
+ qweight : [in_features // pack_factor, out_features] int32
+ qzeros : [num_groups, out_features // pack_factor] int32
+ scales : [num_groups, out_features] float16
+ """
+ assert weight.dim() == 2, f"Expected 2-D weight, got {weight.dim()}-D"
+ out_features, in_features = weight.shape
+ if device is None:
+ device = weight.device
+ weight = weight.to(device).float()
+
+ # --- pad in_features to multiple of group_size ---
+ if in_features % group_size != 0:
+ pad = group_size - (in_features % group_size)
+ weight = torch.nn.functional.pad(weight, (0, pad))
+ in_features = weight.shape[1]
+
+ num_groups = in_features // group_size
+ pack_factor = 32 // bits # values per int32
+
+ # --- pad out_features to multiple of pack_factor (needed for qzeros) ---
+ out_pad = 0
+ if out_features % pack_factor != 0:
+ out_pad = pack_factor - (out_features % pack_factor)
+ weight = torch.nn.functional.pad(weight, (0, 0, 0, out_pad))
+ padded_out = weight.shape[0]
+
+ # Reshape for group-wise ops: [padded_out, num_groups, group_size]
+ w_grouped = weight.reshape(padded_out, num_groups, group_size)
+
+ if sym:
+ half_range = (1 << (bits - 1)) - 1 # e.g. 7 for 4-bit
+ zero_point = 1 << (bits - 1) # e.g. 8 for 4-bit
+
+ w_absmax = w_grouped.abs().amax(dim=-1, keepdim=True).clamp(min=1e-10)
+ scales_full = w_absmax / half_range # [padded_out, num_groups, 1]
+
+ q = torch.round(w_grouped / scales_full).clamp(-half_range - 1, half_range)
+ q = (q + zero_point).to(torch.int32) # shift to unsigned [0, 2^bits-1]
+
+ zp = torch.full((num_groups, padded_out), zero_point, dtype=torch.int32, device=device)
+ else:
+ max_int = (1 << bits) - 1
+ w_min = w_grouped.amin(dim=-1, keepdim=True)
+ w_max = w_grouped.amax(dim=-1, keepdim=True)
+
+ scales_full = ((w_max - w_min).clamp(min=1e-10)) / max_int
+ zp_full = torch.round(-w_min / scales_full).clamp(0, max_int).to(torch.int32)
+
+ q = torch.round(w_grouped / scales_full + zp_full.float()).clamp(0, max_int).to(torch.int32)
+
+ zp = zp_full.squeeze(-1).t().contiguous() # [num_groups, padded_out]
+
+ # scales → [num_groups, padded_out] (float16)
+ scales_out = scales_full.squeeze(-1).t().contiguous().to(torch.float16)
+
+ # q → [in_features, padded_out]
+ q = q.reshape(padded_out, in_features).t().contiguous()
+
+ # ---- Pack qweight: [in_features // pack_factor, padded_out] ----
+ qweight = torch.zeros(in_features // pack_factor, padded_out, dtype=torch.int32, device=device)
+ for k in range(pack_factor):
+ qweight |= q[k::pack_factor, :] << (bits * k)
+
+ # ---- Pack qzeros: [num_groups, padded_out // pack_factor] ----
+ qzeros = torch.zeros(num_groups, padded_out // pack_factor, dtype=torch.int32, device=device)
+ for k in range(pack_factor):
+ qzeros |= zp[:, k::pack_factor] << (bits * k)
+
+ # Remove output padding from qweight / scales (qzeros stays in pack units)
+ if out_pad > 0:
+ qweight = qweight[:, :out_features]
+ scales_out = scales_out[:, :out_features]
+
+ # Always return CPU tensors (safetensors requires CPU)
+ return qweight.cpu(), qzeros.cpu(), scales_out.cpu()
+
+
+def _woq_quantize_missing_tensors(target_dir: str, missing_tensors_dict: dict) -> dict:
+ """Apply WOQ (Weight-Only Quantization) to missing Linear weight tensors.
+
+ Reads ``config.json`` from *target_dir* to obtain ``quantization_config``.
+ Only activates when ``quant_method == "auto-round"`` and
+ ``packing_format == "auto_round:auto_gptq"``.
+
+ Uses :func:`quantize_weight_rtn` for RTN quantisation + packing so that
+ there is **no dependency on the model object** or on QuantLinear classes.
+
+ Non-weight tensors (bias, norms, embeddings, etc.) are kept as-is.
+
+ Per-layer resolution:
+ For each weight, ``extra_config`` is checked first (exact layer name,
+ then regex pattern match), and the result is merged with global
+ defaults. This means entries like ``".*mtp.*": {"bits": 8}`` or
+ ``"mtp.fc": {"bits": 16, "data_type": "fp"}`` are honoured, while
+ layers absent from ``extra_config`` fall back to global ``bits`` /
+ ``group_size`` / ``sym``.
+
+ Args:
+ target_dir: Output directory that contains ``config.json``.
+ missing_tensors_dict: Dict mapping tensor names to tensor values.
+
+ Returns:
+ Updated dict with quantized+packed tensors replacing original weight tensors.
+ """
+ import re as _re
+
+ BLOCK_NAME_TO_IGNORE = ["shared_expert_gate.", "mlp.gate.", "g_proj."]
+ qconfig = _get_woq_config_from_dir(target_dir)
+ if qconfig is None:
+ return missing_tensors_dict
+
+ global_bits = qconfig["bits"]
+ global_group_size = qconfig["group_size"]
+ global_sym = qconfig["sym"]
+ block_name_to_quantize = qconfig.get("block_name_to_quantize", None)
+ extra_config: dict = qconfig.get("extra_config", {}) or {}
+
+ def _resolve_layer_cfg(layer_name: str) -> dict:
+ """Return effective {bits, group_size, sym, data_type} for *layer_name*.
+
+ Lookup order:
+ 1. Exact key match in extra_config.
+ 2. Among all regex keys that match layer_name, pick the longest pattern
+ (longer pattern == more specific, e.g. ``.*mtp\\.fc.*`` beats ``.*mtp.*``).
+ 3. Global defaults.
+ """
+ override: dict = {}
+ # 1. exact match
+ if layer_name in extra_config:
+ override = extra_config[layer_name]
+ else:
+ # 2. collect all matching regex patterns, keep the most specific (longest)
+ best_pattern: str | None = None
+ for pattern, cfg in extra_config.items():
+ if pattern == layer_name:
+ continue # already handled above
+ try:
+ if _re.search(pattern, layer_name):
+ if best_pattern is None or len(pattern) > len(best_pattern):
+ best_pattern = pattern
+ override = cfg
+ except _re.error:
+ pass # not a valid regex → skip
+ return {
+ "bits": override.get("bits", global_bits),
+ "group_size": override.get("group_size", global_group_size),
+ "sym": override.get("sym", global_sym),
+ "data_type": override.get("data_type", "int"),
+ }
+
+ def _is_fp_layer(layer_cfg: dict) -> bool:
+ """Return True when the resolved config indicates full-precision (no quantization)."""
+ dt = layer_cfg.get("data_type", "int")
+ return layer_cfg["bits"] >= 16 or dt in ("fp", "float", "float16", "bfloat16", "float32")
+
+ # Identify weight tensors eligible for quantization (2D Linear weights)
+ def _is_eligible(k: str) -> bool:
+ if not k.endswith(".weight"):
+ return False
+ if missing_tensors_dict[k].dim() != 2:
+ return False
+ layer_name = k[: -len(".weight")]
+ layer_cfg = _resolve_layer_cfg(layer_name)
+ # If extra_config explicitly covers this layer, trust its decision
+ if layer_name in extra_config or any(
+ _re.search(p, layer_name) for p in extra_config if p != layer_name and _is_valid_regex(p)
+ ):
+ return not _is_fp_layer(layer_cfg)
+ # Fall back to BLOCK_NAME_TO_IGNORE heuristic
+ if any(block in k for block in BLOCK_NAME_TO_IGNORE):
+ return False
+ return not _is_fp_layer(layer_cfg)
+
+ weight_keys = [k for k in missing_tensors_dict if _is_eligible(k)]
+
+ # Collect 2-D weight tensors that will NOT be quantized (for extra_config bookkeeping).
+ ignored_weight_keys = [
+ k
+ for k in missing_tensors_dict
+ if k.endswith(".weight") and missing_tensors_dict[k].dim() == 2 and k not in weight_keys
+ ]
+
+ # Update quantization config: extra_config for ignored layers + block_name_to_quantize for new blocks
+ if ignored_weight_keys or weight_keys:
+
+ def _update_qcfg(qcfg):
+ if ignored_weight_keys:
+ ec = qcfg.get("extra_config", {})
+ for k in ignored_weight_keys:
+ layer_name = k[: -len(".weight")]
+ ec.setdefault(layer_name, {"bits": 16, "data_type": "fp"})
+ qcfg["extra_config"] = ec
+ logger.info(
+ f"Updated extra_config for {len(ignored_weight_keys)} ignored layer(s): "
+ f"{[k[: -len('.weight')] for k in ignored_weight_keys]}"
+ )
+ if weight_keys:
+ existing = qcfg.get("block_name_to_quantize") or []
+ if isinstance(existing, str):
+ existing = [b.strip() for b in existing.split(",") if b.strip()]
+ existing_set = set(existing)
+ new_prefixes = {
+ ".".join(k.split(".")[:i])
+ for k in weight_keys
+ for i, p in enumerate(k.split("."))
+ if p.isdigit() and ".".join(k.split(".")[:i])
+ }
+ added = sorted(new_prefixes - existing_set)
+ if added:
+ merged = existing + added
+ merged = [
+ b for b in merged if not any(b != other and b.startswith(other + ".") for other in merged)
+ ]
+ qcfg["block_name_to_quantize"] = merged
+ logger.info(f"Updated block_name_to_quantize: {merged}")
+ return qcfg
+
+ for cfg_file in ["config.json", "quantization_config.json"]:
+ cfg_path = os.path.join(target_dir, cfg_file)
+ if not os.path.exists(cfg_path):
+ continue
+ logger.info(f"Processing {cfg_file} to update quantization_config for missing tensors...")
+ try:
+ with open(cfg_path) as f:
+ cfg_data = json.load(f)
+ if cfg_file == "config.json":
+ cfg_data["quantization_config"] = _update_qcfg(cfg_data.get("quantization_config", {}))
+ else:
+ cfg_data = _update_qcfg(cfg_data)
+ with open(cfg_path, "w") as f:
+ json.dump(cfg_data, f, indent=2)
+ except Exception as e:
+ logger.warning(f"Failed to update {cfg_file}: {e}")
+
+ if not weight_keys:
+ return missing_tensors_dict
+
+ logger.info(
+ f"Applying WOQ to "
+ f"{len(weight_keys)} missing Linear weight(s) (per-layer overrides from extra_config applied)..."
+ )
+
+ new_tensors: dict = {}
+ packed_weight_keys: set = set()
+
+ for weight_key in weight_keys:
+ weight = missing_tensors_dict[weight_key]
+ out_features, in_features = weight.shape
+
+ layer_name = weight_key[: -len(".weight")]
+ layer_cfg = _resolve_layer_cfg(layer_name)
+ bits = layer_cfg["bits"]
+ group_size = layer_cfg["group_size"]
+ sym = layer_cfg["sym"]
+
+ effective_gs = group_size if group_size != -1 else in_features
+ if effective_gs <= 0:
+ continue
+
+ logger.debug(
+ f"WOQ [{layer_name}]: bits={bits}, group_size={effective_gs}, sym={sym}, " f"shape={list(weight.shape)}"
+ )
+
+ base_name = layer_name
+
+ try:
+ qweight, qzeros, scales = quantize_weight_rtn(weight, bits=bits, group_size=effective_gs, sym=sym)
+ except Exception as e:
+ logger.warning(f"Failed to quantize {weight_key}: {e}, keeping original weight")
+ continue
+
+ new_tensors[base_name + ".qweight"] = qweight
+ new_tensors[base_name + ".qzeros"] = qzeros
+ new_tensors[base_name + ".scales"] = scales
+
+ packed_weight_keys.add(weight_key)
+
+ if not packed_weight_keys:
+ return missing_tensors_dict
+
+ # Replace original weights with packed versions
+ result = {}
+ for k, v in missing_tensors_dict.items():
+ if k not in packed_weight_keys:
+ result[k] = v
+ result.update(new_tensors)
+
+ logger.info(
+ f"Successfully packed {len(packed_weight_keys)} weight(s) into WOQ format "
+ f"({len(new_tensors)} packed tensor(s) created)."
+ )
+ return result
+
+
+def _is_valid_regex(pattern: str) -> bool:
+ """Return True if *pattern* is a valid regular expression."""
+ import re as _re
+
+ try:
+ _re.compile(pattern)
+ return True
+ except _re.error:
+ return False
+
+
+def _get_woq_config_from_dir(target_dir: str) -> dict | None:
+ """Read ``quantization_config`` from ``config.json`` in *target_dir*.
+
+ Only supports ``quant_method == "auto-round"`` with
+ ``packing_format == "auto_round:auto_gptq"``. Returns ``None`` (skip
+ quantization) for any other combination or if any required parameter
+ (``bits``, ``group_size``, ``sym``) is missing.
+
+ Returns a dict with keys ``bits``, ``group_size``, ``sym``
+ or ``None``.
+ """
+ config_path = os.path.join(target_dir, "config.json")
+ if not os.path.exists(config_path):
+ return None
+
+ with open(config_path) as f:
+ config_data = json.load(f)
+
+ qcfg = config_data.get("quantization_config", None)
+ if not isinstance(qcfg, dict):
+ return None
+
+ quant_method = qcfg.get("quant_method", "")
+ packing_format = qcfg.get("packing_format", "")
+ bits = qcfg.get("bits", None)
+ group_size = qcfg.get("group_size", None)
+ sym = qcfg.get("sym", None)
+
+ if quant_method != "auto-round":
+ logger.debug(
+ f"Skipping WOQ quantization of missing tensors: "
+ f"quant_method='{quant_method}' (only 'auto-round' is supported)"
+ )
+ return None
+
+ if packing_format != "auto_round:auto_gptq":
+ logger.debug(
+ f"Skipping WOQ quantization of missing tensors: "
+ f"packing_format='{packing_format}' (only 'auto_round:auto_gptq' is supported)"
+ )
+ return None
+
+ if bits is None or group_size is None or sym is None:
+ logger.debug(
+ f"Skipping WOQ quantization of missing tensors: "
+ f"incomplete quantization_config (bits={bits}, group_size={group_size}, sym={sym})"
+ )
+ return None
+
+ if bits > 8:
+ return None
+
+ block_name_to_quantize = qcfg.get("block_name_to_quantize", None)
+ extra_config = qcfg.get("extra_config", {})
+ return {
+ "bits": bits,
+ "group_size": group_size,
+ "sym": sym,
+ "block_name_to_quantize": block_name_to_quantize,
+ "extra_config": extra_config,
+ }
diff --git a/auto_round/utils/model.py b/auto_round/utils/model.py
index 91f2b7e89..0e4a406fb 100644
--- a/auto_round/utils/model.py
+++ b/auto_round/utils/model.py
@@ -257,8 +257,6 @@ def _is_mxfp4_model(model_path, trust_remote_code=True):
Only checks when transformers >= 5.0.0. Returns False immediately for older versions,
adding zero overhead to non-MXFP4 model loading.
"""
- if version.parse(transformers.__version__) < version.parse("5.0.0"):
- return False
from transformers import AutoConfig
try: # in case of config loading failure for new models
@@ -318,17 +316,19 @@ def llm_load_model(
if device_str is not None and "hpu" in device_str:
torch_dtype = torch.bfloat16
- is_mxfp4 = _is_mxfp4_model(pretrained_model_name_or_path, trust_remote_code=trust_remote_code)
load_kwargs = {
"torch_dtype": torch_dtype,
"trust_remote_code": trust_remote_code,
"device_map": "auto" if use_auto_mapping else None,
}
- if is_mxfp4:
- from transformers import Mxfp4Config
- load_kwargs["quantization_config"] = Mxfp4Config(dequantized=True)
- logger.info("Detected MXFP4 quantized model, using Mxfp4Config(dequantized=True) for loading.")
+ if version.parse(transformers.__version__) >= version.parse("5.0.0"):
+ is_mxfp4 = _is_mxfp4_model(pretrained_model_name_or_path, trust_remote_code=trust_remote_code)
+ if is_mxfp4:
+ from transformers import Mxfp4Config
+
+ load_kwargs["quantization_config"] = Mxfp4Config(dequantized=True)
+ logger.info("Detected MXFP4 quantized model, using Mxfp4Config(dequantized=True) for loading.")
is_glm = bool(re.search("chatglm", pretrained_model_name_or_path.lower()))
diff --git a/test/test_cpu/export/test_export.py b/test/test_cpu/export/test_export.py
index f0535ddd3..a2212fbc6 100644
--- a/test/test_cpu/export/test_export.py
+++ b/test/test_cpu/export/test_export.py
@@ -323,13 +323,9 @@ class TestAutoRound:
shutil.rmtree(quantized_model_path, ignore_errors=True)
- @pytest.mark.skipif(
- transformers_version >= version.parse("5.0"),
- reason="PhiConfig missing pad_token_id, https://github.com/huggingface/transformers/pull/43453",
- )
def test_awq_lmhead_export(self, dataloader):
bits, sym, group_size = 4, False, 128
- model_name = get_model_path("microsoft/phi-2")
+ model_name = get_model_path("microsoft/phi-4")
layer_config = {
"lm_head": {"bits": 4}, # set lm_head quant
"layer": {"bits": 16},
@@ -353,14 +349,10 @@ class TestAutoRound:
assert isinstance(lm_head, WQLinear_GEMM), "Illegal AWQ quantization for lm_head layer"
shutil.rmtree(quantized_model_path, ignore_errors=True)
- @pytest.mark.skipif(
- transformers_version >= version.parse("5.0"),
- reason="PhiConfig missing pad_token_id, https://github.com/huggingface/transformers/pull/43453",
- )
def test_gptq_lmhead_export(self, dataloader):
bits, sym, group_size = 4, True, 128
# Note that, to save UT tuning time, the local model is intentionally kept lightweight, using only 2 hidden layers.
- model_name = get_model_path("microsoft/phi-2")
+ model_name = get_model_path("microsoft/phi-4")
layer_config = {
"lm_head": {"bits": 4}, # set lm_head quant
"layer": {"bits": 16},
diff --git a/test/test_cpu/models/test_moe_model.py b/test/test_cpu/models/test_moe_model.py
index ffadaf4f9..66d20da91 100644
--- a/test/test_cpu/models/test_moe_model.py
+++ b/test/test_cpu/models/test_moe_model.py
@@ -1,6 +1,8 @@
import shutil
import pytest
+import torch
+import torch.nn as nn
from packaging import version
from transformers import AutoConfig, AutoProcessor, AutoTokenizer, Llama4ForConditionalGeneration
from transformers.models.gpt_oss.modeling_gpt_oss import GptOssForCausalLM
@@ -62,7 +64,7 @@ def setup_qwen3_vl_moe():
return model, tokenizer, processor, output_dir, config
-def quantize_model(model, tokenizer, output_dir, scheme, iters=0):
+def quantize_model(model, tokenizer, output_dir, scheme, iters=0, ignore_layers="self_attn,router,lm_head,mlp.gate"):
"""Helper function to quantize the model with the given scheme."""
autoround = AutoRound(
model,
@@ -71,7 +73,7 @@ def quantize_model(model, tokenizer, output_dir, scheme, iters=0):
nsamples=2,
iters=iters,
seqlen=32,
- ignore_layers="self_attn,router,lm_head,mlp.gate",
+ ignore_layers=ignore_layers,
)
quantized_model, save_folder = autoround.quantize_and_save(format="auto_round", output_dir=output_dir)
return quantized_model
@@ -98,7 +100,7 @@ def test_gptoss(setup_gpt_oss, scheme):
# Remove it to avoid mismatch during quantized model loading
delattr(model.config, "layer_types")
- quantized_model = quantize_model(model, tokenizer, output_dir, scheme)
+ quantized_model = quantize_model(model, tokenizer, output_dir, scheme, ignore_layers="self_attn,lm_head")
# Ensure the quantized model is not None
assert quantized_model is not None, "Quantized model should not be None."
@@ -136,7 +138,7 @@ def test_llama4(setup_llama4):
delattr(model.config.text_config, "moe_layers")
delattr(model.config.text_config, "layer_types")
- quantized_model = quantize_model(model, tokenizer, output_dir, "MXFP4")
+ quantized_model = quantize_model(model, tokenizer, output_dir, "MXFP4", ignore_layers="self_attn,router,lm_head")
# Ensure the quantized model is not None
assert quantized_model is not None, "Quantized model should not be None."
@@ -177,3 +179,64 @@ def test_qwen3_vl_moe_mxfp(setup_qwen3_vl_moe):
print(tokenizer.decode(loaded_model.generate(**inputs, max_new_tokens=50)[0]))
# clean the output directory after test
shutil.rmtree(output_dir, ignore_errors=True)
+
+
+class _FakeMoELinear(nn.Module):
+ """Mimics the fused MoELinear used in Step3p5MoEMLP."""
+
+ def __init__(self, num_experts, in_features, out_features):
+ super().__init__()
+ self.num_experts = num_experts
+ self.in_features = in_features
+ self.out_features = out_features
+ self.weight = nn.Parameter(torch.randn(num_experts, out_features, in_features))
+
+
+class Step3p5MoEMLP(nn.Module):
+ """Mock of the original Step3p5MoEMLP matched by class name."""
+
+ def __init__(self, hidden_size=32, intermediate_size=64, num_experts=4, top_k=2):
+ super().__init__()
+ self.hidden_size = hidden_size
+ self.moe_intermediate_size = intermediate_size
+ self.num_experts = num_experts
+ self.top_k = top_k
+ self.limit = None
+ self.need_fp32_gate = False
+ self.routed_scaling_factor = 1.0
+ self.use_moe_router_bias = False
+ self.custom_routing_function = None
+ self.gate = nn.Linear(hidden_size, num_experts, bias=False)
+ self.gate_proj = _FakeMoELinear(num_experts, hidden_size, intermediate_size)
+ self.up_proj = _FakeMoELinear(num_experts, hidden_size, intermediate_size)
+ self.down_proj = _FakeMoELinear(num_experts, intermediate_size, hidden_size)
+
+
+def test_step3p5_moe_replacement():
+ """Verify that apply_replacements swaps Step3p5MoEMLP → LinearStep3p5MoEMLP."""
+ from auto_round.modeling.fused_moe.replace_modules import apply_replacements, materialize_model_
+ from auto_round.modeling.fused_moe.step3_5_moe import LinearStep3p5MoEMLP, Step3p5ExpertMLP
+
+ num_experts, hidden_size = 4, 32
+ moe = Step3p5MoEMLP(hidden_size=hidden_size, num_experts=num_experts)
+ # wrap in a model with config.model_type so the registry can find it
+ model = nn.Module()
+ model.config = type("Cfg", (), {"model_type": "step3p5"})()
+ model.moe = moe
+ model.add_module("moe", moe)
+
+ orig_weights = {k: moe.get_parameter(k).clone() for k in ("gate_proj.weight", "up_proj.weight", "down_proj.weight")}
+
+ apply_replacements(model, auto_detect_moe=False)
+ assert isinstance(model.moe, LinearStep3p5MoEMLP)
+ assert count_modules_by_type(model, Step3p5ExpertMLP) == num_experts
+
+ materialize_model_(model)
+ for i, expert in enumerate(model.moe.experts):
+ for proj in ("gate_proj", "up_proj", "down_proj"):
+ w = getattr(expert, proj).weight
+ assert w.device.type != "meta"
+ assert torch.equal(w, orig_weights[f"{proj}.weight"][i])
+
+ out = model.moe(torch.randn(1, 4, hidden_size))
+ assert out.shape == (1, 4, hidden_size)
diff --git a/test/test_cpu/utils/test_cli_usage.py b/test/test_cpu/utils/test_cli_usage.py
index f57e9c102..e444524ae 100644
--- a/test/test_cpu/utils/test_cli_usage.py
+++ b/test/test_cpu/utils/test_cli_usage.py
@@ -2,6 +2,8 @@ import os
import shutil
import sys
+from auto_round.utils import parse_layer_config_arg
+
from ...helpers import get_model_path
AUTO_ROUND_PATH = __file__.split("/")
@@ -71,3 +73,24 @@ class TestAutoRoundCmd:
)
if res > 0 or res == -1:
assert False, "cmd line test fail, please have a check"
+
+ def test_layer_config(self, tiny_opt_model_path):
+ """Test --layer_config with unquoted JSON-like syntax."""
+ python_path = sys.executable
+ layer_cfg = r"{fc1:{bits:8,data_type:int},fc2:{bits:16,data_type:int}}"
+ res = os.system(
+ f"PYTHONPATH='{AUTO_ROUND_PATH}:$PYTHONPATH' {python_path} -m auto_round"
+ f" --model {tiny_opt_model_path} --seqlen 8 --iter 0 --disable_opt_rtn"
+ f" --layer_config '{layer_cfg}' --format auto_round --output_dir ./saved"
+ )
+ if res > 0 or res == -1:
+ assert False, "cmd line test with --layer_config fail, please have a check"
+
+
+def test_parse_layer_config():
+ """Unit test for parse_layer_config_arg parsing logic."""
+ result = parse_layer_config_arg("{mtp:{bits:8,data_type:int},mtp.fc:{bits:16,data_type:int}}")
+ assert result == {
+ "mtp": {"bits": 8, "data_type": "int"},
+ "mtp.fc": {"bits": 16, "data_type": "int"},
+ }
diff --git a/test/test_cpu/utils/test_missing_tensors.py b/test/test_cpu/utils/test_missing_tensors.py
new file mode 100644
index 000000000..1461533db
--- /dev/null
+++ b/test/test_cpu/utils/test_missing_tensors.py
@@ -0,0 +1,604 @@
+# Copyright (c) 2025 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import os
+import tempfile
+import unittest
+
+import torch
+
+from auto_round.utils.missing_tensors import (
+ _get_woq_config_from_dir,
+ copy_missing_tensors_from_source,
+ quantize_weight_rtn,
+)
+
+
+def _save_safetensors(tensors: dict, path: str) -> None:
+ from safetensors.torch import save_file
+
+ save_file(tensors, path)
+
+
+def _load_safetensors(path: str) -> dict:
+ from safetensors import safe_open
+
+ result = {}
+ with safe_open(path, framework="pt", device="cpu") as f:
+ for key in f.keys():
+ result[key] = f.get_tensor(key)
+ return result
+
+
+def _write_config(target_dir: str, qcfg: dict | None = None) -> None:
+ config = {}
+ if qcfg is not None:
+ config["quantization_config"] = qcfg
+ with open(os.path.join(target_dir, "config.json"), "w") as f:
+ json.dump(config, f)
+
+
+class TestQuantizeWeightRtn(unittest.TestCase):
+ """Tests for the standalone quantize_weight_rtn helper."""
+
+ def test_output_shapes_4bit_sym(self):
+ """Check that output tensors have the expected shapes for 4-bit symmetric quantization."""
+ out_features, in_features = 32, 64
+ bits, group_size = 4, 32
+ weight = torch.randn(out_features, in_features)
+
+ pack_factor = 32 // bits # 8
+ num_groups = in_features // group_size # 2
+
+ qweight, qzeros, scales = quantize_weight_rtn(weight, bits=bits, group_size=group_size, sym=True)
+
+ self.assertEqual(qweight.shape, (in_features // pack_factor, out_features))
+ self.assertEqual(qzeros.shape, (num_groups, out_features // pack_factor))
+ self.assertEqual(scales.shape, (num_groups, out_features))
+ self.assertEqual(qweight.dtype, torch.int32)
+ self.assertEqual(qzeros.dtype, torch.int32)
+ self.assertEqual(scales.dtype, torch.float16)
+
+ def test_output_shapes_4bit_asym(self):
+ """Check output shapes for 4-bit asymmetric quantization."""
+ out_features, in_features = 32, 64
+ bits, group_size = 4, 32
+ weight = torch.randn(out_features, in_features)
+
+ pack_factor = 32 // bits
+ num_groups = in_features // group_size
+
+ qweight, qzeros, scales = quantize_weight_rtn(weight, bits=bits, group_size=group_size, sym=False)
+
+ self.assertEqual(qweight.shape, (in_features // pack_factor, out_features))
+ self.assertEqual(qzeros.shape, (num_groups, out_features // pack_factor))
+ self.assertEqual(scales.shape, (num_groups, out_features))
+
+ def test_output_shapes_8bit(self):
+ """Check output shapes for 8-bit symmetric quantization."""
+ out_features, in_features = 16, 128
+ bits, group_size = 8, 128
+ weight = torch.randn(out_features, in_features)
+
+ pack_factor = 32 // bits # 4
+ num_groups = in_features // group_size # 1
+
+ qweight, qzeros, scales = quantize_weight_rtn(weight, bits=bits, group_size=group_size, sym=True)
+
+ self.assertEqual(qweight.shape, (in_features // pack_factor, out_features))
+ self.assertEqual(qzeros.shape, (num_groups, out_features // pack_factor))
+ self.assertEqual(scales.shape, (num_groups, out_features))
+
+ def test_outputs_are_on_cpu(self):
+ """Verify that all returned tensors are on the CPU regardless of input device."""
+ weight = torch.randn(16, 64)
+ qweight, qzeros, scales = quantize_weight_rtn(weight, bits=4, group_size=32)
+ self.assertEqual(qweight.device.type, "cpu")
+ self.assertEqual(qzeros.device.type, "cpu")
+ self.assertEqual(scales.device.type, "cpu")
+
+ def test_raises_on_1d_weight(self):
+ """quantize_weight_rtn must reject 1-D tensors."""
+ with self.assertRaises(AssertionError):
+ quantize_weight_rtn(torch.randn(64), bits=4, group_size=32)
+
+ def test_in_features_padding(self):
+ """in_features that is not a multiple of group_size should be padded internally."""
+ # in_features=70 is not a multiple of group_size=32
+ out_features, in_features = 16, 70
+ bits, group_size = 4, 32
+ weight = torch.randn(out_features, in_features)
+ # padded in_features = 96 (next multiple of 32)
+ padded_in = 96
+ pack_factor = 32 // bits
+
+ qweight, qzeros, scales = quantize_weight_rtn(weight, bits=bits, group_size=group_size)
+ self.assertEqual(qweight.shape, (padded_in // pack_factor, out_features))
+
+
+class TestGetWoqConfigFromDir(unittest.TestCase):
+ """Tests for _get_woq_config_from_dir."""
+
+ def test_returns_none_when_no_config(self):
+ with tempfile.TemporaryDirectory() as d:
+ self.assertIsNone(_get_woq_config_from_dir(d))
+
+ def test_returns_none_for_wrong_quant_method(self):
+ with tempfile.TemporaryDirectory() as d:
+ _write_config(
+ d,
+ {
+ "quant_method": "gptq",
+ "packing_format": "auto_round:auto_gptq",
+ "bits": 4,
+ "group_size": 128,
+ "sym": True,
+ },
+ )
+ self.assertIsNone(_get_woq_config_from_dir(d))
+
+ def test_returns_none_for_wrong_packing_format(self):
+ with tempfile.TemporaryDirectory() as d:
+ _write_config(
+ d,
+ {
+ "quant_method": "auto-round",
+ "packing_format": "other_format",
+ "bits": 4,
+ "group_size": 128,
+ "sym": True,
+ },
+ )
+ self.assertIsNone(_get_woq_config_from_dir(d))
+
+ def test_returns_none_for_missing_bits(self):
+ with tempfile.TemporaryDirectory() as d:
+ _write_config(
+ d,
+ {
+ "quant_method": "auto-round",
+ "packing_format": "auto_round:auto_gptq",
+ "group_size": 128,
+ "sym": True,
+ },
+ )
+ self.assertIsNone(_get_woq_config_from_dir(d))
+
+ def test_returns_none_for_bits_greater_than_8(self):
+ with tempfile.TemporaryDirectory() as d:
+ _write_config(
+ d,
+ {
+ "quant_method": "auto-round",
+ "packing_format": "auto_round:auto_gptq",
+ "bits": 16,
+ "group_size": 128,
+ "sym": True,
+ },
+ )
+ self.assertIsNone(_get_woq_config_from_dir(d))
+
+ def test_valid_config_returns_required_fields(self):
+ """A fully valid config returns a dict with bits, group_size, sym,
+ block_name_to_quantize, and extra_config."""
+ with tempfile.TemporaryDirectory() as d:
+ _write_config(
+ d,
+ {
+ "quant_method": "auto-round",
+ "packing_format": "auto_round:auto_gptq",
+ "bits": 4,
+ "group_size": 128,
+ "sym": True,
+ },
+ )
+ cfg = _get_woq_config_from_dir(d)
+ self.assertIsNotNone(cfg)
+ self.assertEqual(cfg["bits"], 4)
+ self.assertEqual(cfg["group_size"], 128)
+ self.assertTrue(cfg["sym"])
+ # New fields must also be present
+ self.assertIn("block_name_to_quantize", cfg)
+ self.assertIn("extra_config", cfg)
+
+ def test_valid_config_with_extra_config_and_block_name(self):
+ """extra_config and block_name_to_quantize from config.json are propagated."""
+ with tempfile.TemporaryDirectory() as d:
+ _write_config(
+ d,
+ {
+ "quant_method": "auto-round",
+ "packing_format": "auto_round:auto_gptq",
+ "bits": 4,
+ "group_size": 128,
+ "sym": True,
+ "block_name_to_quantize": ["model.layers"],
+ "extra_config": {"mtp.0.fc": {"bits": 8}},
+ },
+ )
+ cfg = _get_woq_config_from_dir(d)
+ self.assertIsNotNone(cfg)
+ self.assertEqual(cfg["block_name_to_quantize"], ["model.layers"])
+ self.assertEqual(cfg["extra_config"], {"mtp.0.fc": {"bits": 8}})
+
+
+class TestCopyMissingTensorsFromSource(unittest.TestCase):
+ """End-to-end tests for copy_missing_tensors_from_source.
+
+ Detection is now purely name-based (parent-layer + block-prefix comparison),
+ so no model loading or mocking is required.
+
+ A source tensor T is "missing" when ALL of:
+ 1. T is not already present in the saved output,
+ 2. T's parent layer (rsplit('.', 1)[0]) is absent from the saved parent layers,
+ 3. T's first numeric block-prefix (e.g. 'mtp.0') is absent from the saved
+ block-prefixes — OR T has no numeric segment at all.
+ Special case: 'lm_head.weight' is always excluded.
+ """
+
+ def _make_auto_round_config(self, bits=4, group_size=128, sym=True) -> dict:
+ return {
+ "quantization_config": {
+ "quant_method": "auto-round",
+ "packing_format": "auto_round:auto_gptq",
+ "bits": bits,
+ "group_size": group_size,
+ "sym": sym,
+ }
+ }
+
+ # ------------------------------------------------------------------ #
+ # Detection correctness #
+ # ------------------------------------------------------------------ #
+
+ def test_plain_copy_of_missing_mtp_tensor(self):
+ """Tensors from blocks entirely absent in the saved output are copied verbatim."""
+ with tempfile.TemporaryDirectory() as source_dir, tempfile.TemporaryDirectory() as target_dir:
+ mtp_norm = torch.randn(64)
+ _save_safetensors({"mtp.0.norm.weight": mtp_norm}, os.path.join(source_dir, "model.safetensors"))
+
+ _save_safetensors(
+ {"model.embed_tokens.weight": torch.randn(32, 64)},
+ os.path.join(target_dir, "model.safetensors"),
+ )
+ _write_config(target_dir)
+
+ copy_missing_tensors_from_source(source_dir, target_dir)
+
+ extra_shard = os.path.join(target_dir, "model_extra_tensors.safetensors")
+ self.assertTrue(os.path.exists(extra_shard), "Extra shard should be created")
+
+ result = _load_safetensors(extra_shard)
+ self.assertIn("mtp.0.norm.weight", result)
+ torch.testing.assert_close(result["mtp.0.norm.weight"], mtp_norm)
+
+ def test_tensor_with_known_parent_layer_not_copied(self):
+ """Source tensor whose parent layer already has tensors in the target is not missing.
+
+ Scenario: source has the original '.weight'; target has the packed '.qweight'
+ under the same parent layer — the layer was processed, not skipped.
+ """
+ with tempfile.TemporaryDirectory() as source_dir, tempfile.TemporaryDirectory() as target_dir:
+ _save_safetensors(
+ {"model.layers.0.self_attn.q_proj.weight": torch.randn(32, 64)},
+ os.path.join(source_dir, "model.safetensors"),
+ )
+ # Target has the quantised form — same parent layer, different suffix
+ _save_safetensors(
+ {"model.layers.0.self_attn.q_proj.qweight": torch.randint(0, 2**31, (8, 32), dtype=torch.int32)},
+ os.path.join(target_dir, "model.safetensors"),
+ )
+ _write_config(target_dir)
+
+ copy_missing_tensors_from_source(source_dir, target_dir)
+
+ extra_shard = os.path.join(target_dir, "model_extra_tensors.safetensors")
+ self.assertFalse(os.path.exists(extra_shard), "No extra shard expected: parent layer is known")
+
+ def test_tensor_with_known_block_prefix_not_copied(self):
+ """Source tensor whose numeric block-prefix exists in the saved output is not missing.
+
+ Scenario: source has an extra gate weight inside 'model.layers.0'; the target
+ already contains other tensors from 'model.layers.0' — the block was processed.
+ """
+ with tempfile.TemporaryDirectory() as source_dir, tempfile.TemporaryDirectory() as target_dir:
+ _save_safetensors(
+ {"model.layers.0.mlp.extra_gate.weight": torch.randn(32, 64)},
+ os.path.join(source_dir, "model.safetensors"),
+ )
+ # Target has a different sub-layer inside the same block
+ _save_safetensors(
+ {"model.layers.0.mlp.gate_proj.qweight": torch.randint(0, 2**31, (8, 32), dtype=torch.int32)},
+ os.path.join(target_dir, "model.safetensors"),
+ )
+ _write_config(target_dir)
+
+ copy_missing_tensors_from_source(source_dir, target_dir)
+
+ extra_shard = os.path.join(target_dir, "model_extra_tensors.safetensors")
+ self.assertFalse(os.path.exists(extra_shard), "No extra shard expected: block prefix is known")
+
+ def test_lm_head_weight_is_never_copied(self):
+ """lm_head.weight is always excluded, even when it is absent from the target."""
+ with tempfile.TemporaryDirectory() as source_dir, tempfile.TemporaryDirectory() as target_dir:
+ _save_safetensors(
+ {"lm_head.weight": torch.randn(32, 64)},
+ os.path.join(source_dir, "model.safetensors"),
+ )
+ _save_safetensors(
+ {"model.embed_tokens.weight": torch.randn(32, 64)},
+ os.path.join(target_dir, "model.safetensors"),
+ )
+ _write_config(target_dir)
+
+ copy_missing_tensors_from_source(source_dir, target_dir)
+
+ extra_shard = os.path.join(target_dir, "model_extra_tensors.safetensors")
+ self.assertFalse(os.path.exists(extra_shard), "lm_head.weight must never be copied")
+
+ def test_already_saved_tensors_are_not_duplicated(self):
+ """Tensors that are already present in the target are not written again."""
+ with tempfile.TemporaryDirectory() as source_dir, tempfile.TemporaryDirectory() as target_dir:
+ mtp_tensor = torch.randn(64)
+ _save_safetensors(
+ {"mtp.0.shared_head.norm.weight": mtp_tensor},
+ os.path.join(source_dir, "model.safetensors"),
+ )
+ # Target already has exactly this tensor
+ _save_safetensors(
+ {"mtp.0.shared_head.norm.weight": mtp_tensor},
+ os.path.join(target_dir, "model.safetensors"),
+ )
+ _write_config(target_dir)
+
+ copy_missing_tensors_from_source(source_dir, target_dir)
+
+ extra_shard = os.path.join(target_dir, "model_extra_tensors.safetensors")
+ self.assertFalse(os.path.exists(extra_shard), "No extra shard expected: tensor already present")
+
+ def test_detects_multiple_missing_tensors_from_different_blocks(self):
+ """All source tensors from blocks absent in the saved output are copied."""
+ with tempfile.TemporaryDirectory() as source_dir, tempfile.TemporaryDirectory() as target_dir:
+ _save_safetensors(
+ {
+ "custom_block.0.norm.weight": torch.randn(64),
+ "mtp.0.norm.weight": torch.randn(64),
+ "model.embed_tokens.weight": torch.randn(8, 64),
+ },
+ os.path.join(source_dir, "model.safetensors"),
+ )
+ _save_safetensors(
+ {"model.embed_tokens.weight": torch.randn(8, 64)},
+ os.path.join(target_dir, "model.safetensors"),
+ )
+ _write_config(target_dir)
+
+ copy_missing_tensors_from_source(source_dir, target_dir)
+
+ extra_shard = os.path.join(target_dir, "model_extra_tensors.safetensors")
+ self.assertTrue(os.path.exists(extra_shard))
+
+ result = _load_safetensors(extra_shard)
+ self.assertIn("custom_block.0.norm.weight", result)
+ self.assertIn("mtp.0.norm.weight", result)
+ # Model-known tensor must NOT be in the extra shard
+ self.assertNotIn("model.embed_tokens.weight", result)
+
+ # ------------------------------------------------------------------ #
+ # FP8 dequantization #
+ # ------------------------------------------------------------------ #
+
+ def test_fp8_weight_is_dequantized_to_bf16(self):
+ """FP8 weights paired with weight_scale_inv are dequantized to BF16 before saving."""
+ with tempfile.TemporaryDirectory() as source_dir, tempfile.TemporaryDirectory() as target_dir:
+ fp8_weight = torch.randn(64, 128).to(torch.float8_e4m3fn)
+ scale_inv = torch.tensor(0.5)
+
+ _save_safetensors(
+ {
+ "mtp.0.ffn.weight": fp8_weight,
+ "mtp.0.ffn.weight_scale_inv": scale_inv,
+ },
+ os.path.join(source_dir, "model.safetensors"),
+ )
+ _save_safetensors(
+ {"model.embed_tokens.weight": torch.randn(8, 64)},
+ os.path.join(target_dir, "model.safetensors"),
+ )
+ _write_config(target_dir)
+
+ copy_missing_tensors_from_source(source_dir, target_dir)
+
+ extra_shard = os.path.join(target_dir, "model_extra_tensors.safetensors")
+ self.assertTrue(os.path.exists(extra_shard))
+
+ result = _load_safetensors(extra_shard)
+ self.assertIn("mtp.0.ffn.weight", result)
+ # Scale tensor must be dropped after dequantization
+ self.assertNotIn("mtp.0.ffn.weight_scale_inv", result)
+ self.assertEqual(result["mtp.0.ffn.weight"].dtype, torch.bfloat16)
+
+ # ------------------------------------------------------------------ #
+ # WOQ quantization #
+ # ------------------------------------------------------------------ #
+
+ def test_woq_quantization_replaces_weight_with_packed_tensors(self):
+ """2-D MTP weight tensors are quantized and packed when an auto-round WOQ config is present."""
+ out_features, in_features = 32, 128
+ with tempfile.TemporaryDirectory() as source_dir, tempfile.TemporaryDirectory() as target_dir:
+ mtp_weight = torch.randn(out_features, in_features)
+ _save_safetensors(
+ {"mtp.0.ffn.weight": mtp_weight},
+ os.path.join(source_dir, "model.safetensors"),
+ )
+ _save_safetensors(
+ {"model.embed_tokens.weight": torch.randn(8, 64)},
+ os.path.join(target_dir, "model.safetensors"),
+ )
+ with open(os.path.join(target_dir, "config.json"), "w") as f:
+ json.dump(self._make_auto_round_config(bits=4, group_size=128, sym=True), f)
+
+ copy_missing_tensors_from_source(source_dir, target_dir)
+
+ extra_shard = os.path.join(target_dir, "model_extra_tensors.safetensors")
+ self.assertTrue(os.path.exists(extra_shard))
+
+ result = _load_safetensors(extra_shard)
+ # Original weight must be replaced by the three packed tensors
+ self.assertNotIn("mtp.0.ffn.weight", result)
+ self.assertIn("mtp.0.ffn.qweight", result)
+ self.assertIn("mtp.0.ffn.qzeros", result)
+ self.assertIn("mtp.0.ffn.scales", result)
+
+ def test_block_name_to_ignore_not_quantized_in_woq_mode(self):
+ """Weights matching BLOCK_NAME_TO_IGNORE patterns are copied as-is, not quantized."""
+ out_features, in_features = 32, 128
+ with tempfile.TemporaryDirectory() as source_dir, tempfile.TemporaryDirectory() as target_dir:
+ gate_weight = torch.randn(out_features, in_features)
+ # "mlp.gate." is in BLOCK_NAME_TO_IGNORE
+ _save_safetensors(
+ {"mtp.0.mlp.gate.weight": gate_weight},
+ os.path.join(source_dir, "model.safetensors"),
+ )
+ _save_safetensors(
+ {"model.embed_tokens.weight": torch.randn(8, 64)},
+ os.path.join(target_dir, "model.safetensors"),
+ )
+ with open(os.path.join(target_dir, "config.json"), "w") as f:
+ json.dump(self._make_auto_round_config(bits=4, group_size=128, sym=True), f)
+
+ copy_missing_tensors_from_source(source_dir, target_dir)
+
+ extra_shard = os.path.join(target_dir, "model_extra_tensors.safetensors")
+ self.assertTrue(os.path.exists(extra_shard))
+
+ result = _load_safetensors(extra_shard)
+ # Weight must be present as-is — not packed
+ self.assertIn("mtp.0.mlp.gate.weight", result)
+ self.assertNotIn("mtp.0.mlp.gate.qweight", result)
+
+ def test_config_updated_with_block_name_to_quantize_after_woq(self):
+ """After WOQ, config.json is updated so the new block appears in block_name_to_quantize."""
+ out_features, in_features = 32, 128
+ with tempfile.TemporaryDirectory() as source_dir, tempfile.TemporaryDirectory() as target_dir:
+ _save_safetensors(
+ {"mtp.0.ffn.weight": torch.randn(out_features, in_features)},
+ os.path.join(source_dir, "model.safetensors"),
+ )
+ _save_safetensors(
+ {"model.embed_tokens.weight": torch.randn(8, 64)},
+ os.path.join(target_dir, "model.safetensors"),
+ )
+ with open(os.path.join(target_dir, "config.json"), "w") as f:
+ json.dump(self._make_auto_round_config(bits=4, group_size=128, sym=True), f)
+
+ copy_missing_tensors_from_source(source_dir, target_dir)
+
+ with open(os.path.join(target_dir, "config.json")) as f:
+ updated_cfg = json.load(f)
+
+ qcfg = updated_cfg.get("quantization_config", {})
+ block_names = qcfg.get("block_name_to_quantize", [])
+ self.assertTrue(
+ any("mtp" in b for b in block_names),
+ f"Expected an 'mtp' block in block_name_to_quantize, got: {block_names}",
+ )
+
+ def test_config_updated_with_extra_config_for_unquantized_2d_weight(self):
+ """When a 2-D weight is skipped by BLOCK_NAME_TO_IGNORE, extra_config is updated
+ to mark it as full-precision so the inference stack knows not to dequantize it."""
+ out_features, in_features = 32, 128
+ with tempfile.TemporaryDirectory() as source_dir, tempfile.TemporaryDirectory() as target_dir:
+ # "mlp.gate." is in BLOCK_NAME_TO_IGNORE → not quantized → should go to extra_config
+ _save_safetensors(
+ {"mtp.0.mlp.gate.weight": torch.randn(out_features, in_features)},
+ os.path.join(source_dir, "model.safetensors"),
+ )
+ _save_safetensors(
+ {"model.embed_tokens.weight": torch.randn(8, 64)},
+ os.path.join(target_dir, "model.safetensors"),
+ )
+ with open(os.path.join(target_dir, "config.json"), "w") as f:
+ json.dump(self._make_auto_round_config(bits=4, group_size=128, sym=True), f)
+
+ copy_missing_tensors_from_source(source_dir, target_dir)
+
+ with open(os.path.join(target_dir, "config.json")) as f:
+ updated_cfg = json.load(f)
+
+ extra_config = updated_cfg.get("quantization_config", {}).get("extra_config", {})
+ self.assertIn("mtp.0.mlp.gate", extra_config)
+ layer_cfg = extra_config["mtp.0.mlp.gate"]
+ self.assertEqual(layer_cfg.get("bits"), 16)
+ self.assertEqual(layer_cfg.get("data_type"), "fp")
+
+ # ------------------------------------------------------------------ #
+ # Index file handling #
+ # ------------------------------------------------------------------ #
+
+ def test_index_json_is_updated_for_sharded_target(self):
+ """When target uses a sharded index, copied tensor names are added to weight_map."""
+ with tempfile.TemporaryDirectory() as source_dir, tempfile.TemporaryDirectory() as target_dir:
+ mtp_norm = torch.randn(64)
+ _save_safetensors({"mtp.0.norm.weight": mtp_norm}, os.path.join(source_dir, "model.safetensors"))
+
+ saved_shard = "model-00001-of-00001.safetensors"
+ _save_safetensors(
+ {"model.embed_tokens.weight": torch.randn(8, 64)},
+ os.path.join(target_dir, saved_shard),
+ )
+ index = {"metadata": {"total_size": 0}, "weight_map": {"model.embed_tokens.weight": saved_shard}}
+ with open(os.path.join(target_dir, "model.safetensors.index.json"), "w") as f:
+ json.dump(index, f)
+ _write_config(target_dir)
+
+ copy_missing_tensors_from_source(source_dir, target_dir)
+
+ with open(os.path.join(target_dir, "model.safetensors.index.json")) as f:
+ updated_index = json.load(f)
+
+ self.assertIn("mtp.0.norm.weight", updated_index["weight_map"])
+ self.assertEqual(updated_index["weight_map"]["mtp.0.norm.weight"], "model_extra_tensors.safetensors")
+
+ def test_index_json_is_created_for_single_file_target(self):
+ """When the target uses a single safetensors file and tensors are copied, a new
+ model.safetensors.index.json is created mapping all tensors to their shards."""
+ with tempfile.TemporaryDirectory() as source_dir, tempfile.TemporaryDirectory() as target_dir:
+ mtp_norm = torch.randn(64)
+ _save_safetensors({"mtp.0.norm.weight": mtp_norm}, os.path.join(source_dir, "model.safetensors"))
+
+ _save_safetensors(
+ {"model.embed_tokens.weight": torch.randn(8, 64)},
+ os.path.join(target_dir, "model.safetensors"),
+ )
+ _write_config(target_dir)
+
+ copy_missing_tensors_from_source(source_dir, target_dir)
+
+ index_path = os.path.join(target_dir, "model.safetensors.index.json")
+ self.assertTrue(os.path.exists(index_path), "An index.json should be created for single-file targets")
+
+ with open(index_path) as f:
+ index = json.load(f)
+
+ weight_map = index["weight_map"]
+ self.assertIn("mtp.0.norm.weight", weight_map)
+ self.assertIn("model.embed_tokens.weight", weight_map)
+ self.assertEqual(weight_map["mtp.0.norm.weight"], "model_extra_tensors.safetensors")
+ self.assertEqual(weight_map["model.embed_tokens.weight"], "model.safetensors")
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/test/test_cuda/models/test_moe_model.py b/test/test_cuda/models/test_moe_model.py
index fa9a91fae..006dc78e6 100644
--- a/test/test_cuda/models/test_moe_model.py
+++ b/test/test_cuda/models/test_moe_model.py
@@ -59,7 +59,7 @@ def setup_qwen3_vl_moe():
return model, tokenizer, processor, output_dir, config
-def quantize_model(model, tokenizer, output_dir, scheme, iters=0):
+def quantize_model(model, tokenizer, output_dir, scheme, iters=0, ignore_layers="self_attn,router,lm_head,mlp.gate"):
"""Helper function to quantize the model with the given scheme."""
autoround = AutoRound(
model,
@@ -68,7 +68,7 @@ def quantize_model(model, tokenizer, output_dir, scheme, iters=0):
nsamples=2,
iters=iters,
seqlen=32,
- ignore_layers="self_attn,router,lm_head,mlp.gate",
+ ignore_layers=ignore_layers,
)
quantized_model, save_folder = autoround.quantize_and_save(format="auto_round", output_dir=output_dir)
return quantized_model
@@ -81,7 +81,7 @@ def test_gptoss(setup_gpt_oss):
# Remove it to avoid mismatch during quantized model loading
delattr(model.config, "layer_types")
- quantized_model = quantize_model(model, tokenizer, output_dir, "MXFP4")
+ quantized_model = quantize_model(model, tokenizer, output_dir, "MXFP4", ignore_layers="self_attn,lm_head")
# Ensure the quantized model is not None
assert quantized_model is not None, "Quantized model should not be None."
@@ -111,7 +111,7 @@ def test_llama4(setup_llama4):
delattr(model.config.text_config, "moe_layers")
delattr(model.config.text_config, "layer_types")
- quantized_model = quantize_model(model, tokenizer, output_dir, "MXFP4")
+ quantized_model = quantize_model(model, tokenizer, output_dir, "MXFP4", ignore_layers="self_attn,router,lm_head")
# Ensure the quantized model is not None
assert quantized_model is not None, "Quantized model should not be None."