PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
support MTP module WOQ quantization and step3p5 model quantization (#1526)
Xin He42fdb06903e1
diff --git a/auto_round/__main__.py b/auto_round/__main__.pyindex 8ee9ed7f6..2254aafa8 100644--- a/auto_round/__main__.py+++ b/auto_round/__main__.py@@ -12,7 +12,9 @@# See the License for the specific language governing permissions and# limitations under the License.import argparse+import jsonimport os+import reimport sysfrom auto_round.auto_scheme import AutoScheme@@ -24,6 +26,7 @@ from auto_round.utils import (clear_memory,get_device_and_parallelism,get_model_dtype,+ parse_layer_config_arg,)RECIPES = {@@ -312,6 +315,15 @@ class BasicArgumentParser(argparse.ArgumentParser):scheme.add_argument("--disable_act_dynamic", action="store_true", help="Use static instead of dynamic activation quantization. ")+ scheme.add_argument(+ "--layer_config",+ default=None,+ type=str,+ help="Per-layer quantization config for missing tensors (e.g., MTP layers) as a JSON string. "+ "Keys are name prefixes, values are config dicts with optional bits/group_size/sym. "+ 'Example: "{mtp:{bits:8,data_type:int},mtp.fc:{bits:16}}". '+ "These settings are saved to extra_config and override the global quantization config.",+ )scheme.add_argument("--shared_layers",type=str,@@ -655,6 +667,9 @@ def tune(args):extra_config.diffusion_config = diffusion_configlayer_config = {}+ if args.layer_config:+ layer_config = parse_layer_config_arg(args.layer_config)+ args.layer_config = layer_configif args.avg_bits is not None:if args.options is None:diff --git a/auto_round/compressors/shard_writer.py b/auto_round/compressors/shard_writer.pyindex 990c5c387..b6dd329e3 100644--- a/auto_round/compressors/shard_writer.py+++ b/auto_round/compressors/shard_writer.py@@ -55,6 +55,10 @@ class ShardWriter:self.global_weight_map = {}self.shard_counter = 0+ # Persistent set of all parameter names already flushed to a shard file.+ # Maintained incrementally in _flush_shard to avoid O(N^2) rebuilds in _add_tensor.+ self._all_saved = set()+# Statsself.total_param_elems = 0self.total_param_size_bytes = 0@@ -99,6 +103,11 @@ class ShardWriter:if isinstance(tensor, torch.Tensor) and tensor.device.type == "meta":self.skipped_meta_tensors.append(name)return++ # Guard against duplicate saving of the same parameter+ if name in self._all_saved or name in self.current_shard_tensors:+ return+t_size = tensor.nbytesself.total_param_elems += tensor.numel()self.total_param_size_bytes += t_size@@ -135,6 +144,7 @@ class ShardWriter:saved_params = list(self.current_shard_tensors.keys())self.shard_meta.append({"tmp_file": tmp_name, "params": saved_params})+ self._all_saved.update(saved_params)# Offload logic: move modules to meta device once all params are savedself._offload_to_meta(saved_params)@@ -144,18 +154,15 @@ class ShardWriter:def _offload_to_meta(self, saved_params):"""Attempts to move fully saved modules to the 'meta' device to free RAM."""- # Using a set for faster lookup of all saved parameters- all_saved = {p for meta in self.shard_meta for p in meta["params"]}-for param_full_name in saved_params:module_path = param_full_name.rsplit(".", 1)[0]module = get_module(self.model, module_path)- # Check if all parameters of this module are now in 'all_saved'+ # Check if all parameters of this module are now in '_all_saved'if (module is not Noneand isinstance(module, torch.nn.Module)- and all(f"{module_path}.{k}" in all_saved for k in module.state_dict().keys())+ and all(f"{module_path}.{k}" in self._all_saved for k in module.state_dict().keys())):module.to("meta")@@ -164,11 +171,10 @@ class ShardWriter:# 1. Capture remaining weights not yet savedfull_sd = self.model.state_dict()tie_word_embeddings = getattr(getattr(self.model, "config", None), "tie_word_embeddings", True)- all_saved_names = {p for meta in self.shard_meta for p in meta["params"]}finalize_skipped_meta_tensors = []for pname, tensor in full_sd.items():- if pname in all_saved_names:+ if pname in self._all_saved:continueif tensor.device.type == "meta":continuediff --git a/auto_round/compressors/utils.py b/auto_round/compressors/utils.pyindex 73ed63206..1aef5bd7c 100644--- a/auto_round/compressors/utils.py+++ b/auto_round/compressors/utils.py@@ -12,6 +12,8 @@# See the License for the specific language governing permissions and# limitations under the License.import copy+import json+import osimport randomimport reimport sys@@ -208,6 +210,63 @@ def infer_bits_by_data_type(data_type: str):return None+def _get_safetensor_layer_names_not_in_model(model, all_module_names: list) -> list:+ """Collect layer names from safetensor files that are not loaded into the model.++ Some tensors (e.g. MTP layers) exist in the original checkpoint but are not+ instantiated by ``transformers``. This function discovers them so that regex+ patterns in ``layer_config`` can still match them.++ Returns:+ List of layer names (the path without the ``.weight`` suffix) for weight+ tensors present in the safetensor files but absent from *all_module_names*.+ """+ name_or_path = None+ if hasattr(model, "config") and hasattr(model.config, "name_or_path"):+ name_or_path = model.config.name_or_path+ if not name_or_path:+ return []++ if not os.path.isdir(name_or_path):+ try:+ from auto_round.utils.model import download_hf_model++ name_or_path = download_hf_model(name_or_path)+ except Exception as e:+ logger.debug(f"Could not resolve source model path to check for missing tensors: {e}")+ return []++ try:+ from safetensors import safe_open+ except ImportError:+ return []++ # Build tensor-name list from the safetensors index or single file+ source_index_file = os.path.join(name_or_path, "model.safetensors.index.json")+ source_single_file = os.path.join(name_or_path, "model.safetensors")++ tensor_names: list = []+ if os.path.exists(source_index_file):+ with open(source_index_file) as f:+ src_index = json.load(f)+ tensor_names = list(src_index["weight_map"].keys())+ elif os.path.exists(source_single_file):+ with safe_open(source_single_file, framework="pt", device="cpu") as f:+ tensor_names = list(f.keys())+ else:+ return []++ module_name_set = set(all_module_names)+ extra_layer_names = []+ for tensor_name in tensor_names:+ if not tensor_name.endswith(".weight"):+ continue+ layer_name = tensor_name[: -len(".weight")]+ if layer_name not in module_name_set:+ extra_layer_names.append(layer_name)+ return extra_layer_names++def set_layer_config(model: torch.nn.Module,layer_config: dict[str, Union[str, dict, "QuantizationScheme"]],@@ -235,6 +294,10 @@ def set_layer_config("""Assign scheme values as attributes to matched modules."""for layer_name, scheme in layer_config.items():module = get_module(model, layer_name)+ if module is None:+ # Layer exists in safetensor files but is not loaded into the model+ # (e.g. MTP layers that transformers does not instantiate). Skip.+ continuefor attr, value in scheme.items():setattr(module, attr, value)@@ -337,6 +400,10 @@ def set_layer_config(if isinstance(m, embedding_types) or m.__class__.__name__.endswith("Embedding"):embedding_layer_names.append(n)+ # Also include layer names from safetensor files not loaded into the model+ # (e.g. MTP layers that transformers does not instantiate).+ safetensor_only_names = _get_safetensor_layer_names_not_in_model(model, all_module_names)+# 6. expand regex configsregex_config = {}for name in list(layer_config.keys()):@@ -346,12 +413,14 @@ def set_layer_config(m = get_module(model, name)if len(list(m.children())) == 0 and type(m) not in supported_types:layer_config.pop(name)- logger.warning(f"{name} is not supported in current scheme, ignoring its setting in `layer_config`")+ logger.debug(f"{name} is not supported in current scheme, ignoring its setting in `layer_config`")continueregex = re.compile(name)matched = [ln for ln in all_supported_layer_names if regex.search(ln)]- if not matched:+ safetensor_only_matched = [ln for ln in safetensor_only_names if regex.search(ln)]+ # skip it for mtp layers not loaded in transformers+ if not matched and not safetensor_only_matched:raise ValueError(f"Invalid '{name}' in layer_config, no match found.")val = layer_config.pop(name)regex_config[name] = val # keep regex config@@ -898,6 +967,7 @@ def get_fp_layer_names(model: torch.nn.Module, ignore_layers: str):for name in all_layer_names:if fp_layer in name:not_to_quantized_layers.append(name)+ not_to_quantized_layers.extend(ignore_layers) # keep regex name for later uselogger.trace(f"not_to_quantized_layers: {not_to_quantized_layers}")return not_to_quantized_layersdiff --git a/auto_round/export/utils.py b/auto_round/export/utils.pyindex 42ae86d5f..df8ebd37a 100644--- a/auto_round/export/utils.py+++ b/auto_round/export/utils.py@@ -16,7 +16,12 @@ import osimport torch.nn as nn-from auto_round.utils import copy_python_files_from_model_cache, logger, unsupported_meta_device+from auto_round.utils import (+ copy_missing_tensors_from_source,+ copy_python_files_from_model_cache,+ logger,+ unsupported_meta_device,+)def save_model(@@ -57,6 +62,14 @@ def save_model(else:model.save_pretrained(save_dir, max_shard_size=max_shard_size, safe_serialization=safe_serialization)+ try:+ copy_missing_tensors_from_source(+ source_dir=model.config._name_or_path,+ target_dir=save_dir,+ )+ except Exception as e:+ logger.warning("Skipping copy of missing tensors from source checkpoint due to error: %s", e)+config_path = os.path.join(save_dir, "config.json")if dtype is not None and dtype != model.dtype and os.path.exists(os.path.join(save_dir, "config.json")):with open(config_path, "r") as file:diff --git a/auto_round/modeling/fused_moe/replace_modules.py b/auto_round/modeling/fused_moe/replace_modules.pyindex 1566ccbc2..f18cc2151 100644--- a/auto_round/modeling/fused_moe/replace_modules.py+++ b/auto_round/modeling/fused_moe/replace_modules.py@@ -37,6 +37,8 @@ BUILTIN_MODULES = {# supports transformers >= 5.0.0"qwen3_5_moe": LazyImport("auto_round.modeling.fused_moe.qwen3_5_moe"),"qwen3_5_moe_text": LazyImport("auto_round.modeling.fused_moe.qwen3_5_moe"),+ # Step 3.5 MoE: splits fused MoELinear into per-expert nn.Linear+ "step3p5": LazyImport("auto_round.modeling.fused_moe.step3_5_moe"),}@@ -99,7 +101,7 @@ def is_custom_model(model: torch.nn.Module) -> bool:def _log_first_moe_block(model: torch.nn.Module, label: str) -> None:"""Log the first experts module found in the model for debugging."""for name, module in model.named_modules():- if name.endswith(".experts"):+ if name.endswith(".experts") or name.endswith(".moe"):logger.info(f"Experts ({label}) [{name}] ({module.__class__.__name__}):\n{module}")return Truereturn Falsediff --git a/auto_round/modeling/fused_moe/step3_5_moe.py b/auto_round/modeling/fused_moe/step3_5_moe.pynew file mode 100755index 000000000..5f622f1e4--- /dev/null+++ b/auto_round/modeling/fused_moe/step3_5_moe.py@@ -0,0 +1,168 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++# Note: adapted from: https://huggingface.co/stepfun-ai/Step-3.5-Flash/blob/main/modeling_step3p5.py++import torch+import torch.nn as nn+import torch.nn.functional as F+from transformers.activations import ACT2FN++from auto_round.modeling.fused_moe.replace_modules import ReplacementModuleBase+from auto_round.modeling.fused_moe.utils import _update_parameter+from auto_round.utils import clear_memory, unsupported_meta_device+++class Step3p5ExpertMLP(nn.Module):+ """Single expert MLP with gate_proj, up_proj, down_proj as nn.Linear."""++ def __init__(self, hidden_size, intermediate_size, limit=None):+ super().__init__()+ self.gate_proj = nn.Linear(hidden_size, intermediate_size, bias=False)+ self.up_proj = nn.Linear(hidden_size, intermediate_size, bias=False)+ self.down_proj = nn.Linear(intermediate_size, hidden_size, bias=False)+ self.act_fn = ACT2FN["silu"]+ self.limit = limit++ def forward(self, x):+ up = self.up_proj(x)+ gate = self.act_fn(self.gate_proj(x))+ if self.limit is not None:+ gate = gate.clamp(min=None, max=self.limit)+ up = up.clamp(min=-self.limit, max=self.limit)+ return self.down_proj(gate * up)+++class SequentialStep3p5MoeExperts(torch.nn.ModuleList):+ """Converts fused MoELinear weights into individual Step3p5ExpertMLP modules."""++ def __init__(self, original):+ super().__init__()+ self.num_experts = original.up_proj.num_experts+ hidden_size = original.up_proj.in_features+ intermediate_size = original.up_proj.out_features+ limit = original.limit++ with torch.device("meta"):+ super().__init__([Step3p5ExpertMLP(hidden_size, intermediate_size, limit) for _ in range(self.num_experts)])++ def _materialize_weights(self, original) -> None:+ """Split fused MoELinear weights into individual expert nn.Linear weights.++ Args:+ original: The original Step3p5MoEMLP module containing fused MoELinear+ modules (gate_proj, up_proj, down_proj).+ """+ if not unsupported_meta_device(original.up_proj):+ for i in range(self.num_experts):+ _update_parameter(self[i].gate_proj, "weight", original.gate_proj.weight[i].contiguous())+ _update_parameter(self[i].up_proj, "weight", original.up_proj.weight[i].contiguous())+ _update_parameter(self[i].down_proj, "weight", original.down_proj.weight[i].contiguous())+ original.gate_proj.to_empty(device="meta")+ original.up_proj.to_empty(device="meta")+ original.down_proj.to_empty(device="meta")+ clear_memory()+++class LinearStep3p5MoEMLP(ReplacementModuleBase):+ """Replacement for Step3p5MoEMLP that splits fused MoELinear into+ individual nn.Linear per expert for quantization support."""++ def __init__(self, original, config=None):+ super().__init__(original)+ self.num_experts = original.num_experts+ self.top_k = original.top_k+ self.hidden_size = original.hidden_size+ self.moe_intermediate_size = original.moe_intermediate_size++ self.gate = original.gate+ self.need_fp32_gate = original.need_fp32_gate+ self.routed_scaling_factor = original.routed_scaling_factor++ # routing function+ self.use_moe_router_bias = original.use_moe_router_bias+ if self.use_moe_router_bias:+ self.router_bias = original.router_bias+ self.custom_routing_function = self.router_bias_func+ else:+ self.custom_routing_function = original.custom_routing_function++ # split fused experts into sequential+ with torch.device("meta"):+ self.experts = SequentialStep3p5MoeExperts(original)++ @classmethod+ def original_module_class(cls) -> str:+ """Return the class name of the module this replaces."""+ return "Step3p5MoEMLP"++ def _materialize_weights(self) -> None:+ original = self._get_original_module()+ self.experts._materialize_weights(original)+ clear_memory()++ def router_bias_func(self, gating_output: torch.Tensor, topk: int, renormalize: bool):+ gate_prob = torch.sigmoid(gating_output.float())+ gate_prob_with_bias = gate_prob + self.router_bias.unsqueeze(0)+ _, indices = torch.topk(gate_prob_with_bias, k=topk, dim=1)+ topk_prob = torch.gather(gate_prob, 1, indices)+ expert_topk_weight = topk_prob+ if renormalize:+ expert_topk_weight = expert_topk_weight / (torch.sum(expert_topk_weight, dim=-1, keepdim=True) + 1e-20)+ return expert_topk_weight, indices++ def forward(self, hidden_states):+ """Forward pass with sequential expert computation."""+ batch_size, sequence_length, hidden_dim = hidden_states.shape+ hidden_states = hidden_states.view(-1, hidden_dim)+ if self.need_fp32_gate:+ router_logits = torch.matmul(hidden_states.to(torch.float32), self.gate.weight.t().to(torch.float32))+ else:+ router_logits = self.gate(hidden_states)++ if self.custom_routing_function:+ routing_weights, selected_experts = self.custom_routing_function(+ router_logits, self.top_k, renormalize=True+ )+ else:+ routing_weights = F.softmax(router_logits, dim=1, dtype=torch.float)+ routing_weights, selected_experts = torch.topk(routing_weights, self.top_k, dim=-1)++ routing_weights = routing_weights * self.routed_scaling_factor++ final_hidden_states = torch.zeros(+ (batch_size * sequence_length, hidden_dim), dtype=hidden_states.dtype, device=hidden_states.device+ )++ expert_mask = torch.nn.functional.one_hot(selected_experts, num_classes=self.num_experts).permute(2, 1, 0)++ for expert_idx in range(self.num_experts):+ idx, top_x = torch.where(expert_mask[expert_idx])++ current_state = hidden_states[None, top_x].reshape(-1, hidden_dim)+ current_hidden_states = self.experts[expert_idx](current_state) * routing_weights[top_x, idx, None]++ final_hidden_states.index_add_(0, top_x, current_hidden_states.to(hidden_states.dtype))+ final_hidden_states = final_hidden_states.reshape(batch_size, sequence_length, hidden_dim)+ return final_hidden_states++ @classmethod+ def from_original(+ cls,+ original,+ config=None,+ **kwargs,+ ):+ """Create an instance from the original module."""+ return cls(original, config)diff --git a/auto_round/special_model_handler.py b/auto_round/special_model_handler.pyindex 9da31a1e3..fa0000bce 100644--- a/auto_round/special_model_handler.py+++ b/auto_round/special_model_handler.py@@ -256,6 +256,21 @@ register_ignore_layers()+# step3p5+register_ignore_layers(+ matchers=[+ ModelTypeMatcher(r"step3p5", mode="full"),+ ],+ ignore_layers=[+ "g_proj", # shape issue [96, 4096], 96 is not divisible by 64+ "moe.gate",+ "eh_proj", # MTP layer+ "shared_head", # MTP layer+ "layers.45", # MTP layer, requiring g_idx in vLLM, skip it+ ],+)++def get_predefined_ignore_layers(model: torch.nn.Module) -> list[str]:layers = []for rule in _PRE_DEFINED_IGNORE_LAYERS:diff --git a/auto_round/utils/__init__.py b/auto_round/utils/__init__.pyindex 32d976702..0d4d5325b 100644--- a/auto_round/utils/__init__.py+++ b/auto_round/utils/__init__.py@@ -16,9 +16,11 @@ from auto_round.utils.device import *from auto_round.utils.common import *from auto_round.utils.model import *from auto_round.utils.weight_handler import (+ convert_module_to_hp_if_necessary,detect_weight_type,is_quantized_input_module,)+from auto_round.utils.missing_tensors import copy_missing_tensors_from_sourceimport transformersfrom packaging.version import Versiondiff --git a/auto_round/utils/common.py b/auto_round/utils/common.pyindex a0dc89e8f..decd5540f 100644--- a/auto_round/utils/common.py+++ b/auto_round/utils/common.py@@ -469,3 +469,42 @@ def is_transformers_version_greater_or_equal_4():from packaging import versionreturn version.parse(transformers.__version__) >= version.parse("4.0.0")+++def parse_layer_config_arg(s: str) -> dict:+ """Parse --layer_config with unquoted keys/values.++ Delimiters are ``{``, ``}``, ``,``, ``:``. Each non-delimiter token is+ auto-typed: numeric strings become ``int``, everything else stays ``str``.++ Example::++ {mtp:{bits:8},mtp.fc:{bits:16,data_type:fp}}+ """+ tokens = re.findall(r"[{}:,]|[^\s{}:,]+", s)+ pos = [0]++ def _val():+ tok = tokens[pos[0]]+ if tok == "{":+ return _dict()+ pos[0] += 1+ try:+ return int(tok)+ except ValueError:+ return tok++ def _dict():+ pos[0] += 1 # consume '{'+ result = {}+ while tokens[pos[0]] != "}":+ key = tokens[pos[0]]+ pos[0] += 1 # key+ pos[0] += 1 # consume ':'+ result[key] = _val()+ if tokens[pos[0]] == ",":+ pos[0] += 1 # consume ','+ pos[0] += 1 # consume '}'+ return result++ return _dict()diff --git a/auto_round/utils/missing_tensors.py b/auto_round/utils/missing_tensors.pynew file mode 100644index 000000000..68c8f8fa3--- /dev/null+++ b/auto_round/utils/missing_tensors.py@@ -0,0 +1,706 @@+# Copyright (c) 2025 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++# Background+# ----------+# When ``transformers`` loads a model, it only instantiates the sub-modules+# declared in the model class. Some architectures include auxiliary+# parameters — most notably **MTP (Multi-Token Prediction)** layers — that are+# stored in the original checkpoint but are *not* part of the default module+# graph. As a result, ``model.save_pretrained`` silently omits those tensors+# from the quantized output.+#+# This module provides :func:`copy_missing_tensors_from_source`, which detects+# and copies such tensors from the original checkpoint into the saved output+# directory. Detection works by comparing the **parent layer names**+# (``tensor_name.rsplit(".", 1)[0]``) present in the source checkpoint against+# those present in the saved output: any source tensor whose parent layer has+# no tensors at all in the saved output (and is itself absent) is considered+# "missing". This requires no model/config loading.+#+# Before writing, the function optionally:+# - **dequantizes FP8 weights** to BF16 (when a matching ``weight_scale_inv``+# tensor is present in the source checkpoint), and+# - **re-quantizes with RTN** into the packed WOQ format used by the rest of+# the saved model (when ``quant_method == "auto-round"`` and+# ``packing_format == "auto_round:auto_gptq"`` are detected in+# ``config.json``).++import json+import os+from typing import Optional, Tuple++import torch++from auto_round.logger import logger+from auto_round.utils.weight_handler import _dequant_fp8_linear_weight+++def copy_missing_tensors_from_source(+ source_dir: str,+ target_dir: str,+) -> None:+ """Copy tensors from the source checkpoint that are absent from the saved output.++ Some parameters (e.g., MTP layers) are not loaded by ``transformers`` and+ therefore not written by ``model.save_pretrained``. This function copies+ those tensors from the original checkpoint into the quantized output dir.++ Detection works by comparing **parent layer names**+ (``tensor_name.rsplit(".", 1)[0]``) between the source checkpoint and the+ saved output. A source tensor is considered "missing" when:+ 1. its name is **not** already present in the saved output, **and**+ 2. its parent layer name is **not** among the parent layer names of any+ saved tensor (i.e. the layer was entirely skipped by+ ``save_pretrained``).++ No model or config loading is required.++ FP8 handling:+ Missing weight tensors in FP8 dtype that have a corresponding+ ``weight_scale_inv`` tensor are dequantized to BF16 before saving.++ WOQ quantization:+ If the saved ``config.json`` contains a ``quantization_config`` with+ ``quant_method == "auto-round"`` and+ ``packing_format == "auto_round:auto_gptq"``, missing 2-D weight tensors+ will be quantized with RTN and packed into qweight/qzeros/scales format.++ Args:+ source_dir: Path to the original (pre-quantization) checkpoint directory+ or a HuggingFace repo id.+ target_dir: Directory to which the quantized model was saved. Must+ contain a ``config.json``.+ """++ try:+ from safetensors import safe_open+ from safetensors.torch import save_file+ except ImportError:+ logger.warning("safetensors not available, skipping copy of missing tensors from source checkpoint")+ return++ # ------------------------------------------------------------------ #+ # Resolve source directory #+ # ------------------------------------------------------------------ #+ config_path = os.path.join(target_dir, "config.json")+ if not os.path.exists(config_path):+ return++ if not os.path.isdir(source_dir):+ try:+ from auto_round.utils.model import download_hf_model++ source_dir = download_hf_model(source_dir)+ except Exception as e:+ logger.debug(f"Could not resolve source model path to check for missing tensors: {e}")+ return++ if not source_dir or not os.path.isdir(source_dir):+ return++ # ------------------------------------------------------------------ #+ # Build a mapping: tensor_name -> source shard file path #+ # ------------------------------------------------------------------ #+ source_index_file = os.path.join(source_dir, "model.safetensors.index.json")+ source_single_file = os.path.join(source_dir, "model.safetensors")++ source_tensor_to_file: dict = {}+ if os.path.exists(source_index_file):+ with open(source_index_file) as f:+ src_index = json.load(f)+ for tensor_name, shard_file in src_index["weight_map"].items():+ source_tensor_to_file[tensor_name] = os.path.join(source_dir, shard_file)+ elif os.path.exists(source_single_file):+ with safe_open(source_single_file, framework="pt", device="cpu") as f:+ for key in f.keys():+ source_tensor_to_file[key] = source_single_file+ else:+ return++ # ------------------------------------------------------------------ #+ # Collect tensor names already present in the saved output #+ # ------------------------------------------------------------------ #+ saved_tensor_names: set = set()+ saved_index_file = os.path.join(target_dir, "model.safetensors.index.json")+ saved_single_file = os.path.join(target_dir, "model.safetensors")++ if os.path.exists(saved_index_file):+ with open(saved_index_file) as f:+ saved_idx = json.load(f)+ saved_tensor_names = set(saved_idx["weight_map"].keys())+ elif os.path.exists(saved_single_file):+ with safe_open(saved_single_file, framework="pt", device="cpu") as f:+ saved_tensor_names = set(f.keys())+ else:+ return++ # ------------------------------------------------------------------ #+ # Identify missing tensors via block-prefix statistics #+ # ------------------------------------------------------------------ #+ # A source tensor is considered "missing" when:+ # 1. its name is absent from the saved output, AND+ # 2. its parent layer name is NOT among the parent layers of any saved+ # tensor (catches layers whose suffix changed, e.g. .weight →+ # .qweight after quantization), AND+ # 3. its "block prefix" — the path up to and including the first+ # numeric segment, e.g. ``model.layers.0`` — is NOT present in the+ # set of block prefixes collected from the saved output.++ saved_parent_layers: set = {name.rsplit(".", 1)[0] for name in saved_tensor_names}++ def _first_numeric_prefix(tensor_name: str) -> str | None:+ """Return the path up to and including the first numeric segment, or None.++ e.g. ``model.layers.0.mlp.gate.qweight`` → ``model.layers.0``+ ``model.embed_tokens.weight`` → ``None``+ """+ parts = tensor_name.split(".")+ for i, p in enumerate(parts):+ if p.isdigit():+ return ".".join(parts[: i + 1])+ return None++ # Collect block prefixes from the saved output: path up to the first+ # numeric segment in each saved tensor name (e.g. "model.layers.0",+ # "mlp.experts.0").+ saved_block_prefix: set = set()+ for tensor_name in saved_tensor_names:+ bp = _first_numeric_prefix(tensor_name)+ if bp is not None:+ saved_block_prefix.add(bp)++ def _is_truly_missing(name: str) -> bool:+ # For Qwen/Qwen3-0.6B-FP8, lm_head is tied but still in source_dir → not missing+ if name == "lm_head.weight":+ return False+ if name in saved_tensor_names:+ return False+ parent = name.rsplit(".", 1)[0]+ if parent in saved_parent_layers:+ return False+ src_block = _first_numeric_prefix(name)+ if src_block is not None:+ # Source tensor belongs to a numbered block; if that exact block+ # appears in the saved output it was processed (possibly renamed+ # or split) rather than omitted.+ return src_block not in saved_block_prefix+ return True++ missing_tensor_names: list = [name for name in source_tensor_to_file if _is_truly_missing(name)]++ if not missing_tensor_names:+ return++ logger.debug(f"Missing tensors detected: {missing_tensor_names}")++ logger.info(+ f"Found {len(missing_tensor_names)} tensor(s) in the source checkpoint that are "+ f"absent from the saved output (e.g., MTP parameters). Copying them now..."+ )++ # ------------------------------------------------------------------ #+ # Load missing tensors from source shards #+ # ------------------------------------------------------------------ #+ shard_to_missing: dict = {}+ for tensor_name in missing_tensor_names:+ shard_file = source_tensor_to_file[tensor_name]+ shard_to_missing.setdefault(shard_file, []).append(tensor_name)++ missing_tensors_dict: dict = {}+ for shard_file, tensor_names in shard_to_missing.items():+ with safe_open(shard_file, framework="pt", device="cpu") as f:+ for tensor_name in tensor_names:+ missing_tensors_dict[tensor_name] = f.get_tensor(tensor_name)++ # ------------------------------------------------------------------ #+ # FP8 dequantization: if a weight is FP8 and its scale_inv is present #+ # ------------------------------------------------------------------ #++ fp8_dtypes = {"torch.float8_e4m3fn", "torch.float8_e5m2", "torch.float8_e4m3fnuz", "torch.float8_e5m2fnuz"}++ dequantized_keys: list = []+ keys_to_remove: list = []+ for tensor_name, tensor in list(missing_tensors_dict.items()):+ if str(tensor.dtype) not in fp8_dtypes:+ continue+ if not tensor_name.endswith(".weight"):+ continue++ # Look for the corresponding scale_inv tensor+ base = tensor_name[: -len(".weight")]+ scale_inv_name = base + ".weight_scale_inv"+ scale_name = base + ".weight_scale"++ weight_scale = missing_tensors_dict.get(scale_inv_name)+ if weight_scale is None:+ weight_scale = missing_tensors_dict.get(scale_name)++ if weight_scale is None:+ continue++ # Try to determine block_size from config or fall back to per-tensor+ block_size = None+ if weight_scale.dim() >= 2 and weight_scale.shape != tensor.shape:+ block_m = tensor.shape[-2] // weight_scale.shape[-2] if weight_scale.shape[-2] > 0 else 1+ block_n = tensor.shape[-1] // weight_scale.shape[-1] if weight_scale.shape[-1] > 0 else 1+ if block_m > 1 or block_n > 1:+ block_size = [block_m, block_n]++ dq_weight = _dequant_fp8_linear_weight(tensor, weight_scale, block_size)+ missing_tensors_dict[tensor_name] = dq_weight.to(torch.bfloat16)+ dequantized_keys.append(tensor_name)++ for s_name in [scale_inv_name, scale_name]:+ if s_name in missing_tensors_dict:+ keys_to_remove.append(s_name)++ for k in keys_to_remove:+ missing_tensors_dict.pop(k, None)++ if dequantized_keys:+ logger.info(+ f"Dequantized {len(dequantized_keys)} FP8 weight(s) to BF16 before saving: "+ f"{dequantized_keys[:5]}{'...' if len(dequantized_keys) > 5 else ''}"+ )++ if not missing_tensors_dict:+ return++ # ------------------------------------------------------------------ #+ # WOQ quantization: quantize missing Linear weights if model is WOQ #+ # ------------------------------------------------------------------ #+ missing_tensors_dict = _woq_quantize_missing_tensors(target_dir, missing_tensors_dict)++ if not missing_tensors_dict:+ return++ # ------------------------------------------------------------------ #+ # Write missing tensors to a new shard and update the index #+ # ------------------------------------------------------------------ #+ is_sharded = os.path.exists(saved_index_file)++ new_shard_name = "model_extra_tensors.safetensors"+ new_shard_path = os.path.join(target_dir, new_shard_name)+ missing_tensors_dict = {k: v.contiguous() for k, v in missing_tensors_dict.items()}+ save_file(missing_tensors_dict, new_shard_path)++ if is_sharded:+ with open(saved_index_file) as f:+ saved_index = json.load(f)+ for tensor_name in missing_tensors_dict:+ saved_index["weight_map"][tensor_name] = new_shard_name+ with open(saved_index_file, "w") as f:+ json.dump(saved_index, f, indent=2)+ elif os.path.exists(saved_single_file):+ weight_map: dict = {}+ with safe_open(saved_single_file, framework="pt", device="cpu") as f:+ for key in f.keys():+ weight_map[key] = "model.safetensors"+ for tensor_name in missing_tensors_dict:+ weight_map[tensor_name] = new_shard_name+ new_index = {"metadata": {"total_size": 0}, "weight_map": weight_map}+ with open(saved_index_file, "w") as f:+ json.dump(new_index, f, indent=2)++ logger.info(+ f"Successfully wrote {len(missing_tensors_dict)} missing tensor(s) to " f"'{new_shard_name}' in {target_dir}."+ )+++def quantize_weight_rtn(+ weight: torch.Tensor,+ bits: int,+ group_size: int,+ sym: bool = True,+ device: Optional[torch.device] = None,+) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:+ """Quantize a 2-D weight tensor and pack into auto_gptq format.++ Parameters+ ----------+ weight : Tensor [out_features, in_features]+ bits : target bit-width (e.g. 4, 8)+ group_size : quantization group size along in_features+ sym : use symmetric quantisation+ device : compute device (cuda / cpu). Results are always returned on CPU.++ Returns+ -------+ qweight : [in_features // pack_factor, out_features] int32+ qzeros : [num_groups, out_features // pack_factor] int32+ scales : [num_groups, out_features] float16+ """+ assert weight.dim() == 2, f"Expected 2-D weight, got {weight.dim()}-D"+ out_features, in_features = weight.shape+ if device is None:+ device = weight.device+ weight = weight.to(device).float()++ # --- pad in_features to multiple of group_size ---+ if in_features % group_size != 0:+ pad = group_size - (in_features % group_size)+ weight = torch.nn.functional.pad(weight, (0, pad))+ in_features = weight.shape[1]++ num_groups = in_features // group_size+ pack_factor = 32 // bits # values per int32++ # --- pad out_features to multiple of pack_factor (needed for qzeros) ---+ out_pad = 0+ if out_features % pack_factor != 0:+ out_pad = pack_factor - (out_features % pack_factor)+ weight = torch.nn.functional.pad(weight, (0, 0, 0, out_pad))+ padded_out = weight.shape[0]++ # Reshape for group-wise ops: [padded_out, num_groups, group_size]+ w_grouped = weight.reshape(padded_out, num_groups, group_size)++ if sym:+ half_range = (1 << (bits - 1)) - 1 # e.g. 7 for 4-bit+ zero_point = 1 << (bits - 1) # e.g. 8 for 4-bit++ w_absmax = w_grouped.abs().amax(dim=-1, keepdim=True).clamp(min=1e-10)+ scales_full = w_absmax / half_range # [padded_out, num_groups, 1]++ q = torch.round(w_grouped / scales_full).clamp(-half_range - 1, half_range)+ q = (q + zero_point).to(torch.int32) # shift to unsigned [0, 2^bits-1]++ zp = torch.full((num_groups, padded_out), zero_point, dtype=torch.int32, device=device)+ else:+ max_int = (1 << bits) - 1+ w_min = w_grouped.amin(dim=-1, keepdim=True)+ w_max = w_grouped.amax(dim=-1, keepdim=True)++ scales_full = ((w_max - w_min).clamp(min=1e-10)) / max_int+ zp_full = torch.round(-w_min / scales_full).clamp(0, max_int).to(torch.int32)++ q = torch.round(w_grouped / scales_full + zp_full.float()).clamp(0, max_int).to(torch.int32)++ zp = zp_full.squeeze(-1).t().contiguous() # [num_groups, padded_out]++ # scales → [num_groups, padded_out] (float16)+ scales_out = scales_full.squeeze(-1).t().contiguous().to(torch.float16)++ # q → [in_features, padded_out]+ q = q.reshape(padded_out, in_features).t().contiguous()++ # ---- Pack qweight: [in_features // pack_factor, padded_out] ----+ qweight = torch.zeros(in_features // pack_factor, padded_out, dtype=torch.int32, device=device)+ for k in range(pack_factor):+ qweight |= q[k::pack_factor, :] << (bits * k)++ # ---- Pack qzeros: [num_groups, padded_out // pack_factor] ----+ qzeros = torch.zeros(num_groups, padded_out // pack_factor, dtype=torch.int32, device=device)+ for k in range(pack_factor):+ qzeros |= zp[:, k::pack_factor] << (bits * k)++ # Remove output padding from qweight / scales (qzeros stays in pack units)+ if out_pad > 0:+ qweight = qweight[:, :out_features]+ scales_out = scales_out[:, :out_features]++ # Always return CPU tensors (safetensors requires CPU)+ return qweight.cpu(), qzeros.cpu(), scales_out.cpu()+++def _woq_quantize_missing_tensors(target_dir: str, missing_tensors_dict: dict) -> dict:+ """Apply WOQ (Weight-Only Quantization) to missing Linear weight tensors.++ Reads ``config.json`` from *target_dir* to obtain ``quantization_config``.+ Only activates when ``quant_method == "auto-round"`` and+ ``packing_format == "auto_round:auto_gptq"``.++ Uses :func:`quantize_weight_rtn` for RTN quantisation + packing so that+ there is **no dependency on the model object** or on QuantLinear classes.++ Non-weight tensors (bias, norms, embeddings, etc.) are kept as-is.++ Per-layer resolution:+ For each weight, ``extra_config`` is checked first (exact layer name,+ then regex pattern match), and the result is merged with global+ defaults. This means entries like ``".*mtp.*": {"bits": 8}`` or+ ``"mtp.fc": {"bits": 16, "data_type": "fp"}`` are honoured, while+ layers absent from ``extra_config`` fall back to global ``bits`` /+ ``group_size`` / ``sym``.++ Args:+ target_dir: Output directory that contains ``config.json``.+ missing_tensors_dict: Dict mapping tensor names to tensor values.++ Returns:+ Updated dict with quantized+packed tensors replacing original weight tensors.+ """+ import re as _re++ BLOCK_NAME_TO_IGNORE = ["shared_expert_gate.", "mlp.gate.", "g_proj."]+ qconfig = _get_woq_config_from_dir(target_dir)+ if qconfig is None:+ return missing_tensors_dict++ global_bits = qconfig["bits"]+ global_group_size = qconfig["group_size"]+ global_sym = qconfig["sym"]+ block_name_to_quantize = qconfig.get("block_name_to_quantize", None)+ extra_config: dict = qconfig.get("extra_config", {}) or {}++ def _resolve_layer_cfg(layer_name: str) -> dict:+ """Return effective {bits, group_size, sym, data_type} for *layer_name*.++ Lookup order:+ 1. Exact key match in extra_config.+ 2. Among all regex keys that match layer_name, pick the longest pattern+ (longer pattern == more specific, e.g. ``.*mtp\\.fc.*`` beats ``.*mtp.*``).+ 3. Global defaults.+ """+ override: dict = {}+ # 1. exact match+ if layer_name in extra_config:+ override = extra_config[layer_name]+ else:+ # 2. collect all matching regex patterns, keep the most specific (longest)+ best_pattern: str | None = None+ for pattern, cfg in extra_config.items():+ if pattern == layer_name:+ continue # already handled above+ try:+ if _re.search(pattern, layer_name):+ if best_pattern is None or len(pattern) > len(best_pattern):+ best_pattern = pattern+ override = cfg+ except _re.error:+ pass # not a valid regex → skip+ return {+ "bits": override.get("bits", global_bits),+ "group_size": override.get("group_size", global_group_size),+ "sym": override.get("sym", global_sym),+ "data_type": override.get("data_type", "int"),+ }++ def _is_fp_layer(layer_cfg: dict) -> bool:+ """Return True when the resolved config indicates full-precision (no quantization)."""+ dt = layer_cfg.get("data_type", "int")+ return layer_cfg["bits"] >= 16 or dt in ("fp", "float", "float16", "bfloat16", "float32")++ # Identify weight tensors eligible for quantization (2D Linear weights)+ def _is_eligible(k: str) -> bool:+ if not k.endswith(".weight"):+ return False+ if missing_tensors_dict[k].dim() != 2:+ return False+ layer_name = k[: -len(".weight")]+ layer_cfg = _resolve_layer_cfg(layer_name)+ # If extra_config explicitly covers this layer, trust its decision+ if layer_name in extra_config or any(+ _re.search(p, layer_name) for p in extra_config if p != layer_name and _is_valid_regex(p)+ ):+ return not _is_fp_layer(layer_cfg)+ # Fall back to BLOCK_NAME_TO_IGNORE heuristic+ if any(block in k for block in BLOCK_NAME_TO_IGNORE):+ return False+ return not _is_fp_layer(layer_cfg)++ weight_keys = [k for k in missing_tensors_dict if _is_eligible(k)]++ # Collect 2-D weight tensors that will NOT be quantized (for extra_config bookkeeping).+ ignored_weight_keys = [+ k+ for k in missing_tensors_dict+ if k.endswith(".weight") and missing_tensors_dict[k].dim() == 2 and k not in weight_keys+ ]++ # Update quantization config: extra_config for ignored layers + block_name_to_quantize for new blocks+ if ignored_weight_keys or weight_keys:++ def _update_qcfg(qcfg):+ if ignored_weight_keys:+ ec = qcfg.get("extra_config", {})+ for k in ignored_weight_keys:+ layer_name = k[: -len(".weight")]+ ec.setdefault(layer_name, {"bits": 16, "data_type": "fp"})+ qcfg["extra_config"] = ec+ logger.info(+ f"Updated extra_config for {len(ignored_weight_keys)} ignored layer(s): "+ f"{[k[: -len('.weight')] for k in ignored_weight_keys]}"+ )+ if weight_keys:+ existing = qcfg.get("block_name_to_quantize") or []+ if isinstance(existing, str):+ existing = [b.strip() for b in existing.split(",") if b.strip()]+ existing_set = set(existing)+ new_prefixes = {+ ".".join(k.split(".")[:i])+ for k in weight_keys+ for i, p in enumerate(k.split("."))+ if p.isdigit() and ".".join(k.split(".")[:i])+ }+ added = sorted(new_prefixes - existing_set)+ if added:+ merged = existing + added+ merged = [+ b for b in merged if not any(b != other and b.startswith(other + ".") for other in merged)+ ]+ qcfg["block_name_to_quantize"] = merged+ logger.info(f"Updated block_name_to_quantize: {merged}")+ return qcfg++ for cfg_file in ["config.json", "quantization_config.json"]:+ cfg_path = os.path.join(target_dir, cfg_file)+ if not os.path.exists(cfg_path):+ continue+ logger.info(f"Processing {cfg_file} to update quantization_config for missing tensors...")+ try:+ with open(cfg_path) as f:+ cfg_data = json.load(f)+ if cfg_file == "config.json":+ cfg_data["quantization_config"] = _update_qcfg(cfg_data.get("quantization_config", {}))+ else:+ cfg_data = _update_qcfg(cfg_data)+ with open(cfg_path, "w") as f:+ json.dump(cfg_data, f, indent=2)+ except Exception as e:+ logger.warning(f"Failed to update {cfg_file}: {e}")++ if not weight_keys:+ return missing_tensors_dict++ logger.info(+ f"Applying WOQ to "+ f"{len(weight_keys)} missing Linear weight(s) (per-layer overrides from extra_config applied)..."+ )++ new_tensors: dict = {}+ packed_weight_keys: set = set()++ for weight_key in weight_keys:+ weight = missing_tensors_dict[weight_key]+ out_features, in_features = weight.shape++ layer_name = weight_key[: -len(".weight")]+ layer_cfg = _resolve_layer_cfg(layer_name)+ bits = layer_cfg["bits"]+ group_size = layer_cfg["group_size"]+ sym = layer_cfg["sym"]++ effective_gs = group_size if group_size != -1 else in_features+ if effective_gs <= 0:+ continue++ logger.debug(+ f"WOQ [{layer_name}]: bits={bits}, group_size={effective_gs}, sym={sym}, " f"shape={list(weight.shape)}"+ )++ base_name = layer_name++ try:+ qweight, qzeros, scales = quantize_weight_rtn(weight, bits=bits, group_size=effective_gs, sym=sym)+ except Exception as e:+ logger.warning(f"Failed to quantize {weight_key}: {e}, keeping original weight")+ continue++ new_tensors[base_name + ".qweight"] = qweight+ new_tensors[base_name + ".qzeros"] = qzeros+ new_tensors[base_name + ".scales"] = scales++ packed_weight_keys.add(weight_key)++ if not packed_weight_keys:+ return missing_tensors_dict++ # Replace original weights with packed versions+ result = {}+ for k, v in missing_tensors_dict.items():+ if k not in packed_weight_keys:+ result[k] = v+ result.update(new_tensors)++ logger.info(+ f"Successfully packed {len(packed_weight_keys)} weight(s) into WOQ format "+ f"({len(new_tensors)} packed tensor(s) created)."+ )+ return result+++def _is_valid_regex(pattern: str) -> bool:+ """Return True if *pattern* is a valid regular expression."""+ import re as _re++ try:+ _re.compile(pattern)+ return True+ except _re.error:+ return False+++def _get_woq_config_from_dir(target_dir: str) -> dict | None:+ """Read ``quantization_config`` from ``config.json`` in *target_dir*.++ Only supports ``quant_method == "auto-round"`` with+ ``packing_format == "auto_round:auto_gptq"``. Returns ``None`` (skip+ quantization) for any other combination or if any required parameter+ (``bits``, ``group_size``, ``sym``) is missing.++ Returns a dict with keys ``bits``, ``group_size``, ``sym``+ or ``None``.+ """+ config_path = os.path.join(target_dir, "config.json")+ if not os.path.exists(config_path):+ return None++ with open(config_path) as f:+ config_data = json.load(f)++ qcfg = config_data.get("quantization_config", None)+ if not isinstance(qcfg, dict):+ return None++ quant_method = qcfg.get("quant_method", "")+ packing_format = qcfg.get("packing_format", "")+ bits = qcfg.get("bits", None)+ group_size = qcfg.get("group_size", None)+ sym = qcfg.get("sym", None)++ if quant_method != "auto-round":+ logger.debug(+ f"Skipping WOQ quantization of missing tensors: "+ f"quant_method='{quant_method}' (only 'auto-round' is supported)"+ )+ return None++ if packing_format != "auto_round:auto_gptq":+ logger.debug(+ f"Skipping WOQ quantization of missing tensors: "+ f"packing_format='{packing_format}' (only 'auto_round:auto_gptq' is supported)"+ )+ return None++ if bits is None or group_size is None or sym is None:+ logger.debug(+ f"Skipping WOQ quantization of missing tensors: "+ f"incomplete quantization_config (bits={bits}, group_size={group_size}, sym={sym})"+ )+ return None++ if bits > 8:+ return None++ block_name_to_quantize = qcfg.get("block_name_to_quantize", None)+ extra_config = qcfg.get("extra_config", {})+ return {+ "bits": bits,+ "group_size": group_size,+ "sym": sym,+ "block_name_to_quantize": block_name_to_quantize,+ "extra_config": extra_config,+ }diff --git a/auto_round/utils/model.py b/auto_round/utils/model.pyindex 91f2b7e89..0e4a406fb 100644--- a/auto_round/utils/model.py+++ b/auto_round/utils/model.py@@ -257,8 +257,6 @@ def _is_mxfp4_model(model_path, trust_remote_code=True):Only checks when transformers >= 5.0.0. Returns False immediately for older versions,adding zero overhead to non-MXFP4 model loading."""- if version.parse(transformers.__version__) < version.parse("5.0.0"):- return Falsefrom transformers import AutoConfigtry: # in case of config loading failure for new models@@ -318,17 +316,19 @@ def llm_load_model(if device_str is not None and "hpu" in device_str:torch_dtype = torch.bfloat16- is_mxfp4 = _is_mxfp4_model(pretrained_model_name_or_path, trust_remote_code=trust_remote_code)load_kwargs = {"torch_dtype": torch_dtype,"trust_remote_code": trust_remote_code,"device_map": "auto" if use_auto_mapping else None,}- if is_mxfp4:- from transformers import Mxfp4Config- load_kwargs["quantization_config"] = Mxfp4Config(dequantized=True)- logger.info("Detected MXFP4 quantized model, using Mxfp4Config(dequantized=True) for loading.")+ if version.parse(transformers.__version__) >= version.parse("5.0.0"):+ is_mxfp4 = _is_mxfp4_model(pretrained_model_name_or_path, trust_remote_code=trust_remote_code)+ if is_mxfp4:+ from transformers import Mxfp4Config++ load_kwargs["quantization_config"] = Mxfp4Config(dequantized=True)+ logger.info("Detected MXFP4 quantized model, using Mxfp4Config(dequantized=True) for loading.")is_glm = bool(re.search("chatglm", pretrained_model_name_or_path.lower()))diff --git a/test/test_cpu/export/test_export.py b/test/test_cpu/export/test_export.pyindex f0535ddd3..a2212fbc6 100644--- a/test/test_cpu/export/test_export.py+++ b/test/test_cpu/export/test_export.py@@ -323,13 +323,9 @@ class TestAutoRound:shutil.rmtree(quantized_model_path, ignore_errors=True)- @pytest.mark.skipif(- transformers_version >= version.parse("5.0"),- reason="PhiConfig missing pad_token_id, https://github.com/huggingface/transformers/pull/43453",- )def test_awq_lmhead_export(self, dataloader):bits, sym, group_size = 4, False, 128- model_name = get_model_path("microsoft/phi-2")+ model_name = get_model_path("microsoft/phi-4")layer_config = {"lm_head": {"bits": 4}, # set lm_head quant"layer": {"bits": 16},@@ -353,14 +349,10 @@ class TestAutoRound:assert isinstance(lm_head, WQLinear_GEMM), "Illegal AWQ quantization for lm_head layer"shutil.rmtree(quantized_model_path, ignore_errors=True)- @pytest.mark.skipif(- transformers_version >= version.parse("5.0"),- reason="PhiConfig missing pad_token_id, https://github.com/huggingface/transformers/pull/43453",- )def test_gptq_lmhead_export(self, dataloader):bits, sym, group_size = 4, True, 128# Note that, to save UT tuning time, the local model is intentionally kept lightweight, using only 2 hidden layers.- model_name = get_model_path("microsoft/phi-2")+ model_name = get_model_path("microsoft/phi-4")layer_config = {"lm_head": {"bits": 4}, # set lm_head quant"layer": {"bits": 16},diff --git a/test/test_cpu/models/test_moe_model.py b/test/test_cpu/models/test_moe_model.pyindex ffadaf4f9..66d20da91 100644--- a/test/test_cpu/models/test_moe_model.py+++ b/test/test_cpu/models/test_moe_model.py@@ -1,6 +1,8 @@import shutilimport pytest+import torch+import torch.nn as nnfrom packaging import versionfrom transformers import AutoConfig, AutoProcessor, AutoTokenizer, Llama4ForConditionalGenerationfrom transformers.models.gpt_oss.modeling_gpt_oss import GptOssForCausalLM@@ -62,7 +64,7 @@ def setup_qwen3_vl_moe():return model, tokenizer, processor, output_dir, config-def quantize_model(model, tokenizer, output_dir, scheme, iters=0):+def quantize_model(model, tokenizer, output_dir, scheme, iters=0, ignore_layers="self_attn,router,lm_head,mlp.gate"):"""Helper function to quantize the model with the given scheme."""autoround = AutoRound(model,@@ -71,7 +73,7 @@ def quantize_model(model, tokenizer, output_dir, scheme, iters=0):nsamples=2,iters=iters,seqlen=32,- ignore_layers="self_attn,router,lm_head,mlp.gate",+ ignore_layers=ignore_layers,)quantized_model, save_folder = autoround.quantize_and_save(format="auto_round", output_dir=output_dir)return quantized_model@@ -98,7 +100,7 @@ def test_gptoss(setup_gpt_oss, scheme):# Remove it to avoid mismatch during quantized model loadingdelattr(model.config, "layer_types")- quantized_model = quantize_model(model, tokenizer, output_dir, scheme)+ quantized_model = quantize_model(model, tokenizer, output_dir, scheme, ignore_layers="self_attn,lm_head")# Ensure the quantized model is not Noneassert quantized_model is not None, "Quantized model should not be None."@@ -136,7 +138,7 @@ def test_llama4(setup_llama4):delattr(model.config.text_config, "moe_layers")delattr(model.config.text_config, "layer_types")- quantized_model = quantize_model(model, tokenizer, output_dir, "MXFP4")+ quantized_model = quantize_model(model, tokenizer, output_dir, "MXFP4", ignore_layers="self_attn,router,lm_head")# Ensure the quantized model is not Noneassert quantized_model is not None, "Quantized model should not be None."@@ -177,3 +179,64 @@ def test_qwen3_vl_moe_mxfp(setup_qwen3_vl_moe):print(tokenizer.decode(loaded_model.generate(**inputs, max_new_tokens=50)[0]))# clean the output directory after testshutil.rmtree(output_dir, ignore_errors=True)+++class _FakeMoELinear(nn.Module):+ """Mimics the fused MoELinear used in Step3p5MoEMLP."""++ def __init__(self, num_experts, in_features, out_features):+ super().__init__()+ self.num_experts = num_experts+ self.in_features = in_features+ self.out_features = out_features+ self.weight = nn.Parameter(torch.randn(num_experts, out_features, in_features))+++class Step3p5MoEMLP(nn.Module):+ """Mock of the original Step3p5MoEMLP matched by class name."""++ def __init__(self, hidden_size=32, intermediate_size=64, num_experts=4, top_k=2):+ super().__init__()+ self.hidden_size = hidden_size+ self.moe_intermediate_size = intermediate_size+ self.num_experts = num_experts+ self.top_k = top_k+ self.limit = None+ self.need_fp32_gate = False+ self.routed_scaling_factor = 1.0+ self.use_moe_router_bias = False+ self.custom_routing_function = None+ self.gate = nn.Linear(hidden_size, num_experts, bias=False)+ self.gate_proj = _FakeMoELinear(num_experts, hidden_size, intermediate_size)+ self.up_proj = _FakeMoELinear(num_experts, hidden_size, intermediate_size)+ self.down_proj = _FakeMoELinear(num_experts, intermediate_size, hidden_size)+++def test_step3p5_moe_replacement():+ """Verify that apply_replacements swaps Step3p5MoEMLP → LinearStep3p5MoEMLP."""+ from auto_round.modeling.fused_moe.replace_modules import apply_replacements, materialize_model_+ from auto_round.modeling.fused_moe.step3_5_moe import LinearStep3p5MoEMLP, Step3p5ExpertMLP++ num_experts, hidden_size = 4, 32+ moe = Step3p5MoEMLP(hidden_size=hidden_size, num_experts=num_experts)+ # wrap in a model with config.model_type so the registry can find it+ model = nn.Module()+ model.config = type("Cfg", (), {"model_type": "step3p5"})()+ model.moe = moe+ model.add_module("moe", moe)++ orig_weights = {k: moe.get_parameter(k).clone() for k in ("gate_proj.weight", "up_proj.weight", "down_proj.weight")}++ apply_replacements(model, auto_detect_moe=False)+ assert isinstance(model.moe, LinearStep3p5MoEMLP)+ assert count_modules_by_type(model, Step3p5ExpertMLP) == num_experts++ materialize_model_(model)+ for i, expert in enumerate(model.moe.experts):+ for proj in ("gate_proj", "up_proj", "down_proj"):+ w = getattr(expert, proj).weight+ assert w.device.type != "meta"+ assert torch.equal(w, orig_weights[f"{proj}.weight"][i])++ out = model.moe(torch.randn(1, 4, hidden_size))+ assert out.shape == (1, 4, hidden_size)diff --git a/test/test_cpu/utils/test_cli_usage.py b/test/test_cpu/utils/test_cli_usage.pyindex f57e9c102..e444524ae 100644--- a/test/test_cpu/utils/test_cli_usage.py+++ b/test/test_cpu/utils/test_cli_usage.py@@ -2,6 +2,8 @@ import osimport shutilimport sys+from auto_round.utils import parse_layer_config_arg+from ...helpers import get_model_pathAUTO_ROUND_PATH = __file__.split("/")@@ -71,3 +73,24 @@ class TestAutoRoundCmd:)if res > 0 or res == -1:assert False, "cmd line test fail, please have a check"++ def test_layer_config(self, tiny_opt_model_path):+ """Test --layer_config with unquoted JSON-like syntax."""+ python_path = sys.executable+ layer_cfg = r"{fc1:{bits:8,data_type:int},fc2:{bits:16,data_type:int}}"+ res = os.system(+ f"PYTHONPATH='{AUTO_ROUND_PATH}:$PYTHONPATH' {python_path} -m auto_round"+ f" --model {tiny_opt_model_path} --seqlen 8 --iter 0 --disable_opt_rtn"+ f" --layer_config '{layer_cfg}' --format auto_round --output_dir ./saved"+ )+ if res > 0 or res == -1:+ assert False, "cmd line test with --layer_config fail, please have a check"+++def test_parse_layer_config():+ """Unit test for parse_layer_config_arg parsing logic."""+ result = parse_layer_config_arg("{mtp:{bits:8,data_type:int},mtp.fc:{bits:16,data_type:int}}")+ assert result == {+ "mtp": {"bits": 8, "data_type": "int"},+ "mtp.fc": {"bits": 16, "data_type": "int"},+ }diff --git a/test/test_cpu/utils/test_missing_tensors.py b/test/test_cpu/utils/test_missing_tensors.pynew file mode 100644index 000000000..1461533db--- /dev/null+++ b/test/test_cpu/utils/test_missing_tensors.py@@ -0,0 +1,604 @@+# Copyright (c) 2025 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++import json+import os+import tempfile+import unittest++import torch++from auto_round.utils.missing_tensors import (+ _get_woq_config_from_dir,+ copy_missing_tensors_from_source,+ quantize_weight_rtn,+)+++def _save_safetensors(tensors: dict, path: str) -> None:+ from safetensors.torch import save_file++ save_file(tensors, path)+++def _load_safetensors(path: str) -> dict:+ from safetensors import safe_open++ result = {}+ with safe_open(path, framework="pt", device="cpu") as f:+ for key in f.keys():+ result[key] = f.get_tensor(key)+ return result+++def _write_config(target_dir: str, qcfg: dict | None = None) -> None:+ config = {}+ if qcfg is not None:+ config["quantization_config"] = qcfg+ with open(os.path.join(target_dir, "config.json"), "w") as f:+ json.dump(config, f)+++class TestQuantizeWeightRtn(unittest.TestCase):+ """Tests for the standalone quantize_weight_rtn helper."""++ def test_output_shapes_4bit_sym(self):+ """Check that output tensors have the expected shapes for 4-bit symmetric quantization."""+ out_features, in_features = 32, 64+ bits, group_size = 4, 32+ weight = torch.randn(out_features, in_features)++ pack_factor = 32 // bits # 8+ num_groups = in_features // group_size # 2++ qweight, qzeros, scales = quantize_weight_rtn(weight, bits=bits, group_size=group_size, sym=True)++ self.assertEqual(qweight.shape, (in_features // pack_factor, out_features))+ self.assertEqual(qzeros.shape, (num_groups, out_features // pack_factor))+ self.assertEqual(scales.shape, (num_groups, out_features))+ self.assertEqual(qweight.dtype, torch.int32)+ self.assertEqual(qzeros.dtype, torch.int32)+ self.assertEqual(scales.dtype, torch.float16)++ def test_output_shapes_4bit_asym(self):+ """Check output shapes for 4-bit asymmetric quantization."""+ out_features, in_features = 32, 64+ bits, group_size = 4, 32+ weight = torch.randn(out_features, in_features)++ pack_factor = 32 // bits+ num_groups = in_features // group_size++ qweight, qzeros, scales = quantize_weight_rtn(weight, bits=bits, group_size=group_size, sym=False)++ self.assertEqual(qweight.shape, (in_features // pack_factor, out_features))+ self.assertEqual(qzeros.shape, (num_groups, out_features // pack_factor))+ self.assertEqual(scales.shape, (num_groups, out_features))++ def test_output_shapes_8bit(self):+ """Check output shapes for 8-bit symmetric quantization."""+ out_features, in_features = 16, 128+ bits, group_size = 8, 128+ weight = torch.randn(out_features, in_features)++ pack_factor = 32 // bits # 4+ num_groups = in_features // group_size # 1++ qweight, qzeros, scales = quantize_weight_rtn(weight, bits=bits, group_size=group_size, sym=True)++ self.assertEqual(qweight.shape, (in_features // pack_factor, out_features))+ self.assertEqual(qzeros.shape, (num_groups, out_features // pack_factor))+ self.assertEqual(scales.shape, (num_groups, out_features))++ def test_outputs_are_on_cpu(self):+ """Verify that all returned tensors are on the CPU regardless of input device."""+ weight = torch.randn(16, 64)+ qweight, qzeros, scales = quantize_weight_rtn(weight, bits=4, group_size=32)+ self.assertEqual(qweight.device.type, "cpu")+ self.assertEqual(qzeros.device.type, "cpu")+ self.assertEqual(scales.device.type, "cpu")++ def test_raises_on_1d_weight(self):+ """quantize_weight_rtn must reject 1-D tensors."""+ with self.assertRaises(AssertionError):+ quantize_weight_rtn(torch.randn(64), bits=4, group_size=32)++ def test_in_features_padding(self):+ """in_features that is not a multiple of group_size should be padded internally."""+ # in_features=70 is not a multiple of group_size=32+ out_features, in_features = 16, 70+ bits, group_size = 4, 32+ weight = torch.randn(out_features, in_features)+ # padded in_features = 96 (next multiple of 32)+ padded_in = 96+ pack_factor = 32 // bits++ qweight, qzeros, scales = quantize_weight_rtn(weight, bits=bits, group_size=group_size)+ self.assertEqual(qweight.shape, (padded_in // pack_factor, out_features))+++class TestGetWoqConfigFromDir(unittest.TestCase):+ """Tests for _get_woq_config_from_dir."""++ def test_returns_none_when_no_config(self):+ with tempfile.TemporaryDirectory() as d:+ self.assertIsNone(_get_woq_config_from_dir(d))++ def test_returns_none_for_wrong_quant_method(self):+ with tempfile.TemporaryDirectory() as d:+ _write_config(+ d,+ {+ "quant_method": "gptq",+ "packing_format": "auto_round:auto_gptq",+ "bits": 4,+ "group_size": 128,+ "sym": True,+ },+ )+ self.assertIsNone(_get_woq_config_from_dir(d))++ def test_returns_none_for_wrong_packing_format(self):+ with tempfile.TemporaryDirectory() as d:+ _write_config(+ d,+ {+ "quant_method": "auto-round",+ "packing_format": "other_format",+ "bits": 4,+ "group_size": 128,+ "sym": True,+ },+ )+ self.assertIsNone(_get_woq_config_from_dir(d))++ def test_returns_none_for_missing_bits(self):+ with tempfile.TemporaryDirectory() as d:+ _write_config(+ d,+ {+ "quant_method": "auto-round",+ "packing_format": "auto_round:auto_gptq",+ "group_size": 128,+ "sym": True,+ },+ )+ self.assertIsNone(_get_woq_config_from_dir(d))++ def test_returns_none_for_bits_greater_than_8(self):+ with tempfile.TemporaryDirectory() as d:+ _write_config(+ d,+ {+ "quant_method": "auto-round",+ "packing_format": "auto_round:auto_gptq",+ "bits": 16,+ "group_size": 128,+ "sym": True,+ },+ )+ self.assertIsNone(_get_woq_config_from_dir(d))++ def test_valid_config_returns_required_fields(self):+ """A fully valid config returns a dict with bits, group_size, sym,+ block_name_to_quantize, and extra_config."""+ with tempfile.TemporaryDirectory() as d:+ _write_config(+ d,+ {+ "quant_method": "auto-round",+ "packing_format": "auto_round:auto_gptq",+ "bits": 4,+ "group_size": 128,+ "sym": True,+ },+ )+ cfg = _get_woq_config_from_dir(d)+ self.assertIsNotNone(cfg)+ self.assertEqual(cfg["bits"], 4)+ self.assertEqual(cfg["group_size"], 128)+ self.assertTrue(cfg["sym"])+ # New fields must also be present+ self.assertIn("block_name_to_quantize", cfg)+ self.assertIn("extra_config", cfg)++ def test_valid_config_with_extra_config_and_block_name(self):+ """extra_config and block_name_to_quantize from config.json are propagated."""+ with tempfile.TemporaryDirectory() as d:+ _write_config(+ d,+ {+ "quant_method": "auto-round",+ "packing_format": "auto_round:auto_gptq",+ "bits": 4,+ "group_size": 128,+ "sym": True,+ "block_name_to_quantize": ["model.layers"],+ "extra_config": {"mtp.0.fc": {"bits": 8}},+ },+ )+ cfg = _get_woq_config_from_dir(d)+ self.assertIsNotNone(cfg)+ self.assertEqual(cfg["block_name_to_quantize"], ["model.layers"])+ self.assertEqual(cfg["extra_config"], {"mtp.0.fc": {"bits": 8}})+++class TestCopyMissingTensorsFromSource(unittest.TestCase):+ """End-to-end tests for copy_missing_tensors_from_source.++ Detection is now purely name-based (parent-layer + block-prefix comparison),+ so no model loading or mocking is required.++ A source tensor T is "missing" when ALL of:+ 1. T is not already present in the saved output,+ 2. T's parent layer (rsplit('.', 1)[0]) is absent from the saved parent layers,+ 3. T's first numeric block-prefix (e.g. 'mtp.0') is absent from the saved+ block-prefixes — OR T has no numeric segment at all.+ Special case: 'lm_head.weight' is always excluded.+ """++ def _make_auto_round_config(self, bits=4, group_size=128, sym=True) -> dict:+ return {+ "quantization_config": {+ "quant_method": "auto-round",+ "packing_format": "auto_round:auto_gptq",+ "bits": bits,+ "group_size": group_size,+ "sym": sym,+ }+ }++ # ------------------------------------------------------------------ #+ # Detection correctness #+ # ------------------------------------------------------------------ #++ def test_plain_copy_of_missing_mtp_tensor(self):+ """Tensors from blocks entirely absent in the saved output are copied verbatim."""+ with tempfile.TemporaryDirectory() as source_dir, tempfile.TemporaryDirectory() as target_dir:+ mtp_norm = torch.randn(64)+ _save_safetensors({"mtp.0.norm.weight": mtp_norm}, os.path.join(source_dir, "model.safetensors"))++ _save_safetensors(+ {"model.embed_tokens.weight": torch.randn(32, 64)},+ os.path.join(target_dir, "model.safetensors"),+ )+ _write_config(target_dir)++ copy_missing_tensors_from_source(source_dir, target_dir)++ extra_shard = os.path.join(target_dir, "model_extra_tensors.safetensors")+ self.assertTrue(os.path.exists(extra_shard), "Extra shard should be created")++ result = _load_safetensors(extra_shard)+ self.assertIn("mtp.0.norm.weight", result)+ torch.testing.assert_close(result["mtp.0.norm.weight"], mtp_norm)++ def test_tensor_with_known_parent_layer_not_copied(self):+ """Source tensor whose parent layer already has tensors in the target is not missing.++ Scenario: source has the original '.weight'; target has the packed '.qweight'+ under the same parent layer — the layer was processed, not skipped.+ """+ with tempfile.TemporaryDirectory() as source_dir, tempfile.TemporaryDirectory() as target_dir:+ _save_safetensors(+ {"model.layers.0.self_attn.q_proj.weight": torch.randn(32, 64)},+ os.path.join(source_dir, "model.safetensors"),+ )+ # Target has the quantised form — same parent layer, different suffix+ _save_safetensors(+ {"model.layers.0.self_attn.q_proj.qweight": torch.randint(0, 2**31, (8, 32), dtype=torch.int32)},+ os.path.join(target_dir, "model.safetensors"),+ )+ _write_config(target_dir)++ copy_missing_tensors_from_source(source_dir, target_dir)++ extra_shard = os.path.join(target_dir, "model_extra_tensors.safetensors")+ self.assertFalse(os.path.exists(extra_shard), "No extra shard expected: parent layer is known")++ def test_tensor_with_known_block_prefix_not_copied(self):+ """Source tensor whose numeric block-prefix exists in the saved output is not missing.++ Scenario: source has an extra gate weight inside 'model.layers.0'; the target+ already contains other tensors from 'model.layers.0' — the block was processed.+ """+ with tempfile.TemporaryDirectory() as source_dir, tempfile.TemporaryDirectory() as target_dir:+ _save_safetensors(+ {"model.layers.0.mlp.extra_gate.weight": torch.randn(32, 64)},+ os.path.join(source_dir, "model.safetensors"),+ )+ # Target has a different sub-layer inside the same block+ _save_safetensors(+ {"model.layers.0.mlp.gate_proj.qweight": torch.randint(0, 2**31, (8, 32), dtype=torch.int32)},+ os.path.join(target_dir, "model.safetensors"),+ )+ _write_config(target_dir)++ copy_missing_tensors_from_source(source_dir, target_dir)++ extra_shard = os.path.join(target_dir, "model_extra_tensors.safetensors")+ self.assertFalse(os.path.exists(extra_shard), "No extra shard expected: block prefix is known")++ def test_lm_head_weight_is_never_copied(self):+ """lm_head.weight is always excluded, even when it is absent from the target."""+ with tempfile.TemporaryDirectory() as source_dir, tempfile.TemporaryDirectory() as target_dir:+ _save_safetensors(+ {"lm_head.weight": torch.randn(32, 64)},+ os.path.join(source_dir, "model.safetensors"),+ )+ _save_safetensors(+ {"model.embed_tokens.weight": torch.randn(32, 64)},+ os.path.join(target_dir, "model.safetensors"),+ )+ _write_config(target_dir)++ copy_missing_tensors_from_source(source_dir, target_dir)++ extra_shard = os.path.join(target_dir, "model_extra_tensors.safetensors")+ self.assertFalse(os.path.exists(extra_shard), "lm_head.weight must never be copied")++ def test_already_saved_tensors_are_not_duplicated(self):+ """Tensors that are already present in the target are not written again."""+ with tempfile.TemporaryDirectory() as source_dir, tempfile.TemporaryDirectory() as target_dir:+ mtp_tensor = torch.randn(64)+ _save_safetensors(+ {"mtp.0.shared_head.norm.weight": mtp_tensor},+ os.path.join(source_dir, "model.safetensors"),+ )+ # Target already has exactly this tensor+ _save_safetensors(+ {"mtp.0.shared_head.norm.weight": mtp_tensor},+ os.path.join(target_dir, "model.safetensors"),+ )+ _write_config(target_dir)++ copy_missing_tensors_from_source(source_dir, target_dir)++ extra_shard = os.path.join(target_dir, "model_extra_tensors.safetensors")+ self.assertFalse(os.path.exists(extra_shard), "No extra shard expected: tensor already present")++ def test_detects_multiple_missing_tensors_from_different_blocks(self):+ """All source tensors from blocks absent in the saved output are copied."""+ with tempfile.TemporaryDirectory() as source_dir, tempfile.TemporaryDirectory() as target_dir:+ _save_safetensors(+ {+ "custom_block.0.norm.weight": torch.randn(64),+ "mtp.0.norm.weight": torch.randn(64),+ "model.embed_tokens.weight": torch.randn(8, 64),+ },+ os.path.join(source_dir, "model.safetensors"),+ )+ _save_safetensors(+ {"model.embed_tokens.weight": torch.randn(8, 64)},+ os.path.join(target_dir, "model.safetensors"),+ )+ _write_config(target_dir)++ copy_missing_tensors_from_source(source_dir, target_dir)++ extra_shard = os.path.join(target_dir, "model_extra_tensors.safetensors")+ self.assertTrue(os.path.exists(extra_shard))++ result = _load_safetensors(extra_shard)+ self.assertIn("custom_block.0.norm.weight", result)+ self.assertIn("mtp.0.norm.weight", result)+ # Model-known tensor must NOT be in the extra shard+ self.assertNotIn("model.embed_tokens.weight", result)++ # ------------------------------------------------------------------ #+ # FP8 dequantization #+ # ------------------------------------------------------------------ #++ def test_fp8_weight_is_dequantized_to_bf16(self):+ """FP8 weights paired with weight_scale_inv are dequantized to BF16 before saving."""+ with tempfile.TemporaryDirectory() as source_dir, tempfile.TemporaryDirectory() as target_dir:+ fp8_weight = torch.randn(64, 128).to(torch.float8_e4m3fn)+ scale_inv = torch.tensor(0.5)++ _save_safetensors(+ {+ "mtp.0.ffn.weight": fp8_weight,+ "mtp.0.ffn.weight_scale_inv": scale_inv,+ },+ os.path.join(source_dir, "model.safetensors"),+ )+ _save_safetensors(+ {"model.embed_tokens.weight": torch.randn(8, 64)},+ os.path.join(target_dir, "model.safetensors"),+ )+ _write_config(target_dir)++ copy_missing_tensors_from_source(source_dir, target_dir)++ extra_shard = os.path.join(target_dir, "model_extra_tensors.safetensors")+ self.assertTrue(os.path.exists(extra_shard))++ result = _load_safetensors(extra_shard)+ self.assertIn("mtp.0.ffn.weight", result)+ # Scale tensor must be dropped after dequantization+ self.assertNotIn("mtp.0.ffn.weight_scale_inv", result)+ self.assertEqual(result["mtp.0.ffn.weight"].dtype, torch.bfloat16)++ # ------------------------------------------------------------------ #+ # WOQ quantization #+ # ------------------------------------------------------------------ #++ def test_woq_quantization_replaces_weight_with_packed_tensors(self):+ """2-D MTP weight tensors are quantized and packed when an auto-round WOQ config is present."""+ out_features, in_features = 32, 128+ with tempfile.TemporaryDirectory() as source_dir, tempfile.TemporaryDirectory() as target_dir:+ mtp_weight = torch.randn(out_features, in_features)+ _save_safetensors(+ {"mtp.0.ffn.weight": mtp_weight},+ os.path.join(source_dir, "model.safetensors"),+ )+ _save_safetensors(+ {"model.embed_tokens.weight": torch.randn(8, 64)},+ os.path.join(target_dir, "model.safetensors"),+ )+ with open(os.path.join(target_dir, "config.json"), "w") as f:+ json.dump(self._make_auto_round_config(bits=4, group_size=128, sym=True), f)++ copy_missing_tensors_from_source(source_dir, target_dir)++ extra_shard = os.path.join(target_dir, "model_extra_tensors.safetensors")+ self.assertTrue(os.path.exists(extra_shard))++ result = _load_safetensors(extra_shard)+ # Original weight must be replaced by the three packed tensors+ self.assertNotIn("mtp.0.ffn.weight", result)+ self.assertIn("mtp.0.ffn.qweight", result)+ self.assertIn("mtp.0.ffn.qzeros", result)+ self.assertIn("mtp.0.ffn.scales", result)++ def test_block_name_to_ignore_not_quantized_in_woq_mode(self):+ """Weights matching BLOCK_NAME_TO_IGNORE patterns are copied as-is, not quantized."""+ out_features, in_features = 32, 128+ with tempfile.TemporaryDirectory() as source_dir, tempfile.TemporaryDirectory() as target_dir:+ gate_weight = torch.randn(out_features, in_features)+ # "mlp.gate." is in BLOCK_NAME_TO_IGNORE+ _save_safetensors(+ {"mtp.0.mlp.gate.weight": gate_weight},+ os.path.join(source_dir, "model.safetensors"),+ )+ _save_safetensors(+ {"model.embed_tokens.weight": torch.randn(8, 64)},+ os.path.join(target_dir, "model.safetensors"),+ )+ with open(os.path.join(target_dir, "config.json"), "w") as f:+ json.dump(self._make_auto_round_config(bits=4, group_size=128, sym=True), f)++ copy_missing_tensors_from_source(source_dir, target_dir)++ extra_shard = os.path.join(target_dir, "model_extra_tensors.safetensors")+ self.assertTrue(os.path.exists(extra_shard))++ result = _load_safetensors(extra_shard)+ # Weight must be present as-is — not packed+ self.assertIn("mtp.0.mlp.gate.weight", result)+ self.assertNotIn("mtp.0.mlp.gate.qweight", result)++ def test_config_updated_with_block_name_to_quantize_after_woq(self):+ """After WOQ, config.json is updated so the new block appears in block_name_to_quantize."""+ out_features, in_features = 32, 128+ with tempfile.TemporaryDirectory() as source_dir, tempfile.TemporaryDirectory() as target_dir:+ _save_safetensors(+ {"mtp.0.ffn.weight": torch.randn(out_features, in_features)},+ os.path.join(source_dir, "model.safetensors"),+ )+ _save_safetensors(+ {"model.embed_tokens.weight": torch.randn(8, 64)},+ os.path.join(target_dir, "model.safetensors"),+ )+ with open(os.path.join(target_dir, "config.json"), "w") as f:+ json.dump(self._make_auto_round_config(bits=4, group_size=128, sym=True), f)++ copy_missing_tensors_from_source(source_dir, target_dir)++ with open(os.path.join(target_dir, "config.json")) as f:+ updated_cfg = json.load(f)++ qcfg = updated_cfg.get("quantization_config", {})+ block_names = qcfg.get("block_name_to_quantize", [])+ self.assertTrue(+ any("mtp" in b for b in block_names),+ f"Expected an 'mtp' block in block_name_to_quantize, got: {block_names}",+ )++ def test_config_updated_with_extra_config_for_unquantized_2d_weight(self):+ """When a 2-D weight is skipped by BLOCK_NAME_TO_IGNORE, extra_config is updated+ to mark it as full-precision so the inference stack knows not to dequantize it."""+ out_features, in_features = 32, 128+ with tempfile.TemporaryDirectory() as source_dir, tempfile.TemporaryDirectory() as target_dir:+ # "mlp.gate." is in BLOCK_NAME_TO_IGNORE → not quantized → should go to extra_config+ _save_safetensors(+ {"mtp.0.mlp.gate.weight": torch.randn(out_features, in_features)},+ os.path.join(source_dir, "model.safetensors"),+ )+ _save_safetensors(+ {"model.embed_tokens.weight": torch.randn(8, 64)},+ os.path.join(target_dir, "model.safetensors"),+ )+ with open(os.path.join(target_dir, "config.json"), "w") as f:+ json.dump(self._make_auto_round_config(bits=4, group_size=128, sym=True), f)++ copy_missing_tensors_from_source(source_dir, target_dir)++ with open(os.path.join(target_dir, "config.json")) as f:+ updated_cfg = json.load(f)++ extra_config = updated_cfg.get("quantization_config", {}).get("extra_config", {})+ self.assertIn("mtp.0.mlp.gate", extra_config)+ layer_cfg = extra_config["mtp.0.mlp.gate"]+ self.assertEqual(layer_cfg.get("bits"), 16)+ self.assertEqual(layer_cfg.get("data_type"), "fp")++ # ------------------------------------------------------------------ #+ # Index file handling #+ # ------------------------------------------------------------------ #++ def test_index_json_is_updated_for_sharded_target(self):+ """When target uses a sharded index, copied tensor names are added to weight_map."""+ with tempfile.TemporaryDirectory() as source_dir, tempfile.TemporaryDirectory() as target_dir:+ mtp_norm = torch.randn(64)+ _save_safetensors({"mtp.0.norm.weight": mtp_norm}, os.path.join(source_dir, "model.safetensors"))++ saved_shard = "model-00001-of-00001.safetensors"+ _save_safetensors(+ {"model.embed_tokens.weight": torch.randn(8, 64)},+ os.path.join(target_dir, saved_shard),+ )+ index = {"metadata": {"total_size": 0}, "weight_map": {"model.embed_tokens.weight": saved_shard}}+ with open(os.path.join(target_dir, "model.safetensors.index.json"), "w") as f:+ json.dump(index, f)+ _write_config(target_dir)++ copy_missing_tensors_from_source(source_dir, target_dir)++ with open(os.path.join(target_dir, "model.safetensors.index.json")) as f:+ updated_index = json.load(f)++ self.assertIn("mtp.0.norm.weight", updated_index["weight_map"])+ self.assertEqual(updated_index["weight_map"]["mtp.0.norm.weight"], "model_extra_tensors.safetensors")++ def test_index_json_is_created_for_single_file_target(self):+ """When the target uses a single safetensors file and tensors are copied, a new+ model.safetensors.index.json is created mapping all tensors to their shards."""+ with tempfile.TemporaryDirectory() as source_dir, tempfile.TemporaryDirectory() as target_dir:+ mtp_norm = torch.randn(64)+ _save_safetensors({"mtp.0.norm.weight": mtp_norm}, os.path.join(source_dir, "model.safetensors"))++ _save_safetensors(+ {"model.embed_tokens.weight": torch.randn(8, 64)},+ os.path.join(target_dir, "model.safetensors"),+ )+ _write_config(target_dir)++ copy_missing_tensors_from_source(source_dir, target_dir)++ index_path = os.path.join(target_dir, "model.safetensors.index.json")+ self.assertTrue(os.path.exists(index_path), "An index.json should be created for single-file targets")++ with open(index_path) as f:+ index = json.load(f)++ weight_map = index["weight_map"]+ self.assertIn("mtp.0.norm.weight", weight_map)+ self.assertIn("model.embed_tokens.weight", weight_map)+ self.assertEqual(weight_map["mtp.0.norm.weight"], "model_extra_tensors.safetensors")+ self.assertEqual(weight_map["model.embed_tokens.weight"], "model.safetensors")+++if __name__ == "__main__":+ unittest.main()diff --git a/test/test_cuda/models/test_moe_model.py b/test/test_cuda/models/test_moe_model.pyindex fa9a91fae..006dc78e6 100644--- a/test/test_cuda/models/test_moe_model.py+++ b/test/test_cuda/models/test_moe_model.py@@ -59,7 +59,7 @@ def setup_qwen3_vl_moe():return model, tokenizer, processor, output_dir, config-def quantize_model(model, tokenizer, output_dir, scheme, iters=0):+def quantize_model(model, tokenizer, output_dir, scheme, iters=0, ignore_layers="self_attn,router,lm_head,mlp.gate"):"""Helper function to quantize the model with the given scheme."""autoround = AutoRound(model,@@ -68,7 +68,7 @@ def quantize_model(model, tokenizer, output_dir, scheme, iters=0):nsamples=2,iters=iters,seqlen=32,- ignore_layers="self_attn,router,lm_head,mlp.gate",+ ignore_layers=ignore_layers,)quantized_model, save_folder = autoround.quantize_and_save(format="auto_round", output_dir=output_dir)return quantized_model@@ -81,7 +81,7 @@ def test_gptoss(setup_gpt_oss):# Remove it to avoid mismatch during quantized model loadingdelattr(model.config, "layer_types")- quantized_model = quantize_model(model, tokenizer, output_dir, "MXFP4")+ quantized_model = quantize_model(model, tokenizer, output_dir, "MXFP4", ignore_layers="self_attn,lm_head")# Ensure the quantized model is not Noneassert quantized_model is not None, "Quantized model should not be None."@@ -111,7 +111,7 @@ def test_llama4(setup_llama4):delattr(model.config.text_config, "moe_layers")delattr(model.config.text_config, "layer_types")- quantized_model = quantize_model(model, tokenizer, output_dir, "MXFP4")+ quantized_model = quantize_model(model, tokenizer, output_dir, "MXFP4", ignore_layers="self_attn,router,lm_head")# Ensure the quantized model is not Noneassert quantized_model is not None, "Quantized model should not be None."