PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
Fix Qwen Omni quantization model issue for long form audio generation (#1698)
Liang Lvb40340734c80
diff --git a/auto_round/compressors/shard_writer.py b/auto_round/compressors/shard_writer.pyindex b25c26739..d4443b080 100644--- a/auto_round/compressors/shard_writer.py+++ b/auto_round/compressors/shard_writer.py@@ -159,11 +159,42 @@ class ShardWriter:param_name = f"{prefix}.{k}"self._add_tensor(param_name, v)- def _add_tensor(self, name: str, tensor: torch.Tensor):+ def _expand_fused_experts(self, name: str, tensor: torch.Tensor) -> list[tuple[str, torch.Tensor]] | None:+ """Expand a fused 3D expert parameter into per-expert 2D weight tensors.- # transformers will handle _checkpoint_conversion_mapping automatically if is_immediate_saving=False- name = revert_checkpoint_conversion_mapping(name, self.reverse_checkpoint_conversion_mapping)+ Non-quantized MoE modules (e.g. thinker in Qwen3-Omni) are kept fused+ during quantization to prevent memory eviction under disk offloading.+ This method converts them to the per-expert checkpoint format at save time.++ Talker MoE modules stay fused during quantization, but save_pretrained+ still needs concrete per-expert 2D checkpoint keys. Saving the original+ fused tensor under a reverse-mapped wildcard key (for example+ ``experts.*.gate_proj.weight``) breaks reload.++ Returns:+ List of (key, 2D tensor) pairs, or None if not a fused expert param.+ """+ from auto_round.modeling.fused_moe.replace_modules import MOE_SKIP_PREFIXES+ from auto_round.utils.missing_tensors import split_fused_expert_tensors++ parts = name.rsplit(".", 1)+ if len(parts) != 2:+ return None+ prefix, attr_name = parts++ model_type = getattr(getattr(self.model, "config", None), "model_type", None)+ skip_prefixes = MOE_SKIP_PREFIXES.get(model_type, []) if model_type is not None else []+ if not any(+ prefix == skip_prefix.rstrip(".") or prefix.startswith(skip_prefix) for skip_prefix in skip_prefixes+ ):+ return None++ expanded = split_fused_expert_tensors({name: tensor})+ if set(expanded) == {name}:+ return None+ return list(expanded.items())+ def _add_tensor(self, name: str, tensor: torch.Tensor):if isinstance(tensor, torch.Tensor) and tensor.device.type == "meta":self.skipped_meta_tensors.append(name)return@@ -172,6 +203,18 @@ class ShardWriter:if name in self._all_saved or name in self.current_shard_tensors:return+ # Expand fused 3D expert parameters into per-expert 2D tensors if necessary+ if tensor.dim() == 3:+ expanded = self._expand_fused_experts(name, tensor)+ if expanded is not None:+ self._all_saved.add(name)+ for sub_name, sub_tensor in expanded:+ self._add_tensor(sub_name, sub_tensor)+ return++ # transformers will handle _checkpoint_conversion_mapping automatically if is_immediate_saving=False+ name = revert_checkpoint_conversion_mapping(name, self.reverse_checkpoint_conversion_mapping)+t_size = tensor.nbytesself.total_param_elems += tensor.numel()self.total_param_size_bytes += t_sizediff --git a/auto_round/modeling/fused_moe/moe_experts_interface.py b/auto_round/modeling/fused_moe/moe_experts_interface.pyindex 854d9e186..f6e901699 100644--- a/auto_round/modeling/fused_moe/moe_experts_interface.py+++ b/auto_round/modeling/fused_moe/moe_experts_interface.py@@ -608,7 +608,11 @@ def _unfuse_experts_weights_inplace(return True-def prepare_model_for_moe_quantization(model: nn.Module, implementation: str = LINEAR_LOOP_IMPL) -> list[str]:+def prepare_model_for_moe_quantization(+ model: nn.Module,+ implementation: str = LINEAR_LOOP_IMPL,+ skip_prefixes: list[str] | None = None,+) -> list[str]:"""Prepare a model for MOE quantization using transformers' experts interface.This function:@@ -622,6 +626,9 @@ def prepare_model_for_moe_quantization(model: nn.Module, implementation: str = LArgs:model: The model to prepareimplementation: The experts implementation to use (default: "linear_loop")+ skip_prefixes: Module name prefixes to skip (e.g. ["talker."]).+ Modules under these prefixes stay in their original fused 3D format.+ ShardWriter handles the fused→per-expert conversion at save time.Returns:List of module names that were unfused@@ -635,6 +642,8 @@ def prepare_model_for_moe_quantization(model: nn.Module, implementation: str = L# Unfuse all fused experts modules (only those supporting @use_experts_implementation)unfused_modules = []for name, module in model.named_modules():+ if skip_prefixes and any(name.startswith(p) for p in skip_prefixes):+ continueif _unfuse_experts_weights_inplace(module):unfused_modules.append(name)logger.debug(f"[MoE Prep] Unfused '{name}'")diff --git a/auto_round/modeling/fused_moe/qwen3_omni.py b/auto_round/modeling/fused_moe/qwen3_omni.pyindex ad2004592..58731f4db 100644--- a/auto_round/modeling/fused_moe/qwen3_omni.py+++ b/auto_round/modeling/fused_moe/qwen3_omni.py@@ -1,14 +1,12 @@# Copyright (c) 2026 Intel Corporation# SPDX-License-Identifier: Apache-2.0-"""MoE module replacements for Qwen3-Omni model.+"""MoE module replacement for Qwen3-Omni thinker.Qwen3-Omni has MoE blocks in both thinker and talker:-- Thinker: Qwen3OmniMoeThinkerTextSparseMoeBlock (experts + gate, no shared expert)-- Talker: Qwen3OmniMoeTalkerTextSparseMoeBlock (experts + gate + shared_expert + shared_expert_gate)--This module provides replacement classes that unfuse fused expert weights (3D Parameters)-into individual nn.Linear layers, enabling per-expert quantization with meta device optimization.+- Thinker: Qwen3OmniMoeThinkerTextSparseMoeBlock — unfused here for quantization.+- Talker: stays in original fused format (excluded via MOE_SKIP_PREFIXES).+ ShardWriter expands talker's fused 3D params to per-expert 2D tensors at save time."""import torch@@ -80,74 +78,6 @@ class LinearQwen3OmniThinkerSparseMoeBlock(ReplacementModuleBase):# ----------------------------------------------------------------------------# Talker MoE replacement (with shared expert, same pattern as qwen3_5_moe)-# ------------------------------------------------------------------------------class LinearQwen3OmniTalkerSparseMoeBlock(ReplacementModuleBase):- """Calibration replacement for Qwen3OmniMoeTalkerTextSparseMoeBlock.-- Unfuses fused expert weights and preserves the shared_expert + shared_expert_gate.- Similar to Qwen3.5-MoE pattern.-- Structure: gate (router) + experts (unfused) + shared_expert + shared_expert_gate.- """-- def __init__(self, original, config):- super().__init__(original)- self.gate = original.gate- self.shared_expert = original.shared_expert- self.shared_expert_gate = original.shared_expert_gate- self.num_experts = original.experts.num_experts- text_config = config.talker_config.text_config- with torch.device("meta"):- self.experts = SequentialQwen3OmniTalkerExperts(text_config, original.experts)-- @classmethod- def original_module_class(cls) -> str:- return "Qwen3OmniMoeTalkerTextSparseMoeBlock"-- def _materialize_weights(self) -> None:- original = self._get_original_module()- self.experts._materialize_weights(original.experts)- clear_memory()-- def experts_forward(self, hidden_states, top_k_index, top_k_weights):- final_hidden_states = torch.zeros_like(hidden_states)- with torch.no_grad():- expert_mask = torch.nn.functional.one_hot(top_k_index, num_classes=self.num_experts)- expert_mask = expert_mask.permute(2, 1, 0)- expert_hit = torch.greater(expert_mask.sum(dim=(-1, -2)), 0).nonzero()-- for expert_idx in expert_hit:- expert_idx = expert_idx[0]- if expert_idx == self.num_experts:- continue- top_k_pos, token_idx = torch.where(expert_mask[expert_idx])- current_state = hidden_states[token_idx]- current_hidden_states = self.experts[expert_idx](current_state)- current_hidden_states = current_hidden_states * top_k_weights[token_idx, top_k_pos, None]- final_hidden_states.index_add_(0, token_idx, current_hidden_states.to(final_hidden_states.dtype))-- return final_hidden_states-- def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:- batch_size, sequence_length, hidden_dim = hidden_states.shape- hidden_states_reshaped = hidden_states.view(-1, hidden_dim)- shared_expert_output = self.shared_expert(hidden_states_reshaped)- _, routing_weights, selected_experts = self.gate(hidden_states_reshaped)- expert_output = self.experts_forward(hidden_states_reshaped, selected_experts, routing_weights)-- shared_expert_output = torch.sigmoid(self.shared_expert_gate(hidden_states_reshaped)) * shared_expert_output-- expert_output = expert_output + shared_expert_output- return expert_output.reshape(batch_size, sequence_length, hidden_dim)-- @classmethod- def from_original(cls, original, config, **kwargs):- return cls(original, config)--# ---------------------------------------------------------------------------# Sequential expert containers (unfused nn.Linear per expert)# ---------------------------------------------------------------------------@@ -193,45 +123,3 @@ class SequentialQwen3OmniThinkerExperts(torch.nn.ModuleList):del gate_up, down, gate_proj, up_projoriginal.to_empty(device="meta") # release original fused parametersclear_memory()---class SequentialQwen3OmniTalkerExperts(torch.nn.ModuleList):- """Unfused per-expert nn.Linear layers for Qwen3-Omni talker MoE.-- Replaces fused 3D Parameters (gate_up_proj, down_proj) with individual- Qwen3OmniMoeTalkerTextMLP modules per expert.- """-- def __init__(self, config, original):- super().__init__()- self.num_experts = original.gate_up_proj.shape[0]- intermediate_size = config.moe_intermediate_size-- from transformers.models.qwen3_omni_moe.modeling_qwen3_omni_moe import (- Qwen3OmniMoeTalkerTextMLP,- )-- with torch.device("meta"):- super().__init__([Qwen3OmniMoeTalkerTextMLP(config, intermediate_size) for _ in range(self.num_experts)])-- def _materialize_weights(self, original) -> None:- """Unfuse fused expert weights into individual nn.Linear layers.-- gate_up_proj shape: (num_experts, 2 * moe_intermediate, hidden)- down_proj shape: (num_experts, hidden, moe_intermediate)- """- intermediate_size = original.down_proj.shape[-1]- if not unsupported_meta_device(original):- for i in range(self.num_experts):- gate_up = original.gate_up_proj[i]- down = original.down_proj[i]-- gate_proj = gate_up[:intermediate_size, :]- up_proj = gate_up[intermediate_size:, :]-- _update_parameter(self[i].gate_proj, "weight", gate_proj.contiguous())- _update_parameter(self[i].up_proj, "weight", up_proj.contiguous())- _update_parameter(self[i].down_proj, "weight", down.contiguous())- del gate_up, down, gate_proj, up_proj- original.to_empty(device="meta") # release original fused parameters- clear_memory()diff --git a/auto_round/modeling/fused_moe/replace_modules.py b/auto_round/modeling/fused_moe/replace_modules.pyindex e7993d109..6f98d83c0 100644--- a/auto_round/modeling/fused_moe/replace_modules.py+++ b/auto_round/modeling/fused_moe/replace_modules.py@@ -39,10 +39,17 @@ BUILTIN_MODULES = {"qwen3_5_moe_text": LazyImport("auto_round.modeling.fused_moe.qwen3_5_moe"),# Step 3.5 MoE: splits fused MoELinear into per-expert nn.Linear"step3p5": LazyImport("auto_round.modeling.fused_moe.step3_5_moe"),- # Qwen3-Omni MoE: thinker (no shared expert) + talker (with shared expert)+ # Qwen3-Omni MoE: thinker (no shared expert)"qwen3_omni_moe": LazyImport("auto_round.modeling.fused_moe.qwen3_omni"),}+# Module name prefixes to exclude from MoE unfusing, keyed by model_type.+# Modules under these prefixes stay in their original fused 3D format during+# quantization. ShardWriter expands them to per-expert 2D tensors at save time.+MOE_SKIP_PREFIXES: dict[str, list[str]] = {+ "qwen3_omni_moe": ["talker."],+}+# transformers >= 5.0.0 supports the general linear_loop experts interface.if not is_transformers_version_greater_or_equal_5():@@ -55,6 +62,9 @@ if not is_transformers_version_greater_or_equal_5():def _handle_moe_modules(model: torch.nn.Module) -> list[str]:"""Handle fused MOE modules using transformers' linear_loop backend.+ Modules under prefixes listed in MOE_SKIP_PREFIXES for the model's+ model_type are left in their original fused format.+Args:model: The model to process@@ -74,8 +84,11 @@ def _handle_moe_modules(model: torch.nn.Module) -> list[str]:)return []+ model_type = getattr(getattr(model, "config", None), "model_type", None)+ skip_prefixes = MOE_SKIP_PREFIXES.get(model_type, [])+# Use transformers' experts interface- unfused = prepare_model_for_moe_quantization(model)+ unfused = prepare_model_for_moe_quantization(model, skip_prefixes=skip_prefixes)if unfused:logger.info(f"Prepared {len(unfused)} MOE modules for quantization")return unfused@@ -337,10 +350,10 @@ def _apply_custom_replacements(model: torch.nn.Module) -> list:if isinstance(module, ReplacementModuleBase):continueclass_name = module.__class__.__name__- if ReplacementModuleBase.is_registered(class_name) and ReplacementModuleBase.get_replacement_class(- class_name- ).is_to_be_replaced(module):- modules_to_replace.append((name, module, class_name))+ if ReplacementModuleBase.is_registered(class_name):+ replacement_cls = ReplacementModuleBase.get_replacement_class(class_name)+ if replacement_cls.is_to_be_replaced(module):+ modules_to_replace.append((name, module, class_name))# Step 2: Replace modulesif modules_to_replace:diff --git a/auto_round/special_model_handler.py b/auto_round/special_model_handler.pyindex 453149f4e..e6079fbed 100644--- a/auto_round/special_model_handler.py+++ b/auto_round/special_model_handler.py@@ -403,8 +403,10 @@ def _get_qwen2_5_omni_multimodal_block(model, quant_vision=False):For quantization, we focus on:- thinker.model.layers (text decoder layers) - main LLM layers- - talker.model.layers (talker decoder layers)- Optionally: visual encoder blocks, audio encoder layers++ talker is excluded by default because quantizing it has been observed to+ degrade audio quality in long-form generation."""block_names = []@@ -422,10 +424,6 @@ def _get_qwen2_5_omni_multimodal_block(model, quant_vision=False):if hasattr(model, "thinker") and hasattr(model.thinker, "model") and hasattr(model.thinker.model, "layers"):block_names.append([f"thinker.model.layers.{i}" for i in range(len(model.thinker.model.layers))])- # Talker model layers (if available)- if hasattr(model, "talker") and hasattr(model.talker, "model") and hasattr(model.talker.model, "layers"):- block_names.append([f"talker.model.layers.{i}" for i in range(len(model.talker.model.layers))])-return block_names@@ -439,8 +437,10 @@ def _get_qwen3_omni_moe_multimodal_block(model, quant_vision=False):For quantization, we focus on:- thinker.model.layers (text decoder layers) - main LLM layers- - talker.model.layers (talker decoder layers)- Optionally: visual encoder blocks, audio encoder layers++ talker is excluded by default because quantizing it has been observed to+ degrade audio quality in long-form generation ."""block_names = []@@ -460,10 +460,6 @@ def _get_qwen3_omni_moe_multimodal_block(model, quant_vision=False):if hasattr(model, "thinker") and hasattr(model.thinker, "model") and hasattr(model.thinker.model, "layers"):block_names.append([f"thinker.model.layers.{i}" for i in range(len(model.thinker.model.layers))])- # Talker model layers (if available)- if hasattr(model, "talker") and hasattr(model.talker, "model") and hasattr(model.talker.model, "layers"):- block_names.append([f"talker.model.layers.{i}" for i in range(len(model.talker.model.layers))])-return block_namesdiff --git a/auto_round/utils/missing_tensors.py b/auto_round/utils/missing_tensors.pyindex 255b7f92f..7f2dc922f 100644--- a/auto_round/utils/missing_tensors.py+++ b/auto_round/utils/missing_tensors.py@@ -326,8 +326,6 @@ def copy_missing_tensors_from_source(if not missing_tensor_names:return- logger.debug(f"Missing tensors detected: {missing_tensor_names}")-try:from tqdm import tqdm as _tqdmexcept ImportError:@@ -721,12 +719,41 @@ def _woq_quantize_missing_tensors(target_dir: str, missing_tensors_dict: dict) -return Truereturn False+ if block_name_to_quantize:+ existing_blocks = (+ block_name_to_quantize+ if isinstance(block_name_to_quantize, list)+ else [b.strip() for b in block_name_to_quantize.split(",") if b.strip()]+ )+ existing_set = set(existing_blocks)+ new_prefixes = {+ ".".join(k.split(".")[:i])+ for k in missing_tensors_dict+ if k.endswith(".weight") and missing_tensors_dict[k].dim() == 2+ for i, p in enumerate(k.split("."))+ if p.isdigit() and ".".join(k.split(".")[:i])+ }+ added = sorted(new_prefixes - existing_set)+ if added:+ block_name_to_quantize = existing_blocks + added+# Identify weight tensors eligible for quantization (2D Linear weights)def _is_eligible(k: str) -> bool:if not k.endswith(".weight"):return Falseif missing_tensors_dict[k].dim() != 2:return False+ if k.startswith("talker."):+ return False+ # Only quantize weights that fall under the quantized blocks.+ if block_name_to_quantize:+ blocks = (+ block_name_to_quantize+ if isinstance(block_name_to_quantize, list)+ else [b.strip() for b in block_name_to_quantize.split(",") if b.strip()]+ )+ if not any(k.startswith(b + ".") or k.startswith(b + "[") for b in blocks):+ return Falselayer_name = k[: -len(".weight")]layer_cfg = _resolve_layer_cfg(layer_name)# If extra_config explicitly covers this layer, trust its decisiondiff --git a/test/test_cpu/models/test_omni_model.py b/test/test_cpu/models/test_omni_model.pyindex 3c5d82548..746ac795f 100644--- a/test/test_cpu/models/test_omni_model.py+++ b/test/test_cpu/models/test_omni_model.py@@ -96,17 +96,17 @@ class TestQwen2_5Omni:shutil.rmtree("runs", ignore_errors=True)def test_block_names_default(self):- """Test that get_block_names returns thinker + talker layers."""+ """Test that get_block_names returns thinker layers only by default."""from auto_round.utils import get_block_namesblock_names = get_block_names(self.model, quant_vision=False)- # Should have thinker.model.layers and talker.model.layers+ # Should have thinker.model.layers but not talker.model.layersassert any("thinker.model.layers" in str(b) for b in block_names), f"Expected thinker.model.layers in block_names, got: {block_names}"- assert any(+ assert not any("talker.model.layers" in str(b) for b in block_names- ), f"Expected talker.model.layers in block_names, got: {block_names}"+ ), f"Did not expect talker.model.layers by default, got: {block_names}"def test_block_names_quant_vision(self):"""Test that quant_vision adds visual and audio blocks."""@@ -167,7 +167,7 @@ class TestQwen3OmniMoeBlockNames:"""Test block name discovery for Qwen3-Omni-MoE."""def test_block_names_default(self):- """Test that get_block_names returns thinker + talker layers."""+ """Test that get_block_names returns thinker layers only by default."""from auto_round.utils import get_block_namesconfig = _make_tiny_qwen3_omni_moe_config()@@ -177,9 +177,9 @@ class TestQwen3OmniMoeBlockNames:assert any("thinker.model.layers" in str(b) for b in block_names), f"Expected thinker.model.layers, got: {block_names}"- assert any(+ assert not any("talker.model.layers" in str(b) for b in block_names- ), f"Expected talker.model.layers, got: {block_names}"+ ), f"Did not expect talker.model.layers by default, got: {block_names}"def test_block_names_quant_vision(self):"""Test that quant_vision adds visual and audio blocks."""@@ -213,15 +213,15 @@ class TestQwen3OmniMoeReplacement:"""Test MoE module replacement for Qwen3-Omni-MoE."""def test_replacement_registered(self):- """Test that both thinker and talker MoE blocks are registered."""+ """Test that thinker MoE block is registered."""from auto_round.modeling.fused_moe.qwen3_omni import (- LinearQwen3OmniTalkerSparseMoeBlock,LinearQwen3OmniThinkerSparseMoeBlock,)from auto_round.modeling.fused_moe.replace_modules import ReplacementModuleBaseassert ReplacementModuleBase.is_registered("Qwen3OmniMoeThinkerTextSparseMoeBlock")- assert ReplacementModuleBase.is_registered("Qwen3OmniMoeTalkerTextSparseMoeBlock")+ # Talker has no replacement class — it stays fused and is excluded via MOE_SKIP_PREFIXES+ assert not ReplacementModuleBase.is_registered("Qwen3OmniMoeTalkerTextSparseMoeBlock")def test_builtin_modules_entry(self):"""Test that qwen3_omni_moe is in BUILTIN_MODULES."""@@ -238,28 +238,38 @@ class TestQwen3OmniMoeReplacement:assert is_custom_model(model)def test_apply_replacements(self):- """Test that MoE blocks are correctly replaced."""+ """Test that only thinker MoE blocks are replaced; talker stays fused."""from auto_round.modeling.fused_moe.replace_modules import apply_replacementsconfig = _make_tiny_qwen3_omni_moe_config()model = Qwen3OmniMoeForConditionalGeneration(config)+ original_talker_class = model.talker.model.layers[0].mlp.__class__.__name__model = apply_replacements(model)- # Check that thinker MoE was replaced+ # Check that thinker MoE was custom-replacedthinker_mlp = model.thinker.model.layers[0].mlpassert ("LinearQwen3OmniThinker" in thinker_mlp.__class__.__name__), f"Expected LinearQwen3OmniThinker, got {thinker_mlp.__class__.__name__}"- # Check that talker MoE was replaced+ # Talker MoE stays fused (excluded from both custom replacement and generic unfuse)+ # to prevent memory eviction during disk-offloaded quantization.+ # ShardWriter handles fused→per-expert conversion at save time.talker_mlp = model.talker.model.layers[0].mlpassert (- "LinearQwen3OmniTalker" in talker_mlp.__class__.__name__- ), f"Expected LinearQwen3OmniTalker, got {talker_mlp.__class__.__name__}"+ talker_mlp.__class__.__name__ == original_talker_class+ ), f"Expected talker MoE to stay original ({original_talker_class}), got {talker_mlp.__class__.__name__}"+ assert hasattr(+ model.talker.model.layers[0].mlp.experts, "gate_up_proj"+ ), "Talker experts should keep fused gate_up_proj"def test_weight_fidelity(self):- """Test that unfused weights match original fused weights."""+ """Test that unfused thinker weights match original fused weights.++ Talker MoE stays fused (excluded from quantization), so only+ thinker experts are verified here.+ """from auto_round.modeling.fused_moe.replace_modules import apply_replacements, materialize_model_torch.manual_seed(42)@@ -270,11 +280,9 @@ class TestQwen3OmniMoeReplacement:config = _make_tiny_qwen3_omni_moe_config()model = Qwen3OmniMoeForConditionalGeneration(config)- # Save original fused weights+ # Save original fused thinker weightsthinker_gate_up = model.thinker.model.layers[0].mlp.experts.gate_up_proj.data.clone()thinker_down = model.thinker.model.layers[0].mlp.experts.down_proj.data.clone()- talker_gate_up = model.talker.model.layers[0].mlp.experts.gate_up_proj.data.clone()- talker_down = model.talker.model.layers[0].mlp.experts.down_proj.data.clone()model = apply_replacements(model)materialize_model_(model)@@ -289,20 +297,13 @@ class TestQwen3OmniMoeReplacement:assert_same_weights(expert.up_proj.weight.data, thinker_gate_up[i, intermediate:, :])assert_same_weights(expert.down_proj.weight.data, thinker_down[i])- # Verify talker expert weights- for i in range(4):- expert = model.talker.model.layers[0].mlp.experts[i]- assert_same_weights(expert.gate_proj.weight.data, talker_gate_up[i, :intermediate, :])- assert_same_weights(expert.up_proj.weight.data, talker_gate_up[i, intermediate:, :])- assert_same_weights(expert.down_proj.weight.data, talker_down[i])-def test_forward_output_match(self):- """Test that replaced MoE forward output matches original."""+ """Test that replaced thinker MoE forward output matches original.++ Talker MoE is not replaced, so only thinker output is verified.+ """from auto_round.modeling.fused_moe.replace_modules import apply_replacements, materialize_model_- # Fix seed for deterministic model weights and use small-scale input to- # prevent numerical overflow with random weights (talker has a larger- # shared_expert MLP that can produce NaN otherwise).torch.manual_seed(0)config = _make_tiny_qwen3_omni_moe_config()model = Qwen3OmniMoeForConditionalGeneration(config)@@ -310,31 +311,22 @@ class TestQwen3OmniMoeReplacement:x = torch.randn(1, 4, 64) * 0.1with torch.no_grad():orig_thinker_out = model.thinker.model.layers[0].mlp(x)- orig_talker_out = model.talker.model.layers[0].mlp(x)model = apply_replacements(model)materialize_model_(model)with torch.no_grad():new_thinker_out = model.thinker.model.layers[0].mlp(x)- new_talker_out = model.talker.model.layers[0].mlp(x)-- # Use a NaN-safe comparison: check that NaN positions match, then- # compare finite values. This avoids false failures when random- # weights cause some outputs to overflow to NaN.- for name, orig, new in [- ("Thinker", orig_thinker_out, new_thinker_out),- ("Talker", orig_talker_out, new_talker_out),- ]:- assert orig.shape == new.shape, f"{name} shape mismatch: {orig.shape} vs {new.shape}"- orig_nan = torch.isnan(orig)- new_nan = torch.isnan(new)- assert torch.equal(orig_nan, new_nan), f"{name} NaN positions differ"- finite_mask = ~orig_nan- if finite_mask.any():- assert torch.allclose(- orig[finite_mask], new[finite_mask], atol=1e-5- ), f"{name} MoE forward mismatch on finite values"++ assert orig_thinker_out.shape == new_thinker_out.shape, "Thinker shape mismatch"+ orig_nan = torch.isnan(orig_thinker_out)+ new_nan = torch.isnan(new_thinker_out)+ assert torch.equal(orig_nan, new_nan), "Thinker NaN positions differ"+ finite_mask = ~orig_nan+ if finite_mask.any():+ assert torch.allclose(+ orig_thinker_out[finite_mask], new_thinker_out[finite_mask], atol=1e-5+ ), "Thinker MoE forward mismatch on finite values"class TestQwen3OmniMoeProcessor:diff --git a/test/test_cpu/utils/test_missing_tensors.py b/test/test_cpu/utils/test_missing_tensors.pyindex 88dd4a329..ae7ad83cc 100644--- a/test/test_cpu/utils/test_missing_tensors.py+++ b/test/test_cpu/utils/test_missing_tensors.py@@ -584,3 +584,88 @@ class TestCopyMissingTensorsFromSource:_write_config(tgt)copy_missing_tensors_from_source(src, tgt)assert not os.path.exists(os.path.join(tgt, "model_extra_tensors.safetensors"))++ def test_talker_missing_weight_is_never_woq_quantized(self):+ """Talker weights must stay BF16/full precision even in WOQ exports."""+ with tempfile.TemporaryDirectory() as source_dir, tempfile.TemporaryDirectory() as target_dir:+ talker_weight = torch.randn(32, 64, dtype=torch.bfloat16)+ _save_safetensors(+ {"talker.model.layers.0.mlp.experts.0.up_proj.weight": talker_weight},+ os.path.join(source_dir, "model.safetensors"),+ )+ _save_safetensors(+ {+ "thinker.model.layers.0.mlp.experts.0.up_proj.qweight": torch.randint(+ 0, 16, (8, 32), dtype=torch.int32+ )+ },+ os.path.join(target_dir, "model.safetensors"),+ )+ with open(os.path.join(target_dir, "config.json"), "w") as f:+ json.dump(+ _make_auto_round_config(bits=4, group_size=64, sym=True)+ | {+ "quantization_config": _make_auto_round_config(bits=4, group_size=64, sym=True)[+ "quantization_config"+ ]+ | {"block_name_to_quantize": "thinker.model.layers"}+ },+ f,+ )++ copy_missing_tensors_from_source(source_dir, target_dir)++ result = _load_safetensors(os.path.join(target_dir, "model_extra_tensors.safetensors"))+ assert "talker.model.layers.0.mlp.experts.0.up_proj.weight" in result+ assert "talker.model.layers.0.mlp.experts.0.up_proj.qweight" not in result++ def test_talker_fused_projection_not_treated_as_missing_when_expanded_components_exist(self):+ """Talker fused projection keys must not be marked missing if their per-expert 2D+ components are already saved (via _expand_fused_experts during export).++ This is the inverse of the previous test: the target already has the per-expert+ 2D weights that _expand_fused_experts produced from the fused 3D source key.+ copy_missing_tensors_from_source should NOT re-split the source fused 3D key+ and add duplicate per-expert weights.+ """+ with tempfile.TemporaryDirectory() as source_dir, tempfile.TemporaryDirectory() as target_dir:+ # Source has fused 3D expert params (what the original checkpoint looks like)+ _save_safetensors(+ {+ # These are the original fused 3D keys that appear in the source checkpoint.+ # In practice num_experts=4 in the tiny test config.+ "talker.model.layers.0.mlp.experts.gate_up_proj": torch.randn(4, 16, 32),+ "talker.model.layers.0.mlp.experts.down_proj": torch.randn(4, 32, 16),+ },+ os.path.join(source_dir, "model.safetensors"),+ )+ # Target already has the per-expert 2D weights that _expand_fused_experts+ # produced during export. These are the correct shape expected by transformers.+ _save_safetensors(+ {+ # gate_up_proj split into gate_proj + up_proj+ "talker.model.layers.0.mlp.experts.0.gate_proj.weight": torch.randn(8, 32),+ "talker.model.layers.0.mlp.experts.0.up_proj.weight": torch.randn(8, 32),+ "talker.model.layers.0.mlp.experts.1.gate_proj.weight": torch.randn(8, 32),+ "talker.model.layers.0.mlp.experts.1.up_proj.weight": torch.randn(8, 32),+ # down_proj as per-expert 2D+ "talker.model.layers.0.mlp.experts.0.down_proj.weight": torch.randn(32, 8),+ "talker.model.layers.0.mlp.experts.1.down_proj.weight": torch.randn(32, 8),+ # Also have some thinker quantized weights+ "thinker.model.layers.0.mlp.experts.0.gate_proj.qweight": torch.randint(+ 0, 16, (8, 32), dtype=torch.int32+ ),+ },+ os.path.join(target_dir, "model.safetensors"),+ )+ _write_config(target_dir)++ copy_missing_tensors_from_source(source_dir, target_dir)++ # The fused keys must NOT be treated as missing — their expanded+ # per-expert 2D components are already in the target.+ extra_shard = os.path.join(target_dir, "model_extra_tensors.safetensors")+ assert not os.path.exists(extra_shard), (+ "Fused talker expert keys should not be treated as missing when "+ "their per-expert 2D components are already saved"+ )diff --git a/test/test_cpu/utils/test_shard_writer.py b/test/test_cpu/utils/test_shard_writer.pyindex 15e78a4ae..dfd092b25 100644--- a/test/test_cpu/utils/test_shard_writer.py+++ b/test/test_cpu/utils/test_shard_writer.py@@ -81,6 +81,22 @@ class _LMStyleModel(torch.nn.Module):self.config = SimpleNamespace(model_type="toy-lm", tie_word_embeddings=True)+class _ToyExperts(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.is_transposed = False+++class _FusedExpertsModel(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.block = torch.nn.Module()+ self.block.experts = _ToyExperts()+ self.talker = torch.nn.Module()+ self.talker.experts = _ToyExperts()+ self.config = SimpleNamespace(model_type="qwen3_omni_moe")++def test_finalize_skips_lm_head_when_tie_word_embeddings_true(tmp_path, monkeypatch):"""Complementary test: when tie_word_embeddings=True the lm_head should beskipped (not written to disk) and offloaded to meta."""@@ -100,6 +116,51 @@ def test_finalize_skips_lm_head_when_tie_word_embeddings_true(tmp_path, monkeypaassert model.lm_head.weight.device.type == "meta"+def test_expand_fused_experts_for_skipped_talker_prefix(tmp_path, monkeypatch):+ """Talker fused 3D weights must be expanded to exact per-expert 2D keys.++ Real Qwen3-Omni-MoE exports attach reverse checkpoint conversion mappings for+ MoE projections. If we apply that mapping to the fused talker tensor before+ expanding it, the fused tensor is saved under a wildcard key such as+ ``talker.experts.*.gate_proj.weight``. That breaks reload because+ transformers expects concrete per-expert 2D keys after save_pretrained.+ """+ model = _FusedExpertsModel()+ writer = _make_writer(model, str(tmp_path), monkeypatch)+ writer.reverse_checkpoint_conversion_mapping = {+ r"experts\.gate_up_proj$": ["experts.*.gate_proj.weight", "experts.*.up_proj.weight"]+ }++ fused_gate_up = torch.arange(2 * 6 * 4, dtype=torch.float32).reshape(2, 6, 4)+ writer._add_tensor("talker.experts.gate_up_proj", fused_gate_up)+ writer.finalize()++ shard_path = os.path.join(tmp_path, "model.bin")+ saved_tensors = torch.load(shard_path, map_location="cpu")++ assert "talker.experts.gate_up_proj" not in saved_tensors+ assert "talker.experts.*.gate_proj.weight" not in saved_tensors+ assert "talker.experts.0.gate_proj.weight" in saved_tensors+ assert "talker.experts.0.up_proj.weight" in saved_tensors+ assert torch.equal(saved_tensors["talker.experts.0.gate_proj.weight"], fused_gate_up[0, :3, :])+ assert torch.equal(saved_tensors["talker.experts.0.up_proj.weight"], fused_gate_up[0, 3:, :])+++def test_do_not_expand_fused_experts_outside_skipped_prefixes(tmp_path, monkeypatch):+ model = _FusedExpertsModel()+ writer = _make_writer(model, str(tmp_path), monkeypatch)++ fused_gate_up = torch.arange(2 * 6 * 4, dtype=torch.float32).reshape(2, 6, 4)+ writer._add_tensor("block.experts.gate_up_proj", fused_gate_up)+ writer.finalize()++ shard_path = os.path.join(tmp_path, "model.bin")+ saved_tensors = torch.load(shard_path, map_location="cpu")++ assert "block.experts.gate_up_proj" in saved_tensors+ assert "block.experts.0.gate_proj.weight" not in saved_tensors++def test_finalize_offloads_module_with_tensor_in_parameters(tmp_path, monkeypatch):model = _DiffusionStyleModel()model.transformer_blocks[0].linear._parameters["weight"] = model.transformer_blocks[0].linear.weight.to("cpu")diff --git a/test/test_xpu/test_autoround.py b/test/test_xpu/test_autoround.pyindex d03c71444..473f73451 100644--- a/test/test_xpu/test_autoround.py+++ b/test/test_xpu/test_autoround.py@@ -155,7 +155,7 @@ class TestAutoRoundXPU:model = Qwen2VLForConditionalGeneration.from_pretrained(quantized_model_path,- torch_dtype="float16",+ torch_dtype="auto",device_map=self.device,quantization_config=quantization_config,)