Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

Fix Qwen Omni quantization model issue for long form audio generation (#1698)

Liang Lvb40340734c80
diff --git a/auto_round/compressors/shard_writer.py b/auto_round/compressors/shard_writer.py
index b25c26739..d4443b080 100644
--- a/auto_round/compressors/shard_writer.py
+++ b/auto_round/compressors/shard_writer.py
@@ -159,11 +159,42 @@ class ShardWriter:
param_name = f"{prefix}.{k}"
self._add_tensor(param_name, v)
- def _add_tensor(self, name: str, tensor: torch.Tensor):
+ def _expand_fused_experts(self, name: str, tensor: torch.Tensor) -> list[tuple[str, torch.Tensor]] | None:
+ """Expand a fused 3D expert parameter into per-expert 2D weight tensors.
- # transformers will handle _checkpoint_conversion_mapping automatically if is_immediate_saving=False
- name = revert_checkpoint_conversion_mapping(name, self.reverse_checkpoint_conversion_mapping)
+ Non-quantized MoE modules (e.g. thinker in Qwen3-Omni) are kept fused
+ during quantization to prevent memory eviction under disk offloading.
+ This method converts them to the per-expert checkpoint format at save time.
+
+ Talker MoE modules stay fused during quantization, but save_pretrained
+ still needs concrete per-expert 2D checkpoint keys. Saving the original
+ fused tensor under a reverse-mapped wildcard key (for example
+ ``experts.*.gate_proj.weight``) breaks reload.
+
+ Returns:
+ List of (key, 2D tensor) pairs, or None if not a fused expert param.
+ """
+ from auto_round.modeling.fused_moe.replace_modules import MOE_SKIP_PREFIXES
+ from auto_round.utils.missing_tensors import split_fused_expert_tensors
+
+ parts = name.rsplit(".", 1)
+ if len(parts) != 2:
+ return None
+ prefix, attr_name = parts
+
+ model_type = getattr(getattr(self.model, "config", None), "model_type", None)
+ skip_prefixes = MOE_SKIP_PREFIXES.get(model_type, []) if model_type is not None else []
+ if not any(
+ prefix == skip_prefix.rstrip(".") or prefix.startswith(skip_prefix) for skip_prefix in skip_prefixes
+ ):
+ return None
+
+ expanded = split_fused_expert_tensors({name: tensor})
+ if set(expanded) == {name}:
+ return None
+ return list(expanded.items())
+ def _add_tensor(self, name: str, tensor: torch.Tensor):
if isinstance(tensor, torch.Tensor) and tensor.device.type == "meta":
self.skipped_meta_tensors.append(name)
return
@@ -172,6 +203,18 @@ class ShardWriter:
if name in self._all_saved or name in self.current_shard_tensors:
return
+ # Expand fused 3D expert parameters into per-expert 2D tensors if necessary
+ if tensor.dim() == 3:
+ expanded = self._expand_fused_experts(name, tensor)
+ if expanded is not None:
+ self._all_saved.add(name)
+ for sub_name, sub_tensor in expanded:
+ self._add_tensor(sub_name, sub_tensor)
+ return
+
+ # transformers will handle _checkpoint_conversion_mapping automatically if is_immediate_saving=False
+ name = revert_checkpoint_conversion_mapping(name, self.reverse_checkpoint_conversion_mapping)
+
t_size = tensor.nbytes
self.total_param_elems += tensor.numel()
self.total_param_size_bytes += t_size
diff --git a/auto_round/modeling/fused_moe/moe_experts_interface.py b/auto_round/modeling/fused_moe/moe_experts_interface.py
index 854d9e186..f6e901699 100644
--- a/auto_round/modeling/fused_moe/moe_experts_interface.py
+++ b/auto_round/modeling/fused_moe/moe_experts_interface.py
@@ -608,7 +608,11 @@ def _unfuse_experts_weights_inplace(
return True
-def prepare_model_for_moe_quantization(model: nn.Module, implementation: str = LINEAR_LOOP_IMPL) -> list[str]:
+def prepare_model_for_moe_quantization(
+ model: nn.Module,
+ implementation: str = LINEAR_LOOP_IMPL,
+ skip_prefixes: list[str] | None = None,
+) -> list[str]:
"""Prepare a model for MOE quantization using transformers' experts interface.
This function:
@@ -622,6 +626,9 @@ def prepare_model_for_moe_quantization(model: nn.Module, implementation: str = L
Args:
model: The model to prepare
implementation: The experts implementation to use (default: "linear_loop")
+ skip_prefixes: Module name prefixes to skip (e.g. ["talker."]).
+ Modules under these prefixes stay in their original fused 3D format.
+ ShardWriter handles the fused→per-expert conversion at save time.
Returns:
List of module names that were unfused
@@ -635,6 +642,8 @@ def prepare_model_for_moe_quantization(model: nn.Module, implementation: str = L
# Unfuse all fused experts modules (only those supporting @use_experts_implementation)
unfused_modules = []
for name, module in model.named_modules():
+ if skip_prefixes and any(name.startswith(p) for p in skip_prefixes):
+ continue
if _unfuse_experts_weights_inplace(module):
unfused_modules.append(name)
logger.debug(f"[MoE Prep] Unfused '{name}'")
diff --git a/auto_round/modeling/fused_moe/qwen3_omni.py b/auto_round/modeling/fused_moe/qwen3_omni.py
index ad2004592..58731f4db 100644
--- a/auto_round/modeling/fused_moe/qwen3_omni.py
+++ b/auto_round/modeling/fused_moe/qwen3_omni.py
@@ -1,14 +1,12 @@
# Copyright (c) 2026 Intel Corporation
# SPDX-License-Identifier: Apache-2.0
-"""MoE module replacements for Qwen3-Omni model.
+"""MoE module replacement for Qwen3-Omni thinker.
Qwen3-Omni has MoE blocks in both thinker and talker:
-- Thinker: Qwen3OmniMoeThinkerTextSparseMoeBlock (experts + gate, no shared expert)
-- Talker: Qwen3OmniMoeTalkerTextSparseMoeBlock (experts + gate + shared_expert + shared_expert_gate)
-
-This module provides replacement classes that unfuse fused expert weights (3D Parameters)
-into individual nn.Linear layers, enabling per-expert quantization with meta device optimization.
+- Thinker: Qwen3OmniMoeThinkerTextSparseMoeBlock — unfused here for quantization.
+- Talker: stays in original fused format (excluded via MOE_SKIP_PREFIXES).
+ ShardWriter expands talker's fused 3D params to per-expert 2D tensors at save time.
"""
import torch
@@ -80,74 +78,6 @@ class LinearQwen3OmniThinkerSparseMoeBlock(ReplacementModuleBase):
# ---------------------------------------------------------------------------
-# Talker MoE replacement (with shared expert, same pattern as qwen3_5_moe)
-# ---------------------------------------------------------------------------
-
-
-class LinearQwen3OmniTalkerSparseMoeBlock(ReplacementModuleBase):
- """Calibration replacement for Qwen3OmniMoeTalkerTextSparseMoeBlock.
-
- Unfuses fused expert weights and preserves the shared_expert + shared_expert_gate.
- Similar to Qwen3.5-MoE pattern.
-
- Structure: gate (router) + experts (unfused) + shared_expert + shared_expert_gate.
- """
-
- def __init__(self, original, config):
- super().__init__(original)
- self.gate = original.gate
- self.shared_expert = original.shared_expert
- self.shared_expert_gate = original.shared_expert_gate
- self.num_experts = original.experts.num_experts
- text_config = config.talker_config.text_config
- with torch.device("meta"):
- self.experts = SequentialQwen3OmniTalkerExperts(text_config, original.experts)
-
- @classmethod
- def original_module_class(cls) -> str:
- return "Qwen3OmniMoeTalkerTextSparseMoeBlock"
-
- def _materialize_weights(self) -> None:
- original = self._get_original_module()
- self.experts._materialize_weights(original.experts)
- clear_memory()
-
- def experts_forward(self, hidden_states, top_k_index, top_k_weights):
- final_hidden_states = torch.zeros_like(hidden_states)
- with torch.no_grad():
- expert_mask = torch.nn.functional.one_hot(top_k_index, num_classes=self.num_experts)
- expert_mask = expert_mask.permute(2, 1, 0)
- expert_hit = torch.greater(expert_mask.sum(dim=(-1, -2)), 0).nonzero()
-
- for expert_idx in expert_hit:
- expert_idx = expert_idx[0]
- if expert_idx == self.num_experts:
- continue
- top_k_pos, token_idx = torch.where(expert_mask[expert_idx])
- current_state = hidden_states[token_idx]
- current_hidden_states = self.experts[expert_idx](current_state)
- current_hidden_states = current_hidden_states * top_k_weights[token_idx, top_k_pos, None]
- final_hidden_states.index_add_(0, token_idx, current_hidden_states.to(final_hidden_states.dtype))
-
- return final_hidden_states
-
- def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
- batch_size, sequence_length, hidden_dim = hidden_states.shape
- hidden_states_reshaped = hidden_states.view(-1, hidden_dim)
- shared_expert_output = self.shared_expert(hidden_states_reshaped)
- _, routing_weights, selected_experts = self.gate(hidden_states_reshaped)
- expert_output = self.experts_forward(hidden_states_reshaped, selected_experts, routing_weights)
-
- shared_expert_output = torch.sigmoid(self.shared_expert_gate(hidden_states_reshaped)) * shared_expert_output
-
- expert_output = expert_output + shared_expert_output
- return expert_output.reshape(batch_size, sequence_length, hidden_dim)
-
- @classmethod
- def from_original(cls, original, config, **kwargs):
- return cls(original, config)
-
-
# ---------------------------------------------------------------------------
# Sequential expert containers (unfused nn.Linear per expert)
# ---------------------------------------------------------------------------
@@ -193,45 +123,3 @@ class SequentialQwen3OmniThinkerExperts(torch.nn.ModuleList):
del gate_up, down, gate_proj, up_proj
original.to_empty(device="meta") # release original fused parameters
clear_memory()
-
-
-class SequentialQwen3OmniTalkerExperts(torch.nn.ModuleList):
- """Unfused per-expert nn.Linear layers for Qwen3-Omni talker MoE.
-
- Replaces fused 3D Parameters (gate_up_proj, down_proj) with individual
- Qwen3OmniMoeTalkerTextMLP modules per expert.
- """
-
- def __init__(self, config, original):
- super().__init__()
- self.num_experts = original.gate_up_proj.shape[0]
- intermediate_size = config.moe_intermediate_size
-
- from transformers.models.qwen3_omni_moe.modeling_qwen3_omni_moe import (
- Qwen3OmniMoeTalkerTextMLP,
- )
-
- with torch.device("meta"):
- super().__init__([Qwen3OmniMoeTalkerTextMLP(config, intermediate_size) for _ in range(self.num_experts)])
-
- def _materialize_weights(self, original) -> None:
- """Unfuse fused expert weights into individual nn.Linear layers.
-
- gate_up_proj shape: (num_experts, 2 * moe_intermediate, hidden)
- down_proj shape: (num_experts, hidden, moe_intermediate)
- """
- intermediate_size = original.down_proj.shape[-1]
- if not unsupported_meta_device(original):
- for i in range(self.num_experts):
- gate_up = original.gate_up_proj[i]
- down = original.down_proj[i]
-
- gate_proj = gate_up[:intermediate_size, :]
- up_proj = gate_up[intermediate_size:, :]
-
- _update_parameter(self[i].gate_proj, "weight", gate_proj.contiguous())
- _update_parameter(self[i].up_proj, "weight", up_proj.contiguous())
- _update_parameter(self[i].down_proj, "weight", down.contiguous())
- del gate_up, down, gate_proj, up_proj
- original.to_empty(device="meta") # release original fused parameters
- clear_memory()
diff --git a/auto_round/modeling/fused_moe/replace_modules.py b/auto_round/modeling/fused_moe/replace_modules.py
index e7993d109..6f98d83c0 100644
--- a/auto_round/modeling/fused_moe/replace_modules.py
+++ b/auto_round/modeling/fused_moe/replace_modules.py
@@ -39,10 +39,17 @@ BUILTIN_MODULES = {
"qwen3_5_moe_text": LazyImport("auto_round.modeling.fused_moe.qwen3_5_moe"),
# Step 3.5 MoE: splits fused MoELinear into per-expert nn.Linear
"step3p5": LazyImport("auto_round.modeling.fused_moe.step3_5_moe"),
- # Qwen3-Omni MoE: thinker (no shared expert) + talker (with shared expert)
+ # Qwen3-Omni MoE: thinker (no shared expert)
"qwen3_omni_moe": LazyImport("auto_round.modeling.fused_moe.qwen3_omni"),
}
+# Module name prefixes to exclude from MoE unfusing, keyed by model_type.
+# Modules under these prefixes stay in their original fused 3D format during
+# quantization. ShardWriter expands them to per-expert 2D tensors at save time.
+MOE_SKIP_PREFIXES: dict[str, list[str]] = {
+ "qwen3_omni_moe": ["talker."],
+}
+
# transformers >= 5.0.0 supports the general linear_loop experts interface.
if not is_transformers_version_greater_or_equal_5():
@@ -55,6 +62,9 @@ if not is_transformers_version_greater_or_equal_5():
def _handle_moe_modules(model: torch.nn.Module) -> list[str]:
"""Handle fused MOE modules using transformers' linear_loop backend.
+ Modules under prefixes listed in MOE_SKIP_PREFIXES for the model's
+ model_type are left in their original fused format.
+
Args:
model: The model to process
@@ -74,8 +84,11 @@ def _handle_moe_modules(model: torch.nn.Module) -> list[str]:
)
return []
+ model_type = getattr(getattr(model, "config", None), "model_type", None)
+ skip_prefixes = MOE_SKIP_PREFIXES.get(model_type, [])
+
# Use transformers' experts interface
- unfused = prepare_model_for_moe_quantization(model)
+ unfused = prepare_model_for_moe_quantization(model, skip_prefixes=skip_prefixes)
if unfused:
logger.info(f"Prepared {len(unfused)} MOE modules for quantization")
return unfused
@@ -337,10 +350,10 @@ def _apply_custom_replacements(model: torch.nn.Module) -> list:
if isinstance(module, ReplacementModuleBase):
continue
class_name = module.__class__.__name__
- if ReplacementModuleBase.is_registered(class_name) and ReplacementModuleBase.get_replacement_class(
- class_name
- ).is_to_be_replaced(module):
- modules_to_replace.append((name, module, class_name))
+ if ReplacementModuleBase.is_registered(class_name):
+ replacement_cls = ReplacementModuleBase.get_replacement_class(class_name)
+ if replacement_cls.is_to_be_replaced(module):
+ modules_to_replace.append((name, module, class_name))
# Step 2: Replace modules
if modules_to_replace:
diff --git a/auto_round/special_model_handler.py b/auto_round/special_model_handler.py
index 453149f4e..e6079fbed 100644
--- a/auto_round/special_model_handler.py
+++ b/auto_round/special_model_handler.py
@@ -403,8 +403,10 @@ def _get_qwen2_5_omni_multimodal_block(model, quant_vision=False):
For quantization, we focus on:
- thinker.model.layers (text decoder layers) - main LLM layers
- - talker.model.layers (talker decoder layers)
- Optionally: visual encoder blocks, audio encoder layers
+
+ talker is excluded by default because quantizing it has been observed to
+ degrade audio quality in long-form generation.
"""
block_names = []
@@ -422,10 +424,6 @@ def _get_qwen2_5_omni_multimodal_block(model, quant_vision=False):
if hasattr(model, "thinker") and hasattr(model.thinker, "model") and hasattr(model.thinker.model, "layers"):
block_names.append([f"thinker.model.layers.{i}" for i in range(len(model.thinker.model.layers))])
- # Talker model layers (if available)
- if hasattr(model, "talker") and hasattr(model.talker, "model") and hasattr(model.talker.model, "layers"):
- block_names.append([f"talker.model.layers.{i}" for i in range(len(model.talker.model.layers))])
-
return block_names
@@ -439,8 +437,10 @@ def _get_qwen3_omni_moe_multimodal_block(model, quant_vision=False):
For quantization, we focus on:
- thinker.model.layers (text decoder layers) - main LLM layers
- - talker.model.layers (talker decoder layers)
- Optionally: visual encoder blocks, audio encoder layers
+
+ talker is excluded by default because quantizing it has been observed to
+ degrade audio quality in long-form generation .
"""
block_names = []
@@ -460,10 +460,6 @@ def _get_qwen3_omni_moe_multimodal_block(model, quant_vision=False):
if hasattr(model, "thinker") and hasattr(model.thinker, "model") and hasattr(model.thinker.model, "layers"):
block_names.append([f"thinker.model.layers.{i}" for i in range(len(model.thinker.model.layers))])
- # Talker model layers (if available)
- if hasattr(model, "talker") and hasattr(model.talker, "model") and hasattr(model.talker.model, "layers"):
- block_names.append([f"talker.model.layers.{i}" for i in range(len(model.talker.model.layers))])
-
return block_names
diff --git a/auto_round/utils/missing_tensors.py b/auto_round/utils/missing_tensors.py
index 255b7f92f..7f2dc922f 100644
--- a/auto_round/utils/missing_tensors.py
+++ b/auto_round/utils/missing_tensors.py
@@ -326,8 +326,6 @@ def copy_missing_tensors_from_source(
if not missing_tensor_names:
return
- logger.debug(f"Missing tensors detected: {missing_tensor_names}")
-
try:
from tqdm import tqdm as _tqdm
except ImportError:
@@ -721,12 +719,41 @@ def _woq_quantize_missing_tensors(target_dir: str, missing_tensors_dict: dict) -
return True
return False
+ if block_name_to_quantize:
+ existing_blocks = (
+ block_name_to_quantize
+ if isinstance(block_name_to_quantize, list)
+ else [b.strip() for b in block_name_to_quantize.split(",") if b.strip()]
+ )
+ existing_set = set(existing_blocks)
+ new_prefixes = {
+ ".".join(k.split(".")[:i])
+ for k in missing_tensors_dict
+ if k.endswith(".weight") and missing_tensors_dict[k].dim() == 2
+ for i, p in enumerate(k.split("."))
+ if p.isdigit() and ".".join(k.split(".")[:i])
+ }
+ added = sorted(new_prefixes - existing_set)
+ if added:
+ block_name_to_quantize = existing_blocks + added
+
# Identify weight tensors eligible for quantization (2D Linear weights)
def _is_eligible(k: str) -> bool:
if not k.endswith(".weight"):
return False
if missing_tensors_dict[k].dim() != 2:
return False
+ if k.startswith("talker."):
+ return False
+ # Only quantize weights that fall under the quantized blocks.
+ if block_name_to_quantize:
+ blocks = (
+ block_name_to_quantize
+ if isinstance(block_name_to_quantize, list)
+ else [b.strip() for b in block_name_to_quantize.split(",") if b.strip()]
+ )
+ if not any(k.startswith(b + ".") or k.startswith(b + "[") for b in blocks):
+ return False
layer_name = k[: -len(".weight")]
layer_cfg = _resolve_layer_cfg(layer_name)
# If extra_config explicitly covers this layer, trust its decision
diff --git a/test/test_cpu/models/test_omni_model.py b/test/test_cpu/models/test_omni_model.py
index 3c5d82548..746ac795f 100644
--- a/test/test_cpu/models/test_omni_model.py
+++ b/test/test_cpu/models/test_omni_model.py
@@ -96,17 +96,17 @@ class TestQwen2_5Omni:
shutil.rmtree("runs", ignore_errors=True)
def test_block_names_default(self):
- """Test that get_block_names returns thinker + talker layers."""
+ """Test that get_block_names returns thinker layers only by default."""
from auto_round.utils import get_block_names
block_names = get_block_names(self.model, quant_vision=False)
- # Should have thinker.model.layers and talker.model.layers
+ # Should have thinker.model.layers but not talker.model.layers
assert any(
"thinker.model.layers" in str(b) for b in block_names
), f"Expected thinker.model.layers in block_names, got: {block_names}"
- assert any(
+ assert not any(
"talker.model.layers" in str(b) for b in block_names
- ), f"Expected talker.model.layers in block_names, got: {block_names}"
+ ), f"Did not expect talker.model.layers by default, got: {block_names}"
def test_block_names_quant_vision(self):
"""Test that quant_vision adds visual and audio blocks."""
@@ -167,7 +167,7 @@ class TestQwen3OmniMoeBlockNames:
"""Test block name discovery for Qwen3-Omni-MoE."""
def test_block_names_default(self):
- """Test that get_block_names returns thinker + talker layers."""
+ """Test that get_block_names returns thinker layers only by default."""
from auto_round.utils import get_block_names
config = _make_tiny_qwen3_omni_moe_config()
@@ -177,9 +177,9 @@ class TestQwen3OmniMoeBlockNames:
assert any(
"thinker.model.layers" in str(b) for b in block_names
), f"Expected thinker.model.layers, got: {block_names}"
- assert any(
+ assert not any(
"talker.model.layers" in str(b) for b in block_names
- ), f"Expected talker.model.layers, got: {block_names}"
+ ), f"Did not expect talker.model.layers by default, got: {block_names}"
def test_block_names_quant_vision(self):
"""Test that quant_vision adds visual and audio blocks."""
@@ -213,15 +213,15 @@ class TestQwen3OmniMoeReplacement:
"""Test MoE module replacement for Qwen3-Omni-MoE."""
def test_replacement_registered(self):
- """Test that both thinker and talker MoE blocks are registered."""
+ """Test that thinker MoE block is registered."""
from auto_round.modeling.fused_moe.qwen3_omni import (
- LinearQwen3OmniTalkerSparseMoeBlock,
LinearQwen3OmniThinkerSparseMoeBlock,
)
from auto_round.modeling.fused_moe.replace_modules import ReplacementModuleBase
assert ReplacementModuleBase.is_registered("Qwen3OmniMoeThinkerTextSparseMoeBlock")
- assert ReplacementModuleBase.is_registered("Qwen3OmniMoeTalkerTextSparseMoeBlock")
+ # Talker has no replacement class — it stays fused and is excluded via MOE_SKIP_PREFIXES
+ assert not ReplacementModuleBase.is_registered("Qwen3OmniMoeTalkerTextSparseMoeBlock")
def test_builtin_modules_entry(self):
"""Test that qwen3_omni_moe is in BUILTIN_MODULES."""
@@ -238,28 +238,38 @@ class TestQwen3OmniMoeReplacement:
assert is_custom_model(model)
def test_apply_replacements(self):
- """Test that MoE blocks are correctly replaced."""
+ """Test that only thinker MoE blocks are replaced; talker stays fused."""
from auto_round.modeling.fused_moe.replace_modules import apply_replacements
config = _make_tiny_qwen3_omni_moe_config()
model = Qwen3OmniMoeForConditionalGeneration(config)
+ original_talker_class = model.talker.model.layers[0].mlp.__class__.__name__
model = apply_replacements(model)
- # Check that thinker MoE was replaced
+ # Check that thinker MoE was custom-replaced
thinker_mlp = model.thinker.model.layers[0].mlp
assert (
"LinearQwen3OmniThinker" in thinker_mlp.__class__.__name__
), f"Expected LinearQwen3OmniThinker, got {thinker_mlp.__class__.__name__}"
- # Check that talker MoE was replaced
+ # Talker MoE stays fused (excluded from both custom replacement and generic unfuse)
+ # to prevent memory eviction during disk-offloaded quantization.
+ # ShardWriter handles fused→per-expert conversion at save time.
talker_mlp = model.talker.model.layers[0].mlp
assert (
- "LinearQwen3OmniTalker" in talker_mlp.__class__.__name__
- ), f"Expected LinearQwen3OmniTalker, got {talker_mlp.__class__.__name__}"
+ talker_mlp.__class__.__name__ == original_talker_class
+ ), f"Expected talker MoE to stay original ({original_talker_class}), got {talker_mlp.__class__.__name__}"
+ assert hasattr(
+ model.talker.model.layers[0].mlp.experts, "gate_up_proj"
+ ), "Talker experts should keep fused gate_up_proj"
def test_weight_fidelity(self):
- """Test that unfused weights match original fused weights."""
+ """Test that unfused thinker weights match original fused weights.
+
+ Talker MoE stays fused (excluded from quantization), so only
+ thinker experts are verified here.
+ """
from auto_round.modeling.fused_moe.replace_modules import apply_replacements, materialize_model_
torch.manual_seed(42)
@@ -270,11 +280,9 @@ class TestQwen3OmniMoeReplacement:
config = _make_tiny_qwen3_omni_moe_config()
model = Qwen3OmniMoeForConditionalGeneration(config)
- # Save original fused weights
+ # Save original fused thinker weights
thinker_gate_up = model.thinker.model.layers[0].mlp.experts.gate_up_proj.data.clone()
thinker_down = model.thinker.model.layers[0].mlp.experts.down_proj.data.clone()
- talker_gate_up = model.talker.model.layers[0].mlp.experts.gate_up_proj.data.clone()
- talker_down = model.talker.model.layers[0].mlp.experts.down_proj.data.clone()
model = apply_replacements(model)
materialize_model_(model)
@@ -289,20 +297,13 @@ class TestQwen3OmniMoeReplacement:
assert_same_weights(expert.up_proj.weight.data, thinker_gate_up[i, intermediate:, :])
assert_same_weights(expert.down_proj.weight.data, thinker_down[i])
- # Verify talker expert weights
- for i in range(4):
- expert = model.talker.model.layers[0].mlp.experts[i]
- assert_same_weights(expert.gate_proj.weight.data, talker_gate_up[i, :intermediate, :])
- assert_same_weights(expert.up_proj.weight.data, talker_gate_up[i, intermediate:, :])
- assert_same_weights(expert.down_proj.weight.data, talker_down[i])
-
def test_forward_output_match(self):
- """Test that replaced MoE forward output matches original."""
+ """Test that replaced thinker MoE forward output matches original.
+
+ Talker MoE is not replaced, so only thinker output is verified.
+ """
from auto_round.modeling.fused_moe.replace_modules import apply_replacements, materialize_model_
- # Fix seed for deterministic model weights and use small-scale input to
- # prevent numerical overflow with random weights (talker has a larger
- # shared_expert MLP that can produce NaN otherwise).
torch.manual_seed(0)
config = _make_tiny_qwen3_omni_moe_config()
model = Qwen3OmniMoeForConditionalGeneration(config)
@@ -310,31 +311,22 @@ class TestQwen3OmniMoeReplacement:
x = torch.randn(1, 4, 64) * 0.1
with torch.no_grad():
orig_thinker_out = model.thinker.model.layers[0].mlp(x)
- orig_talker_out = model.talker.model.layers[0].mlp(x)
model = apply_replacements(model)
materialize_model_(model)
with torch.no_grad():
new_thinker_out = model.thinker.model.layers[0].mlp(x)
- new_talker_out = model.talker.model.layers[0].mlp(x)
-
- # Use a NaN-safe comparison: check that NaN positions match, then
- # compare finite values. This avoids false failures when random
- # weights cause some outputs to overflow to NaN.
- for name, orig, new in [
- ("Thinker", orig_thinker_out, new_thinker_out),
- ("Talker", orig_talker_out, new_talker_out),
- ]:
- assert orig.shape == new.shape, f"{name} shape mismatch: {orig.shape} vs {new.shape}"
- orig_nan = torch.isnan(orig)
- new_nan = torch.isnan(new)
- assert torch.equal(orig_nan, new_nan), f"{name} NaN positions differ"
- finite_mask = ~orig_nan
- if finite_mask.any():
- assert torch.allclose(
- orig[finite_mask], new[finite_mask], atol=1e-5
- ), f"{name} MoE forward mismatch on finite values"
+
+ assert orig_thinker_out.shape == new_thinker_out.shape, "Thinker shape mismatch"
+ orig_nan = torch.isnan(orig_thinker_out)
+ new_nan = torch.isnan(new_thinker_out)
+ assert torch.equal(orig_nan, new_nan), "Thinker NaN positions differ"
+ finite_mask = ~orig_nan
+ if finite_mask.any():
+ assert torch.allclose(
+ orig_thinker_out[finite_mask], new_thinker_out[finite_mask], atol=1e-5
+ ), "Thinker MoE forward mismatch on finite values"
class TestQwen3OmniMoeProcessor:
diff --git a/test/test_cpu/utils/test_missing_tensors.py b/test/test_cpu/utils/test_missing_tensors.py
index 88dd4a329..ae7ad83cc 100644
--- a/test/test_cpu/utils/test_missing_tensors.py
+++ b/test/test_cpu/utils/test_missing_tensors.py
@@ -584,3 +584,88 @@ class TestCopyMissingTensorsFromSource:
_write_config(tgt)
copy_missing_tensors_from_source(src, tgt)
assert not os.path.exists(os.path.join(tgt, "model_extra_tensors.safetensors"))
+
+ def test_talker_missing_weight_is_never_woq_quantized(self):
+ """Talker weights must stay BF16/full precision even in WOQ exports."""
+ with tempfile.TemporaryDirectory() as source_dir, tempfile.TemporaryDirectory() as target_dir:
+ talker_weight = torch.randn(32, 64, dtype=torch.bfloat16)
+ _save_safetensors(
+ {"talker.model.layers.0.mlp.experts.0.up_proj.weight": talker_weight},
+ os.path.join(source_dir, "model.safetensors"),
+ )
+ _save_safetensors(
+ {
+ "thinker.model.layers.0.mlp.experts.0.up_proj.qweight": torch.randint(
+ 0, 16, (8, 32), dtype=torch.int32
+ )
+ },
+ os.path.join(target_dir, "model.safetensors"),
+ )
+ with open(os.path.join(target_dir, "config.json"), "w") as f:
+ json.dump(
+ _make_auto_round_config(bits=4, group_size=64, sym=True)
+ | {
+ "quantization_config": _make_auto_round_config(bits=4, group_size=64, sym=True)[
+ "quantization_config"
+ ]
+ | {"block_name_to_quantize": "thinker.model.layers"}
+ },
+ f,
+ )
+
+ copy_missing_tensors_from_source(source_dir, target_dir)
+
+ result = _load_safetensors(os.path.join(target_dir, "model_extra_tensors.safetensors"))
+ assert "talker.model.layers.0.mlp.experts.0.up_proj.weight" in result
+ assert "talker.model.layers.0.mlp.experts.0.up_proj.qweight" not in result
+
+ def test_talker_fused_projection_not_treated_as_missing_when_expanded_components_exist(self):
+ """Talker fused projection keys must not be marked missing if their per-expert 2D
+ components are already saved (via _expand_fused_experts during export).
+
+ This is the inverse of the previous test: the target already has the per-expert
+ 2D weights that _expand_fused_experts produced from the fused 3D source key.
+ copy_missing_tensors_from_source should NOT re-split the source fused 3D key
+ and add duplicate per-expert weights.
+ """
+ with tempfile.TemporaryDirectory() as source_dir, tempfile.TemporaryDirectory() as target_dir:
+ # Source has fused 3D expert params (what the original checkpoint looks like)
+ _save_safetensors(
+ {
+ # These are the original fused 3D keys that appear in the source checkpoint.
+ # In practice num_experts=4 in the tiny test config.
+ "talker.model.layers.0.mlp.experts.gate_up_proj": torch.randn(4, 16, 32),
+ "talker.model.layers.0.mlp.experts.down_proj": torch.randn(4, 32, 16),
+ },
+ os.path.join(source_dir, "model.safetensors"),
+ )
+ # Target already has the per-expert 2D weights that _expand_fused_experts
+ # produced during export. These are the correct shape expected by transformers.
+ _save_safetensors(
+ {
+ # gate_up_proj split into gate_proj + up_proj
+ "talker.model.layers.0.mlp.experts.0.gate_proj.weight": torch.randn(8, 32),
+ "talker.model.layers.0.mlp.experts.0.up_proj.weight": torch.randn(8, 32),
+ "talker.model.layers.0.mlp.experts.1.gate_proj.weight": torch.randn(8, 32),
+ "talker.model.layers.0.mlp.experts.1.up_proj.weight": torch.randn(8, 32),
+ # down_proj as per-expert 2D
+ "talker.model.layers.0.mlp.experts.0.down_proj.weight": torch.randn(32, 8),
+ "talker.model.layers.0.mlp.experts.1.down_proj.weight": torch.randn(32, 8),
+ # Also have some thinker quantized weights
+ "thinker.model.layers.0.mlp.experts.0.gate_proj.qweight": torch.randint(
+ 0, 16, (8, 32), dtype=torch.int32
+ ),
+ },
+ os.path.join(target_dir, "model.safetensors"),
+ )
+ _write_config(target_dir)
+
+ copy_missing_tensors_from_source(source_dir, target_dir)
+
+ # The fused keys must NOT be treated as missing — their expanded
+ # per-expert 2D components are already in the target.
+ extra_shard = os.path.join(target_dir, "model_extra_tensors.safetensors")
+ assert not os.path.exists(extra_shard), (
+ "Fused talker expert keys should not be treated as missing when "
+ "their per-expert 2D components are already saved"
+ )
diff --git a/test/test_cpu/utils/test_shard_writer.py b/test/test_cpu/utils/test_shard_writer.py
index 15e78a4ae..dfd092b25 100644
--- a/test/test_cpu/utils/test_shard_writer.py
+++ b/test/test_cpu/utils/test_shard_writer.py
@@ -81,6 +81,22 @@ class _LMStyleModel(torch.nn.Module):
self.config = SimpleNamespace(model_type="toy-lm", tie_word_embeddings=True)
+class _ToyExperts(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.is_transposed = False
+
+
+class _FusedExpertsModel(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.block = torch.nn.Module()
+ self.block.experts = _ToyExperts()
+ self.talker = torch.nn.Module()
+ self.talker.experts = _ToyExperts()
+ self.config = SimpleNamespace(model_type="qwen3_omni_moe")
+
+
def test_finalize_skips_lm_head_when_tie_word_embeddings_true(tmp_path, monkeypatch):
"""Complementary test: when tie_word_embeddings=True the lm_head should be
skipped (not written to disk) and offloaded to meta."""
@@ -100,6 +116,51 @@ def test_finalize_skips_lm_head_when_tie_word_embeddings_true(tmp_path, monkeypa
assert model.lm_head.weight.device.type == "meta"
+def test_expand_fused_experts_for_skipped_talker_prefix(tmp_path, monkeypatch):
+ """Talker fused 3D weights must be expanded to exact per-expert 2D keys.
+
+ Real Qwen3-Omni-MoE exports attach reverse checkpoint conversion mappings for
+ MoE projections. If we apply that mapping to the fused talker tensor before
+ expanding it, the fused tensor is saved under a wildcard key such as
+ ``talker.experts.*.gate_proj.weight``. That breaks reload because
+ transformers expects concrete per-expert 2D keys after save_pretrained.
+ """
+ model = _FusedExpertsModel()
+ writer = _make_writer(model, str(tmp_path), monkeypatch)
+ writer.reverse_checkpoint_conversion_mapping = {
+ r"experts\.gate_up_proj$": ["experts.*.gate_proj.weight", "experts.*.up_proj.weight"]
+ }
+
+ fused_gate_up = torch.arange(2 * 6 * 4, dtype=torch.float32).reshape(2, 6, 4)
+ writer._add_tensor("talker.experts.gate_up_proj", fused_gate_up)
+ writer.finalize()
+
+ shard_path = os.path.join(tmp_path, "model.bin")
+ saved_tensors = torch.load(shard_path, map_location="cpu")
+
+ assert "talker.experts.gate_up_proj" not in saved_tensors
+ assert "talker.experts.*.gate_proj.weight" not in saved_tensors
+ assert "talker.experts.0.gate_proj.weight" in saved_tensors
+ assert "talker.experts.0.up_proj.weight" in saved_tensors
+ assert torch.equal(saved_tensors["talker.experts.0.gate_proj.weight"], fused_gate_up[0, :3, :])
+ assert torch.equal(saved_tensors["talker.experts.0.up_proj.weight"], fused_gate_up[0, 3:, :])
+
+
+def test_do_not_expand_fused_experts_outside_skipped_prefixes(tmp_path, monkeypatch):
+ model = _FusedExpertsModel()
+ writer = _make_writer(model, str(tmp_path), monkeypatch)
+
+ fused_gate_up = torch.arange(2 * 6 * 4, dtype=torch.float32).reshape(2, 6, 4)
+ writer._add_tensor("block.experts.gate_up_proj", fused_gate_up)
+ writer.finalize()
+
+ shard_path = os.path.join(tmp_path, "model.bin")
+ saved_tensors = torch.load(shard_path, map_location="cpu")
+
+ assert "block.experts.gate_up_proj" in saved_tensors
+ assert "block.experts.0.gate_proj.weight" not in saved_tensors
+
+
def test_finalize_offloads_module_with_tensor_in_parameters(tmp_path, monkeypatch):
model = _DiffusionStyleModel()
model.transformer_blocks[0].linear._parameters["weight"] = model.transformer_blocks[0].linear.weight.to("cpu")
diff --git a/test/test_xpu/test_autoround.py b/test/test_xpu/test_autoround.py
index d03c71444..473f73451 100644
--- a/test/test_xpu/test_autoround.py
+++ b/test/test_xpu/test_autoround.py
@@ -155,7 +155,7 @@ class TestAutoRoundXPU:
model = Qwen2VLForConditionalGeneration.from_pretrained(
quantized_model_path,
- torch_dtype="float16",
+ torch_dtype="auto",
device_map=self.device,
quantization_config=quantization_config,
)