PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
Support Qwen3 and Qwen2.5 Omni model quantization (#1404)
Liang Lv91af35ff77b0
diff --git a/auto_round/auto_scheme/utils.py b/auto_round/auto_scheme/utils.pyindex df1783d83..51b8da8e1 100644--- a/auto_round/auto_scheme/utils.py+++ b/auto_round/auto_scheme/utils.py@@ -27,6 +27,7 @@ from auto_round.utils import (get_layer_features,get_module,is_hpex_available,+ normalize_no_split_modules,parse_available_devices,)@@ -219,7 +220,7 @@ def dispatch_model_by_all_available_devices(if device_map is None:device_map = 0- no_split_modules = list(getattr(model, "_no_split_modules", []))+ no_split_modules = normalize_no_split_modules(getattr(model, "_no_split_modules", []))if device_map == "auto":max_memory = get_balanced_memory(model,diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.pyindex 20514fba1..e15f918aa 100644--- a/auto_round/compressors/base.py+++ b/auto_round/compressors/base.py@@ -101,6 +101,7 @@ from auto_round.utils import (llm_load_model,memory_monitor,mv_module_from_gpu,+ normalize_no_split_modules,set_amax_for_all_moe_layers,set_module,to_device,diff --git a/auto_round/compressors/mllm/processor.py b/auto_round/compressors/mllm/processor.pyindex 9d98be1e9..1063beed4 100644--- a/auto_round/compressors/mllm/processor.py+++ b/auto_round/compressors/mllm/processor.py@@ -196,6 +196,76 @@ class Qwen2VLProcessor(HFProcessor):return ret+@register_processor("qwen2_5_omni")+class Qwen2_5OmniProcessor(HFProcessor):+ """Processor for Qwen2.5-Omni multimodal models.++ Qwen2.5-Omni supports text, image, video, and audio inputs.+ This processor handles proper tokenization and preprocessing for calibration.+ """++ @staticmethod+ def squeeze_result(ret):+ for key in ret:+ # Skip squeezing for multi-modal data that may have special dimensions+ if key in ["pixel_values", "pixel_values_videos", "input_features"]:+ continue+ ret[key] = ret[key][0]+ return ret++ def _process_v1(self, messages, image):+ """Process messages for Qwen2.5-Omni model."""+ conversation = []+ for content in messages:+ conversation.append(+ {+ "role": content["role"],+ "content": [{"text": content["content"].replace(self.IMAGE_TOKEN, ""), "type": "text"}],+ }+ )+ if self.IMAGE_TOKEN in content["content"]:+ conversation[-1]["content"].append({"image": image, "type": "image"})+ ret = self.processor.apply_chat_template(+ conversation, add_generation_prompt=True, tokenize=True, return_dict=True+ )+ return ret+++@register_processor("qwen3_omni")+class Qwen3OmniProcessor(HFProcessor):+ """Processor for Qwen3-Omni multimodal models.++ Qwen3-Omni supports text, image, video, and audio inputs.+ This processor handles proper tokenization and preprocessing for calibration.+ """++ @staticmethod+ def squeeze_result(ret):+ for key in ret:+ # Skip squeezing for multi-modal data that may have special dimensions+ if key in ["pixel_values", "pixel_values_videos", "input_features"]:+ continue+ ret[key] = ret[key][0]+ return ret++ def _process_v1(self, messages, image):+ """Process messages for Qwen3-Omni model."""+ conversation = []+ for content in messages:+ conversation.append(+ {+ "role": content["role"],+ "content": [{"text": content["content"].replace(self.IMAGE_TOKEN, ""), "type": "text"}],+ }+ )+ if self.IMAGE_TOKEN in content["content"]:+ conversation[-1]["content"].append({"image": image, "type": "image"})+ ret = self.processor.apply_chat_template(+ conversation, add_generation_prompt=True, tokenize=True, return_dict=True+ )+ return ret++@register_processor("cogvlm2")class CogVLM2Processor(BasicProcessor):def get_input(self, text, images, truncation=False, squeeze=True, max_length=None, **kwargs):diff --git a/auto_round/compressors/mllm/template.py b/auto_round/compressors/mllm/template.pyindex 75190a091..ce2825a47 100644--- a/auto_round/compressors/mllm/template.py+++ b/auto_round/compressors/mllm/template.py@@ -119,6 +119,8 @@ def _register_template(_register_template("qwen2_vl", default_dataset="NeelNanda/pile-10k", processor=PROCESSORS["qwen2_vl"])_register_template("qwen2_5_vl", default_dataset="NeelNanda/pile-10k", processor=PROCESSORS["qwen2_vl"])+_register_template("qwen2_5_omni", default_dataset="NeelNanda/pile-10k", processor=PROCESSORS["qwen2_5_omni"])+_register_template("qwen3_omni_moe", default_dataset="NeelNanda/pile-10k", processor=PROCESSORS["qwen3_omni"])_register_template("mllama", default_dataset="liuhaotian/llava", processor=PROCESSORS["hf"])_register_template("deepseek_vl_v2", default_dataset="NeelNanda/pile-10k", processor=PROCESSORS["deepseek_v2"])_register_template("mistral3", default_dataset="NeelNanda/pile-10k", processor=PROCESSORS["hf"])diff --git a/auto_round/compressors/mllm/utils.py b/auto_round/compressors/mllm/utils.pyindex e8535666c..34609d964 100644--- a/auto_round/compressors/mllm/utils.py+++ b/auto_round/compressors/mllm/utils.py@@ -27,6 +27,9 @@ VISUAL_KEYS = ["audio","talker","token2wav",+ "code2wav",+ "audio_tower",+ "code_predictor","multi_modal_projector","vision_tower","multimodal_projector",diff --git a/auto_round/compressors/shard_writer.py b/auto_round/compressors/shard_writer.pyindex 20e959e8e..f61f7394d 100644--- a/auto_round/compressors/shard_writer.py+++ b/auto_round/compressors/shard_writer.py@@ -171,8 +171,18 @@ class ShardWriter:# 1. Capture remaining weights not yet savedfull_sd = self.model.state_dict()tie_word_embeddings = False+ config = getattr(self.model, "config", None)if hasattr(self.model, "config") and hasattr(self.model.config, "tie_word_embeddings"):tie_word_embeddings = self.model.config.tie_word_embeddings+ if tie_word_embeddings is None:+ # For multimodal models, check nested text/thinker configs+ for sub_attr in ("text_config", "thinker_config", "language_config", "llm_config"):+ sub_config = getattr(config, sub_attr, None)+ if sub_config is not None:+ val = getattr(sub_config, "tie_word_embeddings", None)+ if val is not None:+ tie_word_embeddings = val+ breakfinalize_skipped_meta_tensors = []for pname, tensor in full_sd.items():diff --git a/auto_round/inference/convert_model.py b/auto_round/inference/convert_model.pyindex 1e4324244..ba746e1c7 100644--- a/auto_round/inference/convert_model.py+++ b/auto_round/inference/convert_model.py@@ -493,6 +493,10 @@ def post_init(model: torch.nn.Module, used_backends: list[str]) -> None:used_backends (List[str]): List of backend names used for quantization."""+ from auto_round.utils.common import monkey_patch_model++ monkey_patch_model(model)+need_autogptq_init = Falseneed_gptqmodel_init = Falseneed_ipex_init = Falsediff --git a/auto_round/modeling/fused_moe/qwen3_omni.py b/auto_round/modeling/fused_moe/qwen3_omni.pynew file mode 100644index 000000000..ad2004592--- /dev/null+++ b/auto_round/modeling/fused_moe/qwen3_omni.py@@ -0,0 +1,237 @@+# Copyright (c) 2026 Intel Corporation+# SPDX-License-Identifier: Apache-2.0++"""MoE module replacements for Qwen3-Omni model.++Qwen3-Omni has MoE blocks in both thinker and talker:+- Thinker: Qwen3OmniMoeThinkerTextSparseMoeBlock (experts + gate, no shared expert)+- Talker: Qwen3OmniMoeTalkerTextSparseMoeBlock (experts + gate + shared_expert + shared_expert_gate)++This module provides replacement classes that unfuse fused expert weights (3D Parameters)+into individual nn.Linear layers, enabling per-expert quantization with meta device optimization.+"""++import torch++from auto_round.modeling.fused_moe.replace_modules import ReplacementModuleBase+from auto_round.modeling.fused_moe.utils import _update_parameter+from auto_round.utils import clear_memory, unsupported_meta_device++# ---------------------------------------------------------------------------+# Thinker MoE replacement (no shared expert)+# ---------------------------------------------------------------------------+++class LinearQwen3OmniThinkerSparseMoeBlock(ReplacementModuleBase):+ """Calibration replacement for Qwen3OmniMoeThinkerTextSparseMoeBlock.++ Unfuses fused expert weights into individual nn.Linear layers for+ per-expert quantization. Uses meta device to avoid doubling memory.++ Structure: gate (router) + experts (unfused).+ """++ def __init__(self, original, config):+ super().__init__(original)+ self.gate = original.gate+ self.num_experts = original.experts.num_experts+ text_config = config.thinker_config.text_config+ with torch.device("meta"):+ self.experts = SequentialQwen3OmniThinkerExperts(text_config, original.experts)++ @classmethod+ def original_module_class(cls) -> str:+ return "Qwen3OmniMoeThinkerTextSparseMoeBlock"++ def _materialize_weights(self) -> None:+ original = self._get_original_module()+ self.experts._materialize_weights(original.experts)+ clear_memory()++ def experts_forward(self, hidden_states, top_k_index, top_k_weights):+ final_hidden_states = torch.zeros_like(hidden_states)+ with torch.no_grad():+ expert_mask = torch.nn.functional.one_hot(top_k_index, num_classes=self.num_experts)+ expert_mask = expert_mask.permute(2, 1, 0)+ expert_hit = torch.greater(expert_mask.sum(dim=(-1, -2)), 0).nonzero()++ for expert_idx in expert_hit:+ expert_idx = expert_idx[0]+ if expert_idx == self.num_experts:+ continue+ top_k_pos, token_idx = torch.where(expert_mask[expert_idx])+ current_state = hidden_states[token_idx]+ current_hidden_states = self.experts[expert_idx](current_state)+ current_hidden_states = current_hidden_states * top_k_weights[token_idx, top_k_pos, None]+ final_hidden_states.index_add_(0, token_idx, current_hidden_states.to(final_hidden_states.dtype))++ return final_hidden_states++ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:+ batch_size, sequence_length, hidden_dim = hidden_states.shape+ hidden_states_reshaped = hidden_states.view(-1, hidden_dim)+ _, routing_weights, selected_experts = self.gate(hidden_states_reshaped)+ expert_output = self.experts_forward(hidden_states_reshaped, selected_experts, routing_weights)+ return expert_output.reshape(batch_size, sequence_length, hidden_dim)++ @classmethod+ def from_original(cls, original, config, **kwargs):+ return cls(original, config)+++# ---------------------------------------------------------------------------+# Talker MoE replacement (with shared expert, same pattern as qwen3_5_moe)+# ---------------------------------------------------------------------------+++class LinearQwen3OmniTalkerSparseMoeBlock(ReplacementModuleBase):+ """Calibration replacement for Qwen3OmniMoeTalkerTextSparseMoeBlock.++ Unfuses fused expert weights and preserves the shared_expert + shared_expert_gate.+ Similar to Qwen3.5-MoE pattern.++ Structure: gate (router) + experts (unfused) + shared_expert + shared_expert_gate.+ """++ def __init__(self, original, config):+ super().__init__(original)+ self.gate = original.gate+ self.shared_expert = original.shared_expert+ self.shared_expert_gate = original.shared_expert_gate+ self.num_experts = original.experts.num_experts+ text_config = config.talker_config.text_config+ with torch.device("meta"):+ self.experts = SequentialQwen3OmniTalkerExperts(text_config, original.experts)++ @classmethod+ def original_module_class(cls) -> str:+ return "Qwen3OmniMoeTalkerTextSparseMoeBlock"++ def _materialize_weights(self) -> None:+ original = self._get_original_module()+ self.experts._materialize_weights(original.experts)+ clear_memory()++ def experts_forward(self, hidden_states, top_k_index, top_k_weights):+ final_hidden_states = torch.zeros_like(hidden_states)+ with torch.no_grad():+ expert_mask = torch.nn.functional.one_hot(top_k_index, num_classes=self.num_experts)+ expert_mask = expert_mask.permute(2, 1, 0)+ expert_hit = torch.greater(expert_mask.sum(dim=(-1, -2)), 0).nonzero()++ for expert_idx in expert_hit:+ expert_idx = expert_idx[0]+ if expert_idx == self.num_experts:+ continue+ top_k_pos, token_idx = torch.where(expert_mask[expert_idx])+ current_state = hidden_states[token_idx]+ current_hidden_states = self.experts[expert_idx](current_state)+ current_hidden_states = current_hidden_states * top_k_weights[token_idx, top_k_pos, None]+ final_hidden_states.index_add_(0, token_idx, current_hidden_states.to(final_hidden_states.dtype))++ return final_hidden_states++ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:+ batch_size, sequence_length, hidden_dim = hidden_states.shape+ hidden_states_reshaped = hidden_states.view(-1, hidden_dim)+ shared_expert_output = self.shared_expert(hidden_states_reshaped)+ _, routing_weights, selected_experts = self.gate(hidden_states_reshaped)+ expert_output = self.experts_forward(hidden_states_reshaped, selected_experts, routing_weights)++ shared_expert_output = torch.sigmoid(self.shared_expert_gate(hidden_states_reshaped)) * shared_expert_output++ expert_output = expert_output + shared_expert_output+ return expert_output.reshape(batch_size, sequence_length, hidden_dim)++ @classmethod+ def from_original(cls, original, config, **kwargs):+ return cls(original, config)+++# ---------------------------------------------------------------------------+# Sequential expert containers (unfused nn.Linear per expert)+# ---------------------------------------------------------------------------+++class SequentialQwen3OmniThinkerExperts(torch.nn.ModuleList):+ """Unfused per-expert nn.Linear layers for Qwen3-Omni thinker MoE.++ Replaces fused 3D Parameters (gate_up_proj, down_proj) with individual+ Qwen3OmniMoeThinkerTextMLP modules per expert.+ """++ def __init__(self, config, original):+ super().__init__()+ self.num_experts = original.gate_up_proj.shape[0]+ intermediate_size = config.moe_intermediate_size++ from transformers.models.qwen3_omni_moe.modeling_qwen3_omni_moe import (+ Qwen3OmniMoeThinkerTextMLP,+ )++ with torch.device("meta"):+ super().__init__([Qwen3OmniMoeThinkerTextMLP(config, intermediate_size) for _ in range(self.num_experts)])++ def _materialize_weights(self, original) -> None:+ """Unfuse fused expert weights into individual nn.Linear layers.++ gate_up_proj shape: (num_experts, 2 * moe_intermediate, hidden)+ down_proj shape: (num_experts, hidden, moe_intermediate)+ """+ intermediate_size = original.down_proj.shape[-1]+ if not unsupported_meta_device(original):+ for i in range(self.num_experts):+ gate_up = original.gate_up_proj[i]+ down = original.down_proj[i]++ gate_proj = gate_up[:intermediate_size, :]+ up_proj = gate_up[intermediate_size:, :]++ _update_parameter(self[i].gate_proj, "weight", gate_proj.contiguous())+ _update_parameter(self[i].up_proj, "weight", up_proj.contiguous())+ _update_parameter(self[i].down_proj, "weight", down.contiguous())+ del gate_up, down, gate_proj, up_proj+ original.to_empty(device="meta") # release original fused parameters+ clear_memory()+++class SequentialQwen3OmniTalkerExperts(torch.nn.ModuleList):+ """Unfused per-expert nn.Linear layers for Qwen3-Omni talker MoE.++ Replaces fused 3D Parameters (gate_up_proj, down_proj) with individual+ Qwen3OmniMoeTalkerTextMLP modules per expert.+ """++ def __init__(self, config, original):+ super().__init__()+ self.num_experts = original.gate_up_proj.shape[0]+ intermediate_size = config.moe_intermediate_size++ from transformers.models.qwen3_omni_moe.modeling_qwen3_omni_moe import (+ Qwen3OmniMoeTalkerTextMLP,+ )++ with torch.device("meta"):+ super().__init__([Qwen3OmniMoeTalkerTextMLP(config, intermediate_size) for _ in range(self.num_experts)])++ def _materialize_weights(self, original) -> None:+ """Unfuse fused expert weights into individual nn.Linear layers.++ gate_up_proj shape: (num_experts, 2 * moe_intermediate, hidden)+ down_proj shape: (num_experts, hidden, moe_intermediate)+ """+ intermediate_size = original.down_proj.shape[-1]+ if not unsupported_meta_device(original):+ for i in range(self.num_experts):+ gate_up = original.gate_up_proj[i]+ down = original.down_proj[i]++ gate_proj = gate_up[:intermediate_size, :]+ up_proj = gate_up[intermediate_size:, :]++ _update_parameter(self[i].gate_proj, "weight", gate_proj.contiguous())+ _update_parameter(self[i].up_proj, "weight", up_proj.contiguous())+ _update_parameter(self[i].down_proj, "weight", down.contiguous())+ del gate_up, down, gate_proj, up_proj+ original.to_empty(device="meta") # release original fused parameters+ clear_memory()diff --git a/auto_round/modeling/fused_moe/replace_modules.py b/auto_round/modeling/fused_moe/replace_modules.pyindex f18cc2151..e7993d109 100644--- a/auto_round/modeling/fused_moe/replace_modules.py+++ b/auto_round/modeling/fused_moe/replace_modules.py@@ -39,6 +39,8 @@ BUILTIN_MODULES = {"qwen3_5_moe_text": LazyImport("auto_round.modeling.fused_moe.qwen3_5_moe"),# Step 3.5 MoE: splits fused MoELinear into per-expert nn.Linear"step3p5": LazyImport("auto_round.modeling.fused_moe.step3_5_moe"),+ # Qwen3-Omni MoE: thinker (no shared expert) + talker (with shared expert)+ "qwen3_omni_moe": LazyImport("auto_round.modeling.fused_moe.qwen3_omni"),}diff --git a/auto_round/special_model_handler.py b/auto_round/special_model_handler.pyindex fa0000bce..5e271f691 100644--- a/auto_round/special_model_handler.py+++ b/auto_round/special_model_handler.py@@ -21,7 +21,14 @@ from auto_round.formats import OutputFormatfrom auto_round.modeling.fused_moe.replace_modules import apply_replacements, release_original_module_from auto_round.utils import is_moe_model_via_config, logger-mllms_with_limited_bs = ("llava", "qwen2_vl", "phi3_v", "mllama") # Limitations on batch_size+mllms_with_limited_bs = (+ "llava",+ "qwen2_vl",+ "phi3_v",+ "mllama",+ "qwen2_5_omni",+ "qwen3_omni_moe",+) # Limitations on batch_sizeSUPPORT_ONLY_TEXT_MODELS = ["phi3_v",@@ -36,6 +43,8 @@ SUPPORT_ONLY_TEXT_MODELS = ["internvl_chat","glm4v_moe","qwen3_vl_moe",+ "qwen2_5_omni",+ "qwen3_omni_moe","gemma3",]@@ -53,6 +62,14 @@ def _handle_special_model(model):from functools import partialmodel.forward = partial(_deepseek_vl2_forward, model)+ if hasattr(model, "config") and model.config.model_type == "qwen2_5_omni":+ from functools import partial++ model.forward = partial(_qwen2_5_omni_forward, model)+ if hasattr(model, "config") and model.config.model_type == "qwen3_omni_moe":+ from functools import partial++ model.forward = partial(_qwen3_omni_moe_forward, model)return model@@ -80,7 +97,85 @@ def _get_deepseek_vl2_multimodal_block(model, quant_vision=False):return block_names-SPECIAL_MULTIMODAL_BLOCK = {"deepseek_vl_v2": _get_deepseek_vl2_multimodal_block}+def _get_qwen2_5_omni_multimodal_block(model, quant_vision=False):+ """Get block names for Qwen2.5-Omni model.++ Qwen2.5-Omni has the following structure:+ - thinker: Contains audio_tower, visual, model (text decoder)+ - talker: Contains model (talker decoder)+ - token2wav: Audio decoder++ For quantization, we focus on:+ - thinker.model.layers (text decoder layers) - main LLM layers+ - talker.model.layers (talker decoder layers)+ - Optionally: visual encoder blocks, audio encoder layers+ """+ block_names = []++ # Quantize visual encoder blocks if quant_vision is enabled+ if quant_vision:+ if hasattr(model, "thinker") and hasattr(model.thinker, "visual") and hasattr(model.thinker.visual, "blocks"):+ block_names.append([f"thinker.visual.blocks.{i}" for i in range(len(model.thinker.visual.blocks))])+ if hasattr(model, "thinker") and hasattr(model.thinker, "audio_tower"):+ if hasattr(model.thinker.audio_tower, "layers"):+ block_names.append(+ [f"thinker.audio_tower.layers.{i}" for i in range(len(model.thinker.audio_tower.layers))]+ )++ # Thinker text model layers (main LLM decoder)+ if hasattr(model, "thinker") and hasattr(model.thinker, "model") and hasattr(model.thinker.model, "layers"):+ block_names.append([f"thinker.model.layers.{i}" for i in range(len(model.thinker.model.layers))])++ # Talker model layers (if available)+ if hasattr(model, "talker") and hasattr(model.talker, "model") and hasattr(model.talker.model, "layers"):+ block_names.append([f"talker.model.layers.{i}" for i in range(len(model.talker.model.layers))])++ return block_names+++def _get_qwen3_omni_moe_multimodal_block(model, quant_vision=False):+ """Get block names for Qwen3-Omni MoE model.++ Qwen3-Omni has the following structure:+ - thinker: Contains audio_tower, visual, model (text decoder)+ - talker: Contains model (talker decoder), code_predictor+ - code2wav: Audio decoder++ For quantization, we focus on:+ - thinker.model.layers (text decoder layers) - main LLM layers+ - talker.model.layers (talker decoder layers)+ - Optionally: visual encoder blocks, audio encoder layers+ """+ block_names = []++ # Quantize visual encoder blocks if quant_vision is enabled+ if quant_vision:+ # Vision encoder blocks+ if hasattr(model, "thinker") and hasattr(model.thinker, "visual") and hasattr(model.thinker.visual, "blocks"):+ block_names.append([f"thinker.visual.blocks.{i}" for i in range(len(model.thinker.visual.blocks))])+ # Audio encoder layers+ if hasattr(model, "thinker") and hasattr(model.thinker, "audio_tower"):+ if hasattr(model.thinker.audio_tower, "layers"):+ block_names.append(+ [f"thinker.audio_tower.layers.{i}" for i in range(len(model.thinker.audio_tower.layers))]+ )++ # Thinker text model layers (main LLM decoder)+ if hasattr(model, "thinker") and hasattr(model.thinker, "model") and hasattr(model.thinker.model, "layers"):+ block_names.append([f"thinker.model.layers.{i}" for i in range(len(model.thinker.model.layers))])++ # Talker model layers (if available)+ if hasattr(model, "talker") and hasattr(model.talker, "model") and hasattr(model.talker.model, "layers"):+ block_names.append([f"talker.model.layers.{i}" for i in range(len(model.talker.model.layers))])++ return block_names+++SPECIAL_MULTIMODAL_BLOCK = {+ "deepseek_vl_v2": _get_deepseek_vl2_multimodal_block,+ "qwen2_5_omni": _get_qwen2_5_omni_multimodal_block,+ "qwen3_omni_moe": _get_qwen3_omni_moe_multimodal_block,+}def _deepseek_vl2_forward(@@ -122,6 +217,200 @@ def _deepseek_vl2_forward()+def _qwen2_5_omni_forward(+ model,+ input_ids=None,+ input_features=None,+ pixel_values=None,+ pixel_values_videos=None,+ image_grid_thw=None,+ video_grid_thw=None,+ attention_mask=None,+ feature_attention_mask=None,+ audio_feature_lengths=None,+ position_ids=None,+ past_key_values=None,+ inputs_embeds=None,+ rope_deltas=None,+ labels=None,+ use_cache=None,+ use_audio_in_video=None,+ cache_position=None,+ video_second_per_grid=None,+ **kwargs,+):+ """Forward function for Qwen2.5-Omni model.++ This delegates to the thinker module for calibration, then optionally+ runs a forward through the talker to ensure its layers are also calibrated.+ """+ thinker_output = model.thinker(+ input_ids=input_ids,+ input_features=input_features,+ pixel_values=pixel_values,+ pixel_values_videos=pixel_values_videos,+ image_grid_thw=image_grid_thw,+ video_grid_thw=video_grid_thw,+ attention_mask=attention_mask,+ feature_attention_mask=feature_attention_mask,+ audio_feature_lengths=audio_feature_lengths,+ position_ids=position_ids,+ past_key_values=past_key_values,+ inputs_embeds=inputs_embeds,+ rope_deltas=rope_deltas,+ labels=labels,+ use_cache=use_cache,+ use_audio_in_video=use_audio_in_video,+ cache_position=cache_position,+ video_second_per_grid=video_second_per_grid,+ output_hidden_states=True,+ return_dict=True,+ **kwargs,+ )++ # Run talker forward if available (for calibration purposes)+ if hasattr(model, "talker") and model.has_talker:+ try:+ thinker_hidden = thinker_output.hidden_states[-1] if thinker_output.hidden_states else None++ if thinker_hidden is not None:+ batch_size, seq_len, _ = thinker_hidden.shape++ if hasattr(model.talker, "thinker_to_talker_proj"):+ thinker_embeds = model.thinker.get_input_embeddings()(input_ids)+ proj_dtype = next(model.talker.thinker_to_talker_proj.parameters()).dtype+ talker_inputs_embeds = model.talker.thinker_to_talker_proj(thinker_embeds.to(proj_dtype))+ else:+ talker_hidden_size = model.talker.model.config.hidden_size+ talker_inputs_embeds = torch.zeros(+ batch_size,+ seq_len,+ talker_hidden_size,+ device=thinker_hidden.device,+ dtype=thinker_hidden.dtype,+ )++ # Align dtype with talker model weights+ talker_dtype = next(model.talker.model.parameters()).dtype+ _ = model.talker.model(+ inputs_embeds=talker_inputs_embeds.to(talker_dtype),+ attention_mask=attention_mask,+ use_cache=False,+ )+ except Exception as exc:+ logger.warning(+ "Qwen2.5-Omni talker forward failed during calibration; "+ "continuing with thinker-only quantization. Error: %s",+ exc,+ exc_info=True,+ )++ return thinker_output+++def _qwen3_omni_moe_forward(+ model,+ input_ids=None,+ input_features=None,+ pixel_values=None,+ pixel_values_videos=None,+ image_grid_thw=None,+ video_grid_thw=None,+ attention_mask=None,+ feature_attention_mask=None,+ audio_feature_lengths=None,+ position_ids=None,+ past_key_values=None,+ inputs_embeds=None,+ rope_deltas=None,+ labels=None,+ use_cache=None,+ output_router_logits=None,+ use_audio_in_video=None,+ cache_position=None,+ video_second_per_grid=None,+ **kwargs,+):+ """Forward function for Qwen3-Omni-MoE model.++ This runs forward through both thinker and talker modules for calibration.+ The thinker processes text/vision/audio input, and talker uses thinker's+ hidden states to prepare for speech synthesis.+ """+ # Run thinker forward with output_hidden_states to get hidden states for talker+ thinker_output = model.thinker(+ input_ids=input_ids,+ input_features=input_features,+ pixel_values=pixel_values,+ pixel_values_videos=pixel_values_videos,+ image_grid_thw=image_grid_thw,+ video_grid_thw=video_grid_thw,+ attention_mask=attention_mask,+ feature_attention_mask=feature_attention_mask,+ audio_feature_lengths=audio_feature_lengths,+ position_ids=position_ids,+ past_key_values=past_key_values,+ inputs_embeds=inputs_embeds,+ rope_deltas=rope_deltas,+ labels=labels,+ use_cache=use_cache,+ output_router_logits=output_router_logits,+ use_audio_in_video=use_audio_in_video,+ cache_position=cache_position,+ video_second_per_grid=video_second_per_grid,+ output_hidden_states=True,+ return_dict=True,+ **kwargs,+ )++ # Run talker forward if available (for calibration purposes)+ if getattr(model, "has_talker", False) and getattr(model, "talker", None) is not None:+ try:+ # Get thinker hidden states for talker input+ # Use the last hidden state from thinker as input to talker+ thinker_hidden = thinker_output.hidden_states[-1] if thinker_output.hidden_states else None++ if thinker_hidden is not None:+ # Create simple talker input from thinker hidden states+ # Project thinker hidden states to talker dimension+ batch_size, seq_len, _ = thinker_hidden.shape+ talker_hidden_size = model.config.talker_config.text_config.hidden_size++ # Use text projection to convert thinker embeddings to talker space+ if hasattr(model.talker, "text_projection"):+ # Get thinker embeddings and align dtype with text_projection weights+ thinker_embeds = model.thinker.get_input_embeddings()(input_ids)+ proj_dtype = next(model.talker.text_projection.parameters()).dtype+ talker_inputs_embeds = model.talker.text_projection(thinker_embeds.to(proj_dtype))+ else:+ # Fallback: create zero embeddings of correct size+ talker_inputs_embeds = torch.zeros(+ batch_size,+ seq_len,+ talker_hidden_size,+ device=thinker_hidden.device,+ dtype=thinker_hidden.dtype,+ )++ # Run talker model forward — align dtype with talker model weights+ talker_dtype = next(model.talker.model.parameters()).dtype+ _ = model.talker.model(+ inputs_embeds=talker_inputs_embeds.to(talker_dtype),+ attention_mask=attention_mask,+ use_cache=False,+ )+ except Exception as exc:+ # Log talker forward errors during calibration without interrupting thinker quantization+ logger.warning(+ "Qwen3-Omni-MoE talker forward failed during calibration; "+ "continuing with thinker-only quantization. Error: %s",+ exc,+ exc_info=True,+ )++ return thinker_output++def check_mllm_model_batch(model, batch_size, gradient_accumulate_steps=1):"""Checks model configuration to determine if it's necessary to limit bs to avoid potential input shape mismatches.diff --git a/auto_round/utils/common.py b/auto_round/utils/common.pyindex 457a0e6a4..23685d2f2 100644--- a/auto_round/utils/common.py+++ b/auto_round/utils/common.py@@ -104,6 +104,25 @@ def _patch_classmethod_kwargs(cls, method_name, **name_map):setattr(cls, method_name, classmethod(patched))+def normalize_no_split_modules(no_split_modules):+ if not no_split_modules:+ return []++ def flatten_items(value):+ if isinstance(value, (list, tuple, set)):+ for item in value:+ yield from flatten_items(item)+ else:+ yield value++ flattened = []+ for item in flatten_items(no_split_modules):+ if item is None:+ continue+ flattened.append(item)+ return flattened++def _patch_transpose_for_buffers():"""Patch Transpose.convert() to skip transposition for buffer tensors.@@ -220,6 +239,191 @@ def monkey_patch():monkey_patch_transformers()+def monkey_patch_model(model) -> None:+ """Apply model-instance-level monkey patches after a model is loaded.++ This is the central place for all instance-level patches (as opposed to the+ class-level patches in ``monkey_patch_transformers``).+ """+ _patch_prepare_inputs_for_generation(model)+++def _patch_prepare_inputs_for_generation(model) -> None:+ """Fix positional-arg mismatch in models whose prepare_inputs_for_generation+ passes arguments positionally to GenerationMixin.prepare_inputs_for_generation.++ transformers >= 5.1 inserted ``next_sequence_length`` as the 2nd positional+ parameter in the base ``GenerationMixin.prepare_inputs_for_generation``.+ Some model implementations (e.g. Qwen2.5-Omni / Qwen3-Omni MoE talker)+ still pass ``past_key_values`` etc. positionally, causing a+ "got multiple values for argument 'next_sequence_length'" TypeError.++ This function monkey-patches the affected sub-model to use keyword arguments+ instead, which works with both old and new transformers.+ """+ model_type = getattr(getattr(model, "config", None), "model_type", None)++ if model_type == "qwen2_5_omni":+ _patch_qwen25_omni_talker(model)+ elif model_type == "qwen3_omni_moe":+ _patch_qwen3_omni_moe_talker(model)+++def _patch_qwen25_omni_talker(model) -> None:+ """Patch Qwen2.5-Omni talker prepare_inputs_for_generation."""+ talker = getattr(model, "talker", None)+ if talker is None:+ return++ import types++ def _fixed_prepare_inputs_for_generation(+ self,+ input_ids,+ input_text_ids=None,+ past_key_values=None,+ attention_mask=None,+ inputs_embeds=None,+ thinker_reply_part=None,+ cache_position=None,+ position_ids=None,+ use_cache=True,+ pixel_values=None,+ pixel_values_videos=None,+ image_grid_thw=None,+ video_grid_thw=None,+ input_audio_features=None,+ audio_feature_attention_mask=None,+ audio_feature_lengths=None,+ use_audio_in_video=None,+ video_second_per_grid=None,+ **kwargs,+ ):+ from transformers.generation.utils import GenerationMixin++ model_inputs = GenerationMixin.prepare_inputs_for_generation(+ self,+ input_ids,+ past_key_values=past_key_values,+ attention_mask=attention_mask,+ inputs_embeds=inputs_embeds,+ cache_position=cache_position,+ use_cache=use_cache,+ thinker_reply_part=thinker_reply_part,+ input_text_ids=input_text_ids,+ pixel_values=pixel_values,+ pixel_values_videos=pixel_values_videos,+ image_grid_thw=image_grid_thw,+ video_grid_thw=video_grid_thw,+ input_audio_features=input_audio_features,+ audio_feature_attention_mask=audio_feature_attention_mask,+ audio_feature_lengths=audio_feature_lengths,+ use_audio_in_video=use_audio_in_video,+ video_second_per_grid=video_second_per_grid,+ **kwargs,+ )+ model_inputs["position_ids"] = None+ return model_inputs++ talker.prepare_inputs_for_generation = types.MethodType(_fixed_prepare_inputs_for_generation, talker)+ from auto_round.logger import logger++ logger.info("Patched Qwen2.5-Omni talker prepare_inputs_for_generation for transformers compat.")+++def _patch_qwen3_omni_moe_talker(model) -> None:+ """Patch Qwen3-Omni MoE talker prepare_inputs_for_generation.++ The talker passes past_key_values, attention_mask, inputs_embeds positionally+ to super().prepare_inputs_for_generation(), colliding with the new+ next_sequence_length parameter in transformers >= 5.1.+ """+ talker = getattr(model, "talker", None)+ if talker is None:+ return++ import types++ _orig_prepare = (+ talker.prepare_inputs_for_generation.__func__+ if hasattr(talker.prepare_inputs_for_generation, "__func__")+ else talker.prepare_inputs_for_generation+ )++ def _fixed_prepare_inputs_for_generation(+ self,+ input_ids,+ past_key_values=None,+ attention_mask=None,+ inputs_embeds=None,+ is_first_iteration=False,+ **kwargs,+ ):+ from transformers.generation.utils import GenerationMixin++ hidden_states = kwargs.pop("hidden_states", None)+ inputs = GenerationMixin.prepare_inputs_for_generation(+ self,+ input_ids,+ past_key_values=past_key_values,+ attention_mask=attention_mask,+ inputs_embeds=inputs_embeds,+ is_first_iteration=is_first_iteration,+ **kwargs,+ )++ # Qwen3-Omni will prepare position ids in forward with deltas+ inputs["position_ids"] = None++ # Reproduce the talker's codec logic for non-first iterations+ if not is_first_iteration and kwargs.get("use_cache", True):+ import torch++ input_ids_last = input_ids[:, -1:]+ generation_step = kwargs.get("generation_step")+ trailing_text_hidden = kwargs.get("trailing_text_hidden")+ tts_pad_embed = kwargs.get("tts_pad_embed")+ last_id_hidden = self.get_input_embeddings()(input_ids_last)++ past_hidden = hidden_states[0][-1][:, -1:].to(last_id_hidden.device)+ predictor_result = self.code_predictor.generate(+ inputs_embeds=torch.cat((past_hidden, last_id_hidden), dim=1),+ max_new_tokens=self.config.num_code_groups - 1,+ do_sample=True,+ top_k=50,+ top_p=0.8,+ output_hidden_states=True,+ return_dict_in_generate=True,+ )+ residual_codes = torch.cat((input_ids_last, predictor_result.sequences.to(input_ids_last.device)), dim=-1)++ mid_residual_hiddens = [hid[0].to(last_id_hidden.device) for hid in predictor_result.hidden_states[1:]]+ last_residual_hidden = self.code_predictor.get_input_embeddings()[-1](+ predictor_result.sequences[..., -1:]+ ).to(last_id_hidden.device)+ codec_hiddens = torch.cat(+ [last_id_hidden] + mid_residual_hiddens + [last_residual_hidden],+ dim=1,+ )+ inputs_embeds_new = codec_hiddens.sum(1, keepdim=True)++ if generation_step < trailing_text_hidden.shape[1]:+ inputs_embeds_new = inputs_embeds_new + trailing_text_hidden[:, generation_step].unsqueeze(1).to(+ inputs_embeds_new.device+ )+ else:+ inputs_embeds_new = inputs_embeds_new + tts_pad_embed.to(inputs_embeds_new.device)+ inputs["inputs_embeds"] = inputs_embeds_new+ inputs["residual_codes"] = residual_codes++ return inputs++ talker.prepare_inputs_for_generation = types.MethodType(_fixed_prepare_inputs_for_generation, talker)+ from auto_round.logger import logger++ logger.info("Patched Qwen3-Omni MoE talker prepare_inputs_for_generation for transformers compat.")++auto_gptq = LazyImport("auto_gptq")htcore = LazyImport("habana_frameworks.torch.core")@@ -280,6 +484,8 @@ MM_KEYS = ["audio","talker","token2wav",+ "code2wav",+ "code_predictor","vision_model","audio_tower","vision_encoder",diff --git a/auto_round/utils/model.py b/auto_round/utils/model.pyindex 0e4a406fb..357e7c9df 100644--- a/auto_round/utils/model.py+++ b/auto_round/utils/model.py@@ -431,6 +431,20 @@ def mllm_load_model(else:model_type = None+ if model_type == "qwen2_5_omni":+ if version.parse(transformers.__version__) < version.parse("4.52.0"):+ raise RuntimeError(+ f"Qwen2.5-Omni requires transformers >= 4.52.0, but found {transformers.__version__}. "+ "Please upgrade: pip install transformers>=4.52.0"+ )++ if model_type == "qwen3_omni_moe":+ if version.parse(transformers.__version__) < version.parse("5.1.0"):+ raise RuntimeError(+ f"Qwen3-Omni requires transformers >= 5.1.0, but found {transformers.__version__}. "+ "Please upgrade: pip install transformers>=5.1.0"+ )+processor, image_processor = None, Noneif "deepseek_vl_v2" == model_type:from deepseek_vl2.models import DeepseekVLV2ForCausalLM, DeepseekVLV2Processor # pylint: disable=E0401@@ -712,6 +726,8 @@ def is_moe_layer(module: torch.nn.Module) -> bool:"Qwen2MoeSparseMoeBlock".lower(),"Qwen3MoeSparseMoeBlock".lower(),"Qwen3VLMoeTextSparseMoeBlock".lower(),+ "Qwen3OmniMoeThinkerTextSparseMoeBlock".lower(),+ "Qwen3OmniMoeTalkerTextSparseMoeBlock".lower(),])@@ -811,6 +827,8 @@ def get_expert_linear_names(module: torch.nn.Module) -> list[str]:"DeepseekV2MoE","DeepseekV3MoE","Qwen3VLMoeTextSparseMoeBlock",+ "Qwen3OmniMoeThinkerTextSparseMoeBlock",+ "Qwen3OmniMoeTalkerTextSparseMoeBlock",],):return ["gate_proj", "down_proj", "up_proj"]@@ -846,6 +864,8 @@ def get_expert_input_proj_names(module: torch.nn.Module) -> list[str]:"Qwen2MoeSparseMoeBlock","Qwen3MoeSparseMoeBlock","Qwen3VLMoeTextSparseMoeBlock",+ "Qwen3OmniMoeThinkerTextSparseMoeBlock",+ "Qwen3OmniMoeTalkerTextSparseMoeBlock","DeepseekMoE","DeepseekV2MoE","DeepseekV3MoE",@@ -1525,6 +1545,28 @@ def _get_reference_amax_from_experts(moe_module: torch.nn.Module, attr_name: strreturn torch.max(all_values)+# Extra non-weight files that some models require at load time but are not saved+# by model.save_pretrained(). These are copied from the source model cache to+# the quantized output directory so that from_pretrained() works out of the box.+_EXTRA_MODEL_FILES = {+ "spk_dict.pt", # Qwen2.5-Omni speaker dictionary for audio output+}+++def _copy_extra_model_files(src_dir: str, dst_dir: str):+ """Copy known extra model files from *src_dir* to *dst_dir* if they exist."""+ import os+ import shutil++ for file in os.listdir(src_dir):+ if file in _EXTRA_MODEL_FILES:+ src_file = os.path.join(src_dir, file)+ dst_file = os.path.join(dst_dir, file)+ if os.path.isfile(src_file) and not os.path.exists(dst_file):+ logger.debug(f"Transferring extra model file {src_file} to {dst_dir}")+ shutil.copy(src_file, dst_dir)++# Adapted from https://github.com/vllm-project/llm-compressor/blob/# 5b3ddff74cae9651f24bef15d3255c4ee053fc60/src/llmcompressor/pytorch/model_load/helpers.py#L144def copy_python_files_from_model_cache(model, save_path: str):@@ -1563,6 +1605,8 @@ def copy_python_files_from_model_cache(model, save_path: str):logger.debug(f"Transferring {full_file_name} to {save_path}")shutil.copy(full_file_name, save_path)+ _copy_extra_model_files(cache_path, save_path)+def extract_block_names_to_str(quant_block_list):if not isinstance(quant_block_list, (list, tuple)):diff --git a/pyproject.toml b/pyproject.tomlindex 014a453f6..9249c6ad4 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -23,6 +23,7 @@ extend-exclude = [[tool.typos.default.extend-words]ue = "ue"endianess = "endianess"+thw = "thw"[tool.ruff]# Exclude a variety of commonly ignored directories.diff --git a/test/fixtures.py b/test/fixtures.pyindex 4ec157b1f..624ee1a19 100644--- a/test/fixtures.py+++ b/test/fixtures.py@@ -15,6 +15,8 @@ from .helpers import (lamini_name_or_path,opt_name_or_path,phi2_name_or_path,+ qwen2_5_omni_name_or_path,+ qwen3_omni_name_or_path,qwen_2_5_vl_name_or_path,qwen_moe_name_or_path,qwen_name_or_path,@@ -123,6 +125,75 @@ def tiny_qwen_2_5_vl_model_path():shutil.rmtree(tiny_model_path, ignore_errors=True)[email protected](scope="session")+def tiny_qwen2_5_omni():+ """Tiny Qwen2.5-Omni-3B model built from real config with reduced layers.++ Uses random weights (no checkpoint loading) so it is fast for CPU unit+ tests while still exercising the real config structure.+ Skipped automatically when the model path does not exist locally.+ """++ from transformers import AutoConfig, AutoProcessor, AutoTokenizer, Qwen2_5OmniForConditionalGeneration++ model_name = qwen2_5_omni_name_or_path+ if not os.path.isdir(model_name):+ pytest.skip(f"Qwen2.5-Omni-3B not found at {model_name}")++ tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)+ processor = AutoProcessor.from_pretrained(model_name, trust_remote_code=True)+ config = AutoConfig.from_pretrained(model_name, trust_remote_code=True)++ # Reduce layers — keeps real config structure but uses random weights+ config.thinker_config.text_config.num_hidden_layers = 1+ config.thinker_config.vision_config.depth = 1+ config.thinker_config.audio_config.num_hidden_layers = 1+ config.talker_config.num_hidden_layers = 1+ if hasattr(config.thinker_config.text_config, "layer_types"):+ config.thinker_config.text_config.layer_types = config.thinker_config.text_config.layer_types[:1]+ if hasattr(config.talker_config, "layer_types"):+ config.talker_config.layer_types = config.talker_config.layer_types[:1]++ model = Qwen2_5OmniForConditionalGeneration(config)+ yield model, tokenizer, processor++[email protected](scope="session")+def tiny_qwen3_omni_moe():+ """Tiny Qwen3-Omni-MoE model built from real config with reduced layers.++ Uses random weights (no checkpoint loading) so it is fast for CI while+ still exercising the real config structure.+ Skipped automatically when the model path does not exist locally.+ """++ from transformers import AutoConfig, AutoProcessor, AutoTokenizer, Qwen3OmniMoeForConditionalGeneration++ model_name = qwen3_omni_name_or_path+ if not os.path.isdir(model_name):+ pytest.skip(f"Qwen3-Omni-30B-A3B-Instruct not found at {model_name}")++ tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)+ processor = AutoProcessor.from_pretrained(model_name, trust_remote_code=True)+ config = AutoConfig.from_pretrained(model_name, trust_remote_code=True)++ # Reduce layers — keeps real config structure but uses random weights+ config.thinker_config.text_config.num_hidden_layers = 1+ config.thinker_config.vision_config.depth = 1+ config.thinker_config.audio_config.num_hidden_layers = 1+ if hasattr(config.thinker_config.text_config, "layer_types"):+ config.thinker_config.text_config.layer_types = config.thinker_config.text_config.layer_types[:1]+ # Talker+ if hasattr(config, "talker_config"):+ if hasattr(config.talker_config, "text_config"):+ config.talker_config.text_config.num_hidden_layers = 1+ elif hasattr(config.talker_config, "num_hidden_layers"):+ config.talker_config.num_hidden_layers = 1++ model = Qwen3OmniMoeForConditionalGeneration(config)+ yield model, tokenizer, processor++# Mock torch.cuda.get_device_capability to always return (9, 0) like H100@pytest.fixture()def mock_fp8_capable_device():diff --git a/test/helpers.py b/test/helpers.pyindex 240791a6a..8a5d24649 100644--- a/test/helpers.py+++ b/test/helpers.py@@ -98,6 +98,8 @@ qwen_moe_name_or_path = get_model_path("Qwen/Qwen1.5-MoE-A2.7B")qwen_vl_name_or_path = get_model_path("Qwen/Qwen2-VL-2B-Instruct")qwen_2_5_vl_name_or_path = get_model_path("Qwen/Qwen2.5-VL-3B-Instruct")gemma_name_or_path = get_model_path("benzart/gemma-2b-it-fine-tuning-for-code-test")+qwen2_5_omni_name_or_path = get_model_path("Qwen/Qwen2.5-Omni-3B")+qwen3_omni_name_or_path = get_model_path("Qwen/Qwen3-Omni-30B-A3B-Instruct")# Slice model into tiny model for speedupdiff --git a/test/test_cpu/models/test_omni_model.py b/test/test_cpu/models/test_omni_model.pynew file mode 100644index 000000000..8cd83b8bf--- /dev/null+++ b/test/test_cpu/models/test_omni_model.py@@ -0,0 +1,399 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++"""Unit tests for Qwen2.5-Omni and Qwen3-Omni-MoE model support.++Tests cover:+- Block name discovery (thinker + talker layers)+- MoE module replacement and weight fidelity (Qwen3-Omni)+- Forward function patching (_handle_special_model)+- Processor and template registration+- MoE utility functions (is_moe_layer, get_expert_linear_names, etc.)+- Ignore layer registration (mlp.gate for MoE)+"""++import copy+import shutil++import pytest+import torch+from transformers import (+ Qwen3OmniMoeConfig,+ Qwen3OmniMoeForConditionalGeneration,+)++from ...helpers import check_version, transformers_version++pytestmark = pytest.mark.skipif(+ not check_version("transformers>=5.1.0"),+ reason="Qwen-Omni models require transformers >= 5.1.0",+)+++# ---------------------------------------------------------------------------+# Helper: create tiny Qwen3-Omni-MoE config (no real checkpoint needed for MoE)+# ---------------------------------------------------------------------------+def _make_tiny_qwen3_omni_moe_config():+ config = Qwen3OmniMoeConfig()+ # Thinker+ config.thinker_config.text_config.num_hidden_layers = 1+ config.thinker_config.text_config.hidden_size = 64+ config.thinker_config.text_config.intermediate_size = 128+ config.thinker_config.text_config.moe_intermediate_size = 32+ config.thinker_config.text_config.num_attention_heads = 4+ config.thinker_config.text_config.num_key_value_heads = 2+ config.thinker_config.text_config.num_experts = 4+ config.thinker_config.text_config.num_experts_per_tok = 2+ config.thinker_config.vision_config.depth = 1+ config.thinker_config.vision_config.embed_dim = 64+ config.thinker_config.vision_config.hidden_size = 64+ config.thinker_config.vision_config.num_heads = 4+ config.thinker_config.audio_config.num_hidden_layers = 1+ # Talker+ config.talker_config.text_config.num_hidden_layers = 1+ config.talker_config.text_config.hidden_size = 64+ config.talker_config.text_config.intermediate_size = 128+ config.talker_config.text_config.moe_intermediate_size = 32+ config.talker_config.text_config.num_attention_heads = 4+ config.talker_config.text_config.num_key_value_heads = 2+ config.talker_config.text_config.num_experts = 4+ config.talker_config.text_config.num_local_experts = 4+ config.talker_config.text_config.num_experts_per_tok = 2+ config.talker_config.text_config.shared_expert_intermediate_size = 64+ config.talker_config.thinker_hidden_size = 64+ config.talker_config.spatial_merge_size = 2+ # Code2wav (minimal)+ config.code2wav_config.hidden_size = 64+ config.code2wav_config.num_hidden_layers = 1+ config.code2wav_config.num_attention_heads = 4+ config.code2wav_config.num_key_value_heads = 4+ config.code2wav_config.intermediate_size = 128+ return config+++# ========================= Qwen2.5-Omni Tests =============================+# NOTE: Tests use the `tiny_qwen2_5_omni` session-scoped fixture from fixtures.py+# (real config, reduced layers, random weights). Skipped if model not available.+++class TestQwen2_5OmniBlockNames:+ """Test block name discovery for Qwen2.5-Omni (dense, not MoE)."""++ def test_block_names_default(self, tiny_qwen2_5_omni):+ """Test that get_block_names returns thinker + talker layers."""+ from auto_round.utils import get_block_names++ model, _, _ = tiny_qwen2_5_omni+ block_names = get_block_names(model, quant_vision=False)+ # Should have thinker.model.layers and talker.model.layers+ assert any(+ "thinker.model.layers" in str(b) for b in block_names+ ), f"Expected thinker.model.layers in block_names, got: {block_names}"+ assert any(+ "talker.model.layers" in str(b) for b in block_names+ ), f"Expected talker.model.layers in block_names, got: {block_names}"++ def test_block_names_quant_vision(self, tiny_qwen2_5_omni):+ """Test that quant_vision adds visual and audio blocks."""+ from auto_round.utils import get_block_names++ model, _, _ = tiny_qwen2_5_omni+ blocks_no_vision = get_block_names(model, quant_vision=False)+ blocks_with_vision = get_block_names(model, quant_vision=True)++ assert len(blocks_with_vision) > len(blocks_no_vision), "quant_vision=True should add visual/audio blocks"+++class TestQwen2_5OmniForward:+ """Test forward function patching for Qwen2.5-Omni."""++ def test_handle_special_model(self, tiny_qwen2_5_omni):+ """Test that _handle_special_model patches the forward function."""+ from auto_round.special_model_handler import _handle_special_model++ # Deepcopy to avoid mutating the shared session-scoped fixture+ model = copy.deepcopy(tiny_qwen2_5_omni[0])+ original_forward = model.forward+ model = _handle_special_model(model)+ assert model.forward != original_forward, "Forward should be patched for qwen2_5_omni"+++class TestQwen2_5OmniProcessor:+ """Test processor and template registration for Qwen2.5-Omni."""++ def test_processor_registered(self):+ from auto_round.compressors.mllm.processor import PROCESSORS++ assert "qwen2_5_omni" in PROCESSORS, "qwen2_5_omni processor not registered"++ def test_template_registered(self):+ from auto_round.compressors.mllm.template import TEMPLATES++ assert "qwen2_5_omni" in TEMPLATES, "qwen2_5_omni template not registered"++ def test_template_default_dataset(self):+ from auto_round.compressors.mllm.template import TEMPLATES++ template = TEMPLATES["qwen2_5_omni"]+ assert template.default_dataset is not None+++# ========================= Qwen3-Omni-MoE Tests ===========================+++class TestQwen3OmniMoeBlockNames:+ """Test block name discovery for Qwen3-Omni-MoE."""++ def test_block_names_default(self):+ """Test that get_block_names returns thinker + talker layers."""+ from auto_round.utils import get_block_names++ config = _make_tiny_qwen3_omni_moe_config()+ model = Qwen3OmniMoeForConditionalGeneration(config)++ block_names = get_block_names(model, quant_vision=False)+ assert any(+ "thinker.model.layers" in str(b) for b in block_names+ ), f"Expected thinker.model.layers, got: {block_names}"+ assert any(+ "talker.model.layers" in str(b) for b in block_names+ ), f"Expected talker.model.layers, got: {block_names}"++ def test_block_names_quant_vision(self):+ """Test that quant_vision adds visual and audio blocks."""+ from auto_round.utils import get_block_names++ config = _make_tiny_qwen3_omni_moe_config()+ model = Qwen3OmniMoeForConditionalGeneration(config)++ blocks_no_vision = get_block_names(model, quant_vision=False)+ blocks_with_vision = get_block_names(model, quant_vision=True)++ assert len(blocks_with_vision) > len(blocks_no_vision), "quant_vision=True should add visual/audio blocks"+++class TestQwen3OmniMoeForward:+ """Test forward function patching for Qwen3-Omni-MoE."""++ def test_handle_special_model(self):+ """Test that _handle_special_model patches the forward function."""+ from auto_round.special_model_handler import _handle_special_model++ config = _make_tiny_qwen3_omni_moe_config()+ model = Qwen3OmniMoeForConditionalGeneration(config)++ original_forward = model.forward+ model = _handle_special_model(model)+ assert model.forward != original_forward, "Forward should be patched for qwen3_omni_moe"+++class TestQwen3OmniMoeReplacement:+ """Test MoE module replacement for Qwen3-Omni-MoE."""++ def test_replacement_registered(self):+ """Test that both thinker and talker MoE blocks are registered."""+ from auto_round.modeling.fused_moe.qwen3_omni import (+ LinearQwen3OmniTalkerSparseMoeBlock,+ LinearQwen3OmniThinkerSparseMoeBlock,+ )+ from auto_round.modeling.fused_moe.replace_modules import ReplacementModuleBase++ assert ReplacementModuleBase.is_registered("Qwen3OmniMoeThinkerTextSparseMoeBlock")+ assert ReplacementModuleBase.is_registered("Qwen3OmniMoeTalkerTextSparseMoeBlock")++ def test_builtin_modules_entry(self):+ """Test that qwen3_omni_moe is in BUILTIN_MODULES."""+ from auto_round.modeling.fused_moe.replace_modules import BUILTIN_MODULES++ assert "qwen3_omni_moe" in BUILTIN_MODULES++ def test_is_custom_model(self):+ """Test that is_custom_model returns True for Qwen3-Omni-MoE."""+ from auto_round.modeling.fused_moe.replace_modules import is_custom_model++ config = _make_tiny_qwen3_omni_moe_config()+ model = Qwen3OmniMoeForConditionalGeneration(config)+ assert is_custom_model(model)++ def test_apply_replacements(self):+ """Test that MoE blocks are correctly replaced."""+ from auto_round.modeling.fused_moe.replace_modules import apply_replacements++ config = _make_tiny_qwen3_omni_moe_config()+ model = Qwen3OmniMoeForConditionalGeneration(config)++ model = apply_replacements(model)++ # Check that thinker MoE was replaced+ thinker_mlp = model.thinker.model.layers[0].mlp+ assert (+ "LinearQwen3OmniThinker" in thinker_mlp.__class__.__name__+ ), f"Expected LinearQwen3OmniThinker, got {thinker_mlp.__class__.__name__}"++ # Check that talker MoE was replaced+ talker_mlp = model.talker.model.layers[0].mlp+ assert (+ "LinearQwen3OmniTalker" in talker_mlp.__class__.__name__+ ), f"Expected LinearQwen3OmniTalker, got {talker_mlp.__class__.__name__}"++ def test_weight_fidelity(self):+ """Test that unfused weights match original fused weights."""+ from auto_round.modeling.fused_moe.replace_modules import apply_replacements, materialize_model_++ config = _make_tiny_qwen3_omni_moe_config()+ model = Qwen3OmniMoeForConditionalGeneration(config)++ # Save original fused weights+ thinker_gate_up = model.thinker.model.layers[0].mlp.experts.gate_up_proj.data.clone()+ thinker_down = model.thinker.model.layers[0].mlp.experts.down_proj.data.clone()+ talker_gate_up = model.talker.model.layers[0].mlp.experts.gate_up_proj.data.clone()+ talker_down = model.talker.model.layers[0].mlp.experts.down_proj.data.clone()++ model = apply_replacements(model)+ materialize_model_(model)++ intermediate = 32 # moe_intermediate_size+ # Verify thinker expert weights+ for i in range(4):+ expert = model.thinker.model.layers[0].mlp.experts[i]+ assert torch.allclose(expert.gate_proj.weight.data, thinker_gate_up[i, :intermediate, :])+ assert torch.allclose(expert.up_proj.weight.data, thinker_gate_up[i, intermediate:, :])+ assert torch.allclose(expert.down_proj.weight.data, thinker_down[i])++ # Verify talker expert weights+ for i in range(4):+ expert = model.talker.model.layers[0].mlp.experts[i]+ assert torch.allclose(expert.gate_proj.weight.data, talker_gate_up[i, :intermediate, :])+ assert torch.allclose(expert.up_proj.weight.data, talker_gate_up[i, intermediate:, :])+ assert torch.allclose(expert.down_proj.weight.data, talker_down[i])++ def test_forward_output_match(self):+ """Test that replaced MoE forward output matches original."""+ from auto_round.modeling.fused_moe.replace_modules import apply_replacements, materialize_model_++ config = _make_tiny_qwen3_omni_moe_config()+ model = Qwen3OmniMoeForConditionalGeneration(config)++ x = torch.randn(1, 4, 64)+ with torch.no_grad():+ orig_thinker_out = model.thinker.model.layers[0].mlp(x)+ orig_talker_out = model.talker.model.layers[0].mlp(x)++ model = apply_replacements(model)+ materialize_model_(model)++ with torch.no_grad():+ new_thinker_out = model.thinker.model.layers[0].mlp(x)+ new_talker_out = model.talker.model.layers[0].mlp(x)++ assert torch.allclose(orig_thinker_out, new_thinker_out, atol=1e-5), "Thinker MoE forward mismatch"+ assert torch.allclose(orig_talker_out, new_talker_out, atol=1e-5), "Talker MoE forward mismatch"+++class TestQwen3OmniMoeProcessor:+ """Test processor and template registration for Qwen3-Omni-MoE."""++ def test_processor_registered(self):+ from auto_round.compressors.mllm.processor import PROCESSORS++ assert "qwen3_omni" in PROCESSORS, "qwen3_omni processor not registered"++ def test_template_registered(self):+ from auto_round.compressors.mllm.template import TEMPLATES++ assert "qwen3_omni_moe" in TEMPLATES, "qwen3_omni_moe template not registered"+++class TestQwen3OmniMoeUtils:+ """Test MoE utility functions for Qwen3-Omni-MoE classes."""++ def test_is_moe_layer_thinker(self):+ from auto_round.utils.model import is_moe_layer++ config = _make_tiny_qwen3_omni_moe_config()+ model = Qwen3OmniMoeForConditionalGeneration(config)+ moe_block = model.thinker.model.layers[0].mlp+ assert is_moe_layer(moe_block), f"Thinker MoE block ({moe_block.__class__.__name__}) should be detected as MoE"++ def test_is_moe_layer_talker(self):+ from auto_round.utils.model import is_moe_layer++ config = _make_tiny_qwen3_omni_moe_config()+ model = Qwen3OmniMoeForConditionalGeneration(config)+ moe_block = model.talker.model.layers[0].mlp+ assert is_moe_layer(moe_block), f"Talker MoE block ({moe_block.__class__.__name__}) should be detected as MoE"++ def test_get_expert_linear_names(self):+ from auto_round.utils.model import get_expert_linear_names++ config = _make_tiny_qwen3_omni_moe_config()+ model = Qwen3OmniMoeForConditionalGeneration(config)++ thinker_mlp = model.thinker.model.layers[0].mlp+ names = get_expert_linear_names(thinker_mlp)+ assert set(names) == {"gate_proj", "down_proj", "up_proj"}++ talker_mlp = model.talker.model.layers[0].mlp+ names = get_expert_linear_names(talker_mlp)+ assert set(names) == {"gate_proj", "down_proj", "up_proj"}++ def test_get_expert_input_proj_names(self):+ from auto_round.utils.model import get_expert_input_proj_names++ config = _make_tiny_qwen3_omni_moe_config()+ model = Qwen3OmniMoeForConditionalGeneration(config)++ thinker_mlp = model.thinker.model.layers[0].mlp+ names = get_expert_input_proj_names(thinker_mlp)+ assert set(names) == {"gate_proj", "up_proj"}++ def test_ignore_layers_registered(self):+ from auto_round.special_model_handler import get_predefined_ignore_layers++ config = _make_tiny_qwen3_omni_moe_config()+ model = Qwen3OmniMoeForConditionalGeneration(config)++ ignore_layers = get_predefined_ignore_layers(model)+ assert (+ "mlp.gate" in ignore_layers+ ), f"Expected mlp.gate in ignore_layers for qwen3_omni_moe, got: {ignore_layers}"+++class TestQwen2_5OmniNotMoe:+ """Verify Qwen2.5-Omni is not treated as MoE."""++ def test_not_moe(self, tiny_qwen2_5_omni):+ from auto_round.utils.model import is_moe_layer++ model, _, _ = tiny_qwen2_5_omni+ # Thinker uses dense MLP, not MoE+ thinker_mlp = model.thinker.model.layers[0].mlp+ assert not is_moe_layer(thinker_mlp), "Qwen2.5-Omni should not be detected as MoE"++ def test_not_custom_model(self, tiny_qwen2_5_omni):+ from auto_round.modeling.fused_moe.replace_modules import is_custom_model++ model, _, _ = tiny_qwen2_5_omni+ assert not is_custom_model(model), "Qwen2.5-Omni should not be in BUILTIN_MODULES"+++class TestVisualKeysExclusion:+ """Test that omni sub-modules are properly excluded from quantization."""++ def test_visual_keys_contain_omni_keys(self):+ from auto_round.compressors.mllm.utils import VISUAL_KEYS++ expected_keys = ["thinker", "talker", "audio", "token2wav", "code2wav", "audio_tower", "code_predictor"]+ for key in expected_keys:+ assert key in VISUAL_KEYS, f"'{key}' should be in VISUAL_KEYS"diff --git a/test/test_cuda/models/test_omni_model.py b/test/test_cuda/models/test_omni_model.pynew file mode 100644index 000000000..3b31de75f--- /dev/null+++ b/test/test_cuda/models/test_omni_model.py@@ -0,0 +1,176 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++"""CUDA integration tests for Qwen2.5-Omni and Qwen3-Omni-MoE quantization.++Tests cover end-to-end quantization flow:+- Qwen2.5-Omni: loads real config from pretrained, reduces layers, random weights+ (uses the shared ``tiny_qwen2_5_omni`` session-scoped fixture)+- Qwen3-Omni-MoE: fully synthetic tiny config with random weights+ (no pretrained checkpoint needed — model is too large for CI)+- Quantize with AutoRound+- Save and reload+- Run inference on reloaded model+"""++import os+import shutil++import pytest+import torch+from transformers import (+ AutoTokenizer,+ Qwen2_5OmniForConditionalGeneration,+ Qwen3OmniMoeConfig,+ Qwen3OmniMoeForConditionalGeneration,+)++from auto_round import AutoRound++from ...helpers import check_version, qwen2_5_omni_name_or_path++pytestmark = [+ pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available"),+ pytest.mark.skipif(+ not check_version("transformers>=5.1.0"),+ reason="Qwen-Omni models require transformers >= 5.1.0",+ ),+]+++# ---------------------------------------------------------------------------+# Fixture: tiny Qwen3-Omni-MoE+# Priority: use real config from qwen3_omni_name_or_path (skipped if absent);+# fall back to fully synthetic config using qwen_name_or_path tokenizer.+# ---------------------------------------------------------------------------+def setup_qwen3_omni_moe(tiny_qwen3_omni_moe):+ """Create a tiny Qwen3-Omni-MoE model.++ Uses the session-scoped ``tiny_qwen3_omni_moe`` fixture which loads the+ real tokenizer/processor from ``qwen3_omni_name_or_path`` and builds a+ model with reduced layers and random weights.+ """+ model, tokenizer, processor = tiny_qwen3_omni_moe+ output_dir = "./tmp/test_quantized_qwen3_omni_moe"+ return model, tokenizer, processor, output_dir, model.config+++# ========================= Qwen2.5-Omni Integration Tests ==================+++class TestQwen2_5OmniQuantization:+ """End-to-end quantization test for Qwen2.5-Omni (dense model)."""++ def test_quantize_and_reload(self, tiny_qwen2_5_omni):+ """Quantize, save, reload, verify weights, and run inference."""+ model, tokenizer, processor = tiny_qwen2_5_omni+ output_dir = "./tmp/test_quantized_qwen2_5_omni"++ # Quantize+ autoround = AutoRound(+ model,+ tokenizer,+ processor=processor,+ nsamples=2,+ iters=1,+ seqlen=32,+ ignore_layers="self_attn,lm_head",+ )+ quantized_model, save_folder = autoround.quantize_and_save(format="auto_round", output_dir=output_dir)+ assert quantized_model is not None, "Quantized model should not be None"++ # Copy model-specific files required for from_pretrained (e.g. spk_dict.pt for token2wav)+ for extra_file in ["spk_dict.pt"]:+ src = os.path.join(qwen2_5_omni_name_or_path, extra_file)+ if os.path.exists(src):+ shutil.copy2(src, output_dir)++ # Reload+ loaded_model = Qwen2_5OmniForConditionalGeneration.from_pretrained(output_dir)+ loaded_model.to("cuda")++ # Run inference on thinker+ inp = torch.randint(0, 100, (1, 64)).to("cuda")+ with torch.inference_mode():+ output = loaded_model.thinker(input_ids=inp)+ assert output is not None, "Inference failed on reloaded model"++ # Cleanup+ shutil.rmtree(output_dir, ignore_errors=True)+++# ========================= Qwen3-Omni-MoE Integration Tests ================+++class TestQwen3OmniMoeQuantization:+ """End-to-end quantization test for Qwen3-Omni-MoE."""++ def test_quantize_and_reload(self, setup_qwen3_omni_moe):+ """Quantize, save, reload, verify weights, and run inference."""+ model, tokenizer, processor, output_dir, config = setup_qwen3_omni_moe++ # Quantize+ autoround = AutoRound(+ model,+ tokenizer,+ processor=processor,+ nsamples=2,+ iters=1,+ seqlen=32,+ ignore_layers="self_attn,lm_head,mlp.gate",+ )+ quantized_model, save_folder = autoround.quantize_and_save(format="auto_round", output_dir=output_dir)+ assert quantized_model is not None, "Quantized model should not be None"++ # Reload+ loaded_model = Qwen3OmniMoeForConditionalGeneration.from_pretrained(output_dir)+ loaded_model.to("cuda")++ # Run inference on thinker+ inp = torch.randint(0, 100, (1, 64)).to("cuda")+ with torch.inference_mode():+ output = loaded_model.thinker(input_ids=inp)+ assert output is not None, "Inference failed on reloaded model (thinker)"++ # Cleanup+ shutil.rmtree(output_dir, ignore_errors=True)++ def test_quantize_mxfp4(self, setup_qwen3_omni_moe):+ """Quantize with MXFP4 scheme and verify."""+ model, tokenizer, processor, output_dir, config = setup_qwen3_omni_moe++ autoround = AutoRound(+ model,+ tokenizer,+ processor=processor,+ scheme="MXFP4",+ nsamples=2,+ iters=1,+ seqlen=32,+ ignore_layers="self_attn,lm_head,mlp.gate",+ )+ quantized_model, save_folder = autoround.quantize_and_save(format="auto_round", output_dir=output_dir)+ assert quantized_model is not None, "MXFP4 quantized model should not be None"++ # Reload and inference+ loaded_model = Qwen3OmniMoeForConditionalGeneration.from_pretrained(output_dir)+ loaded_model.to("cuda")++ inp = torch.randint(0, 100, (1, 64)).to("cuda")+ with torch.inference_mode():+ output = loaded_model.thinker(input_ids=inp)+ assert output is not None++ shutil.rmtree(output_dir, ignore_errors=True)