PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
fix: refactor and fix Omni model tests to improve CI stability (#1598)
Xin He8f10ac54285b
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.pyindex 3922789f3..eb7f6ffcc 100644--- a/auto_round/compressors/base.py+++ b/auto_round/compressors/base.py@@ -1890,7 +1890,7 @@ class BaseCompressor(object):)if len(unquantized_layers) > 0:compressed_unquantized_layers = compress_layer_names(unquantized_layers)- summary_info += f", {compressed_unquantized_layers} have not been quantized"+ summary_info += f", unquantized layers: {compressed_unquantized_layers}"logger.info(summary_info)self.quantized = True@@ -3166,7 +3166,7 @@ class BaseCompressor(object):if self.low_gpu_mem_usage:clear_memory(device_list=self.device_list) # clear cached memory during trainingif len(unquantized_layer_names) != 0:- logger.info(f"{unquantized_layer_names} have not been quantized")+ logger.info(f"Unquantized layers: {unquantized_layer_names}")with torch.no_grad():unwrapper_block(block, best_params)diff --git a/auto_round/special_model_handler.py b/auto_round/special_model_handler.pyindex 73dbf4896..b6343e048 100644--- a/auto_round/special_model_handler.py+++ b/auto_round/special_model_handler.py@@ -298,40 +298,30 @@ def _qwen2_5_omni_forward(# Run talker forward if available (for calibration purposes)if hasattr(model, "talker") and model.has_talker:- try:- thinker_hidden = thinker_output.hidden_states[-1] if thinker_output.hidden_states else None-- if thinker_hidden is not None:- batch_size, seq_len, _ = thinker_hidden.shape+ thinker_hidden = thinker_output.hidden_states[-1] if thinker_output.hidden_states else None++ if thinker_hidden is not None:+ # ---- calibrate thinker_to_talker_proj (nn.Linear) ----+ thinker_embeds = model.thinker.get_input_embeddings()(input_ids)+ proj_dtype = next(model.talker.thinker_to_talker_proj.parameters()).dtype+ talker_inputs_embeds = model.talker.thinker_to_talker_proj(thinker_embeds.to(proj_dtype))++ # ---- calibrate talker decoder layers ----+ talker_dtype = next(model.talker.model.parameters()).dtype+ talker_output = model.talker.model(+ inputs_embeds=talker_inputs_embeds.to(talker_dtype),+ attention_mask=attention_mask,+ use_cache=False,+ )- if hasattr(model.talker, "thinker_to_talker_proj"):- thinker_embeds = model.thinker.get_input_embeddings()(input_ids)- proj_dtype = next(model.talker.thinker_to_talker_proj.parameters()).dtype- talker_inputs_embeds = model.talker.thinker_to_talker_proj(thinker_embeds.to(proj_dtype))- else:- talker_hidden_size = model.talker.model.config.hidden_size- talker_inputs_embeds = torch.zeros(- batch_size,- seq_len,- talker_hidden_size,- device=thinker_hidden.device,- dtype=thinker_hidden.dtype,- )-- # Align dtype with talker model weights- talker_dtype = next(model.talker.model.parameters()).dtype- _ = model.talker.model(- inputs_embeds=talker_inputs_embeds.to(talker_dtype),- attention_mask=attention_mask,- use_cache=False,+ # ---- calibrate codec_head (nn.Linear) ----+ if hasattr(model.talker, "codec_head"):+ talker_last_hidden = (+ talker_output[0]+ if not hasattr(talker_output, "last_hidden_state")+ else talker_output.last_hidden_state)- except Exception as exc:- logger.warning(- "Qwen2.5-Omni talker forward failed during calibration; "- "continuing with thinker-only quantization. Error: %s",- exc,- exc_info=True,- )+ _ = model.talker.codec_head(talker_last_hidden.to(talker_dtype))return thinker_output@@ -364,6 +354,16 @@ def _qwen3_omni_moe_forward(This runs forward through both thinker and talker modules for calibration.The thinker processes text/vision/audio input, and talker uses thinker'shidden states to prepare for speech synthesis.++ In real inference the talker receives inputs built from two projections:+ - ``text_projection``: maps thinker word embeddings → talker hidden dim+ (used for pure-text tokens).+ - ``hidden_projection``: maps thinker intermediate hidden states → talker+ hidden dim (used for multimodal tokens).++ During calibration we exercise both projection paths and then run one+ forward pass through the talker decoder so that every linear layer+ (attention, MoE/MLP, codec_head) observes representative activations."""# Run thinker forward with output_hidden_states to get hidden states for talkerthinker_output = model.thinker(@@ -393,49 +393,36 @@ def _qwen3_omni_moe_forward(# Run talker forward if available (for calibration purposes)if getattr(model, "has_talker", False) and getattr(model, "talker", None) is not None:- try:- # Get thinker hidden states for talker input- # Use the last hidden state from thinker as input to talker- thinker_hidden = thinker_output.hidden_states[-1] if thinker_output.hidden_states else None-- if thinker_hidden is not None:- # Create simple talker input from thinker hidden states- # Project thinker hidden states to talker dimension- batch_size, seq_len, _ = thinker_hidden.shape- talker_hidden_size = model.config.talker_config.text_config.hidden_size-- # Use text projection to convert thinker embeddings to talker space- if hasattr(model.talker, "text_projection"):- # Get thinker embeddings and align dtype with text_projection weights- thinker_embeds = model.thinker.get_input_embeddings()(input_ids)- proj_dtype = next(model.talker.text_projection.parameters()).dtype- talker_inputs_embeds = model.talker.text_projection(thinker_embeds.to(proj_dtype))- else:- # Fallback: create zero embeddings of correct size- talker_inputs_embeds = torch.zeros(- batch_size,- seq_len,- talker_hidden_size,- device=thinker_hidden.device,- dtype=thinker_hidden.dtype,- )-- # Run talker model forward — align dtype with talker model weights- talker_dtype = next(model.talker.model.parameters()).dtype- _ = model.talker.model(- inputs_embeds=talker_inputs_embeds.to(talker_dtype),- attention_mask=attention_mask,- use_cache=False,- )- except Exception as exc:- # Log talker forward errors during calibration without interrupting thinker quantization- logger.warning(- "Qwen3-Omni-MoE talker forward failed during calibration; "- "continuing with thinker-only quantization. Error: %s",- exc,- exc_info=True,+ thinker_hidden = thinker_output.hidden_states[-1] if thinker_output.hidden_states else None++ if thinker_hidden is not None:+ # ---- calibrate text_projection (ResizeMLP) ----+ thinker_embeds = model.thinker.get_input_embeddings()(input_ids)+ proj_dtype = next(model.talker.text_projection.parameters()).dtype+ talker_inputs_embeds = model.talker.text_projection(thinker_embeds.to(proj_dtype))++ # ---- calibrate hidden_projection (ResizeMLP) ----+ if hasattr(model.talker, "hidden_projection"):+ hidden_proj_dtype = next(model.talker.hidden_projection.parameters()).dtype+ _ = model.talker.hidden_projection(thinker_hidden.to(hidden_proj_dtype))++ # ---- calibrate talker decoder layers ----+ talker_dtype = next(model.talker.model.parameters()).dtype+ talker_output = model.talker.model(+ inputs_embeds=talker_inputs_embeds.to(talker_dtype),+ attention_mask=attention_mask,+ use_cache=False,)+ # ---- calibrate codec_head ----+ if hasattr(model.talker, "codec_head"):+ talker_last_hidden = (+ talker_output[0]+ if not hasattr(talker_output, "last_hidden_state")+ else talker_output.last_hidden_state+ )+ _ = model.talker.codec_head(talker_last_hidden.to(talker_dtype))+return thinker_outputdiff --git a/test/fixtures.py b/test/fixtures.pyindex b2a4068c5..c570abf36 100644--- a/test/fixtures.py+++ b/test/fixtures.py@@ -282,74 +282,45 @@ def tiny_tiny_llama_model_path():@pytest.fixture(scope="session")-def tiny_qwen2_5_omni():+def tiny_qwen2_5_omni_model_path():"""Tiny Qwen2.5-Omni-3B model built from real config with reduced layers.Uses random weights (no checkpoint loading) so it is fast for CPU unittests while still exercising the real config structure.Skipped automatically when the model path does not exist locally."""-- from transformers import AutoConfig, AutoProcessor, AutoTokenizer, Qwen2_5OmniForConditionalGeneration+ from huggingface_hub import hf_hub_downloadmodel_name = qwen2_5_omni_name_or_path- if not os.path.isdir(model_name):- pytest.skip(f"Qwen2.5-Omni-3B not found at {model_name}")-- tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)- processor = AutoProcessor.from_pretrained(model_name, trust_remote_code=True)- config = AutoConfig.from_pretrained(model_name, trust_remote_code=True)-- # Reduce layers — keeps real config structure but uses random weights- config.thinker_config.text_config.num_hidden_layers = 1- config.thinker_config.vision_config.depth = 1- config.thinker_config.audio_config.num_hidden_layers = 1- config.talker_config.num_hidden_layers = 1- if hasattr(config.thinker_config.text_config, "layer_types"):- config.thinker_config.text_config.layer_types = config.thinker_config.text_config.layer_types[:1]- if hasattr(config.talker_config, "layer_types"):- config.talker_config.layer_types = config.talker_config.layer_types[:1]-- model = Qwen2_5OmniForConditionalGeneration(config)- model.config.name_or_path = None- yield model, tokenizer, processor+ tiny_model_path = "./tmp/tiny_qwen2_5_omni_model_path"+ tiny_model_path = save_tiny_model(model_name, tiny_model_path, num_layers=1, is_mllm=True, from_config=True)+ tokenizer = transformers.AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)+ processor = transformers.AutoProcessor.from_pretrained(model_name, trust_remote_code=True)+ tokenizer.save_pretrained(tiny_model_path)+ processor.save_pretrained(tiny_model_path)+ # Copy model-specific files required for from_pretrained (e.g. spk_dict.pt for token2wav)+ file_path = hf_hub_download(repo_id="Qwen/Qwen2.5-Omni-3B", filename="spk_dict.pt", local_dir=tiny_model_path)+ yield tiny_model_path+ shutil.rmtree(tiny_model_path, ignore_errors=True)@pytest.fixture(scope="session")-def tiny_qwen3_omni_moe():+def tiny_qwen3_omni_moe_model_path():"""Tiny Qwen3-Omni-MoE model built from real config with reduced layers.Uses random weights (no checkpoint loading) so it is fast for CI whilestill exercising the real config structure.Skipped automatically when the model path does not exist locally."""-- from transformers import AutoConfig, AutoProcessor, AutoTokenizer, Qwen3OmniMoeForConditionalGeneration-model_name = qwen3_omni_name_or_path- if not os.path.isdir(model_name):- pytest.skip(f"Qwen3-Omni-30B-A3B-Instruct not found at {model_name}")-- tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)- processor = AutoProcessor.from_pretrained(model_name, trust_remote_code=True)- config = AutoConfig.from_pretrained(model_name, trust_remote_code=True)-- # Reduce layers — keeps real config structure but uses random weights- config.thinker_config.text_config.num_hidden_layers = 1- config.thinker_config.vision_config.depth = 1- config.thinker_config.audio_config.num_hidden_layers = 1- if hasattr(config.thinker_config.text_config, "layer_types"):- config.thinker_config.text_config.layer_types = config.thinker_config.text_config.layer_types[:1]- # Talker- if hasattr(config, "talker_config"):- if hasattr(config.talker_config, "text_config"):- config.talker_config.text_config.num_hidden_layers = 1- elif hasattr(config.talker_config, "num_hidden_layers"):- config.talker_config.num_hidden_layers = 1-- model = Qwen3OmniMoeForConditionalGeneration(config)- model.config.name_or_path = None- yield model, tokenizer, processor+ tiny_model_path = "./tmp/tiny_qwen3_omni_moe_model_path"+ tiny_model_path = save_tiny_model(model_name, tiny_model_path, num_layers=1, is_mllm=True, from_config=True)+ tokenizer = transformers.AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)+ processor = transformers.AutoProcessor.from_pretrained(model_name, trust_remote_code=True)+ tokenizer.save_pretrained(tiny_model_path)+ processor.save_pretrained(tiny_model_path)+ yield tiny_model_path+ shutil.rmtree(tiny_model_path, ignore_errors=True)# Mock torch.cuda.get_device_capability to always return (9, 0) like H100diff --git a/test/helpers.py b/test/helpers.pyindex 040fdf82d..045a7eeb9 100644--- a/test/helpers.py+++ b/test/helpers.py@@ -160,10 +160,88 @@ qwen3_omni_name_or_path = get_model_path("Qwen/Qwen3-Omni-30B-A3B-Instruct")# Slice model into tiny model for speedup-def get_tiny_model(model_name_or_path, num_layers=2, is_mllm=False, **kwargs):- """Generate a tiny model by slicing layers from the original model."""+def _reduce_config_layers(config, num_layers, num_experts=None):+ """Reduce num_layers and num_experts in a config object (in-place).++ Handles nested sub-configs (text_config, vision_config, audio_config,+ thinker_config, talker_config, etc.) commonly found in multi-modal and+ MoE architectures.+ """+ n_block_keys = ["n_layers", "num_hidden_layers", "n_layer", "num_layers", "depth", "encoder_layers"]+ n_expert_keys = ["num_experts", "num_local_experts"]++ def _apply(cfg):+ for key in n_block_keys:+ if hasattr(cfg, key):+ setattr(cfg, key, num_layers)+ if hasattr(cfg, "layer_types"):+ cfg.layer_types = cfg.layer_types[:num_layers]+ if num_experts is not None:+ for key in n_expert_keys:+ if hasattr(cfg, key):+ setattr(cfg, key, num_experts)++ # Top-level config+ _apply(config)++ # Common sub-configs+ for sub_name in [+ "text_config",+ "vision_config",+ "audio_config",+ "thinker_config",+ "talker_config",+ ]:+ sub_cfg = getattr(config, sub_name, None)+ if sub_cfg is not None:+ _apply(sub_cfg)+ # Handle deeper nesting, e.g. thinker_config.text_config+ for inner_name in ["text_config", "vision_config", "audio_config"]:+ inner_cfg = getattr(sub_cfg, inner_name, None)+ if inner_cfg is not None:+ _apply(inner_cfg)+++def get_tiny_model(model_name_or_path, num_layers=2, num_experts=None, is_mllm=False, from_config=False, **kwargs):+ """Generate a tiny model by slicing layers from the original model.++ Args:+ model_name_or_path: HuggingFace model name or local path.+ num_layers: Number of layers to keep.+ num_experts: Number of experts to keep for MoE models (None = unchanged).+ is_mllm: Whether the model is a multi-modal model.+ from_config: If True, initialise the model from config with random+ weights instead of downloading the full checkpoint. This is much+ faster and avoids large downloads.+ **kwargs: Extra keyword arguments forwarded to the model loader or+ ``AutoConfig.from_pretrained``.+ """model_name_or_path = get_model_path(model_name_or_path)+ if from_config:+ # ---- lightweight path: config-only, random weights ----+ trust_remote_code = kwargs.pop("trust_remote_code", True)+ config = transformers.AutoConfig.from_pretrained(model_name_or_path, trust_remote_code=trust_remote_code)+ _reduce_config_layers(config, num_layers, num_experts)++ # Pick the right model class+ base_lib = transformers+ architectures = getattr(config, "architectures", [None])[0]+ if (+ is_mllm+ and architectures.endswith("Model")+ and hasattr(base_lib, n := architectures.replace("Model", "ForConditionalGeneration"))+ ):+ model_cls = getattr(base_lib, n)+ elif hasattr(base_lib, architectures):+ model_cls = getattr(base_lib, architectures)+ else:+ model_cls = transformers.AutoModelForCausalLM # default to causal LM if we can't find a better match+ model = model_cls._from_config(config)+ model = model.eval()+ return model++ # ---- original path: load pretrained weights then slice ----def slice_layers(module):"""slice layers in the model."""sliced = False@@ -190,30 +268,43 @@ def get_tiny_model(model_name_or_path, num_layers=2, is_mllm=False, **kwargs):slice_layers(model)- if hasattr(model.config, "num_hidden_layers"):- model.config.num_hidden_layers = num_layers- if hasattr(model.config, "text_config"):- n_block_keys = ["n_layers", "num_hidden_layers", "n_layer", "num_layers", "depth", "encoder_layers"]- for key in n_block_keys:- if hasattr(model.config.text_config, key):- setattr(model.config.text_config, key, num_layers)- if hasattr(model.config.text_config, "layer_types"):- model.config.text_config.layer_types = model.config.text_config.layer_types[:num_layers]- if hasattr(model.config, "vision_config"):- n_block_keys = ["n_layers", "num_hidden_layers", "n_layer", "num_layers", "depth", "encoder_layers"]- for key in n_block_keys:- if hasattr(model.config.vision_config, key):- setattr(model.config.vision_config, key, num_layers)- if hasattr(model.config, "layer_types"):- model.config.layer_types = model.config.layer_types[:num_layers]+ _reduce_config_layers(model.config, num_layers, num_experts)return model# for fixture usage only-def save_tiny_model(model_name_or_path, tiny_model_path, num_layers=2, is_mllm=False, force_untie=False, **kwargs):- """Generate a tiny model and save to the specified path."""- model = get_tiny_model(model_name_or_path, num_layers=num_layers, is_mllm=is_mllm, **kwargs)+def save_tiny_model(+ model_name_or_path,+ tiny_model_path,+ num_layers=2,+ num_experts=None,+ is_mllm=False,+ force_untie=False,+ from_config=False,+ **kwargs,+):+ """Generate a tiny model and save to the specified path.++ Args:+ model_name_or_path: HuggingFace model name or local path.+ tiny_model_path: Where to save the tiny model.+ num_layers: Number of layers to keep.+ num_experts: Number of experts to keep for MoE models (None = unchanged).+ is_mllm: Whether the model is a multi-modal model.+ force_untie: Force untie word embeddings.+ from_config: If True, initialise the model from config with random+ weights instead of downloading the full checkpoint.+ **kwargs: Extra keyword arguments forwarded to the model loader.+ """+ model = get_tiny_model(+ model_name_or_path,+ num_layers=num_layers,+ num_experts=num_experts,+ is_mllm=is_mllm,+ from_config=from_config,+ **kwargs,+ )if force_untie:if getattr(getattr(model, "config", None), "tie_word_embeddings", False):model.config.tie_word_embeddings = Falsediff --git a/test/test_cuda/models/test_omni_model.py b/test/test_cuda/models/test_omni_model.pyindex 3b31de75f..b4ccdb81c 100644--- a/test/test_cuda/models/test_omni_model.py+++ b/test/test_cuda/models/test_omni_model.py@@ -38,7 +38,7 @@ from transformers import (from auto_round import AutoRound-from ...helpers import check_version, qwen2_5_omni_name_or_path+from ...helpers import check_versionpytestmark = [pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available"),@@ -49,57 +49,34 @@ pytestmark = []-# ----------------------------------------------------------------------------# Fixture: tiny Qwen3-Omni-MoE-# Priority: use real config from qwen3_omni_name_or_path (skipped if absent);-# fall back to fully synthetic config using qwen_name_or_path tokenizer.-# ----------------------------------------------------------------------------def setup_qwen3_omni_moe(tiny_qwen3_omni_moe):- """Create a tiny Qwen3-Omni-MoE model.-- Uses the session-scoped ``tiny_qwen3_omni_moe`` fixture which loads the- real tokenizer/processor from ``qwen3_omni_name_or_path`` and builds a- model with reduced layers and random weights.- """- model, tokenizer, processor = tiny_qwen3_omni_moe- output_dir = "./tmp/test_quantized_qwen3_omni_moe"- return model, tokenizer, processor, output_dir, model.config--# ========================= Qwen2.5-Omni Integration Tests ==================[email protected]_ci(reason="Not necessary to test all cases in CI")class TestQwen2_5OmniQuantization:"""End-to-end quantization test for Qwen2.5-Omni (dense model)."""- def test_quantize_and_reload(self, tiny_qwen2_5_omni):+ def test_quantize_and_reload(self, tiny_qwen2_5_omni_model_path, tmp_path):"""Quantize, save, reload, verify weights, and run inference."""- model, tokenizer, processor = tiny_qwen2_5_omni- output_dir = "./tmp/test_quantized_qwen2_5_omni"-# Quantizeautoround = AutoRound(- model,- tokenizer,- processor=processor,+ tiny_qwen2_5_omni_model_path,nsamples=2,iters=1,seqlen=32,ignore_layers="self_attn,lm_head",)- quantized_model, save_folder = autoround.quantize_and_save(format="auto_round", output_dir=output_dir)+ quantized_model, save_folder = autoround.quantize_and_save(format="auto_round", output_dir=tmp_path)assert quantized_model is not None, "Quantized model should not be None"# Copy model-specific files required for from_pretrained (e.g. spk_dict.pt for token2wav)for extra_file in ["spk_dict.pt"]:- src = os.path.join(qwen2_5_omni_name_or_path, extra_file)+ src = os.path.join(tiny_qwen2_5_omni_model_path, extra_file)if os.path.exists(src):- shutil.copy2(src, output_dir)+ shutil.copy2(src, tmp_path)# Reload- loaded_model = Qwen2_5OmniForConditionalGeneration.from_pretrained(output_dir)- loaded_model.to("cuda")+ loaded_model = Qwen2_5OmniForConditionalGeneration.from_pretrained(tmp_path, device_map="cuda")# Run inference on thinkerinp = torch.randint(0, 100, (1, 64)).to("cuda")@@ -107,9 +84,6 @@ class TestQwen2_5OmniQuantization:output = loaded_model.thinker(input_ids=inp)assert output is not None, "Inference failed on reloaded model"- # Cleanup- shutil.rmtree(output_dir, ignore_errors=True)-# ========================= Qwen3-Omni-MoE Integration Tests ================@@ -117,26 +91,27 @@ class TestQwen2_5OmniQuantization:class TestQwen3OmniMoeQuantization:"""End-to-end quantization test for Qwen3-Omni-MoE."""- def test_quantize_and_reload(self, setup_qwen3_omni_moe):- """Quantize, save, reload, verify weights, and run inference."""- model, tokenizer, processor, output_dir, config = setup_qwen3_omni_moe+ @pytest.fixture(autouse=True)+ def _save_dir(self, tmp_path):+ self.save_dir = str(tmp_path / "saved")+ yield+ shutil.rmtree(self.save_dir, ignore_errors=True)+ def test_quantize_and_reload(self, tiny_qwen3_omni_moe_model_path):+ """Quantize, save, reload, verify weights, and run inference."""# Quantizeautoround = AutoRound(- model,- tokenizer,- processor=processor,+ tiny_qwen3_omni_moe_model_path,nsamples=2,iters=1,seqlen=32,ignore_layers="self_attn,lm_head,mlp.gate",)- quantized_model, save_folder = autoround.quantize_and_save(format="auto_round", output_dir=output_dir)+ quantized_model, save_folder = autoround.quantize_and_save(format="auto_round", output_dir=self.save_dir)assert quantized_model is not None, "Quantized model should not be None"# Reload- loaded_model = Qwen3OmniMoeForConditionalGeneration.from_pretrained(output_dir)- loaded_model.to("cuda")+ loaded_model = Qwen3OmniMoeForConditionalGeneration.from_pretrained(self.save_dir, device_map="cuda")# Run inference on thinkerinp = torch.randint(0, 100, (1, 64)).to("cuda")@@ -144,33 +119,24 @@ class TestQwen3OmniMoeQuantization:output = loaded_model.thinker(input_ids=inp)assert output is not None, "Inference failed on reloaded model (thinker)"- # Cleanup- shutil.rmtree(output_dir, ignore_errors=True)-- def test_quantize_mxfp4(self, setup_qwen3_omni_moe):+ @pytest.mark.skip_ci(reason="Not necessary to test all schemes in CI")+ def test_quantize_mxfp4(self, tiny_qwen3_omni_moe_model_path):"""Quantize with MXFP4 scheme and verify."""- model, tokenizer, processor, output_dir, config = setup_qwen3_omni_moe-autoround = AutoRound(- model,- tokenizer,- processor=processor,+ tiny_qwen3_omni_moe_model_path,scheme="MXFP4",nsamples=2,iters=1,seqlen=32,ignore_layers="self_attn,lm_head,mlp.gate",)- quantized_model, save_folder = autoround.quantize_and_save(format="auto_round", output_dir=output_dir)+ quantized_model, save_folder = autoround.quantize_and_save(format="auto_round", output_dir=self.save_dir)assert quantized_model is not None, "MXFP4 quantized model should not be None"# Reload and inference- loaded_model = Qwen3OmniMoeForConditionalGeneration.from_pretrained(output_dir)- loaded_model.to("cuda")+ loaded_model = Qwen3OmniMoeForConditionalGeneration.from_pretrained(self.save_dir, device_map="cuda")inp = torch.randint(0, 100, (1, 64)).to("cuda")with torch.inference_mode():output = loaded_model.thinker(input_ids=inp)assert output is not None-- shutil.rmtree(output_dir, ignore_errors=True)