Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

add mimo-audio, Qwen-TTS model backbone quantization (#1810)

Weiwei04e146937264
diff --git a/auto_round/algorithms/quantization/base.py b/auto_round/algorithms/quantization/base.py
index 765991844..e41c1223a 100644
--- a/auto_round/algorithms/quantization/base.py
+++ b/auto_round/algorithms/quantization/base.py
@@ -57,6 +57,7 @@ class BaseQuantizers:
"FluxSingleTransformerBlock": ["encoder_hidden_states", "hidden_states"],
"OvisImageTransformerBlock": ["encoder_hidden_states", "hidden_states"],
"OvisImageSingleTransformerBlock": ["encoder_hidden_states", "hidden_states"],
+ "StableAudioDiTBlock": ["hidden_states"],
"WanTransformerBlock": ["hidden_states"],
}
diff --git a/auto_round/calib_dataset.py b/auto_round/calib_dataset.py
index 5b48e814c..7d2e3a2f8 100644
--- a/auto_round/calib_dataset.py
+++ b/auto_round/calib_dataset.py
@@ -552,6 +552,71 @@ def get_mbpp_dataset(
return calib_dataset
+@register_dataset(["audiocaps", "AudioCaps"])
+def get_audiocaps_dataset(
+ tokenizer,
+ seqlen,
+ dataset_name="audiocaps",
+ split=None,
+ seed=42,
+ apply_chat_template=False,
+ system_prompt=None,
+):
+ """Returns a calibration dataset from AudioCaps (audio caption text).
+
+ AudioCaps provides human-written captions describing audio events. The captions
+ are rich in descriptive language about sounds, music, and speech — making them
+ well-suited as calibration text for audio-related model quantization.
+
+ The dataset CSV is downloaded from the official GitHub repository on first use.
+
+ Args:
+ tokenizer: The tokenizer to be used for tokenization.
+ seqlen: The maximum sequence length.
+ dataset_name: Dataset identifier (ignored, always loads AudioCaps).
+ split: Unused (AudioCaps train split is always used).
+ seed: The random seed for shuffling.
+ apply_chat_template: Whether to apply chat template in tokenization.
+ system_prompt: Optional system prompt for chat template.
+
+ Returns:
+ A HuggingFace Dataset with tokenized audio captions.
+ """
+ import csv
+
+ from auto_round.utils import download_audiocaps_csv
+
+ tokenizer_function = get_tokenizer_function(
+ tokenizer, seqlen, apply_chat_template=apply_chat_template, system_prompt=system_prompt
+ )
+
+ cache_file = download_audiocaps_csv()
+
+ samples = []
+ with open(cache_file, "r", encoding="utf-8") as f:
+ reader = csv.DictReader(f)
+ for row in reader:
+ caption = row.get("caption", "").strip()
+ if caption:
+ samples.append({"text": caption})
+
+ if not samples:
+ raise RuntimeError("AudioCaps dataset is empty or could not be parsed.")
+
+ random.Random(seed).shuffle(samples)
+ import datasets as ds
+
+ calib_dataset = ds.Dataset.from_list(samples)
+ calib_dataset = calib_dataset.map(
+ tokenizer_function,
+ batched=True,
+ new_fingerprint=_make_map_fingerprint(
+ calib_dataset, tokenizer, seqlen, apply_chat_template, system_prompt, "text"
+ ),
+ )
+ return calib_dataset
+
+
@register_dataset("local")
def get_local_dataset(
tokenizer, seqlen, dataset_name="./tmp.json", split=None, seed=42, apply_chat_template=False, system_prompt=None
diff --git a/auto_round/calibration/diffusion.py b/auto_round/calibration/diffusion.py
index fe3ae863f..4a63631df 100644
--- a/auto_round/calibration/diffusion.py
+++ b/auto_round/calibration/diffusion.py
@@ -95,41 +95,45 @@ class DiffusionCalibrator(LLMCalibrator):
)
exit(-1)
- if pipe.device != c.model.device:
- pipe.to(c.model.device)
-
+ target_device = c.compress_context.device
+ if pipe.device != torch.device(target_device):
+ pipe.to(target_device)
+ pipeline_fn = getattr(pipe, "_autoround_pipeline_fn", None)
# Check if this is an I2V pipeline (needs calibration image)
requires_image = hasattr(c, "_requires_calibration_image") and c._requires_calibration_image()
-
with tqdm(range(1, total + 1), desc="cache block inputs") as pbar:
for ids, prompts in c.dataloader:
if isinstance(prompts, tuple):
prompts = list(prompts)
try:
+ generator = (
+ None
+ if c.generator_seed is None
+ else torch.Generator(device=pipe.device).manual_seed(c.generator_seed)
+ )
if requires_image:
- # I2V pipeline: pass image and prompt as kwargs
image = c._get_calibration_image(len(prompts) if isinstance(prompts, list) else 1)
pipe(
image,
prompt=prompts,
guidance_scale=c.guidance_scale,
num_inference_steps=c.num_inference_steps,
- generator=(
- None
- if c.generator_seed is None
- else torch.Generator(device=pipe.device).manual_seed(c.generator_seed)
- ),
+ generator=generator,
+ )
+ elif pipeline_fn is not None:
+ pipeline_fn(
+ pipe,
+ prompts,
+ guidance_scale=c.guidance_scale,
+ num_inference_steps=c.num_inference_steps,
+ generator=generator,
)
else:
pipe(
prompts,
guidance_scale=c.guidance_scale,
num_inference_steps=c.num_inference_steps,
- generator=(
- None
- if c.generator_seed is None
- else torch.Generator(device=pipe.device).manual_seed(c.generator_seed)
- ),
+ generator=generator,
)
except NotImplementedError:
pass
diff --git a/auto_round/calibration/inputs.py b/auto_round/calibration/inputs.py
index 6893ce5d5..b397bdd05 100644
--- a/auto_round/calibration/inputs.py
+++ b/auto_round/calibration/inputs.py
@@ -27,6 +27,7 @@ def split_inputs(
first_input_name: str,
*,
is_diffusion: bool,
+ shared_cache_keys: tuple = (),
) -> Tuple[object, dict]:
"""Split a captured ``inputs`` dict into ``(input_ids, input_others)``.
@@ -34,7 +35,8 @@ def split_inputs(
- For diffusion models, every key containing ``"hidden_state"`` is pulled
out into a dict and returned as ``input_ids``; the remaining kwargs are
- returned as ``input_others``.
+ returned as ``input_others``. Keys in *shared_cache_keys* are kept in
+ ``input_others`` even if they match ``"hidden_state"``.
- Otherwise, ``inputs[first_input_name]`` is popped and returned as
``input_ids`` (may be ``None``); the remainder is ``input_others``.
@@ -42,7 +44,7 @@ def split_inputs(
behaviour that downstream code relies on.
"""
if is_diffusion:
- input_id_str = [key for key in inputs.keys() if "hidden_state" in key]
+ input_id_str = [key for key in inputs.keys() if "hidden_state" in key and key not in shared_cache_keys]
input_ids = {k: inputs.pop(k, None) for k in input_id_str}
input_others = inputs
return input_ids, input_others
@@ -66,7 +68,12 @@ def preprocess_block_inputs(
``is_diffusion``) and ``compress_context`` (for ``cache_device`` /
``device_list``) so it does not require a Compressor ``self``.
"""
- input_ids, input_others = split_inputs(inputs, first_input_name, is_diffusion=model_context.is_diffusion)
+ input_ids, input_others = split_inputs(
+ inputs,
+ first_input_name,
+ is_diffusion=model_context.is_diffusion,
+ shared_cache_keys=getattr(model_context, "shared_cache_keys", ()),
+ )
clear_memory(device_list=compress_context.device_list)
tmp_dtype = model_context.amp_dtype if model_context.amp else torch.float32
if input_ids is not None:
diff --git a/auto_round/calibration/mllm.py b/auto_round/calibration/mllm.py
index 0c403a54a..95156834e 100644
--- a/auto_round/calibration/mllm.py
+++ b/auto_round/calibration/mllm.py
@@ -45,6 +45,7 @@ class MLLMCalibrator(LLMCalibrator):
from auto_round.compressors.mllm.dataset import get_mllm_dataloader
from auto_round.compressors.mllm.template import get_template
from auto_round.special_model_handler import MISTRAL_3_2_MODELS, NOT_SUPPORT_ONLY_TEXT_MODELS
+ from auto_round.utils.model import resolve_model_type
c = self.compressor
mc = c.model_context
@@ -64,8 +65,8 @@ class MLLMCalibrator(LLMCalibrator):
c.template = "mistral3_2"
template_name = c.template
- if template_name is None and hasattr(mc.model.config, "model_type"):
- template_name = mc.model.config.model_type
+ if template_name is None:
+ template_name = resolve_model_type(mc.model) or getattr(mc.model.config, "model_type", None)
if template_name is None:
template_name = "default"
diff --git a/auto_round/compressors/data_driven.py b/auto_round/compressors/data_driven.py
index efd69c8bc..a5303371a 100644
--- a/auto_round/compressors/data_driven.py
+++ b/auto_round/compressors/data_driven.py
@@ -221,7 +221,12 @@ class DataDrivenCompressor(BaseCompressor):
# Thin wrapper around auto_round.calibration.inputs.split_inputs.
from auto_round.calibration.inputs import split_inputs
- return split_inputs(inputs, first_input_name, is_diffusion=self.model_context.is_diffusion)
+ return split_inputs(
+ inputs,
+ first_input_name,
+ is_diffusion=self.model_context.is_diffusion,
+ shared_cache_keys=self.model_context.shared_cache_keys,
+ )
def normalize_decoding_layer_inputs_(self, decoding_layer_inputs: list[tuple[tuple[Any, dict[str, Any]]]]) -> None:
"""Replay captured decoding-layer calls to populate ``self.inputs``.
diff --git a/auto_round/compressors/diffusion/README.md b/auto_round/compressors/diffusion/README.md
index 96501c9b6..ae78a0567 100644
--- a/auto_round/compressors/diffusion/README.md
+++ b/auto_round/compressors/diffusion/README.md
@@ -67,6 +67,7 @@ For diffusion models, currently we validate quantization on the following models
| Tongyi-MAI/Z-Image-Turb | COCO2014 | - |
| stepfun-ai/NextStep-1.1 | COCO2014 | - |
| AIDC-AI/Ovis-Image-7B | COCO2014 | - |
+| stabilityai/stable-audio-open-1.0 | audiocaps | - |
| Wan-AI/Wan2.2-I2V-A14B-Diffusers | COCO2014 | - |
| Wan-AI/Wan2.2-TI2V-5B-Diffusers | COCO2014 | - |
| Wan-AI/Wan2.2-T2V-A14B-Diffusers | COCO2014 | - |
@@ -88,4 +89,4 @@ id caption
- `id`: the id used to map generated images and prompts.
- `caption`: the text prompt used to generate the images.
-</details>
\ No newline at end of file
+</details>
diff --git a/auto_round/compressors/diffusion/dataset.py b/auto_round/compressors/diffusion/dataset.py
index a716a8a58..b3ee20d34 100644
--- a/auto_round/compressors/diffusion/dataset.py
+++ b/auto_round/compressors/diffusion/dataset.py
@@ -20,7 +20,7 @@ import torch
from torch.utils.data import DataLoader, Dataset
from transformers import set_seed
-from auto_round.utils import logger
+from auto_round.utils import download_audiocaps_csv, logger
DIFFUSION_DATASET: Dict[str, Dataset] = {}
@@ -84,6 +84,44 @@ class Text2ImgDataset(Dataset):
return self.caption_ids[i], self.captions[i]
+@register_dataset("audiocaps")
+class AudioCapsDataset(Dataset):
+ """Dataset for AudioCaps caption-based calibration.
+
+ AudioCaps CSV contains columns like ``audiocap_id``, ``youtube_id``,
+ ``start_time``, and ``caption``. For diffusion calibration we use
+ ``audiocap_id`` as the sample id and ``caption`` as the text prompt.
+ """
+
+ def __init__(self, dataset_path, nsamples=128) -> None:
+ super().__init__()
+ self.captions = []
+ self.caption_ids = []
+
+ logger.info(f"use dataset {dataset_path}, loading from disk...")
+ df = pd.read_csv(dataset_path)
+
+ id_col = "audiocap_id" if "audiocap_id" in df.columns else "id"
+ if "caption" not in df.columns:
+ raise ValueError("AudioCaps dataset must contain a 'caption' column")
+
+ for index, row in df.iterrows():
+ if nsamples > 0 and index + 1 > nsamples:
+ break
+ caption_id = row.get(id_col, index)
+ caption_text = str(row.get("caption", "")).strip()
+ if not caption_text:
+ continue
+ self.caption_ids.append(caption_id)
+ self.captions.append(caption_text)
+
+ def __len__(self):
+ return len(self.captions)
+
+ def __getitem__(self, i) -> Dict[str, torch.Tensor]:
+ return self.caption_ids[i], self.captions[i]
+
+
def get_diffusion_dataloader(
dataset="coco2014",
bs=1,
@@ -107,10 +145,16 @@ def get_diffusion_dataloader(
f.write(text_data)
dataset = "captions_source.tsv"
+ if dataset in ("audiocaps",):
+ dataset = download_audiocaps_csv()
+
if isinstance(dataset, str) and os.path.exists(dataset):
- dataset = DIFFUSION_DATASET["local"](dataset, nsamples)
+ if dataset.endswith(".csv"):
+ dataset = DIFFUSION_DATASET["audiocaps"](dataset, nsamples)
+ else:
+ dataset = DIFFUSION_DATASET["local"](dataset, nsamples)
else:
- raise ValueError("Only support coco2014 dataset or loading local tsv file now.")
+ raise ValueError("Only support coco2014/audiocaps dataset or loading local tsv/csv file now.")
set_seed(seed)
dataloader_params = {"batch_size": bs, "shuffle": True}
diff --git a/auto_round/compressors/diffusion_mixin.py b/auto_round/compressors/diffusion_mixin.py
index 84cde97e3..e044415b4 100644
--- a/auto_round/compressors/diffusion_mixin.py
+++ b/auto_round/compressors/diffusion_mixin.py
@@ -617,7 +617,17 @@ class DiffusionMixin:
folders.append(target_output_dir)
- pipe.save_config(output_dir)
+ if hasattr(pipe, "save_config"):
+ pipe.save_config(output_dir)
+ elif hasattr(pipe.config, "save_pretrained"):
+ pipe.config.save_pretrained(output_dir)
+ else:
+ # FrozenDict / plain dict — write model_index.json manually
+ import json
+
+ model_index_path = os.path.join(output_dir, "model_index.json")
+ with open(model_index_path, "w", encoding="utf-8") as f:
+ f.write(json.dumps(dict(pipe.config), indent=2, sort_keys=True) + "\n")
if return_folders:
return compressed_model, folders
diff --git a/auto_round/compressors/mllm/README.md b/auto_round/compressors/mllm/README.md
index ee44675db..de7a106ab 100644
--- a/auto_round/compressors/mllm/README.md
+++ b/auto_round/compressors/mllm/README.md
@@ -44,12 +44,13 @@ auto-round \
--output_dir ./tmp_autoround
```
-### VLM Support Matrix
+### VLM & Omni Support Matrix
For most VLMs, we typically support the default quantization configuration, which involves quantizing only the language
component while excluding the visual component. Besides, we also support quantizing non-text modules of models that
follow the Hugging Face standard, i.e., those with a typical processor, though inference may have some issues due to
-model architecture or kernel limitations.
+model architecture or kernel limitations. Audio models (MiMo-Audio, Qwen3-TTS, etc.) use text-only calibration
+by default.
| Model | calibration dataset | quant nontext module | Quantized Model Link |
|-----------------------------------------------|---------------------|----------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
@@ -67,6 +68,8 @@ model architecture or kernel limitations.
| Qwen/Qwen2-VL | pile/llava | - | [Qwen2-VL-7B-Instruct-int4-sym](https://huggingface.co/OPEA/Qwen2-VL-7B-Instruct-int4-sym-inc), [Qwen2-VL-72B-Instruct-int4-sym](https://huggingface.co/OPEA/Qwen2-VL-72B-Instruct-int4-sym-inc), [Qwen2-VL-72B-Instruct-int2-sym](https://huggingface.co/OPEA/Qwen2-VL-72B-Instruct-int2-sym-inc) |
| Qwen/Qwen2.5-Omni | pile | √ | [Qwen2.5-Omni-7B](https://huggingface.co/Intel/Qwen2.5-Omni-7B-int4-AutoRound) |
| Qwen/Qwen3-Omni | pile | √ | [Qwen3-Omni-30B-A3B-Instruct](https://huggingface.co/Intel/Qwen3-Omni-30B-A3B-Instruct-int4-AutoRound) |
+| Qwen/Qwen3-TTS | pile/audiocaps | X | |
+| XiaomiMiMo/MiMo-Audio | pile/audiocaps | X | |
| rhymes-ai/Aria | pile/llava | √ | |
| THUDM/CogVLM2 | pile/llava | √ | [cogvlm2-llama3-chat-19B-int4-sym](https://huggingface.co/OPEA/cogvlm2-llama3-chat-19B-int4-sym-inc), [cogvlm2-llama3-chat-19B-qvision-int4-sym](https://huggingface.co/OPEA/cogvlm2-llama3-chat-19B-qvision-int4-sym-inc) |
| THUDM/glm-4v | pile | X | [glm-4v-9b-int4-sym](https://huggingface.co/OPEA/glm-4v-9b-int4-sym-inc) |
diff --git a/auto_round/compressors/mllm/processor.py b/auto_round/compressors/mllm/processor.py
index ffd6ae387..2165e5456 100644
--- a/auto_round/compressors/mllm/processor.py
+++ b/auto_round/compressors/mllm/processor.py
@@ -367,6 +367,47 @@ class Qwen3OmniProcessor(HFProcessor):
return ret
+@register_processor("mimo_audio")
+@register_processor("qwen3_tts")
+class AudioTextProcessor(BasicProcessor):
+ """Processor for audio/TTS models that use text-only calibration.
+
+ Supports MiMo-Audio (MiMoAudioForCausalLM) and Qwen3-TTS
+ (Qwen3TTSForConditionalGeneration). Both models use text data for
+ quantization calibration since audio calibration is not yet supported.
+ """
+
+ def post_init(self, model, tokenizer, processor=None, image_processor=None, use_rtn=False, **kwargs):
+ assert tokenizer is not None, "tokenizer should not be None"
+ self.model = model
+ self.tokenizer = tokenizer
+ self.processor = processor
+ self.image_processor = None # Audio/TTS model, no image processor needed
+ self.use_rtn = use_rtn
+
+ def check_image_processor(self):
+ pass
+
+ def get_input(self, text, images, squeeze=True, max_length=None, truncation=False, **kwargs):
+ if isinstance(text, list):
+ if hasattr(self.tokenizer, "chat_template") and self.tokenizer.chat_template:
+ text = self.tokenizer.apply_chat_template(text, tokenize=False, add_generation_prompt=True)
+ else:
+ text = (
+ " ".join(msg["content"] for msg in text if isinstance(msg, dict) and "content" in msg)
+ if all(isinstance(msg, dict) for msg in text)
+ else " ".join(str(t) for t in text)
+ )
+
+ if max_length is not None:
+ ret = self.tokenizer(text, return_tensors="pt", truncation=True, max_length=max_length)
+ else:
+ ret = self.tokenizer(text, return_tensors="pt")
+ if squeeze:
+ ret = self.squeeze_result(ret)
+ return ret
+
+
@register_processor("cogvlm2")
class CogVLM2Processor(BasicProcessor):
def get_input(self, text, images, truncation=False, squeeze=True, max_length=None, **kwargs):
diff --git a/auto_round/compressors/mllm/template.py b/auto_round/compressors/mllm/template.py
index eb4a56e24..dc5f346c8 100644
--- a/auto_round/compressors/mllm/template.py
+++ b/auto_round/compressors/mllm/template.py
@@ -128,6 +128,8 @@ _register_template("mistral3", default_dataset="NeelNanda/pile-10k", processor=P
_register_template("mistral3_2", default_dataset="liuhaotian/llava", processor=PROCESSORS["mistral3_2"])
_register_template("gemma3", default_dataset="NeelNanda/pile-10k", processor=PROCESSORS["hf"])
_register_template("longcat_next", default_dataset="liuhaotian/llava", processor=PROCESSORS["longcat_next"])
+_register_template("mimo_audio", default_dataset="NeelNanda/pile-10k", processor=PROCESSORS["mimo_audio"])
+_register_template("qwen3_tts", default_dataset="NeelNanda/pile-10k", processor=PROCESSORS["qwen3_tts"])
def load_template(path: str):
diff --git a/auto_round/export/utils.py b/auto_round/export/utils.py
index 85765d11b..395cd7fa4 100644
--- a/auto_round/export/utils.py
+++ b/auto_round/export/utils.py
@@ -29,7 +29,16 @@ from auto_round.utils import (
def _save_model_configs(model: nn.Module, save_dir: str) -> None:
if hasattr(model, "config") and model.config is not None:
- model.config.save_pretrained(save_dir)
+ try:
+ model.config.save_pretrained(save_dir)
+ except (KeyError, TypeError):
+ # Some third-party configs (e.g. qwen-tts) fail with use_diff=True
+ # due to missing keys in recursive_diff_dict. Fall back to full config.
+ import json
+
+ config_path = os.path.join(save_dir, "config.json")
+ with open(config_path, "w", encoding="utf-8") as f:
+ f.write(model.config.to_json_string(use_diff=False))
if hasattr(model, "generation_config") and model.generation_config is not None:
model.generation_config.save_pretrained(save_dir)
@@ -222,7 +231,17 @@ def save_model(
)
_save_model_configs(model, save_dir)
else:
- model.save_pretrained(save_dir, max_shard_size=max_shard_size, safe_serialization=safe_serialization)
+ try:
+ model.save_pretrained(save_dir, max_shard_size=max_shard_size, safe_serialization=safe_serialization)
+ except (KeyError, TypeError) as e:
+ # Some third-party configs fail during config serialization in save_pretrained.
+ # Fall back to saving weights separately + config without diff.
+ logger.warning("model.save_pretrained failed (%s), falling back to manual save.", e)
+ from safetensors.torch import save_file
+
+ state_dict = model.state_dict()
+ save_file(state_dict, os.path.join(save_dir, "model.safetensors"))
+ _save_model_configs(model, save_dir)
source_dir = _resolve_model_source_dir(model)
diff --git a/auto_round/special_model_handler.py b/auto_round/special_model_handler.py
index a23bac533..1d94f6354 100644
--- a/auto_round/special_model_handler.py
+++ b/auto_round/special_model_handler.py
@@ -29,6 +29,8 @@ mllms_with_limited_bs = (
"qwen2_5_omni",
"qwen3_omni_moe",
"glm_image",
+ "mimo_audio",
+ "qwen3_tts",
) # Limitations on batch_size
SUPPORT_ONLY_TEXT_MODELS = [
@@ -48,16 +50,22 @@ SUPPORT_ONLY_TEXT_MODELS = [
"qwen2_5_omni",
"qwen3_omni_moe",
"gemma3",
- "gemma4",
"bagel",
+ "mimo_audio",
+ "qwen3_tts",
]
NOT_SUPPORT_ONLY_TEXT_MODELS = ["mllama", "mistral3_2"]
+# Maps architecture class names to virtual model_type keys.
+# Used when config.model_type doesn't uniquely identify the model (e.g. MiMo-Audio).
+from auto_round.utils.model import ARCHITECTURE_MODEL_TYPE_MAP, resolve_model_type # noqa: E402
+
SPECIAL_SHARED_CACHE_KEYS = {
"Gemma3ForConditionalGeneration": ("position_embeddings_global", "position_embeddings_local")
}
SPECIAL_SHARED_CACHE_KEYS["MiniMaxText01ForCausalLM"] = ("slope_rate",)
+SPECIAL_SHARED_CACHE_KEYS["StableAudioDiTModel"] = ("encoder_hidden_states",)
SPECIAL_SHARED_CACHE_KEYS["Gemma4ForConditionalGeneration"] = ("position_ids",)
SPECIAL_SHARED_CACHE_KEYS["WanTransformer3DModel"] = ("rotary_emb",)
MISTRAL_3_2_MODELS = ["Mistral-Small-3.2", "Magistral-Small", "Devstral-Small"]
@@ -311,7 +319,7 @@ def _patch_gemma4_model(model):
def _handle_special_model(model):
- model_type = getattr(getattr(model, "config", None), "model_type", None)
+ model_type = resolve_model_type(model)
if model_type == "deepseek_vl_v2":
from functools import partial
@@ -324,6 +332,14 @@ def _handle_special_model(model):
from functools import partial
model.forward = partial(_qwen3_omni_moe_forward, model)
+ if model_type == "qwen3_tts":
+ from functools import partial
+
+ model.forward = partial(_qwen3_tts_forward, model)
+ if model_type == "mimo_audio":
+ from functools import partial
+
+ model.forward = partial(_mimo_audio_forward, model)
if hasattr(model, "config") and model_type == "gemma4":
import transformers
from packaging import version
@@ -463,6 +479,64 @@ def _get_glm_image_multimodal_block(model, quant_vision=False):
return block_names
+def _get_mimo_audio_multimodal_block(model, quant_vision=False):
+ """Get block names for MiMo-Audio model.
+
+ MiMo-Audio (MiMoAudioForCausalLM) has the following structure:
+ - model.model.layers: Main Qwen2-based LLM decoder (28 layers)
+ - input_local_transformer.layers: Input audio encoder (6 layers)
+ - local_transformer.layers: Audio decoder / patch decoder (16 layers)
+
+ Currently only the main LLM backbone is quantized because:
+ - Audio encoder/decoder require audio calibration data (not yet supported)
+ - Audio modules use smaller hidden dim (1024 vs 4096) with limited quantization benefit
+
+ Args:
+ model: The MiMoAudioForCausalLM or MiMoAudioModel instance.
+ """
+ block_names = []
+
+ # Main LLM decoder layers (backbone - always quantized)
+ if hasattr(model, "model") and hasattr(model.model, "layers"):
+ # MiMoAudioForCausalLM: layers at model.model.layers
+ block_names.append([f"model.layers.{i}" for i in range(len(model.model.layers))])
+ elif hasattr(model, "layers"):
+ # MiMoAudioModel (base): layers at model.layers directly
+ block_names.append([f"layers.{i}" for i in range(len(model.layers))])
+
+ return block_names
+
+
+def _get_qwen3_tts_multimodal_block(model, quant_vision=False):
+ """Get block names for Qwen3-TTS model.
+
+ Qwen3-TTS (Qwen3TTSForConditionalGeneration) has a talker backbone with the
+ following possible module layouts depending on the modeling code version:
+ - tts_model.model.layers: Main TTS transformer decoder layers
+ - talker.model.layers: Alternative attribute name (matches talker_config)
+ - model.model.layers: Standard fallback
+
+ Only the talker backbone is quantized. Audio codec layers are not quantized
+ because audio calibration data is not yet supported.
+
+ Args:
+ model: The Qwen3TTSForConditionalGeneration model instance.
+ quant_vision: Unused. Accepted for interface compatibility with
+ SPECIAL_MULTIMODAL_BLOCK registry.
+ """
+ block_names = []
+ # Try tts_model.model.layers
+ if hasattr(model, "tts_model") and hasattr(getattr(model.tts_model, "model", None), "layers"):
+ block_names.append([f"tts_model.model.layers.{i}" for i in range(len(model.tts_model.model.layers))])
+ # Try talker.model.layers (alternative attr name from talker_config)
+ if not block_names and hasattr(model, "talker") and hasattr(getattr(model.talker, "model", None), "layers"):
+ block_names.append([f"talker.model.layers.{i}" for i in range(len(model.talker.model.layers))])
+ # Fallback: model.model.layers (standard structure)
+ if not block_names and hasattr(model, "model") and hasattr(model.model, "layers"):
+ block_names.append([f"model.layers.{i}" for i in range(len(model.model.layers))])
+ return block_names
+
+
def _get_bagel_multimodal_block(model, quant_vision=False):
"""Get block names for BAGEL MoT (Mixture of Transformers) model.
@@ -491,6 +565,8 @@ SPECIAL_MULTIMODAL_BLOCK = {
"qwen2_5_omni": _get_qwen2_5_omni_multimodal_block,
"qwen3_omni_moe": _get_qwen3_omni_moe_multimodal_block,
"glm_image": _get_glm_image_multimodal_block,
+ "mimo_audio": _get_mimo_audio_multimodal_block,
+ "qwen3_tts": _get_qwen3_tts_multimodal_block,
"bagel": _get_bagel_multimodal_block,
}
@@ -715,12 +791,133 @@ def _qwen3_omni_moe_forward(
return thinker_output
+def _mimo_audio_forward(
+ model,
+ input_ids=None,
+ attention_mask=None,
+ position_ids=None,
+ past_key_values=None,
+ inputs_embeds=None,
+ labels=None,
+ use_cache=None,
+ output_attentions=None,
+ output_hidden_states=None,
+ return_dict=None,
+ cache_position=None,
+ **kwargs,
+):
+ """Forward function for MiMo-Audio model during calibration.
+
+ MiMo-Audio's native forward expects 3D input_ids [B, audio_channels+1, T] with
+ interleaved text and audio tokens. During calibration with text-only data, we
+ route through the Qwen2Model backbone directly to calibrate the main decoder layers.
+ """
+ backbone = model.model
+
+ if input_ids is not None and inputs_embeds is None:
+ inputs_embeds = backbone.embed_tokens(input_ids)
+ input_ids = None
+
+ return backbone(
+ input_ids=input_ids,
+ attention_mask=attention_mask,
+ position_ids=position_ids,
+ past_key_values=past_key_values,
+ inputs_embeds=inputs_embeds,
+ use_cache=use_cache,
+ output_attentions=output_attentions,
+ output_hidden_states=output_hidden_states,
+ return_dict=return_dict,
+ cache_position=cache_position,
+ )
+
+
+def _qwen3_tts_forward(
+ model,
+ input_ids=None,
+ attention_mask=None,
+ position_ids=None,
+ past_key_values=None,
+ inputs_embeds=None,
+ labels=None,
+ use_cache=None,
+ output_attentions=None,
+ output_hidden_states=None,
+ return_dict=None,
+ cache_position=None,
+ **kwargs,
+):
+ """Forward function for Qwen3-TTS model.
+
+ Qwen3-TTS uses a discrete multi-codebook LM architecture. During calibration
+ we route the forward through the talker, converting text input_ids to
+ inputs_embeds via text_embedding + text_projection.
+ """
+ tts_backbone = None
+ for candidate in [getattr(model, "tts_model", None), getattr(model, "talker", None)]:
+ if candidate is None:
+ continue
+ has_text_embed = hasattr(getattr(candidate, "model", None), "text_embedding")
+ has_text_proj = hasattr(candidate, "text_projection")
+ if has_text_embed and has_text_proj:
+ tts_backbone = candidate
+ break
+ # Fallback: use whichever backbone exists even without text pathway
+ if tts_backbone is None:
+ tts_backbone = getattr(model, "tts_model", None) or getattr(model, "talker", None)
+
+ if tts_backbone is not None:
+ # Convert text input_ids to inputs_embeds through text pathway
+ if input_ids is not None and inputs_embeds is None:
+ text_embedding = getattr(tts_backbone.model, "text_embedding", None)
+ text_projection = getattr(tts_backbone, "text_projection", None)
+ if text_embedding is None or text_projection is None:
+ raise RuntimeError(
+ "Qwen3-TTS backbone is missing text_embedding or text_projection. "
+ "Cannot convert text input_ids to inputs_embeds for calibration."
+ )
+ inputs_embeds = text_projection(text_embedding(input_ids))
+ input_ids = None
+
+ return tts_backbone(
+ input_ids=input_ids,
+ attention_mask=attention_mask,
+ position_ids=position_ids,
+ past_key_values=past_key_values,
+ inputs_embeds=inputs_embeds,
+ labels=labels,
+ use_cache=use_cache,
+ output_attentions=output_attentions,
+ output_hidden_states=output_hidden_states,
+ return_dict=return_dict,
+ cache_position=cache_position,
+ **kwargs,
+ )
+ # Fallback: model has standard forward
+ return model.__class__.forward(
+ model,
+ input_ids=input_ids,
+ attention_mask=attention_mask,
+ position_ids=position_ids,
+ past_key_values=past_key_values,
+ inputs_embeds=inputs_embeds,
+ labels=labels,
+ use_cache=use_cache,
+ output_attentions=output_attentions,
+ output_hidden_states=output_hidden_states,
+ return_dict=return_dict,
+ cache_position=cache_position,
+ **kwargs,
+ )
+
+
def check_mllm_model_batch(model, batch_size, gradient_accumulate_steps=1):
"""
Checks model configuration to determine if it's necessary to limit bs to avoid potential input shape mismatches.
"""
+ effective_type = resolve_model_type(model) or ""
for key in mllms_with_limited_bs:
- if hasattr(model, "config") and key in model.config.model_type and batch_size != 1:
+ if key in effective_type and batch_size != 1:
accumulate_steps = batch_size * gradient_accumulate_steps
logger.warning(
"To avoid the tensor concat mismatch problem, modified parameters to "
diff --git a/auto_round/utils/common.py b/auto_round/utils/common.py
index 5e6d18c09..14e9033ac 100644
--- a/auto_round/utils/common.py
+++ b/auto_round/utils/common.py
@@ -29,6 +29,44 @@ from auto_round.export.export_to_gguf.config import GGUF_CONFIG
from auto_round.logger import logger
+def download_audiocaps_csv():
+ """Download AudioCaps train.csv and return the local cache path.
+
+ Downloads from GitHub on first use and caches in a temporary directory.
+
+ Returns:
+ str: Path to the cached CSV file.
+ """
+ import tempfile
+
+ import requests
+
+ url = "https://raw.githubusercontent.com/cdjkim/audiocaps/master/dataset2.0/train.csv"
+ cache_dir = os.path.join(tempfile.gettempdir(), "audiocaps_cache")
+ os.makedirs(cache_dir, exist_ok=True)
+ cache_file = os.path.join(cache_dir, "train.csv")
+
+ if os.path.exists(cache_file) and os.path.getsize(cache_file) > 0:
+ logger.debug(f"Using cached AudioCaps dataset: {cache_file}")
+ return cache_file
+
+ logger.info("Downloading AudioCaps dataset from GitHub...")
+ try:
+ resp = requests.get(url, timeout=60)
+ resp.raise_for_status()
+ if not resp.text or len(resp.text.strip()) == 0:
+ raise RuntimeError("Downloaded AudioCaps dataset is empty")
+ with open(cache_file, "w", encoding="utf-8") as f:
+ f.write(resp.text)
+ logger.info(f"AudioCaps dataset cached at: {cache_file}")
+ except requests.RequestException as e:
+ raise RuntimeError(f"Failed to download AudioCaps from {url}: {e}") from e
+ except IOError as e:
+ raise RuntimeError(f"Failed to write AudioCaps cache to {cache_file}: {e}") from e
+
+ return cache_file
+
+
def compare_versions(v1, v2):
return version.parse(v1) >= version.parse(v2)
diff --git a/auto_round/utils/model.py b/auto_round/utils/model.py
index 8569fab9c..8cf67dab8 100644
--- a/auto_round/utils/model.py
+++ b/auto_round/utils/model.py
@@ -36,6 +36,40 @@ from auto_round.utils.weight_handler import (
is_quantized_input_module,
)
+# Maps architecture class names to virtual model_type keys.
+# Used when config.model_type doesn't uniquely identify the model (e.g. MiMo-Audio
+# has model_type="qwen2" but needs audio-specific handling).
+ARCHITECTURE_MODEL_TYPE_MAP = {
+ "MiMoAudioModel": "mimo_audio",
+ "MiMoAudioForCausalLM": "mimo_audio",
+}
+
+
+def resolve_model_type(model):
+ """Resolve the effective model type using architecture class name as primary source.
+
+ This function prioritizes the model's architecture class name (from config.architectures)
+ over config.model_type to handle models where the two diverge (e.g., MiMo-Audio has
+ architecture="MiMoAudioModel" but model_type="qwen2" on HuggingFace).
+
+ Args:
+ model: A model instance with optional config attribute.
+
+ Returns:
+ str or None: The resolved model type identifier, or None if config is missing.
+ """
+ config = getattr(model, "config", None)
+ if config is None:
+ return None
+ # Check architecture-based override first
+ archs = getattr(config, "architectures", None)
+ if archs:
+ for arch in archs:
+ if arch in ARCHITECTURE_MODEL_TYPE_MAP:
+ return ARCHITECTURE_MODEL_TYPE_MAP[arch]
+ return getattr(config, "model_type", None)
+
+
FIX_MISTRAL_REGEX_MODEL_TYPE_LIST = ["longcat_next"]
if TYPE_CHECKING:
@@ -486,12 +520,18 @@ def mllm_load_model(
if platform == "model_scope":
import modelscope # pylint: disable=E0401
- from modelscope import AutoModel, AutoModelForCausalLM, AutoProcessor, AutoTokenizer # pylint: disable=E0401
+ from modelscope import ( # pylint: disable=E0401
+ AutoConfig,
+ AutoModel,
+ AutoModelForCausalLM,
+ AutoProcessor,
+ AutoTokenizer,
+ )
base_lib = modelscope
else:
import transformers
- from transformers import AutoModel, AutoModelForCausalLM, AutoProcessor, AutoTokenizer
+ from transformers import AutoConfig, AutoModel, AutoModelForCausalLM, AutoProcessor, AutoTokenizer
base_lib = transformers
@@ -552,7 +592,84 @@ def mllm_load_model(
)
processor, image_processor = None, None
- if "deepseek_vl_v2" == model_type:
+ if "qwen3_tts" == model_type:
+ try:
+ from qwen_tts.core.models import Qwen3TTSConfig, Qwen3TTSForConditionalGeneration
+ from qwen_tts.core.models.processing_qwen3_tts import Qwen3TTSProcessor
+
+ AutoConfig.register("qwen3_tts", Qwen3TTSConfig)
+ AutoModel.register(Qwen3TTSConfig, Qwen3TTSForConditionalGeneration)
+ AutoModelForCausalLM.register(Qwen3TTSConfig, Qwen3TTSForConditionalGeneration)
+ AutoProcessor.register(Qwen3TTSConfig, Qwen3TTSProcessor)
+ except ImportError:
+ raise ImportError("Qwen3-TTS requires the 'qwen-tts' package. " "Please install it: pip install qwen-tts")
+ except TypeError as e:
+ if "check_model_inputs" in str(e):
+ raise ImportError(
+ f"Qwen3-TTS 'qwen-tts' package is incompatible with transformers {transformers.__version__}. "
+ "Please upgrade qwen-tts: pip install -U qwen-tts"
+ ) from e
+ raise
+
+ # MiMo-Audio: architectures=["MiMoAudioModel"] but model_type="qwen2".
+ # Requires MiMo-Audio SDK from https://github.com/XiaomiMiMo/MiMo-Audio
+ # Set MIMO_AUDIO_PATH env var to the cloned repo root (containing src/mimo_audio/).
+ architectures = config.get("architectures", [])
+ _is_mimo_audio = any(a in ("MiMoAudioModel", "MiMoAudioForCausalLM") for a in architectures)
+
+ if _is_mimo_audio:
+ try:
+ from mimo_audio.modeling_mimo_audio import MiMoAudioArguments, MiMoAudioForCausalLM
+ except ImportError:
+ # Try adding MIMO_AUDIO_PATH/src to sys.path
+ mimo_path = os.environ.get("MIMO_AUDIO_PATH")
+ if mimo_path:
+ import sys
+
+ src_path = os.path.join(mimo_path, "src")
+ if src_path not in sys.path:
+ sys.path.insert(0, src_path)
+ try:
+ from mimo_audio.modeling_mimo_audio import MiMoAudioArguments, MiMoAudioForCausalLM
+ except ImportError:
+ raise ImportError(
+ "MiMo-Audio requires the MiMo-Audio SDK. "
+ "Please clone it: git clone https://github.com/XiaomiMiMo/MiMo-Audio.git "
+ "and set MIMO_AUDIO_PATH to the repo root."
+ )
+ else:
+ raise ImportError(
+ "MiMo-Audio requires the MiMo-Audio SDK. "
+ "Please clone https://github.com/XiaomiMiMo/MiMo-Audio and set env var "
+ "MIMO_AUDIO_PATH to the repo root (e.g. export MIMO_AUDIO_PATH=/path/to/MiMo-Audio)."
+ )
+
+ tokenizer = AutoTokenizer.from_pretrained(pretrained_model_name_or_path, trust_remote_code=trust_remote_code)
+ # Ensure special tokens are registered
+ special_tokens = ["<|sosp|>", "<|eosp|>", "<|empty|>", "<|sostm|>", "<|eostm|>", "<|eot|>"]
+ for token in special_tokens:
+ if token not in tokenizer.get_vocab():
+ tokenizer.add_tokens([token], special_tokens=True)
+
+ model_args = MiMoAudioArguments(
+ model_name_or_path=pretrained_model_name_or_path,
+ sosp_idx=tokenizer.convert_tokens_to_ids("<|sosp|>"),
+ eosp_idx=tokenizer.convert_tokens_to_ids("<|eosp|>"),
+ sostm_idx=tokenizer.convert_tokens_to_ids("<|sostm|>"),
+ eostm_idx=tokenizer.convert_tokens_to_ids("<|eostm|>"),
+ eot_idx=tokenizer.convert_tokens_to_ids("<|eot|>"),
+ empty_idx=tokenizer.convert_tokens_to_ids("<|empty|>"),
+ )
+
+ model = MiMoAudioForCausalLM.from_pretrained(
+ pretrained_model_name_or_path,
+ args=model_args,
+ torch_dtype=torch_dtype,
+ device_map="auto" if use_auto_mapping else None,
+ )
+ processor = None
+
+ elif "deepseek_vl_v2" == model_type:
from deepseek_vl2.models import DeepseekVLV2ForCausalLM, DeepseekVLV2Processor # pylint: disable=E0401
processor = DeepseekVLV2Processor.from_pretrained(pretrained_model_name_or_path)
@@ -675,6 +792,26 @@ def mllm_load_model(
return model, processor, tokenizer, image_processor
+def _attach_diffusion_pipeline_fn(pipe):
+ """Attach a custom pipeline function for diffusion models that need special API calls."""
+ pipe_class_name = type(pipe).__name__
+ if pipe_class_name == "StableAudioPipeline":
+
+ def _stable_audio_pipeline_fn(
+ pipe, prompts, guidance_scale=7.0, num_inference_steps=100, generator=None, **kwargs
+ ):
+ audio_end_in_s = kwargs.pop("audio_end_in_s", 10.0)
+ return pipe(
+ prompts,
+ guidance_scale=guidance_scale,
+ num_inference_steps=num_inference_steps,
+ audio_end_in_s=audio_end_in_s,
+ generator=generator,
+ )
+
+ pipe._autoround_pipeline_fn = _stable_audio_pipeline_fn
+
+
def diffusion_load_model(
pretrained_model_name_or_path: str,
platform: str = "hf",
@@ -719,15 +856,15 @@ def diffusion_load_model(
pipelines = LazyImport("diffusers.pipelines")
if isinstance(pretrained_model_name_or_path, str):
+ model_index = os.path.join(pretrained_model_name_or_path, "model_index.json")
+ with open(model_index, "r", encoding="utf-8") as file:
+ config = json.load(file)
+
if torch_dtype == "auto":
torch_dtype = {}
- model_index = os.path.join(pretrained_model_name_or_path, "model_index.json")
- with open(model_index, "r", encoding="utf-8") as file:
- config = json.load(file)
for k, v in config.items():
component_folder = os.path.join(pretrained_model_name_or_path, k)
if isinstance(v, list) and os.path.exists(os.path.join(component_folder, "config.json")):
- component_folder = os.path.join(pretrained_model_name_or_path, k)
with open(os.path.join(component_folder, "config.json"), "r", encoding="utf-8") as file:
component_config = json.load(file)
torch_dtype[k] = component_config.get("torch_dtype", "auto")
@@ -754,6 +891,9 @@ def diffusion_load_model(
pipe = _to_model_dtype(pipe, model_dtype)
model = pipe.transformer
+ # Attach custom pipeline function for models that need special API calls
+ _attach_diffusion_pipeline_fn(pipe)
+
# meta model uses model.config.save_pretrained for config saving
setattr(model.config, "save_pretrained", partial(config_save_pretrained, model.config, "config.json", model=model))
setattr(pipe.config, "save_pretrained", partial(config_save_pretrained, pipe.config, "model_index.json"))
@@ -977,12 +1117,9 @@ def get_block_names(model, quant_vision=False):
def _get_vlm_block_names(model, quant_vision=False, ignore_audio=True):
# Since calibration dataset doesn't contain audio data, audio-related blocks will be ignored by default.
- if (
- hasattr(model, "config")
- and hasattr(model.config, "model_type")
- and model.config.model_type in SPECIAL_MULTIMODAL_BLOCK.keys()
- ):
- return SPECIAL_MULTIMODAL_BLOCK.get(model.config.model_type)(model, quant_vision=quant_vision)
+ effective_type = resolve_model_type(model)
+ if effective_type and effective_type in SPECIAL_MULTIMODAL_BLOCK:
+ return SPECIAL_MULTIMODAL_BLOCK[effective_type](model, quant_vision=quant_vision)
block_names = []
target_modules = []
vision_blocks_tuple = ("vision", "visual", "image", "img")
@@ -998,6 +1135,12 @@ def get_block_names(model, quant_vision=False):
block_names[-1].append(target_m[0] + "." + n)
return block_names
+ # Check architecture-based special handlers first (e.g. MiMo-Audio has model_type="qwen2"
+ # but is_pure_text_model returns True since it has no vision modules — only audio ones).
+ effective_type = resolve_model_type(model)
+ if effective_type and effective_type in SPECIAL_MULTIMODAL_BLOCK:
+ return SPECIAL_MULTIMODAL_BLOCK[effective_type](model, quant_vision=quant_vision)
+
if quant_vision or not is_pure_text_model(model):
return _get_vlm_block_names(model, quant_vision=quant_vision)
else:
@@ -1492,7 +1635,10 @@ def is_moe_model(model: torch.nn.Module) -> bool:
def is_moe_model_via_config(config) -> bool:
- config_str = str(config).lower()
+ try:
+ config_str = str(config).lower()
+ except Exception:
+ config_str = str(config.to_dict()).lower() if hasattr(config, "to_dict") else ""
if "moe" in config_str or "expert" in config_str:
return True
return False
diff --git a/test/README.md b/test/README.md
index ee0af61f1..5fc5ecfcf 100644
--- a/test/README.md
+++ b/test/README.md
@@ -18,7 +18,7 @@ Tests are organized by hardware backend (`test_cpu/`, `test_cuda/`) and function
- **quantization/** - Quantization techniques (mixed-bit, MXFP, NVFP4, activation quant)
- **export/** - Model serialization (GGUF, AutoGPTQ, AutoRound format)
- **backends/** - Inference backends (Torch, Marlin, Triton, ExLlamaV2)
-- **models/** - Architecture-specific tests (MLLMs, VLMs, MoE, Diffusion)
+- **models/** - Architecture-specific tests (MLLMs, VLMs, MoE, Diffusion, Omni)
- **integrations/** - Third-party frameworks (vLLM, SGLang, LLMC, Transformers)
- **schemes/** - Quantization scheme selection and configuration
- **utils/** - Calibration datasets, logging, CLI, model loading
diff --git a/test/fixtures.py b/test/fixtures.py
index bbd601a87..ffc628401 100644
--- a/test/fixtures.py
+++ b/test/fixtures.py
@@ -442,3 +442,84 @@ def tokenizer():
@pytest.fixture(scope="session")
def dataloader():
return DataLoader()
+
+
[email protected](scope="session")
+def tiny_stable_audio_pipe():
+ """Build a tiny StableAudioPipeline from scratch (random weights, no download).
+
+ StableAudioPipeline is a text-to-audio pipeline not supported by AutoPipelineForText2Image,
+ so we construct it manually from individual components rather than using save_tiny_model.
+ Saves to a temp directory and reloads via from_pretrained so that
+ ``name_or_path`` and ``model_index.json`` are set correctly.
+ """
+ from diffusers import AutoencoderOobleck, StableAudioDiTModel, StableAudioPipeline
+ from diffusers.pipelines.stable_audio.modeling_stable_audio import StableAudioProjectionModel
+ from diffusers.schedulers import EDMDPMSolverMultistepScheduler
+ from transformers import AutoTokenizer, T5Config, T5EncoderModel
+
+ tiny_model_path = "./tmp/tiny_stable_audio_pipe"
+
+ transformer = StableAudioDiTModel(
+ sample_size=64,
+ in_channels=8,
+ num_layers=1,
+ attention_head_dim=32,
+ num_attention_heads=2,
+ num_key_value_attention_heads=2,
+ out_channels=8,
+ cross_attention_dim=64,
+ time_proj_dim=32,
+ global_states_input_dim=64,
+ cross_attention_input_dim=64,
+ )
+ t5_config = T5Config(vocab_size=100, d_model=64, d_ff=128, num_heads=2, num_layers=1, d_kv=32)
+ text_encoder = T5EncoderModel(t5_config)
+ projection = StableAudioProjectionModel(text_encoder_dim=64, conditioning_dim=64, min_value=0.0, max_value=47.0)
+ vae = AutoencoderOobleck(
+ encoder_hidden_size=32,
+ downsampling_ratios=[2, 4],
+ channel_multiples=[1, 2],
+ decoder_channels=16,
+ decoder_input_channels=8,
+ audio_channels=1,
+ sampling_rate=16000,
+ )
+ scheduler = EDMDPMSolverMultistepScheduler()
+ tokenizer = AutoTokenizer.from_pretrained(get_model_path("google-t5/t5-small"))
+ pipe = StableAudioPipeline(
+ vae=vae,
+ text_encoder=text_encoder,
+ projection_model=projection,
+ tokenizer=tokenizer,
+ transformer=transformer,
+ scheduler=scheduler,
+ )
+ pipe.save_pretrained(tiny_model_path, is_diffusers=True)
+ yield tiny_model_path
+ shutil.rmtree(tiny_model_path, ignore_errors=True)
+
+
[email protected](scope="session")
+def tiny_mimo_audio_model_path():
+ """Build a tiny MiMo-Audio model by patching a Qwen backbone with MiMo-Audio config.
+
+ Follows the pattern of omni models (is_mllm=True, from_config=True) but uses Qwen
+ as the base since MiMo-Audio requires custom code not available in standard transformers.
+ Patches config.architectures to ["MiMoAudioModel"] so that resolve_model_type returns 'mimo_audio'.
+ """
+ model_name_or_path = qwen_name_or_path
+ tiny_model_path = "./tmp/tiny_mimo_audio_model_path"
+ tiny_model_path = save_tiny_model(
+ model_name_or_path,
+ tiny_model_path,
+ num_layers=2,
+ is_mllm=True,
+ from_config=True,
+ )
+ # Patch the config to simulate MiMo-Audio architecture
+ config = transformers.AutoConfig.from_pretrained(tiny_model_path)
+ config.architectures = ["MiMoAudioModel"]
+ config.save_pretrained(tiny_model_path)
+ yield tiny_model_path
+ shutil.rmtree(tiny_model_path, ignore_errors=True)
diff --git a/test/helpers.py b/test/helpers.py
index cfbc7ceee..e16b758ab 100644
--- a/test/helpers.py
+++ b/test/helpers.py
@@ -153,6 +153,8 @@ def _reduce_config_layers(config, num_layers, num_experts=None):
"encoder_layers",
"num_single_layers",
"num_decoder_layers",
+ "input_local_layers",
+ "local_layers",
]
n_expert_keys = ["num_experts", "num_local_experts"]
@@ -185,6 +187,7 @@ def _reduce_config_layers(config, num_layers, num_experts=None):
"audio_config",
"thinker_config",
"talker_config",
+ "code_predictor_config",
]:
sub_cfg = getattr(config, sub_name, None)
if sub_cfg is not None:
diff --git a/test/test_cpu/models/test_audio_model.py b/test/test_cpu/models/test_audio_model.py
new file mode 100644
index 000000000..aaa29bc26
--- /dev/null
+++ b/test/test_cpu/models/test_audio_model.py
@@ -0,0 +1,266 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""CPU unit tests for Audio model support.
+
+Tests cover:
+- MiMo-Audio: architecture-based model_type resolution, block detection,
+ forward patching, processor/template registration, quantization with
+ a tiny dummy Qwen model patched to MiMo-Audio config.
+- StableAudio: diffusion pipeline function attachment, output config
+ registration, shared cache keys, and quantization (RTN) with a tiny
+ dummy StableAudioPipeline built from random weights.
+"""
+
+import shutil
+from types import SimpleNamespace
+from unittest.mock import MagicMock
+
+import pytest
+import torch
+
+from auto_round import AutoRound
+from auto_round.compressors.mllm.processor import PROCESSORS
+from auto_round.special_model_handler import (
+ ARCHITECTURE_MODEL_TYPE_MAP,
+ SPECIAL_MULTIMODAL_BLOCK,
+ SPECIAL_SHARED_CACHE_KEYS,
+ SUPPORT_ONLY_TEXT_MODELS,
+ _get_mimo_audio_multimodal_block,
+ _handle_special_model,
+ check_mllm_model_batch,
+ mllms_with_limited_bs,
+ resolve_model_type,
+)
+
+# ====================== Mock Helpers =========================================
+
+
+def _make_mock_config(model_type, architectures=None):
+ cfg = SimpleNamespace(model_type=model_type)
+ if architectures is not None:
+ cfg.architectures = architectures
+ return cfg
+
+
+def _make_mimo_audio_mock(n_main_layers=2, n_input_local_layers=1, n_local_layers=2):
+ """Build a mock MiMo-Audio model with correct module hierarchy."""
+ model = MagicMock()
+ model.config = _make_mock_config("qwen2", architectures=["MiMoAudioModel"])
+
+ main_layers = torch.nn.ModuleList([torch.nn.Linear(8, 8) for _ in range(n_main_layers)])
+ inner_model = MagicMock()
+ inner_model.layers = main_layers
+ model.model = inner_model
+
+ input_local = MagicMock()
+ input_local.layers = torch.nn.ModuleList([torch.nn.Linear(8, 8) for _ in range(n_input_local_layers)])
+ model.input_local_transformer = input_local
+
+ local_xfm = MagicMock()
+ local_xfm.layers = torch.nn.ModuleList([torch.nn.Linear(8, 8) for _ in range(n_local_layers)])
+ model.local_transformer = local_xfm
+
+ return model
+
+
+# ====================== MiMo-Audio Unit Tests ================================
+
+
+class TestResolveModelType:
+ """Test architecture-based model_type resolution."""
+
+ def test_mimo_audio_architecture_override(self):
+ model = MagicMock()
+ model.config = _make_mock_config("qwen2", architectures=["MiMoAudioModel"])
+ assert resolve_model_type(model) == "mimo_audio"
+
+ def test_mimo_audio_causal_lm_variant(self):
+ model = MagicMock()
+ model.config = _make_mock_config("qwen2", architectures=["MiMoAudioForCausalLM"])
+ assert resolve_model_type(model) == "mimo_audio"
+
+ def test_standard_qwen2_not_overridden(self):
+ model = MagicMock()
+ model.config = _make_mock_config("qwen2", architectures=["Qwen2ForCausalLM"])
+ assert resolve_model_type(model) == "qwen2"
+
+
+class TestMiMoAudioBlockDetection:
+ """Test _get_mimo_audio_multimodal_block with mock models."""
+
+ def test_main_decoder_only(self):
+ model = _make_mimo_audio_mock(n_main_layers=4)
+ blocks = _get_mimo_audio_multimodal_block(model)
+ assert len(blocks) == 1
+ assert blocks[0] == [f"model.layers.{i}" for i in range(4)]
+
+
+class TestMiMoAudioRegistration:
+ """Verify MiMo-Audio is properly registered in all required registries."""
+
+ def test_special_map(self):
+ assert "MiMoAudioModel" in ARCHITECTURE_MODEL_TYPE_MAP
+ assert "MiMoAudioForCausalLM" in ARCHITECTURE_MODEL_TYPE_MAP
+ assert "mimo_audio" in SPECIAL_MULTIMODAL_BLOCK
+ assert "mimo_audio" in SUPPORT_ONLY_TEXT_MODELS
+
+ assert "mimo_audio" in PROCESSORS
+ from auto_round.compressors.mllm.template import TEMPLATES
+
+ assert "mimo_audio" in TEMPLATES
+
+
+class TestMiMoAudioForwardPatching:
+ """Test that _handle_special_model patches the forward function."""
+
+ def test_forward_is_patched(self, tiny_mimo_audio_model_path):
+ import transformers
+
+ model = transformers.AutoModelForCausalLM.from_pretrained(tiny_mimo_audio_model_path)
+ original_forward = model.forward
+ model = _handle_special_model(model)
+ assert model.forward != original_forward
+
+
+class TestMiMoAudioQuantization:
+ """End-to-end quantization with a tiny Qwen model patched to MiMo-Audio config."""
+
+ def test_quantize_rtn(self, tiny_mimo_audio_model_path, tmp_path):
+ import transformers
+
+ model = transformers.AutoModelForCausalLM.from_pretrained(
+ tiny_mimo_audio_model_path, torch_dtype="auto", trust_remote_code=True
+ )
+ tokenizer = transformers.AutoTokenizer.from_pretrained(tiny_mimo_audio_model_path)
+
+ assert resolve_model_type(model) == "mimo_audio"
+
+ autoround = AutoRound(
+ model,
+ tokenizer=tokenizer,
+ scheme="W4A16",
+ nsamples=1,
+ iters=0,
+ seqlen=32,
+ enable_torch_compile=True,
+ disable_opt_rtn=True,
+ )
+ quantized_model, save_folder = autoround.quantize_and_save(output_dir=str(tmp_path / "saved"))
+ assert quantized_model is not None
+ assert save_folder is not None
+ # Verify quantized model has QuantLinear layers
+ has_quantlinear = any(m.__class__.__name__ == "QuantLinear" for m in quantized_model.modules())
+ assert has_quantlinear, "Quantized model should contain QuantLinear layers"
+
+ def test_quantize_with_tuning(self, tiny_mimo_audio_model_path, tmp_path):
+ import transformers
+
+ model = transformers.AutoModelForCausalLM.from_pretrained(
+ tiny_mimo_audio_model_path, torch_dtype="auto", trust_remote_code=True
+ )
+ tokenizer = transformers.AutoTokenizer.from_pretrained(tiny_mimo_audio_model_path)
+
+ autoround = AutoRound(
+ model,
+ tokenizer=tokenizer,
+ scheme="W4A16",
+ nsamples=1,
+ iters=1,
+ seqlen=32,
+ enable_torch_compile=True,
+ disable_opt_rtn=True,
+ )
+ quantized_model, _ = autoround.quantize_and_save(format="auto_round", output_dir=str(tmp_path / "saved"))
+ assert quantized_model is not None
+ # Verify quantized model has QuantLinear layers
+ has_quantlinear = any(m.__class__.__name__ == "QuantLinear" for m in quantized_model.modules())
+ assert has_quantlinear, "Quantized model should contain QuantLinear layers after tuning"
+
+
+# ====================== StableAudio Unit Tests ===============================
+
+
+class TestStableAudioRegistration:
+ """Verify StableAudio-specific registrations."""
+
+ def test_config_and_special_registered(self):
+ from auto_round.algorithms.quantization.base import BaseQuantizers
+
+ assert "StableAudioDiTBlock" in BaseQuantizers.DIFFUSION_OUTPUT_CONFIGS
+ assert BaseQuantizers.DIFFUSION_OUTPUT_CONFIGS["StableAudioDiTBlock"] == ["hidden_states"]
+
+ assert "StableAudioDiTModel" in SPECIAL_SHARED_CACHE_KEYS
+ assert "encoder_hidden_states" in SPECIAL_SHARED_CACHE_KEYS["StableAudioDiTModel"]
+
+
+class TestStableAudioPipelineFunction:
+ """Test custom pipeline function attachment for StableAudio."""
+
+ def test_attach_pipeline_fn(self):
+ from auto_round.utils.model import _attach_diffusion_pipeline_fn
+
+ pipe = MagicMock()
+ type(pipe).__name__ = "StableAudioPipeline"
+ _attach_diffusion_pipeline_fn(pipe)
+ assert hasattr(pipe, "_autoround_pipeline_fn")
+ pipe._autoround_pipeline_fn(pipe, ["test prompt"], guidance_scale=3.5, num_inference_steps=10)
+ pipe.assert_called_once()
+ _, kwargs = pipe.call_args
+ assert "audio_end_in_s" in kwargs
+
+ def test_noop_for_non_stable_audio(self):
+ from auto_round.utils.model import _attach_diffusion_pipeline_fn
+
+ pipe = MagicMock(spec=["config", "__call__"])
+ type(pipe).__name__ = "FluxPipeline"
+ _attach_diffusion_pipeline_fn(pipe)
+ assert not hasattr(pipe, "_autoround_pipeline_fn")
+
+
+class TestStableAudioQuantization:
+ """End-to-end quantization with a tiny StableAudioPipeline (random weights)."""
+
+ def test_quantize_rtn(self, tiny_stable_audio_pipe, tmp_path):
+ """RTN quantization on tiny StableAudio pipeline."""
+ from diffusers import StableAudioPipeline
+
+ pipe = StableAudioPipeline.from_pretrained(tiny_stable_audio_pipe)
+ print(pipe)
+ output_dir = str(tmp_path / "stable_audio_rtn")
+
+ autoround = AutoRound(
+ pipe,
+ tokenizer=None,
+ scheme="W4A16",
+ nsamples=1,
+ iters=0,
+ disable_opt_rtn=True,
+ num_inference_steps=2,
+ )
+ autoround.quantize_and_save(output_dir)
+ # Verify quantized transformer has QuantLinear layers
+ has_quantlinear = any(m.__class__.__name__ == "QuantLinear" for m in pipe.transformer.modules())
+ assert has_quantlinear, "Quantized transformer should contain QuantLinear layers"
+ # Verify StableAudio-specific output: pipeline config and quantized transformer
+ import os
+
+ assert os.path.exists(os.path.join(output_dir, "model_index.json")), "model_index.json missing"
+ assert os.path.exists(
+ os.path.join(output_dir, "transformer", "quantization_config.json")
+ ), "quantization_config.json missing in transformer directory"
+ # Verify non-quantized pipeline components are also saved
+ assert os.path.exists(os.path.join(output_dir, "text_encoder")), "text_encoder directory missing"
+ assert os.path.exists(os.path.join(output_dir, "vae")), "vae directory missing"
+ assert os.path.exists(os.path.join(output_dir, "projection_model")), "projection_model directory missing"
diff --git a/test/test_cuda/models/test_audio_model.py b/test/test_cuda/models/test_audio_model.py
new file mode 100644
index 000000000..2d32053e3
--- /dev/null
+++ b/test/test_cuda/models/test_audio_model.py
@@ -0,0 +1,388 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""CUDA integration tests for audio model quantization.
+
+Tests cover:
+- MiMo-Audio: architecture-based model_type resolution, block detection,
+ and end-to-end quantization using a tiny Qwen model with MiMo-Audio config.
+- Qwen3-TTS: block detection, processor/template registration, and forward
+ routing via mock objects (model class is not in transformers).
+- Stable-Audio: diffusion pipeline quantization, output config registration,
+ shared cache keys, and custom pipeline function attachment.
+"""
+
+import os
+import shutil
+from types import SimpleNamespace
+from unittest.mock import MagicMock
+
+import pytest
+import torch
+
+from auto_round import AutoRound
+from auto_round.special_model_handler import (
+ ARCHITECTURE_MODEL_TYPE_MAP,
+ SPECIAL_MULTIMODAL_BLOCK,
+ SPECIAL_SHARED_CACHE_KEYS,
+ SUPPORT_ONLY_TEXT_MODELS,
+ _get_mimo_audio_multimodal_block,
+ _get_qwen3_tts_multimodal_block,
+ _handle_special_model,
+ _qwen3_tts_forward,
+ check_mllm_model_batch,
+ mllms_with_limited_bs,
+ resolve_model_type,
+)
+
+pytestmark = [
+ pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available"),
+]
+
+
+# ====================== Mock Helpers =========================================
+
+
+def _make_mock_config(model_type, architectures=None):
+ """Create a minimal config object for testing resolve_model_type."""
+ cfg = SimpleNamespace(model_type=model_type)
+ if architectures is not None:
+ cfg.architectures = architectures
+ return cfg
+
+
+def _make_mimo_audio_mock(n_main_layers=2, n_input_local_layers=1, n_local_layers=2):
+ """Build a mock MiMo-Audio model with the correct module hierarchy."""
+ model = MagicMock()
+ model.config = _make_mock_config("qwen2", architectures=["MiMoAudioModel"])
+
+ main_layers = torch.nn.ModuleList([torch.nn.Linear(8, 8) for _ in range(n_main_layers)])
+ inner_model = MagicMock()
+ inner_model.layers = main_layers
+ model.model = inner_model
+
+ input_local = MagicMock()
+ input_local.layers = torch.nn.ModuleList([torch.nn.Linear(8, 8) for _ in range(n_input_local_layers)])
+ model.input_local_transformer = input_local
+
+ local_xfm = MagicMock()
+ local_xfm.layers = torch.nn.ModuleList([torch.nn.Linear(8, 8) for _ in range(n_local_layers)])
+ model.local_transformer = local_xfm
+
+ return model
+
+
+def _make_qwen3_tts_mock(n_layers=2, use_tts_model=True):
+ """Build a mock Qwen3-TTS model."""
+ model = MagicMock()
+ model.config = _make_mock_config("qwen3_tts", architectures=["Qwen3TTSForConditionalGeneration"])
+
+ layers = torch.nn.ModuleList([torch.nn.Linear(8, 8) for _ in range(n_layers)])
+ inner = MagicMock()
+ inner.layers = layers
+
+ if use_tts_model:
+ tts = MagicMock()
+ tts.model = inner
+ model.tts_model = tts
+ del model.talker
+ else:
+ talker = MagicMock()
+ talker.model = inner
+ model.talker = talker
+ del model.tts_model
+
+ return model
+
+
+# ====================== MiMo-Audio Unit Tests ================================
+
+
+class TestResolveModelType:
+ """Test the architecture-based model_type resolution mechanism."""
+
+ def test_mimo_audio_architecture_override(self):
+ model = MagicMock()
+ model.config = _make_mock_config("qwen2", architectures=["MiMoAudioModel"])
+ assert resolve_model_type(model) == "mimo_audio"
+
+ def test_mimo_audio_causal_lm_variant(self):
+ model = MagicMock()
+ model.config = _make_mock_config("qwen2", architectures=["MiMoAudioForCausalLM"])
+ assert resolve_model_type(model) == "mimo_audio"
+
+ def test_qwen3_tts_uses_config_model_type(self):
+ model = MagicMock()
+ model.config = _make_mock_config("qwen3_tts", architectures=["Qwen3TTSForConditionalGeneration"])
+ assert resolve_model_type(model) == "qwen3_tts"
+
+ def test_standard_qwen2_not_overridden(self):
+ model = MagicMock()
+ model.config = _make_mock_config("qwen2", architectures=["Qwen2ForCausalLM"])
+ assert resolve_model_type(model) == "qwen2"
+
+ def test_no_architectures_falls_back(self):
+ model = MagicMock()
+ model.config = _make_mock_config("qwen2")
+ assert resolve_model_type(model) == "qwen2"
+
+ def test_no_config_returns_none(self):
+ model = MagicMock(spec=[])
+ assert resolve_model_type(model) is None
+
+
+class TestMiMoAudioBlockDetection:
+ """Test _get_mimo_audio_multimodal_block with mock models."""
+
+ def test_main_decoder_only(self):
+ model = _make_mimo_audio_mock(n_main_layers=4)
+ blocks = _get_mimo_audio_multimodal_block(model)
+ assert len(blocks) == 1
+ assert blocks[0] == [f"model.layers.{i}" for i in range(4)]
+
+ def test_no_audio_modules(self):
+ model = MagicMock()
+ model.config = _make_mock_config("qwen2", architectures=["MiMoAudioModel"])
+ inner = MagicMock()
+ inner.layers = torch.nn.ModuleList([torch.nn.Linear(8, 8) for _ in range(2)])
+ model.model = inner
+ del model.input_local_transformer
+ del model.local_transformer
+ blocks = _get_mimo_audio_multimodal_block(model)
+ assert len(blocks) == 1
+
+
+class TestMiMoAudioRegistration:
+ """Verify MiMo-Audio is properly registered in all required registries."""
+
+ def test_special_map(self):
+ assert "MiMoAudioModel" in ARCHITECTURE_MODEL_TYPE_MAP
+ assert "MiMoAudioForCausalLM" in ARCHITECTURE_MODEL_TYPE_MAP
+ assert ARCHITECTURE_MODEL_TYPE_MAP["MiMoAudioModel"] == "mimo_audio"
+ assert "mimo_audio" in SPECIAL_MULTIMODAL_BLOCK
+ assert "mimo_audio" in SUPPORT_ONLY_TEXT_MODELS
+ from auto_round.compressors.mllm.processor import PROCESSORS
+ from auto_round.compressors.mllm.template import TEMPLATES
+
+ assert "mimo_audio" in PROCESSORS
+ assert "mimo_audio" in TEMPLATES
+
+
+class TestMiMoAudioForwardPatching:
+ """Test that _handle_special_model patches the forward function."""
+
+ def test_forward_is_patched(self, tiny_mimo_audio_model_path):
+ import transformers
+
+ model = transformers.AutoModelForCausalLM.from_pretrained(tiny_mimo_audio_model_path)
+ original_forward = model.forward
+ model = _handle_special_model(model)
+ assert model.forward != original_forward
+
+
+# ====================== Qwen3-TTS Unit Tests =================================
+
+
+class TestQwen3TTSDetection:
+ """Test _get_qwen3_tts_multimodal_block with mock models."""
+
+ def test_special_map(self):
+ assert "qwen3_tts" in SPECIAL_MULTIMODAL_BLOCK
+ assert "qwen3_tts" in SUPPORT_ONLY_TEXT_MODELS
+ assert "qwen3_tts" in mllms_with_limited_bs
+ from auto_round.compressors.mllm.processor import PROCESSORS
+ from auto_round.compressors.mllm.template import TEMPLATES
+
+ assert "qwen3_tts" in PROCESSORS
+ assert "qwen3_tts" in TEMPLATES
+
+ def test_tts_model_path(self):
+ model = _make_qwen3_tts_mock(n_layers=3, use_tts_model=True)
+ blocks = _get_qwen3_tts_multimodal_block(model)
+ assert len(blocks) == 1
+ assert blocks[0] == [f"tts_model.model.layers.{i}" for i in range(3)]
+
+ def test_talker_path(self):
+ model = _make_qwen3_tts_mock(n_layers=4, use_tts_model=False)
+ blocks = _get_qwen3_tts_multimodal_block(model)
+ assert len(blocks) == 1
+ assert blocks[0] == [f"talker.model.layers.{i}" for i in range(4)]
+
+ def test_fallback_to_model_layers(self):
+ model = MagicMock()
+ model.config = _make_mock_config("qwen3_tts")
+ inner = MagicMock()
+ inner.layers = torch.nn.ModuleList([torch.nn.Linear(8, 8) for _ in range(2)])
+ model.model = inner
+ del model.tts_model
+ del model.talker
+ blocks = _get_qwen3_tts_multimodal_block(model)
+ assert len(blocks) == 1
+ assert blocks[0] == [f"model.layers.{i}" for i in range(2)]
+
+
+class TestQwen3TTSForward:
+ """Test the _qwen3_tts_forward routing logic."""
+
+ def test_routes_through_tts_model(self):
+ model = MagicMock()
+ tts = MagicMock()
+ model.tts_model = tts
+ del model.talker
+ dummy_ids = torch.randint(0, 100, (1, 4))
+ _qwen3_tts_forward(model, input_ids=dummy_ids)
+ tts.assert_called_once()
+
+ def test_routes_through_talker(self):
+ model = MagicMock()
+ talker = MagicMock()
+ model.talker = talker
+ model.tts_model = None
+ dummy_ids = torch.randint(0, 100, (1, 4))
+ _qwen3_tts_forward(model, input_ids=dummy_ids)
+ talker.assert_called_once()
+
+
+# ====================== StableAudio Unit Tests ===============================
+
+
+class TestStableAudioRegistration:
+ """Verify StableAudio-specific registrations."""
+
+ def test_config_and_special_registered(self):
+ from auto_round.algorithms.quantization.base import BaseQuantizers
+
+ assert "StableAudioDiTBlock" in BaseQuantizers.DIFFUSION_OUTPUT_CONFIGS
+ assert BaseQuantizers.DIFFUSION_OUTPUT_CONFIGS["StableAudioDiTBlock"] == ["hidden_states"]
+
+ assert "StableAudioDiTModel" in SPECIAL_SHARED_CACHE_KEYS
+ assert "encoder_hidden_states" in SPECIAL_SHARED_CACHE_KEYS["StableAudioDiTModel"]
+
+
+class TestStableAudioPipelineFunction:
+ """Test custom pipeline function attachment for StableAudio."""
+
+ def test_attach_pipeline_fn(self):
+ from auto_round.utils.model import _attach_diffusion_pipeline_fn
+
+ pipe = MagicMock()
+ type(pipe).__name__ = "StableAudioPipeline"
+ _attach_diffusion_pipeline_fn(pipe)
+ assert hasattr(pipe, "_autoround_pipeline_fn")
+ pipe._autoround_pipeline_fn(pipe, ["test prompt"], guidance_scale=3.5, num_inference_steps=10)
+ pipe.assert_called_once()
+ _, kwargs = pipe.call_args
+ assert "audio_end_in_s" in kwargs
+
+ def test_noop_for_non_stable_audio(self):
+ from auto_round.utils.model import _attach_diffusion_pipeline_fn
+
+ pipe = MagicMock(spec=["config", "__call__"])
+ type(pipe).__name__ = "FluxPipeline"
+ _attach_diffusion_pipeline_fn(pipe)
+ assert not hasattr(pipe, "_autoround_pipeline_fn")
+
+
+# ====================== Integration Tests ====================================
+
+
+class TestMiMoAudioQuantization:
+ """End-to-end quantization with a tiny Qwen model patched to MiMo-Audio config."""
+
+ def test_quantize_rtn(self, tiny_mimo_audio_model_path, tmp_path):
+ import transformers
+
+ model = transformers.AutoModelForCausalLM.from_pretrained(
+ tiny_mimo_audio_model_path, torch_dtype="auto", trust_remote_code=True
+ )
+ print(model)
+ tokenizer = transformers.AutoTokenizer.from_pretrained(tiny_mimo_audio_model_path)
+
+ assert resolve_model_type(model) == "mimo_audio"
+
+ autoround = AutoRound(
+ model,
+ tokenizer=tokenizer,
+ scheme="W4A16",
+ nsamples=1,
+ iters=0,
+ seqlen=32,
+ disable_opt_rtn=True,
+ enable_torch_compile=True,
+ )
+ quantized_model, save_folder = autoround.quantize_and_save(output_dir=str(tmp_path / "saved"))
+ assert quantized_model is not None
+ assert save_folder is not None
+ # Verify quantized model has QuantLinear layers
+ has_quantlinear = any(m.__class__.__name__ == "QuantLinear" for m in quantized_model.modules())
+ assert has_quantlinear, "Quantized model should contain QuantLinear layers"
+
+ def test_quantize_with_tuning(self, tiny_mimo_audio_model_path, tmp_path):
+ import transformers
+
+ model = transformers.AutoModelForCausalLM.from_pretrained(
+ tiny_mimo_audio_model_path, torch_dtype="auto", trust_remote_code=True
+ )
+ tokenizer = transformers.AutoTokenizer.from_pretrained(tiny_mimo_audio_model_path)
+
+ autoround = AutoRound(
+ model,
+ tokenizer=tokenizer,
+ scheme="W4A16",
+ nsamples=1,
+ iters=1,
+ seqlen=32,
+ disable_opt_rtn=True,
+ enable_torch_compile=True,
+ )
+ quantized_model, _ = autoround.quantize_and_save(output_dir=str(tmp_path / "saved"))
+ assert quantized_model is not None
+ # Verify quantized model has QuantLinear layers
+ has_quantlinear = any(m.__class__.__name__ == "QuantLinear" for m in quantized_model.modules())
+ assert has_quantlinear, "Quantized model should contain QuantLinear layers after tuning"
+
+
+class TestStableAudioQuantization:
+ """End-to-end quantization with a tiny StableAudioPipeline (random weights)."""
+
+ def test_quantize_rtn(self, tiny_stable_audio_pipe, tmp_path):
+ """RTN quantization on tiny StableAudio pipeline."""
+ from diffusers import StableAudioPipeline
+
+ pipe = StableAudioPipeline.from_pretrained(tiny_stable_audio_pipe)
+ print(pipe)
+ output_dir = str(tmp_path / "stable_audio_rtn")
+
+ autoround = AutoRound(
+ pipe,
+ tokenizer=None,
+ scheme="W4A16",
+ nsamples=1,
+ iters=0,
+ disable_opt_rtn=True,
+ num_inference_steps=2,
+ enable_torch_compile=True,
+ )
+ autoround.quantize_and_save(output_dir)
+ has_quantlinear = any(m.__class__.__name__ == "QuantLinear" for m in pipe.transformer.modules())
+ assert has_quantlinear, "Quantized transformer should contain QuantLinear layers"
+ # Verify StableAudio-specific output: pipeline config and quantized transformer
+ assert os.path.exists(os.path.join(output_dir, "model_index.json")), "model_index.json missing"
+ assert os.path.exists(
+ os.path.join(output_dir, "transformer", "quantization_config.json")
+ ), "quantization_config.json missing in transformer directory"
+ # Verify non-quantized pipeline components are also saved
+ assert os.path.exists(os.path.join(output_dir, "text_encoder")), "text_encoder directory missing"
+ assert os.path.exists(os.path.join(output_dir, "vae")), "vae directory missing"
+ assert os.path.exists(os.path.join(output_dir, "projection_model")), "projection_model directory missing"