PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
import json
import os
import shutil
from unittest.mock import patch
import datasets
import pytest
import torch
import transformers
from auto_round.utils import is_transformers_version_greater_or_equal_5_4_0
from .helpers import (
DataLoader,
deepseek_v2_name_or_path,
flux_name_or_path,
gemma_name_or_path,
get_model_path,
get_tiny_model,
gptj_name_or_path,
lamini_name_or_path,
make_tiny_qwen3_omni_moe_config,
opt_name_or_path,
phi2_name_or_path,
qwen2_5_omni_name_or_path,
qwen_2_5_vl_name_or_path,
qwen_moe_name_or_path,
qwen_name_or_path,
qwen_vl_name_or_path,
save_tiny_model,
)
_save_tiny_model = save_tiny_model
TINY_MODEL_ROOT = os.path.join(os.path.dirname(__file__), "tmp", "tiny_models")
def tiny_model_dir(name):
return os.path.join(TINY_MODEL_ROOT, os.path.basename(os.path.normpath(name)))
def _tiny_model_config_fingerprint(path):
config_path = os.path.join(path, "config.json")
if not os.path.isfile(config_path):
return None
with open(config_path, encoding="utf-8") as config_file:
return json.load(config_file)
def _tiny_model_ready(path, signature=None):
marker_path = os.path.join(path, ".autoround_ready")
if not os.path.isfile(marker_path):
return False
if signature is None:
return True
with open(marker_path, encoding="utf-8") as marker:
try:
metadata = json.load(marker)
except json.JSONDecodeError:
return False
return metadata.get("signature") == signature and metadata.get("config") == _tiny_model_config_fingerprint(path)
def _mark_tiny_model(path, signature=None):
os.makedirs(path, exist_ok=True)
with open(os.path.join(path, ".autoround_ready"), "w", encoding="utf-8") as marker:
if signature is None:
marker.write("ready\n")
else:
json.dump({"signature": signature, "config": _tiny_model_config_fingerprint(path)}, marker, sort_keys=True)
def _tiny_model_signature(args, kwargs):
return json.dumps({"args": args, "kwargs": kwargs}, default=str, sort_keys=True)
def save_tiny_model(*args, **kwargs):
requested_path = args[1] if len(args) > 1 else kwargs["tiny_model_path"]
tiny_model_path = tiny_model_dir(requested_path)
signature = _tiny_model_signature(args, kwargs)
if _tiny_model_ready(tiny_model_path, signature):
return tiny_model_path
args = list(args)
if len(args) > 1:
args[1] = tiny_model_path
else:
kwargs = dict(kwargs)
kwargs["tiny_model_path"] = tiny_model_path
result = _save_tiny_model(*args, **kwargs)
_mark_tiny_model(result, signature)
return result
# Create tiny model path fixtures for testing
@pytest.fixture(scope="session")
def tiny_opt_model_path():
model_name_or_path = opt_name_or_path
tiny_model_path = tiny_model_dir("tiny_opt_model_path")
tiny_model_path = save_tiny_model(model_name_or_path, tiny_model_path)
yield tiny_model_path
@pytest.fixture(scope="session")
def tiny_lamini_model_path():
model_name_or_path = lamini_name_or_path
tiny_model_path = tiny_model_dir("tiny_lamini_model_path")
tiny_model_path = save_tiny_model(model_name_or_path, tiny_model_path)
yield tiny_model_path
@pytest.fixture(scope="session")
def micro_conv1d_model_path():
"""One-layer Conv1D model for backend-agnostic save/reload smoke tests."""
tiny_model_path = tiny_model_dir("micro_conv1d_model_path")
tiny_model_path = save_tiny_model(lamini_name_or_path, tiny_model_path, num_layers=1)
yield tiny_model_path
@pytest.fixture(scope="session")
def tiny_gptj_model_path():
model_name_or_path = gptj_name_or_path
tiny_model_path = tiny_model_dir("tiny_gptj_model_path")
tiny_model_path = save_tiny_model(model_name_or_path, tiny_model_path)
yield tiny_model_path
@pytest.fixture(scope="session")
def tiny_phi2_model_path():
model_name_or_path = phi2_name_or_path
tiny_model_path = tiny_model_dir("tiny_phi2_model_path")
tiny_model_path = save_tiny_model(model_name_or_path, tiny_model_path)
yield tiny_model_path
@pytest.fixture(scope="session")
def tiny_deepseek_v2_model_path():
model_name_or_path = deepseek_v2_name_or_path
tiny_model_path = tiny_model_dir("tiny_deepseek_v2_model_path")
tiny_model_path = save_tiny_model(
model_name_or_path, tiny_model_path, num_layers=2, trust_remote_code=False, use_config=True
)
yield tiny_model_path
@pytest.fixture(scope="session")
def tiny_deepseek_v2_model_path_cpu():
"""Reduced fixture for CPU-only tests (2 MoE layers, 8 experts)."""
model_name_or_path = deepseek_v2_name_or_path
tiny_model_path = tiny_model_dir("tiny_deepseek_v2_model_path_cpu")
tiny_model_path = save_tiny_model(
model_name_or_path,
tiny_model_path,
num_layers=2,
num_experts=8,
trust_remote_code=False,
use_config=True,
config_overrides={"first_k_dense_replace": 0},
)
yield tiny_model_path
@pytest.fixture(scope="session")
def tiny_gemma_model_path():
model_name_or_path = gemma_name_or_path
tiny_model_path = tiny_model_dir("tiny_gemma_model_path")
tiny_model_path = save_tiny_model(model_name_or_path, tiny_model_path, num_layers=2)
yield tiny_model_path
@pytest.fixture(scope="session")
def tiny_qwen_model_path():
model_name_or_path = qwen_name_or_path
tiny_model_path = tiny_model_dir("tiny_qwen_model_path")
tiny_model_path = save_tiny_model(model_name_or_path, tiny_model_path)
yield tiny_model_path
@pytest.fixture(scope="session")
def tiny_fp8_qwen_model_path():
from unittest.mock import patch
with patch("torch.cuda.get_device_capability", return_value=(9, 0)):
model_name_or_path = get_model_path("Qwen/Qwen3-0.6B-FP8")
tiny_model_path = tiny_model_dir("tiny_fp8_qwen_model_path")
tiny_model_path = save_tiny_model(model_name_or_path, tiny_model_path)
yield tiny_model_path
@pytest.fixture(scope="session")
def tiny_flux_model_path():
model_name_or_path = flux_name_or_path
tiny_model_path = tiny_model_dir("tiny_flux_model_path")
tiny_model_path = save_tiny_model(
model_name_or_path,
tiny_model_path,
num_layers=1,
is_diffusion=True,
from_config=True,
config_overrides={
"num_attention_heads": 2,
"attention_head_dim": 128,
"joint_attention_dim": 256,
"pooled_projection_dim": 256,
"hidden_size": 256,
"max_position_embeddings": 128,
"intermediate_size": 256,
},
)
yield tiny_model_path
@pytest.fixture(scope="session")
def tiny_z_image_model_path():
model_name_or_path = "Tongyi-MAI/Z-Image"
tiny_model_path = tiny_model_dir("tiny_z_image_model_path")
tiny_model_path = save_tiny_model(
model_name_or_path,
tiny_model_path,
num_layers=1,
is_diffusion=True,
from_config=True,
config_overrides={
"dim": 256,
"n_heads": 2,
"n_kv_heads": 2,
"n_layers": 1,
"n_refiner_layers": 1,
"cap_feat_dim": 512,
"in_channels": 16,
"num_attention_heads": 2,
"num_key_value_heads": 2,
"attention_head_dim": 128,
"joint_attention_dim": 256,
"pooled_projection_dim": 256,
"hidden_size": 512,
"intermediate_size": 256,
},
)
yield tiny_model_path
@pytest.fixture(scope="session")
def tiny_untied_qwen_model_path():
model_name_or_path = qwen_name_or_path
tiny_model_path = tiny_model_dir("tiny_untied_qwen_model_path")
tiny_model_path = save_tiny_model(model_name_or_path, tiny_model_path, force_untie=True)
yield tiny_model_path
@pytest.fixture(scope="session")
def tiny_qwen_moe_model_path():
model_name_or_path = qwen_moe_name_or_path
tiny_model_path = tiny_model_dir("tiny_qwen_moe_model_path")
tiny_model_path = save_tiny_model(model_name_or_path, tiny_model_path, num_layers=2)
yield tiny_model_path
@pytest.fixture(scope="session")
def tiny_qwen_vl_model_path():
model_name_or_path = qwen_vl_name_or_path
tiny_model_path = tiny_model_dir("tiny_qwen_vl_model_path")
tiny_model_path = save_tiny_model(model_name_or_path, tiny_model_path, num_layers=3, is_mllm=True)
yield tiny_model_path
@pytest.fixture(scope="session")
def tiny_qwen_2_5_vl_model_path():
model_name_or_path = qwen_2_5_vl_name_or_path
tiny_model_path = tiny_model_dir("tiny_qwen_2_5_vl_model_path")
tiny_model_path = save_tiny_model(model_name_or_path, tiny_model_path, num_layers=2, is_mllm=True)
yield tiny_model_path
@pytest.fixture(scope="session")
def tiny_fp8_qwen_moe_model_path():
with patch("torch.cuda.get_device_capability", return_value=(9, 0)):
tiny_model_path = tiny_model_dir("tiny_fp8_qwen_moe_model_path")
if _tiny_model_ready(tiny_model_path):
yield tiny_model_path
return
model_name = get_model_path("Qwen/Qwen3-30B-A3B-FP8")
config = transformers.AutoConfig.from_pretrained(model_name, trust_remote_code=True)
config.num_experts, config.num_hidden_layers, config.vocab_size = 4, 2, 2048
model = transformers.AutoModelForCausalLM.from_config(config, trust_remote_code=True)
tokenizer = transformers.AutoTokenizer.from_pretrained(model_name)
from transformers.integrations.finegrained_fp8 import FP8Linear
if is_transformers_version_greater_or_equal_5_4_0():
from transformers.integrations.finegrained_fp8 import FP8Experts as FP8Expert
else:
from transformers.integrations.finegrained_fp8 import FP8Expert
for name, module in model.named_modules():
if name == "lm_head":
continue
if "mlp.gate" in name:
continue
if isinstance(module, torch.nn.Linear):
fp8_linear = FP8Linear(
module.in_features,
module.out_features,
block_size=[128, 128],
)
model.set_submodule(name, fp8_linear)
if name.endswith("mlp.experts"):
fp8_expert = FP8Expert(
config=model.config.get_text_config(),
block_size=[128, 128],
)
model.set_submodule(name, fp8_expert)
model.save_pretrained(tiny_model_path)
print(model)
tokenizer.save_pretrained(tiny_model_path)
_mark_tiny_model(tiny_model_path)
yield tiny_model_path
@pytest.fixture(scope="session")
def tiny_gpt_oss_model_path():
tiny_model_path = tiny_model_dir("tiny_gpt_oss")
if _tiny_model_ready(tiny_model_path):
yield tiny_model_path
return
from transformers import GptOssForCausalLM
model_name = get_model_path("unsloth/gpt-oss-20b")
config = transformers.AutoConfig.from_pretrained(model_name, trust_remote_code=True)
config.num_hidden_layers = 1 # Reduce layers for testing
config.layer_types = config.layer_types[:1] # Keep only the first layer type for testing
delattr(config, "quantization_config")
model = GptOssForCausalLM(config)
model.save_pretrained(tiny_model_path)
tokenizer = transformers.AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
tokenizer.save_pretrained(tiny_model_path)
_mark_tiny_model(tiny_model_path)
yield tiny_model_path
@pytest.fixture(scope="session")
def tiny_llama4_model_path():
tiny_model_path = tiny_model_dir("tiny_llama4")
if _tiny_model_ready(tiny_model_path):
yield tiny_model_path
return
from transformers import Llama4ForConditionalGeneration
model_name = get_model_path("meta-llama/Llama-4-Scout-17B-16E-Instruct")
config = transformers.AutoConfig.from_pretrained(model_name, trust_remote_code=True)
# TODO: Remove after https://github.com/huggingface/transformers/issues/43525 is resolved
config.pad_token_id = None
config.vision_config.num_hidden_layers = 1 # Reduce layers for testing
config.text_config.num_hidden_layers = 1
config.text_config.num_hidden_layers = 1
model = Llama4ForConditionalGeneration(config)
# Remove these parameters to avoid mismatch during quantized model loading
model.config.text_config.no_rope_layers = []
if hasattr(model.config.text_config, "moe_layers"):
delattr(model.config.text_config, "moe_layers")
if hasattr(model.config.text_config, "layer_types"):
delattr(model.config.text_config, "layer_types")
model.save_pretrained(tiny_model_path)
tokenizer = transformers.AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
tokenizer.save_pretrained(tiny_model_path)
processor = transformers.AutoProcessor.from_pretrained(model_name, trust_remote_code=True)
processor.save_pretrained(tiny_model_path)
_mark_tiny_model(tiny_model_path)
yield tiny_model_path
@pytest.fixture(scope="session")
def tiny_qwen3_vl_moe_model_path():
tiny_model_path = tiny_model_dir("tiny_qwen3_vl_moe")
if _tiny_model_ready(tiny_model_path):
yield tiny_model_path
return
from transformers.models.qwen3_vl_moe.modeling_qwen3_vl_moe import Qwen3VLMoeForConditionalGeneration
model_name = get_model_path("Qwen/Qwen3-VL-30B-A3B-Instruct")
config = transformers.AutoConfig.from_pretrained(model_name)
config.vision_config.depth = 1 # Reduce layers for testing
config.text_config.num_hidden_layers = 1
config.text_config.num_experts = 16
config.num_hidden_layers = 1
model = Qwen3VLMoeForConditionalGeneration(config)
model.save_pretrained(tiny_model_path)
tokenizer = transformers.AutoTokenizer.from_pretrained(model_name)
tokenizer.save_pretrained(tiny_model_path)
processor = transformers.AutoProcessor.from_pretrained(model_name)
processor.save_pretrained(tiny_model_path)
_mark_tiny_model(tiny_model_path)
yield tiny_model_path
@pytest.fixture(scope="session")
def tiny_qwen35_moe_model_path():
tiny_model_path = tiny_model_dir("tiny_qwen35_moe")
if _tiny_model_ready(tiny_model_path):
yield tiny_model_path
return
from transformers import Qwen3_5MoeForConditionalGeneration
model_name = get_model_path("Qwen/Qwen3.5-35B-A3B")
config = transformers.AutoConfig.from_pretrained(model_name)
config.text_config.pad_token_id = None
config.vision_config.depth = 1 # Reduce layers for testing
config.text_config.num_hidden_layers = 4
config.num_hidden_layers = 1
config.text_config.layer_types = config.text_config.layer_types[: config.text_config.num_hidden_layers]
config.text_config.use_cache = False
# This tiny model doesn't materialize the MTP block, so keep block_count aligned
# with the actual number of exported layers to avoid a gguf tensor mismatch
# (e.g. missing "blk.N.attn_norm.weight") when loading with llama.cpp.
config.text_config.mtp_num_hidden_layers = 0
model = Qwen3_5MoeForConditionalGeneration(config)
model.save_pretrained(tiny_model_path)
tokenizer = transformers.AutoTokenizer.from_pretrained(model_name)
tokenizer.save_pretrained(tiny_model_path)
processor = transformers.AutoProcessor.from_pretrained(model_name)
processor.save_pretrained(tiny_model_path)
_mark_tiny_model(tiny_model_path)
yield tiny_model_path
@pytest.fixture(scope="session")
def tiny_qwen35_moe_text_model_path(tiny_opt_model_path):
"""Small text-only Qwen3.5 MoE fixture for the PR CUDA smoke test."""
tiny_model_path = tiny_model_dir("tiny_qwen35_moe_text")
if _tiny_model_ready(tiny_model_path):
yield tiny_model_path
return
from transformers.models.qwen3_5_moe.configuration_qwen3_5_moe import Qwen3_5MoeTextConfig
from transformers.models.qwen3_5_moe.modeling_qwen3_5_moe import Qwen3_5MoeForCausalLM
tokenizer = transformers.AutoTokenizer.from_pretrained(tiny_opt_model_path)
config = Qwen3_5MoeTextConfig(
architectures=["Qwen3_5MoeForCausalLM"],
vocab_size=len(tokenizer),
hidden_size=128,
num_hidden_layers=2,
num_attention_heads=4,
num_key_value_heads=2,
head_dim=32,
linear_key_head_dim=32,
linear_value_head_dim=32,
linear_num_key_heads=4,
linear_num_value_heads=4,
moe_intermediate_size=64,
shared_expert_intermediate_size=64,
num_experts=2,
num_experts_per_tok=2,
layer_types=["linear_attention", "full_attention"],
max_position_embeddings=64,
use_cache=False,
pad_token_id=tokenizer.pad_token_id,
bos_token_id=tokenizer.bos_token_id,
eos_token_id=tokenizer.eos_token_id,
)
model = Qwen3_5MoeForCausalLM(config)
model.save_pretrained(tiny_model_path)
tokenizer.save_pretrained(tiny_model_path)
_mark_tiny_model(tiny_model_path)
yield tiny_model_path
@pytest.fixture(scope="session")
def tiny_tiny_llama_model_path():
tiny_model_path = tiny_model_dir("tiny_TinyLlama")
if _tiny_model_ready(tiny_model_path):
yield tiny_model_path
return
model_name = get_model_path("TinyLlama/TinyLlama-1.1B-Chat-v1.0")
config = transformers.AutoConfig.from_pretrained(model_name)
config.num_hidden_layers = 4
model = transformers.AutoModelForCausalLM.from_config(config)
model.save_pretrained(tiny_model_path)
tokenizer = transformers.AutoTokenizer.from_pretrained(model_name)
tokenizer.save_pretrained(tiny_model_path)
_mark_tiny_model(tiny_model_path)
yield tiny_model_path
@pytest.fixture(scope="session")
def tiny_qwen2_5_omni_model_path():
"""Tiny Qwen2.5-Omni-3B model built from real config with reduced layers.
Uses random weights (no checkpoint loading) so it is fast for CPU unit
tests while still exercising the real config structure.
Skipped automatically when the model path does not exist locally.
"""
model_name_or_path = get_model_path(qwen2_5_omni_name_or_path)
if not os.path.isdir(model_name_or_path):
pytest.skip("Qwen2.5-Omni fixture is not available locally")
tiny_model_path = tiny_model_dir("tiny_qwen2_5_omni_model_path")
if _tiny_model_ready(tiny_model_path):
yield tiny_model_path
return
tiny_model_path = save_tiny_model(model_name_or_path, tiny_model_path, num_layers=1, is_mllm=True, from_config=True)
tokenizer = transformers.AutoTokenizer.from_pretrained(model_name_or_path, trust_remote_code=True)
processor = transformers.AutoProcessor.from_pretrained(model_name_or_path, trust_remote_code=True)
tokenizer.save_pretrained(tiny_model_path)
processor.save_pretrained(tiny_model_path)
# Copy model-specific files required for from_pretrained (e.g. spk_dict.pt for token2wav)
local_spk_dict = os.path.join(model_name_or_path, "spk_dict.pt")
if os.path.exists(local_spk_dict):
shutil.copy(local_spk_dict, tiny_model_path)
else:
pytest.skip("Qwen2.5-Omni spk_dict.pt is not available locally")
yield tiny_model_path
@pytest.fixture(scope="session")
def tiny_qwen3_omni_moe_model_path():
"""Self-contained tiny Qwen3-Omni-MoE model for CUDA smoke coverage."""
tiny_model_path = tiny_model_dir("tiny_qwen3_omni_moe_smoke_v2_model_path")
if not _tiny_model_ready(tiny_model_path):
model = transformers.Qwen3OmniMoeForConditionalGeneration(make_tiny_qwen3_omni_moe_config())
model.save_pretrained(tiny_model_path)
_mark_tiny_model(tiny_model_path)
yield tiny_model_path
# Mock FP8 capability checks without letting the fake capability affect Inductor code generation.
@pytest.fixture()
def mock_fp8_capable_device():
from unittest.mock import patch
with patch("torch.cuda.get_device_capability", return_value=(9, 0)), patch(
"torch.compile", side_effect=lambda function, *args, **kwargs: function
):
yield
@pytest.fixture(autouse=True, scope="session")
def clean_tmp_model_folder():
yield
tmp_root = os.path.join(os.path.dirname(__file__), "tmp")
tiny_model_cache = os.path.abspath(TINY_MODEL_ROOT)
for entry in os.scandir(tmp_root) if os.path.isdir(tmp_root) else []:
if os.path.abspath(entry.path) == tiny_model_cache:
continue
if entry.is_dir():
shutil.rmtree(entry.path, ignore_errors=True)
else:
try:
os.unlink(entry.path)
except FileNotFoundError:
pass
shutil.rmtree("./ar_work_space", ignore_errors=True)
shutil.rmtree("./tmp_autoround", ignore_errors=True)
shutil.rmtree(os.path.expanduser("~/.cache/auto_round"), ignore_errors=True)
# Create objective fixtures for testing
@pytest.fixture(scope="function")
def tiny_opt_model():
model_name_or_path = opt_name_or_path
return get_tiny_model(model_name_or_path, num_layers=2)
@pytest.fixture(scope="function")
def opt_model():
model_name_or_path = opt_name_or_path
model = transformers.AutoModelForCausalLM.from_pretrained(model_name_or_path, dtype="auto", trust_remote_code=True)
return model
@pytest.fixture(scope="session")
def opt_tokenizer():
model_name_or_path = opt_name_or_path
tokenizer = transformers.AutoTokenizer.from_pretrained(model_name_or_path, trust_remote_code=True)
return tokenizer
@pytest.fixture(scope="function")
def model():
model_name_or_path = opt_name_or_path
model = transformers.AutoModelForCausalLM.from_pretrained(model_name_or_path, dtype="auto", trust_remote_code=True)
return model
@pytest.fixture(scope="session")
def tokenizer():
model_name_or_path = opt_name_or_path
tokenizer = transformers.AutoTokenizer.from_pretrained(model_name_or_path, trust_remote_code=True)
return tokenizer
@pytest.fixture(scope="session")
def dataloader():
return DataLoader()
@pytest.fixture(scope="session")
def tiny_stable_audio_pipe():
"""Build a tiny StableAudioPipeline from scratch (random weights, no download).
StableAudioPipeline is a text-to-audio pipeline not supported by AutoPipelineForText2Image,
so we construct it manually from individual components rather than using save_tiny_model.
Saves to a temp directory and reloads via from_pretrained so that
``name_or_path`` and ``model_index.json`` are set correctly.
"""
from diffusers import AutoencoderOobleck, StableAudioDiTModel, StableAudioPipeline
from diffusers.pipelines.stable_audio.modeling_stable_audio import StableAudioProjectionModel
from diffusers.schedulers import EDMDPMSolverMultistepScheduler
from transformers import AutoTokenizer, T5Config, T5EncoderModel
tiny_model_path = tiny_model_dir("tiny_stable_audio_pipe")
if _tiny_model_ready(tiny_model_path):
yield tiny_model_path
return
transformer = StableAudioDiTModel(
sample_size=64,
in_channels=8,
num_layers=1,
attention_head_dim=32,
num_attention_heads=2,
num_key_value_attention_heads=2,
out_channels=8,
cross_attention_dim=64,
time_proj_dim=32,
global_states_input_dim=64,
cross_attention_input_dim=64,
)
t5_config = T5Config(vocab_size=100, d_model=64, d_ff=128, num_heads=2, num_layers=1, d_kv=32)
text_encoder = T5EncoderModel(t5_config)
projection = StableAudioProjectionModel(text_encoder_dim=64, conditioning_dim=64, min_value=0.0, max_value=47.0)
vae = AutoencoderOobleck(
encoder_hidden_size=32,
downsampling_ratios=[2, 4],
channel_multiples=[1, 2],
decoder_channels=16,
decoder_input_channels=8,
audio_channels=1,
sampling_rate=16000,
)
scheduler = EDMDPMSolverMultistepScheduler()
tokenizer = AutoTokenizer.from_pretrained(get_model_path("google-t5/t5-small"))
pipe = StableAudioPipeline(
vae=vae,
text_encoder=text_encoder,
projection_model=projection,
tokenizer=tokenizer,
transformer=transformer,
scheduler=scheduler,
)
pipe.save_pretrained(tiny_model_path, is_diffusers=True)
_mark_tiny_model(tiny_model_path)
yield tiny_model_path
@pytest.fixture(scope="session")
def tiny_mimo_audio_model_path():
"""Build a tiny MiMo-Audio model by patching a Qwen backbone with MiMo-Audio config.
Follows the pattern of omni models (is_mllm=True, from_config=True) but uses Qwen
as the base since MiMo-Audio requires custom code not available in standard transformers.
Patches config.architectures to ["MiMoAudioModel"] so that resolve_model_type returns 'mimo_audio'.
"""
model_name_or_path = qwen_name_or_path
tiny_model_path = tiny_model_dir("tiny_mimo_audio_model_path")
tiny_model_path = save_tiny_model(
model_name_or_path,
tiny_model_path,
num_layers=2,
is_mllm=True,
from_config=True,
)
# Patch the config to simulate MiMo-Audio architecture
config = transformers.AutoConfig.from_pretrained(tiny_model_path)
config.architectures = ["MiMoAudioModel"]
config.save_pretrained(tiny_model_path)
yield tiny_model_path
def _micro_model_path(model_name_or_path, name, config_overrides, num_experts=None):
"""Build a reusable architecture-level model with random micro-sized weights."""
model_path = tiny_model_dir(name)
if _tiny_model_ready(model_path):
config = transformers.AutoConfig.from_pretrained(model_path)
if any(getattr(config, key, None) != value for key, value in config_overrides.items()):
os.remove(os.path.join(model_path, ".autoround_ready"))
model_path = save_tiny_model(
model_name_or_path,
model_path,
num_layers=2,
from_config=True,
num_experts=num_experts,
config_overrides=config_overrides,
)
_save_micro_tokenizer(model_path)
return model_path
def _save_micro_tokenizer(model_path):
from tokenizers import Tokenizer
from tokenizers.models import WordLevel
from tokenizers.pre_tokenizers import Whitespace
tokenizer = Tokenizer(
WordLevel(
{
"<pad>": 0,
"<bos>": 1,
"<eos>": 2,
"<unk>": 3,
"auto": 4,
"round": 5,
"calibration": 6,
"sample": 7,
"keeps": 8,
"each": 9,
"token": 10,
"distinct": 11,
"for": 12,
"scoring": 13,
"another": 14,
"local": 15,
"dataset": 16,
"verifies": 17,
"scheme": 18,
"behavior": 19,
"sequence": 20,
},
unk_token="<unk>",
)
)
tokenizer.pre_tokenizer = Whitespace()
transformers.PreTrainedTokenizerFast(
tokenizer_object=tokenizer,
bos_token="<bos>",
eos_token="<eos>",
pad_token="<pad>",
unk_token="<unk>",
).save_pretrained(model_path)
@pytest.fixture(scope="session")
def micro_opt_model_path():
yield _micro_model_path(
opt_name_or_path,
"micro_opt_model_path",
{
"vocab_size": 256,
"hidden_size": 64,
"bos_token_id": 1,
"eos_token_id": 2,
"pad_token_id": 0,
"word_embed_proj_dim": 64,
"ffn_dim": 128,
"num_attention_heads": 4,
"max_position_embeddings": 128,
},
)
@pytest.fixture(scope="session")
def micro_qwen_model_path():
yield _micro_model_path(
qwen_name_or_path,
"micro_qwen_model_path",
{
"vocab_size": 32768,
"hidden_size": 256,
"intermediate_size": 512,
"num_attention_heads": 4,
"num_key_value_heads": 2,
"head_dim": 64,
"bos_token_id": 1,
"eos_token_id": 2,
"pad_token_id": 0,
"max_position_embeddings": 128,
"tie_word_embeddings": True,
},
)
@pytest.fixture(scope="session")
def micro_qwen_moe_model_path():
yield _micro_model_path(
qwen_moe_name_or_path,
"micro_qwen_moe_model_path",
{
"vocab_size": 256,
"hidden_size": 64,
"intermediate_size": 128,
"moe_intermediate_size": 32,
"shared_expert_intermediate_size": 128,
"num_attention_heads": 4,
"num_key_value_heads": 4,
"num_experts_per_tok": 2,
"decoder_sparse_step": 1,
"bos_token_id": 1,
"eos_token_id": 2,
"pad_token_id": 0,
"max_position_embeddings": 128,
},
num_experts=4,
)