Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

10.5 KB
Raw Blame
# Copyright (c) 2026 Intel Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Tests for audio model support: MiMo-Audio, Qwen3-TTS, and StableAudio."""

from types import SimpleNamespace
from unittest.mock import MagicMock

import torch

from auto_round.compressors.mllm.processor import PROCESSORS
from auto_round.compressors.mllm.template import TEMPLATES
from auto_round.special_model_handler import (
    ARCHITECTURE_MODEL_TYPE_MAP,
    SPECIAL_MULTIMODAL_BLOCK,
    SPECIAL_SHARED_CACHE_KEYS,
    SUPPORT_ONLY_TEXT_MODELS,
    _get_mimo_audio_multimodal_block,
    _get_qwen3_tts_multimodal_block,
    _handle_special_model,
    _qwen3_tts_forward,
    mllms_with_limited_bs,
    resolve_model_type,
)


def _make_mock_config(model_type, architectures=None):
    cfg = SimpleNamespace(model_type=model_type)
    if architectures is not None:
        cfg.architectures = architectures
    return cfg


def _make_mimo_audio_mock(n_main_layers=2, n_input_local_layers=1, n_local_layers=2):
    """Build a mock MiMo-Audio model with the expected module hierarchy."""
    model = MagicMock()
    model.config = _make_mock_config("qwen2", architectures=["MiMoAudioModel"])

    main_layers = torch.nn.ModuleList([torch.nn.Linear(8, 8) for _ in range(n_main_layers)])
    inner_model = MagicMock()
    inner_model.layers = main_layers
    model.model = inner_model

    input_local = MagicMock()
    input_local.layers = torch.nn.ModuleList([torch.nn.Linear(8, 8) for _ in range(n_input_local_layers)])
    model.input_local_transformer = input_local

    local_xfm = MagicMock()
    local_xfm.layers = torch.nn.ModuleList([torch.nn.Linear(8, 8) for _ in range(n_local_layers)])
    model.local_transformer = local_xfm

    return model


def _make_qwen3_tts_mock(n_layers=2, use_tts_model=True):
    """Build a mock Qwen3-TTS model, routed through either ``tts_model`` or ``talker``."""
    model = MagicMock()
    model.config = _make_mock_config("qwen3_tts", architectures=["Qwen3TTSForConditionalGeneration"])

    layers = torch.nn.ModuleList([torch.nn.Linear(8, 8) for _ in range(n_layers)])
    inner = MagicMock()
    inner.layers = layers

    if use_tts_model:
        tts = MagicMock()
        tts.model = inner
        model.tts_model = tts
        del model.talker
    else:
        talker = MagicMock()
        talker.model = inner
        model.talker = talker
        del model.tts_model

    return model


# ====================== MiMo-Audio ======================


class TestResolveModelType:
    def test_mimo_audio_architecture_override(self):
        """architectures=[MiMoAudioModel] resolves to the mimo_audio model type."""
        model = MagicMock()
        model.config = _make_mock_config("qwen2", architectures=["MiMoAudioModel"])
        assert resolve_model_type(model) == "mimo_audio"

    def test_mimo_audio_causal_lm_variant(self):
        """architectures=[MiMoAudioForCausalLM] also resolves to mimo_audio."""
        model = MagicMock()
        model.config = _make_mock_config("qwen2", architectures=["MiMoAudioForCausalLM"])
        assert resolve_model_type(model) == "mimo_audio"

    def test_qwen3_tts_uses_config_model_type(self):
        """qwen3_tts architecture resolves via its config model_type."""
        model = MagicMock()
        model.config = _make_mock_config("qwen3_tts", architectures=["Qwen3TTSForConditionalGeneration"])
        assert resolve_model_type(model) == "qwen3_tts"

    def test_standard_qwen2_not_overridden(self):
        """A plain Qwen2ForCausalLM is not remapped to a special model type."""
        model = MagicMock()
        model.config = _make_mock_config("qwen2", architectures=["Qwen2ForCausalLM"])
        assert resolve_model_type(model) == "qwen2"

    def test_no_architectures_falls_back(self):
        """Missing `architectures` falls back to the config's model_type."""
        model = MagicMock()
        model.config = _make_mock_config("qwen2")
        assert resolve_model_type(model) == "qwen2"

    def test_no_config_returns_none(self):
        """A model without a config resolves to None instead of raising."""
        model = MagicMock(spec=[])
        assert resolve_model_type(model) is None


class TestMiMoAudioBlockDetection:
    def test_main_decoder_only(self):
        """Main decoder layers are detected as a single multimodal block."""
        model = _make_mimo_audio_mock(n_main_layers=4)
        blocks = _get_mimo_audio_multimodal_block(model)
        assert len(blocks) == 1
        assert blocks[0] == [f"model.layers.{i}" for i in range(4)]

    def test_no_audio_modules(self):
        """Detection degrades gracefully when the audio-specific submodules are absent."""
        model = MagicMock()
        model.config = _make_mock_config("qwen2", architectures=["MiMoAudioModel"])
        inner = MagicMock()
        inner.layers = torch.nn.ModuleList([torch.nn.Linear(8, 8) for _ in range(2)])
        model.model = inner
        del model.input_local_transformer
        del model.local_transformer
        blocks = _get_mimo_audio_multimodal_block(model)
        assert len(blocks) == 1


class TestMiMoAudioRegistration:
    def test_special_map(self):
        """MiMo-Audio is registered in every registry it needs to be (block/text/processor/template)."""
        assert "MiMoAudioModel" in ARCHITECTURE_MODEL_TYPE_MAP
        assert "MiMoAudioForCausalLM" in ARCHITECTURE_MODEL_TYPE_MAP
        assert ARCHITECTURE_MODEL_TYPE_MAP["MiMoAudioModel"] == "mimo_audio"
        assert "mimo_audio" in SPECIAL_MULTIMODAL_BLOCK
        assert "mimo_audio" in SUPPORT_ONLY_TEXT_MODELS
        assert "mimo_audio" in PROCESSORS
        assert "mimo_audio" in TEMPLATES


class TestMiMoAudioForwardPatching:
    def test_forward_is_patched(self, tiny_mimo_audio_model_path):
        """_handle_special_model replaces forward on a real (tiny) MiMo-Audio model."""
        import transformers

        model = transformers.AutoModelForCausalLM.from_pretrained(tiny_mimo_audio_model_path)
        original_forward = model.forward
        model = _handle_special_model(model)
        assert model.forward != original_forward


# ====================== Qwen3-TTS ======================


class TestQwen3TTSDetection:
    def test_special_map(self):
        """Qwen3-TTS is registered in every registry it needs to be."""
        assert "qwen3_tts" in SPECIAL_MULTIMODAL_BLOCK
        assert "qwen3_tts" in SUPPORT_ONLY_TEXT_MODELS
        assert "qwen3_tts" in mllms_with_limited_bs
        assert "qwen3_tts" in PROCESSORS
        assert "qwen3_tts" in TEMPLATES

    def test_tts_model_path(self):
        """Block detection routes through `tts_model` when present."""
        model = _make_qwen3_tts_mock(n_layers=3, use_tts_model=True)
        blocks = _get_qwen3_tts_multimodal_block(model)
        assert len(blocks) == 1
        assert blocks[0] == [f"tts_model.model.layers.{i}" for i in range(3)]

    def test_talker_path(self):
        """Block detection routes through `talker` when `tts_model` is absent."""
        model = _make_qwen3_tts_mock(n_layers=4, use_tts_model=False)
        blocks = _get_qwen3_tts_multimodal_block(model)
        assert len(blocks) == 1
        assert blocks[0] == [f"talker.model.layers.{i}" for i in range(4)]

    def test_fallback_to_model_layers(self):
        """Block detection falls back to `model.layers` when neither submodule is present."""
        model = MagicMock()
        model.config = _make_mock_config("qwen3_tts")
        inner = MagicMock()
        inner.layers = torch.nn.ModuleList([torch.nn.Linear(8, 8) for _ in range(2)])
        model.model = inner
        del model.tts_model
        del model.talker
        blocks = _get_qwen3_tts_multimodal_block(model)
        assert len(blocks) == 1
        assert blocks[0] == [f"model.layers.{i}" for i in range(2)]


class TestQwen3TTSForward:
    def test_routes_through_tts_model(self):
        """Forward is routed through `tts_model` when present."""
        model = MagicMock()
        tts = MagicMock()
        model.tts_model = tts
        del model.talker
        dummy_ids = torch.randint(0, 100, (1, 4))
        _qwen3_tts_forward(model, input_ids=dummy_ids)
        tts.assert_called_once()

    def test_routes_through_talker(self):
        """Forward is routed through `talker` when `tts_model` is None."""
        model = MagicMock()
        talker = MagicMock()
        model.talker = talker
        model.tts_model = None
        dummy_ids = torch.randint(0, 100, (1, 4))
        _qwen3_tts_forward(model, input_ids=dummy_ids)
        talker.assert_called_once()


# ====================== StableAudio ======================


class TestStableAudioRegistration:
    def test_config_and_special_registered(self):
        """StableAudio is registered for diffusion output handling and shared cache keys."""
        from auto_round.algorithms.block_runner import _DIFFUSION_OUTPUT_REGISTRY

        assert "StableAudioDiTBlock" in _DIFFUSION_OUTPUT_REGISTRY
        assert _DIFFUSION_OUTPUT_REGISTRY["StableAudioDiTBlock"] == ["hidden_states"]

        assert "StableAudioDiTModel" in SPECIAL_SHARED_CACHE_KEYS
        assert "encoder_hidden_states" in SPECIAL_SHARED_CACHE_KEYS["StableAudioDiTModel"]


class TestStableAudioPipelineFunction:
    def test_attach_pipeline_fn(self):
        """StableAudio pipelines get a custom call wrapper injecting audio_end_in_s."""
        from auto_round.utils.model import _attach_diffusion_pipeline_fn

        pipe = MagicMock()
        type(pipe).__name__ = "StableAudioPipeline"
        _attach_diffusion_pipeline_fn(pipe)
        assert hasattr(pipe, "_autoround_pipeline_fn")
        pipe._autoround_pipeline_fn(pipe, ["test prompt"], guidance_scale=3.5, num_inference_steps=10)
        pipe.assert_called_once()
        _, kwargs = pipe.call_args
        assert "audio_end_in_s" in kwargs

    def test_noop_for_non_stable_audio(self):
        """Non-StableAudio pipelines are left untouched."""
        from auto_round.utils.model import _attach_diffusion_pipeline_fn

        pipe = MagicMock(spec=["config", "__call__"])
        type(pipe).__name__ = "FluxPipeline"
        _attach_diffusion_pipeline_fn(pipe)
        assert not hasattr(pipe, "_autoround_pipeline_fn")