PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
# Copyright (c) 2026 Intel Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Tests for auto_round/special_model_handler.py"""
from unittest.mock import MagicMock, patch
import pytest
import torch
class TestMllmsWithLimitedBs:
"""Test mllms_with_limited_bs tuple."""
def test_is_tuple(self):
from auto_round.special_model_handler import mllms_with_limited_bs
assert isinstance(mllms_with_limited_bs, tuple)
def test_llava_in_tuple(self):
from auto_round.special_model_handler import mllms_with_limited_bs
assert "llava" in mllms_with_limited_bs
def test_qwen2_vl_in_tuple(self):
from auto_round.special_model_handler import mllms_with_limited_bs
assert "qwen2_vl" in mllms_with_limited_bs
def test_phi3_v_in_tuple(self):
from auto_round.special_model_handler import mllms_with_limited_bs
assert "phi3_v" in mllms_with_limited_bs
def test_mllama_in_tuple(self):
from auto_round.special_model_handler import mllms_with_limited_bs
assert "mllama" in mllms_with_limited_bs
def test_qwen2_5_omni_in_tuple(self):
from auto_round.special_model_handler import mllms_with_limited_bs
assert "qwen2_5_omni" in mllms_with_limited_bs
def test_qwen3_omni_moe_in_tuple(self):
from auto_round.special_model_handler import mllms_with_limited_bs
assert "qwen3_omni_moe" in mllms_with_limited_bs
def test_glm_image_in_tuple(self):
from auto_round.special_model_handler import mllms_with_limited_bs
assert "glm_image" in mllms_with_limited_bs
def test_mimo_audio_in_tuple(self):
from auto_round.special_model_handler import mllms_with_limited_bs
assert "mimo_audio" in mllms_with_limited_bs
def test_qwen3_tts_in_tuple(self):
from auto_round.special_model_handler import mllms_with_limited_bs
assert "qwen3_tts" in mllms_with_limited_bs
def test_tuple_length(self):
from auto_round.special_model_handler import mllms_with_limited_bs
assert len(mllms_with_limited_bs) == 9
class TestSupportOnlyTextModels:
"""Test SUPPORT_ONLY_TEXT_MODELS list."""
def test_is_list(self):
from auto_round.special_model_handler import SUPPORT_ONLY_TEXT_MODELS
assert isinstance(SUPPORT_ONLY_TEXT_MODELS, list)
def test_phi3_v_in_list(self):
from auto_round.special_model_handler import SUPPORT_ONLY_TEXT_MODELS
assert "phi3_v" in SUPPORT_ONLY_TEXT_MODELS
def test_cogvlm2_in_list(self):
from auto_round.special_model_handler import SUPPORT_ONLY_TEXT_MODELS
assert "cogvlm2" in SUPPORT_ONLY_TEXT_MODELS
def test_llava_in_list(self):
from auto_round.special_model_handler import SUPPORT_ONLY_TEXT_MODELS
assert "llava" in SUPPORT_ONLY_TEXT_MODELS
def test_qwen2_vl_in_list(self):
from auto_round.special_model_handler import SUPPORT_ONLY_TEXT_MODELS
assert "qwen2_vl" in SUPPORT_ONLY_TEXT_MODELS
def test_qwen2_5_vl_in_list(self):
from auto_round.special_model_handler import SUPPORT_ONLY_TEXT_MODELS
assert "qwen2_5_vl" in SUPPORT_ONLY_TEXT_MODELS
def test_deepseek_vl_v2_in_list(self):
from auto_round.special_model_handler import SUPPORT_ONLY_TEXT_MODELS
assert "deepseek_vl_v2" in SUPPORT_ONLY_TEXT_MODELS
def test_chatglm_in_list(self):
from auto_round.special_model_handler import SUPPORT_ONLY_TEXT_MODELS
assert "chatglm" in SUPPORT_ONLY_TEXT_MODELS
def test_idefics3_in_list(self):
from auto_round.special_model_handler import SUPPORT_ONLY_TEXT_MODELS
assert "idefics3" in SUPPORT_ONLY_TEXT_MODELS
def test_llama4_in_list(self):
from auto_round.special_model_handler import SUPPORT_ONLY_TEXT_MODELS
assert "llama4" in SUPPORT_ONLY_TEXT_MODELS
def test_internvl_chat_in_list(self):
from auto_round.special_model_handler import SUPPORT_ONLY_TEXT_MODELS
assert "internvl_chat" in SUPPORT_ONLY_TEXT_MODELS
def test_glm4v_moe_in_list(self):
from auto_round.special_model_handler import SUPPORT_ONLY_TEXT_MODELS
assert "glm4v_moe" in SUPPORT_ONLY_TEXT_MODELS
def test_glm_image_in_list(self):
from auto_round.special_model_handler import SUPPORT_ONLY_TEXT_MODELS
assert "glm_image" in SUPPORT_ONLY_TEXT_MODELS
def test_qwen3_vl_moe_in_list(self):
from auto_round.special_model_handler import SUPPORT_ONLY_TEXT_MODELS
assert "qwen3_vl_moe" in SUPPORT_ONLY_TEXT_MODELS
def test_qwen2_5_omni_in_list(self):
from auto_round.special_model_handler import SUPPORT_ONLY_TEXT_MODELS
assert "qwen2_5_omni" in SUPPORT_ONLY_TEXT_MODELS
def test_qwen3_omni_moe_in_list(self):
from auto_round.special_model_handler import SUPPORT_ONLY_TEXT_MODELS
assert "qwen3_omni_moe" in SUPPORT_ONLY_TEXT_MODELS
def test_gemma3_in_list(self):
from auto_round.special_model_handler import SUPPORT_ONLY_TEXT_MODELS
assert "gemma3" in SUPPORT_ONLY_TEXT_MODELS
def test_bagel_in_list(self):
from auto_round.special_model_handler import SUPPORT_ONLY_TEXT_MODELS
assert "bagel" in SUPPORT_ONLY_TEXT_MODELS
def test_mimo_audio_in_list(self):
from auto_round.special_model_handler import SUPPORT_ONLY_TEXT_MODELS
assert "mimo_audio" in SUPPORT_ONLY_TEXT_MODELS
def test_qwen3_tts_in_list(self):
from auto_round.special_model_handler import SUPPORT_ONLY_TEXT_MODELS
assert "qwen3_tts" in SUPPORT_ONLY_TEXT_MODELS
class TestNotSupportOnlyTextModels:
"""Test NOT_SUPPORT_ONLY_TEXT_MODELS list."""
def test_is_list(self):
from auto_round.special_model_handler import NOT_SUPPORT_ONLY_TEXT_MODELS
assert isinstance(NOT_SUPPORT_ONLY_TEXT_MODELS, list)
def test_mllama_in_list(self):
from auto_round.special_model_handler import NOT_SUPPORT_ONLY_TEXT_MODELS
assert "mllama" in NOT_SUPPORT_ONLY_TEXT_MODELS
def test_mistral3_2_in_list(self):
from auto_round.special_model_handler import NOT_SUPPORT_ONLY_TEXT_MODELS
assert "mistral3_2" in NOT_SUPPORT_ONLY_TEXT_MODELS
class TestSpecialSharedCacheKeys:
"""Test SPECIAL_SHARED_CACHE_KEYS dict."""
def test_is_dict(self):
from auto_round.special_model_handler import SPECIAL_SHARED_CACHE_KEYS
assert isinstance(SPECIAL_SHARED_CACHE_KEYS, dict)
def test_gemma3_for_conditional_generation_key(self):
from auto_round.special_model_handler import SPECIAL_SHARED_CACHE_KEYS
assert "Gemma3ForConditionalGeneration" in SPECIAL_SHARED_CACHE_KEYS
keys = SPECIAL_SHARED_CACHE_KEYS["Gemma3ForConditionalGeneration"]
assert "position_embeddings_global" in keys
assert "position_embeddings_local" in keys
def test_minimax_key(self):
from auto_round.special_model_handler import SPECIAL_SHARED_CACHE_KEYS
assert "MiniMaxText01ForCausalLM" in SPECIAL_SHARED_CACHE_KEYS
keys = SPECIAL_SHARED_CACHE_KEYS["MiniMaxText01ForCausalLM"]
assert "slope_rate" in keys
def test_stable_audio_dit_model_key(self):
from auto_round.special_model_handler import SPECIAL_SHARED_CACHE_KEYS
assert "StableAudioDiTModel" in SPECIAL_SHARED_CACHE_KEYS
keys = SPECIAL_SHARED_CACHE_KEYS["StableAudioDiTModel"]
assert "encoder_hidden_states" in keys
def test_gemma4_for_conditional_generation_key(self):
from auto_round.special_model_handler import SPECIAL_SHARED_CACHE_KEYS
assert "Gemma4ForConditionalGeneration" in SPECIAL_SHARED_CACHE_KEYS
keys = SPECIAL_SHARED_CACHE_KEYS["Gemma4ForConditionalGeneration"]
assert "position_ids" in keys
def test_wan_transformer_3d_model_key(self):
from auto_round.special_model_handler import SPECIAL_SHARED_CACHE_KEYS
assert "WanTransformer3DModel" in SPECIAL_SHARED_CACHE_KEYS
keys = SPECIAL_SHARED_CACHE_KEYS["WanTransformer3DModel"]
assert "rotary_emb" in keys
class TestMistral32Models:
"""Test MISTRAL_3_2_MODELS list."""
def test_is_list(self):
from auto_round.special_model_handler import MISTRAL_3_2_MODELS
assert isinstance(MISTRAL_3_2_MODELS, list)
def test_mistral_small_3_2_in_list(self):
from auto_round.special_model_handler import MISTRAL_3_2_MODELS
assert "Mistral-Small-3.2" in MISTRAL_3_2_MODELS
def test_magistral_small_in_list(self):
from auto_round.special_model_handler import MISTRAL_3_2_MODELS
assert "Magistral-Small" in MISTRAL_3_2_MODELS
def test_devstral_small_in_list(self):
from auto_round.special_model_handler import MISTRAL_3_2_MODELS
assert "Devstral-Small" in MISTRAL_3_2_MODELS
class TestModelNameMatcher:
"""Test ModelNameMatcher class."""
def test_match_qwen_in_mode(self):
from auto_round.special_model_handler import ModelNameMatcher
matcher = ModelNameMatcher("Qwen3-0.6B", mode="in")
mock_model = MagicMock()
mock_model.config.name_or_path = "Qwen/Qwen3-0.6B"
assert matcher(mock_model) is True
def test_match_qwen_case_insensitive(self):
from auto_round.special_model_handler import ModelNameMatcher
matcher = ModelNameMatcher("Qwen", mode="in")
mock_model = MagicMock()
mock_model.config.name_or_path = "Qwen/Qwen2.5-3B"
assert matcher(mock_model) is True
def test_match_deepseek_in_mode(self):
from auto_round.special_model_handler import ModelNameMatcher
matcher = ModelNameMatcher("deepseek-ai", mode="in")
mock_model = MagicMock()
mock_model.config.name_or_path = "deepseek-ai/DeepSeek-V2-Lite"
assert matcher(mock_model) is True
def test_match_gemma_in_mode(self):
from auto_round.special_model_handler import ModelNameMatcher
matcher = ModelNameMatcher("gemma", mode="in")
mock_model = MagicMock()
mock_model.config.name_or_path = "google/gemma-2b-it"
assert matcher(mock_model) is True
def test_no_match(self):
from auto_round.special_model_handler import ModelNameMatcher
matcher = ModelNameMatcher("Qwen", mode="in")
mock_model = MagicMock()
mock_model.config.name_or_path = "facebook/opt-125m"
assert matcher(mock_model) is False
def test_full_mode_match(self):
from auto_round.special_model_handler import ModelNameMatcher
matcher = ModelNameMatcher("Qwen/Qwen3-0.6B", mode="full")
mock_model = MagicMock()
mock_model.config.name_or_path = "Qwen/Qwen3-0.6B"
assert matcher(mock_model) is True
def test_full_mode_no_match(self):
from auto_round.special_model_handler import ModelNameMatcher
matcher = ModelNameMatcher("Qwen/Qwen3-0.6B", mode="full")
mock_model = MagicMock()
mock_model.config.name_or_path = "Qwen/Qwen2.5-3B"
assert matcher(mock_model) is False
def test_regex_mode_match(self):
from auto_round.special_model_handler import ModelNameMatcher
matcher = ModelNameMatcher(r"Qwen\d*-", mode="regex")
mock_model = MagicMock()
mock_model.config.name_or_path = "Qwen/Qwen3-0.6B"
assert matcher(mock_model) is True
def test_regex_mode_no_match(self):
from auto_round.special_model_handler import ModelNameMatcher
matcher = ModelNameMatcher(r"Mistral-", mode="regex")
mock_model = MagicMock()
mock_model.config.name_or_path = "Qwen/Qwen3-0.6B"
assert matcher(mock_model) is False
def test_unsupported_mode_raises(self):
from auto_round.special_model_handler import ModelNameMatcher
matcher = ModelNameMatcher("test", mode="unsupported")
mock_model = MagicMock()
with pytest.raises(ValueError, match="unsupported mode"):
matcher(mock_model)
class TestArchitectureMatcher:
"""Test ArchitectureMatcher class."""
def test_match_qwen3_5_moe_in_mode(self):
from auto_round.special_model_handler import ArchitectureMatcher
matcher = ArchitectureMatcher("Qwen3_5Moe", mode="in")
mock_model = MagicMock()
mock_model.config.architectures = ["Qwen3_5MoeForConditionalGeneration"]
assert matcher(mock_model) is True
def test_match_qwen3_omni_moe_in_mode(self):
from auto_round.special_model_handler import ArchitectureMatcher
matcher = ArchitectureMatcher("Qwen3OmniMoe", mode="in")
mock_model = MagicMock()
mock_model.config.architectures = ["Qwen3OmniMoeForConditionalGeneration"]
assert matcher(mock_model) is True
def test_match_deepseek_v2_in_mode(self):
from auto_round.special_model_handler import ArchitectureMatcher
matcher = ArchitectureMatcher("DeepSeekV2", mode="in")
mock_model = MagicMock()
mock_model.config.architectures = ["DeepSeekV2ForCausalLM"]
assert matcher(mock_model) is True
def test_match_gemma3_in_mode(self):
from auto_round.special_model_handler import ArchitectureMatcher
matcher = ArchitectureMatcher("Gemma3", mode="in")
mock_model = MagicMock()
mock_model.config.architectures = ["Gemma3ForConditionalGeneration"]
assert matcher(mock_model) is True
def test_no_match(self):
from auto_round.special_model_handler import ArchitectureMatcher
matcher = ArchitectureMatcher("Qwen", mode="in")
mock_model = MagicMock()
mock_model.config.architectures = ["OPTForCausalLM"]
assert matcher(mock_model) is False
def test_full_mode_match(self):
from auto_round.special_model_handler import ArchitectureMatcher
matcher = ArchitectureMatcher("Qwen3_5MoeForConditionalGeneration", mode="full")
mock_model = MagicMock()
mock_model.config.architectures = ["Qwen3_5MoeForConditionalGeneration"]
assert matcher(mock_model) is True
def test_full_mode_no_match(self):
from auto_round.special_model_handler import ArchitectureMatcher
matcher = ArchitectureMatcher("Qwen3_5MoeForConditionalGeneration", mode="full")
mock_model = MagicMock()
mock_model.config.architectures = ["Qwen3_5MoEForConditionalGeneration"]
assert matcher(mock_model) is False
def test_regex_mode_match(self):
from auto_round.special_model_handler import ArchitectureMatcher
matcher = ArchitectureMatcher(r"Qwen\d*_?\d*Moe", mode="regex")
mock_model = MagicMock()
mock_model.config.architectures = ["Qwen3_5MoeForConditionalGeneration"]
assert matcher(mock_model) is True
def test_regex_mode_no_match(self):
from auto_round.special_model_handler import ArchitectureMatcher
matcher = ArchitectureMatcher(r"Mistral-", mode="regex")
mock_model = MagicMock()
mock_model.config.architectures = ["Qwen3_5MoeForConditionalGeneration"]
assert matcher(mock_model) is False
def test_unsupported_mode_raises(self):
from auto_round.special_model_handler import ArchitectureMatcher
matcher = ArchitectureMatcher("test", mode="unsupported")
mock_model = MagicMock()
with pytest.raises(ValueError, match="unsupported mode"):
matcher(mock_model)
class TestPreDefinedIgnoreLayers:
"""Test PreDefinedIgnoreLayers dataclass."""
def test_dataclass_fields(self):
from auto_round.special_model_handler import PreDefinedIgnoreLayers
ignore = PreDefinedIgnoreLayers(matchers=[], ignore_layers=[])
assert hasattr(ignore, "matchers")
assert hasattr(ignore, "ignore_layers")
def test_dataclass_assignment(self):
from auto_round.special_model_handler import PreDefinedIgnoreLayers
matcher = MagicMock()
ignore = PreDefinedIgnoreLayers(matchers=[matcher], ignore_layers=["layer.0", "layer.1"])
assert ignore.matchers == [matcher]
assert ignore.ignore_layers == ["layer.0", "layer.1"]
def test_default_ignore_layers_empty_list(self):
from auto_round.special_model_handler import PreDefinedIgnoreLayers
ignore = PreDefinedIgnoreLayers(matchers=[])
assert ignore.ignore_layers == []
class TestCheckMllmModelBatch:
"""Test check_mllm_only_support_bs1 function.
Note: ``check_mllm_model_batch`` was replaced by ``check_mllm_only_support_bs1``
during the compressor/quantizer refactor (#2039). The new helper returns a bool
indicating whether the model only supports ``batch_size == 1`` instead of
clamping the batch size itself.
"""
def _create_mock_model(self, model_type, architectures=None):
mock_model = MagicMock()
mock_model.config.model_type = model_type
if architectures is not None:
mock_model.config.architectures = architectures
return mock_model
@pytest.mark.parametrize(
"model_type",
[
"llava",
"qwen2_vl",
"phi3_v",
"mllama",
"qwen2_5_omni",
"qwen3_omni_moe",
"glm_image",
"mimo_audio",
"qwen3_tts",
],
)
def test_mllm_requires_batch_size_1(self, model_type):
from auto_round.special_model_handler import check_mllm_only_support_bs1
mock_model = self._create_mock_model(model_type)
assert check_mllm_only_support_bs1(mock_model) is True
def test_non_mllm_allows_batch_greater_than_1(self):
from auto_round.special_model_handler import check_mllm_only_support_bs1
mock_model = self._create_mock_model("gpt2")
assert check_mllm_only_support_bs1(mock_model) is False
def test_model_without_config_returns_false(self):
from auto_round.special_model_handler import check_mllm_only_support_bs1
mock_model = MagicMock()
mock_model.config = None
assert check_mllm_only_support_bs1(mock_model) is False
def test_architecture_based_override(self):
"""MiMo-Audio has architecture MiMoAudioModel but model_type qwen2."""
from auto_round.special_model_handler import check_mllm_only_support_bs1
mock_model = self._create_mock_model("qwen2", architectures=["MiMoAudioModel"])
assert check_mllm_only_support_bs1(mock_model) is True
class TestNormalizeGemma4PerLayerInput:
"""Test _normalize_gemma4_per_layer_input function."""
def test_none_input(self):
from auto_round.special_model_handler import _normalize_gemma4_per_layer_input
result = _normalize_gemma4_per_layer_input(None, torch.randn(1, 10, 128))
assert result is None
def test_empty_list(self):
from auto_round.special_model_handler import _normalize_gemma4_per_layer_input
result = _normalize_gemma4_per_layer_input([], torch.randn(1, 10, 128))
assert result == []
def test_same_shape(self):
from auto_round.special_model_handler import _normalize_gemma4_per_layer_input
positional_inputs = (torch.randn(1, 10, 128),)
hidden_states = torch.randn(1, 10, 128)
result = _normalize_gemma4_per_layer_input(positional_inputs, hidden_states)
assert torch.equal(result[0], positional_inputs[0])
def test_truncate_longer_input(self):
from auto_round.special_model_handler import _normalize_gemma4_per_layer_input
positional_inputs = (torch.randn(1, 20, 128),)
hidden_states = torch.randn(1, 10, 128)
result = _normalize_gemma4_per_layer_input(positional_inputs, hidden_states)
assert result[0].shape[1] == 10
def test_pad_shorter_input(self):
from auto_round.special_model_handler import _normalize_gemma4_per_layer_input
positional_inputs = (torch.randn(1, 5, 128),)
hidden_states = torch.randn(1, 10, 128)
result = _normalize_gemma4_per_layer_input(positional_inputs, hidden_states)
assert result[0].shape[1] == 10
def test_returns_tuple_when_input_is_tuple(self):
from auto_round.special_model_handler import _normalize_gemma4_per_layer_input
positional_inputs = (torch.randn(1, 10, 128), torch.randn(1, 10, 128))
hidden_states = torch.randn(1, 10, 128)
result = _normalize_gemma4_per_layer_input(positional_inputs, hidden_states)
assert isinstance(result, tuple)
def test_returns_list_when_input_is_list(self):
from auto_round.special_model_handler import _normalize_gemma4_per_layer_input
positional_inputs = [torch.randn(1, 10, 128), torch.randn(1, 10, 128)]
hidden_states = torch.randn(1, 5, 128)
result = _normalize_gemma4_per_layer_input(positional_inputs, hidden_states)
assert isinstance(result, list)
def test_non_tensor_input_not_modified(self):
from auto_round.special_model_handler import _normalize_gemma4_per_layer_input
positional_inputs = ("not_a_tensor",)
hidden_states = torch.randn(1, 10, 128)
result = _normalize_gemma4_per_layer_input(positional_inputs, hidden_states)
assert result == positional_inputs
class TestPrepareSpecialModelBlockInputs:
"""Test prepare_special_model_block_inputs function."""
def test_position_ids_none_creates_tensor(self):
from auto_round.special_model_handler import prepare_special_model_block_inputs
block = MagicMock()
block._autoround_special_replay = None
rotary_input = torch.randn(1, 10, 128)
input_others = {"position_ids": None}
positional_inputs = None
result_others, result_pos = prepare_special_model_block_inputs(
block, rotary_input, input_others, positional_inputs
)
assert result_others["position_ids"] is not None
assert result_others["position_ids"].shape == (1, 10)
def test_position_ids_list_single_element(self):
from auto_round.special_model_handler import prepare_special_model_block_inputs
block = MagicMock()
block._autoround_special_replay = None
rotary_input = torch.randn(1, 10, 128)
input_others = {"position_ids": [torch.tensor([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])]}
positional_inputs = None
result_others, result_pos = prepare_special_model_block_inputs(
block, rotary_input, input_others, positional_inputs
)
assert isinstance(result_others["position_ids"], torch.Tensor)
def test_position_ids_list_empty_creates_tensor(self):
from auto_round.special_model_handler import prepare_special_model_block_inputs
block = MagicMock()
block._autoround_special_replay = None
rotary_input = torch.randn(1, 10, 128)
input_others = {"position_ids": []}
positional_inputs = None
result_others, result_pos = prepare_special_model_block_inputs(
block, rotary_input, input_others, positional_inputs
)
assert result_others["position_ids"].shape == (1, 10)
def test_position_ids_already_tensor_unchanged(self):
from auto_round.special_model_handler import prepare_special_model_block_inputs
block = MagicMock()
block._autoround_special_replay = None
rotary_input = torch.randn(1, 10, 128)
input_others = {"position_ids": torch.tensor([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])}
positional_inputs = None
result_others, result_pos = prepare_special_model_block_inputs(
block, rotary_input, input_others, positional_inputs
)
assert torch.equal(result_others["position_ids"], input_others["position_ids"])
def test_position_ids_not_in_input_others(self):
from auto_round.special_model_handler import prepare_special_model_block_inputs
block = MagicMock()
block._autoround_special_replay = None
rotary_input = torch.randn(1, 10, 128)
input_others = {}
positional_inputs = None
result_others, result_pos = prepare_special_model_block_inputs(
block, rotary_input, input_others, positional_inputs
)
assert "position_ids" not in result_others
class TestGetDeepseekVl2MultimodalBlock:
"""Test _get_deepseek_vl2_multimodal_block function."""
def test_returns_language_layers(self):
from auto_round.special_model_handler import _get_deepseek_vl2_multimodal_block
mock_model = MagicMock()
mock_model.language.model.layers = [MagicMock() for _ in range(10)]
block_names = _get_deepseek_vl2_multimodal_block(mock_model)
assert len(block_names) == 1
assert len(block_names[0]) == 10
def test_with_quant_vision_true(self):
from auto_round.special_model_handler import _get_deepseek_vl2_multimodal_block
mock_model = MagicMock()
mock_model.vision.blocks = [MagicMock() for _ in range(5)]
mock_model.projector.layers = [MagicMock() for _ in range(3)]
mock_model.language.model.layers = [MagicMock() for _ in range(10)]
block_names = _get_deepseek_vl2_multimodal_block(mock_model, quant_vision=True)
assert len(block_names) == 3
assert len(block_names[0]) == 5 # vision blocks
assert len(block_names[1]) == 3 # projector layers
assert len(block_names[2]) == 10 # language layers
def test_model_forward_replaced(self):
from auto_round.special_model_handler import _get_deepseek_vl2_multimodal_block
mock_model = MagicMock()
mock_model.language.forward = MagicMock()
mock_model.language.model.layers = [MagicMock() for _ in range(10)]
original_forward = mock_model.forward
_get_deepseek_vl2_multimodal_block(mock_model)
assert mock_model.forward == mock_model.language.forward
class TestGetQwen25OmniMultimodalBlock:
"""Test _get_qwen2_5_omni_multimodal_block function."""
def test_returns_thinker_layers(self):
from auto_round.special_model_handler import _get_qwen2_5_omni_multimodal_block
mock_model = MagicMock()
mock_model.thinker.model.layers = [MagicMock() for _ in range(8)]
block_names = _get_qwen2_5_omni_multimodal_block(mock_model)
assert len(block_names) == 1
assert len(block_names[0]) == 8
def test_with_quant_vision_true(self):
from auto_round.special_model_handler import _get_qwen2_5_omni_multimodal_block
mock_model = MagicMock()
mock_model.thinker.visual.blocks = [MagicMock() for _ in range(5)]
mock_model.thinker.audio_tower.layers = [MagicMock() for _ in range(3)]
mock_model.thinker.model.layers = [MagicMock() for _ in range(8)]
block_names = _get_qwen2_5_omni_multimodal_block(mock_model, quant_vision=True)
assert len(block_names) == 3
assert len(block_names[0]) == 5 # visual blocks
assert len(block_names[1]) == 3 # audio tower layers
assert len(block_names[2]) == 8 # thinker model layers
def test_no_thinker_model(self):
from auto_round.special_model_handler import _get_qwen2_5_omni_multimodal_block
mock_model = MagicMock(spec=[])
block_names = _get_qwen2_5_omni_multimodal_block(mock_model)
assert block_names == []
class TestGetQwen3OmniMoeMultimodalBlock:
"""Test _get_qwen3_omni_moe_multimodal_block function."""
def test_with_nested_vision_and_audio_towers(self):
from types import SimpleNamespace
from auto_round.special_model_handler import _get_qwen3_omni_moe_multimodal_block
model = SimpleNamespace(
thinker=SimpleNamespace(
model=SimpleNamespace(
visual=SimpleNamespace(blocks=[MagicMock() for _ in range(2)]),
audio_tower=SimpleNamespace(layers=[MagicMock() for _ in range(3)]),
language_model=SimpleNamespace(layers=[MagicMock() for _ in range(4)]),
)
)
)
block_names = _get_qwen3_omni_moe_multimodal_block(model, quant_vision=True)
assert block_names == [
["thinker.model.visual.blocks.0", "thinker.model.visual.blocks.1"],
[
"thinker.model.audio_tower.layers.0",
"thinker.model.audio_tower.layers.1",
"thinker.model.audio_tower.layers.2",
],
[
"thinker.model.language_model.layers.0",
"thinker.model.language_model.layers.1",
"thinker.model.language_model.layers.2",
"thinker.model.language_model.layers.3",
],
]
def test_returns_thinker_layers(self):
from auto_round.special_model_handler import _get_qwen3_omni_moe_multimodal_block
mock_model = MagicMock()
mock_model.thinker.model.layers = [MagicMock() for _ in range(8)]
block_names = _get_qwen3_omni_moe_multimodal_block(mock_model)
assert len(block_names) == 1
assert len(block_names[0]) == 8
def test_with_quant_vision_true(self):
from auto_round.special_model_handler import _get_qwen3_omni_moe_multimodal_block
mock_model = MagicMock()
mock_model.thinker.visual.blocks = [MagicMock() for _ in range(5)]
mock_model.thinker.audio_tower.layers = [MagicMock() for _ in range(3)]
mock_model.thinker.model.layers = [MagicMock() for _ in range(8)]
block_names = _get_qwen3_omni_moe_multimodal_block(mock_model, quant_vision=True)
assert len(block_names) == 3
class TestGetGlmImageMultimodalBlock:
"""Test _get_glm_image_multimodal_block function."""
def test_returns_language_model_layers(self):
from auto_round.special_model_handler import _get_glm_image_multimodal_block
mock_model = MagicMock()
mock_model.model.language_model.layers = [MagicMock() for _ in range(12)]
block_names = _get_glm_image_multimodal_block(mock_model)
assert len(block_names) == 1
assert len(block_names[0]) == 12
def test_with_quant_vision_true(self):
from auto_round.special_model_handler import _get_glm_image_multimodal_block
mock_model = MagicMock()
mock_model.model.visual.blocks = [MagicMock() for _ in range(8)]
mock_model.model.language_model.layers = [MagicMock() for _ in range(12)]
block_names = _get_glm_image_multimodal_block(mock_model, quant_vision=True)
assert len(block_names) == 2
assert len(block_names[0]) == 8 # visual blocks
assert len(block_names[1]) == 12 # language model layers
def test_no_language_model(self):
from auto_round.special_model_handler import _get_glm_image_multimodal_block
mock_model = MagicMock(spec=[])
block_names = _get_glm_image_multimodal_block(mock_model)
assert block_names == []
class TestGetMimoAudioMultimodalBlock:
"""Test _get_mimo_audio_multimodal_block function."""
def test_returns_model_layers(self):
from auto_round.special_model_handler import _get_mimo_audio_multimodal_block
mock_model = MagicMock()
mock_model.model.layers = [MagicMock() for _ in range(28)]
block_names = _get_mimo_audio_multimodal_block(mock_model)
assert len(block_names) == 1
assert len(block_names[0]) == 28
def test_with_base_model(self):
from auto_round.special_model_handler import _get_mimo_audio_multimodal_block
mock_model = MagicMock(spec=[])
mock_model.layers = [MagicMock() for _ in range(20)]
block_names = _get_mimo_audio_multimodal_block(mock_model)
assert len(block_names) == 1
assert len(block_names[0]) == 20
def test_no_layers(self):
from auto_round.special_model_handler import _get_mimo_audio_multimodal_block
mock_model = MagicMock(spec=[])
block_names = _get_mimo_audio_multimodal_block(mock_model)
assert block_names == []
class TestGetQwen3TtsMultimodalBlock:
"""Test _get_qwen3_tts_multimodal_block function."""
def test_tts_model_model_layers(self):
from auto_round.special_model_handler import _get_qwen3_tts_multimodal_block
mock_model = MagicMock()
mock_model.tts_model.model.layers = [MagicMock() for _ in range(6)]
block_names = _get_qwen3_tts_multimodal_block(mock_model)
assert len(block_names) == 1
assert len(block_names[0]) == 6
def test_talker_model_layers(self):
from auto_round.special_model_handler import _get_qwen3_tts_multimodal_block
mock_model = MagicMock()
mock_model.tts_model = MagicMock(spec=[])
mock_model.talker.model.layers = [MagicMock() for _ in range(6)]
block_names = _get_qwen3_tts_multimodal_block(mock_model)
assert len(block_names) == 1
assert len(block_names[0]) == 6
def test_model_model_layers_fallback(self):
from auto_round.special_model_handler import _get_qwen3_tts_multimodal_block
mock_model = MagicMock()
mock_model.tts_model = MagicMock(spec=[])
mock_model.talker = MagicMock(spec=[])
mock_model.model.layers = [MagicMock() for _ in range(6)]
block_names = _get_qwen3_tts_multimodal_block(mock_model)
assert len(block_names) == 1
assert len(block_names[0]) == 6
class TestSpecialMultimodalBlockRegistry:
"""Test SPECIAL_MULTIMODAL_BLOCK registry."""
def test_registry_contains_deepseek_vl_v2(self):
from auto_round.special_model_handler import SPECIAL_MULTIMODAL_BLOCK
assert "deepseek_vl_v2" in SPECIAL_MULTIMODAL_BLOCK
def test_registry_contains_qwen2_5_omni(self):
from auto_round.special_model_handler import SPECIAL_MULTIMODAL_BLOCK
assert "qwen2_5_omni" in SPECIAL_MULTIMODAL_BLOCK
def test_registry_contains_qwen3_omni_moe(self):
from auto_round.special_model_handler import SPECIAL_MULTIMODAL_BLOCK
assert "qwen3_omni_moe" in SPECIAL_MULTIMODAL_BLOCK
def test_registry_contains_glm_image(self):
from auto_round.special_model_handler import SPECIAL_MULTIMODAL_BLOCK
assert "glm_image" in SPECIAL_MULTIMODAL_BLOCK
def test_registry_contains_mimo_audio(self):
from auto_round.special_model_handler import SPECIAL_MULTIMODAL_BLOCK
assert "mimo_audio" in SPECIAL_MULTIMODAL_BLOCK
def test_registry_contains_qwen3_tts(self):
from auto_round.special_model_handler import SPECIAL_MULTIMODAL_BLOCK
assert "qwen3_tts" in SPECIAL_MULTIMODAL_BLOCK
def test_registry_contains_bagel(self):
from auto_round.special_model_handler import SPECIAL_MULTIMODAL_BLOCK
assert "bagel" in SPECIAL_MULTIMODAL_BLOCK
class TestRegisterIgnoreLayers:
"""Test register_ignore_layers function."""
def test_register_ignore_layers(self):
from auto_round.special_model_handler import _PRE_DEFINED_IGNORE_LAYERS, register_ignore_layers
initial_count = len(_PRE_DEFINED_IGNORE_LAYERS)
matcher = MagicMock(return_value=True)
try:
register_ignore_layers(matchers=[matcher], ignore_layers=["layer.0"])
assert len(_PRE_DEFINED_IGNORE_LAYERS) == initial_count + 1
finally:
del _PRE_DEFINED_IGNORE_LAYERS[initial_count:]
class TestGetPredefinedIgnoreLayers:
"""Test get_predefined_ignore_layers function."""
def test_longcat_matcher(self):
from auto_round.special_model_handler import get_predefined_ignore_layers
mock_model = MagicMock()
mock_model.config.architectures = ["LongcatConfig"]
layers = get_predefined_ignore_layers(mock_model)
assert "classifier" in layers
def test_glm_flash_matcher(self):
from auto_round.special_model_handler import get_predefined_ignore_layers
mock_model = MagicMock()
mock_model.config.model_type = "glm_moe_dsa"
mock_model.config.first_k_dense_replace = 2
layers = get_predefined_ignore_layers(mock_model)
assert "layers.0.mlp" in layers
assert "layers.1.mlp" in layers
# The whole DSA indexer stays FP: its forward is @torch.no_grad() and
# transformers keeps indexer.weights_proj in _keep_in_fp32_modules.
assert "weights_proj" in layers
assert "indexer" in layers
def test_glm5_next_matcher(self):
"""GLM-5.3-Flash keeps its whole DSA indexer in full precision.
``Glm5NextTextIndexer.forward`` is ``@torch.no_grad()``, so quantizing its
``wq_b`` / ``wk`` / ``weights_proj`` linears makes the same compiled quant
function run under both grad modes and breaks the AOTAutograd backward.
The leading dense MLPs are excluded for the same vLLM reason as glm_moe_dsa.
"""
from auto_round.special_model_handler import get_predefined_ignore_layers
mock_model = MagicMock()
mock_model.config.model_type = "glm5_next"
mock_model.config.first_k_dense_replace = 3
layers = get_predefined_ignore_layers(mock_model)
assert "weights_proj" in layers
assert "indexer" in layers
assert "layers.0.mlp" in layers
assert "layers.1.mlp" in layers
assert "layers.2.mlp" in layers
def test_step3p5_matcher(self):
from auto_round.special_model_handler import get_predefined_ignore_layers
mock_model = MagicMock()
mock_model.config.model_type = "step3p5"
layers = get_predefined_ignore_layers(mock_model)
assert "g_proj" in layers
assert "moe.gate" in layers
assert "eh_proj" in layers
assert "shared_head" in layers
assert "layers.45" in layers
def test_kimi_k25_matcher(self):
from auto_round.special_model_handler import get_predefined_ignore_layers
mock_model = MagicMock()
mock_model.config.model_type = "kimi_k25"
layers = get_predefined_ignore_layers(mock_model)
assert "vision_tower" in layers
assert "mm_projector" in layers
def test_bagel_matcher(self, monkeypatch):
from auto_round.special_model_handler import get_predefined_ignore_layers
monkeypatch.delenv("AR_QUANTIZE_BAGEL_MOE_GEN", raising=False)
mock_model = MagicMock()
mock_model.config.model_type = "bagel"
mock_model.language_model.model.layers = [MagicMock() for _ in range(32)]
layers = get_predefined_ignore_layers(mock_model)
assert layers == ["moe_gen"]
def test_moe_model_via_config(self):
from auto_round.special_model_handler import get_predefined_ignore_layers
mock_model = MagicMock()
mock_model.config.model_type = "test_moe"
mock_model.config.architectures = ["TestMoE"]
mock_model.named_modules.return_value = iter([])
layers = get_predefined_ignore_layers(mock_model)
# Should not add any layers without matching rules
assert layers == []
def test_generic_moe_ignores_router_and_shared_expert_gates(self):
from types import SimpleNamespace
from auto_round.special_model_handler import get_predefined_ignore_layers
mock_model = MagicMock()
mock_model.config = SimpleNamespace(model_type="test_moe", architectures=[])
mock_model.named_modules.return_value = iter(
[
("layers.0.mlp.gate", MagicMock()),
("layers.0.mlp.shared_expert_gate", MagicMock()),
("layers.0.mlp.gate", MagicMock()),
("layers.0.mlp.up_proj", MagicMock()),
]
)
layers = get_predefined_ignore_layers(mock_model)
assert layers == ["layers.0.mlp.gate", "layers.0.mlp.shared_expert_gate"]
class TestTorchCompileOff:
"""DeepSeek / GLM-5.3-Flash DSA families must not run under torch.compile on torch < 2.14.0.
From torch 2.14.0 onward the recompile_limit issue is resolved, so these families no
longer force ``torch.compile`` off and instead honor the default / user-provided setting.
"""
def _cfg(self, model_type="", architectures=None):
from types import SimpleNamespace
return SimpleNamespace(config=SimpleNamespace(model_type=model_type, architectures=architectures or []))
def _expected_off(self):
from auto_round.utils import torch_version_at_least
# On torch >= 2.14.0 the rules are not registered, so the reason is ``None``.
return not torch_version_at_least("2.14.0")
def test_deepseek_disabled(self):
from auto_round.special_model_handler import get_torch_compile_off_reason
for mt in ("deepseek_v3", "deepseek_v32"):
off = get_torch_compile_off_reason(self._cfg(model_type=mt)) is not None
assert off == self._expected_off()
def test_glm5_family_disabled(self):
from auto_round.special_model_handler import get_torch_compile_off_reason
expected = self._expected_off()
assert (get_torch_compile_off_reason(self._cfg(model_type="glm5_next")) is not None) == expected
assert (get_torch_compile_off_reason(self._cfg(model_type="glm_moe_dsa")) is not None) == expected
# Architecture name alone is enough (config before weights are materialized).
assert (
get_torch_compile_off_reason(self._cfg(architectures=["Glm5NextForConditionalGeneration"])) is not None
) == expected
def test_other_models_unaffected(self):
from auto_round.special_model_handler import get_torch_compile_off_reason
assert get_torch_compile_off_reason(self._cfg(model_type="qwen3", architectures=["Qwen3ForCausalLM"])) is None
assert get_torch_compile_off_reason(self._cfg(model_type="glm4", architectures=["Glm4ForCausalLM"])) is None
assert get_torch_compile_off_reason(None) is None
class TestGetBagelIgnoreLayers:
"""Test get_bagel_ignore_layers function."""
@pytest.mark.parametrize(
("env_value", "expected"),
[(None, ["moe_gen"]), ("0", ["moe_gen"]), ("1", [])],
)
def test_moe_gen_policy(self, monkeypatch, env_value, expected):
from auto_round.special_model_handler import get_bagel_ignore_layers
if env_value is None:
monkeypatch.delenv("AR_QUANTIZE_BAGEL_MOE_GEN", raising=False)
else:
monkeypatch.setenv("AR_QUANTIZE_BAGEL_MOE_GEN", env_value)
assert get_bagel_ignore_layers(MagicMock()) == expected
class TestGetGlmFlashIgnoreLayers:
"""Test get_glm_flash_ignore_layers function."""
def test_default_num_dense_layer(self):
from auto_round.special_model_handler import get_glm_flash_ignore_layers
mock_model = MagicMock(spec=[])
layers = get_glm_flash_ignore_layers(mock_model)
assert "layers.0.mlp" in layers
def test_custom_num_dense_layer(self):
from auto_round.special_model_handler import get_glm_flash_ignore_layers
mock_model = MagicMock()
mock_model.config.first_k_dense_replace = 3
layers = get_glm_flash_ignore_layers(mock_model)
assert "layers.0.mlp" in layers
assert "layers.1.mlp" in layers
assert "layers.2.mlp" in layers
class TestGetPredefinedFixedAttr:
"""Test get_predefined_fixed_attr function."""
def test_gemma4_unified_returns_attrs(self):
from auto_round.special_model_handler import get_predefined_fixed_attr
mock_model = MagicMock()
mock_model.config.model_type = "gemma4_unified"
attrs = get_predefined_fixed_attr(mock_model)
assert attrs is not None
assert "has_variable_block_shape" in attrs
def test_unknown_model_type_returns_none(self):
from auto_round.special_model_handler import get_predefined_fixed_attr
mock_model = MagicMock()
mock_model.config.model_type = "unknown_model"
attrs = get_predefined_fixed_attr(mock_model)
assert attrs is None
def test_no_config_returns_none(self):
from auto_round.special_model_handler import get_predefined_fixed_attr
mock_model = MagicMock(spec=[])
del mock_model.config
attrs = get_predefined_fixed_attr(mock_model)
assert attrs is None
def test_config_without_model_type_returns_none(self):
from auto_round.special_model_handler import get_predefined_fixed_attr
mock_model = MagicMock()
mock_model.config.model_type = None
attrs = get_predefined_fixed_attr(mock_model)
assert attrs is None
class TestUpdateModule:
"""Test update_module function."""
def test_gguf_format_returns_unchanged(self):
from auto_round.formats import OutputFormat
from auto_round.special_model_handler import update_module
mock_model = MagicMock()
gguf_format = MagicMock(spec=OutputFormat)
gguf_format.is_gguf.return_value = True
result = update_module(mock_model, formats=[gguf_format])
assert result is mock_model
def test_non_gguf_format_applies_replacements(self):
from auto_round.formats import OutputFormat
from auto_round.special_model_handler import update_module
mock_model = MagicMock()
non_gguf_format = MagicMock(spec=OutputFormat)
non_gguf_format.is_gguf.return_value = False
result = update_module(mock_model, formats=[non_gguf_format])
# The function should call apply_replacements
assert result is not None
def test_no_formats_applies_replacements(self):
from auto_round.special_model_handler import update_module
mock_model = MagicMock()
result = update_module(mock_model, formats=None)
# The function should call apply_replacements
assert result is not None
class TestDeepseekVl2Forward:
"""Test _deepseek_vl2_forward function."""
def test_calls_prepare_inputs_embeds(self):
from auto_round.special_model_handler import _deepseek_vl2_forward
mock_model = MagicMock()
mock_model.prepare_inputs_embeds.return_value = torch.randn(1, 10, 128)
mock_model.language.return_value = MagicMock()
input_ids = torch.tensor([1, 2, 3])
_deepseek_vl2_forward(mock_model, input_ids=input_ids, images=None)
mock_model.prepare_inputs_embeds.assert_called_once()
mock_model.language.assert_called_once()
class TestQwen25OmniForward:
"""Test _qwen2_5_omni_forward function."""
def test_calls_thinker_forward(self):
from auto_round.special_model_handler import _qwen2_5_omni_forward
mock_model = MagicMock()
mock_model.thinker.return_value = MagicMock(hidden_states=[torch.randn(1, 10, 128)])
mock_model.has_talker = False
mock_model.thinker.get_input_embeddings.return_value = MagicMock(return_value=torch.randn(1, 10, 128))
input_ids = torch.tensor([1, 2, 3])
_qwen2_5_omni_forward(mock_model, input_ids=input_ids)
mock_model.thinker.assert_called_once()
class TestMimoAudioForward:
"""Test _mimo_audio_forward function."""
def test_converts_input_ids_to_embeds(self):
from auto_round.special_model_handler import _mimo_audio_forward
mock_model = MagicMock()
mock_model.model.embed_tokens.return_value = torch.randn(1, 10, 128)
mock_model.model.return_value = MagicMock()
input_ids = torch.tensor([[1, 2, 3]])
_mimo_audio_forward(mock_model, input_ids=input_ids)
mock_model.model.embed_tokens.assert_called_once_with(input_ids)
mock_model.model.assert_called_once()
class TestQwen3TtsForward:
"""Test _qwen3_tts_forward function."""
def test_uses_tts_model_backbone(self):
from auto_round.special_model_handler import _qwen3_tts_forward
mock_model = MagicMock(spec=[])
mock_tts_backbone = MagicMock()
mock_model.tts_model = mock_tts_backbone
mock_tts_backbone.model.text_embedding = MagicMock(return_value=torch.randn(1, 10, 128))
mock_tts_backbone.text_projection = MagicMock(return_value=torch.randn(1, 10, 128))
mock_tts_backbone.return_value = MagicMock()
input_ids = torch.tensor([[1, 2, 3]])
_qwen3_tts_forward(mock_model, input_ids=input_ids)
mock_tts_backbone.assert_called_once()
def test_uses_talker_backbone(self):
from auto_round.special_model_handler import _qwen3_tts_forward
mock_model = MagicMock(spec=[])
mock_talker = MagicMock()
mock_model.talker = mock_talker
mock_model.tts_model = None
mock_talker.model.text_embedding = MagicMock(return_value=torch.randn(1, 10, 128))
mock_talker.text_projection = MagicMock(return_value=torch.randn(1, 10, 128))
mock_talker.return_value = MagicMock()
input_ids = torch.tensor([[1, 2, 3]])
_qwen3_tts_forward(mock_model, input_ids=input_ids)
mock_talker.assert_called_once()
def test_raises_if_missing_text_embedding(self):
from auto_round.special_model_handler import _qwen3_tts_forward
mock_model = MagicMock(spec=[])
mock_tts_backbone = MagicMock()
mock_model.tts_model = mock_tts_backbone
mock_tts_backbone.model.text_embedding = None
mock_tts_backbone.text_projection = MagicMock()
mock_model.talker = None
input_ids = torch.tensor([[1, 2, 3]])
with pytest.raises(RuntimeError, match="missing text_embedding"):
_qwen3_tts_forward(mock_model, input_ids=input_ids)
class TestPredefinedIgnoreLayersRegistry:
"""Test _PRE_DEFINED_IGNORE_LAYERS global list has expected entries."""
def test_registry_has_longcat_rule(self):
from auto_round.special_model_handler import _PRE_DEFINED_IGNORE_LAYERS
assert len(_PRE_DEFINED_IGNORE_LAYERS) > 0
def test_registry_contains_multiple_rules(self):
from auto_round.special_model_handler import _PRE_DEFINED_IGNORE_LAYERS
assert len(_PRE_DEFINED_IGNORE_LAYERS) >= 5
class TestPredefinedFixedAttr:
"""Test _PRE_DEFINED_FIXED_ATTR global dict."""
def test_gemma4_unified_in_dict(self):
from auto_round.special_model_handler import _PRE_DEFINED_FIXED_ATTR
assert "gemma4_unified" in _PRE_DEFINED_FIXED_ATTR
class TestGemma4HelperFunctions:
"""Test Gemma4 helper functions."""
def test_get_gemma4_shared_kv_states_global_with_ref(self):
from auto_round.special_model_handler import _get_gemma4_shared_kv_states_global
mock_block = MagicMock()
mock_block._shared_kv_states_global_ref = {"test": "value"}
result = _get_gemma4_shared_kv_states_global(mock_block)
assert result == {"test": "value"}
def test_get_gemma4_shared_kv_states_global_without_ref(self):
from auto_round.special_model_handler import _get_gemma4_shared_kv_states_global
mock_block = MagicMock()
mock_block._shared_kv_states_global_ref = None
result = _get_gemma4_shared_kv_states_global(mock_block)
assert result == {}
def test_get_gemma4_rotary_emb_with_ref(self):
from auto_round.special_model_handler import _get_gemma4_rotary_emb
mock_block = MagicMock()
mock_rotary_emb = MagicMock()
mock_block._rotary_emb_ref = [mock_rotary_emb]
result = _get_gemma4_rotary_emb(mock_block)
assert result is mock_rotary_emb
def test_get_gemma4_rotary_emb_without_ref(self):
from auto_round.special_model_handler import _get_gemma4_rotary_emb
mock_block = MagicMock()
mock_block._rotary_emb_ref = None
mock_block._rotary_emb = "default_rotary"
result = _get_gemma4_rotary_emb(mock_block, default_rotary_emb="default_rotary")
assert result == "default_rotary"