Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

Fix gemma4 crash issue during quantizing (#1860)

Liang Lv2bb9042e9705
diff --git a/auto_round/compressors/utils.py b/auto_round/compressors/utils.py
index eda8b749e..b31202c22 100644
--- a/auto_round/compressors/utils.py
+++ b/auto_round/compressors/utils.py
@@ -196,27 +196,6 @@ def block_forward(
input_others[param_name] = val
positional_inputs = ()
- # Guard: ensure position_ids is a tensor, not a list or None.
- if "position_ids" in input_others:
- pid = input_others["position_ids"]
- if isinstance(pid, list):
- if len(pid) == 1:
- input_others["position_ids"] = pid[0]
- elif len(pid) == 0:
- # Generate position_ids from hidden_states shape when it's empty.
- input_others["position_ids"] = (
- torch.arange(input_ids.shape[1], device=input_ids.device, dtype=torch.long)
- .unsqueeze(0)
- .expand(input_ids.shape[0], -1)
- )
- elif pid is None:
- # Generate position_ids from hidden_states shape when it's None.
- input_others["position_ids"] = (
- torch.arange(input_ids.shape[1], device=input_ids.device, dtype=torch.long)
- .unsqueeze(0)
- .expand(input_ids.shape[0], -1)
- )
-
if amp:
with autocast(device_type=str(device).split(":")[0], dtype=amp_dtype): # pragma: no cover
output = block(**input_others)
diff --git a/auto_round/special_model_handler.py b/auto_round/special_model_handler.py
index e6079fbed..4bdd5b2e2 100644
--- a/auto_round/special_model_handler.py
+++ b/auto_round/special_model_handler.py
@@ -95,6 +95,25 @@ def _normalize_gemma4_per_layer_input(positional_inputs, hidden_states):
def prepare_special_model_block_inputs(block, rotary_input, input_others, positional_inputs=None):
"""Rewrite replay inputs for blocks that need model-specific handling."""
+ # Guard: ensure position_ids is a tensor, not a list or None.
+ if "position_ids" in input_others:
+ pid = input_others["position_ids"]
+ if isinstance(pid, list):
+ if len(pid) == 1:
+ input_others["position_ids"] = pid[0]
+ elif len(pid) == 0:
+ input_others["position_ids"] = (
+ torch.arange(rotary_input.shape[1], device=rotary_input.device, dtype=torch.long)
+ .unsqueeze(0)
+ .expand(rotary_input.shape[0], -1)
+ )
+ elif pid is None:
+ input_others["position_ids"] = (
+ torch.arange(rotary_input.shape[1], device=rotary_input.device, dtype=torch.long)
+ .unsqueeze(0)
+ .expand(rotary_input.shape[0], -1)
+ )
+
special_replay_type = getattr(block, "_autoround_special_replay", None)
if special_replay_type == "gemma4":
prepared_inputs = _prepare_gemma4_replay_inputs(
diff --git a/test/test_cpu/models/test_gemma4_special_handler.py b/test/test_cpu/models/test_gemma4_special_handler.py
new file mode 100644
index 000000000..61abd39d7
--- /dev/null
+++ b/test/test_cpu/models/test_gemma4_special_handler.py
@@ -0,0 +1,633 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Unit tests for Gemma4 special model handler changes.
+
+Covers:
+1. _get_gemma4_shared_kv_states_global
+2. position_ids guard in prepare_special_model_block_inputs
+3. _attach_gemma4_rotary_emb (transformers >= 5.6 path)
+4. _patch_gemma4_model (transformers < 5.6 path)
+5. Gemma4 position embedding replay (block_forward level)
+6. _prepare_gemma4_replay_inputs helper
+7. prepare_special_model_block_inputs Gemma4 dispatch
+8. block_forward Gemma4 integration
+
+Uses real Gemma4TextModel from transformers where available; falls back to
+mocks for untestable paths.
+"""
+
+import types
+from unittest.mock import patch
+
+import pytest
+import torch
+import torch.nn as nn
+
+# ---------------------------------------------------------------------------
+# 1. Tests for _get_gemma4_shared_kv_states_global
+# ---------------------------------------------------------------------------
+
+
+class TestGemma4SharedKvStatesGlobal:
+ """Unit tests for _get_gemma4_shared_kv_states_global."""
+
+ def test_returns_shared_ref_when_attached(self):
+ """Returns the attached _shared_kv_states_global_ref."""
+ from auto_round.special_model_handler import _get_gemma4_shared_kv_states_global
+
+ class FakeBlock(nn.Module):
+ pass
+
+ block = FakeBlock()
+ shared_dict = {}
+ object.__setattr__(block, "_shared_kv_states_global_ref", shared_dict)
+
+ result = _get_gemma4_shared_kv_states_global(block)
+ assert result is shared_dict
+
+ def test_returns_empty_dict_when_no_ref(self):
+ """Returns a fresh {} when no ref is attached."""
+ from auto_round.special_model_handler import _get_gemma4_shared_kv_states_global
+
+ class FakeBlock(nn.Module):
+ pass
+
+ block = FakeBlock()
+ result = _get_gemma4_shared_kv_states_global(block)
+ assert result == {}
+
+ def test_shared_dict_is_mutated_across_layers(self):
+ """The same dict reference is returned for multiple layers."""
+ from auto_round.special_model_handler import _get_gemma4_shared_kv_states_global
+
+ class FakeBlock(nn.Module):
+ pass
+
+ shared_dict = {}
+ block0 = FakeBlock()
+ block1 = FakeBlock()
+ object.__setattr__(block0, "_shared_kv_states_global_ref", shared_dict)
+ object.__setattr__(block1, "_shared_kv_states_global_ref", shared_dict)
+
+ result0 = _get_gemma4_shared_kv_states_global(block0)
+ result1 = _get_gemma4_shared_kv_states_global(block1)
+
+ assert result0 is result1 is shared_dict
+ # Mutating via one layer is visible to the other
+ result0["key"] = "value"
+ assert result1["key"] == "value"
+
+
+# ---------------------------------------------------------------------------
+# 2. Tests for position_ids guard in prepare_special_model_block_inputs
+# ---------------------------------------------------------------------------
+
+
+class TestPrepareSpecialModelBlockInputsPositionIdsGuard:
+ """Unit tests for position_ids normalization in prepare_special_model_block_inputs.
+
+ These tests cover the defensive guard added to handle position_ids arriving
+ as a list or None instead of a tensor — which occurs in the MLLM calibration
+ pipeline on transformers >= 5.6.
+ """
+
+ def _make_fake_block(self):
+ class FakeRotaryEmbedding:
+ def __call__(self, input_ids, position_ids, lt):
+ return torch.ones(*input_ids.shape, 512), torch.ones(*input_ids.shape, 512)
+
+ class FakeAttention(nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.layer_type = "full_attention"
+ self.head_dim = 512
+
+ class FakeBlock(nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.self_attn = FakeAttention()
+
+ block = FakeBlock()
+ block._rotary_emb_ref = [FakeRotaryEmbedding()]
+ block._autoround_special_replay = "gemma4"
+ block.layer_idx = 0
+ return block
+
+ def test_position_ids_as_single_element_list_is_unwrapped(self):
+ """Single-element list [tensor] is unwrapped to bare tensor."""
+ from auto_round.special_model_handler import prepare_special_model_block_inputs
+
+ block = self._make_fake_block()
+ hidden_states = torch.zeros((1, 4, 8), dtype=torch.float32)
+ inner_tensor = torch.arange(4).unsqueeze(0)
+ input_others = {"position_ids": [inner_tensor], "positional_inputs": []}
+
+ updated_inputs, _ = prepare_special_model_block_inputs(
+ block, hidden_states, input_others, positional_inputs=None
+ )
+
+ assert updated_inputs["position_ids"] is inner_tensor
+ assert not isinstance(updated_inputs["position_ids"], list)
+
+ def test_position_ids_as_empty_list_is_rebuilt_from_seq_len(self):
+ """Empty list [] causes position_ids to be rebuilt from sequence length."""
+ from auto_round.special_model_handler import prepare_special_model_block_inputs
+
+ block = self._make_fake_block()
+ hidden_states = torch.zeros((2, 8, 8), dtype=torch.float32)
+ input_others = {"position_ids": [], "positional_inputs": []}
+
+ updated_inputs, _ = prepare_special_model_block_inputs(
+ block, hidden_states, input_others, positional_inputs=None
+ )
+
+ rebuilt = updated_inputs["position_ids"]
+ assert isinstance(rebuilt, torch.Tensor)
+ assert rebuilt.shape == (2, 8)
+ assert rebuilt[0].tolist() == list(range(8))
+
+ def test_position_ids_as_none_is_rebuilt_from_seq_len(self):
+ """position_ids=None causes position_ids to be rebuilt from sequence length."""
+ from auto_round.special_model_handler import prepare_special_model_block_inputs
+
+ block = self._make_fake_block()
+ hidden_states = torch.zeros((1, 6, 8), dtype=torch.float32)
+ input_others = {"position_ids": None, "positional_inputs": []}
+
+ updated_inputs, _ = prepare_special_model_block_inputs(
+ block, hidden_states, input_others, positional_inputs=None
+ )
+
+ rebuilt = updated_inputs["position_ids"]
+ assert isinstance(rebuilt, torch.Tensor)
+ assert rebuilt.shape == (1, 6)
+ assert rebuilt[0].tolist() == list(range(6))
+
+ def test_position_ids_as_tensor_passes_through_unchanged(self):
+ """Bare tensor position_ids is passed through without modification."""
+ from auto_round.special_model_handler import prepare_special_model_block_inputs
+
+ block = self._make_fake_block()
+ hidden_states = torch.zeros((1, 4, 8), dtype=torch.float32)
+ original = torch.arange(4).unsqueeze(0)
+ input_others = {"position_ids": original, "positional_inputs": []}
+
+ updated_inputs, _ = prepare_special_model_block_inputs(
+ block, hidden_states, input_others, positional_inputs=None
+ )
+
+ assert updated_inputs["position_ids"] is original
+
+
+# ---------------------------------------------------------------------------
+# 3. Tests for _attach_gemma4_rotary_emb (transformers >= 5.6 path)
+# ---------------------------------------------------------------------------
+
+
+class TestAttachGemma4RotaryEmb:
+ """Unit tests for _attach_gemma4_rotary_emb.
+
+ On transformers >= 5.6 this function attaches _rotary_emb_ref and
+ _shared_kv_states_global_ref to every Gemma4 layer.
+ """
+
+ @pytest.fixture
+ def gemma4_model(self):
+ """Minimal real Gemma4TextModel (transformers >= 5.6 path)."""
+ from transformers.models.gemma4.modeling_gemma4 import Gemma4TextConfig, Gemma4TextModel
+
+ config = Gemma4TextConfig(
+ num_hidden_layers=3,
+ num_attention_heads=2,
+ intermediate_size=8,
+ hidden_size=8,
+ )
+ model = Gemma4TextModel(config).cpu().eval()
+ wrapper = nn.Module()
+ wrapper.language_model = model
+ return wrapper
+
+ def test_attaches_rotary_emb_ref_to_all_layers(self, gemma4_model):
+ from auto_round.special_model_handler import _attach_gemma4_rotary_emb
+
+ _attach_gemma4_rotary_emb(gemma4_model)
+
+ text_model = gemma4_model.language_model
+ for i, layer in enumerate(text_model.layers):
+ ref = getattr(layer, "_rotary_emb_ref", None)
+ assert ref is not None, f"layer {i} missing _rotary_emb_ref"
+ assert ref[0] is text_model.rotary_emb
+
+ def test_attaches_shared_kv_states_global_ref_to_all_layers(self, gemma4_model):
+ from auto_round.special_model_handler import _attach_gemma4_rotary_emb
+
+ _attach_gemma4_rotary_emb(gemma4_model)
+
+ text_model = gemma4_model.language_model
+ shared_refs = [getattr(layer, "_shared_kv_states_global_ref", None) for layer in text_model.layers]
+ assert all(r is not None for r in shared_refs), "all layers must have _shared_kv_states_global_ref"
+ for i in range(1, len(shared_refs)):
+ assert shared_refs[0] is shared_refs[i], "layers must share the same dict"
+ shared_refs[0]["_test_marker"] = 99
+ for r in shared_refs[1:]:
+ assert r.get("_test_marker") == 99
+
+ def test_attaches_special_replay_marker_to_all_layers(self, gemma4_model):
+ from auto_round.special_model_handler import _attach_gemma4_rotary_emb
+
+ _attach_gemma4_rotary_emb(gemma4_model)
+
+ text_model = gemma4_model.language_model
+ for i, layer in enumerate(text_model.layers):
+ marker = getattr(layer, "_autoround_special_replay", None)
+ assert marker == "gemma4", f"layer {i} missing gemma4 marker"
+
+ def test_noops_when_gemma4textmodel_not_found(self):
+ from auto_round.special_model_handler import _attach_gemma4_rotary_emb
+
+ model = nn.Module()
+ result = _attach_gemma4_rotary_emb(model)
+ assert result is None
+
+
+# ---------------------------------------------------------------------------
+# 4. Tests for _patch_gemma4_model (transformers < 5.6 path)
+# ---------------------------------------------------------------------------
+
+
+class TestPatchGemma4Model:
+ """Unit tests for _patch_gemma4_model.
+
+ On transformers < 5.6 this function replaces each layer's forward with
+ a patched version that rebuilds position_embeddings and attention_mask.
+ """
+
+ @pytest.fixture
+ def gemma4_model(self):
+ """Minimal real Gemma4TextModel (transformers >= 5.6, but _patch_gemma4_model
+ is called for < 5.6 — we test the patching logic directly)."""
+ from transformers.models.gemma4.modeling_gemma4 import Gemma4TextConfig, Gemma4TextModel
+
+ config = Gemma4TextConfig(
+ num_hidden_layers=3,
+ num_attention_heads=2,
+ intermediate_size=8,
+ hidden_size=8,
+ )
+ model = Gemma4TextModel(config).cpu().eval()
+ wrapper = nn.Module()
+ wrapper.language_model = model
+ return wrapper
+
+ def test_patches_all_layers_with_special_replay_marker(self, gemma4_model):
+ from auto_round.special_model_handler import _patch_gemma4_model
+
+ result = _patch_gemma4_model(gemma4_model)
+
+ text_model = result.language_model
+ for i, layer in enumerate(text_model.layers):
+ marker = getattr(layer, "_autoround_special_replay", None)
+ assert marker == "gemma4", f"layer {i} missing gemma4 marker"
+
+ def test_patched_forward_accepts_position_ids_none(self):
+ """Patched forward must not crash when position_ids is None."""
+ from auto_round.special_model_handler import _patch_gemma4_model
+
+ class FakeRotaryEmb(nn.Module):
+ def __call__(self, x, position_ids, layer_type):
+ return torch.ones(x.shape[0], x.shape[1], 512), torch.ones(x.shape[0], x.shape[1], 512)
+
+ class FakeAttention(nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.layer_type = "full_attention"
+ self.head_dim = 512
+ self.store_full_length_kv = True
+
+ class FakeLayer(nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.self_attn = FakeAttention()
+ self.layer_idx = 0
+
+ def forward(self, hidden_states, **kwargs):
+ return hidden_states
+
+ class FakeTextModel(nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.rotary_emb = FakeRotaryEmb()
+ self.layers = nn.ModuleList([FakeLayer()])
+ self.config = types.SimpleNamespace()
+
+ model = nn.Module()
+ object.__setattr__(model, "language_model", FakeTextModel())
+
+ _patch_gemma4_model(model)
+
+ layer = model.language_model.layers[0]
+ hidden_states = torch.zeros((1, 4, 8))
+ try:
+ layer.forward(hidden_states, position_ids=None)
+ except AttributeError:
+ pytest.fail("Patched forward crashed on position_ids=None — guard missing or broken")
+
+ def test_noops_when_gemma4textmodel_not_found(self):
+ from auto_round.special_model_handler import _patch_gemma4_model
+
+ model = nn.Module()
+ result = _patch_gemma4_model(model)
+ assert result is model
+
+
+# ---------------------------------------------------------------------------
+# 5. Tests for Gemma4 position embedding replay (block_forward level)
+# ---------------------------------------------------------------------------
+
+
+class TestGemma4PositionEmbeddingReplay:
+ """Unit tests for Gemma4 position embedding recomputation in block_forward.
+
+ These test the full block_forward -> prepare_special_model_block_inputs ->
+ _prepare_gemma4_replay_inputs -> rotary_emb pipeline with fake blocks.
+ """
+
+ def _make_fake_block(self, layer_type="full_attention", head_dim=512):
+ """Helper to build a fake Gemma4-like block with _rotary_emb attached."""
+
+ class FakeRotaryEmbedding:
+ def __call__(self, input_ids, position_ids, lt):
+ hd = 512 if lt == "full_attention" else 256
+ shape = (*input_ids.shape, hd)
+ return torch.ones(shape), torch.ones(shape)
+
+ class FakeAttention(nn.Module):
+ def __init__(self, lt, hd):
+ super().__init__()
+ self.layer_type = lt
+ self.head_dim = hd
+
+ class FakeBlock(nn.Module):
+ def __init__(self, lt, hd):
+ super().__init__()
+ self.self_attn = FakeAttention(lt, hd)
+
+ def forward(self, input_ids, **kwargs):
+ return kwargs["position_embeddings"][0]
+
+ block = FakeBlock(layer_type, head_dim)
+ block._rotary_emb_ref = [FakeRotaryEmbedding()]
+ block._autoround_special_replay = "gemma4"
+ return block
+
+ def test_recomputes_when_position_embeddings_missing(self):
+ """position_embeddings absent → should be recomputed from position_ids."""
+ from auto_round.compressors.utils import block_forward
+
+ block = self._make_fake_block("full_attention", 512)
+ input_ids = torch.zeros((1, 4), dtype=torch.float32)
+ position_ids = torch.arange(4).unsqueeze(0)
+ output = block_forward(
+ block,
+ input_ids,
+ {"position_ids": position_ids, "positional_inputs": []},
+ device=torch.device("cpu"),
+ )
+ assert output.shape[-1] == 512
+
+ def test_recomputes_when_position_embeddings_shape_mismatches(self):
+ """position_embeddings present but wrong dim (cached from sliding layer) → recompute."""
+ from auto_round.compressors.utils import block_forward
+
+ block = self._make_fake_block("full_attention", 512)
+ input_ids = torch.zeros((1, 4), dtype=torch.float32)
+ position_ids = torch.arange(4).unsqueeze(0)
+ wrong_pe = (torch.ones(1, 4, 256), torch.ones(1, 4, 256))
+ output = block_forward(
+ block,
+ input_ids,
+ {"position_ids": position_ids, "position_embeddings": wrong_pe, "positional_inputs": []},
+ device=torch.device("cpu"),
+ )
+ assert output.shape[-1] == 512
+
+ def test_keeps_position_embeddings_when_shape_matches(self):
+ """position_embeddings present with correct dim → no recompute."""
+ from auto_round.compressors.utils import block_forward
+
+ block = self._make_fake_block("sliding_attention", 256)
+ input_ids = torch.zeros((1, 4), dtype=torch.float32)
+ position_ids = torch.arange(4).unsqueeze(0)
+ correct_pe = (torch.full((1, 4, 256), 42.0), torch.full((1, 4, 256), 42.0))
+ output = block_forward(
+ block,
+ input_ids,
+ {"position_ids": position_ids, "position_embeddings": correct_pe, "positional_inputs": []},
+ device=torch.device("cpu"),
+ )
+ assert output.shape[-1] == 256
+ assert torch.allclose(output, torch.full((1, 4, 256), 42.0))
+
+
+# ---------------------------------------------------------------------------
+# 6. Tests for _prepare_gemma4_replay_inputs helper
+# ---------------------------------------------------------------------------
+
+
+class TestGemma4ReplayInputHelper:
+ """Unit tests for _prepare_gemma4_replay_inputs."""
+
+ def _make_fake_block(self, layer_type="full_attention", head_dim=512, store_full_length_kv=True):
+ class FakeRotaryEmbedding:
+ def __call__(self, input_states, position_ids, lt):
+ hd = 512 if lt == "full_attention" else 256
+ shape = (*input_states.shape, hd)
+ return torch.ones(shape), torch.ones(shape)
+
+ class FakeAttention(nn.Module):
+ def __init__(self, lt, hd, store_kv):
+ super().__init__()
+ self.layer_type = lt
+ self.head_dim = hd
+ if store_kv:
+ self.store_full_length_kv = True
+
+ class FakeBlock(nn.Module):
+ def __init__(self, lt, hd, store_kv):
+ super().__init__()
+ self.self_attn = FakeAttention(lt, hd, store_kv)
+
+ block = FakeBlock(layer_type, head_dim, store_full_length_kv)
+ block._rotary_emb_ref = [FakeRotaryEmbedding()]
+ block._autoround_special_replay = "gemma4"
+ return block
+
+ def test_injects_shared_kv_and_rebuilds_missing_position_embeddings(self):
+ from auto_round.special_model_handler import _prepare_gemma4_replay_inputs
+
+ block = self._make_fake_block("full_attention", 512, store_full_length_kv=True)
+ hidden_states = torch.zeros((1, 4, 8), dtype=torch.float32)
+ position_ids = torch.arange(4).unsqueeze(0)
+ shared_kv_states = {}
+
+ prepared = _prepare_gemma4_replay_inputs(
+ block,
+ hidden_states,
+ position_ids=position_ids,
+ default_shared_kv_states=shared_kv_states,
+ )
+
+ assert prepared["shared_kv_states"] is shared_kv_states
+ assert prepared["position_embeddings"][0].shape[-1] == 512
+
+ def test_rebuilds_attention_mask_for_sliding_layers(self):
+ from auto_round.special_model_handler import _prepare_gemma4_replay_inputs
+
+ block = self._make_fake_block("sliding_attention", 256, store_full_length_kv=False)
+ hidden_states = torch.zeros((1, 4, 8), dtype=torch.float32)
+ position_ids = torch.arange(4).unsqueeze(0)
+ expected_mask = torch.full((1, 1, 4, 4), 7.0)
+
+ with patch(
+ "auto_round.special_model_handler._rebuild_gemma4_attention_mask",
+ return_value=expected_mask,
+ ) as rebuild_mask:
+ prepared = _prepare_gemma4_replay_inputs(
+ block,
+ hidden_states,
+ position_ids=position_ids,
+ attention_mask=None,
+ config=types.SimpleNamespace(),
+ )
+
+ rebuild_mask.assert_called_once()
+ assert torch.equal(prepared["attention_mask"], expected_mask)
+
+ def test_keeps_other_layer_types_when_position_embeddings_is_dict(self):
+ from auto_round.special_model_handler import _prepare_gemma4_replay_inputs
+
+ block = self._make_fake_block("full_attention", 512, store_full_length_kv=False)
+ hidden_states = torch.zeros((1, 4, 8), dtype=torch.float32)
+ position_ids = torch.arange(4).unsqueeze(0)
+ cached_dict = {"sliding_attention": (torch.full((1, 4, 256), 3.0), torch.full((1, 4, 256), 3.0))}
+
+ prepared = _prepare_gemma4_replay_inputs(
+ block,
+ hidden_states,
+ position_ids=position_ids,
+ position_embeddings=cached_dict,
+ )
+
+ assert prepared["position_embeddings"]["sliding_attention"][0].shape[-1] == 256
+ assert prepared["position_embeddings"]["full_attention"][0].shape[-1] == 512
+
+
+# ---------------------------------------------------------------------------
+# 7. Tests for prepare_special_model_block_inputs Gemma4 dispatch
+# ---------------------------------------------------------------------------
+
+
+class TestPrepareSpecialModelBlockInputsDispatch:
+ """Unit tests for Gemma4-specific dispatch in prepare_special_model_block_inputs."""
+
+ def test_gemma4_dispatch_rebuilds_mask_and_normalizes_per_layer_input(self):
+ from auto_round.special_model_handler import prepare_special_model_block_inputs
+
+ class FakeRotaryEmbedding:
+ def __call__(self, input_ids, position_ids, lt):
+ hd = 512 if lt == "full_attention" else 256
+ shape = (*input_ids.shape, hd)
+ return torch.ones(shape), torch.ones(shape)
+
+ class FakeAttention(nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.layer_type = "full_attention"
+ self.head_dim = 512
+
+ class FakeBlock(nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.self_attn = FakeAttention()
+ self._rotary_emb_ref = [FakeRotaryEmbedding()]
+ self._autoround_special_replay = "gemma4"
+ self._gemma4_config_ref = types.SimpleNamespace()
+
+ block = FakeBlock()
+ hidden_states = torch.zeros((1, 4, 8), dtype=torch.float32)
+ position_ids = torch.arange(4).unsqueeze(0)
+ per_layer_input = torch.ones((1, 2, 8), dtype=torch.float32)
+ expected_mask = torch.full((1, 1, 4, 4), 9.0)
+
+ with patch(
+ "auto_round.special_model_handler._rebuild_gemma4_attention_mask",
+ return_value=expected_mask,
+ ) as rebuild_mask:
+ updated_inputs, updated_positional = prepare_special_model_block_inputs(
+ block,
+ hidden_states,
+ {"position_ids": position_ids},
+ [per_layer_input],
+ )
+
+ rebuild_mask.assert_called_once()
+ assert torch.equal(updated_inputs["attention_mask"], expected_mask)
+ assert updated_positional[0].shape[1] == hidden_states.shape[1]
+
+
+# ---------------------------------------------------------------------------
+# 8. Tests for block_forward Gemma4 integration
+# ---------------------------------------------------------------------------
+
+
+class TestBlockForwardGemma4Dispatch:
+ """Unit tests for Gemma4 integration in block_forward."""
+
+ def test_legacy_block_forward_uses_special_model_dispatch(self):
+ from auto_round.compressors.utils import block_forward
+
+ class FakeRotaryEmbedding:
+ def __call__(self, input_ids, position_ids, lt):
+ hd = 512 if lt == "full_attention" else 256
+ shape = (*input_ids.shape, hd)
+ return torch.ones(shape), torch.ones(shape)
+
+ class FakeAttention(nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.layer_type = "full_attention"
+ self.head_dim = 512
+
+ class FakeBlock(nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.self_attn = FakeAttention()
+ self._rotary_emb_ref = [FakeRotaryEmbedding()]
+ self._autoround_special_replay = "gemma4"
+
+ def forward(self, input_ids, **kwargs):
+ return kwargs["position_embeddings"][0]
+
+ block = FakeBlock()
+ input_ids = torch.zeros((1, 4), dtype=torch.float32)
+ position_ids = torch.arange(4).unsqueeze(0)
+
+ output = block_forward(
+ block,
+ input_ids,
+ {"position_ids": position_ids, "positional_inputs": []},
+ device=torch.device("cpu"),
+ )
+
+ assert output.shape[-1] == 512
diff --git a/test/test_cpu/models/test_vlm_ram_reduction.py b/test/test_cpu/models/test_vlm_ram_reduction.py
index 9846b4857..2e78813d9 100644
--- a/test/test_cpu/models/test_vlm_ram_reduction.py
+++ b/test/test_cpu/models/test_vlm_ram_reduction.py
@@ -24,7 +24,7 @@ are loaded or downloaded.
"""
import types
-from unittest.mock import MagicMock, patch
+from unittest.mock import patch
import pytest
import torch
@@ -116,174 +116,6 @@ class TestBlockForwardTensorSkipping:
assert not should_skip, "hidden_states must never be skipped"
-class TestGemma4PositionEmbeddingReplay:
- def _make_fake_block(self, layer_type="full_attention", head_dim=512):
- """Helper to build a fake Gemma4-like block with _rotary_emb attached."""
-
- class FakeRotaryEmbedding:
- def __call__(self, input_ids, position_ids, lt):
- hd = 512 if lt == "full_attention" else 256
- shape = (*input_ids.shape, hd)
- return torch.ones(shape), torch.ones(shape)
-
- class FakeAttention(nn.Module):
- def __init__(self, lt, hd):
- super().__init__()
- self.layer_type = lt
- self.head_dim = hd
-
- class FakeBlock(nn.Module):
- def __init__(self, lt, hd):
- super().__init__()
- self.self_attn = FakeAttention(lt, hd)
-
- def forward(self, input_ids, **kwargs):
- return kwargs["position_embeddings"][0]
-
- block = FakeBlock(layer_type, head_dim)
- block._rotary_emb_ref = [FakeRotaryEmbedding()]
- block._autoround_special_replay = "gemma4"
- return block
-
- def test_recomputes_when_position_embeddings_missing(self):
- """position_embeddings absent → should be recomputed from position_ids."""
- from auto_round.compressors.utils import block_forward
-
- block = self._make_fake_block("full_attention", 512)
- input_ids = torch.zeros((1, 4), dtype=torch.float32)
- position_ids = torch.arange(4).unsqueeze(0)
- output = block_forward(
- block,
- input_ids,
- {"position_ids": position_ids, "positional_inputs": []},
- device=torch.device("cpu"),
- )
- assert output.shape[-1] == 512
-
- def test_recomputes_when_position_embeddings_shape_mismatches(self):
- """position_embeddings present but wrong dim (cached from sliding layer) → recompute."""
- from auto_round.compressors.utils import block_forward
-
- block = self._make_fake_block("full_attention", 512)
- input_ids = torch.zeros((1, 4), dtype=torch.float32)
- position_ids = torch.arange(4).unsqueeze(0)
- # Simulate cached position_embeddings from a sliding_attention layer (dim=256)
- wrong_pe = (torch.ones(1, 4, 256), torch.ones(1, 4, 256))
- output = block_forward(
- block,
- input_ids,
- {"position_ids": position_ids, "position_embeddings": wrong_pe, "positional_inputs": []},
- device=torch.device("cpu"),
- )
- # Should have been recomputed to dim=512
- assert output.shape[-1] == 512
-
- def test_keeps_position_embeddings_when_shape_matches(self):
- """position_embeddings present with correct dim → no recompute."""
- from auto_round.compressors.utils import block_forward
-
- block = self._make_fake_block("sliding_attention", 256)
- input_ids = torch.zeros((1, 4), dtype=torch.float32)
- position_ids = torch.arange(4).unsqueeze(0)
- # Correct dim for sliding_attention
- correct_pe = (torch.full((1, 4, 256), 42.0), torch.full((1, 4, 256), 42.0))
- output = block_forward(
- block,
- input_ids,
- {"position_ids": position_ids, "position_embeddings": correct_pe, "positional_inputs": []},
- device=torch.device("cpu"),
- )
- # Should use the original (value=42), not recomputed (value=1)
- assert output.shape[-1] == 256
- assert torch.allclose(output, torch.full((1, 4, 256), 42.0))
-
-
-class TestGemma4ReplayInputHelper:
- def _make_fake_block(self, layer_type="full_attention", head_dim=512, store_full_length_kv=True):
- class FakeRotaryEmbedding:
- def __call__(self, input_states, position_ids, lt):
- hd = 512 if lt == "full_attention" else 256
- shape = (*input_states.shape, hd)
- return torch.ones(shape), torch.ones(shape)
-
- class FakeAttention(nn.Module):
- def __init__(self, lt, hd, store_kv):
- super().__init__()
- self.layer_type = lt
- self.head_dim = hd
- if store_kv:
- self.store_full_length_kv = True
-
- class FakeBlock(nn.Module):
- def __init__(self, lt, hd, store_kv):
- super().__init__()
- self.self_attn = FakeAttention(lt, hd, store_kv)
-
- block = FakeBlock(layer_type, head_dim, store_full_length_kv)
- block._rotary_emb_ref = [FakeRotaryEmbedding()]
- block._autoround_special_replay = "gemma4"
- return block
-
- def test_injects_shared_kv_and_rebuilds_missing_position_embeddings(self):
- from auto_round.special_model_handler import _prepare_gemma4_replay_inputs
-
- block = self._make_fake_block("full_attention", 512, store_full_length_kv=True)
- hidden_states = torch.zeros((1, 4, 8), dtype=torch.float32)
- position_ids = torch.arange(4).unsqueeze(0)
- shared_kv_states = {}
-
- prepared = _prepare_gemma4_replay_inputs(
- block,
- hidden_states,
- position_ids=position_ids,
- default_shared_kv_states=shared_kv_states,
- )
-
- assert prepared["shared_kv_states"] is shared_kv_states
- assert prepared["position_embeddings"][0].shape[-1] == 512
-
- def test_rebuilds_attention_mask_for_sliding_layers(self):
- from auto_round.special_model_handler import _prepare_gemma4_replay_inputs
-
- block = self._make_fake_block("sliding_attention", 256, store_full_length_kv=False)
- hidden_states = torch.zeros((1, 4, 8), dtype=torch.float32)
- position_ids = torch.arange(4).unsqueeze(0)
- expected_mask = torch.full((1, 1, 4, 4), 7.0)
-
- with patch(
- "auto_round.special_model_handler._rebuild_gemma4_attention_mask",
- return_value=expected_mask,
- ) as rebuild_mask:
- prepared = _prepare_gemma4_replay_inputs(
- block,
- hidden_states,
- position_ids=position_ids,
- attention_mask=None,
- config=types.SimpleNamespace(),
- )
-
- rebuild_mask.assert_called_once()
- assert torch.equal(prepared["attention_mask"], expected_mask)
-
- def test_keeps_other_layer_types_when_position_embeddings_is_dict(self):
- from auto_round.special_model_handler import _prepare_gemma4_replay_inputs
-
- block = self._make_fake_block("full_attention", 512, store_full_length_kv=False)
- hidden_states = torch.zeros((1, 4, 8), dtype=torch.float32)
- position_ids = torch.arange(4).unsqueeze(0)
- cached_dict = {"sliding_attention": (torch.full((1, 4, 256), 3.0), torch.full((1, 4, 256), 3.0))}
-
- prepared = _prepare_gemma4_replay_inputs(
- block,
- hidden_states,
- position_ids=position_ids,
- position_embeddings=cached_dict,
- )
-
- assert prepared["position_embeddings"]["sliding_attention"][0].shape[-1] == 256
- assert prepared["position_embeddings"]["full_attention"][0].shape[-1] == 512
-
-
class TestSpecialModelReplayDispatch:
def test_non_special_block_is_untouched(self):
from auto_round.special_model_handler import prepare_special_model_block_inputs
@@ -299,90 +131,6 @@ class TestSpecialModelReplayDispatch:
assert positional_inputs == []
assert torch.equal(output["attention_mask"], torch.ones(1, 1, 4, 4))
- def test_gemma4_dispatch_rebuilds_mask_and_normalizes_per_layer_input(self):
- from auto_round.special_model_handler import prepare_special_model_block_inputs
-
- class FakeRotaryEmbedding:
- def __call__(self, input_ids, position_ids, lt):
- hd = 512 if lt == "full_attention" else 256
- shape = (*input_ids.shape, hd)
- return torch.ones(shape), torch.ones(shape)
-
- class FakeAttention(nn.Module):
- def __init__(self):
- super().__init__()
- self.layer_type = "full_attention"
- self.head_dim = 512
-
- class FakeBlock(nn.Module):
- def __init__(self):
- super().__init__()
- self.self_attn = FakeAttention()
- self._rotary_emb_ref = [FakeRotaryEmbedding()]
- self._autoround_special_replay = "gemma4"
- self._gemma4_config_ref = types.SimpleNamespace()
-
- block = FakeBlock()
- hidden_states = torch.zeros((1, 4, 8), dtype=torch.float32)
- position_ids = torch.arange(4).unsqueeze(0)
- per_layer_input = torch.ones((1, 2, 8), dtype=torch.float32)
- expected_mask = torch.full((1, 1, 4, 4), 9.0)
-
- with patch(
- "auto_round.special_model_handler._rebuild_gemma4_attention_mask",
- return_value=expected_mask,
- ) as rebuild_mask:
- updated_inputs, updated_positional = prepare_special_model_block_inputs(
- block,
- hidden_states,
- {"position_ids": position_ids},
- [per_layer_input],
- )
-
- rebuild_mask.assert_called_once()
- assert torch.equal(updated_inputs["attention_mask"], expected_mask)
- assert updated_positional[0].shape[1] == hidden_states.shape[1]
-
-
-class TestLegacyBlockForwardDispatch:
- def test_legacy_block_forward_uses_special_model_dispatch(self):
- from auto_round.compressors.utils import block_forward
-
- class FakeRotaryEmbedding:
- def __call__(self, input_ids, position_ids, lt):
- hd = 512 if lt == "full_attention" else 256
- shape = (*input_ids.shape, hd)
- return torch.ones(shape), torch.ones(shape)
-
- class FakeAttention(nn.Module):
- def __init__(self):
- super().__init__()
- self.layer_type = "full_attention"
- self.head_dim = 512
-
- class FakeBlock(nn.Module):
- def __init__(self):
- super().__init__()
- self.self_attn = FakeAttention()
- self._rotary_emb_ref = [FakeRotaryEmbedding()]
- self._autoround_special_replay = "gemma4"
-
- def forward(self, input_ids, **kwargs):
- return kwargs["position_embeddings"][0]
-
- block = FakeBlock()
- input_ids = torch.zeros((1, 4), dtype=torch.float32)
- position_ids = torch.arange(4).unsqueeze(0)
-
- output = block_forward(
- block,
- input_ids,
- {"position_ids": position_ids, "positional_inputs": []},
- device=torch.device("cpu"),
- )
-
- assert output.shape[-1] == 512
-
# ---------------------------------------------------------------------------
# 2. Tests for diffusion multi-device memory reservation