Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

Reduce RAM usage of quantizing VLM models and fix some issues of quantizing gemma4 (#1791)

Liang Lv9696884c3de0
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py
index c48618c4a..6f5dc2fda 100644
--- a/auto_round/compressors/base.py
+++ b/auto_round/compressors/base.py
@@ -40,7 +40,7 @@ from auto_round.schemes import (
get_gguf_scheme,
preset_name_to_scheme,
)
-from auto_round.special_model_handler import get_predefined_fixed_attr, get_predefined_ignore_layers
+from auto_round.special_model_handler import get_predefined_ignore_layers, update_module
from auto_round.utils import (
INNER_SUPPORTED_LAYER_TYPES,
SUPPORTED_LAYER_TYPES,
@@ -338,9 +338,6 @@ class BaseCompressor(object):
# batch_size from kwargs) have already routed through it.
self.has_variable_block_shape = False
- fixed_attr = get_predefined_fixed_attr(self.model_context.model) or {}
- for key, value in fixed_attr.items():
- setattr(self, key, value)
# ── Scheme resolution ─────────────────────────────────────────────────────
diff --git a/auto_round/compressors/data_driven.py b/auto_round/compressors/data_driven.py
index c78e11d6e..efd69c8bc 100644
--- a/auto_round/compressors/data_driven.py
+++ b/auto_round/compressors/data_driven.py
@@ -645,6 +645,7 @@ class DataDrivenCompressor(BaseCompressor):
to_cache_block_names = [block[0] for block in all_blocks]
else:
to_cache_block_names = flatten_list(all_blocks)
+ _last_cache_name = to_cache_block_names[-1] if len(to_cache_block_names) > 1 else None
to_cache_layer_names = layer_names
if self.super_group_size is not None:
to_cache_layer_names = []
@@ -658,6 +659,7 @@ class DataDrivenCompressor(BaseCompressor):
to_cache_block_names,
self.nsamples,
to_cache_layer_names,
+ last_cache_name=_last_cache_name,
)
self.inputs = all_inputs
is_quantized_embedding = self._quantize_embedding_layer()
@@ -666,7 +668,9 @@ class DataDrivenCompressor(BaseCompressor):
if is_quantized_embedding:
all_inputs = copy.deepcopy(self.inputs)
clear_memory(self.inputs, device_list=self.compress_context.device_list)
- all_q_inputs = self.try_cache_inter_data_gpucpu(to_cache_block_names, self.nsamples, to_cache_layer_names)
+ all_q_inputs = self.try_cache_inter_data_gpucpu(
+ to_cache_block_names, self.nsamples, to_cache_layer_names, last_cache_name=_last_cache_name
+ )
# Remove accelerate dispatch hooks before moving parameters.
# hf_device_map is kept for reference but hooks are no longer needed.
if hasattr(self.model_context.model, "hf_device_map") and len(self.model_context.model.hf_device_map) > 1:
diff --git a/auto_round/compressors/utils.py b/auto_round/compressors/utils.py
index 59c3e5264..6cc00dd79 100644
--- a/auto_round/compressors/utils.py
+++ b/auto_round/compressors/utils.py
@@ -172,6 +172,11 @@ def block_forward(
if "alibi" in input_others.keys() and input_others["alibi"] is not None:
alibi = input_others["alibi"]
input_others["alibi"] = alibi.reshape(-1, alibi.shape[2], alibi.shape[3])
+
+ from auto_round.special_model_handler import prepare_special_model_block_inputs
+
+ input_others, input_tuple = prepare_special_model_block_inputs(block, input_ids, input_others, input_tuple)
+
if amp:
with autocast(device_type=str(device).split(":")[0], dtype=amp_dtype): # pragma: no cover
output = block(input_ids, *input_tuple, **input_others)
diff --git a/auto_round/special_model_handler.py b/auto_round/special_model_handler.py
index 68ac5b540..a23bac533 100644
--- a/auto_round/special_model_handler.py
+++ b/auto_round/special_model_handler.py
@@ -58,10 +58,138 @@ SPECIAL_SHARED_CACHE_KEYS = {
"Gemma3ForConditionalGeneration": ("position_embeddings_global", "position_embeddings_local")
}
SPECIAL_SHARED_CACHE_KEYS["MiniMaxText01ForCausalLM"] = ("slope_rate",)
+SPECIAL_SHARED_CACHE_KEYS["Gemma4ForConditionalGeneration"] = ("position_ids",)
SPECIAL_SHARED_CACHE_KEYS["WanTransformer3DModel"] = ("rotary_emb",)
MISTRAL_3_2_MODELS = ["Mistral-Small-3.2", "Magistral-Small", "Devstral-Small"]
+def _normalize_gemma4_per_layer_input(positional_inputs, hidden_states):
+ if positional_inputs is None or len(positional_inputs) == 0:
+ return positional_inputs
+
+ per_layer_input = positional_inputs[0]
+ if not isinstance(per_layer_input, torch.Tensor) or per_layer_input.shape[1] == hidden_states.shape[1]:
+ return positional_inputs
+
+ hs_seq = hidden_states.shape[1]
+ pl_seq = per_layer_input.shape[1]
+ if hs_seq <= pl_seq:
+ per_layer_input = per_layer_input[:, :hs_seq, :]
+ else:
+ pad = per_layer_input[:, -1:, :].expand(-1, hs_seq - pl_seq, -1)
+ per_layer_input = torch.cat([per_layer_input, pad], dim=1)
+
+ normalized_inputs = list(positional_inputs)
+ normalized_inputs[0] = per_layer_input
+ return type(positional_inputs)(normalized_inputs) if isinstance(positional_inputs, tuple) else normalized_inputs
+
+
+def prepare_special_model_block_inputs(block, rotary_input, input_others, positional_inputs=None):
+ """Rewrite replay inputs for blocks that need model-specific handling."""
+
+ special_replay_type = getattr(block, "_autoround_special_replay", None)
+ if special_replay_type == "gemma4":
+ prepared_inputs = _prepare_gemma4_replay_inputs(
+ block,
+ rotary_input,
+ position_ids=input_others.get("position_ids"),
+ position_embeddings=input_others.get("position_embeddings"),
+ attention_mask=input_others.get("attention_mask"),
+ shared_kv_states=input_others.get("shared_kv_states"),
+ past_key_values=input_others.get("past_key_values"),
+ config=getattr(block, "_gemma4_config_ref", None),
+ )
+ for key, value in prepared_inputs.items():
+ if value is not None or key in input_others or key == "shared_kv_states":
+ input_others[key] = value
+ positional_inputs = _normalize_gemma4_per_layer_input(positional_inputs, rotary_input)
+ return input_others, positional_inputs
+
+
+def _get_gemma4_rotary_emb(block, default_rotary_emb=None):
+ rotary_emb_ref = getattr(block, "_rotary_emb_ref", None)
+ if rotary_emb_ref:
+ return rotary_emb_ref[0]
+ return getattr(block, "_rotary_emb", default_rotary_emb)
+
+
+def _rebuild_gemma4_attention_mask(config, hidden_states, position_ids, layer_type, past_key_values=None):
+ from transformers.masking_utils import create_causal_mask, create_sliding_window_causal_mask
+
+ mask_builder = create_causal_mask if layer_type == "full_attention" else create_sliding_window_causal_mask
+ return mask_builder(
+ config=config,
+ inputs_embeds=hidden_states,
+ attention_mask=None,
+ past_key_values=past_key_values,
+ position_ids=position_ids,
+ )
+
+
+def _prepare_gemma4_replay_inputs(
+ block,
+ rotary_input,
+ *,
+ position_ids=None,
+ position_embeddings=None,
+ attention_mask=None,
+ shared_kv_states=None,
+ past_key_values=None,
+ config=None,
+ default_rotary_emb=None,
+ default_shared_kv_states=None,
+):
+ attn = getattr(block, "self_attn", None)
+ layer_type = getattr(attn, "layer_type", None)
+ head_dim = getattr(attn, "head_dim", None)
+
+ if attn is not None and hasattr(attn, "store_full_length_kv") and shared_kv_states is None:
+ shared_kv_states = default_shared_kv_states if default_shared_kv_states is not None else {}
+
+ need_position_embeddings = position_embeddings is None
+ if isinstance(position_embeddings, dict):
+ cached_position_embeddings = position_embeddings.get(layer_type) if layer_type is not None else None
+ need_position_embeddings = cached_position_embeddings is None
+ else:
+ cached_position_embeddings = position_embeddings
+
+ if (
+ not need_position_embeddings
+ and head_dim is not None
+ and isinstance(cached_position_embeddings, (tuple, list))
+ and cached_position_embeddings
+ ):
+ need_position_embeddings = cached_position_embeddings[0].shape[-1] != head_dim
+
+ if need_position_embeddings and layer_type is not None and position_ids is not None:
+ rotary_emb = _get_gemma4_rotary_emb(block, default_rotary_emb)
+ if rotary_emb is not None:
+ rebuilt_position_embeddings = rotary_emb(rotary_input, position_ids, layer_type)
+ if isinstance(position_embeddings, dict):
+ position_embeddings = dict(position_embeddings)
+ position_embeddings[layer_type] = rebuilt_position_embeddings
+ else:
+ position_embeddings = rebuilt_position_embeddings
+
+ if config is not None and layer_type is not None and position_ids is not None:
+ try:
+ attention_mask = _rebuild_gemma4_attention_mask(
+ config,
+ hidden_states=rotary_input,
+ position_ids=position_ids,
+ layer_type=layer_type,
+ past_key_values=past_key_values,
+ )
+ except Exception:
+ pass
+
+ return {
+ "position_embeddings": position_embeddings,
+ "attention_mask": attention_mask,
+ "shared_kv_states": shared_kv_states,
+ }
+
+
def _patch_gemma4_model(model):
"""Patch each Gemma4 decoder layer so it recomputes position_embeddings and
attention_mask from the cached position_ids when the cached versions have
@@ -75,7 +203,6 @@ def _patch_gemma4_model(model):
import types as _types
try:
- from transformers.masking_utils import create_causal_mask, create_sliding_window_causal_mask
from transformers.models.gemma4.modeling_gemma4 import Gemma4TextModel
except ImportError:
return model
@@ -96,6 +223,8 @@ def _patch_gemma4_model(model):
shared_kv_states_global = {}
for layer in text_model.layers:
+ object.__setattr__(layer, "_autoround_special_replay", "gemma4")
+ object.__setattr__(layer, "_gemma4_config_ref", text_model.config)
original_forward = layer.forward
layer_type = getattr(getattr(layer, "self_attn", None), "layer_type", None)
if layer_type is None:
@@ -119,20 +248,25 @@ def _patch_gemma4_model(model):
position_ids=None,
**kwargs,
):
- # Ensure shared_kv_states is always a dict (newer transformers
- # versions pass it as positional arg 3; None causes item assignment crash)
- if shared_kv_states is None:
- shared_kv_states = skv_global
- # Recompute position_embeddings when cached dim doesn't match
- if (
- hd is not None
- and position_embeddings is not None
- and isinstance(position_embeddings, (tuple, list))
- and len(position_embeddings) == 2
- ):
- cos, _ = position_embeddings
- if cos.shape[-1] != hd and position_ids is not None:
- position_embeddings = re(hidden_states, position_ids, lt)
+ # Rebuild Gemma4 layer-specific replay inputs from the minimal
+ # shared cache so later layers do not need variable block inputs.
+ if shared_kv_states is None and getattr(self, "layer_idx", None) == 0:
+ skv_global.clear()
+ prepared_inputs = _prepare_gemma4_replay_inputs(
+ self,
+ hidden_states,
+ position_ids=position_ids,
+ position_embeddings=position_embeddings,
+ attention_mask=attention_mask,
+ shared_kv_states=shared_kv_states,
+ past_key_values=kwargs.get("past_key_values"),
+ config=cfg,
+ default_rotary_emb=re,
+ default_shared_kv_states=skv_global,
+ )
+ position_embeddings = prepared_inputs["position_embeddings"]
+ attention_mask = prepared_inputs["attention_mask"]
+ shared_kv_states = prepared_inputs["shared_kv_states"]
# per_layer_input is token-specific but is cached as shared positional
# input (only 1st batch stored). Truncate/pad to match hidden_states seq_len.
@@ -145,22 +279,6 @@ def _patch_gemma4_model(model):
pad = per_layer_input[:, -1:, :].expand(-1, hs_seq - pl_seq, -1)
per_layer_input = torch.cat([per_layer_input, pad], dim=1)
- # Recompute attention_mask for full-attention layers when a
- # sliding-window mask was cached (it would be too restrictive)
- if is_full_attn and attention_mask is not None and position_ids is not None:
- # Only rebuild if the mask was created for a shorter context
- # (sliding window masks have finite bandwidth)
- try:
- attention_mask = create_causal_mask(
- config=cfg,
- inputs_embeds=hidden_states,
- attention_mask=None,
- past_key_values=kwargs.get("past_key_values"),
- position_ids=position_ids,
- )
- except Exception:
- pass # fall back to whatever was cached
-
if orig_has_shared_kv:
return orig_fwd(
hidden_states,
@@ -212,6 +330,13 @@ def _handle_special_model(model):
if version.parse(transformers.__version__) < version.parse("5.6"):
_patch_gemma4_model(model)
+ else:
+ _attach_gemma4_rotary_emb(model)
+ logger.warning(
+ "Applying a monkey patch to Gemma4 to reduce RAM usage. "
+ "This patch has only been validated with limited Transformers versions. "
+ "Proceed with caution."
+ )
return model
@@ -810,32 +935,34 @@ def get_predefined_ignore_layers(model: torch.nn.Module) -> list[str]:
return list(dict.fromkeys(layers))
-# Maps model_type -> fixed attributes to set on the BaseCompressor instance.
-# Only used when transformers >= 5.6, which natively supports per-block
-# position_embedding recomputation (has_variable_block_shape).
-_PRE_DEFINED_FIXED_ATTR = {"gemma4": {"has_variable_block_shape": True}}
+def _attach_gemma4_rotary_emb(model):
+ """Attach ``_rotary_emb`` to each Gemma4 decoder layer.
+ For transformers >= 5.6 the per-layer forward patch is unnecessary, but
+ ``block_forward`` still needs access to ``rotary_emb`` (which lives on the
+ parent ``Gemma4TextModel``) to recompute ``position_embeddings`` when the
+ cached version from block 0 has the wrong dimension.
+ """
+ try:
+ from transformers.models.gemma4.modeling_gemma4 import Gemma4TextModel
+ except ImportError:
+ return
-def get_predefined_fixed_attr(model: torch.nn.Module) -> dict | None:
- """Return fixed compressor attributes for models that need special caching.
+ text_model = None
+ for _, submodule in model.named_modules():
+ if isinstance(submodule, Gemma4TextModel):
+ text_model = submodule
+ break
- For Gemma4 with transformers >= 5.6, each decoder block must cache its own
- inputs because sliding vs full-attention layers require different
- position_embeddings. Returns ``None`` for older transformers, which instead
- rely on the per-layer forward patch applied in ``_handle_special_model``.
- """
- import transformers
- from packaging import version
+ if text_model is None:
+ return
- config = getattr(model, "config", None)
- if config is None or not hasattr(config, "model_type"):
- return None
- attrs = _PRE_DEFINED_FIXED_ATTR.get(config.model_type)
- if attrs is None:
- return None
- if version.parse(transformers.__version__) >= version.parse("5.6"):
- return attrs
- return None
+ for layer in text_model.layers:
+ # Store in a plain list to prevent nn.Module from registering rotary_emb
+ # as a child submodule (which would cause meta-tensor errors during .to(device)).
+ object.__setattr__(layer, "_rotary_emb_ref", [text_model.rotary_emb])
+ object.__setattr__(layer, "_autoround_special_replay", "gemma4")
+ object.__setattr__(layer, "_gemma4_config_ref", text_model.config)
def load_next_step_diffusion(pretrained_model_name_or_path, device_str):
diff --git a/test/test_cpu/models/test_vlm_ram_reduction.py b/test/test_cpu/models/test_vlm_ram_reduction.py
new file mode 100644
index 000000000..9846b4857
--- /dev/null
+++ b/test/test_cpu/models/test_vlm_ram_reduction.py
@@ -0,0 +1,589 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Unit tests for VLM RAM reduction changes.
+
+These tests target the specific logic changes introduced by the VLM RAM fix:
+1. Per-sample-constant tensor skipping in block forward cache
+2. Diffusion multi-device memory reservation
+3. MLLM calibration memory cleanup
+4. last_cache_name RAM reduction
+
+All tests use fake model hierarchies and mocks — no real model weights
+are loaded or downloaded.
+"""
+
+import types
+from unittest.mock import MagicMock, patch
+
+import pytest
+import torch
+import torch.nn as nn
+
+from auto_round.utils.common import flatten_list
+
+# ---------------------------------------------------------------------------
+# 1. Tests for per-sample-constant tensor skipping in _get_block_forward_func
+# ---------------------------------------------------------------------------
+
+
+class TestBlockForwardTensorSkipping:
+ """Unit tests for the per-sample-constant tensor skipping logic in calib.py.
+
+ The fix skips caching non-hidden_states tensor args that are per-sample
+ constants (e.g. position_embeddings, attention_mask) to avoid accumulating
+ batch_size * nsamples items in RAM.
+ """
+
+ def test_hidden_states_tensor_is_cached(self):
+ """hidden_states must always be cached regardless of shared_cache_keys."""
+ # Simulate the new logic: skip only if:
+ # 1. key != "hidden_states"
+ # 2. isinstance(tensor)
+ # 3. key not in shared_cache_keys
+ key = "hidden_states"
+ is_tensor = True
+ shared_cache_keys = set()
+
+ should_skip = key != "hidden_states" and is_tensor and key not in shared_cache_keys
+ assert not should_skip, "hidden_states must not be skipped"
+
+ def test_per_sample_constant_tensor_is_skipped(self):
+ """Per-sample-constant tensors (e.g. attention_mask) must be skipped."""
+ key = "attention_mask"
+ is_tensor = True
+ shared_cache_keys = set()
+
+ should_skip = key != "hidden_states" and is_tensor and key not in shared_cache_keys
+ assert should_skip, "attention_mask (per-sample constant) must be skipped"
+
+ def test_position_embeddings_skipped_when_not_shared_no_variable(self):
+ """position_embeddings skipped when not in shared_cache_keys and no variable block shape."""
+ key = "position_embeddings"
+ is_tensor = True
+ shared_cache_keys = set() # empty = position_embeddings not shared
+ has_variable_block_shape = False
+
+ should_skip = (
+ key != "hidden_states" and is_tensor and key not in shared_cache_keys and not has_variable_block_shape
+ )
+ assert should_skip, "position_embeddings must be skipped when not shared and no variable block shape"
+
+ def test_position_embeddings_kept_when_variable_shape(self):
+ """position_embeddings not skipped when has_variable_block_shape=True (cached per-block)."""
+ key = "position_embeddings"
+ is_tensor = True
+ shared_cache_keys = set() # not in shared_cache_keys
+ has_variable_block_shape = True
+
+ should_skip = (
+ key != "hidden_states" and is_tensor and key not in shared_cache_keys and not has_variable_block_shape
+ )
+ assert not should_skip, "position_embeddings must be cached per-block when variable block shape is enabled"
+
+ def test_position_embeddings_kept_when_shared(self):
+ """position_embeddings in shared_cache_keys must not be skipped (non-variable path)."""
+ key = "position_embeddings"
+ is_tensor = True
+ shared_cache_keys = {"position_embeddings"}
+ has_variable_block_shape = False
+
+ should_skip = (
+ key != "hidden_states" and is_tensor and key not in shared_cache_keys and not has_variable_block_shape
+ )
+ assert not should_skip, "position_embeddings must be kept when in shared_cache_keys"
+
+ def test_hidden_states_always_cached_variable_shape(self):
+ """hidden_states must always be cached even when has_variable_block_shape=True."""
+ key = "hidden_states"
+ is_tensor = True
+ shared_cache_keys = set()
+ has_variable_block_shape = True
+
+ should_skip = (
+ key != "hidden_states" and is_tensor and key not in shared_cache_keys and not has_variable_block_shape
+ )
+ assert not should_skip, "hidden_states must never be skipped"
+
+
+class TestGemma4PositionEmbeddingReplay:
+ def _make_fake_block(self, layer_type="full_attention", head_dim=512):
+ """Helper to build a fake Gemma4-like block with _rotary_emb attached."""
+
+ class FakeRotaryEmbedding:
+ def __call__(self, input_ids, position_ids, lt):
+ hd = 512 if lt == "full_attention" else 256
+ shape = (*input_ids.shape, hd)
+ return torch.ones(shape), torch.ones(shape)
+
+ class FakeAttention(nn.Module):
+ def __init__(self, lt, hd):
+ super().__init__()
+ self.layer_type = lt
+ self.head_dim = hd
+
+ class FakeBlock(nn.Module):
+ def __init__(self, lt, hd):
+ super().__init__()
+ self.self_attn = FakeAttention(lt, hd)
+
+ def forward(self, input_ids, **kwargs):
+ return kwargs["position_embeddings"][0]
+
+ block = FakeBlock(layer_type, head_dim)
+ block._rotary_emb_ref = [FakeRotaryEmbedding()]
+ block._autoround_special_replay = "gemma4"
+ return block
+
+ def test_recomputes_when_position_embeddings_missing(self):
+ """position_embeddings absent → should be recomputed from position_ids."""
+ from auto_round.compressors.utils import block_forward
+
+ block = self._make_fake_block("full_attention", 512)
+ input_ids = torch.zeros((1, 4), dtype=torch.float32)
+ position_ids = torch.arange(4).unsqueeze(0)
+ output = block_forward(
+ block,
+ input_ids,
+ {"position_ids": position_ids, "positional_inputs": []},
+ device=torch.device("cpu"),
+ )
+ assert output.shape[-1] == 512
+
+ def test_recomputes_when_position_embeddings_shape_mismatches(self):
+ """position_embeddings present but wrong dim (cached from sliding layer) → recompute."""
+ from auto_round.compressors.utils import block_forward
+
+ block = self._make_fake_block("full_attention", 512)
+ input_ids = torch.zeros((1, 4), dtype=torch.float32)
+ position_ids = torch.arange(4).unsqueeze(0)
+ # Simulate cached position_embeddings from a sliding_attention layer (dim=256)
+ wrong_pe = (torch.ones(1, 4, 256), torch.ones(1, 4, 256))
+ output = block_forward(
+ block,
+ input_ids,
+ {"position_ids": position_ids, "position_embeddings": wrong_pe, "positional_inputs": []},
+ device=torch.device("cpu"),
+ )
+ # Should have been recomputed to dim=512
+ assert output.shape[-1] == 512
+
+ def test_keeps_position_embeddings_when_shape_matches(self):
+ """position_embeddings present with correct dim → no recompute."""
+ from auto_round.compressors.utils import block_forward
+
+ block = self._make_fake_block("sliding_attention", 256)
+ input_ids = torch.zeros((1, 4), dtype=torch.float32)
+ position_ids = torch.arange(4).unsqueeze(0)
+ # Correct dim for sliding_attention
+ correct_pe = (torch.full((1, 4, 256), 42.0), torch.full((1, 4, 256), 42.0))
+ output = block_forward(
+ block,
+ input_ids,
+ {"position_ids": position_ids, "position_embeddings": correct_pe, "positional_inputs": []},
+ device=torch.device("cpu"),
+ )
+ # Should use the original (value=42), not recomputed (value=1)
+ assert output.shape[-1] == 256
+ assert torch.allclose(output, torch.full((1, 4, 256), 42.0))
+
+
+class TestGemma4ReplayInputHelper:
+ def _make_fake_block(self, layer_type="full_attention", head_dim=512, store_full_length_kv=True):
+ class FakeRotaryEmbedding:
+ def __call__(self, input_states, position_ids, lt):
+ hd = 512 if lt == "full_attention" else 256
+ shape = (*input_states.shape, hd)
+ return torch.ones(shape), torch.ones(shape)
+
+ class FakeAttention(nn.Module):
+ def __init__(self, lt, hd, store_kv):
+ super().__init__()
+ self.layer_type = lt
+ self.head_dim = hd
+ if store_kv:
+ self.store_full_length_kv = True
+
+ class FakeBlock(nn.Module):
+ def __init__(self, lt, hd, store_kv):
+ super().__init__()
+ self.self_attn = FakeAttention(lt, hd, store_kv)
+
+ block = FakeBlock(layer_type, head_dim, store_full_length_kv)
+ block._rotary_emb_ref = [FakeRotaryEmbedding()]
+ block._autoround_special_replay = "gemma4"
+ return block
+
+ def test_injects_shared_kv_and_rebuilds_missing_position_embeddings(self):
+ from auto_round.special_model_handler import _prepare_gemma4_replay_inputs
+
+ block = self._make_fake_block("full_attention", 512, store_full_length_kv=True)
+ hidden_states = torch.zeros((1, 4, 8), dtype=torch.float32)
+ position_ids = torch.arange(4).unsqueeze(0)
+ shared_kv_states = {}
+
+ prepared = _prepare_gemma4_replay_inputs(
+ block,
+ hidden_states,
+ position_ids=position_ids,
+ default_shared_kv_states=shared_kv_states,
+ )
+
+ assert prepared["shared_kv_states"] is shared_kv_states
+ assert prepared["position_embeddings"][0].shape[-1] == 512
+
+ def test_rebuilds_attention_mask_for_sliding_layers(self):
+ from auto_round.special_model_handler import _prepare_gemma4_replay_inputs
+
+ block = self._make_fake_block("sliding_attention", 256, store_full_length_kv=False)
+ hidden_states = torch.zeros((1, 4, 8), dtype=torch.float32)
+ position_ids = torch.arange(4).unsqueeze(0)
+ expected_mask = torch.full((1, 1, 4, 4), 7.0)
+
+ with patch(
+ "auto_round.special_model_handler._rebuild_gemma4_attention_mask",
+ return_value=expected_mask,
+ ) as rebuild_mask:
+ prepared = _prepare_gemma4_replay_inputs(
+ block,
+ hidden_states,
+ position_ids=position_ids,
+ attention_mask=None,
+ config=types.SimpleNamespace(),
+ )
+
+ rebuild_mask.assert_called_once()
+ assert torch.equal(prepared["attention_mask"], expected_mask)
+
+ def test_keeps_other_layer_types_when_position_embeddings_is_dict(self):
+ from auto_round.special_model_handler import _prepare_gemma4_replay_inputs
+
+ block = self._make_fake_block("full_attention", 512, store_full_length_kv=False)
+ hidden_states = torch.zeros((1, 4, 8), dtype=torch.float32)
+ position_ids = torch.arange(4).unsqueeze(0)
+ cached_dict = {"sliding_attention": (torch.full((1, 4, 256), 3.0), torch.full((1, 4, 256), 3.0))}
+
+ prepared = _prepare_gemma4_replay_inputs(
+ block,
+ hidden_states,
+ position_ids=position_ids,
+ position_embeddings=cached_dict,
+ )
+
+ assert prepared["position_embeddings"]["sliding_attention"][0].shape[-1] == 256
+ assert prepared["position_embeddings"]["full_attention"][0].shape[-1] == 512
+
+
+class TestSpecialModelReplayDispatch:
+ def test_non_special_block_is_untouched(self):
+ from auto_round.special_model_handler import prepare_special_model_block_inputs
+
+ class PlainBlock(nn.Module):
+ pass
+
+ block = PlainBlock()
+ input_others = {"attention_mask": torch.ones(1, 1, 4, 4), "positional_inputs": []}
+ output, positional_inputs = prepare_special_model_block_inputs(block, torch.zeros((1, 4, 8)), input_others, [])
+
+ assert output is input_others
+ assert positional_inputs == []
+ assert torch.equal(output["attention_mask"], torch.ones(1, 1, 4, 4))
+
+ def test_gemma4_dispatch_rebuilds_mask_and_normalizes_per_layer_input(self):
+ from auto_round.special_model_handler import prepare_special_model_block_inputs
+
+ class FakeRotaryEmbedding:
+ def __call__(self, input_ids, position_ids, lt):
+ hd = 512 if lt == "full_attention" else 256
+ shape = (*input_ids.shape, hd)
+ return torch.ones(shape), torch.ones(shape)
+
+ class FakeAttention(nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.layer_type = "full_attention"
+ self.head_dim = 512
+
+ class FakeBlock(nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.self_attn = FakeAttention()
+ self._rotary_emb_ref = [FakeRotaryEmbedding()]
+ self._autoround_special_replay = "gemma4"
+ self._gemma4_config_ref = types.SimpleNamespace()
+
+ block = FakeBlock()
+ hidden_states = torch.zeros((1, 4, 8), dtype=torch.float32)
+ position_ids = torch.arange(4).unsqueeze(0)
+ per_layer_input = torch.ones((1, 2, 8), dtype=torch.float32)
+ expected_mask = torch.full((1, 1, 4, 4), 9.0)
+
+ with patch(
+ "auto_round.special_model_handler._rebuild_gemma4_attention_mask",
+ return_value=expected_mask,
+ ) as rebuild_mask:
+ updated_inputs, updated_positional = prepare_special_model_block_inputs(
+ block,
+ hidden_states,
+ {"position_ids": position_ids},
+ [per_layer_input],
+ )
+
+ rebuild_mask.assert_called_once()
+ assert torch.equal(updated_inputs["attention_mask"], expected_mask)
+ assert updated_positional[0].shape[1] == hidden_states.shape[1]
+
+
+class TestLegacyBlockForwardDispatch:
+ def test_legacy_block_forward_uses_special_model_dispatch(self):
+ from auto_round.compressors.utils import block_forward
+
+ class FakeRotaryEmbedding:
+ def __call__(self, input_ids, position_ids, lt):
+ hd = 512 if lt == "full_attention" else 256
+ shape = (*input_ids.shape, hd)
+ return torch.ones(shape), torch.ones(shape)
+
+ class FakeAttention(nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.layer_type = "full_attention"
+ self.head_dim = 512
+
+ class FakeBlock(nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.self_attn = FakeAttention()
+ self._rotary_emb_ref = [FakeRotaryEmbedding()]
+ self._autoround_special_replay = "gemma4"
+
+ def forward(self, input_ids, **kwargs):
+ return kwargs["position_embeddings"][0]
+
+ block = FakeBlock()
+ input_ids = torch.zeros((1, 4), dtype=torch.float32)
+ position_ids = torch.arange(4).unsqueeze(0)
+
+ output = block_forward(
+ block,
+ input_ids,
+ {"position_ids": position_ids, "positional_inputs": []},
+ device=torch.device("cpu"),
+ )
+
+ assert output.shape[-1] == 512
+
+
+# ---------------------------------------------------------------------------
+# 2. Tests for diffusion multi-device memory reservation
+# ---------------------------------------------------------------------------
+
+
+class TestDiffusionMultiDeviceDispatch:
+ """Unit tests for diffusion multi-device memory reservation.
+
+ The fix reserves memory on the primary device for non-target pipeline
+ components (text encoder, VAE) before computing the balanced device map,
+ avoiding OOM when the main model is also dispatched to that device.
+ """
+
+ def test_reserve_memory_on_primary_device(self):
+ """Non-target components must reserve memory on the primary device."""
+ from auto_round.utils.device import dispatch_model_by_all_available_devices
+
+ # Build a fake pipeline with two components: main + auxiliary
+ class FakeComponent(nn.Module):
+ def __init__(self, param_count):
+ # Each param is 2 bytes (bf16), total = param_count * 2 bytes
+ super().__init__()
+ self.register_parameter("p", nn.Parameter(torch.empty(param_count, dtype=torch.bfloat16)))
+
+ class FakePipe(nn.Module):
+ def __init__(self):
+ super().__init__()
+ # Main model: 1000 params (2KB)
+ self.transformer = FakeComponent(1000)
+ # Non-main: 100 params (0.2KB)
+ self.vae = FakeComponent(100)
+
+ pipe = FakePipe()
+ pipe.components = {"transformer": pipe.transformer, "vae": pipe.vae}
+
+ with patch("torch.cuda.is_available", return_value=False):
+ with patch("torch.cuda.device_count", return_value=0):
+ # Single device path (falls back to pipe.to)
+ result = dispatch_model_by_all_available_devices(pipe, "cpu")
+ assert result is pipe
+
+ def test_multi_device_respects_non_main_memory(self):
+ """With multiple devices, memory reservation must be computed for non-main components."""
+ from auto_round.utils.device import dispatch_model_by_all_available_devices
+
+ class FakeComponent(nn.Module):
+ def __init__(self, param_count):
+ super().__init__()
+ self.register_parameter("p", nn.Parameter(torch.empty(param_count, dtype=torch.bfloat16)))
+
+ class FakePipe(nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.transformer = FakeComponent(1000) # main: 1000 * 2 = 2000 bytes
+ self.text_encoder = FakeComponent(100) # non-main: 100 * 2 = 200 bytes
+ self.vae = FakeComponent(50) # non-main: 50 * 2 = 100 bytes
+ # total non-main: 300 params = 600 bytes * 1.2 buffer = 720 bytes
+
+ pipe = FakePipe()
+ pipe.components = {
+ "transformer": pipe.transformer,
+ "text_encoder": pipe.text_encoder,
+ "vae": pipe.vae,
+ }
+
+ # We can't easily test the full multi-device path without accelerate mocks,
+ # but we can verify the non_main_bytes calculation logic in isolation
+ non_main_bytes = 0
+ for attr, component in pipe.components.items():
+ if attr == "transformer":
+ continue
+ if isinstance(component, nn.Module):
+ non_main_bytes += sum(p.numel() * p.element_size() for p in component.parameters())
+ non_main_reserved = int(non_main_bytes * 1.2)
+ assert non_main_reserved > 0, "non-main memory must be computed"
+
+
+# ---------------------------------------------------------------------------
+# 3. Tests for MLLM calibration memory cleanup
+# ---------------------------------------------------------------------------
+
+
+class TestMLLMCalibMemoryCleanup:
+ """Unit tests for MLLM calibration memory cleanup.
+
+ The fix adds gc.collect() + clear_memory() after each forward pass in
+ MLLMMixin.calib to reduce memory fragmentation. Also re-raises OOM so
+ the try_cache_inter_data_gpucpu handler can switch to CPU.
+ """
+
+ def test_get_calibrator_kind_returns_mllm(self):
+ """MLLMMixin._get_calibrator_kind must return 'mllm' so the right calibrator is used."""
+ from auto_round.compressors.mllm_mixin import MLLMMixin
+
+ class DummyCompressor(MLLMMixin):
+ def __init__(self):
+ # Only initialise what MLLMMixin needs — skip super().__init__
+ # to avoid model loading side-effects.
+ self.template = "qwen2_vl"
+ self.extra_data_dir = None
+ self.quant_nontext_module = False
+ self.template_obj = None
+
+ compressor = DummyCompressor()
+ assert compressor._get_calibrator_kind() == "mllm"
+
+ def test_quant_nontext_module_batch_size_accumulation(self):
+ """quant_nontext_module=True must reset batch_size=1 and accumulate gradient steps."""
+ # Directly test the batch_size accumulation logic from MLLMMixin.__init__.
+ kwargs = {"batch_size": 4, "gradient_accumulate_steps": 2}
+
+ batch_size = kwargs.get("batch_size", 8)
+ grad_acc = kwargs.get("gradient_accumulate_steps", 1)
+ new_grad_acc = batch_size * grad_acc
+ kwargs["gradient_accumulate_steps"] = new_grad_acc
+ kwargs["batch_size"] = 1
+
+ assert kwargs["batch_size"] == 1
+ assert kwargs["gradient_accumulate_steps"] == 8
+
+
+# ---------------------------------------------------------------------------
+# 3. Tests for last_cache_name RAM reduction
+# ---------------------------------------------------------------------------
+
+
+class TestLastCacheNameRAMReduction:
+ """Unit tests for last_cache_name logic that limits caching to the first block.
+
+ The fix passes last_cache_name=block_names[0] to try_cache_inter_data_gpucpu
+ so that only the first block's inputs are cached, reducing RAM from
+ O(n_blocks * nsamples) to O(nsamples).
+ """
+
+ def test_should_stop_after_first_block(self):
+ """With last_cache_name set to the first block, caching must stop there."""
+
+ class FakeCompressor:
+ def __init__(self):
+ self.last_cache_name = "model.layers.0"
+ self._cache_target_set = {"model.layers.0"}
+ self._cache_seen_targets = set()
+
+ def _should_stop_cache_forward(self, name):
+ if name == self.last_cache_name:
+ return True
+ if self.last_cache_name is not None:
+ return False
+ if not hasattr(self, "_cache_target_set") or not hasattr(self, "_cache_seen_targets"):
+ return False
+ if name in self._cache_target_set:
+ self._cache_seen_targets.add(name)
+ if not self._cache_target_set.issubset(self._cache_seen_targets):
+ return False
+ self.last_cache_name = name
+ return True
+
+ c = FakeCompressor()
+ assert c._should_stop_cache_forward("model.layers.0") is True
+ assert c.last_cache_name == "model.layers.0"
+
+ def test_without_last_cache_name_all_blocks_cached(self):
+ """Without last_cache_name, all blocks must be cached before stopping."""
+
+ class FakeCompressor:
+ def __init__(self):
+ self.last_cache_name = None
+ self._cache_target_set = {"model.layers.0", "model.layers.1"}
+ self._cache_seen_targets = set()
+
+ def _should_stop_cache_forward(self, name):
+ if name == self.last_cache_name:
+ return True
+ if self.last_cache_name is not None:
+ return False
+ if not hasattr(self, "_cache_target_set") or not hasattr(self, "_cache_seen_targets"):
+ return False
+ if name in self._cache_target_set:
+ self._cache_seen_targets.add(name)
+ if not self._cache_target_set.issubset(self._cache_seen_targets):
+ return False
+ self.last_cache_name = name
+ return True
+
+ c = FakeCompressor()
+ assert c._should_stop_cache_forward("model.layers.0") is False
+ assert c._should_stop_cache_forward("model.layers.1") is True
+
+ def test_last_cache_name_logic_in_quantize(self):
+ """Verify that last_cache_name is set to first block when there are multiple blocks."""
+ all_blocks = [["model.layers.0", "model.layers.1"], ["model.layers.2"]]
+ to_cache_block_names = [b[0] for b in all_blocks]
+ assert to_cache_block_names == ["model.layers.0", "model.layers.2"]
+
+ # Without variable block shape: last_cache_name = first block
+ _last_cache_name = to_cache_block_names[0] if len(to_cache_block_names) > 1 else None
+ assert _last_cache_name == "model.layers.0"
+
+ single_group = [["model.layers.0"]]
+ single_cache = [b[0] for b in single_group]
+ assert len(single_cache) == 1
+ _last_cache_name_single = single_cache[0] if len(single_cache) > 1 else None
+ assert _last_cache_name_single is None