PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
Reduce RAM usage of quantizing VLM models and fix some issues of quantizing gemma4 (#1791)
Liang Lv9696884c3de0
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.pyindex c48618c4a..6f5dc2fda 100644--- a/auto_round/compressors/base.py+++ b/auto_round/compressors/base.py@@ -40,7 +40,7 @@ from auto_round.schemes import (get_gguf_scheme,preset_name_to_scheme,)-from auto_round.special_model_handler import get_predefined_fixed_attr, get_predefined_ignore_layers+from auto_round.special_model_handler import get_predefined_ignore_layers, update_modulefrom auto_round.utils import (INNER_SUPPORTED_LAYER_TYPES,SUPPORTED_LAYER_TYPES,@@ -338,9 +338,6 @@ class BaseCompressor(object):# batch_size from kwargs) have already routed through it.self.has_variable_block_shape = False- fixed_attr = get_predefined_fixed_attr(self.model_context.model) or {}- for key, value in fixed_attr.items():- setattr(self, key, value)# ── Scheme resolution ─────────────────────────────────────────────────────diff --git a/auto_round/compressors/data_driven.py b/auto_round/compressors/data_driven.pyindex c78e11d6e..efd69c8bc 100644--- a/auto_round/compressors/data_driven.py+++ b/auto_round/compressors/data_driven.py@@ -645,6 +645,7 @@ class DataDrivenCompressor(BaseCompressor):to_cache_block_names = [block[0] for block in all_blocks]else:to_cache_block_names = flatten_list(all_blocks)+ _last_cache_name = to_cache_block_names[-1] if len(to_cache_block_names) > 1 else Noneto_cache_layer_names = layer_namesif self.super_group_size is not None:to_cache_layer_names = []@@ -658,6 +659,7 @@ class DataDrivenCompressor(BaseCompressor):to_cache_block_names,self.nsamples,to_cache_layer_names,+ last_cache_name=_last_cache_name,)self.inputs = all_inputsis_quantized_embedding = self._quantize_embedding_layer()@@ -666,7 +668,9 @@ class DataDrivenCompressor(BaseCompressor):if is_quantized_embedding:all_inputs = copy.deepcopy(self.inputs)clear_memory(self.inputs, device_list=self.compress_context.device_list)- all_q_inputs = self.try_cache_inter_data_gpucpu(to_cache_block_names, self.nsamples, to_cache_layer_names)+ all_q_inputs = self.try_cache_inter_data_gpucpu(+ to_cache_block_names, self.nsamples, to_cache_layer_names, last_cache_name=_last_cache_name+ )# Remove accelerate dispatch hooks before moving parameters.# hf_device_map is kept for reference but hooks are no longer needed.if hasattr(self.model_context.model, "hf_device_map") and len(self.model_context.model.hf_device_map) > 1:diff --git a/auto_round/compressors/utils.py b/auto_round/compressors/utils.pyindex 59c3e5264..6cc00dd79 100644--- a/auto_round/compressors/utils.py+++ b/auto_round/compressors/utils.py@@ -172,6 +172,11 @@ def block_forward(if "alibi" in input_others.keys() and input_others["alibi"] is not None:alibi = input_others["alibi"]input_others["alibi"] = alibi.reshape(-1, alibi.shape[2], alibi.shape[3])++ from auto_round.special_model_handler import prepare_special_model_block_inputs++ input_others, input_tuple = prepare_special_model_block_inputs(block, input_ids, input_others, input_tuple)+if amp:with autocast(device_type=str(device).split(":")[0], dtype=amp_dtype): # pragma: no coveroutput = block(input_ids, *input_tuple, **input_others)diff --git a/auto_round/special_model_handler.py b/auto_round/special_model_handler.pyindex 68ac5b540..a23bac533 100644--- a/auto_round/special_model_handler.py+++ b/auto_round/special_model_handler.py@@ -58,10 +58,138 @@ SPECIAL_SHARED_CACHE_KEYS = {"Gemma3ForConditionalGeneration": ("position_embeddings_global", "position_embeddings_local")}SPECIAL_SHARED_CACHE_KEYS["MiniMaxText01ForCausalLM"] = ("slope_rate",)+SPECIAL_SHARED_CACHE_KEYS["Gemma4ForConditionalGeneration"] = ("position_ids",)SPECIAL_SHARED_CACHE_KEYS["WanTransformer3DModel"] = ("rotary_emb",)MISTRAL_3_2_MODELS = ["Mistral-Small-3.2", "Magistral-Small", "Devstral-Small"]+def _normalize_gemma4_per_layer_input(positional_inputs, hidden_states):+ if positional_inputs is None or len(positional_inputs) == 0:+ return positional_inputs++ per_layer_input = positional_inputs[0]+ if not isinstance(per_layer_input, torch.Tensor) or per_layer_input.shape[1] == hidden_states.shape[1]:+ return positional_inputs++ hs_seq = hidden_states.shape[1]+ pl_seq = per_layer_input.shape[1]+ if hs_seq <= pl_seq:+ per_layer_input = per_layer_input[:, :hs_seq, :]+ else:+ pad = per_layer_input[:, -1:, :].expand(-1, hs_seq - pl_seq, -1)+ per_layer_input = torch.cat([per_layer_input, pad], dim=1)++ normalized_inputs = list(positional_inputs)+ normalized_inputs[0] = per_layer_input+ return type(positional_inputs)(normalized_inputs) if isinstance(positional_inputs, tuple) else normalized_inputs+++def prepare_special_model_block_inputs(block, rotary_input, input_others, positional_inputs=None):+ """Rewrite replay inputs for blocks that need model-specific handling."""++ special_replay_type = getattr(block, "_autoround_special_replay", None)+ if special_replay_type == "gemma4":+ prepared_inputs = _prepare_gemma4_replay_inputs(+ block,+ rotary_input,+ position_ids=input_others.get("position_ids"),+ position_embeddings=input_others.get("position_embeddings"),+ attention_mask=input_others.get("attention_mask"),+ shared_kv_states=input_others.get("shared_kv_states"),+ past_key_values=input_others.get("past_key_values"),+ config=getattr(block, "_gemma4_config_ref", None),+ )+ for key, value in prepared_inputs.items():+ if value is not None or key in input_others or key == "shared_kv_states":+ input_others[key] = value+ positional_inputs = _normalize_gemma4_per_layer_input(positional_inputs, rotary_input)+ return input_others, positional_inputs+++def _get_gemma4_rotary_emb(block, default_rotary_emb=None):+ rotary_emb_ref = getattr(block, "_rotary_emb_ref", None)+ if rotary_emb_ref:+ return rotary_emb_ref[0]+ return getattr(block, "_rotary_emb", default_rotary_emb)+++def _rebuild_gemma4_attention_mask(config, hidden_states, position_ids, layer_type, past_key_values=None):+ from transformers.masking_utils import create_causal_mask, create_sliding_window_causal_mask++ mask_builder = create_causal_mask if layer_type == "full_attention" else create_sliding_window_causal_mask+ return mask_builder(+ config=config,+ inputs_embeds=hidden_states,+ attention_mask=None,+ past_key_values=past_key_values,+ position_ids=position_ids,+ )+++def _prepare_gemma4_replay_inputs(+ block,+ rotary_input,+ *,+ position_ids=None,+ position_embeddings=None,+ attention_mask=None,+ shared_kv_states=None,+ past_key_values=None,+ config=None,+ default_rotary_emb=None,+ default_shared_kv_states=None,+):+ attn = getattr(block, "self_attn", None)+ layer_type = getattr(attn, "layer_type", None)+ head_dim = getattr(attn, "head_dim", None)++ if attn is not None and hasattr(attn, "store_full_length_kv") and shared_kv_states is None:+ shared_kv_states = default_shared_kv_states if default_shared_kv_states is not None else {}++ need_position_embeddings = position_embeddings is None+ if isinstance(position_embeddings, dict):+ cached_position_embeddings = position_embeddings.get(layer_type) if layer_type is not None else None+ need_position_embeddings = cached_position_embeddings is None+ else:+ cached_position_embeddings = position_embeddings++ if (+ not need_position_embeddings+ and head_dim is not None+ and isinstance(cached_position_embeddings, (tuple, list))+ and cached_position_embeddings+ ):+ need_position_embeddings = cached_position_embeddings[0].shape[-1] != head_dim++ if need_position_embeddings and layer_type is not None and position_ids is not None:+ rotary_emb = _get_gemma4_rotary_emb(block, default_rotary_emb)+ if rotary_emb is not None:+ rebuilt_position_embeddings = rotary_emb(rotary_input, position_ids, layer_type)+ if isinstance(position_embeddings, dict):+ position_embeddings = dict(position_embeddings)+ position_embeddings[layer_type] = rebuilt_position_embeddings+ else:+ position_embeddings = rebuilt_position_embeddings++ if config is not None and layer_type is not None and position_ids is not None:+ try:+ attention_mask = _rebuild_gemma4_attention_mask(+ config,+ hidden_states=rotary_input,+ position_ids=position_ids,+ layer_type=layer_type,+ past_key_values=past_key_values,+ )+ except Exception:+ pass++ return {+ "position_embeddings": position_embeddings,+ "attention_mask": attention_mask,+ "shared_kv_states": shared_kv_states,+ }++def _patch_gemma4_model(model):"""Patch each Gemma4 decoder layer so it recomputes position_embeddings andattention_mask from the cached position_ids when the cached versions have@@ -75,7 +203,6 @@ def _patch_gemma4_model(model):import types as _typestry:- from transformers.masking_utils import create_causal_mask, create_sliding_window_causal_maskfrom transformers.models.gemma4.modeling_gemma4 import Gemma4TextModelexcept ImportError:return model@@ -96,6 +223,8 @@ def _patch_gemma4_model(model):shared_kv_states_global = {}for layer in text_model.layers:+ object.__setattr__(layer, "_autoround_special_replay", "gemma4")+ object.__setattr__(layer, "_gemma4_config_ref", text_model.config)original_forward = layer.forwardlayer_type = getattr(getattr(layer, "self_attn", None), "layer_type", None)if layer_type is None:@@ -119,20 +248,25 @@ def _patch_gemma4_model(model):position_ids=None,**kwargs,):- # Ensure shared_kv_states is always a dict (newer transformers- # versions pass it as positional arg 3; None causes item assignment crash)- if shared_kv_states is None:- shared_kv_states = skv_global- # Recompute position_embeddings when cached dim doesn't match- if (- hd is not None- and position_embeddings is not None- and isinstance(position_embeddings, (tuple, list))- and len(position_embeddings) == 2- ):- cos, _ = position_embeddings- if cos.shape[-1] != hd and position_ids is not None:- position_embeddings = re(hidden_states, position_ids, lt)+ # Rebuild Gemma4 layer-specific replay inputs from the minimal+ # shared cache so later layers do not need variable block inputs.+ if shared_kv_states is None and getattr(self, "layer_idx", None) == 0:+ skv_global.clear()+ prepared_inputs = _prepare_gemma4_replay_inputs(+ self,+ hidden_states,+ position_ids=position_ids,+ position_embeddings=position_embeddings,+ attention_mask=attention_mask,+ shared_kv_states=shared_kv_states,+ past_key_values=kwargs.get("past_key_values"),+ config=cfg,+ default_rotary_emb=re,+ default_shared_kv_states=skv_global,+ )+ position_embeddings = prepared_inputs["position_embeddings"]+ attention_mask = prepared_inputs["attention_mask"]+ shared_kv_states = prepared_inputs["shared_kv_states"]# per_layer_input is token-specific but is cached as shared positional# input (only 1st batch stored). Truncate/pad to match hidden_states seq_len.@@ -145,22 +279,6 @@ def _patch_gemma4_model(model):pad = per_layer_input[:, -1:, :].expand(-1, hs_seq - pl_seq, -1)per_layer_input = torch.cat([per_layer_input, pad], dim=1)- # Recompute attention_mask for full-attention layers when a- # sliding-window mask was cached (it would be too restrictive)- if is_full_attn and attention_mask is not None and position_ids is not None:- # Only rebuild if the mask was created for a shorter context- # (sliding window masks have finite bandwidth)- try:- attention_mask = create_causal_mask(- config=cfg,- inputs_embeds=hidden_states,- attention_mask=None,- past_key_values=kwargs.get("past_key_values"),- position_ids=position_ids,- )- except Exception:- pass # fall back to whatever was cached-if orig_has_shared_kv:return orig_fwd(hidden_states,@@ -212,6 +330,13 @@ def _handle_special_model(model):if version.parse(transformers.__version__) < version.parse("5.6"):_patch_gemma4_model(model)+ else:+ _attach_gemma4_rotary_emb(model)+ logger.warning(+ "Applying a monkey patch to Gemma4 to reduce RAM usage. "+ "This patch has only been validated with limited Transformers versions. "+ "Proceed with caution."+ )return model@@ -810,32 +935,34 @@ def get_predefined_ignore_layers(model: torch.nn.Module) -> list[str]:return list(dict.fromkeys(layers))-# Maps model_type -> fixed attributes to set on the BaseCompressor instance.-# Only used when transformers >= 5.6, which natively supports per-block-# position_embedding recomputation (has_variable_block_shape).-_PRE_DEFINED_FIXED_ATTR = {"gemma4": {"has_variable_block_shape": True}}+def _attach_gemma4_rotary_emb(model):+ """Attach ``_rotary_emb`` to each Gemma4 decoder layer.+ For transformers >= 5.6 the per-layer forward patch is unnecessary, but+ ``block_forward`` still needs access to ``rotary_emb`` (which lives on the+ parent ``Gemma4TextModel``) to recompute ``position_embeddings`` when the+ cached version from block 0 has the wrong dimension.+ """+ try:+ from transformers.models.gemma4.modeling_gemma4 import Gemma4TextModel+ except ImportError:+ return-def get_predefined_fixed_attr(model: torch.nn.Module) -> dict | None:- """Return fixed compressor attributes for models that need special caching.+ text_model = None+ for _, submodule in model.named_modules():+ if isinstance(submodule, Gemma4TextModel):+ text_model = submodule+ break- For Gemma4 with transformers >= 5.6, each decoder block must cache its own- inputs because sliding vs full-attention layers require different- position_embeddings. Returns ``None`` for older transformers, which instead- rely on the per-layer forward patch applied in ``_handle_special_model``.- """- import transformers- from packaging import version+ if text_model is None:+ return- config = getattr(model, "config", None)- if config is None or not hasattr(config, "model_type"):- return None- attrs = _PRE_DEFINED_FIXED_ATTR.get(config.model_type)- if attrs is None:- return None- if version.parse(transformers.__version__) >= version.parse("5.6"):- return attrs- return None+ for layer in text_model.layers:+ # Store in a plain list to prevent nn.Module from registering rotary_emb+ # as a child submodule (which would cause meta-tensor errors during .to(device)).+ object.__setattr__(layer, "_rotary_emb_ref", [text_model.rotary_emb])+ object.__setattr__(layer, "_autoround_special_replay", "gemma4")+ object.__setattr__(layer, "_gemma4_config_ref", text_model.config)def load_next_step_diffusion(pretrained_model_name_or_path, device_str):diff --git a/test/test_cpu/models/test_vlm_ram_reduction.py b/test/test_cpu/models/test_vlm_ram_reduction.pynew file mode 100644index 000000000..9846b4857--- /dev/null+++ b/test/test_cpu/models/test_vlm_ram_reduction.py@@ -0,0 +1,589 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+"""Unit tests for VLM RAM reduction changes.++These tests target the specific logic changes introduced by the VLM RAM fix:+1. Per-sample-constant tensor skipping in block forward cache+2. Diffusion multi-device memory reservation+3. MLLM calibration memory cleanup+4. last_cache_name RAM reduction++All tests use fake model hierarchies and mocks — no real model weights+are loaded or downloaded.+"""++import types+from unittest.mock import MagicMock, patch++import pytest+import torch+import torch.nn as nn++from auto_round.utils.common import flatten_list++# ---------------------------------------------------------------------------+# 1. Tests for per-sample-constant tensor skipping in _get_block_forward_func+# ---------------------------------------------------------------------------+++class TestBlockForwardTensorSkipping:+ """Unit tests for the per-sample-constant tensor skipping logic in calib.py.++ The fix skips caching non-hidden_states tensor args that are per-sample+ constants (e.g. position_embeddings, attention_mask) to avoid accumulating+ batch_size * nsamples items in RAM.+ """++ def test_hidden_states_tensor_is_cached(self):+ """hidden_states must always be cached regardless of shared_cache_keys."""+ # Simulate the new logic: skip only if:+ # 1. key != "hidden_states"+ # 2. isinstance(tensor)+ # 3. key not in shared_cache_keys+ key = "hidden_states"+ is_tensor = True+ shared_cache_keys = set()++ should_skip = key != "hidden_states" and is_tensor and key not in shared_cache_keys+ assert not should_skip, "hidden_states must not be skipped"++ def test_per_sample_constant_tensor_is_skipped(self):+ """Per-sample-constant tensors (e.g. attention_mask) must be skipped."""+ key = "attention_mask"+ is_tensor = True+ shared_cache_keys = set()++ should_skip = key != "hidden_states" and is_tensor and key not in shared_cache_keys+ assert should_skip, "attention_mask (per-sample constant) must be skipped"++ def test_position_embeddings_skipped_when_not_shared_no_variable(self):+ """position_embeddings skipped when not in shared_cache_keys and no variable block shape."""+ key = "position_embeddings"+ is_tensor = True+ shared_cache_keys = set() # empty = position_embeddings not shared+ has_variable_block_shape = False++ should_skip = (+ key != "hidden_states" and is_tensor and key not in shared_cache_keys and not has_variable_block_shape+ )+ assert should_skip, "position_embeddings must be skipped when not shared and no variable block shape"++ def test_position_embeddings_kept_when_variable_shape(self):+ """position_embeddings not skipped when has_variable_block_shape=True (cached per-block)."""+ key = "position_embeddings"+ is_tensor = True+ shared_cache_keys = set() # not in shared_cache_keys+ has_variable_block_shape = True++ should_skip = (+ key != "hidden_states" and is_tensor and key not in shared_cache_keys and not has_variable_block_shape+ )+ assert not should_skip, "position_embeddings must be cached per-block when variable block shape is enabled"++ def test_position_embeddings_kept_when_shared(self):+ """position_embeddings in shared_cache_keys must not be skipped (non-variable path)."""+ key = "position_embeddings"+ is_tensor = True+ shared_cache_keys = {"position_embeddings"}+ has_variable_block_shape = False++ should_skip = (+ key != "hidden_states" and is_tensor and key not in shared_cache_keys and not has_variable_block_shape+ )+ assert not should_skip, "position_embeddings must be kept when in shared_cache_keys"++ def test_hidden_states_always_cached_variable_shape(self):+ """hidden_states must always be cached even when has_variable_block_shape=True."""+ key = "hidden_states"+ is_tensor = True+ shared_cache_keys = set()+ has_variable_block_shape = True++ should_skip = (+ key != "hidden_states" and is_tensor and key not in shared_cache_keys and not has_variable_block_shape+ )+ assert not should_skip, "hidden_states must never be skipped"+++class TestGemma4PositionEmbeddingReplay:+ def _make_fake_block(self, layer_type="full_attention", head_dim=512):+ """Helper to build a fake Gemma4-like block with _rotary_emb attached."""++ class FakeRotaryEmbedding:+ def __call__(self, input_ids, position_ids, lt):+ hd = 512 if lt == "full_attention" else 256+ shape = (*input_ids.shape, hd)+ return torch.ones(shape), torch.ones(shape)++ class FakeAttention(nn.Module):+ def __init__(self, lt, hd):+ super().__init__()+ self.layer_type = lt+ self.head_dim = hd++ class FakeBlock(nn.Module):+ def __init__(self, lt, hd):+ super().__init__()+ self.self_attn = FakeAttention(lt, hd)++ def forward(self, input_ids, **kwargs):+ return kwargs["position_embeddings"][0]++ block = FakeBlock(layer_type, head_dim)+ block._rotary_emb_ref = [FakeRotaryEmbedding()]+ block._autoround_special_replay = "gemma4"+ return block++ def test_recomputes_when_position_embeddings_missing(self):+ """position_embeddings absent → should be recomputed from position_ids."""+ from auto_round.compressors.utils import block_forward++ block = self._make_fake_block("full_attention", 512)+ input_ids = torch.zeros((1, 4), dtype=torch.float32)+ position_ids = torch.arange(4).unsqueeze(0)+ output = block_forward(+ block,+ input_ids,+ {"position_ids": position_ids, "positional_inputs": []},+ device=torch.device("cpu"),+ )+ assert output.shape[-1] == 512++ def test_recomputes_when_position_embeddings_shape_mismatches(self):+ """position_embeddings present but wrong dim (cached from sliding layer) → recompute."""+ from auto_round.compressors.utils import block_forward++ block = self._make_fake_block("full_attention", 512)+ input_ids = torch.zeros((1, 4), dtype=torch.float32)+ position_ids = torch.arange(4).unsqueeze(0)+ # Simulate cached position_embeddings from a sliding_attention layer (dim=256)+ wrong_pe = (torch.ones(1, 4, 256), torch.ones(1, 4, 256))+ output = block_forward(+ block,+ input_ids,+ {"position_ids": position_ids, "position_embeddings": wrong_pe, "positional_inputs": []},+ device=torch.device("cpu"),+ )+ # Should have been recomputed to dim=512+ assert output.shape[-1] == 512++ def test_keeps_position_embeddings_when_shape_matches(self):+ """position_embeddings present with correct dim → no recompute."""+ from auto_round.compressors.utils import block_forward++ block = self._make_fake_block("sliding_attention", 256)+ input_ids = torch.zeros((1, 4), dtype=torch.float32)+ position_ids = torch.arange(4).unsqueeze(0)+ # Correct dim for sliding_attention+ correct_pe = (torch.full((1, 4, 256), 42.0), torch.full((1, 4, 256), 42.0))+ output = block_forward(+ block,+ input_ids,+ {"position_ids": position_ids, "position_embeddings": correct_pe, "positional_inputs": []},+ device=torch.device("cpu"),+ )+ # Should use the original (value=42), not recomputed (value=1)+ assert output.shape[-1] == 256+ assert torch.allclose(output, torch.full((1, 4, 256), 42.0))+++class TestGemma4ReplayInputHelper:+ def _make_fake_block(self, layer_type="full_attention", head_dim=512, store_full_length_kv=True):+ class FakeRotaryEmbedding:+ def __call__(self, input_states, position_ids, lt):+ hd = 512 if lt == "full_attention" else 256+ shape = (*input_states.shape, hd)+ return torch.ones(shape), torch.ones(shape)++ class FakeAttention(nn.Module):+ def __init__(self, lt, hd, store_kv):+ super().__init__()+ self.layer_type = lt+ self.head_dim = hd+ if store_kv:+ self.store_full_length_kv = True++ class FakeBlock(nn.Module):+ def __init__(self, lt, hd, store_kv):+ super().__init__()+ self.self_attn = FakeAttention(lt, hd, store_kv)++ block = FakeBlock(layer_type, head_dim, store_full_length_kv)+ block._rotary_emb_ref = [FakeRotaryEmbedding()]+ block._autoround_special_replay = "gemma4"+ return block++ def test_injects_shared_kv_and_rebuilds_missing_position_embeddings(self):+ from auto_round.special_model_handler import _prepare_gemma4_replay_inputs++ block = self._make_fake_block("full_attention", 512, store_full_length_kv=True)+ hidden_states = torch.zeros((1, 4, 8), dtype=torch.float32)+ position_ids = torch.arange(4).unsqueeze(0)+ shared_kv_states = {}++ prepared = _prepare_gemma4_replay_inputs(+ block,+ hidden_states,+ position_ids=position_ids,+ default_shared_kv_states=shared_kv_states,+ )++ assert prepared["shared_kv_states"] is shared_kv_states+ assert prepared["position_embeddings"][0].shape[-1] == 512++ def test_rebuilds_attention_mask_for_sliding_layers(self):+ from auto_round.special_model_handler import _prepare_gemma4_replay_inputs++ block = self._make_fake_block("sliding_attention", 256, store_full_length_kv=False)+ hidden_states = torch.zeros((1, 4, 8), dtype=torch.float32)+ position_ids = torch.arange(4).unsqueeze(0)+ expected_mask = torch.full((1, 1, 4, 4), 7.0)++ with patch(+ "auto_round.special_model_handler._rebuild_gemma4_attention_mask",+ return_value=expected_mask,+ ) as rebuild_mask:+ prepared = _prepare_gemma4_replay_inputs(+ block,+ hidden_states,+ position_ids=position_ids,+ attention_mask=None,+ config=types.SimpleNamespace(),+ )++ rebuild_mask.assert_called_once()+ assert torch.equal(prepared["attention_mask"], expected_mask)++ def test_keeps_other_layer_types_when_position_embeddings_is_dict(self):+ from auto_round.special_model_handler import _prepare_gemma4_replay_inputs++ block = self._make_fake_block("full_attention", 512, store_full_length_kv=False)+ hidden_states = torch.zeros((1, 4, 8), dtype=torch.float32)+ position_ids = torch.arange(4).unsqueeze(0)+ cached_dict = {"sliding_attention": (torch.full((1, 4, 256), 3.0), torch.full((1, 4, 256), 3.0))}++ prepared = _prepare_gemma4_replay_inputs(+ block,+ hidden_states,+ position_ids=position_ids,+ position_embeddings=cached_dict,+ )++ assert prepared["position_embeddings"]["sliding_attention"][0].shape[-1] == 256+ assert prepared["position_embeddings"]["full_attention"][0].shape[-1] == 512+++class TestSpecialModelReplayDispatch:+ def test_non_special_block_is_untouched(self):+ from auto_round.special_model_handler import prepare_special_model_block_inputs++ class PlainBlock(nn.Module):+ pass++ block = PlainBlock()+ input_others = {"attention_mask": torch.ones(1, 1, 4, 4), "positional_inputs": []}+ output, positional_inputs = prepare_special_model_block_inputs(block, torch.zeros((1, 4, 8)), input_others, [])++ assert output is input_others+ assert positional_inputs == []+ assert torch.equal(output["attention_mask"], torch.ones(1, 1, 4, 4))++ def test_gemma4_dispatch_rebuilds_mask_and_normalizes_per_layer_input(self):+ from auto_round.special_model_handler import prepare_special_model_block_inputs++ class FakeRotaryEmbedding:+ def __call__(self, input_ids, position_ids, lt):+ hd = 512 if lt == "full_attention" else 256+ shape = (*input_ids.shape, hd)+ return torch.ones(shape), torch.ones(shape)++ class FakeAttention(nn.Module):+ def __init__(self):+ super().__init__()+ self.layer_type = "full_attention"+ self.head_dim = 512++ class FakeBlock(nn.Module):+ def __init__(self):+ super().__init__()+ self.self_attn = FakeAttention()+ self._rotary_emb_ref = [FakeRotaryEmbedding()]+ self._autoround_special_replay = "gemma4"+ self._gemma4_config_ref = types.SimpleNamespace()++ block = FakeBlock()+ hidden_states = torch.zeros((1, 4, 8), dtype=torch.float32)+ position_ids = torch.arange(4).unsqueeze(0)+ per_layer_input = torch.ones((1, 2, 8), dtype=torch.float32)+ expected_mask = torch.full((1, 1, 4, 4), 9.0)++ with patch(+ "auto_round.special_model_handler._rebuild_gemma4_attention_mask",+ return_value=expected_mask,+ ) as rebuild_mask:+ updated_inputs, updated_positional = prepare_special_model_block_inputs(+ block,+ hidden_states,+ {"position_ids": position_ids},+ [per_layer_input],+ )++ rebuild_mask.assert_called_once()+ assert torch.equal(updated_inputs["attention_mask"], expected_mask)+ assert updated_positional[0].shape[1] == hidden_states.shape[1]+++class TestLegacyBlockForwardDispatch:+ def test_legacy_block_forward_uses_special_model_dispatch(self):+ from auto_round.compressors.utils import block_forward++ class FakeRotaryEmbedding:+ def __call__(self, input_ids, position_ids, lt):+ hd = 512 if lt == "full_attention" else 256+ shape = (*input_ids.shape, hd)+ return torch.ones(shape), torch.ones(shape)++ class FakeAttention(nn.Module):+ def __init__(self):+ super().__init__()+ self.layer_type = "full_attention"+ self.head_dim = 512++ class FakeBlock(nn.Module):+ def __init__(self):+ super().__init__()+ self.self_attn = FakeAttention()+ self._rotary_emb_ref = [FakeRotaryEmbedding()]+ self._autoround_special_replay = "gemma4"++ def forward(self, input_ids, **kwargs):+ return kwargs["position_embeddings"][0]++ block = FakeBlock()+ input_ids = torch.zeros((1, 4), dtype=torch.float32)+ position_ids = torch.arange(4).unsqueeze(0)++ output = block_forward(+ block,+ input_ids,+ {"position_ids": position_ids, "positional_inputs": []},+ device=torch.device("cpu"),+ )++ assert output.shape[-1] == 512+++# ---------------------------------------------------------------------------+# 2. Tests for diffusion multi-device memory reservation+# ---------------------------------------------------------------------------+++class TestDiffusionMultiDeviceDispatch:+ """Unit tests for diffusion multi-device memory reservation.++ The fix reserves memory on the primary device for non-target pipeline+ components (text encoder, VAE) before computing the balanced device map,+ avoiding OOM when the main model is also dispatched to that device.+ """++ def test_reserve_memory_on_primary_device(self):+ """Non-target components must reserve memory on the primary device."""+ from auto_round.utils.device import dispatch_model_by_all_available_devices++ # Build a fake pipeline with two components: main + auxiliary+ class FakeComponent(nn.Module):+ def __init__(self, param_count):+ # Each param is 2 bytes (bf16), total = param_count * 2 bytes+ super().__init__()+ self.register_parameter("p", nn.Parameter(torch.empty(param_count, dtype=torch.bfloat16)))++ class FakePipe(nn.Module):+ def __init__(self):+ super().__init__()+ # Main model: 1000 params (2KB)+ self.transformer = FakeComponent(1000)+ # Non-main: 100 params (0.2KB)+ self.vae = FakeComponent(100)++ pipe = FakePipe()+ pipe.components = {"transformer": pipe.transformer, "vae": pipe.vae}++ with patch("torch.cuda.is_available", return_value=False):+ with patch("torch.cuda.device_count", return_value=0):+ # Single device path (falls back to pipe.to)+ result = dispatch_model_by_all_available_devices(pipe, "cpu")+ assert result is pipe++ def test_multi_device_respects_non_main_memory(self):+ """With multiple devices, memory reservation must be computed for non-main components."""+ from auto_round.utils.device import dispatch_model_by_all_available_devices++ class FakeComponent(nn.Module):+ def __init__(self, param_count):+ super().__init__()+ self.register_parameter("p", nn.Parameter(torch.empty(param_count, dtype=torch.bfloat16)))++ class FakePipe(nn.Module):+ def __init__(self):+ super().__init__()+ self.transformer = FakeComponent(1000) # main: 1000 * 2 = 2000 bytes+ self.text_encoder = FakeComponent(100) # non-main: 100 * 2 = 200 bytes+ self.vae = FakeComponent(50) # non-main: 50 * 2 = 100 bytes+ # total non-main: 300 params = 600 bytes * 1.2 buffer = 720 bytes++ pipe = FakePipe()+ pipe.components = {+ "transformer": pipe.transformer,+ "text_encoder": pipe.text_encoder,+ "vae": pipe.vae,+ }++ # We can't easily test the full multi-device path without accelerate mocks,+ # but we can verify the non_main_bytes calculation logic in isolation+ non_main_bytes = 0+ for attr, component in pipe.components.items():+ if attr == "transformer":+ continue+ if isinstance(component, nn.Module):+ non_main_bytes += sum(p.numel() * p.element_size() for p in component.parameters())+ non_main_reserved = int(non_main_bytes * 1.2)+ assert non_main_reserved > 0, "non-main memory must be computed"+++# ---------------------------------------------------------------------------+# 3. Tests for MLLM calibration memory cleanup+# ---------------------------------------------------------------------------+++class TestMLLMCalibMemoryCleanup:+ """Unit tests for MLLM calibration memory cleanup.++ The fix adds gc.collect() + clear_memory() after each forward pass in+ MLLMMixin.calib to reduce memory fragmentation. Also re-raises OOM so+ the try_cache_inter_data_gpucpu handler can switch to CPU.+ """++ def test_get_calibrator_kind_returns_mllm(self):+ """MLLMMixin._get_calibrator_kind must return 'mllm' so the right calibrator is used."""+ from auto_round.compressors.mllm_mixin import MLLMMixin++ class DummyCompressor(MLLMMixin):+ def __init__(self):+ # Only initialise what MLLMMixin needs — skip super().__init__+ # to avoid model loading side-effects.+ self.template = "qwen2_vl"+ self.extra_data_dir = None+ self.quant_nontext_module = False+ self.template_obj = None++ compressor = DummyCompressor()+ assert compressor._get_calibrator_kind() == "mllm"++ def test_quant_nontext_module_batch_size_accumulation(self):+ """quant_nontext_module=True must reset batch_size=1 and accumulate gradient steps."""+ # Directly test the batch_size accumulation logic from MLLMMixin.__init__.+ kwargs = {"batch_size": 4, "gradient_accumulate_steps": 2}++ batch_size = kwargs.get("batch_size", 8)+ grad_acc = kwargs.get("gradient_accumulate_steps", 1)+ new_grad_acc = batch_size * grad_acc+ kwargs["gradient_accumulate_steps"] = new_grad_acc+ kwargs["batch_size"] = 1++ assert kwargs["batch_size"] == 1+ assert kwargs["gradient_accumulate_steps"] == 8+++# ---------------------------------------------------------------------------+# 3. Tests for last_cache_name RAM reduction+# ---------------------------------------------------------------------------+++class TestLastCacheNameRAMReduction:+ """Unit tests for last_cache_name logic that limits caching to the first block.++ The fix passes last_cache_name=block_names[0] to try_cache_inter_data_gpucpu+ so that only the first block's inputs are cached, reducing RAM from+ O(n_blocks * nsamples) to O(nsamples).+ """++ def test_should_stop_after_first_block(self):+ """With last_cache_name set to the first block, caching must stop there."""++ class FakeCompressor:+ def __init__(self):+ self.last_cache_name = "model.layers.0"+ self._cache_target_set = {"model.layers.0"}+ self._cache_seen_targets = set()++ def _should_stop_cache_forward(self, name):+ if name == self.last_cache_name:+ return True+ if self.last_cache_name is not None:+ return False+ if not hasattr(self, "_cache_target_set") or not hasattr(self, "_cache_seen_targets"):+ return False+ if name in self._cache_target_set:+ self._cache_seen_targets.add(name)+ if not self._cache_target_set.issubset(self._cache_seen_targets):+ return False+ self.last_cache_name = name+ return True++ c = FakeCompressor()+ assert c._should_stop_cache_forward("model.layers.0") is True+ assert c.last_cache_name == "model.layers.0"++ def test_without_last_cache_name_all_blocks_cached(self):+ """Without last_cache_name, all blocks must be cached before stopping."""++ class FakeCompressor:+ def __init__(self):+ self.last_cache_name = None+ self._cache_target_set = {"model.layers.0", "model.layers.1"}+ self._cache_seen_targets = set()++ def _should_stop_cache_forward(self, name):+ if name == self.last_cache_name:+ return True+ if self.last_cache_name is not None:+ return False+ if not hasattr(self, "_cache_target_set") or not hasattr(self, "_cache_seen_targets"):+ return False+ if name in self._cache_target_set:+ self._cache_seen_targets.add(name)+ if not self._cache_target_set.issubset(self._cache_seen_targets):+ return False+ self.last_cache_name = name+ return True++ c = FakeCompressor()+ assert c._should_stop_cache_forward("model.layers.0") is False+ assert c._should_stop_cache_forward("model.layers.1") is True++ def test_last_cache_name_logic_in_quantize(self):+ """Verify that last_cache_name is set to first block when there are multiple blocks."""+ all_blocks = [["model.layers.0", "model.layers.1"], ["model.layers.2"]]+ to_cache_block_names = [b[0] for b in all_blocks]+ assert to_cache_block_names == ["model.layers.0", "model.layers.2"]++ # Without variable block shape: last_cache_name = first block+ _last_cache_name = to_cache_block_names[0] if len(to_cache_block_names) > 1 else None+ assert _last_cache_name == "model.layers.0"++ single_group = [["model.layers.0"]]+ single_cache = [b[0] for b in single_group]+ assert len(single_cache) == 1+ _last_cache_name_single = single_cache[0] if len(single_cache) > 1 else None+ assert _last_cache_name_single is None