Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Fix Gemma2 padding masks during batched cached decoding (#12008)

cherry77-cloudf7c9c68c411e
diff --git a/tests/test_gemma2_attention_masks.py b/tests/test_gemma2_attention_masks.py
new file mode 100644
index 00000000000..db806eeecb7
--- /dev/null
+++ b/tests/test_gemma2_attention_masks.py
@@ -0,0 +1,201 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""Gemma2 decode and prefill masks: padding, single-row decode and unwindowed global layers."""
+
+import types
+
+import pytest
+
+import unsloth # noqa: F401 (must precede transformers)
+import unsloth.models.gemma2 as g2
+from real_accelerator import has_real_cuda
+
+torch = pytest.importorskip("torch")
+
+
+class _Captured(Exception):
+ pass
+
+
+def _masks_reaching_attention(
+ monkeypatch,
+ flash,
+ bsz = 2,
+ second_device = "cpu",
+):
+ seen = []
+
+ def attention(self, hidden_states, past_key_value, position_ids, attention_mask, **kw):
+ seen.append((kw["use_sliding_window"], attention_mask))
+ if len(seen) == 2:
+ raise _Captured
+ return hidden_states, past_key_value
+
+ monkeypatch.setattr(g2, "HAS_FLASH_ATTENTION_SOFTCAPPING", flash)
+ monkeypatch.setattr(g2, "DEVICE_COUNT", 0)
+ monkeypatch.setattr(g2, "Gemma2Attention_fast_forward_inference", attention)
+ monkeypatch.setattr(g2, "fast_rms_layernorm_inference_gemma", lambda ln, x, w: x)
+ monkeypatch.setattr(g2, "fast_geglu_inference", lambda mlp, x: x)
+ # No per-device fp32 buffers on CPU; the stubbed norms never read them.
+ monkeypatch.setattr(
+ g2, "per_layer_device", lambda layer: (torch.device(layer.device), slice(None))
+ )
+
+ hidden, cached = 8, 5
+ layer = types.SimpleNamespace(
+ device = "cpu",
+ input_layernorm = types.SimpleNamespace(weight = torch.ones(hidden)),
+ post_attention_layernorm = None,
+ pre_feedforward_layernorm = None,
+ post_feedforward_layernorm = None,
+ self_attn = None,
+ mlp = None,
+ )
+ config = types.SimpleNamespace(
+ hidden_size = hidden, sliding_window = 4096, torch_dtype = torch.float32
+ )
+ model = types.SimpleNamespace(
+ layers = [layer, types.SimpleNamespace(**{**vars(layer), "device": second_device})],
+ embed_tokens = lambda ids: torch.zeros(*ids.shape, hidden),
+ )
+ self = types.SimpleNamespace(model = model, config = config, max_seq_length = 64)
+ past = [(torch.zeros(bsz, 1, cached, 4), torch.zeros(bsz, 1, cached, 4))] * 2
+ attention_mask = torch.tensor([[1] * (cached + 1), [0, 0] + [1] * (cached - 1)])[:bsz]
+ with pytest.raises(_Captured):
+ g2.Gemma2Model_fast_forward_inference(
+ self,
+ torch.zeros(bsz, 1, dtype = torch.long),
+ past,
+ torch.full((bsz, 1), cached),
+ attention_mask = attention_mask,
+ )
+ return seen
+
+
[email protected]("flash", [False, True])
+def test_batched_decode_masks_left_padding(monkeypatch, flash):
+ seen = _masks_reaching_attention(monkeypatch, flash)
+ assert [sliding for sliding, _ in seen] == [True, False]
+ for _, mask in seen:
+ assert isinstance(mask, torch.Tensor)
+ assert mask.shape[-1] == 6
+ assert torch.all(mask[0] == 0)
+ assert torch.all(mask[1, ..., :2] < -1e30)
+ assert torch.all(mask[1, ..., 2:] == 0)
+
+
[email protected]("flash", [False, True])
+def test_single_row_decode_passes_no_mask(monkeypatch, flash):
+ # A 2D mask reaching the sliding layer raised IndexError past the window.
+ seen = _masks_reaching_attention(monkeypatch, flash, bsz = 1)
+ assert [mask for _, mask in seen] == [None, None]
+
+
+ not has_real_cuda(), reason = "loads tiny Gemma checkpoints through FastLanguageModel on CUDA"
+)
+def test_prefill_global_layers_see_past_the_window():
+ from unsloth import FastLanguageModel
+
+ model, _ = FastLanguageModel.from_pretrained(
+ "trl-internal-testing/tiny-Gemma2ForCausalLM",
+ max_seq_length = 64,
+ load_in_4bit = False,
+ dtype = torch.float32,
+ )
+ FastLanguageModel.for_inference(model)
+ window, n = 4, 12
+ model.config.sliding_window = window
+ masks = {}
+ for idx, layer in enumerate(model.model.layers[:2]):
+ layer.register_forward_pre_hook(
+ lambda mod, args, kwargs, idx = idx: masks.__setitem__(idx, kwargs["causal_mask"]),
+ with_kwargs = True,
+ )
+ ids = torch.arange(10, 10 + n, device = "cuda")[None]
+ attention_mask = torch.ones_like(ids)
+ attention_mask[:, 0] = 0
+ with torch.no_grad():
+ model(input_ids = ids, attention_mask = attention_mask)
+ q = torch.arange(n)[:, None]
+ k = torch.arange(n)[None]
+ causal = (k <= q) & (k >= 1) # key 0 is padding
+ kept = {idx: (m.reshape(-1, n, n)[0] == 0).cpu() for idx, m in masks.items()}
+ assert torch.equal(kept[1][1:], causal[1:])
+ assert not kept[0][-1, 1]
+ assert torch.equal(kept[0][1:], (causal & (q - k <= window))[1:])
+
+
+def test_decode_masks_follow_each_layer_device(monkeypatch):
+ # Pipeline-parallel device maps: a mask left on the first device fails on the next GPU's layer.
+ seen = _masks_reaching_attention(monkeypatch, flash = False, second_device = "meta")
+ assert [mask.device.type for _, mask in seen] == ["cpu", "meta"]
+
+
+ not has_real_cuda(), reason = "loads tiny Gemma checkpoints through FastLanguageModel on CUDA"
+)
+def test_prefill_masks_short_batches_and_4d(monkeypatch):
+ import unsloth.models.llama as llama_module
+ from unsloth import FastLanguageModel
+
+ # The masks under test feed the softcapping kernels, not flash-attn.
+ monkeypatch.setattr(llama_module, "HAS_FLASH_ATTENTION_SOFTCAPPING", False)
+ monkeypatch.setattr(g2, "HAS_FLASH_ATTENTION_SOFTCAPPING", False)
+
+ model, _ = FastLanguageModel.from_pretrained(
+ "trl-internal-testing/tiny-Gemma2ForCausalLM",
+ max_seq_length = 64,
+ load_in_4bit = False,
+ dtype = torch.float32,
+ )
+ FastLanguageModel.for_inference(model)
+ torch.manual_seed(0)
+ with torch.no_grad():
+ # More rows than tokens: a 4D mask sliced as mask[:q_len, :q_len] used to cut the batch axis.
+ ids = torch.randint(5, 100, (8, 2), device = "cuda")
+ batched = model(input_ids = ids, attention_mask = torch.ones_like(ids)).logits
+ single = model(input_ids = ids[3:4], attention_mask = torch.ones_like(ids[3:4])).logits
+ torch.testing.assert_close(batched[3:4], single, rtol = 1e-4, atol = 1e-4)
+ ids = torch.randint(5, 100, (2, 5), device = "cuda")
+ keep = torch.ones(5, 5, dtype = torch.bool, device = "cuda").tril()
+ mask4d = torch.zeros(2, 1, 5, 5, device = "cuda").masked_fill(
+ ~keep, torch.finfo(torch.float32).min
+ )
+ from_4d = model(input_ids = ids, attention_mask = mask4d).logits
+ from_2d = model(input_ids = ids, attention_mask = torch.ones_like(ids)).logits
+ torch.testing.assert_close(from_4d, from_2d, rtol = 1e-4, atol = 1e-4)
+
+
+ not has_real_cuda(), reason = "loads tiny Gemma checkpoints through FastLanguageModel on CUDA"
+)
+def test_unpadded_prefill_uses_static_masks(monkeypatch):
+ import unsloth.models.llama as llama_module
+ from unsloth import FastLanguageModel
+
+ monkeypatch.setattr(llama_module, "HAS_FLASH_ATTENTION_SOFTCAPPING", False)
+ monkeypatch.setattr(g2, "HAS_FLASH_ATTENTION_SOFTCAPPING", False)
+ model, _ = FastLanguageModel.from_pretrained(
+ "trl-internal-testing/tiny-Gemma2ForCausalLM",
+ max_seq_length = 64,
+ load_in_4bit = False,
+ dtype = torch.float32,
+ )
+ FastLanguageModel.for_inference(model)
+ window, n = 4, 12
+ model.config.sliding_window = window
+ masks = {}
+ for idx, layer in enumerate(model.model.layers[:2]):
+ layer.register_forward_pre_hook(
+ lambda mod, args, kwargs, idx = idx: masks.__setitem__(idx, kwargs["causal_mask"]),
+ with_kwargs = True,
+ )
+ ids = torch.arange(10, 10 + n, device = "cuda").repeat(8, 1)
+ with torch.no_grad():
+ model(input_ids = ids, attention_mask = torch.ones_like(ids))
+ assert all(m.dim() == 2 for m in masks.values())
+ kept = {idx: (m[:n, :n] == 0).cpu() for idx, m in masks.items()}
+ assert kept[1][-1].sum() == n
+ assert kept[0][-1].sum() < n
diff --git a/unsloth/kernels/flex_attention.py b/unsloth/kernels/flex_attention.py
index 193d3f7f3bb..a1fc03a02a3 100644
--- a/unsloth/kernels/flex_attention.py
+++ b/unsloth/kernels/flex_attention.py
@@ -58,7 +58,7 @@ if not HAS_FLEX_ATTENTION:
Q = Q * torch.tensor(s**-0.5, dtype = Q.dtype)
A = torch.matmul(Q, K.transpose(2, 3))
A = t * torch.tanh(A / t)
- A += causal_mask[:q_len, :q_len]
+ A += causal_mask[..., :q_len, :q_len]
# Much slower under torch compile than the masked_fill_ it replaces.
A = torch.nn.functional.softmax(A, dim = -1, dtype = torch.float32).to(Q.dtype)
A = torch.matmul(A, V)
@@ -163,7 +163,7 @@ def slow_inference_attention_softcapping(Q, K, V, causal_mask, self, bsz, q_len)
A /= t
torch_tanh(A, out = A)
A *= t
- A += causal_mask[:q_len, :q_len]
+ A += causal_mask[..., :q_len, :q_len]
# Much slower under torch compile than the masked_fill_ it replaces.
A = torch_nn_functional_softmax(A, dim = -1, dtype = torch.float32).to(Q.dtype)
A = torch_matmul(A, V)
diff --git a/unsloth/models/gemma2.py b/unsloth/models/gemma2.py
index 91b3f893973..c101a0aa861 100644
--- a/unsloth/models/gemma2.py
+++ b/unsloth/models/gemma2.py
@@ -463,31 +463,32 @@ def Gemma2Model_fast_forward_inference(
bsz, q_len, hd = hidden_states.shape
seq_len = past_key_values[0][0].shape[-2]
if bsz != 1:
- if HAS_FLASH_ATTENTION_SOFTCAPPING:
- SWA = True
- GA = False
- else:
- SWA = _prepare_4d_causal_attention_mask_for_sdpa(
- attention_mask,
- (bsz, q_len),
- hidden_states,
- seq_len,
- sliding_window = self.config.sliding_window,
- )
- GA = _prepare_4d_causal_attention_mask_for_sdpa(
- attention_mask,
- (bsz, q_len),
- hidden_states,
- seq_len,
- )
+ SWA = _prepare_4d_causal_attention_mask_for_sdpa(
+ attention_mask,
+ (bsz, q_len),
+ hidden_states,
+ seq_len,
+ sliding_window = self.config.sliding_window,
+ )
+ GA = _prepare_4d_causal_attention_mask_for_sdpa(
+ attention_mask,
+ (bsz, q_len),
+ hidden_states,
+ seq_len,
+ )
else:
- SWA = attention_mask
- GA = attention_mask
+ # One row has no padding; the raw 2D mask cannot be sliced to the sliding window.
+ SWA = None
+ GA = None
next_decoder_cache = []
for idx, decoder_layer in enumerate(self.model.layers):
# For pipeline parallelism every tensor must be on the same device; this movement happens once per GPU in PP.
layer_device, device_index = per_layer_device(decoder_layer)
hidden_states, position_ids = move_to_device(layer_device, hidden_states, position_ids)
+ if SWA is not None:
+ SWA = move_to_device(layer_device, SWA)
+ if GA is not None:
+ GA = move_to_device(layer_device, GA)
use_sliding_window = idx % 2 == 0
diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py
index 86d956d9271..76af64068d7 100644
--- a/unsloth/models/llama.py
+++ b/unsloth/models/llama.py
@@ -1016,13 +1016,23 @@ def LlamaModel_fast_forward(
else:
padding_mask = None
- attention_mask = _prepare_4d_causal_attention_mask_for_sdpa(
- attention_mask,
- (batch_size, seq_length),
- inputs_embeds,
- past_key_values_length,
- sliding_window = getattr(self.config, "sliding_window", None),
- )
+ # Gemma2 builds its own masks below; a 4D sliding mask here would also window the global layers.
+ if IS_GEMMA2:
+ # No padding: keep the flash path, which windows each layer itself.
+ if (
+ HAS_FLASH_ATTENTION_SOFTCAPPING
+ and attention_mask.dim() == 2
+ and bool(attention_mask.all())
+ ):
+ attention_mask = None
+ else:
+ attention_mask = _prepare_4d_causal_attention_mask_for_sdpa(
+ attention_mask,
+ (batch_size, seq_length),
+ inputs_embeds,
+ past_key_values_length,
+ sliding_window = getattr(self.config, "sliding_window", None),
+ )
# Must NOT convert to bool; that weirdly causes errors.
hidden_states = inputs_embeds
@@ -1052,27 +1062,32 @@ def LlamaModel_fast_forward(
dynamic_SWA_mask = None
dynamic_GA_mask = None
if IS_GEMMA2:
+ # An unpadded prefill shares the static [n, n] masks instead of two [bsz, 1, q, q] copies.
+ unpadded_prefill = (
+ attention_mask is not None
+ and past_key_values_length == 0
+ and attention_mask.dim() == 2
+ and bool(attention_mask.all())
+ )
if HAS_FLASH_ATTENTION_SOFTCAPPING and attention_mask is None:
self.SWA_mask = True
self.GA_mask = False
- elif attention_mask is not None:
+ elif attention_mask is not None and not unpadded_prefill:
# Unsloth needs a 2D mask, not [2, 1, n, n] (#853), converted to float not bool
# (pytorch/pytorch#103749).
- dynamic_SWA_mask = _prepare_4d_causal_attention_mask_for_sdpa(
- attention_mask,
- (batch_size, seq_length),
- inputs_embeds,
- past_key_values_length,
- sliding_window = self.config.sliding_window,
- )
- dynamic_GA_mask = _prepare_4d_causal_attention_mask_for_sdpa(
- attention_mask,
- (batch_size, seq_length),
- inputs_embeds,
- past_key_values_length,
- sliding_window = None,
- )
+ if attention_mask.dim() == 2:
+ # The SDPA helper returns None for an all-ones mask; the softcapping kernels need a tensor.
+ key_value_length = past_key_values_length + seq_length
+ dynamic_SWA_mask = AttentionMaskConverter(
+ is_causal = True, sliding_window = self.config.sliding_window
+ ).to_4d(attention_mask, seq_length, inputs_embeds.dtype, key_value_length)
+ dynamic_GA_mask = AttentionMaskConverter(is_causal = True).to_4d(
+ attention_mask, seq_length, inputs_embeds.dtype, key_value_length
+ )
+ else:
+ dynamic_SWA_mask = attention_mask
+ dynamic_GA_mask = attention_mask
use_static_mask = False
elif not hasattr(self, "SWA_mask"):