Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Fix Gemma2 padding masks during batched cached decoding (#12008)
cherry77-cloudf7c9c68c411e
diff --git a/tests/test_gemma2_attention_masks.py b/tests/test_gemma2_attention_masks.pynew file mode 100644index 00000000000..db806eeecb7--- /dev/null+++ b/tests/test_gemma2_attention_masks.py@@ -0,0 +1,201 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""Gemma2 decode and prefill masks: padding, single-row decode and unwindowed global layers."""++import types++import pytest++import unsloth # noqa: F401 (must precede transformers)+import unsloth.models.gemma2 as g2+from real_accelerator import has_real_cuda++torch = pytest.importorskip("torch")+++class _Captured(Exception):+ pass+++def _masks_reaching_attention(+ monkeypatch,+ flash,+ bsz = 2,+ second_device = "cpu",+):+ seen = []++ def attention(self, hidden_states, past_key_value, position_ids, attention_mask, **kw):+ seen.append((kw["use_sliding_window"], attention_mask))+ if len(seen) == 2:+ raise _Captured+ return hidden_states, past_key_value++ monkeypatch.setattr(g2, "HAS_FLASH_ATTENTION_SOFTCAPPING", flash)+ monkeypatch.setattr(g2, "DEVICE_COUNT", 0)+ monkeypatch.setattr(g2, "Gemma2Attention_fast_forward_inference", attention)+ monkeypatch.setattr(g2, "fast_rms_layernorm_inference_gemma", lambda ln, x, w: x)+ monkeypatch.setattr(g2, "fast_geglu_inference", lambda mlp, x: x)+ # No per-device fp32 buffers on CPU; the stubbed norms never read them.+ monkeypatch.setattr(+ g2, "per_layer_device", lambda layer: (torch.device(layer.device), slice(None))+ )++ hidden, cached = 8, 5+ layer = types.SimpleNamespace(+ device = "cpu",+ input_layernorm = types.SimpleNamespace(weight = torch.ones(hidden)),+ post_attention_layernorm = None,+ pre_feedforward_layernorm = None,+ post_feedforward_layernorm = None,+ self_attn = None,+ mlp = None,+ )+ config = types.SimpleNamespace(+ hidden_size = hidden, sliding_window = 4096, torch_dtype = torch.float32+ )+ model = types.SimpleNamespace(+ layers = [layer, types.SimpleNamespace(**{**vars(layer), "device": second_device})],+ embed_tokens = lambda ids: torch.zeros(*ids.shape, hidden),+ )+ self = types.SimpleNamespace(model = model, config = config, max_seq_length = 64)+ past = [(torch.zeros(bsz, 1, cached, 4), torch.zeros(bsz, 1, cached, 4))] * 2+ attention_mask = torch.tensor([[1] * (cached + 1), [0, 0] + [1] * (cached - 1)])[:bsz]+ with pytest.raises(_Captured):+ g2.Gemma2Model_fast_forward_inference(+ self,+ torch.zeros(bsz, 1, dtype = torch.long),+ past,+ torch.full((bsz, 1), cached),+ attention_mask = attention_mask,+ )+ return seen++[email protected]("flash", [False, True])+def test_batched_decode_masks_left_padding(monkeypatch, flash):+ seen = _masks_reaching_attention(monkeypatch, flash)+ assert [sliding for sliding, _ in seen] == [True, False]+ for _, mask in seen:+ assert isinstance(mask, torch.Tensor)+ assert mask.shape[-1] == 6+ assert torch.all(mask[0] == 0)+ assert torch.all(mask[1, ..., :2] < -1e30)+ assert torch.all(mask[1, ..., 2:] == 0)++[email protected]("flash", [False, True])+def test_single_row_decode_passes_no_mask(monkeypatch, flash):+ # A 2D mask reaching the sliding layer raised IndexError past the window.+ seen = _masks_reaching_attention(monkeypatch, flash, bsz = 1)+ assert [mask for _, mask in seen] == [None, None]+++ not has_real_cuda(), reason = "loads tiny Gemma checkpoints through FastLanguageModel on CUDA"+)+def test_prefill_global_layers_see_past_the_window():+ from unsloth import FastLanguageModel++ model, _ = FastLanguageModel.from_pretrained(+ "trl-internal-testing/tiny-Gemma2ForCausalLM",+ max_seq_length = 64,+ load_in_4bit = False,+ dtype = torch.float32,+ )+ FastLanguageModel.for_inference(model)+ window, n = 4, 12+ model.config.sliding_window = window+ masks = {}+ for idx, layer in enumerate(model.model.layers[:2]):+ layer.register_forward_pre_hook(+ lambda mod, args, kwargs, idx = idx: masks.__setitem__(idx, kwargs["causal_mask"]),+ with_kwargs = True,+ )+ ids = torch.arange(10, 10 + n, device = "cuda")[None]+ attention_mask = torch.ones_like(ids)+ attention_mask[:, 0] = 0+ with torch.no_grad():+ model(input_ids = ids, attention_mask = attention_mask)+ q = torch.arange(n)[:, None]+ k = torch.arange(n)[None]+ causal = (k <= q) & (k >= 1) # key 0 is padding+ kept = {idx: (m.reshape(-1, n, n)[0] == 0).cpu() for idx, m in masks.items()}+ assert torch.equal(kept[1][1:], causal[1:])+ assert not kept[0][-1, 1]+ assert torch.equal(kept[0][1:], (causal & (q - k <= window))[1:])+++def test_decode_masks_follow_each_layer_device(monkeypatch):+ # Pipeline-parallel device maps: a mask left on the first device fails on the next GPU's layer.+ seen = _masks_reaching_attention(monkeypatch, flash = False, second_device = "meta")+ assert [mask.device.type for _, mask in seen] == ["cpu", "meta"]+++ not has_real_cuda(), reason = "loads tiny Gemma checkpoints through FastLanguageModel on CUDA"+)+def test_prefill_masks_short_batches_and_4d(monkeypatch):+ import unsloth.models.llama as llama_module+ from unsloth import FastLanguageModel++ # The masks under test feed the softcapping kernels, not flash-attn.+ monkeypatch.setattr(llama_module, "HAS_FLASH_ATTENTION_SOFTCAPPING", False)+ monkeypatch.setattr(g2, "HAS_FLASH_ATTENTION_SOFTCAPPING", False)++ model, _ = FastLanguageModel.from_pretrained(+ "trl-internal-testing/tiny-Gemma2ForCausalLM",+ max_seq_length = 64,+ load_in_4bit = False,+ dtype = torch.float32,+ )+ FastLanguageModel.for_inference(model)+ torch.manual_seed(0)+ with torch.no_grad():+ # More rows than tokens: a 4D mask sliced as mask[:q_len, :q_len] used to cut the batch axis.+ ids = torch.randint(5, 100, (8, 2), device = "cuda")+ batched = model(input_ids = ids, attention_mask = torch.ones_like(ids)).logits+ single = model(input_ids = ids[3:4], attention_mask = torch.ones_like(ids[3:4])).logits+ torch.testing.assert_close(batched[3:4], single, rtol = 1e-4, atol = 1e-4)+ ids = torch.randint(5, 100, (2, 5), device = "cuda")+ keep = torch.ones(5, 5, dtype = torch.bool, device = "cuda").tril()+ mask4d = torch.zeros(2, 1, 5, 5, device = "cuda").masked_fill(+ ~keep, torch.finfo(torch.float32).min+ )+ from_4d = model(input_ids = ids, attention_mask = mask4d).logits+ from_2d = model(input_ids = ids, attention_mask = torch.ones_like(ids)).logits+ torch.testing.assert_close(from_4d, from_2d, rtol = 1e-4, atol = 1e-4)+++ not has_real_cuda(), reason = "loads tiny Gemma checkpoints through FastLanguageModel on CUDA"+)+def test_unpadded_prefill_uses_static_masks(monkeypatch):+ import unsloth.models.llama as llama_module+ from unsloth import FastLanguageModel++ monkeypatch.setattr(llama_module, "HAS_FLASH_ATTENTION_SOFTCAPPING", False)+ monkeypatch.setattr(g2, "HAS_FLASH_ATTENTION_SOFTCAPPING", False)+ model, _ = FastLanguageModel.from_pretrained(+ "trl-internal-testing/tiny-Gemma2ForCausalLM",+ max_seq_length = 64,+ load_in_4bit = False,+ dtype = torch.float32,+ )+ FastLanguageModel.for_inference(model)+ window, n = 4, 12+ model.config.sliding_window = window+ masks = {}+ for idx, layer in enumerate(model.model.layers[:2]):+ layer.register_forward_pre_hook(+ lambda mod, args, kwargs, idx = idx: masks.__setitem__(idx, kwargs["causal_mask"]),+ with_kwargs = True,+ )+ ids = torch.arange(10, 10 + n, device = "cuda").repeat(8, 1)+ with torch.no_grad():+ model(input_ids = ids, attention_mask = torch.ones_like(ids))+ assert all(m.dim() == 2 for m in masks.values())+ kept = {idx: (m[:n, :n] == 0).cpu() for idx, m in masks.items()}+ assert kept[1][-1].sum() == n+ assert kept[0][-1].sum() < ndiff --git a/unsloth/kernels/flex_attention.py b/unsloth/kernels/flex_attention.pyindex 193d3f7f3bb..a1fc03a02a3 100644--- a/unsloth/kernels/flex_attention.py+++ b/unsloth/kernels/flex_attention.py@@ -58,7 +58,7 @@ if not HAS_FLEX_ATTENTION:Q = Q * torch.tensor(s**-0.5, dtype = Q.dtype)A = torch.matmul(Q, K.transpose(2, 3))A = t * torch.tanh(A / t)- A += causal_mask[:q_len, :q_len]+ A += causal_mask[..., :q_len, :q_len]# Much slower under torch compile than the masked_fill_ it replaces.A = torch.nn.functional.softmax(A, dim = -1, dtype = torch.float32).to(Q.dtype)A = torch.matmul(A, V)@@ -163,7 +163,7 @@ def slow_inference_attention_softcapping(Q, K, V, causal_mask, self, bsz, q_len)A /= ttorch_tanh(A, out = A)A *= t- A += causal_mask[:q_len, :q_len]+ A += causal_mask[..., :q_len, :q_len]# Much slower under torch compile than the masked_fill_ it replaces.A = torch_nn_functional_softmax(A, dim = -1, dtype = torch.float32).to(Q.dtype)A = torch_matmul(A, V)diff --git a/unsloth/models/gemma2.py b/unsloth/models/gemma2.pyindex 91b3f893973..c101a0aa861 100644--- a/unsloth/models/gemma2.py+++ b/unsloth/models/gemma2.py@@ -463,31 +463,32 @@ def Gemma2Model_fast_forward_inference(bsz, q_len, hd = hidden_states.shapeseq_len = past_key_values[0][0].shape[-2]if bsz != 1:- if HAS_FLASH_ATTENTION_SOFTCAPPING:- SWA = True- GA = False- else:- SWA = _prepare_4d_causal_attention_mask_for_sdpa(- attention_mask,- (bsz, q_len),- hidden_states,- seq_len,- sliding_window = self.config.sliding_window,- )- GA = _prepare_4d_causal_attention_mask_for_sdpa(- attention_mask,- (bsz, q_len),- hidden_states,- seq_len,- )+ SWA = _prepare_4d_causal_attention_mask_for_sdpa(+ attention_mask,+ (bsz, q_len),+ hidden_states,+ seq_len,+ sliding_window = self.config.sliding_window,+ )+ GA = _prepare_4d_causal_attention_mask_for_sdpa(+ attention_mask,+ (bsz, q_len),+ hidden_states,+ seq_len,+ )else:- SWA = attention_mask- GA = attention_mask+ # One row has no padding; the raw 2D mask cannot be sliced to the sliding window.+ SWA = None+ GA = Nonenext_decoder_cache = []for idx, decoder_layer in enumerate(self.model.layers):# For pipeline parallelism every tensor must be on the same device; this movement happens once per GPU in PP.layer_device, device_index = per_layer_device(decoder_layer)hidden_states, position_ids = move_to_device(layer_device, hidden_states, position_ids)+ if SWA is not None:+ SWA = move_to_device(layer_device, SWA)+ if GA is not None:+ GA = move_to_device(layer_device, GA)use_sliding_window = idx % 2 == 0diff --git a/unsloth/models/llama.py b/unsloth/models/llama.pyindex 86d956d9271..76af64068d7 100644--- a/unsloth/models/llama.py+++ b/unsloth/models/llama.py@@ -1016,13 +1016,23 @@ def LlamaModel_fast_forward(else:padding_mask = None- attention_mask = _prepare_4d_causal_attention_mask_for_sdpa(- attention_mask,- (batch_size, seq_length),- inputs_embeds,- past_key_values_length,- sliding_window = getattr(self.config, "sliding_window", None),- )+ # Gemma2 builds its own masks below; a 4D sliding mask here would also window the global layers.+ if IS_GEMMA2:+ # No padding: keep the flash path, which windows each layer itself.+ if (+ HAS_FLASH_ATTENTION_SOFTCAPPING+ and attention_mask.dim() == 2+ and bool(attention_mask.all())+ ):+ attention_mask = None+ else:+ attention_mask = _prepare_4d_causal_attention_mask_for_sdpa(+ attention_mask,+ (batch_size, seq_length),+ inputs_embeds,+ past_key_values_length,+ sliding_window = getattr(self.config, "sliding_window", None),+ )# Must NOT convert to bool; that weirdly causes errors.hidden_states = inputs_embeds@@ -1052,27 +1062,32 @@ def LlamaModel_fast_forward(dynamic_SWA_mask = Nonedynamic_GA_mask = Noneif IS_GEMMA2:+ # An unpadded prefill shares the static [n, n] masks instead of two [bsz, 1, q, q] copies.+ unpadded_prefill = (+ attention_mask is not None+ and past_key_values_length == 0+ and attention_mask.dim() == 2+ and bool(attention_mask.all())+ )if HAS_FLASH_ATTENTION_SOFTCAPPING and attention_mask is None:self.SWA_mask = Trueself.GA_mask = False- elif attention_mask is not None:+ elif attention_mask is not None and not unpadded_prefill:# Unsloth needs a 2D mask, not [2, 1, n, n] (#853), converted to float not bool# (pytorch/pytorch#103749).- dynamic_SWA_mask = _prepare_4d_causal_attention_mask_for_sdpa(- attention_mask,- (batch_size, seq_length),- inputs_embeds,- past_key_values_length,- sliding_window = self.config.sliding_window,- )- dynamic_GA_mask = _prepare_4d_causal_attention_mask_for_sdpa(- attention_mask,- (batch_size, seq_length),- inputs_embeds,- past_key_values_length,- sliding_window = None,- )+ if attention_mask.dim() == 2:+ # The SDPA helper returns None for an all-ones mask; the softcapping kernels need a tensor.+ key_value_length = past_key_values_length + seq_length+ dynamic_SWA_mask = AttentionMaskConverter(+ is_causal = True, sliding_window = self.config.sliding_window+ ).to_4d(attention_mask, seq_length, inputs_embeds.dtype, key_value_length)+ dynamic_GA_mask = AttentionMaskConverter(is_causal = True).to_4d(+ attention_mask, seq_length, inputs_embeds.dtype, key_value_length+ )+ else:+ dynamic_SWA_mask = attention_mask+ dynamic_GA_mask = attention_maskuse_static_mask = Falseelif not hasattr(self, "SWA_mask"):