Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Fix Gemma2 sliding-window boundaries across attention backends (#13730)

cherry77-cloud7453f8563846
diff --git a/tests/test_gemma2_attention_masks.py b/tests/test_gemma2_attention_masks.py
index db806eeecb7..68192cfeae7 100644
--- a/tests/test_gemma2_attention_masks.py
+++ b/tests/test_gemma2_attention_masks.py
@@ -124,7 +124,7 @@ def test_prefill_global_layers_see_past_the_window():
kept = {idx: (m.reshape(-1, n, n)[0] == 0).cpu() for idx, m in masks.items()}
assert torch.equal(kept[1][1:], causal[1:])
assert not kept[0][-1, 1]
- assert torch.equal(kept[0][1:], (causal & (q - k <= window))[1:])
+ assert torch.equal(kept[0][1:], (causal & (q - k < window))[1:])
def test_decode_masks_follow_each_layer_device(monkeypatch):
diff --git a/unsloth/kernels/flex_attention.py b/unsloth/kernels/flex_attention.py
index cf7fe8f20db..45b91164bb9 100644
--- a/unsloth/kernels/flex_attention.py
+++ b/unsloth/kernels/flex_attention.py
@@ -129,7 +129,7 @@ else:
def sliding_window_masker(size = 4096):
def sliding_window(b, h, q_idx, kv_idx):
causal_mask = q_idx >= kv_idx
- window_mask = q_idx - kv_idx <= size
+ window_mask = q_idx - kv_idx < size
return causal_mask & window_mask
return sliding_window
diff --git a/unsloth/models/gemma2.py b/unsloth/models/gemma2.py
index c744cad20fc..d45c2a816ec 100644
--- a/unsloth/models/gemma2.py
+++ b/unsloth/models/gemma2.py
@@ -206,7 +206,7 @@ def Gemma2Attention_fast_forward(
sliding_window = getattr(self.config, "sliding_window", None)
if has_sliding_window:
sliding_window = sliding_window if sliding_window is not None else kv_seq_len
- window = (-1, -1) if kv_seq_len <= sliding_window else (sliding_window, sliding_window)
+ window = (-1, -1) if kv_seq_len <= sliding_window else (sliding_window - 1, 0)
if not hasattr(self, "_flash_attention_softmax_scale"):
self._flash_attention_softmax_scale = 1.0 / (self.config.query_pre_attn_scalar**0.5)
diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py
index 77dc277ed80..7b1fa1c00fa 100644
--- a/unsloth/models/llama.py
+++ b/unsloth/models/llama.py
@@ -1081,6 +1081,27 @@ def _is_left_padded(attention_mask):
return not bool(attention_mask[:, 0].all())
+class Gemma2AttentionMaskConverter(AttentionMaskConverter):
+ """Gemma2's window counts the current key as well as the preceding keys."""
+
+ @staticmethod
+ def _make_causal_mask(
+ input_ids_shape,
+ dtype,
+ device,
+ past_key_values_length = 0,
+ sliding_window = None,
+ ):
+ # The legacy converter counts preceding keys; its static helper also accepts zero for a one-key window.
+ return AttentionMaskConverter._make_causal_mask(
+ input_ids_shape,
+ dtype,
+ device,
+ past_key_values_length = past_key_values_length,
+ sliding_window = sliding_window - 1 if sliding_window is not None else None,
+ )
+
+
# Ported from transformers models/llama/modeling_llama.py#L825
def LlamaModel_fast_forward(
self,
@@ -1297,7 +1318,7 @@ def LlamaModel_fast_forward(
if attention_mask.dim() == 2:
# The SDPA helper returns None for an all-ones mask; the softcapping kernels need a tensor.
key_value_length = past_key_values_length + seq_length
- dynamic_SWA_mask = AttentionMaskConverter(
+ dynamic_SWA_mask = Gemma2AttentionMaskConverter(
is_causal = True, sliding_window = self.config.sliding_window
).to_4d(attention_mask, seq_length, inputs_embeds.dtype, key_value_length)
dynamic_GA_mask = AttentionMaskConverter(is_causal = True).to_4d(
@@ -1318,7 +1339,7 @@ def LlamaModel_fast_forward(
n = self.max_seq_length
# masked_fill makes things slower.
self.SWA_mask = (
- AttentionMaskConverter(
+ Gemma2AttentionMaskConverter(
is_causal = True,
sliding_window = self.config.sliding_window,
)