Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Fix Gemma2 sliding-window boundaries across attention backends (#13730)
cherry77-cloud7453f8563846
diff --git a/tests/test_gemma2_attention_masks.py b/tests/test_gemma2_attention_masks.pyindex db806eeecb7..68192cfeae7 100644--- a/tests/test_gemma2_attention_masks.py+++ b/tests/test_gemma2_attention_masks.py@@ -124,7 +124,7 @@ def test_prefill_global_layers_see_past_the_window():kept = {idx: (m.reshape(-1, n, n)[0] == 0).cpu() for idx, m in masks.items()}assert torch.equal(kept[1][1:], causal[1:])assert not kept[0][-1, 1]- assert torch.equal(kept[0][1:], (causal & (q - k <= window))[1:])+ assert torch.equal(kept[0][1:], (causal & (q - k < window))[1:])def test_decode_masks_follow_each_layer_device(monkeypatch):diff --git a/unsloth/kernels/flex_attention.py b/unsloth/kernels/flex_attention.pyindex cf7fe8f20db..45b91164bb9 100644--- a/unsloth/kernels/flex_attention.py+++ b/unsloth/kernels/flex_attention.py@@ -129,7 +129,7 @@ else:def sliding_window_masker(size = 4096):def sliding_window(b, h, q_idx, kv_idx):causal_mask = q_idx >= kv_idx- window_mask = q_idx - kv_idx <= size+ window_mask = q_idx - kv_idx < sizereturn causal_mask & window_maskreturn sliding_windowdiff --git a/unsloth/models/gemma2.py b/unsloth/models/gemma2.pyindex c744cad20fc..d45c2a816ec 100644--- a/unsloth/models/gemma2.py+++ b/unsloth/models/gemma2.py@@ -206,7 +206,7 @@ def Gemma2Attention_fast_forward(sliding_window = getattr(self.config, "sliding_window", None)if has_sliding_window:sliding_window = sliding_window if sliding_window is not None else kv_seq_len- window = (-1, -1) if kv_seq_len <= sliding_window else (sliding_window, sliding_window)+ window = (-1, -1) if kv_seq_len <= sliding_window else (sliding_window - 1, 0)if not hasattr(self, "_flash_attention_softmax_scale"):self._flash_attention_softmax_scale = 1.0 / (self.config.query_pre_attn_scalar**0.5)diff --git a/unsloth/models/llama.py b/unsloth/models/llama.pyindex 77dc277ed80..7b1fa1c00fa 100644--- a/unsloth/models/llama.py+++ b/unsloth/models/llama.py@@ -1081,6 +1081,27 @@ def _is_left_padded(attention_mask):return not bool(attention_mask[:, 0].all())+class Gemma2AttentionMaskConverter(AttentionMaskConverter):+ """Gemma2's window counts the current key as well as the preceding keys."""++ @staticmethod+ def _make_causal_mask(+ input_ids_shape,+ dtype,+ device,+ past_key_values_length = 0,+ sliding_window = None,+ ):+ # The legacy converter counts preceding keys; its static helper also accepts zero for a one-key window.+ return AttentionMaskConverter._make_causal_mask(+ input_ids_shape,+ dtype,+ device,+ past_key_values_length = past_key_values_length,+ sliding_window = sliding_window - 1 if sliding_window is not None else None,+ )++# Ported from transformers models/llama/modeling_llama.py#L825def LlamaModel_fast_forward(self,@@ -1297,7 +1318,7 @@ def LlamaModel_fast_forward(if attention_mask.dim() == 2:# The SDPA helper returns None for an all-ones mask; the softcapping kernels need a tensor.key_value_length = past_key_values_length + seq_length- dynamic_SWA_mask = AttentionMaskConverter(+ dynamic_SWA_mask = Gemma2AttentionMaskConverter(is_causal = True, sliding_window = self.config.sliding_window).to_4d(attention_mask, seq_length, inputs_embeds.dtype, key_value_length)dynamic_GA_mask = AttentionMaskConverter(is_causal = True).to_4d(@@ -1318,7 +1339,7 @@ def LlamaModel_fast_forward(n = self.max_seq_length# masked_fill makes things slower.self.SWA_mask = (- AttentionMaskConverter(+ Gemma2AttentionMaskConverter(is_causal = True,sliding_window = self.config.sliding_window,)