Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Gemma2: use flash_attn_with_kvcache for cached decoding (#12112)

Daniel Han379f14a8d831
diff --git a/tests/test_gemma2_flash_decode.py b/tests/test_gemma2_flash_decode.py
new file mode 100644
index 00000000000..e39f7998e62
--- /dev/null
+++ b/tests/test_gemma2_flash_decode.py
@@ -0,0 +1,170 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""Gemma2 cached decode through flash_attn_with_kvcache: routing, left padding and numerics."""
+
+import types
+
+import pytest
+
+import unsloth # noqa: F401 (must precede transformers)
+import unsloth.models.gemma2 as g2
+from real_accelerator import has_real_cuda
+
+torch = pytest.importorskip("torch")
+
+
+class _Captured(Exception):
+ pass
+
+
+def _decode_kwargs(
+ monkeypatch,
+ bsz,
+ flash,
+ second_device = "cpu",
+ mask_rows = None,
+ mask = None,
+):
+ seen = []
+
+ def attention(self, hidden_states, past_key_value, position_ids, attention_mask, **kw):
+ seen.append(dict(kw, attention_mask = attention_mask))
+ if len(seen) == 2:
+ raise _Captured
+ return hidden_states, past_key_value
+
+ monkeypatch.setattr(g2, "_flash_decode_usable", lambda config, hidden, mask: flash)
+ monkeypatch.setattr(g2, "DEVICE_COUNT", 0)
+ monkeypatch.setattr(g2, "Gemma2Attention_fast_forward_inference", attention)
+ monkeypatch.setattr(g2, "fast_rms_layernorm_inference_gemma", lambda ln, x, w: x)
+ monkeypatch.setattr(g2, "fast_geglu_inference", lambda mlp, x: x)
+ # No per-device fp32 buffers on CPU; the stubbed norms never read them.
+ monkeypatch.setattr(
+ g2, "per_layer_device", lambda layer: (torch.device(layer.device), slice(None))
+ )
+
+ hidden, cached = 8, 5
+ layer = types.SimpleNamespace(
+ device = "cpu",
+ input_layernorm = types.SimpleNamespace(weight = torch.ones(hidden)),
+ post_attention_layernorm = None,
+ pre_feedforward_layernorm = None,
+ post_feedforward_layernorm = None,
+ self_attn = None,
+ mlp = None,
+ )
+ config = types.SimpleNamespace(
+ hidden_size = hidden, sliding_window = 4096, torch_dtype = torch.float32
+ )
+ model = types.SimpleNamespace(
+ layers = [layer, types.SimpleNamespace(**{**vars(layer), "device": second_device})],
+ embed_tokens = lambda ids: torch.zeros(*ids.shape, hidden),
+ )
+ self = types.SimpleNamespace(model = model, config = config, max_seq_length = 64)
+ past = [(torch.zeros(bsz, 1, cached, 4), torch.zeros(bsz, 1, cached, 4))] * 2
+ attention_mask = torch.tensor(
+ [[1] * (cached + 1), [0, 0] + [1] * (cached - 1), [0] * 4 + [1] * 2]
+ )[mask_rows if mask_rows is not None else slice(0, bsz)]
+ if mask is not None:
+ attention_mask = torch.tensor(mask)
+ with pytest.raises(_Captured):
+ g2.Gemma2Model_fast_forward_inference(
+ self,
+ torch.zeros(bsz, 1, dtype = torch.long),
+ past,
+ torch.full((bsz, 1), cached),
+ attention_mask = attention_mask,
+ )
+ return seen
+
+
+def test_flash_decode_passes_left_padding_not_masks(monkeypatch):
+ seen = _decode_kwargs(monkeypatch, bsz = 3, flash = True)
+ for kw in seen:
+ assert kw["flash_decode"] is True
+ assert kw["attention_mask"] is None
+ assert kw["leftpad"].dtype == torch.int32
+ assert kw["leftpad"].tolist() == [0, 2, 4]
+
+
+def test_flash_decode_single_padded_row_keeps_leftpad(monkeypatch):
+ seen = _decode_kwargs(monkeypatch, bsz = 1, flash = True, mask_rows = [1])
+ assert [(kw["attention_mask"], kw["leftpad"].tolist()) for kw in seen] == [(None, [2])] * 2
+
+
+def test_flash_decode_leftpad_follows_layer_device(monkeypatch):
+ # Pipeline-parallel device maps: flash-attn rejects a cache_leftpad on another GPU.
+ seen = _decode_kwargs(monkeypatch, bsz = 3, flash = True, second_device = "meta")
+ assert [kw["leftpad"].device.type for kw in seen] == ["cpu", "meta"]
+
+
[email protected]("rows", [[[0, 1, 1, 1, 1, 1]], []])
[email protected]("gap_row", [[1, 1, 0, 1, 1, 1], [1, 1, 1, 1, 1, 0]])
+def test_flash_decode_falls_back_for_non_left_padding(monkeypatch, gap_row, rows):
+ mask = rows + [gap_row]
+ seen = _decode_kwargs(monkeypatch, bsz = len(mask), flash = True, mask = mask)
+ for kw in seen:
+ assert kw["flash_decode"] is False
+ assert kw["leftpad"] is None
+ assert isinstance(kw["attention_mask"], torch.Tensor)
+
+
+def test_manual_path_keeps_masks(monkeypatch):
+ seen = _decode_kwargs(monkeypatch, bsz = 2, flash = False)
+ for kw in seen:
+ assert kw["flash_decode"] is False
+ assert kw["leftpad"] is None
+ assert isinstance(kw["attention_mask"], torch.Tensor)
+
+
+def test_usable_refuses_unsupported_inputs(monkeypatch):
+ config = types.SimpleNamespace(head_dim = 256, hidden_size = 2304, num_attention_heads = 8)
+ monkeypatch.setattr(g2, "_FLASH_DECODE", True)
+ monkeypatch.setattr(g2, "_FLASH_DECODE_PROBED", {})
+ mask = torch.ones(2, 6, dtype = torch.long)
+ assert not g2._flash_decode_usable(config, torch.zeros(2, 1, 8, dtype = torch.float32), mask)
+ assert not g2._flash_decode_usable(
+ config, torch.zeros(2, 1, 8, dtype = torch.bfloat16), mask[:, None, None]
+ )
+ odd = types.SimpleNamespace(head_dim = 100, hidden_size = 800, num_attention_heads = 8)
+ assert not g2._flash_decode_usable(odd, torch.zeros(2, 1, 8, dtype = torch.bfloat16), mask)
+ monkeypatch.setattr(g2, "_FLASH_DECODE", False)
+ assert not g2._flash_decode_usable(config, torch.zeros(2, 1, 8, dtype = torch.bfloat16), mask)
+
+
+def _reference(Q, K_cache, V_cache, kv, leftpad, scale, softcap, window):
+ K = K_cache[:kv].permute(1, 2, 0, 3).float()
+ V = V_cache[:kv].permute(1, 2, 0, 3).float()
+ groups = Q.shape[1] // K.shape[1]
+ K, V = K.repeat_interleave(groups, 1), V.repeat_interleave(groups, 1)
+ A = (Q.float() * scale) @ K.transpose(2, 3)
+ A = softcap * torch.tanh(A / softcap)
+ j = torch.arange(kv, device = Q.device)
+ keep = j[None] >= leftpad[:, None].long()
+ if window is not None:
+ keep &= j[None] >= kv - window
+ A = A.masked_fill(~keep[:, None, None], float("-inf"))
+ return (torch.softmax(A, -1) @ V).transpose(1, 2)
+
+
[email protected](not has_real_cuda(), reason = "runs the flash_attn_with_kvcache CUDA kernel")
+ "n_heads,n_kv_heads,head_dim,scalar", [(8, 4, 256, 256), (32, 16, 128, 144)]
+)
[email protected]("window", [None, 64])
+def test_flash_decode_matches_reference(n_heads, n_kv_heads, head_dim, scalar, window):
+ pytest.importorskip("flash_attn")
+ if not g2._FLASH_DECODE:
+ pytest.skip("flash-attn softcapping unavailable")
+ torch.manual_seed(0)
+ bsz, kv, capacity = 3, 200, 456
+ cache = torch.randn(capacity, 2, bsz, n_kv_heads, head_dim, device = "cuda", dtype = torch.bfloat16)
+ K_cache, V_cache = cache[:, 0], cache[:, 1]
+ Q = torch.randn(bsz, n_heads, 1, head_dim, device = "cuda", dtype = torch.bfloat16)
+ leftpad = torch.tensor([0, 17, 150], device = "cuda", dtype = torch.int32)
+ scale = scalar**-0.5
+ out = g2._gemma2_flash_decode(Q, K_cache, V_cache, kv, leftpad, scale, 50.0, window)
+ ref = _reference(Q, K_cache, V_cache, kv, leftpad, scale, 50.0, window)
+ assert out.shape == (bsz, 1, n_heads, head_dim)
+ torch.testing.assert_close(out.float(), ref, atol = 2e-2, rtol = 2e-2)
diff --git a/unsloth/models/gemma2.py b/unsloth/models/gemma2.py
index c101a0aa861..916bb27f6ee 100644
--- a/unsloth/models/gemma2.py
+++ b/unsloth/models/gemma2.py
@@ -15,7 +15,7 @@ from .llama import (
original_apply_o,
)
from ._utils import __version__
-from ._utils import move_to_device, per_layer_device
+from ._utils import move_to_device, per_layer_device, DEVICE_TYPE
from unsloth_zoo.utils import _get_dtype, Version
from unsloth_zoo.hf_utils import dtype_from_config
from ..utils.packing import get_packed_info_from_kwargs
@@ -70,6 +70,62 @@ except:
if HAS_FLASH_ATTENTION_SOFTCAPPING:
from flash_attn import flash_attn_func
+# softcap, window_size and cache_leftpad all exist from flash-attn 2.6.3, the HAS_FLASH_ATTENTION_SOFTCAPPING floor.
+_FLASH_DECODE = (
+ HAS_FLASH_ATTENTION_SOFTCAPPING
+ and DEVICE_TYPE == "cuda"
+ and os.environ.get("UNSLOTH_GEMMA2_FLASH_DECODE", "1") != "0"
+)
+if _FLASH_DECODE:
+ try:
+ from flash_attn import flash_attn_with_kvcache
+ except Exception:
+ _FLASH_DECODE = False
+_FLASH_DECODE_PROBED = {}
+
+
+def _gemma2_flash_decode(Q, K_cache, V_cache, kv_seq_len, leftpad, scale, softcap, window):
+ """One-token attention over the paged cache [max_len, bsz, n_kv_heads, head_dim], left padded by leftpad."""
+ return flash_attn_with_kvcache(
+ Q.transpose(1, 2),
+ K_cache.permute(1, 0, 2, 3),
+ V_cache.permute(1, 0, 2, 3),
+ cache_seqlens = kv_seq_len,
+ cache_leftpad = leftpad,
+ softmax_scale = scale,
+ softcap = softcap,
+ # Keeps window - 1 earlier keys plus the current one, as HF and the manual path do.
+ window_size = (window - 1, 0) if window is not None else (-1, -1),
+ )
+
+
+def _flash_decode_usable(config, hidden_states, attention_mask):
+ if not _FLASH_DECODE or hidden_states.dtype not in (torch.float16, torch.bfloat16):
+ return False
+ if attention_mask is not None and attention_mask.dim() != 2:
+ return False
+ head_dim = getattr(config, "head_dim", config.hidden_size // config.num_attention_heads)
+ if head_dim > 256 or head_dim % 8 != 0:
+ return False
+ key = (hidden_states.device, hidden_states.dtype)
+ if key not in _FLASH_DECODE_PROBED:
+ # A wheel built without this GPU's arch fails at launch; find out before any layer relies on it.
+ try:
+ q = torch.zeros(
+ 1, 1, 1, head_dim, device = hidden_states.device, dtype = hidden_states.dtype
+ )
+ kv = torch.zeros(
+ 2, 1, 1, head_dim, device = hidden_states.device, dtype = hidden_states.dtype
+ )
+ _gemma2_flash_decode(q, kv, kv, 1, None, 1.0, 50.0, None)
+ _FLASH_DECODE_PROBED[key] = True
+ except Exception as error:
+ logger.warning_once(
+ f"Unsloth: flash-attn decoding failed on {key}, using the manual Gemma2 decode path: {error}"
+ )
+ _FLASH_DECODE_PROBED[key] = False
+ return _FLASH_DECODE_PROBED[key]
+
# Logit softcapping.
def Gemma2Attention_fast_forward(
@@ -394,6 +450,21 @@ def Gemma2Attention_fast_forward_inference(
Kn = self.paged_attention_K[:kv_seq_len].permute(1, 2, 0, 3)
Vn = self.paged_attention_V[:kv_seq_len].permute(1, 2, 0, 3)
+ if kwargs.get("flash_decode", False):
+ A = _gemma2_flash_decode(
+ Qn,
+ self.paged_attention_K,
+ self.paged_attention_V,
+ kv_seq_len,
+ kwargs.get("leftpad"),
+ self.scalar,
+ self.t,
+ self.config.sliding_window if use_sliding_window else None,
+ )
+ A = A.reshape(bsz, 1, attention_size)
+ A = fast_linear_forward(self.o_proj, A, out = self.temp_O)
+ return A, (Kn, Vn)
+
# Handle sliding windows
sliding_window = self.config.sliding_window
if use_sliding_window and kv_seq_len > sliding_window:
@@ -462,7 +533,21 @@ def Gemma2Model_fast_forward_inference(
bsz, q_len, hd = hidden_states.shape
seq_len = past_key_values[0][0].shape[-2]
- if bsz != 1:
+ flash_decode = _flash_decode_usable(self.config, hidden_states, attention_mask)
+ leftpad = None
+ masked_keys = False
+ if flash_decode and attention_mask is not None:
+ leftpad = (attention_mask.cumsum(-1) == 0).sum(-1, dtype = torch.int32)
+ # cache_leftpad only skips leading zeros; any other masked key needs the manual masks.
+ if not bool(((attention_mask != 0).sum(-1) + leftpad == attention_mask.shape[-1]).all()):
+ flash_decode = False
+ leftpad = None
+ masked_keys = True
+ if flash_decode:
+ # The kernel applies the window itself and skips each row's left padding.
+ SWA = None
+ GA = None
+ elif bsz != 1 or masked_keys:
SWA = _prepare_4d_causal_attention_mask_for_sdpa(
attention_mask,
(bsz, q_len),
@@ -489,6 +574,8 @@ def Gemma2Model_fast_forward_inference(
SWA = move_to_device(layer_device, SWA)
if GA is not None:
GA = move_to_device(layer_device, GA)
+ if leftpad is not None:
+ leftpad = move_to_device(layer_device, leftpad)
use_sliding_window = idx % 2 == 0
@@ -504,6 +591,8 @@ def Gemma2Model_fast_forward_inference(
attention_mask = SWA if use_sliding_window else GA,
do_prefill = not hasattr(decoder_layer.self_attn, "paged_attention"),
use_sliding_window = use_sliding_window,
+ flash_decode = flash_decode,
+ leftpad = leftpad,
)
hidden_states = fast_rms_layernorm_inference_gemma(
decoder_layer.post_attention_layernorm,