Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Gemma2: use flash_attn_with_kvcache for cached decoding (#12112)
Daniel Han379f14a8d831
diff --git a/tests/test_gemma2_flash_decode.py b/tests/test_gemma2_flash_decode.pynew file mode 100644index 00000000000..e39f7998e62--- /dev/null+++ b/tests/test_gemma2_flash_decode.py@@ -0,0 +1,170 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""Gemma2 cached decode through flash_attn_with_kvcache: routing, left padding and numerics."""++import types++import pytest++import unsloth # noqa: F401 (must precede transformers)+import unsloth.models.gemma2 as g2+from real_accelerator import has_real_cuda++torch = pytest.importorskip("torch")+++class _Captured(Exception):+ pass+++def _decode_kwargs(+ monkeypatch,+ bsz,+ flash,+ second_device = "cpu",+ mask_rows = None,+ mask = None,+):+ seen = []++ def attention(self, hidden_states, past_key_value, position_ids, attention_mask, **kw):+ seen.append(dict(kw, attention_mask = attention_mask))+ if len(seen) == 2:+ raise _Captured+ return hidden_states, past_key_value++ monkeypatch.setattr(g2, "_flash_decode_usable", lambda config, hidden, mask: flash)+ monkeypatch.setattr(g2, "DEVICE_COUNT", 0)+ monkeypatch.setattr(g2, "Gemma2Attention_fast_forward_inference", attention)+ monkeypatch.setattr(g2, "fast_rms_layernorm_inference_gemma", lambda ln, x, w: x)+ monkeypatch.setattr(g2, "fast_geglu_inference", lambda mlp, x: x)+ # No per-device fp32 buffers on CPU; the stubbed norms never read them.+ monkeypatch.setattr(+ g2, "per_layer_device", lambda layer: (torch.device(layer.device), slice(None))+ )++ hidden, cached = 8, 5+ layer = types.SimpleNamespace(+ device = "cpu",+ input_layernorm = types.SimpleNamespace(weight = torch.ones(hidden)),+ post_attention_layernorm = None,+ pre_feedforward_layernorm = None,+ post_feedforward_layernorm = None,+ self_attn = None,+ mlp = None,+ )+ config = types.SimpleNamespace(+ hidden_size = hidden, sliding_window = 4096, torch_dtype = torch.float32+ )+ model = types.SimpleNamespace(+ layers = [layer, types.SimpleNamespace(**{**vars(layer), "device": second_device})],+ embed_tokens = lambda ids: torch.zeros(*ids.shape, hidden),+ )+ self = types.SimpleNamespace(model = model, config = config, max_seq_length = 64)+ past = [(torch.zeros(bsz, 1, cached, 4), torch.zeros(bsz, 1, cached, 4))] * 2+ attention_mask = torch.tensor(+ [[1] * (cached + 1), [0, 0] + [1] * (cached - 1), [0] * 4 + [1] * 2]+ )[mask_rows if mask_rows is not None else slice(0, bsz)]+ if mask is not None:+ attention_mask = torch.tensor(mask)+ with pytest.raises(_Captured):+ g2.Gemma2Model_fast_forward_inference(+ self,+ torch.zeros(bsz, 1, dtype = torch.long),+ past,+ torch.full((bsz, 1), cached),+ attention_mask = attention_mask,+ )+ return seen+++def test_flash_decode_passes_left_padding_not_masks(monkeypatch):+ seen = _decode_kwargs(monkeypatch, bsz = 3, flash = True)+ for kw in seen:+ assert kw["flash_decode"] is True+ assert kw["attention_mask"] is None+ assert kw["leftpad"].dtype == torch.int32+ assert kw["leftpad"].tolist() == [0, 2, 4]+++def test_flash_decode_single_padded_row_keeps_leftpad(monkeypatch):+ seen = _decode_kwargs(monkeypatch, bsz = 1, flash = True, mask_rows = [1])+ assert [(kw["attention_mask"], kw["leftpad"].tolist()) for kw in seen] == [(None, [2])] * 2+++def test_flash_decode_leftpad_follows_layer_device(monkeypatch):+ # Pipeline-parallel device maps: flash-attn rejects a cache_leftpad on another GPU.+ seen = _decode_kwargs(monkeypatch, bsz = 3, flash = True, second_device = "meta")+ assert [kw["leftpad"].device.type for kw in seen] == ["cpu", "meta"]++[email protected]("rows", [[[0, 1, 1, 1, 1, 1]], []])[email protected]("gap_row", [[1, 1, 0, 1, 1, 1], [1, 1, 1, 1, 1, 0]])+def test_flash_decode_falls_back_for_non_left_padding(monkeypatch, gap_row, rows):+ mask = rows + [gap_row]+ seen = _decode_kwargs(monkeypatch, bsz = len(mask), flash = True, mask = mask)+ for kw in seen:+ assert kw["flash_decode"] is False+ assert kw["leftpad"] is None+ assert isinstance(kw["attention_mask"], torch.Tensor)+++def test_manual_path_keeps_masks(monkeypatch):+ seen = _decode_kwargs(monkeypatch, bsz = 2, flash = False)+ for kw in seen:+ assert kw["flash_decode"] is False+ assert kw["leftpad"] is None+ assert isinstance(kw["attention_mask"], torch.Tensor)+++def test_usable_refuses_unsupported_inputs(monkeypatch):+ config = types.SimpleNamespace(head_dim = 256, hidden_size = 2304, num_attention_heads = 8)+ monkeypatch.setattr(g2, "_FLASH_DECODE", True)+ monkeypatch.setattr(g2, "_FLASH_DECODE_PROBED", {})+ mask = torch.ones(2, 6, dtype = torch.long)+ assert not g2._flash_decode_usable(config, torch.zeros(2, 1, 8, dtype = torch.float32), mask)+ assert not g2._flash_decode_usable(+ config, torch.zeros(2, 1, 8, dtype = torch.bfloat16), mask[:, None, None]+ )+ odd = types.SimpleNamespace(head_dim = 100, hidden_size = 800, num_attention_heads = 8)+ assert not g2._flash_decode_usable(odd, torch.zeros(2, 1, 8, dtype = torch.bfloat16), mask)+ monkeypatch.setattr(g2, "_FLASH_DECODE", False)+ assert not g2._flash_decode_usable(config, torch.zeros(2, 1, 8, dtype = torch.bfloat16), mask)+++def _reference(Q, K_cache, V_cache, kv, leftpad, scale, softcap, window):+ K = K_cache[:kv].permute(1, 2, 0, 3).float()+ V = V_cache[:kv].permute(1, 2, 0, 3).float()+ groups = Q.shape[1] // K.shape[1]+ K, V = K.repeat_interleave(groups, 1), V.repeat_interleave(groups, 1)+ A = (Q.float() * scale) @ K.transpose(2, 3)+ A = softcap * torch.tanh(A / softcap)+ j = torch.arange(kv, device = Q.device)+ keep = j[None] >= leftpad[:, None].long()+ if window is not None:+ keep &= j[None] >= kv - window+ A = A.masked_fill(~keep[:, None, None], float("-inf"))+ return (torch.softmax(A, -1) @ V).transpose(1, 2)++[email protected](not has_real_cuda(), reason = "runs the flash_attn_with_kvcache CUDA kernel")+ "n_heads,n_kv_heads,head_dim,scalar", [(8, 4, 256, 256), (32, 16, 128, 144)]+)[email protected]("window", [None, 64])+def test_flash_decode_matches_reference(n_heads, n_kv_heads, head_dim, scalar, window):+ pytest.importorskip("flash_attn")+ if not g2._FLASH_DECODE:+ pytest.skip("flash-attn softcapping unavailable")+ torch.manual_seed(0)+ bsz, kv, capacity = 3, 200, 456+ cache = torch.randn(capacity, 2, bsz, n_kv_heads, head_dim, device = "cuda", dtype = torch.bfloat16)+ K_cache, V_cache = cache[:, 0], cache[:, 1]+ Q = torch.randn(bsz, n_heads, 1, head_dim, device = "cuda", dtype = torch.bfloat16)+ leftpad = torch.tensor([0, 17, 150], device = "cuda", dtype = torch.int32)+ scale = scalar**-0.5+ out = g2._gemma2_flash_decode(Q, K_cache, V_cache, kv, leftpad, scale, 50.0, window)+ ref = _reference(Q, K_cache, V_cache, kv, leftpad, scale, 50.0, window)+ assert out.shape == (bsz, 1, n_heads, head_dim)+ torch.testing.assert_close(out.float(), ref, atol = 2e-2, rtol = 2e-2)diff --git a/unsloth/models/gemma2.py b/unsloth/models/gemma2.pyindex c101a0aa861..916bb27f6ee 100644--- a/unsloth/models/gemma2.py+++ b/unsloth/models/gemma2.py@@ -15,7 +15,7 @@ from .llama import (original_apply_o,)from ._utils import __version__-from ._utils import move_to_device, per_layer_device+from ._utils import move_to_device, per_layer_device, DEVICE_TYPEfrom unsloth_zoo.utils import _get_dtype, Versionfrom unsloth_zoo.hf_utils import dtype_from_configfrom ..utils.packing import get_packed_info_from_kwargs@@ -70,6 +70,62 @@ except:if HAS_FLASH_ATTENTION_SOFTCAPPING:from flash_attn import flash_attn_func+# softcap, window_size and cache_leftpad all exist from flash-attn 2.6.3, the HAS_FLASH_ATTENTION_SOFTCAPPING floor.+_FLASH_DECODE = (+ HAS_FLASH_ATTENTION_SOFTCAPPING+ and DEVICE_TYPE == "cuda"+ and os.environ.get("UNSLOTH_GEMMA2_FLASH_DECODE", "1") != "0"+)+if _FLASH_DECODE:+ try:+ from flash_attn import flash_attn_with_kvcache+ except Exception:+ _FLASH_DECODE = False+_FLASH_DECODE_PROBED = {}+++def _gemma2_flash_decode(Q, K_cache, V_cache, kv_seq_len, leftpad, scale, softcap, window):+ """One-token attention over the paged cache [max_len, bsz, n_kv_heads, head_dim], left padded by leftpad."""+ return flash_attn_with_kvcache(+ Q.transpose(1, 2),+ K_cache.permute(1, 0, 2, 3),+ V_cache.permute(1, 0, 2, 3),+ cache_seqlens = kv_seq_len,+ cache_leftpad = leftpad,+ softmax_scale = scale,+ softcap = softcap,+ # Keeps window - 1 earlier keys plus the current one, as HF and the manual path do.+ window_size = (window - 1, 0) if window is not None else (-1, -1),+ )+++def _flash_decode_usable(config, hidden_states, attention_mask):+ if not _FLASH_DECODE or hidden_states.dtype not in (torch.float16, torch.bfloat16):+ return False+ if attention_mask is not None and attention_mask.dim() != 2:+ return False+ head_dim = getattr(config, "head_dim", config.hidden_size // config.num_attention_heads)+ if head_dim > 256 or head_dim % 8 != 0:+ return False+ key = (hidden_states.device, hidden_states.dtype)+ if key not in _FLASH_DECODE_PROBED:+ # A wheel built without this GPU's arch fails at launch; find out before any layer relies on it.+ try:+ q = torch.zeros(+ 1, 1, 1, head_dim, device = hidden_states.device, dtype = hidden_states.dtype+ )+ kv = torch.zeros(+ 2, 1, 1, head_dim, device = hidden_states.device, dtype = hidden_states.dtype+ )+ _gemma2_flash_decode(q, kv, kv, 1, None, 1.0, 50.0, None)+ _FLASH_DECODE_PROBED[key] = True+ except Exception as error:+ logger.warning_once(+ f"Unsloth: flash-attn decoding failed on {key}, using the manual Gemma2 decode path: {error}"+ )+ _FLASH_DECODE_PROBED[key] = False+ return _FLASH_DECODE_PROBED[key]+# Logit softcapping.def Gemma2Attention_fast_forward(@@ -394,6 +450,21 @@ def Gemma2Attention_fast_forward_inference(Kn = self.paged_attention_K[:kv_seq_len].permute(1, 2, 0, 3)Vn = self.paged_attention_V[:kv_seq_len].permute(1, 2, 0, 3)+ if kwargs.get("flash_decode", False):+ A = _gemma2_flash_decode(+ Qn,+ self.paged_attention_K,+ self.paged_attention_V,+ kv_seq_len,+ kwargs.get("leftpad"),+ self.scalar,+ self.t,+ self.config.sliding_window if use_sliding_window else None,+ )+ A = A.reshape(bsz, 1, attention_size)+ A = fast_linear_forward(self.o_proj, A, out = self.temp_O)+ return A, (Kn, Vn)+# Handle sliding windowssliding_window = self.config.sliding_windowif use_sliding_window and kv_seq_len > sliding_window:@@ -462,7 +533,21 @@ def Gemma2Model_fast_forward_inference(bsz, q_len, hd = hidden_states.shapeseq_len = past_key_values[0][0].shape[-2]- if bsz != 1:+ flash_decode = _flash_decode_usable(self.config, hidden_states, attention_mask)+ leftpad = None+ masked_keys = False+ if flash_decode and attention_mask is not None:+ leftpad = (attention_mask.cumsum(-1) == 0).sum(-1, dtype = torch.int32)+ # cache_leftpad only skips leading zeros; any other masked key needs the manual masks.+ if not bool(((attention_mask != 0).sum(-1) + leftpad == attention_mask.shape[-1]).all()):+ flash_decode = False+ leftpad = None+ masked_keys = True+ if flash_decode:+ # The kernel applies the window itself and skips each row's left padding.+ SWA = None+ GA = None+ elif bsz != 1 or masked_keys:SWA = _prepare_4d_causal_attention_mask_for_sdpa(attention_mask,(bsz, q_len),@@ -489,6 +574,8 @@ def Gemma2Model_fast_forward_inference(SWA = move_to_device(layer_device, SWA)if GA is not None:GA = move_to_device(layer_device, GA)+ if leftpad is not None:+ leftpad = move_to_device(layer_device, leftpad)use_sliding_window = idx % 2 == 0@@ -504,6 +591,8 @@ def Gemma2Model_fast_forward_inference(attention_mask = SWA if use_sliding_window else GA,do_prefill = not hasattr(decoder_layer.self_attn, "paged_attention"),use_sliding_window = use_sliding_window,+ flash_decode = flash_decode,+ leftpad = leftpad,)hidden_states = fast_rms_layernorm_inference_gemma(decoder_layer.post_attention_layernorm,