Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Native decode: read GQA KV heads in place instead of copying them per query head (#13271)
Daniel Hancc7e8aaeb2eb
diff --git a/tests/test_decode_gqa_in_place.py b/tests/test_decode_gqa_in_place.pynew file mode 100644index 00000000000..f6111f055c7--- /dev/null+++ b/tests/test_decode_gqa_in_place.py@@ -0,0 +1,251 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""CPU checks for the native decode path: grouped attention reads the KV heads in place instead of+repeating them per query head, and per-step scratch is only made on devices that hold layers."""++import types++import pytest++import unsloth # noqa: F401 (must precede transformers)+import torch+from torch.utils._python_dispatch import TorchDispatchMode++from unsloth.models import llama+import unsloth.models.gemma2 as g2++N_HEADS, N_KV, HEAD_DIM, HIDDEN = 8, 2, 16, 32+++def _attention(bsz, dtype):+ torch.manual_seed(0)++ def linear(i, o):+ layer = torch.nn.Linear(i, o, bias = False).to(dtype)+ layer.weight.requires_grad_(False)+ return layer++ config = types.SimpleNamespace(+ num_attention_heads = N_HEADS,+ num_key_value_heads = N_KV,+ hidden_size = HIDDEN,+ sliding_window = None,+ )+ # cos = 1 and sin = 0 make RoPE the identity, so the reference below needs no rotation.+ rotary = types.SimpleNamespace(+ extend_rope_embedding = lambda x, n: None,+ get_cached = lambda n, i: (torch.ones(4096, HEAD_DIM), torch.zeros(4096, HEAD_DIM)),+ )+ return types.SimpleNamespace(+ config = config,+ num_key_value_groups = N_HEADS // N_KV,+ head_dim = HEAD_DIM,+ q_proj = linear(HIDDEN, N_HEADS * HEAD_DIM),+ k_proj = linear(HIDDEN, N_KV * HEAD_DIM),+ v_proj = linear(HIDDEN, N_KV * HEAD_DIM),+ o_proj = linear(N_HEADS * HEAD_DIM, HIDDEN),+ rotary_emb = rotary,+ )+++def _step(attn, X, past, seq_len, do_prefill):+ bsz = X.shape[0]+ return llama.LlamaAttention_fast_forward_inference(+ attn,+ X,+ past,+ torch.full((bsz, 1), seq_len),+ attention_mask = None,+ do_prefill = do_prefill,+ )+++def _reference(attn, X, K, V):+ # Repeat K/V per query head, fp32 softmax, as the expand path did.+ bsz = X.shape[0]+ Q = (X @ attn.q_proj.weight.t()).view(bsz, N_HEADS, 1, HEAD_DIM).float()+ K = K.float().repeat_interleave(N_HEADS // N_KV, dim = 1)+ V = V.float().repeat_interleave(N_HEADS // N_KV, dim = 1)+ A = torch.softmax((Q / HEAD_DIM**0.5) @ K.transpose(2, 3), dim = -1) @ V+ return A.transpose(1, 2).reshape(bsz, 1, -1) @ attn.o_proj.weight.float().t()++[email protected]("dtype", [torch.float32, torch.bfloat16])+def test_bsz1_decode_matches_repeated_kv(dtype):+ attn = _attention(1, dtype)+ cached = 7+ past = (+ torch.randn(1, N_KV, cached, HEAD_DIM, dtype = dtype),+ torch.randn(1, N_KV, cached, HEAD_DIM, dtype = dtype),+ )+ X = torch.randn(1, 1, HIDDEN, dtype = dtype)+ out, (K, V) = _step(attn, X, past, cached, do_prefill = True)+ assert K.shape == (1, N_KV, cached + 1, HEAD_DIM)+ tol = 1e-5 if dtype == torch.float32 else 3e-2+ torch.testing.assert_close(out.float(), _reference(attn, X, K, V), atol = tol, rtol = tol)++ # Second step reuses the paged cache and the preallocated score buffer.+ X = torch.randn(1, 1, HIDDEN, dtype = dtype)+ out, (K, V) = _step(attn, X, (K, V), cached + 1, do_prefill = False)+ torch.testing.assert_close(out.float(), _reference(attn, X, K, V), atol = tol, rtol = tol)+++class _ShapeRecorder(TorchDispatchMode):+ def __init__(self):+ super().__init__()+ self.shapes = []++ def __torch_dispatch__(+ self,+ func,+ types,+ args = (),+ kwargs = None,+ ):+ out = func(*args, **(kwargs or {}))+ for t in out if isinstance(out, (tuple, list)) else (out,):+ if isinstance(t, torch.Tensor):+ self.shapes.append((str(func), tuple(t.shape)))+ return out+++def test_bsz1_decode_does_not_materialise_kv_per_query_head():+ attn = _attention(1, torch.float32)+ cached = 9+ past = (torch.randn(1, N_KV, cached, HEAD_DIM), torch.randn(1, N_KV, cached, HEAD_DIM))+ _, past = _step(attn, torch.randn(1, 1, HIDDEN), past, cached, do_prefill = True)+ with _ShapeRecorder() as rec:+ _step(attn, torch.randn(1, 1, HIDDEN), past, cached + 1, do_prefill = False)+ kv_len = cached + 2+ repeated = [+ (op, shape)+ for op, shape in rec.shapes+ if shape in ((1, N_HEADS, kv_len, HEAD_DIM), (1, N_KV, N_HEADS // N_KV, kv_len, HEAD_DIM))+ ]+ assert repeated == [], repeated++[email protected]("L", [5, 16, 21])+def test_gqa_helpers_write_into_head_layout_buffers(L):+ # 5: shorter than one aligned block; 16: aligned; 21: aligned prefix plus a remainder.+ torch.manual_seed(0)+ bsz, M = 2, 32+ Q = torch.randn(bsz, 1, N_HEADS, HEAD_DIM, dtype = torch.float64).transpose(1, 2)+ paged = torch.randn(M, 2, bsz, N_KV, HEAD_DIM, dtype = torch.float64)+ K = paged[:L, 0].permute(1, 2, 0, 3)+ V = paged[:L, 1].permute(1, 2, 0, 3)+ scores = torch.empty(bsz, N_HEADS, 1, M, dtype = torch.float64)[:, :, :, :L]+ A = llama._gqa_decode_scores(Q, K, scores)+ assert A.data_ptr() == scores.data_ptr()+ Kr = K.repeat_interleave(N_HEADS // N_KV, dim = 1)+ Vr = V.repeat_interleave(N_HEADS // N_KV, dim = 1)+ torch.testing.assert_close(A, Q @ Kr.transpose(2, 3))+ expected = A @ Vr+ out = llama._gqa_decode_values(A, V, Q)+ assert out.data_ptr() == Q.data_ptr()+ torch.testing.assert_close(out, expected)+++def _record_to(monkeypatch):+ seen = []+ original = torch.Tensor.to++ def to(self, *args, **kwargs):+ target = args[0] if args else kwargs.get("device")+ if isinstance(target, torch.device):+ seen.append(target)+ # No real accelerator here: stand in for the copy.+ return self if target.type == "cpu" else self.clone()+ return original(self, *args, **kwargs)++ monkeypatch.setattr(torch.Tensor, "to", to)+ return seen+++def test_llama_decode_scratch_only_on_layer_devices(monkeypatch):+ monkeypatch.setattr(llama, "DEVICE_COUNT", 4)+ monkeypatch.setattr(llama, "DEVICE_TYPE_TORCH", "cpu")+ monkeypatch.setattr(llama, "per_layer_device", lambda layer: (torch.device("cpu"), 0))+ monkeypatch.setattr(llama, "fast_rms_layernorm_inference", lambda ln, X, **kw: X)+ gates = []++ def attention(self, hidden_states, past_key_value, **kw):+ return hidden_states, past_key_value++ def mlp(+ self,+ X,+ temp_gate = None,+ temp_up = None,+ ):+ gates.append(temp_gate)+ return X++ forward = llama._LlamaModel_fast_forward_inference(attention, mlp)+ hidden, layers = 8, 3+ config = types.SimpleNamespace(+ hidden_size = hidden, intermediate_size = 16, torch_dtype = torch.float32+ )+ model = types.SimpleNamespace(+ layers = [+ types.SimpleNamespace(+ self_attn = None, mlp = None, input_layernorm = None, post_attention_layernorm = None+ )+ ]+ * layers,+ embed_tokens = lambda ids: torch.zeros(*ids.shape, hidden),+ norm = None,+ )+ self = types.SimpleNamespace(model = model, config = config, max_seq_length = 64)+ past = [(torch.zeros(1, 1, 5, 4), torch.zeros(1, 1, 5, 4))] * layers+ seen = _record_to(monkeypatch)+ forward(self, torch.zeros(1, 1, dtype = torch.long), past, torch.full((1, 1), 5))+ # Every layer is on device 0, so devices 1-3 must never receive a scratch copy.+ assert {d.index for d in seen if d.index is not None} == {0}, seen+ assert len(gates) == layers and all(g is gates[0] for g in gates)+++def test_gemma2_decode_norm_scratch_only_on_layer_devices(monkeypatch):+ made = []+ empty_like = torch.empty_like++ def recording_empty_like(+ t,+ *args,+ device = None,+ **kwargs,+ ):+ made.append(device)+ return empty_like(t, *args, **kwargs)++ class _Stop(Exception):+ pass++ def attention(self, hidden_states, past_key_value, position_ids, attention_mask, **kw):+ raise _Stop++ monkeypatch.setattr(torch, "empty_like", recording_empty_like)+ monkeypatch.setattr(g2, "DEVICE_COUNT", 3)+ monkeypatch.setattr(llama, "DEVICE_TYPE_TORCH", "cpu")+ monkeypatch.setattr(g2, "HAS_FLASH_ATTENTION_SOFTCAPPING", False)+ monkeypatch.setattr(g2, "Gemma2Attention_fast_forward_inference", attention)+ monkeypatch.setattr(g2, "fast_rms_layernorm_inference_gemma", lambda ln, x, w: x)+ monkeypatch.setattr(g2, "per_layer_device", lambda layer: (torch.device("cpu"), 0))+ hidden = 8+ layer = types.SimpleNamespace(+ input_layernorm = types.SimpleNamespace(weight = torch.ones(hidden)), self_attn = None+ )+ config = types.SimpleNamespace(+ hidden_size = hidden, sliding_window = 4096, torch_dtype = torch.float32+ )+ model = types.SimpleNamespace(+ layers = [layer, layer], embed_tokens = lambda ids: torch.zeros(*ids.shape, hidden)+ )+ self = types.SimpleNamespace(model = model, config = config, max_seq_length = 64)+ past = [(torch.zeros(1, 1, 5, 4), torch.zeros(1, 1, 5, 4))] * 2+ with pytest.raises(_Stop):+ g2.Gemma2Model_fast_forward_inference(+ self, torch.zeros(1, 1, dtype = torch.long), past, torch.full((1, 1), 5)+ )+ assert [getattr(d, "index", None) for d in made if d is not None] == [0], madediff --git a/tests/utils/test_single_row_cached_attention.py b/tests/utils/test_single_row_cached_attention.pyindex 7ef50fb951d..8b57cabdd84 100644--- a/tests/utils/test_single_row_cached_attention.py+++ b/tests/utils/test_single_row_cached_attention.py@@ -62,6 +62,15 @@ def decoder(request):"scaled_dot_product_attention": F.scaled_dot_product_attention,"SDPA_HAS_GQA": True,}+ # The decode paths share llama.py's grouped KV helpers (qwen3 imports them from there).+ llama_source = source.with_name("llama.py")+ helpers = [+ node+ for node in ast.parse(llama_source.read_text(encoding = "utf-8")).body+ if isinstance(node, ast.FunctionDef)+ and node.name in ("_gqa_decode_scores", "_gqa_decode_values")+ ]+ exec(compile(ast.Module(body = helpers, type_ignores = []), str(llama_source), "exec"), namespace)exec(compile(ast.Module(body = [function], type_ignores = []), str(source), "exec"), namespace)return namespace[name]diff --git a/unsloth/models/cohere.py b/unsloth/models/cohere.pyindex d770f23c6ba..f0519c0dd19 100644--- a/unsloth/models/cohere.py+++ b/unsloth/models/cohere.py@@ -13,6 +13,9 @@ from .llama import *from .llama import (original_apply_qkv,original_apply_o,+ _gqa_decode_scores,+ _gqa_decode_values,+ _LazyDeviceScratch,)from ._utils import __version__from ._utils import move_to_device, per_layer_device@@ -305,7 +308,7 @@ def CohereAttention_fast_forward_inference(self.temp_O = self.temp_QA[1][:, :, :hidden_size]self.attention = torch.empty(- (bsz, n_heads, 1, KV_CACHE_INCREMENT + seq_len),+ (bsz, n_heads, 1, KV_CACHE_INCREMENT + seq_len + (-seq_len) % 8),dtype = dtype,device = f"{DEVICE_TYPE_TORCH}:0",)@@ -388,9 +391,9 @@ def CohereAttention_fast_forward_inference(else:Knn, Vnn = Kn, Vn- # Grouped query attention.+ # Grouped query attention; bsz == 1 reads the KV heads in place below._, _, cached_len, _ = Knn.shape- if n_groups != 1:+ if bsz != 1 and n_groups != 1:Knn = Knn[:, :, None, :, :].expand(bsz, n_kv_heads, n_groups, cached_len, head_dim)Vnn = Vnn[:, :, None, :, :].expand(bsz, n_kv_heads, n_groups, cached_len, head_dim)Knn = Knn.reshape(bsz, n_heads, cached_len, head_dim)@@ -400,9 +403,9 @@ def CohereAttention_fast_forward_inference(Qn *= self.scalar# (Q * scalar) @ K beats (Q @ K) * scalar for stopping overflows; see ggerganov/llama.cpp#7805# (comment 2153349963).- A = torch_matmul(Qn, Knn.transpose(2, 3), out = self.attention[:, :, :, :cached_len])+ A = _gqa_decode_scores(Qn, Knn, self.attention[:, :, :, :cached_len])A[:] = torch_nn_functional_softmax(A, dim = -1, dtype = torch.float32)- A = torch_matmul(A, Vnn, out = Qn)+ A = _gqa_decode_values(A, Vnn, Qn)else:A = scaled_dot_product_attention(Qn, Knn, Vnn, attn_mask = attention_mask, is_causal = False)A = A.transpose(1, 2)@@ -419,13 +422,10 @@ def CohereModel_fast_forward_inference(position_ids,attention_mask = None,):- out_weights = tuple(- torch.empty_like(- self.model.layers[0].input_layernorm.weight,- dtype = torch.float32,- device = torch.device(x),- )- for x in range(DEVICE_COUNT)+ norm_weight = self.model.layers[0].input_layernorm.weight+ out_weights = _LazyDeviceScratch(+ DEVICE_COUNT,+ lambda device: torch.empty_like(norm_weight, dtype = torch.float32, device = device),)input_ids = input_ids[:, : self.max_seq_length]hidden_states = self.model.embed_tokens(input_ids)diff --git a/unsloth/models/falcon_h1.py b/unsloth/models/falcon_h1.pyindex 2c9302195a1..d93f0a6cb23 100644--- a/unsloth/models/falcon_h1.py+++ b/unsloth/models/falcon_h1.py@@ -26,6 +26,8 @@ from ..utils.attention_dispatch import (from .llama import (LlamaRotaryEmbedding,LlamaLinearScalingRotaryEmbedding,+ _gqa_decode_scores,+ _gqa_decode_values,_LlamaModel_fast_forward_inference,original_apply_qkv,original_apply_o,@@ -253,7 +255,9 @@ def FalconH1Attention_fast_forward_inference(self.temp_O = self.temp_QA[1][:, :, :hidden_size]self.attention = torch.empty(- (bsz, n_heads, 1, KV_CACHE_INCREMENT + seq_len), dtype = dtype, device = device+ (bsz, n_heads, 1, KV_CACHE_INCREMENT + seq_len + (-seq_len) % 8),+ dtype = dtype,+ device = device,)self.scalar = 1.0 / math_sqrt(self.head_dim)self.half_head_dim = head_dim // 2@@ -322,9 +326,9 @@ def FalconH1Attention_fast_forward_inference(else:Knn, Vnn = Kn, Vn- # Grouped query attention.+ # Grouped query attention; bsz == 1 reads the KV heads in place below._, _, cached_len, _ = Knn.shape- if bsz == 1 or not SDPA_HAS_GQA and n_groups != 1:+ if bsz != 1 and (not SDPA_HAS_GQA) and n_groups != 1:Knn = Knn[:, :, None, :, :].expand(bsz, n_kv_heads, n_groups, cached_len, head_dim)Vnn = Vnn[:, :, None, :, :].expand(bsz, n_kv_heads, n_groups, cached_len, head_dim)Knn = Knn.reshape(bsz, n_heads, cached_len, head_dim)@@ -334,9 +338,9 @@ def FalconH1Attention_fast_forward_inference(Qn *= self.scalar# (Q * scalar) @ K beats (Q @ K) * scalar for stopping overflows; see ggerganov/llama.cpp#7805# (comment 2153349963).- A = torch_matmul(Qn, Knn.transpose(2, 3), out = self.attention[:, :, :, :cached_len])+ A = _gqa_decode_scores(Qn, Knn, self.attention[:, :, :, :cached_len])A[:] = torch_nn_functional_softmax(A, dim = -1, dtype = torch.float32)- A = torch_matmul(A, Vnn, out = Qn)+ A = _gqa_decode_values(A, Vnn, Qn)else:if SDPA_HAS_GQA:A = scaled_dot_product_attention(diff --git a/unsloth/models/gemma.py b/unsloth/models/gemma.pyindex 2df1d8e6c3e..83778ae0f64 100644--- a/unsloth/models/gemma.py+++ b/unsloth/models/gemma.py@@ -10,7 +10,7 @@# limitations under the License.from .llama import *-from .llama import _get_rope_theta+from .llama import _get_rope_theta, _LazyDeviceScratchfrom ._utils import __version__from ._utils import move_to_device, per_layer_device, embedding_applies_scalefrom unsloth_zoo.utils import _get_dtype, Version@@ -157,13 +157,10 @@ def GemmaModel_fast_forward_inference(attention_mask = None,**kwargs,):- out_weights = tuple(- torch.empty_like(- self.model.layers[0].input_layernorm.weight,- dtype = torch.float32,- device = torch.device(x),- )- for x in range(DEVICE_COUNT)+ norm_weight = self.model.layers[0].input_layernorm.weight+ out_weights = _LazyDeviceScratch(+ DEVICE_COUNT,+ lambda device: torch.empty_like(norm_weight, dtype = torch.float32, device = device),)input_ids = input_ids[:, : self.max_seq_length]hidden_states = self.model.embed_tokens(input_ids)diff --git a/unsloth/models/gemma2.py b/unsloth/models/gemma2.pyindex 57a7bbb72d2..c744cad20fc 100644--- a/unsloth/models/gemma2.py+++ b/unsloth/models/gemma2.py@@ -13,6 +13,9 @@ from .llama import *from .llama import (original_apply_qkv,original_apply_o,+ _gqa_decode_scores,+ _gqa_decode_values,+ _LazyDeviceScratch,)from ._utils import __version__from ._utils import move_to_device, per_layer_device, DEVICE_TYPE@@ -395,7 +398,9 @@ def Gemma2Attention_fast_forward_inference(# Only for Gemma2.self.temp_O = torch.empty((bsz, 1, hidden_size), dtype = dtype, device = device)self.attention = torch.empty(- (bsz, n_heads, 1, KV_CACHE_INCREMENT + seq_len), dtype = dtype, device = device+ (bsz, n_heads, 1, KV_CACHE_INCREMENT + seq_len + (-seq_len) % 8),+ dtype = dtype,+ device = device,)# Gemma 9b should use 256, not hidden_size // num_attention_heads (224); 27b uses the derived value,@@ -477,19 +482,14 @@ def Gemma2Attention_fast_forward_inference(else:Knn, Vnn = Kn, Vn- # Grouped query attention._, _, cached_len, _ = Knn.shape- if n_groups != 1:- Knn = Knn[:, :, None, :, :].expand(bsz, n_kv_heads, n_groups, cached_len, head_dim)- Vnn = Vnn[:, :, None, :, :].expand(bsz, n_kv_heads, n_groups, cached_len, head_dim)- Knn = Knn.reshape(bsz, n_heads, cached_len, head_dim)- Vnn = Vnn.reshape(bsz, n_heads, cached_len, head_dim)# Gemma2 uses manual matmul for all batch sizes since SDPA lacks softcapping (tanh logit scaling);# if PyTorch adds a softcap param, consider SDPA for bsz > 1 to match llama/qwen3.Qn *= self.scalar# (Q * scalar) @ K beats (Q @ K) * scalar for stopping overflows; see ggerganov/llama.cpp#7805 (comment 2153349963).- A = torch_matmul(Qn, Knn.transpose(2, 3), out = self.attention[:, :, :, :cached_len])+ # Grouped query attention: the KV heads are read in place, not repeated per query head.+ A = _gqa_decode_scores(Qn, Knn, self.attention[:, :, :, :cached_len])# Softcapping must happen BEFORE the mask is applied; see google-deepmind/gemma _modules.py and# transformers gemma2 eager_attention_forward.@@ -504,7 +504,7 @@ def Gemma2Attention_fast_forward_inference(A += attention_maskA[:] = torch_nn_functional_softmax(A, dim = -1, dtype = torch.float32)- A = torch_matmul(A, Vnn, out = Qn)+ A = _gqa_decode_values(A, Vnn, Qn)A = A.transpose(1, 2)A = A.reshape(bsz, 1, attention_size)A = fast_linear_forward(self.o_proj, A, out = self.temp_O)@@ -520,13 +520,10 @@ def Gemma2Model_fast_forward_inference(attention_mask = None,**kwargs,):- out_weights = tuple(- torch.empty_like(- self.model.layers[0].input_layernorm.weight,- dtype = torch.float32,- device = torch.device(x),- )- for x in range(DEVICE_COUNT)+ norm_weight = self.model.layers[0].input_layernorm.weight+ out_weights = _LazyDeviceScratch(+ DEVICE_COUNT,+ lambda device: torch.empty_like(norm_weight, dtype = torch.float32, device = device),)input_ids = input_ids[:, : self.max_seq_length]hidden_states = self.model.embed_tokens(input_ids)diff --git a/unsloth/models/granite.py b/unsloth/models/granite.pyindex 51fc0c135bc..8b82ed25860 100644--- a/unsloth/models/granite.py+++ b/unsloth/models/granite.py@@ -27,6 +27,8 @@ from ..utils.attention_dispatch import (from .llama import (LlamaRotaryEmbedding,LlamaLinearScalingRotaryEmbedding,+ _gqa_decode_scores,+ _gqa_decode_values,original_apply_qkv,original_apply_o,)@@ -321,7 +323,9 @@ def GraniteAttention_fast_forward_inference(self.RH_Q = torch.empty((bsz, n_heads, 1, head_dim), dtype = dtype, device = device)self.temp_O = torch.empty((bsz, 1, hidden_size), dtype = dtype, device = device)self.attention = torch.empty(- (bsz, n_heads, 1, KV_CACHE_INCREMENT + seq_len), dtype = dtype, device = device+ (bsz, n_heads, 1, KV_CACHE_INCREMENT + seq_len + (-seq_len) % 8),+ dtype = dtype,+ device = device,)self.half_head_dim = head_dim // 2@@ -372,9 +376,9 @@ def GraniteAttention_fast_forward_inference(Kn = self.paged_attention_K[:kv_seq_len].permute(1, 2, 0, 3)Vn = self.paged_attention_V[:kv_seq_len].permute(1, 2, 0, 3)- # Grouped query attention.+ # Grouped query attention; bsz == 1 reads the KV heads in place below._, _, cached_len, _ = Kn.shape- if bsz == 1 or ((not SDPA_HAS_GQA) and n_groups != 1):+ if bsz != 1 and (not SDPA_HAS_GQA) and n_groups != 1:Kn = Kn[:, :, None, :, :].expand(bsz, n_kv_heads, n_groups, cached_len, head_dim)Vn = Vn[:, :, None, :, :].expand(bsz, n_kv_heads, n_groups, cached_len, head_dim)Kn = Kn.reshape(bsz, n_heads, cached_len, head_dim)@@ -382,9 +386,9 @@ def GraniteAttention_fast_forward_inference(if bsz == 1:Qn *= self.scaling- A = torch_matmul(Qn, Kn.transpose(2, 3), out = self.attention[:, :, :, :cached_len])+ A = _gqa_decode_scores(Qn, Kn, self.attention[:, :, :, :cached_len])A[:] = torch_nn_functional_softmax(A, dim = -1, dtype = torch.float32)- A = torch_matmul(A, Vn, out = Qn)+ A = _gqa_decode_values(A, Vn, Qn)else:if (attention_mask is not Nonediff --git a/unsloth/models/llama.py b/unsloth/models/llama.pyindex 18322792fd5..f0405d48af2 100644--- a/unsloth/models/llama.py+++ b/unsloth/models/llama.py@@ -531,6 +531,57 @@ def fix_prepare_inputs_for_generation(module):torch_matmul = torch.matmul+def _gqa_decode_scores(Qn, Kn, out):+ """Q @ K^T for one decode token without repeating K per query head. Q (bsz, n_heads, 1, hd)+ is viewed as (bsz, n_kv_heads, n_groups, hd) so each KV head is read in place; the scores+ land in out's (bsz, n_heads, 1, cached_len) layout."""+ bsz, n_heads, _, head_dim = Qn.shape+ _, n_kv_heads, cached_len, _ = Kn.shape+ n_groups = n_heads // n_kv_heads+ torch_matmul(+ Qn.view(bsz, n_kv_heads, n_groups, head_dim),+ Kn.transpose(2, 3),+ out = out.view(bsz, n_kv_heads, n_groups, cached_len),+ )+ return out+++def _gqa_decode_values(A, Vn, out):+ """softmax(QK^T) @ V for one decode token, grouped like _gqa_decode_scores; writes out.+ cuBLAS drops to a slow kernel when the reduction over the cache is not a multiple of 8, so+ the aligned prefix is one bmm and the last few positions are accumulated on top."""+ bsz, n_heads, _, cached_len = A.shape+ _, n_kv_heads, _, head_dim = Vn.shape+ n_groups = n_heads // n_kv_heads+ A = A.view(bsz * n_kv_heads, n_groups, cached_len)+ V = Vn.reshape(bsz * n_kv_heads, cached_len, head_dim)+ O = out.view(bsz * n_kv_heads, n_groups, head_dim)+ aligned = cached_len - cached_len % 8+ if aligned == 0 or aligned == cached_len:+ torch.bmm(A, V, out = O)+ else:+ torch.bmm(A[:, :, :aligned], V[:, :aligned], out = O)+ O.baddbmm_(A[:, :, aligned:], V[:, aligned:])+ return out+++class _LazyDeviceScratch(list):+ """Per-device decode scratch, made on first use, so only devices holding layers get one."""++ __slots__ = ("_make",)++ def __init__(self, device_count, make):+ super().__init__((None,) * device_count)+ self._make = make++ def __getitem__(self, index):+ buffer = list.__getitem__(self, index)+ if buffer is None:+ buffer = self._make(torch.device(DEVICE_TYPE_TORCH, index))+ list.__setitem__(self, index, buffer)+ return buffer++def LlamaAttention_fast_forward_inference(self,hidden_states: torch.Tensor,@@ -604,8 +655,11 @@ def LlamaAttention_fast_forward_inference(else:self.temp_O = self.temp_QA[1][:, :, :hidden_size]+ # Row stride kept a multiple of 8 so the grouped decode bmm stays on the fast cuBLAS path.self.attention = torch.empty(- (bsz, n_heads, 1, KV_CACHE_INCREMENT + seq_len), dtype = dtype, device = device+ (bsz, n_heads, 1, KV_CACHE_INCREMENT + seq_len + (-seq_len) % 8),+ dtype = dtype,+ device = device,)self.scalar = 1.0 / math_sqrt(self.head_dim)self.half_head_dim = head_dim // 2@@ -689,9 +743,9 @@ def LlamaAttention_fast_forward_inference(# Decode is more stable with boolean keep masks than additive bf16 masks.attention_mask = attention_mask.eq(0)- # Grouped query attention.+ # Grouped query attention; bsz == 1 reads the KV heads in place below._, _, cached_len, _ = Knn.shape- if bsz == 1 or ((not SDPA_HAS_GQA) and n_groups != 1):+ if bsz != 1 and (not SDPA_HAS_GQA) and n_groups != 1:Knn = Knn[:, :, None, :, :].expand(bsz, n_kv_heads, n_groups, cached_len, head_dim)Vnn = Vnn[:, :, None, :, :].expand(bsz, n_kv_heads, n_groups, cached_len, head_dim)Knn = Knn.reshape(bsz, n_heads, cached_len, head_dim)@@ -707,11 +761,11 @@ def LlamaAttention_fast_forward_inference(if bsz == 1:Qn *= self.scalar# (Q * scalar) @ K beats (Q @ K) * scalar for stopping overflows; see ggerganov/llama.cpp#7805.- A = torch_matmul(Qn, Knn.transpose(2, 3), out = self.attention[:, :, :, :cached_len])+ A = _gqa_decode_scores(Qn, Knn, self.attention[:, :, :, :cached_len])if attention_mask is not None:A.masked_fill_(~attention_mask, float("-inf"))A[:] = torch_nn_functional_softmax(A, dim = -1, dtype = torch.float32)- A = torch_matmul(A, Vnn, out = Qn)+ A = _gqa_decode_values(A, Vnn, Qn)else:if SDPA_HAS_GQA:A = scaled_dot_product_attention(@@ -1434,10 +1488,8 @@ def _LlamaModel_fast_forward_inference(temp_mlp = torch.empty((2, bsz, 1, mlp_size), dtype = X.dtype, device = f"{DEVICE_TYPE_TORCH}:0")- temp_gates, temp_ups = (- tuple(temp_mlp[0].to(torch.device(x)) for x in range(DEVICE_COUNT)),- tuple(temp_mlp[1].to(torch.device(x)) for x in range(DEVICE_COUNT)),- )+ temp_gates = _LazyDeviceScratch(DEVICE_COUNT, temp_mlp[0].to)+ temp_ups = _LazyDeviceScratch(DEVICE_COUNT, temp_mlp[1].to)seq_len = past_key_values[0][0].shape[-2]kv_seq_len = seq_len + 1diff --git a/unsloth/models/qwen3.py b/unsloth/models/qwen3.pyindex a1ffbb49700..7bcb519b1d1 100644--- a/unsloth/models/qwen3.py+++ b/unsloth/models/qwen3.py@@ -26,6 +26,8 @@ from ..utils.attention_dispatch import (from .llama import (LlamaRotaryEmbedding,LlamaLinearScalingRotaryEmbedding,+ _gqa_decode_scores,+ _gqa_decode_values,_LlamaModel_fast_forward_inference,original_apply_qkv,original_apply_o,@@ -222,7 +224,9 @@ def Qwen3Attention_fast_forward_inference(self.temp_O = self.temp_QA[1][:, :, :hidden_size]self.attention = torch.empty(- (bsz, n_heads, 1, KV_CACHE_INCREMENT + seq_len), dtype = dtype, device = device+ (bsz, n_heads, 1, KV_CACHE_INCREMENT + seq_len + (-seq_len) % 8),+ dtype = dtype,+ device = device,)self.scalar = 1.0 / math_sqrt(self.head_dim)self.half_head_dim = head_dim // 2@@ -318,9 +322,9 @@ def Qwen3Attention_fast_forward_inference(# Avoid SDPA GQA drift for batched masked decode.use_sdpa_gqa = False- # Grouped query attention.+ # Grouped query attention; bsz == 1 reads the KV heads in place below._, _, cached_len, _ = Knn.shape- if bsz == 1 or ((not use_sdpa_gqa) and n_groups != 1):+ if bsz != 1 and (not use_sdpa_gqa) and n_groups != 1:Knn = Knn[:, :, None, :, :].expand(bsz, n_kv_heads, n_groups, cached_len, head_dim)Vnn = Vnn[:, :, None, :, :].expand(bsz, n_kv_heads, n_groups, cached_len, head_dim)Knn = Knn.reshape(bsz, n_heads, cached_len, head_dim)@@ -330,11 +334,11 @@ def Qwen3Attention_fast_forward_inference(Qn *= self.scalar# (Q * scalar) @ K beats (Q @ K) * scalar for stopping overflows; see ggerganov/llama.cpp#7805# (comment 2153349963).- A = torch_matmul(Qn, Knn.transpose(2, 3), out = self.attention[:, :, :, :cached_len])+ A = _gqa_decode_scores(Qn, Knn, self.attention[:, :, :, :cached_len])if attention_mask is not None:A.masked_fill_(~attention_mask, float("-inf"))A[:] = torch_nn_functional_softmax(A, dim = -1, dtype = torch.float32)- A = torch_matmul(A, Vnn, out = Qn)+ A = _gqa_decode_values(A, Vnn, Qn)else:if use_sdpa_gqa:A = scaled_dot_product_attention(