Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Native decode: read GQA KV heads in place instead of copying them per query head (#13271)

Daniel Hancc7e8aaeb2eb
diff --git a/tests/test_decode_gqa_in_place.py b/tests/test_decode_gqa_in_place.py
new file mode 100644
index 00000000000..f6111f055c7
--- /dev/null
+++ b/tests/test_decode_gqa_in_place.py
@@ -0,0 +1,251 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""CPU checks for the native decode path: grouped attention reads the KV heads in place instead of
+repeating them per query head, and per-step scratch is only made on devices that hold layers."""
+
+import types
+
+import pytest
+
+import unsloth # noqa: F401 (must precede transformers)
+import torch
+from torch.utils._python_dispatch import TorchDispatchMode
+
+from unsloth.models import llama
+import unsloth.models.gemma2 as g2
+
+N_HEADS, N_KV, HEAD_DIM, HIDDEN = 8, 2, 16, 32
+
+
+def _attention(bsz, dtype):
+ torch.manual_seed(0)
+
+ def linear(i, o):
+ layer = torch.nn.Linear(i, o, bias = False).to(dtype)
+ layer.weight.requires_grad_(False)
+ return layer
+
+ config = types.SimpleNamespace(
+ num_attention_heads = N_HEADS,
+ num_key_value_heads = N_KV,
+ hidden_size = HIDDEN,
+ sliding_window = None,
+ )
+ # cos = 1 and sin = 0 make RoPE the identity, so the reference below needs no rotation.
+ rotary = types.SimpleNamespace(
+ extend_rope_embedding = lambda x, n: None,
+ get_cached = lambda n, i: (torch.ones(4096, HEAD_DIM), torch.zeros(4096, HEAD_DIM)),
+ )
+ return types.SimpleNamespace(
+ config = config,
+ num_key_value_groups = N_HEADS // N_KV,
+ head_dim = HEAD_DIM,
+ q_proj = linear(HIDDEN, N_HEADS * HEAD_DIM),
+ k_proj = linear(HIDDEN, N_KV * HEAD_DIM),
+ v_proj = linear(HIDDEN, N_KV * HEAD_DIM),
+ o_proj = linear(N_HEADS * HEAD_DIM, HIDDEN),
+ rotary_emb = rotary,
+ )
+
+
+def _step(attn, X, past, seq_len, do_prefill):
+ bsz = X.shape[0]
+ return llama.LlamaAttention_fast_forward_inference(
+ attn,
+ X,
+ past,
+ torch.full((bsz, 1), seq_len),
+ attention_mask = None,
+ do_prefill = do_prefill,
+ )
+
+
+def _reference(attn, X, K, V):
+ # Repeat K/V per query head, fp32 softmax, as the expand path did.
+ bsz = X.shape[0]
+ Q = (X @ attn.q_proj.weight.t()).view(bsz, N_HEADS, 1, HEAD_DIM).float()
+ K = K.float().repeat_interleave(N_HEADS // N_KV, dim = 1)
+ V = V.float().repeat_interleave(N_HEADS // N_KV, dim = 1)
+ A = torch.softmax((Q / HEAD_DIM**0.5) @ K.transpose(2, 3), dim = -1) @ V
+ return A.transpose(1, 2).reshape(bsz, 1, -1) @ attn.o_proj.weight.float().t()
+
+
[email protected]("dtype", [torch.float32, torch.bfloat16])
+def test_bsz1_decode_matches_repeated_kv(dtype):
+ attn = _attention(1, dtype)
+ cached = 7
+ past = (
+ torch.randn(1, N_KV, cached, HEAD_DIM, dtype = dtype),
+ torch.randn(1, N_KV, cached, HEAD_DIM, dtype = dtype),
+ )
+ X = torch.randn(1, 1, HIDDEN, dtype = dtype)
+ out, (K, V) = _step(attn, X, past, cached, do_prefill = True)
+ assert K.shape == (1, N_KV, cached + 1, HEAD_DIM)
+ tol = 1e-5 if dtype == torch.float32 else 3e-2
+ torch.testing.assert_close(out.float(), _reference(attn, X, K, V), atol = tol, rtol = tol)
+
+ # Second step reuses the paged cache and the preallocated score buffer.
+ X = torch.randn(1, 1, HIDDEN, dtype = dtype)
+ out, (K, V) = _step(attn, X, (K, V), cached + 1, do_prefill = False)
+ torch.testing.assert_close(out.float(), _reference(attn, X, K, V), atol = tol, rtol = tol)
+
+
+class _ShapeRecorder(TorchDispatchMode):
+ def __init__(self):
+ super().__init__()
+ self.shapes = []
+
+ def __torch_dispatch__(
+ self,
+ func,
+ types,
+ args = (),
+ kwargs = None,
+ ):
+ out = func(*args, **(kwargs or {}))
+ for t in out if isinstance(out, (tuple, list)) else (out,):
+ if isinstance(t, torch.Tensor):
+ self.shapes.append((str(func), tuple(t.shape)))
+ return out
+
+
+def test_bsz1_decode_does_not_materialise_kv_per_query_head():
+ attn = _attention(1, torch.float32)
+ cached = 9
+ past = (torch.randn(1, N_KV, cached, HEAD_DIM), torch.randn(1, N_KV, cached, HEAD_DIM))
+ _, past = _step(attn, torch.randn(1, 1, HIDDEN), past, cached, do_prefill = True)
+ with _ShapeRecorder() as rec:
+ _step(attn, torch.randn(1, 1, HIDDEN), past, cached + 1, do_prefill = False)
+ kv_len = cached + 2
+ repeated = [
+ (op, shape)
+ for op, shape in rec.shapes
+ if shape in ((1, N_HEADS, kv_len, HEAD_DIM), (1, N_KV, N_HEADS // N_KV, kv_len, HEAD_DIM))
+ ]
+ assert repeated == [], repeated
+
+
[email protected]("L", [5, 16, 21])
+def test_gqa_helpers_write_into_head_layout_buffers(L):
+ # 5: shorter than one aligned block; 16: aligned; 21: aligned prefix plus a remainder.
+ torch.manual_seed(0)
+ bsz, M = 2, 32
+ Q = torch.randn(bsz, 1, N_HEADS, HEAD_DIM, dtype = torch.float64).transpose(1, 2)
+ paged = torch.randn(M, 2, bsz, N_KV, HEAD_DIM, dtype = torch.float64)
+ K = paged[:L, 0].permute(1, 2, 0, 3)
+ V = paged[:L, 1].permute(1, 2, 0, 3)
+ scores = torch.empty(bsz, N_HEADS, 1, M, dtype = torch.float64)[:, :, :, :L]
+ A = llama._gqa_decode_scores(Q, K, scores)
+ assert A.data_ptr() == scores.data_ptr()
+ Kr = K.repeat_interleave(N_HEADS // N_KV, dim = 1)
+ Vr = V.repeat_interleave(N_HEADS // N_KV, dim = 1)
+ torch.testing.assert_close(A, Q @ Kr.transpose(2, 3))
+ expected = A @ Vr
+ out = llama._gqa_decode_values(A, V, Q)
+ assert out.data_ptr() == Q.data_ptr()
+ torch.testing.assert_close(out, expected)
+
+
+def _record_to(monkeypatch):
+ seen = []
+ original = torch.Tensor.to
+
+ def to(self, *args, **kwargs):
+ target = args[0] if args else kwargs.get("device")
+ if isinstance(target, torch.device):
+ seen.append(target)
+ # No real accelerator here: stand in for the copy.
+ return self if target.type == "cpu" else self.clone()
+ return original(self, *args, **kwargs)
+
+ monkeypatch.setattr(torch.Tensor, "to", to)
+ return seen
+
+
+def test_llama_decode_scratch_only_on_layer_devices(monkeypatch):
+ monkeypatch.setattr(llama, "DEVICE_COUNT", 4)
+ monkeypatch.setattr(llama, "DEVICE_TYPE_TORCH", "cpu")
+ monkeypatch.setattr(llama, "per_layer_device", lambda layer: (torch.device("cpu"), 0))
+ monkeypatch.setattr(llama, "fast_rms_layernorm_inference", lambda ln, X, **kw: X)
+ gates = []
+
+ def attention(self, hidden_states, past_key_value, **kw):
+ return hidden_states, past_key_value
+
+ def mlp(
+ self,
+ X,
+ temp_gate = None,
+ temp_up = None,
+ ):
+ gates.append(temp_gate)
+ return X
+
+ forward = llama._LlamaModel_fast_forward_inference(attention, mlp)
+ hidden, layers = 8, 3
+ config = types.SimpleNamespace(
+ hidden_size = hidden, intermediate_size = 16, torch_dtype = torch.float32
+ )
+ model = types.SimpleNamespace(
+ layers = [
+ types.SimpleNamespace(
+ self_attn = None, mlp = None, input_layernorm = None, post_attention_layernorm = None
+ )
+ ]
+ * layers,
+ embed_tokens = lambda ids: torch.zeros(*ids.shape, hidden),
+ norm = None,
+ )
+ self = types.SimpleNamespace(model = model, config = config, max_seq_length = 64)
+ past = [(torch.zeros(1, 1, 5, 4), torch.zeros(1, 1, 5, 4))] * layers
+ seen = _record_to(monkeypatch)
+ forward(self, torch.zeros(1, 1, dtype = torch.long), past, torch.full((1, 1), 5))
+ # Every layer is on device 0, so devices 1-3 must never receive a scratch copy.
+ assert {d.index for d in seen if d.index is not None} == {0}, seen
+ assert len(gates) == layers and all(g is gates[0] for g in gates)
+
+
+def test_gemma2_decode_norm_scratch_only_on_layer_devices(monkeypatch):
+ made = []
+ empty_like = torch.empty_like
+
+ def recording_empty_like(
+ t,
+ *args,
+ device = None,
+ **kwargs,
+ ):
+ made.append(device)
+ return empty_like(t, *args, **kwargs)
+
+ class _Stop(Exception):
+ pass
+
+ def attention(self, hidden_states, past_key_value, position_ids, attention_mask, **kw):
+ raise _Stop
+
+ monkeypatch.setattr(torch, "empty_like", recording_empty_like)
+ monkeypatch.setattr(g2, "DEVICE_COUNT", 3)
+ monkeypatch.setattr(llama, "DEVICE_TYPE_TORCH", "cpu")
+ monkeypatch.setattr(g2, "HAS_FLASH_ATTENTION_SOFTCAPPING", False)
+ monkeypatch.setattr(g2, "Gemma2Attention_fast_forward_inference", attention)
+ monkeypatch.setattr(g2, "fast_rms_layernorm_inference_gemma", lambda ln, x, w: x)
+ monkeypatch.setattr(g2, "per_layer_device", lambda layer: (torch.device("cpu"), 0))
+ hidden = 8
+ layer = types.SimpleNamespace(
+ input_layernorm = types.SimpleNamespace(weight = torch.ones(hidden)), self_attn = None
+ )
+ config = types.SimpleNamespace(
+ hidden_size = hidden, sliding_window = 4096, torch_dtype = torch.float32
+ )
+ model = types.SimpleNamespace(
+ layers = [layer, layer], embed_tokens = lambda ids: torch.zeros(*ids.shape, hidden)
+ )
+ self = types.SimpleNamespace(model = model, config = config, max_seq_length = 64)
+ past = [(torch.zeros(1, 1, 5, 4), torch.zeros(1, 1, 5, 4))] * 2
+ with pytest.raises(_Stop):
+ g2.Gemma2Model_fast_forward_inference(
+ self, torch.zeros(1, 1, dtype = torch.long), past, torch.full((1, 1), 5)
+ )
+ assert [getattr(d, "index", None) for d in made if d is not None] == [0], made
diff --git a/tests/utils/test_single_row_cached_attention.py b/tests/utils/test_single_row_cached_attention.py
index 7ef50fb951d..8b57cabdd84 100644
--- a/tests/utils/test_single_row_cached_attention.py
+++ b/tests/utils/test_single_row_cached_attention.py
@@ -62,6 +62,15 @@ def decoder(request):
"scaled_dot_product_attention": F.scaled_dot_product_attention,
"SDPA_HAS_GQA": True,
}
+ # The decode paths share llama.py's grouped KV helpers (qwen3 imports them from there).
+ llama_source = source.with_name("llama.py")
+ helpers = [
+ node
+ for node in ast.parse(llama_source.read_text(encoding = "utf-8")).body
+ if isinstance(node, ast.FunctionDef)
+ and node.name in ("_gqa_decode_scores", "_gqa_decode_values")
+ ]
+ exec(compile(ast.Module(body = helpers, type_ignores = []), str(llama_source), "exec"), namespace)
exec(compile(ast.Module(body = [function], type_ignores = []), str(source), "exec"), namespace)
return namespace[name]
diff --git a/unsloth/models/cohere.py b/unsloth/models/cohere.py
index d770f23c6ba..f0519c0dd19 100644
--- a/unsloth/models/cohere.py
+++ b/unsloth/models/cohere.py
@@ -13,6 +13,9 @@ from .llama import *
from .llama import (
original_apply_qkv,
original_apply_o,
+ _gqa_decode_scores,
+ _gqa_decode_values,
+ _LazyDeviceScratch,
)
from ._utils import __version__
from ._utils import move_to_device, per_layer_device
@@ -305,7 +308,7 @@ def CohereAttention_fast_forward_inference(
self.temp_O = self.temp_QA[1][:, :, :hidden_size]
self.attention = torch.empty(
- (bsz, n_heads, 1, KV_CACHE_INCREMENT + seq_len),
+ (bsz, n_heads, 1, KV_CACHE_INCREMENT + seq_len + (-seq_len) % 8),
dtype = dtype,
device = f"{DEVICE_TYPE_TORCH}:0",
)
@@ -388,9 +391,9 @@ def CohereAttention_fast_forward_inference(
else:
Knn, Vnn = Kn, Vn
- # Grouped query attention.
+ # Grouped query attention; bsz == 1 reads the KV heads in place below.
_, _, cached_len, _ = Knn.shape
- if n_groups != 1:
+ if bsz != 1 and n_groups != 1:
Knn = Knn[:, :, None, :, :].expand(bsz, n_kv_heads, n_groups, cached_len, head_dim)
Vnn = Vnn[:, :, None, :, :].expand(bsz, n_kv_heads, n_groups, cached_len, head_dim)
Knn = Knn.reshape(bsz, n_heads, cached_len, head_dim)
@@ -400,9 +403,9 @@ def CohereAttention_fast_forward_inference(
Qn *= self.scalar
# (Q * scalar) @ K beats (Q @ K) * scalar for stopping overflows; see ggerganov/llama.cpp#7805
# (comment 2153349963).
- A = torch_matmul(Qn, Knn.transpose(2, 3), out = self.attention[:, :, :, :cached_len])
+ A = _gqa_decode_scores(Qn, Knn, self.attention[:, :, :, :cached_len])
A[:] = torch_nn_functional_softmax(A, dim = -1, dtype = torch.float32)
- A = torch_matmul(A, Vnn, out = Qn)
+ A = _gqa_decode_values(A, Vnn, Qn)
else:
A = scaled_dot_product_attention(Qn, Knn, Vnn, attn_mask = attention_mask, is_causal = False)
A = A.transpose(1, 2)
@@ -419,13 +422,10 @@ def CohereModel_fast_forward_inference(
position_ids,
attention_mask = None,
):
- out_weights = tuple(
- torch.empty_like(
- self.model.layers[0].input_layernorm.weight,
- dtype = torch.float32,
- device = torch.device(x),
- )
- for x in range(DEVICE_COUNT)
+ norm_weight = self.model.layers[0].input_layernorm.weight
+ out_weights = _LazyDeviceScratch(
+ DEVICE_COUNT,
+ lambda device: torch.empty_like(norm_weight, dtype = torch.float32, device = device),
)
input_ids = input_ids[:, : self.max_seq_length]
hidden_states = self.model.embed_tokens(input_ids)
diff --git a/unsloth/models/falcon_h1.py b/unsloth/models/falcon_h1.py
index 2c9302195a1..d93f0a6cb23 100644
--- a/unsloth/models/falcon_h1.py
+++ b/unsloth/models/falcon_h1.py
@@ -26,6 +26,8 @@ from ..utils.attention_dispatch import (
from .llama import (
LlamaRotaryEmbedding,
LlamaLinearScalingRotaryEmbedding,
+ _gqa_decode_scores,
+ _gqa_decode_values,
_LlamaModel_fast_forward_inference,
original_apply_qkv,
original_apply_o,
@@ -253,7 +255,9 @@ def FalconH1Attention_fast_forward_inference(
self.temp_O = self.temp_QA[1][:, :, :hidden_size]
self.attention = torch.empty(
- (bsz, n_heads, 1, KV_CACHE_INCREMENT + seq_len), dtype = dtype, device = device
+ (bsz, n_heads, 1, KV_CACHE_INCREMENT + seq_len + (-seq_len) % 8),
+ dtype = dtype,
+ device = device,
)
self.scalar = 1.0 / math_sqrt(self.head_dim)
self.half_head_dim = head_dim // 2
@@ -322,9 +326,9 @@ def FalconH1Attention_fast_forward_inference(
else:
Knn, Vnn = Kn, Vn
- # Grouped query attention.
+ # Grouped query attention; bsz == 1 reads the KV heads in place below.
_, _, cached_len, _ = Knn.shape
- if bsz == 1 or not SDPA_HAS_GQA and n_groups != 1:
+ if bsz != 1 and (not SDPA_HAS_GQA) and n_groups != 1:
Knn = Knn[:, :, None, :, :].expand(bsz, n_kv_heads, n_groups, cached_len, head_dim)
Vnn = Vnn[:, :, None, :, :].expand(bsz, n_kv_heads, n_groups, cached_len, head_dim)
Knn = Knn.reshape(bsz, n_heads, cached_len, head_dim)
@@ -334,9 +338,9 @@ def FalconH1Attention_fast_forward_inference(
Qn *= self.scalar
# (Q * scalar) @ K beats (Q @ K) * scalar for stopping overflows; see ggerganov/llama.cpp#7805
# (comment 2153349963).
- A = torch_matmul(Qn, Knn.transpose(2, 3), out = self.attention[:, :, :, :cached_len])
+ A = _gqa_decode_scores(Qn, Knn, self.attention[:, :, :, :cached_len])
A[:] = torch_nn_functional_softmax(A, dim = -1, dtype = torch.float32)
- A = torch_matmul(A, Vnn, out = Qn)
+ A = _gqa_decode_values(A, Vnn, Qn)
else:
if SDPA_HAS_GQA:
A = scaled_dot_product_attention(
diff --git a/unsloth/models/gemma.py b/unsloth/models/gemma.py
index 2df1d8e6c3e..83778ae0f64 100644
--- a/unsloth/models/gemma.py
+++ b/unsloth/models/gemma.py
@@ -10,7 +10,7 @@
# limitations under the License.
from .llama import *
-from .llama import _get_rope_theta
+from .llama import _get_rope_theta, _LazyDeviceScratch
from ._utils import __version__
from ._utils import move_to_device, per_layer_device, embedding_applies_scale
from unsloth_zoo.utils import _get_dtype, Version
@@ -157,13 +157,10 @@ def GemmaModel_fast_forward_inference(
attention_mask = None,
**kwargs,
):
- out_weights = tuple(
- torch.empty_like(
- self.model.layers[0].input_layernorm.weight,
- dtype = torch.float32,
- device = torch.device(x),
- )
- for x in range(DEVICE_COUNT)
+ norm_weight = self.model.layers[0].input_layernorm.weight
+ out_weights = _LazyDeviceScratch(
+ DEVICE_COUNT,
+ lambda device: torch.empty_like(norm_weight, dtype = torch.float32, device = device),
)
input_ids = input_ids[:, : self.max_seq_length]
hidden_states = self.model.embed_tokens(input_ids)
diff --git a/unsloth/models/gemma2.py b/unsloth/models/gemma2.py
index 57a7bbb72d2..c744cad20fc 100644
--- a/unsloth/models/gemma2.py
+++ b/unsloth/models/gemma2.py
@@ -13,6 +13,9 @@ from .llama import *
from .llama import (
original_apply_qkv,
original_apply_o,
+ _gqa_decode_scores,
+ _gqa_decode_values,
+ _LazyDeviceScratch,
)
from ._utils import __version__
from ._utils import move_to_device, per_layer_device, DEVICE_TYPE
@@ -395,7 +398,9 @@ def Gemma2Attention_fast_forward_inference(
# Only for Gemma2.
self.temp_O = torch.empty((bsz, 1, hidden_size), dtype = dtype, device = device)
self.attention = torch.empty(
- (bsz, n_heads, 1, KV_CACHE_INCREMENT + seq_len), dtype = dtype, device = device
+ (bsz, n_heads, 1, KV_CACHE_INCREMENT + seq_len + (-seq_len) % 8),
+ dtype = dtype,
+ device = device,
)
# Gemma 9b should use 256, not hidden_size // num_attention_heads (224); 27b uses the derived value,
@@ -477,19 +482,14 @@ def Gemma2Attention_fast_forward_inference(
else:
Knn, Vnn = Kn, Vn
- # Grouped query attention.
_, _, cached_len, _ = Knn.shape
- if n_groups != 1:
- Knn = Knn[:, :, None, :, :].expand(bsz, n_kv_heads, n_groups, cached_len, head_dim)
- Vnn = Vnn[:, :, None, :, :].expand(bsz, n_kv_heads, n_groups, cached_len, head_dim)
- Knn = Knn.reshape(bsz, n_heads, cached_len, head_dim)
- Vnn = Vnn.reshape(bsz, n_heads, cached_len, head_dim)
# Gemma2 uses manual matmul for all batch sizes since SDPA lacks softcapping (tanh logit scaling);
# if PyTorch adds a softcap param, consider SDPA for bsz > 1 to match llama/qwen3.
Qn *= self.scalar
# (Q * scalar) @ K beats (Q @ K) * scalar for stopping overflows; see ggerganov/llama.cpp#7805 (comment 2153349963).
- A = torch_matmul(Qn, Knn.transpose(2, 3), out = self.attention[:, :, :, :cached_len])
+ # Grouped query attention: the KV heads are read in place, not repeated per query head.
+ A = _gqa_decode_scores(Qn, Knn, self.attention[:, :, :, :cached_len])
# Softcapping must happen BEFORE the mask is applied; see google-deepmind/gemma _modules.py and
# transformers gemma2 eager_attention_forward.
@@ -504,7 +504,7 @@ def Gemma2Attention_fast_forward_inference(
A += attention_mask
A[:] = torch_nn_functional_softmax(A, dim = -1, dtype = torch.float32)
- A = torch_matmul(A, Vnn, out = Qn)
+ A = _gqa_decode_values(A, Vnn, Qn)
A = A.transpose(1, 2)
A = A.reshape(bsz, 1, attention_size)
A = fast_linear_forward(self.o_proj, A, out = self.temp_O)
@@ -520,13 +520,10 @@ def Gemma2Model_fast_forward_inference(
attention_mask = None,
**kwargs,
):
- out_weights = tuple(
- torch.empty_like(
- self.model.layers[0].input_layernorm.weight,
- dtype = torch.float32,
- device = torch.device(x),
- )
- for x in range(DEVICE_COUNT)
+ norm_weight = self.model.layers[0].input_layernorm.weight
+ out_weights = _LazyDeviceScratch(
+ DEVICE_COUNT,
+ lambda device: torch.empty_like(norm_weight, dtype = torch.float32, device = device),
)
input_ids = input_ids[:, : self.max_seq_length]
hidden_states = self.model.embed_tokens(input_ids)
diff --git a/unsloth/models/granite.py b/unsloth/models/granite.py
index 51fc0c135bc..8b82ed25860 100644
--- a/unsloth/models/granite.py
+++ b/unsloth/models/granite.py
@@ -27,6 +27,8 @@ from ..utils.attention_dispatch import (
from .llama import (
LlamaRotaryEmbedding,
LlamaLinearScalingRotaryEmbedding,
+ _gqa_decode_scores,
+ _gqa_decode_values,
original_apply_qkv,
original_apply_o,
)
@@ -321,7 +323,9 @@ def GraniteAttention_fast_forward_inference(
self.RH_Q = torch.empty((bsz, n_heads, 1, head_dim), dtype = dtype, device = device)
self.temp_O = torch.empty((bsz, 1, hidden_size), dtype = dtype, device = device)
self.attention = torch.empty(
- (bsz, n_heads, 1, KV_CACHE_INCREMENT + seq_len), dtype = dtype, device = device
+ (bsz, n_heads, 1, KV_CACHE_INCREMENT + seq_len + (-seq_len) % 8),
+ dtype = dtype,
+ device = device,
)
self.half_head_dim = head_dim // 2
@@ -372,9 +376,9 @@ def GraniteAttention_fast_forward_inference(
Kn = self.paged_attention_K[:kv_seq_len].permute(1, 2, 0, 3)
Vn = self.paged_attention_V[:kv_seq_len].permute(1, 2, 0, 3)
- # Grouped query attention.
+ # Grouped query attention; bsz == 1 reads the KV heads in place below.
_, _, cached_len, _ = Kn.shape
- if bsz == 1 or ((not SDPA_HAS_GQA) and n_groups != 1):
+ if bsz != 1 and (not SDPA_HAS_GQA) and n_groups != 1:
Kn = Kn[:, :, None, :, :].expand(bsz, n_kv_heads, n_groups, cached_len, head_dim)
Vn = Vn[:, :, None, :, :].expand(bsz, n_kv_heads, n_groups, cached_len, head_dim)
Kn = Kn.reshape(bsz, n_heads, cached_len, head_dim)
@@ -382,9 +386,9 @@ def GraniteAttention_fast_forward_inference(
if bsz == 1:
Qn *= self.scaling
- A = torch_matmul(Qn, Kn.transpose(2, 3), out = self.attention[:, :, :, :cached_len])
+ A = _gqa_decode_scores(Qn, Kn, self.attention[:, :, :, :cached_len])
A[:] = torch_nn_functional_softmax(A, dim = -1, dtype = torch.float32)
- A = torch_matmul(A, Vn, out = Qn)
+ A = _gqa_decode_values(A, Vn, Qn)
else:
if (
attention_mask is not None
diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py
index 18322792fd5..f0405d48af2 100644
--- a/unsloth/models/llama.py
+++ b/unsloth/models/llama.py
@@ -531,6 +531,57 @@ def fix_prepare_inputs_for_generation(module):
torch_matmul = torch.matmul
+def _gqa_decode_scores(Qn, Kn, out):
+ """Q @ K^T for one decode token without repeating K per query head. Q (bsz, n_heads, 1, hd)
+ is viewed as (bsz, n_kv_heads, n_groups, hd) so each KV head is read in place; the scores
+ land in out's (bsz, n_heads, 1, cached_len) layout."""
+ bsz, n_heads, _, head_dim = Qn.shape
+ _, n_kv_heads, cached_len, _ = Kn.shape
+ n_groups = n_heads // n_kv_heads
+ torch_matmul(
+ Qn.view(bsz, n_kv_heads, n_groups, head_dim),
+ Kn.transpose(2, 3),
+ out = out.view(bsz, n_kv_heads, n_groups, cached_len),
+ )
+ return out
+
+
+def _gqa_decode_values(A, Vn, out):
+ """softmax(QK^T) @ V for one decode token, grouped like _gqa_decode_scores; writes out.
+ cuBLAS drops to a slow kernel when the reduction over the cache is not a multiple of 8, so
+ the aligned prefix is one bmm and the last few positions are accumulated on top."""
+ bsz, n_heads, _, cached_len = A.shape
+ _, n_kv_heads, _, head_dim = Vn.shape
+ n_groups = n_heads // n_kv_heads
+ A = A.view(bsz * n_kv_heads, n_groups, cached_len)
+ V = Vn.reshape(bsz * n_kv_heads, cached_len, head_dim)
+ O = out.view(bsz * n_kv_heads, n_groups, head_dim)
+ aligned = cached_len - cached_len % 8
+ if aligned == 0 or aligned == cached_len:
+ torch.bmm(A, V, out = O)
+ else:
+ torch.bmm(A[:, :, :aligned], V[:, :aligned], out = O)
+ O.baddbmm_(A[:, :, aligned:], V[:, aligned:])
+ return out
+
+
+class _LazyDeviceScratch(list):
+ """Per-device decode scratch, made on first use, so only devices holding layers get one."""
+
+ __slots__ = ("_make",)
+
+ def __init__(self, device_count, make):
+ super().__init__((None,) * device_count)
+ self._make = make
+
+ def __getitem__(self, index):
+ buffer = list.__getitem__(self, index)
+ if buffer is None:
+ buffer = self._make(torch.device(DEVICE_TYPE_TORCH, index))
+ list.__setitem__(self, index, buffer)
+ return buffer
+
+
def LlamaAttention_fast_forward_inference(
self,
hidden_states: torch.Tensor,
@@ -604,8 +655,11 @@ def LlamaAttention_fast_forward_inference(
else:
self.temp_O = self.temp_QA[1][:, :, :hidden_size]
+ # Row stride kept a multiple of 8 so the grouped decode bmm stays on the fast cuBLAS path.
self.attention = torch.empty(
- (bsz, n_heads, 1, KV_CACHE_INCREMENT + seq_len), dtype = dtype, device = device
+ (bsz, n_heads, 1, KV_CACHE_INCREMENT + seq_len + (-seq_len) % 8),
+ dtype = dtype,
+ device = device,
)
self.scalar = 1.0 / math_sqrt(self.head_dim)
self.half_head_dim = head_dim // 2
@@ -689,9 +743,9 @@ def LlamaAttention_fast_forward_inference(
# Decode is more stable with boolean keep masks than additive bf16 masks.
attention_mask = attention_mask.eq(0)
- # Grouped query attention.
+ # Grouped query attention; bsz == 1 reads the KV heads in place below.
_, _, cached_len, _ = Knn.shape
- if bsz == 1 or ((not SDPA_HAS_GQA) and n_groups != 1):
+ if bsz != 1 and (not SDPA_HAS_GQA) and n_groups != 1:
Knn = Knn[:, :, None, :, :].expand(bsz, n_kv_heads, n_groups, cached_len, head_dim)
Vnn = Vnn[:, :, None, :, :].expand(bsz, n_kv_heads, n_groups, cached_len, head_dim)
Knn = Knn.reshape(bsz, n_heads, cached_len, head_dim)
@@ -707,11 +761,11 @@ def LlamaAttention_fast_forward_inference(
if bsz == 1:
Qn *= self.scalar
# (Q * scalar) @ K beats (Q @ K) * scalar for stopping overflows; see ggerganov/llama.cpp#7805.
- A = torch_matmul(Qn, Knn.transpose(2, 3), out = self.attention[:, :, :, :cached_len])
+ A = _gqa_decode_scores(Qn, Knn, self.attention[:, :, :, :cached_len])
if attention_mask is not None:
A.masked_fill_(~attention_mask, float("-inf"))
A[:] = torch_nn_functional_softmax(A, dim = -1, dtype = torch.float32)
- A = torch_matmul(A, Vnn, out = Qn)
+ A = _gqa_decode_values(A, Vnn, Qn)
else:
if SDPA_HAS_GQA:
A = scaled_dot_product_attention(
@@ -1434,10 +1488,8 @@ def _LlamaModel_fast_forward_inference(
temp_mlp = torch.empty(
(2, bsz, 1, mlp_size), dtype = X.dtype, device = f"{DEVICE_TYPE_TORCH}:0"
)
- temp_gates, temp_ups = (
- tuple(temp_mlp[0].to(torch.device(x)) for x in range(DEVICE_COUNT)),
- tuple(temp_mlp[1].to(torch.device(x)) for x in range(DEVICE_COUNT)),
- )
+ temp_gates = _LazyDeviceScratch(DEVICE_COUNT, temp_mlp[0].to)
+ temp_ups = _LazyDeviceScratch(DEVICE_COUNT, temp_mlp[1].to)
seq_len = past_key_values[0][0].shape[-2]
kv_seq_len = seq_len + 1
diff --git a/unsloth/models/qwen3.py b/unsloth/models/qwen3.py
index a1ffbb49700..7bcb519b1d1 100644
--- a/unsloth/models/qwen3.py
+++ b/unsloth/models/qwen3.py
@@ -26,6 +26,8 @@ from ..utils.attention_dispatch import (
from .llama import (
LlamaRotaryEmbedding,
LlamaLinearScalingRotaryEmbedding,
+ _gqa_decode_scores,
+ _gqa_decode_values,
_LlamaModel_fast_forward_inference,
original_apply_qkv,
original_apply_o,
@@ -222,7 +224,9 @@ def Qwen3Attention_fast_forward_inference(
self.temp_O = self.temp_QA[1][:, :, :hidden_size]
self.attention = torch.empty(
- (bsz, n_heads, 1, KV_CACHE_INCREMENT + seq_len), dtype = dtype, device = device
+ (bsz, n_heads, 1, KV_CACHE_INCREMENT + seq_len + (-seq_len) % 8),
+ dtype = dtype,
+ device = device,
)
self.scalar = 1.0 / math_sqrt(self.head_dim)
self.half_head_dim = head_dim // 2
@@ -318,9 +322,9 @@ def Qwen3Attention_fast_forward_inference(
# Avoid SDPA GQA drift for batched masked decode.
use_sdpa_gqa = False
- # Grouped query attention.
+ # Grouped query attention; bsz == 1 reads the KV heads in place below.
_, _, cached_len, _ = Knn.shape
- if bsz == 1 or ((not use_sdpa_gqa) and n_groups != 1):
+ if bsz != 1 and (not use_sdpa_gqa) and n_groups != 1:
Knn = Knn[:, :, None, :, :].expand(bsz, n_kv_heads, n_groups, cached_len, head_dim)
Vnn = Vnn[:, :, None, :, :].expand(bsz, n_kv_heads, n_groups, cached_len, head_dim)
Knn = Knn.reshape(bsz, n_heads, cached_len, head_dim)
@@ -330,11 +334,11 @@ def Qwen3Attention_fast_forward_inference(
Qn *= self.scalar
# (Q * scalar) @ K beats (Q @ K) * scalar for stopping overflows; see ggerganov/llama.cpp#7805
# (comment 2153349963).
- A = torch_matmul(Qn, Knn.transpose(2, 3), out = self.attention[:, :, :, :cached_len])
+ A = _gqa_decode_scores(Qn, Knn, self.attention[:, :, :, :cached_len])
if attention_mask is not None:
A.masked_fill_(~attention_mask, float("-inf"))
A[:] = torch_nn_functional_softmax(A, dim = -1, dtype = torch.float32)
- A = torch_matmul(A, Vnn, out = Qn)
+ A = _gqa_decode_values(A, Vnn, Qn)
else:
if use_sdpa_gqa:
A = scaled_dot_product_attention(