Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Decode CUDA graphs: opt-in whole-step capture for batch 1 (UNSLOTH_DECODE_CUDA_GRAPHS_FULL=1) (#15976)

Daniel Haneed6da82bf1a
diff --git a/tests/test_decode_full_graph.py b/tests/test_decode_full_graph.py
new file mode 100644
index 00000000000..977b2b37543
--- /dev/null
+++ b/tests/test_decode_full_graph.py
@@ -0,0 +1,156 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""Whole-step CUDA-graph decode (UNSLOTH_DECODE_CUDA_GRAPHS_FULL=1): the attention cores are captured
+too, over a power-of-two span of cache rows with a device-side keep mask. Teacher-forced logits
+match the eager step to bf16 summation-order noise, every step after the first is one replay,
+spans re-capture as the cache fills, and sliding-window models keep the segmented capture."""
+
+import os
+
+os.environ.setdefault("UNSLOTH_DECODE_CUDA_GRAPHS", "1")
+
+import types
+
+import pytest
+
+import unsloth # noqa: F401 (must precede transformers)
+import torch
+
+import unsloth.models._decode_graphs as G
+from tests.test_decode_cuda_graphs import MODELS, PROMPTS, _causal, _load
+
+cuda = pytest.mark.skipif(not torch.cuda.is_available(), reason = "CUDA only")
+
+
+def _forced(model, enc, tokens, enabled, full, monkeypatch):
+ """Logits of every decode step with the next tokens forced (so arms see the same inputs)."""
+ monkeypatch.setattr(G, "ENABLED", enabled)
+ monkeypatch.setattr(G, "FULL", full)
+ ids, mask = enc.input_ids, enc.attention_mask
+ with torch.inference_mode():
+ out = model(input_ids = ids, attention_mask = mask, use_cache = True, return_dict = True)
+ past, logits = out.past_key_values, [out.logits[:, -1].float().cpu()]
+ pos = mask.long().cumsum(-1)[:, -1:] - 1
+ for t in range(tokens.shape[1] - 1):
+ nxt = tokens[:, t : t + 1]
+ mask = torch.cat([mask, torch.ones_like(nxt)], 1)
+ pos = pos + 1
+ out = model(
+ input_ids = nxt, attention_mask = mask, position_ids = pos,
+ past_key_values = past, use_cache = True, return_dict = True,
+ ) # fmt: skip
+ past = out.past_key_values
+ logits.append(out.logits[:, -1].float().cpu())
+ G.release(model)
+ return torch.stack(logits, 1)
+
+
+def _kl(p_logits, q_logits):
+ lp, lq = torch.log_softmax(p_logits, -1), torch.log_softmax(q_logits, -1)
+ return float((lp.exp() * (lp - lq)).sum(-1).mean())
+
+
+@cuda
[email protected]("name", MODELS)
[email protected]("load_in_4bit", [False, True])
[email protected]("bsz", [1, 4])
+def test_full_graph_matches_eager(name, load_in_4bit, bsz, monkeypatch):
+ monkeypatch.setattr(G, "FULL_MAX_BSZ", 8) # the batched whole-step path too
+ model, tok = _load(name, load_in_4bit)
+ enc = tok(PROMPTS[:bsz], return_tensors = "pt", padding = True).to("cuda")
+ g = torch.Generator().manual_seed(0)
+ tokens = torch.randint(10, 1000, (bsz, 300), generator = g).cuda()
+ eager = _forced(model, enc, tokens, False, False, monkeypatch)
+ before = dict(G.stats(_causal(model)))
+ full = _forced(model, enc, tokens, True, True, monkeypatch)
+ stats = G.stats(_causal(model))
+ assert stats["disabled"] is None, stats
+ # Every step but the eager ones (the first fills the cache, a cache growth reallocates it)
+ # was one whole-step replay.
+ replays = stats["replays"] - before.get("replays", 0)
+ eager_steps = stats["eager_steps"] - before.get("eager_steps", 0)
+ assert replays + eager_steps == tokens.shape[1] - 1, stats
+ assert eager_steps <= 3, stats
+ # 300 steps from a short prompt cross the 256-row span: at least two spans captured.
+ assert stats["captures"] - before.get("captures", 0) >= 2, stats
+ assert torch.isfinite(full).all()
+ assert _kl(eager, full) < 1e-3
+ # Summation-order noise only: a few bf16 ulps on average (tiny random models have tiny logits).
+ assert float((eager - full).abs().mean()) < 0.02 * float(eager.abs().mean())
+
+
+@cuda
+def test_full_and_segmented_graphs_coexist(monkeypatch):
+ """Toggling FULL between generations: each mode keeps its own graphs; segmented stays bitwise."""
+ model, tok = _load(MODELS[0], False)
+ enc = tok(PROMPTS[:2], return_tensors = "pt", padding = True).to("cuda")
+ g = torch.Generator().manual_seed(1)
+ tokens = torch.randint(10, 1000, (2, 40), generator = g).cuda()
+ monkeypatch.setattr(G, "FULL_MAX_BSZ", 8)
+ eager = _forced(model, enc, tokens, False, False, monkeypatch)
+ seg = _forced(model, enc, tokens, True, False, monkeypatch)
+ full = _forced(model, enc, tokens, True, True, monkeypatch)
+ seg2 = _forced(model, enc, tokens, True, False, monkeypatch)
+ assert torch.equal(eager, seg)
+ assert torch.equal(eager, seg2)
+ assert _kl(eager, full) < 1e-3
+
+
+@cuda
+def test_full_graph_generate_and_lora(monkeypatch):
+ """generate() with LoRA adapters: one capture per span, replays, finite logits close to eager."""
+ model, tok = _load(MODELS[1], False, lora = True)
+ enc = tok(PROMPTS[:1], return_tensors = "pt").to("cuda")
+
+ def gen(enabled, full):
+ monkeypatch.setattr(G, "ENABLED", enabled)
+ monkeypatch.setattr(G, "FULL", full)
+ out = model.generate(
+ **enc, max_new_tokens = 40, min_new_tokens = 40, do_sample = False,
+ pad_token_id = tok.pad_token_id, output_logits = True, return_dict_in_generate = True,
+ ) # fmt: skip
+ return torch.stack(out.logits).float().cpu()
+
+ eager = gen(False, False)
+ before = dict(G.stats(_causal(model)))
+ full = gen(True, True)
+ stats = G.stats(_causal(model))
+ assert stats["disabled"] is None, stats
+ assert stats["replays"] - before.get("replays", 0) == 40 - 2
+ # Greedy tokens may part on a near-tie; compare the first step that diverges onwards only
+ # through its logits, which see the same prefix until then.
+ assert _kl(eager[:5], full[:5]) < 1e-3
+
+
+def test_full_span_policy(monkeypatch):
+ monkeypatch.setattr(G, "FULL", True)
+ model = types.SimpleNamespace(config = types.SimpleNamespace(sliding_window = None))
+ assert G._full_span(model, 1, 512) == 256
+ assert G._full_span(model, 256, 512) == 256
+ assert G._full_span(model, 257, 512) == 512
+ assert G._full_span(model, 600, 768) == 768 # capped at the cache rows
+ assert G._full_span(model, 1025, 4096) == 2048
+ model.config.sliding_window = 4096
+ assert G._full_span(model, 10, 2048) == 256
+ # A window shorter than the cache: the segmented capture slices it eagerly.
+ assert G._full_span(model, 10, 8192) is None
+ model.config.sliding_window = None
+ # Batched decode keeps the segmented capture unless widened.
+ assert G._full_span(model, 10, 512, bsz = 2) is None
+ monkeypatch.setattr(G, "FULL_MAX_BSZ", 4)
+ assert G._full_span(model, 10, 512, bsz = 4) == 256
+ assert G._full_span(model, 10, 512, bsz = 5) is None
+ monkeypatch.setattr(G, "FULL", False)
+ assert G._full_span(model, 10, 512) is None
+
+
+@cuda
+def test_batched_default_stays_bitwise(monkeypatch):
+ """FULL on, default FULL_MAX_BSZ: a batch of 2 runs the segmented capture, bit for bit eager."""
+ model, tok = _load(MODELS[0], False)
+ enc = tok(PROMPTS[:2], return_tensors = "pt", padding = True).to("cuda")
+ tokens = torch.randint(10, 1000, (2, 30), generator = torch.Generator().manual_seed(2)).cuda()
+ eager = _forced(model, enc, tokens, False, False, monkeypatch)
+ full = _forced(model, enc, tokens, True, True, monkeypatch)
+ assert torch.equal(eager, full)
diff --git a/tests/utils/test_decode_full_graph_stale_tail.py b/tests/utils/test_decode_full_graph_stale_tail.py
new file mode 100644
index 00000000000..9f75b5a18a1
--- /dev/null
+++ b/tests/utils/test_decode_full_graph_stale_tail.py
@@ -0,0 +1,101 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""Whole-step graph attention reads every row of its span, rows past the KV length masked. A masked
+row still multiplies its V by 0 (NaN for inf / NaN), so a reused graph KV buffer must not carry an
+earlier generation's rows into a later one. CPU: the real buffer helpers and attention, by path."""
+
+import ast
+import importlib.util
+from pathlib import Path
+from types import SimpleNamespace
+
+import pytest
+import torch
+from torch.nn import functional as F
+
+MODELS = Path(__file__).resolve().parents[2] / "unsloth" / "models"
+
+
+def _decode_graphs():
+ spec = importlib.util.spec_from_file_location(
+ "_unsloth_decode_graphs_stale_tail", MODELS / "_decode_graphs.py"
+ )
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+ module.ENABLED = True # size buffers as the graph path does (replay itself is not run here)
+ module.FULL = True
+ return module
+
+
+def _attention_functions():
+ names = {"_decode_attention_core", "_decode_attention_span", "_gqa_decode_scores",
+ "_gqa_decode_values"} # fmt: skip
+ tree = ast.parse((MODELS / "llama.py").read_text(encoding = "utf-8"))
+ nodes = [n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name in names]
+ assert {n.name for n in nodes} == names
+ namespace = dict(
+ torch = torch,
+ torch_matmul = torch.matmul,
+ torch_nn_functional_softmax = F.softmax,
+ scaled_dot_product_attention = F.scaled_dot_product_attention,
+ SDPA_HAS_GQA = True,
+ )
+ exec(compile(ast.Module(nodes, []), str(MODELS / "llama.py"), "exec"), namespace)
+ return namespace
+
+
[email protected]("dtype", [torch.float32, torch.float16, torch.bfloat16])
[email protected]("bsz", [1, 2])
[email protected]("bad", [float("inf"), float("nan")])
+def test_reused_buffer_does_not_leak_an_earlier_generation(dtype, bsz, bad):
+ G, fns = _decode_graphs(), _attention_functions()
+ attn = SimpleNamespace(
+ config = SimpleNamespace(
+ model_type = "llama", num_attention_heads = 2, num_key_value_heads = 1,
+ sliding_window = None,
+ ),
+ num_key_value_groups = 2,
+ head_dim = 2,
+ ) # fmt: skip
+ args = (bsz, 2, 1, 2, 4, dtype, torch.device("cpu"), 256)
+ # An earlier, longer generation left a nonfinite K and V row (e.g. an fp16 overflow).
+ k = torch.zeros(bsz, 1, 8, 2, dtype = dtype)
+ v = torch.ones_like(k)
+ k[:, :, 5, :] = bad
+ v[:, :, 4, :] = bad
+ G.alloc_decode_buffers(attn, k, v, *args)
+ first = attn.paged_attention.data_ptr()
+ # A new, short and finite generation reuses the same buffer.
+ G.alloc_decode_buffers(attn, k[:, :, :2].nan_to_num(0.0), v[:, :, :2], *args)
+ assert attn.paged_attention.data_ptr() == first
+
+ kv_len, span = 2, 256
+ keep = torch.zeros(bsz, 1, 1, span, dtype = torch.bool)
+ keep[..., :kv_len] = True
+ static = SimpleNamespace(span = span, keep = keep, drop = ~keep)
+ q = torch.ones(bsz, 2, 1, 2, dtype = dtype)
+ segmented = fns["_decode_attention_core"](attn, q.clone(), kv_len, None, bsz, True)[0].clone()
+ whole_step = fns["_decode_attention_span"](attn, q.clone(), static, bsz, True).clone()
+ assert torch.isfinite(segmented).all()
+ assert torch.isfinite(whole_step).all()
+ torch.testing.assert_close(whole_step, segmented)
+ assert torch.isfinite(attn.paged_attention).all()
+
+
+def test_segmented_only_keeps_buffer_untouched_past_the_prompt():
+ # FULL off: only the segmented capture reads the cache, over the true length; no tail memset.
+ G = _decode_graphs()
+ G.FULL = False
+ attn = SimpleNamespace(
+ config = SimpleNamespace(model_type = "llama", num_attention_heads = 2,
+ num_key_value_heads = 1, sliding_window = None),
+ num_key_value_groups = 2,
+ head_dim = 2,
+ ) # fmt: skip
+ args = (1, 2, 1, 2, 4, torch.float32, torch.device("cpu"), 256)
+ G.alloc_decode_buffers(
+ attn, torch.full((1, 1, 8, 2), 3.0), torch.full((1, 1, 8, 2), 3.0), *args
+ )
+ G.alloc_decode_buffers(attn, torch.ones(1, 1, 2, 2), torch.ones(1, 1, 2, 2), *args)
+ assert (attn.paged_attention_K[2:8] == 3.0).all()
diff --git a/tests/utils/test_single_row_cached_attention.py b/tests/utils/test_single_row_cached_attention.py
index 72ed6ee22e0..005be7083c5 100644
--- a/tests/utils/test_single_row_cached_attention.py
+++ b/tests/utils/test_single_row_cached_attention.py
@@ -85,6 +85,7 @@ def decoder(request):
# The cache and scratch buffers come from the real helpers; only the captured path is out.
"_decode_graphs": _decode_graphs_module(source),
"_decode_attention_core": _captured_attention_only,
+ "_decode_attention_span": _captured_attention_only,
}
# The decode paths share llama.py's grouped KV helpers (qwen3 imports them from there).
llama_source = source.with_name("llama.py")
diff --git a/unsloth/models/_decode_graphs.py b/unsloth/models/_decode_graphs.py
index 0496977bd8d..ab16312b398 100644
--- a/unsloth/models/_decode_graphs.py
+++ b/unsloth/models/_decode_graphs.py
@@ -2,7 +2,14 @@
# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
"""CUDA-graph decode; UNSLOTH_DECODE_CUDA_GRAPHS=1 forces, =0 disables; unset, only RL-trainer
-generate on NVIDIA CUDA; graphs dropped when params, LoRA, quant or RoPE change."""
+generate on NVIDIA CUDA; graphs dropped when params, LoRA, quant or RoPE change.
+
+UNSLOTH_DECODE_CUDA_GRAPHS_FULL=1 also captures the attention cores: each step is then ONE graph that
+attends over a fixed span of cache rows (the next power of two >= the KV length, at least BUCKET, at
+most the cache) with a device-side keep mask, so no eager launch is left between the layers. Rows past
+the true length are masked to -inf, so the result matches the eager step up to the summation order of
+the softmax / PV reductions (not bit for bit); one graph is captured per (batch size, span). Only
+batch 1 by default (UNSLOTH_DECODE_FULL_GRAPH_MAX_BSZ): larger batches keep the segmented capture."""
import math
import os
@@ -13,10 +20,17 @@ import torch
# Programmatic override (tests, harnesses): True / False force graphs on / off, None (default) follows
# the environment variable read at call time and the RL generation scope (enabled()).
ENABLED = None
+# Whole-step graphs (attention captured too, over a power-of-two span of cache rows).
+FULL = os.environ.get("UNSLOTH_DECODE_CUDA_GRAPHS_FULL", "0") == "1"
+# Largest batch the whole-step capture takes; larger batches keep the segmented (bit-exact) capture.
+# Batched decode measured slower whole-step on a T4 (+9-15%: GPU bound, nothing to hide).
+FULL_MAX_BSZ = int(os.environ.get("UNSLOTH_DECODE_FULL_GRAPH_MAX_BSZ", "1"))
# KV-cache rows are allocated in multiples of this with graphs on.
BUCKET = 256
# Captured steps (one per batch size / autocast state) kept per model.
MAX_GRAPHS = int(os.environ.get("UNSLOTH_DECODE_GRAPH_MAX", "8"))
+# Whole-step graphs keep one graph per span too (log2(cache / BUCKET) + 1 per batch size).
+MAX_FULL_GRAPHS = int(os.environ.get("UNSLOTH_DECODE_FULL_GRAPH_MAX", "16"))
__all__ = [
"ENABLED",
@@ -113,14 +127,26 @@ class _StepStatics:
length, and (per replayed step) the true KV length + mask for the eager attention cores.
attention(fn) is the split point: eager it runs fn, capturing it closes the current segment."""
- __slots__ = ("rope_len", "slot", "kv_len", "mask", "attention")
+ __slots__ = ("rope_len", "slot", "kv_len", "mask", "attention", "span", "keep", "drop")
- def __init__(self, rope_len, slot):
+ def __init__(
+ self,
+ rope_len,
+ slot,
+ span = None,
+ keep = None,
+ drop = None,
+ ):
self.rope_len = rope_len
self.slot = slot
self.kv_len = None
self.mask = None
self.attention = _run_now
+ # Whole-step graphs: cache rows the captured attention reads, and their bool keep mask
+ # [bsz, 1, 1, rows] (refilled outside the graph before every replay).
+ self.span = span
+ self.keep = keep
+ self.drop = drop # ~keep, so the graph does not invert it once per layer
def _run_now(fn):
@@ -206,6 +232,10 @@ def alloc_decode_buffers(
store.clear()
BUFFER_EPOCH += 1
bufs = store[key] = (paged, temp_QA, temp_KV, RH_Q, temp_O, scores)
+ elif FULL:
+ # A reused buffer still holds the last generation's rows: a whole-step graph reads them
+ # past this one's length, and a masked inf / NaN there still turns 0 * V into NaN.
+ bufs[0][seq_len:].zero_()
paged, temp_QA, temp_KV, RH_Q, temp_O, scores = bufs
# A new sequence: its first graph step re-checks the fingerprint (adapters toggled or
# merged since the last one, also in decode loops that do not go through generate()).
@@ -434,6 +464,43 @@ def _eligible(model, layers):
return None
+def _full_span(
+ model,
+ kv_len,
+ cap,
+ bsz = 1,
+):
+ """The cache rows a whole-step graph attends over, or None for the segmented capture."""
+ if not FULL or bsz > FULL_MAX_BSZ:
+ return None
+ sliding_window = getattr(model.config, "sliding_window", None)
+ if sliding_window is not None and cap > sliding_window:
+ # The segmented path slices the window off eagerly; a span would read rows outside it.
+ return None
+ return min(cap, max(BUCKET, 1 << (kv_len - 1).bit_length()))
+
+
+def _fill_keep(state, s, mask, bsz, kv_len, cap, device):
+ """Writes this step's keep mask (true KV rows, padding applied) into the static buffer."""
+ keep = s.get("keep")
+ if keep is None or keep.shape[0] != bsz or keep.shape[-1] != cap:
+ with _no_inference_tensors():
+ keep = torch.zeros((bsz, 1, 1, cap), dtype = torch.bool, device = device)
+ drop = torch.ones((bsz, 1, 1, cap), dtype = torch.bool, device = device)
+ if s.get("keep") is not None:
+ # Graphs captured against the old buffers would read freed memory.
+ state.clear()
+ s["keep"], s["drop"] = keep, drop
+ drop = s["drop"]
+ keep[..., kv_len:] = False
+ if mask is None:
+ keep[..., :kv_len] = True
+ else:
+ keep[..., :kv_len] = mask[..., -kv_len:]
+ torch.logical_not(keep, out = drop)
+ return keep
+
+
def _static_inputs(state, bsz, device, dtype, hidden_size):
key = (bsz, device)
s = state.statics.get(key)
@@ -594,17 +661,24 @@ def decode_step(
s["pos"].copy_(last.view(bsz, 1))
s["slot"].fill_(seq_len)
+ span = _full_span(model, kv_len, cap, bsz)
+ if span is not None:
+ mask = mask_fn(model, attention_mask, bsz, 1, s["like"], seq_len)
+ if mask is not None and mask.shape[-2] != 1:
+ return None
+ _fill_keep(state, s, mask, bsz, kv_len, cap, device)
+
autocast = _autocast_state()
- key = (bsz, device, autocast, torch.is_inference_mode_enabled())
+ key = (bsz, device, autocast, torch.is_inference_mode_enabled(), span)
step = state.graphs.get(key)
if step is None:
- if len(state.graphs) >= MAX_GRAPHS:
+ if len(state.graphs) >= (MAX_GRAPHS if span is None else MAX_FULL_GRAPHS):
state.graphs.clear()
state.pool = None
try:
step = _capture(
state, model, fast_forward_inference, logits_fn, mask_fn, s, rope_len,
- past_key_values, autocast, attention_mask, seq_len,
+ past_key_values, autocast, attention_mask, seq_len, span,
) # fmt: skip
# The warmup may have built LoRA cast caches or RoPE rows the graph now reads.
state.fingerprint = _fingerprint(model, layers, rope_len)
@@ -618,7 +692,9 @@ def decode_step(
state.captures += 1
st = step.statics
st.kv_len = kv_len
- st.mask = mask_fn(model, attention_mask, bsz, 1, s["like"], seq_len)
+ st.mask = (
+ None if span is not None else mask_fn(model, attention_mask, bsz, 1, s["like"], seq_len)
+ )
try:
step.replay()
finally:
@@ -656,6 +732,7 @@ def _capture(
autocast,
attention_mask,
seq_len,
+ span = None,
):
if state.stream is not None and state.stream.device.index != torch.cuda.current_device():
state.stream = None
@@ -665,9 +742,13 @@ def _capture(
if state.stream is None:
state.stream = torch.cuda.Stream()
stream = state.stream
- st = _StepStatics(rope_len, s["slot"])
+ st = _StepStatics(
+ rope_len, s["slot"], span,
+ s.get("keep") if span is not None else None, s.get("drop") if span is not None else None,
+ ) # fmt: skip
st.kv_len = seq_len + 1
- st.mask = mask_fn(model, attention_mask, s["ids"].shape[0], 1, s["like"], seq_len)
+ if span is None:
+ st.mask = mask_fn(model, attention_mask, s["ids"].shape[0], 1, s["like"], seq_len)
enabled, dtype = autocast
# No autocast weight cache: a cast cached during capture would be freed into the graph's pool.
no_cache = lambda: torch.autocast("cuda", dtype = dtype, enabled = enabled, cache_enabled = False)
@@ -696,7 +777,8 @@ def _capture(
graph = torch.cuda.CUDAGraph()
step.segments.append(graph)
- st.attention = split
+ if span is None:
+ st.attention = split
_begin(graph, state.pool)
try:
step.logits = _run_step(
diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py
index 5fcb2e55d1b..f2b9341e40f 100644
--- a/unsloth/models/llama.py
+++ b/unsloth/models/llama.py
@@ -776,15 +776,17 @@ def LlamaAttention_fast_forward_inference(
if static is not None:
# Captured step: the cache slot is on the device; the attention itself runs eagerly
# between two captured segments (static.attention), over the true length.
- self.paged_attention_K.index_copy_(
- 0, static.slot, K_slot.view(1, bsz, n_kv_heads, head_dim)
- )
- self.paged_attention_V.index_copy_(
- 0, static.slot, V_slot.view(1, bsz, n_kv_heads, head_dim)
- )
- A = static.attention(
- lambda: _decode_attention_core(self, Qn, static.kv_len, static.mask, bsz, True)[0]
+ # K and V scratch are adjacent ([2, bsz, 1, n_kv_heads * head_dim]), as are their cache
+ # rows ([rows, 2, bsz, n_kv_heads, head_dim]): one index_copy_ writes both.
+ self.paged_attention.index_copy_(
+ 0, static.slot, self.temp_KV.view(1, 2, bsz, n_kv_heads, head_dim)
)
+ if static.span is not None:
+ A = _decode_attention_span(self, Qn, static, bsz, True)
+ else:
+ A = static.attention(
+ lambda: _decode_attention_core(self, Qn, static.kv_len, static.mask, bsz, True)[0]
+ )
A = A.transpose(1, 2).reshape(bsz, 1, attention_size)
return fast_linear_forward(self.o_proj, A, out = self.temp_O), (None, None)
@@ -940,6 +942,36 @@ def _decode_attention_core(self, Qn, kv_seq_len, attention_mask, bsz, sdpa_gqa_w
return A, Kn, Vn
+def _decode_attention_span(self, Qn, static, bsz, sdpa_gqa_with_batch_mask):
+ """softmax(Q K^T) V of one decode token inside a whole-step graph: over the first static.span
+ cache rows, rows past the true length masked by the device-side static.keep. The same ops as
+ _decode_attention_core on a longer (masked) row, so only the reduction order differs."""
+ span = static.span
+ keep = static.keep[..., :span]
+ drop = static.drop[..., :span]
+ n_heads = self.__dict__.get("_unsloth_tp_heads", self.config.num_attention_heads)
+ n_groups = self.num_key_value_groups
+ n_kv_heads = self.__dict__.get("_unsloth_tp_kv_heads", self.config.num_key_value_heads)
+ head_dim = self.head_dim
+ Knn = self.paged_attention_K[:span].permute(1, 2, 0, 3)
+ Vnn = self.paged_attention_V[:span].permute(1, 2, 0, 3)
+ if bsz == 1:
+ Qn *= self.scalar
+ A = _gqa_decode_scores(Qn, Knn, self.attention[:, :, :, :span])
+ A.masked_fill_(drop, float("-inf"))
+ A[:] = torch_nn_functional_softmax(A, dim = -1, dtype = torch.float32)
+ return _gqa_decode_values(A, Vnn, Qn)
+ # The mask is always batched here: as the eager core, Qwen3 expands K / V instead of SDPA GQA.
+ if SDPA_HAS_GQA and sdpa_gqa_with_batch_mask:
+ return scaled_dot_product_attention(Qn, Knn, Vnn, attn_mask = keep, enable_gqa = True)
+ if n_groups != 1:
+ Knn = Knn[:, :, None, :, :].expand(bsz, n_kv_heads, n_groups, span, head_dim)
+ Vnn = Vnn[:, :, None, :, :].expand(bsz, n_kv_heads, n_groups, span, head_dim)
+ Knn = Knn.reshape(bsz, n_heads, span, head_dim)
+ Vnn = Vnn.reshape(bsz, n_heads, span, head_dim)
+ return scaled_dot_product_attention(Qn, Knn, Vnn, attn_mask = keep)
+
+
_decode_graphs.register_attention(LlamaAttention_fast_forward_inference)
torch_nn_functional_silu = torch.nn.functional.silu
diff --git a/unsloth/models/qwen3.py b/unsloth/models/qwen3.py
index 40a8f37f191..67d43c9a3a6 100644
--- a/unsloth/models/qwen3.py
+++ b/unsloth/models/qwen3.py
@@ -18,7 +18,7 @@ from ..utils.packing import get_packed_info_from_kwargs
from ..kernels import qk_norm_rope as _qk_norm_rope
from ..kernels.qk_norm_rope import fast_qk_norm_rope, decode_qk_norm_rope
from . import _decode_graphs
-from .llama import _decode_attention_core
+from .llama import _decode_attention_core, _decode_attention_span
from ..utils.attention_dispatch import (
AttentionConfig,
AttentionContext,
@@ -272,9 +272,12 @@ def Qwen3Attention_fast_forward_inference(
# between two captured segments (static.attention), over the true length.
self.paged_attention_K.index_copy_(0, static.slot, Kn.permute(2, 0, 1, 3))
self.paged_attention_V.index_copy_(0, static.slot, Vn.permute(2, 0, 1, 3))
- A = static.attention(
- lambda: _decode_attention_core(self, Qn, static.kv_len, static.mask, bsz, False)[0]
- )
+ if static.span is not None:
+ A = _decode_attention_span(self, Qn, static, bsz, False)
+ else:
+ A = static.attention(
+ lambda: _decode_attention_core(self, Qn, static.kv_len, static.mask, bsz, False)[0]
+ )
A = A.transpose(1, 2).reshape(bsz, 1, attention_size)
return fast_linear_forward(self.o_proj, A, out = self.temp_O), (None, None)
self.paged_attention_K[seq_len] = Kn.permute(2, 0, 1, 3)