Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Decode CUDA graphs: opt-in whole-step capture for batch 1 (UNSLOTH_DECODE_CUDA_GRAPHS_FULL=1) (#15976)
Daniel Haneed6da82bf1a
diff --git a/tests/test_decode_full_graph.py b/tests/test_decode_full_graph.pynew file mode 100644index 00000000000..977b2b37543--- /dev/null+++ b/tests/test_decode_full_graph.py@@ -0,0 +1,156 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""Whole-step CUDA-graph decode (UNSLOTH_DECODE_CUDA_GRAPHS_FULL=1): the attention cores are captured+too, over a power-of-two span of cache rows with a device-side keep mask. Teacher-forced logits+match the eager step to bf16 summation-order noise, every step after the first is one replay,+spans re-capture as the cache fills, and sliding-window models keep the segmented capture."""++import os++os.environ.setdefault("UNSLOTH_DECODE_CUDA_GRAPHS", "1")++import types++import pytest++import unsloth # noqa: F401 (must precede transformers)+import torch++import unsloth.models._decode_graphs as G+from tests.test_decode_cuda_graphs import MODELS, PROMPTS, _causal, _load++cuda = pytest.mark.skipif(not torch.cuda.is_available(), reason = "CUDA only")+++def _forced(model, enc, tokens, enabled, full, monkeypatch):+ """Logits of every decode step with the next tokens forced (so arms see the same inputs)."""+ monkeypatch.setattr(G, "ENABLED", enabled)+ monkeypatch.setattr(G, "FULL", full)+ ids, mask = enc.input_ids, enc.attention_mask+ with torch.inference_mode():+ out = model(input_ids = ids, attention_mask = mask, use_cache = True, return_dict = True)+ past, logits = out.past_key_values, [out.logits[:, -1].float().cpu()]+ pos = mask.long().cumsum(-1)[:, -1:] - 1+ for t in range(tokens.shape[1] - 1):+ nxt = tokens[:, t : t + 1]+ mask = torch.cat([mask, torch.ones_like(nxt)], 1)+ pos = pos + 1+ out = model(+ input_ids = nxt, attention_mask = mask, position_ids = pos,+ past_key_values = past, use_cache = True, return_dict = True,+ ) # fmt: skip+ past = out.past_key_values+ logits.append(out.logits[:, -1].float().cpu())+ G.release(model)+ return torch.stack(logits, 1)+++def _kl(p_logits, q_logits):+ lp, lq = torch.log_softmax(p_logits, -1), torch.log_softmax(q_logits, -1)+ return float((lp.exp() * (lp - lq)).sum(-1).mean())+++@cuda[email protected]("name", MODELS)[email protected]("load_in_4bit", [False, True])[email protected]("bsz", [1, 4])+def test_full_graph_matches_eager(name, load_in_4bit, bsz, monkeypatch):+ monkeypatch.setattr(G, "FULL_MAX_BSZ", 8) # the batched whole-step path too+ model, tok = _load(name, load_in_4bit)+ enc = tok(PROMPTS[:bsz], return_tensors = "pt", padding = True).to("cuda")+ g = torch.Generator().manual_seed(0)+ tokens = torch.randint(10, 1000, (bsz, 300), generator = g).cuda()+ eager = _forced(model, enc, tokens, False, False, monkeypatch)+ before = dict(G.stats(_causal(model)))+ full = _forced(model, enc, tokens, True, True, monkeypatch)+ stats = G.stats(_causal(model))+ assert stats["disabled"] is None, stats+ # Every step but the eager ones (the first fills the cache, a cache growth reallocates it)+ # was one whole-step replay.+ replays = stats["replays"] - before.get("replays", 0)+ eager_steps = stats["eager_steps"] - before.get("eager_steps", 0)+ assert replays + eager_steps == tokens.shape[1] - 1, stats+ assert eager_steps <= 3, stats+ # 300 steps from a short prompt cross the 256-row span: at least two spans captured.+ assert stats["captures"] - before.get("captures", 0) >= 2, stats+ assert torch.isfinite(full).all()+ assert _kl(eager, full) < 1e-3+ # Summation-order noise only: a few bf16 ulps on average (tiny random models have tiny logits).+ assert float((eager - full).abs().mean()) < 0.02 * float(eager.abs().mean())+++@cuda+def test_full_and_segmented_graphs_coexist(monkeypatch):+ """Toggling FULL between generations: each mode keeps its own graphs; segmented stays bitwise."""+ model, tok = _load(MODELS[0], False)+ enc = tok(PROMPTS[:2], return_tensors = "pt", padding = True).to("cuda")+ g = torch.Generator().manual_seed(1)+ tokens = torch.randint(10, 1000, (2, 40), generator = g).cuda()+ monkeypatch.setattr(G, "FULL_MAX_BSZ", 8)+ eager = _forced(model, enc, tokens, False, False, monkeypatch)+ seg = _forced(model, enc, tokens, True, False, monkeypatch)+ full = _forced(model, enc, tokens, True, True, monkeypatch)+ seg2 = _forced(model, enc, tokens, True, False, monkeypatch)+ assert torch.equal(eager, seg)+ assert torch.equal(eager, seg2)+ assert _kl(eager, full) < 1e-3+++@cuda+def test_full_graph_generate_and_lora(monkeypatch):+ """generate() with LoRA adapters: one capture per span, replays, finite logits close to eager."""+ model, tok = _load(MODELS[1], False, lora = True)+ enc = tok(PROMPTS[:1], return_tensors = "pt").to("cuda")++ def gen(enabled, full):+ monkeypatch.setattr(G, "ENABLED", enabled)+ monkeypatch.setattr(G, "FULL", full)+ out = model.generate(+ **enc, max_new_tokens = 40, min_new_tokens = 40, do_sample = False,+ pad_token_id = tok.pad_token_id, output_logits = True, return_dict_in_generate = True,+ ) # fmt: skip+ return torch.stack(out.logits).float().cpu()++ eager = gen(False, False)+ before = dict(G.stats(_causal(model)))+ full = gen(True, True)+ stats = G.stats(_causal(model))+ assert stats["disabled"] is None, stats+ assert stats["replays"] - before.get("replays", 0) == 40 - 2+ # Greedy tokens may part on a near-tie; compare the first step that diverges onwards only+ # through its logits, which see the same prefix until then.+ assert _kl(eager[:5], full[:5]) < 1e-3+++def test_full_span_policy(monkeypatch):+ monkeypatch.setattr(G, "FULL", True)+ model = types.SimpleNamespace(config = types.SimpleNamespace(sliding_window = None))+ assert G._full_span(model, 1, 512) == 256+ assert G._full_span(model, 256, 512) == 256+ assert G._full_span(model, 257, 512) == 512+ assert G._full_span(model, 600, 768) == 768 # capped at the cache rows+ assert G._full_span(model, 1025, 4096) == 2048+ model.config.sliding_window = 4096+ assert G._full_span(model, 10, 2048) == 256+ # A window shorter than the cache: the segmented capture slices it eagerly.+ assert G._full_span(model, 10, 8192) is None+ model.config.sliding_window = None+ # Batched decode keeps the segmented capture unless widened.+ assert G._full_span(model, 10, 512, bsz = 2) is None+ monkeypatch.setattr(G, "FULL_MAX_BSZ", 4)+ assert G._full_span(model, 10, 512, bsz = 4) == 256+ assert G._full_span(model, 10, 512, bsz = 5) is None+ monkeypatch.setattr(G, "FULL", False)+ assert G._full_span(model, 10, 512) is None+++@cuda+def test_batched_default_stays_bitwise(monkeypatch):+ """FULL on, default FULL_MAX_BSZ: a batch of 2 runs the segmented capture, bit for bit eager."""+ model, tok = _load(MODELS[0], False)+ enc = tok(PROMPTS[:2], return_tensors = "pt", padding = True).to("cuda")+ tokens = torch.randint(10, 1000, (2, 30), generator = torch.Generator().manual_seed(2)).cuda()+ eager = _forced(model, enc, tokens, False, False, monkeypatch)+ full = _forced(model, enc, tokens, True, True, monkeypatch)+ assert torch.equal(eager, full)diff --git a/tests/utils/test_decode_full_graph_stale_tail.py b/tests/utils/test_decode_full_graph_stale_tail.pynew file mode 100644index 00000000000..9f75b5a18a1--- /dev/null+++ b/tests/utils/test_decode_full_graph_stale_tail.py@@ -0,0 +1,101 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""Whole-step graph attention reads every row of its span, rows past the KV length masked. A masked+row still multiplies its V by 0 (NaN for inf / NaN), so a reused graph KV buffer must not carry an+earlier generation's rows into a later one. CPU: the real buffer helpers and attention, by path."""++import ast+import importlib.util+from pathlib import Path+from types import SimpleNamespace++import pytest+import torch+from torch.nn import functional as F++MODELS = Path(__file__).resolve().parents[2] / "unsloth" / "models"+++def _decode_graphs():+ spec = importlib.util.spec_from_file_location(+ "_unsloth_decode_graphs_stale_tail", MODELS / "_decode_graphs.py"+ )+ module = importlib.util.module_from_spec(spec)+ spec.loader.exec_module(module)+ module.ENABLED = True # size buffers as the graph path does (replay itself is not run here)+ module.FULL = True+ return module+++def _attention_functions():+ names = {"_decode_attention_core", "_decode_attention_span", "_gqa_decode_scores",+ "_gqa_decode_values"} # fmt: skip+ tree = ast.parse((MODELS / "llama.py").read_text(encoding = "utf-8"))+ nodes = [n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name in names]+ assert {n.name for n in nodes} == names+ namespace = dict(+ torch = torch,+ torch_matmul = torch.matmul,+ torch_nn_functional_softmax = F.softmax,+ scaled_dot_product_attention = F.scaled_dot_product_attention,+ SDPA_HAS_GQA = True,+ )+ exec(compile(ast.Module(nodes, []), str(MODELS / "llama.py"), "exec"), namespace)+ return namespace++[email protected]("dtype", [torch.float32, torch.float16, torch.bfloat16])[email protected]("bsz", [1, 2])[email protected]("bad", [float("inf"), float("nan")])+def test_reused_buffer_does_not_leak_an_earlier_generation(dtype, bsz, bad):+ G, fns = _decode_graphs(), _attention_functions()+ attn = SimpleNamespace(+ config = SimpleNamespace(+ model_type = "llama", num_attention_heads = 2, num_key_value_heads = 1,+ sliding_window = None,+ ),+ num_key_value_groups = 2,+ head_dim = 2,+ ) # fmt: skip+ args = (bsz, 2, 1, 2, 4, dtype, torch.device("cpu"), 256)+ # An earlier, longer generation left a nonfinite K and V row (e.g. an fp16 overflow).+ k = torch.zeros(bsz, 1, 8, 2, dtype = dtype)+ v = torch.ones_like(k)+ k[:, :, 5, :] = bad+ v[:, :, 4, :] = bad+ G.alloc_decode_buffers(attn, k, v, *args)+ first = attn.paged_attention.data_ptr()+ # A new, short and finite generation reuses the same buffer.+ G.alloc_decode_buffers(attn, k[:, :, :2].nan_to_num(0.0), v[:, :, :2], *args)+ assert attn.paged_attention.data_ptr() == first++ kv_len, span = 2, 256+ keep = torch.zeros(bsz, 1, 1, span, dtype = torch.bool)+ keep[..., :kv_len] = True+ static = SimpleNamespace(span = span, keep = keep, drop = ~keep)+ q = torch.ones(bsz, 2, 1, 2, dtype = dtype)+ segmented = fns["_decode_attention_core"](attn, q.clone(), kv_len, None, bsz, True)[0].clone()+ whole_step = fns["_decode_attention_span"](attn, q.clone(), static, bsz, True).clone()+ assert torch.isfinite(segmented).all()+ assert torch.isfinite(whole_step).all()+ torch.testing.assert_close(whole_step, segmented)+ assert torch.isfinite(attn.paged_attention).all()+++def test_segmented_only_keeps_buffer_untouched_past_the_prompt():+ # FULL off: only the segmented capture reads the cache, over the true length; no tail memset.+ G = _decode_graphs()+ G.FULL = False+ attn = SimpleNamespace(+ config = SimpleNamespace(model_type = "llama", num_attention_heads = 2,+ num_key_value_heads = 1, sliding_window = None),+ num_key_value_groups = 2,+ head_dim = 2,+ ) # fmt: skip+ args = (1, 2, 1, 2, 4, torch.float32, torch.device("cpu"), 256)+ G.alloc_decode_buffers(+ attn, torch.full((1, 1, 8, 2), 3.0), torch.full((1, 1, 8, 2), 3.0), *args+ )+ G.alloc_decode_buffers(attn, torch.ones(1, 1, 2, 2), torch.ones(1, 1, 2, 2), *args)+ assert (attn.paged_attention_K[2:8] == 3.0).all()diff --git a/tests/utils/test_single_row_cached_attention.py b/tests/utils/test_single_row_cached_attention.pyindex 72ed6ee22e0..005be7083c5 100644--- a/tests/utils/test_single_row_cached_attention.py+++ b/tests/utils/test_single_row_cached_attention.py@@ -85,6 +85,7 @@ def decoder(request):# The cache and scratch buffers come from the real helpers; only the captured path is out."_decode_graphs": _decode_graphs_module(source),"_decode_attention_core": _captured_attention_only,+ "_decode_attention_span": _captured_attention_only,}# The decode paths share llama.py's grouped KV helpers (qwen3 imports them from there).llama_source = source.with_name("llama.py")diff --git a/unsloth/models/_decode_graphs.py b/unsloth/models/_decode_graphs.pyindex 0496977bd8d..ab16312b398 100644--- a/unsloth/models/_decode_graphs.py+++ b/unsloth/models/_decode_graphs.py@@ -2,7 +2,14 @@# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved."""CUDA-graph decode; UNSLOTH_DECODE_CUDA_GRAPHS=1 forces, =0 disables; unset, only RL-trainer-generate on NVIDIA CUDA; graphs dropped when params, LoRA, quant or RoPE change."""+generate on NVIDIA CUDA; graphs dropped when params, LoRA, quant or RoPE change.++UNSLOTH_DECODE_CUDA_GRAPHS_FULL=1 also captures the attention cores: each step is then ONE graph that+attends over a fixed span of cache rows (the next power of two >= the KV length, at least BUCKET, at+most the cache) with a device-side keep mask, so no eager launch is left between the layers. Rows past+the true length are masked to -inf, so the result matches the eager step up to the summation order of+the softmax / PV reductions (not bit for bit); one graph is captured per (batch size, span). Only+batch 1 by default (UNSLOTH_DECODE_FULL_GRAPH_MAX_BSZ): larger batches keep the segmented capture."""import mathimport os@@ -13,10 +20,17 @@ import torch# Programmatic override (tests, harnesses): True / False force graphs on / off, None (default) follows# the environment variable read at call time and the RL generation scope (enabled()).ENABLED = None+# Whole-step graphs (attention captured too, over a power-of-two span of cache rows).+FULL = os.environ.get("UNSLOTH_DECODE_CUDA_GRAPHS_FULL", "0") == "1"+# Largest batch the whole-step capture takes; larger batches keep the segmented (bit-exact) capture.+# Batched decode measured slower whole-step on a T4 (+9-15%: GPU bound, nothing to hide).+FULL_MAX_BSZ = int(os.environ.get("UNSLOTH_DECODE_FULL_GRAPH_MAX_BSZ", "1"))# KV-cache rows are allocated in multiples of this with graphs on.BUCKET = 256# Captured steps (one per batch size / autocast state) kept per model.MAX_GRAPHS = int(os.environ.get("UNSLOTH_DECODE_GRAPH_MAX", "8"))+# Whole-step graphs keep one graph per span too (log2(cache / BUCKET) + 1 per batch size).+MAX_FULL_GRAPHS = int(os.environ.get("UNSLOTH_DECODE_FULL_GRAPH_MAX", "16"))__all__ = ["ENABLED",@@ -113,14 +127,26 @@ class _StepStatics:length, and (per replayed step) the true KV length + mask for the eager attention cores.attention(fn) is the split point: eager it runs fn, capturing it closes the current segment."""- __slots__ = ("rope_len", "slot", "kv_len", "mask", "attention")+ __slots__ = ("rope_len", "slot", "kv_len", "mask", "attention", "span", "keep", "drop")- def __init__(self, rope_len, slot):+ def __init__(+ self,+ rope_len,+ slot,+ span = None,+ keep = None,+ drop = None,+ ):self.rope_len = rope_lenself.slot = slotself.kv_len = Noneself.mask = Noneself.attention = _run_now+ # Whole-step graphs: cache rows the captured attention reads, and their bool keep mask+ # [bsz, 1, 1, rows] (refilled outside the graph before every replay).+ self.span = span+ self.keep = keep+ self.drop = drop # ~keep, so the graph does not invert it once per layerdef _run_now(fn):@@ -206,6 +232,10 @@ def alloc_decode_buffers(store.clear()BUFFER_EPOCH += 1bufs = store[key] = (paged, temp_QA, temp_KV, RH_Q, temp_O, scores)+ elif FULL:+ # A reused buffer still holds the last generation's rows: a whole-step graph reads them+ # past this one's length, and a masked inf / NaN there still turns 0 * V into NaN.+ bufs[0][seq_len:].zero_()paged, temp_QA, temp_KV, RH_Q, temp_O, scores = bufs# A new sequence: its first graph step re-checks the fingerprint (adapters toggled or# merged since the last one, also in decode loops that do not go through generate()).@@ -434,6 +464,43 @@ def _eligible(model, layers):return None+def _full_span(+ model,+ kv_len,+ cap,+ bsz = 1,+):+ """The cache rows a whole-step graph attends over, or None for the segmented capture."""+ if not FULL or bsz > FULL_MAX_BSZ:+ return None+ sliding_window = getattr(model.config, "sliding_window", None)+ if sliding_window is not None and cap > sliding_window:+ # The segmented path slices the window off eagerly; a span would read rows outside it.+ return None+ return min(cap, max(BUCKET, 1 << (kv_len - 1).bit_length()))+++def _fill_keep(state, s, mask, bsz, kv_len, cap, device):+ """Writes this step's keep mask (true KV rows, padding applied) into the static buffer."""+ keep = s.get("keep")+ if keep is None or keep.shape[0] != bsz or keep.shape[-1] != cap:+ with _no_inference_tensors():+ keep = torch.zeros((bsz, 1, 1, cap), dtype = torch.bool, device = device)+ drop = torch.ones((bsz, 1, 1, cap), dtype = torch.bool, device = device)+ if s.get("keep") is not None:+ # Graphs captured against the old buffers would read freed memory.+ state.clear()+ s["keep"], s["drop"] = keep, drop+ drop = s["drop"]+ keep[..., kv_len:] = False+ if mask is None:+ keep[..., :kv_len] = True+ else:+ keep[..., :kv_len] = mask[..., -kv_len:]+ torch.logical_not(keep, out = drop)+ return keep++def _static_inputs(state, bsz, device, dtype, hidden_size):key = (bsz, device)s = state.statics.get(key)@@ -594,17 +661,24 @@ def decode_step(s["pos"].copy_(last.view(bsz, 1))s["slot"].fill_(seq_len)+ span = _full_span(model, kv_len, cap, bsz)+ if span is not None:+ mask = mask_fn(model, attention_mask, bsz, 1, s["like"], seq_len)+ if mask is not None and mask.shape[-2] != 1:+ return None+ _fill_keep(state, s, mask, bsz, kv_len, cap, device)+autocast = _autocast_state()- key = (bsz, device, autocast, torch.is_inference_mode_enabled())+ key = (bsz, device, autocast, torch.is_inference_mode_enabled(), span)step = state.graphs.get(key)if step is None:- if len(state.graphs) >= MAX_GRAPHS:+ if len(state.graphs) >= (MAX_GRAPHS if span is None else MAX_FULL_GRAPHS):state.graphs.clear()state.pool = Nonetry:step = _capture(state, model, fast_forward_inference, logits_fn, mask_fn, s, rope_len,- past_key_values, autocast, attention_mask, seq_len,+ past_key_values, autocast, attention_mask, seq_len, span,) # fmt: skip# The warmup may have built LoRA cast caches or RoPE rows the graph now reads.state.fingerprint = _fingerprint(model, layers, rope_len)@@ -618,7 +692,9 @@ def decode_step(state.captures += 1st = step.staticsst.kv_len = kv_len- st.mask = mask_fn(model, attention_mask, bsz, 1, s["like"], seq_len)+ st.mask = (+ None if span is not None else mask_fn(model, attention_mask, bsz, 1, s["like"], seq_len)+ )try:step.replay()finally:@@ -656,6 +732,7 @@ def _capture(autocast,attention_mask,seq_len,+ span = None,):if state.stream is not None and state.stream.device.index != torch.cuda.current_device():state.stream = None@@ -665,9 +742,13 @@ def _capture(if state.stream is None:state.stream = torch.cuda.Stream()stream = state.stream- st = _StepStatics(rope_len, s["slot"])+ st = _StepStatics(+ rope_len, s["slot"], span,+ s.get("keep") if span is not None else None, s.get("drop") if span is not None else None,+ ) # fmt: skipst.kv_len = seq_len + 1- st.mask = mask_fn(model, attention_mask, s["ids"].shape[0], 1, s["like"], seq_len)+ if span is None:+ st.mask = mask_fn(model, attention_mask, s["ids"].shape[0], 1, s["like"], seq_len)enabled, dtype = autocast# No autocast weight cache: a cast cached during capture would be freed into the graph's pool.no_cache = lambda: torch.autocast("cuda", dtype = dtype, enabled = enabled, cache_enabled = False)@@ -696,7 +777,8 @@ def _capture(graph = torch.cuda.CUDAGraph()step.segments.append(graph)- st.attention = split+ if span is None:+ st.attention = split_begin(graph, state.pool)try:step.logits = _run_step(diff --git a/unsloth/models/llama.py b/unsloth/models/llama.pyindex 5fcb2e55d1b..f2b9341e40f 100644--- a/unsloth/models/llama.py+++ b/unsloth/models/llama.py@@ -776,15 +776,17 @@ def LlamaAttention_fast_forward_inference(if static is not None:# Captured step: the cache slot is on the device; the attention itself runs eagerly# between two captured segments (static.attention), over the true length.- self.paged_attention_K.index_copy_(- 0, static.slot, K_slot.view(1, bsz, n_kv_heads, head_dim)- )- self.paged_attention_V.index_copy_(- 0, static.slot, V_slot.view(1, bsz, n_kv_heads, head_dim)- )- A = static.attention(- lambda: _decode_attention_core(self, Qn, static.kv_len, static.mask, bsz, True)[0]+ # K and V scratch are adjacent ([2, bsz, 1, n_kv_heads * head_dim]), as are their cache+ # rows ([rows, 2, bsz, n_kv_heads, head_dim]): one index_copy_ writes both.+ self.paged_attention.index_copy_(+ 0, static.slot, self.temp_KV.view(1, 2, bsz, n_kv_heads, head_dim))+ if static.span is not None:+ A = _decode_attention_span(self, Qn, static, bsz, True)+ else:+ A = static.attention(+ lambda: _decode_attention_core(self, Qn, static.kv_len, static.mask, bsz, True)[0]+ )A = A.transpose(1, 2).reshape(bsz, 1, attention_size)return fast_linear_forward(self.o_proj, A, out = self.temp_O), (None, None)@@ -940,6 +942,36 @@ def _decode_attention_core(self, Qn, kv_seq_len, attention_mask, bsz, sdpa_gqa_wreturn A, Kn, Vn+def _decode_attention_span(self, Qn, static, bsz, sdpa_gqa_with_batch_mask):+ """softmax(Q K^T) V of one decode token inside a whole-step graph: over the first static.span+ cache rows, rows past the true length masked by the device-side static.keep. The same ops as+ _decode_attention_core on a longer (masked) row, so only the reduction order differs."""+ span = static.span+ keep = static.keep[..., :span]+ drop = static.drop[..., :span]+ n_heads = self.__dict__.get("_unsloth_tp_heads", self.config.num_attention_heads)+ n_groups = self.num_key_value_groups+ n_kv_heads = self.__dict__.get("_unsloth_tp_kv_heads", self.config.num_key_value_heads)+ head_dim = self.head_dim+ Knn = self.paged_attention_K[:span].permute(1, 2, 0, 3)+ Vnn = self.paged_attention_V[:span].permute(1, 2, 0, 3)+ if bsz == 1:+ Qn *= self.scalar+ A = _gqa_decode_scores(Qn, Knn, self.attention[:, :, :, :span])+ A.masked_fill_(drop, float("-inf"))+ A[:] = torch_nn_functional_softmax(A, dim = -1, dtype = torch.float32)+ return _gqa_decode_values(A, Vnn, Qn)+ # The mask is always batched here: as the eager core, Qwen3 expands K / V instead of SDPA GQA.+ if SDPA_HAS_GQA and sdpa_gqa_with_batch_mask:+ return scaled_dot_product_attention(Qn, Knn, Vnn, attn_mask = keep, enable_gqa = True)+ if n_groups != 1:+ Knn = Knn[:, :, None, :, :].expand(bsz, n_kv_heads, n_groups, span, head_dim)+ Vnn = Vnn[:, :, None, :, :].expand(bsz, n_kv_heads, n_groups, span, head_dim)+ Knn = Knn.reshape(bsz, n_heads, span, head_dim)+ Vnn = Vnn.reshape(bsz, n_heads, span, head_dim)+ return scaled_dot_product_attention(Qn, Knn, Vnn, attn_mask = keep)++_decode_graphs.register_attention(LlamaAttention_fast_forward_inference)torch_nn_functional_silu = torch.nn.functional.siludiff --git a/unsloth/models/qwen3.py b/unsloth/models/qwen3.pyindex 40a8f37f191..67d43c9a3a6 100644--- a/unsloth/models/qwen3.py+++ b/unsloth/models/qwen3.py@@ -18,7 +18,7 @@ from ..utils.packing import get_packed_info_from_kwargsfrom ..kernels import qk_norm_rope as _qk_norm_ropefrom ..kernels.qk_norm_rope import fast_qk_norm_rope, decode_qk_norm_ropefrom . import _decode_graphs-from .llama import _decode_attention_core+from .llama import _decode_attention_core, _decode_attention_spanfrom ..utils.attention_dispatch import (AttentionConfig,AttentionContext,@@ -272,9 +272,12 @@ def Qwen3Attention_fast_forward_inference(# between two captured segments (static.attention), over the true length.self.paged_attention_K.index_copy_(0, static.slot, Kn.permute(2, 0, 1, 3))self.paged_attention_V.index_copy_(0, static.slot, Vn.permute(2, 0, 1, 3))- A = static.attention(- lambda: _decode_attention_core(self, Qn, static.kv_len, static.mask, bsz, False)[0]- )+ if static.span is not None:+ A = _decode_attention_span(self, Qn, static, bsz, False)+ else:+ A = static.attention(+ lambda: _decode_attention_core(self, Qn, static.kv_len, static.mask, bsz, False)[0]+ )A = A.transpose(1, 2).reshape(bsz, 1, attention_size)return fast_linear_forward(self.o_proj, A, out = self.temp_O), (None, None)self.paged_attention_K[seq_len] = Kn.permute(2, 0, 1, 3)