Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Decode: write K/V straight into the cache slot, and opt-in CUDA-graph replay of the decode step (#15171)
Daniel Hanaae9f5472386
diff --git a/tests/test_decode_cuda_graphs.py b/tests/test_decode_cuda_graphs.pynew file mode 100644index 00000000000..687c5a015a3--- /dev/null+++ b/tests/test_decode_cuda_graphs.py@@ -0,0 +1,343 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""CUDA-graph decode (UNSLOTH_DECODE_CUDA_GRAPHS=1): a replayed step is bit for bit the eager step+(greedy tokens and raw logits, 16-bit and 4-bit, batch 1 and a left-padded batch), graphs are+re-captured when LoRA weights change between generate() calls, and the KV cache may grow."""++import os++os.environ.setdefault("UNSLOTH_DECODE_CUDA_GRAPHS", "1")++import pytest++import unsloth # noqa: F401 (must precede transformers)+from unsloth import FastLanguageModel+import torch++import unsloth.models._decode_graphs as G+from unsloth.kernels import utils as kernel_utils++cuda = pytest.mark.skipif(not torch.cuda.is_available(), reason = "CUDA only")+MODELS = (+ "trl-internal-testing/tiny-LlamaForCausalLM-3.2",+ "trl-internal-testing/tiny-Qwen3ForCausalLM",+)+PROMPTS = [+ "Once upon a time there was",+ "The capital of France is Paris and",+ "def f(x):",+ "1, 2, 3,",+]+++def _load(+ name,+ load_in_4bit,+ lora = False,+):+ model, tok = FastLanguageModel.from_pretrained(+ name, max_seq_length = 2048, load_in_4bit = load_in_4bit+ )+ if lora:+ model = FastLanguageModel.get_peft_model(+ model, r = 8, lora_alpha = 16, random_state = 0,+ target_modules = ["q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj"],+ ) # fmt: skip+ g = torch.Generator().manual_seed(0)+ with torch.no_grad():+ for n, p in model.named_parameters():+ if "lora_B" in n:+ p.copy_(torch.randn(p.shape, generator = g) * 0.05)+ FastLanguageModel.for_inference(model)+ tok.padding_side = "left"+ if tok.pad_token is None:+ tok.pad_token = tok.eos_token+ return model, tok+++def _causal(model):+ base = getattr(model, "base_model", model)+ return base.model if hasattr(base, "model") and hasattr(base.model, "lm_head") else model+++def _generate(+ model,+ tok,+ enc,+ enabled,+ monkeypatch,+ new = 40,+):+ monkeypatch.setattr(G, "ENABLED", enabled)+ out = model.generate(+ **enc, max_new_tokens = new, min_new_tokens = new, do_sample = False, use_cache = True,+ pad_token_id = tok.pad_token_id, output_logits = True, return_dict_in_generate = True,+ ) # fmt: skip+ return out.sequences.cpu(), torch.stack(out.logits).cpu()+++def _plain(+ model,+ tok,+ enc,+ new = 20,+):+ return model.generate(+ **enc,+ max_new_tokens = new,+ min_new_tokens = new,+ do_sample = False,+ pad_token_id = tok.pad_token_id,+ )+++def _manual_loop(+ model,+ ids,+ enabled,+ monkeypatch,+ steps = 12,+ start = None,+):+ monkeypatch.setattr(G, "ENABLED", enabled)+ with torch.inference_mode():+ out = model(ids, use_cache = True, return_dict = True)+ past, nxt, logits = out.past_key_values, out.logits[:, -1:].argmax(-1), []+ for i in range(steps):+ p = (ids.shape[1] if start is None else start) + i+ pos = torch.full((ids.shape[0], 1), p, device = "cuda", dtype = torch.long)+ out = model(+ nxt, past_key_values = past, position_ids = pos, use_cache = True, return_dict = True+ )+ past = out.past_key_values+ logits.append(out.logits[:, -1].float().cpu())+ nxt = out.logits[:, -1:].argmax(-1)+ return torch.stack(logits)+++@cuda[email protected]("name", MODELS)[email protected]("load_in_4bit", [False, True])[email protected]("bsz", [1, 4])+def test_graph_decode_is_bitwise_eager(name, load_in_4bit, bsz, monkeypatch):+ model, tok = _load(name, load_in_4bit)+ enc = tok(PROMPTS[:bsz], return_tensors = "pt", padding = True).to("cuda")+ eager = _generate(model, tok, enc, False, monkeypatch)+ before = G.stats(_causal(model)).get("replays", 0)+ graph = _generate(model, tok, enc, True, monkeypatch)+ stats = G.stats(_causal(model))+ assert stats["disabled"] is None, stats+ # Every step but the first (eager: it fills the cache) was replayed.+ assert stats["replays"] - before == 40 - 2+ assert torch.equal(eager[0], graph[0])+ assert torch.equal(eager[1], graph[1])+ # generate() calls that return no cache reuse the buffers and the captured step.+ _plain(model, tok, enc)+ captures = G.stats(_causal(model))["captures"]+ _plain(model, tok, enc)+ assert G.stats(_causal(model))["captures"] == captures+ again = _generate(model, tok, enc, True, monkeypatch)+ assert torch.equal(eager[1], again[1])+++@cuda+def test_returned_cache_is_not_overwritten(monkeypatch):+ """A cache handed back by generate(return_dict_in_generate=True) keeps its values through a+ later generate(): the persistent buffers it views are released to the caller."""+ model, tok = _load(MODELS[0], False)+ monkeypatch.setattr(G, "ENABLED", True)+ enc = tok(PROMPTS[:1], return_tensors = "pt").to("cuda")+ out = model.generate(+ **enc,+ max_new_tokens = 20,+ do_sample = False,+ pad_token_id = tok.pad_token_id,+ return_dict_in_generate = True,+ )+ K, V = out.past_key_values[0]+ kept = K.clone(), V.clone()+ other = tok(PROMPTS[1:2], return_tensors = "pt").to("cuda")+ _plain(model, tok, other)+ assert torch.equal(K, kept[0]) and torch.equal(V, kept[1])+++@cuda+def test_shift_labels_decode_loss(monkeypatch):+ model, tok = _load(MODELS[0], False)+ ids = tok(PROMPTS[:1], return_tensors = "pt").input_ids.to("cuda")++ def losses(enabled):+ monkeypatch.setattr(G, "ENABLED", enabled)+ with torch.inference_mode():+ out = model(ids, use_cache = True, return_dict = True)+ past, nxt, res = out.past_key_values, out.logits[:, -1:].argmax(-1), []+ for i in range(6):+ pos = torch.full((1, 1), ids.shape[1] + i, device = "cuda", dtype = torch.long)+ out = model(+ nxt,+ past_key_values = past,+ position_ids = pos,+ use_cache = True,+ return_dict = True,+ shift_labels = torch.full((1, 1), 7, device = "cuda", dtype = torch.long),+ )+ past, nxt = out.past_key_values, out.logits[:, -1:].argmax(-1)+ res.append(out.loss)+ return res++ eager, graph = losses(False), losses(True)+ assert all(l is not None for l in graph)+ assert all(torch.equal(a, b) for a, b in zip(eager, graph))+++@cuda[email protected]("name", MODELS)[email protected]("stale_control", [False, True])+def test_lora_change_between_generations_recaptures(name, stale_control, monkeypatch):+ """GRPO: an optimizer step, then an in-place adapter edit, then a disabled adapter between+ decode runs. stale_control pins the fingerprint, so the old graphs replay: proves the check is+ what keeps the graph path on the live adapter."""+ model, tok = _load(name, False, lora = True)+ enc = tok(PROMPTS[:1], return_tensors = "pt").to("cuda")+ ids = enc.input_ids+ _plain(model, tok, enc)+ _manual_loop(model, ids, True, monkeypatch)+ if stale_control:+ pinned = G._state(_causal(model)).fingerprint+ monkeypatch.setattr(G, "_fingerprint", lambda *a: pinned)+ params = [p for n, p in model.named_parameters() if "lora_" in n and p.requires_grad]+ FastLanguageModel.for_training(model)+ opt = torch.optim.SGD(params, lr = 1.0)+ for p in params:+ p.grad = torch.randn_like(p) * 0.05+ opt.step()+ opt.zero_grad(set_to_none = True)+ FastLanguageModel.for_inference(model)+ _plain(model, tok, enc)+ eager = _manual_loop(model, ids, False, monkeypatch)+ graph = _manual_loop(model, ids, True, monkeypatch)+ if stale_control:+ assert not torch.equal(eager, graph)+ return+ assert torch.equal(eager, graph)+ with torch.no_grad():+ for p in params:+ p.mul_(-1.0)+ assert torch.equal(+ _manual_loop(model, ids, False, monkeypatch), _manual_loop(model, ids, True, monkeypatch)+ )+ with model.disable_adapter():+ assert torch.equal(+ _manual_loop(model, ids, False, monkeypatch),+ _manual_loop(model, ids, True, monkeypatch),+ )+++@cuda+def test_manual_decode_loop_grows_cache(monkeypatch):+ """Direct forward calls (no generate() length hint): the cache grows past its first allocation+ (new addresses), and the graphs re-capture instead of replaying the old buffers."""+ model, tok = _load(MODELS[0], False)+ ids = tok(PROMPTS[:1], return_tensors = "pt").input_ids.to("cuda")++ def loop(enabled, steps = 800):+ monkeypatch.setattr(G, "ENABLED", enabled)+ with torch.inference_mode():+ out = model(ids, use_cache = True, return_dict = True)+ past, nxt, logits = out.past_key_values, out.logits[:, -1:].argmax(-1), []+ for i in range(steps):+ pos = torch.full((1, 1), ids.shape[1] + i, device = "cuda", dtype = torch.long)+ out = model(+ nxt, past_key_values = past, position_ids = pos, use_cache = True, return_dict = True+ )+ past = out.past_key_values+ logits.append(out.logits[:, -1].float().cpu())+ nxt = out.logits[:, -1:].argmax(-1)+ return torch.stack(logits)++ eager = loop(False)+ stats0 = dict(G.stats(_causal(model)))+ graph = loop(True)+ stats = G.stats(_causal(model))+ assert stats["captures"] - stats0.get("captures", 0) >= 2, stats+ assert torch.equal(eager, graph)+++@cuda+def test_manual_loop_adapter_toggle_and_merge(monkeypatch):+ """Decode loops outside generate(): each prefill re-checks the fingerprint, so a disabled or+ merged adapter is never replayed from graphs captured with it active."""+ model, tok = _load(MODELS[0], False, lora = True)+ ids = tok(PROMPTS[:1], return_tensors = "pt").input_ids.to("cuda")+ assert torch.equal(+ _manual_loop(model, ids, False, monkeypatch), _manual_loop(model, ids, True, monkeypatch)+ )+ with model.disable_adapter():+ assert torch.equal(+ _manual_loop(model, ids, False, monkeypatch),+ _manual_loop(model, ids, True, monkeypatch),+ )+ model.merge_adapter()+ try:+ assert torch.equal(+ _manual_loop(model, ids, False, monkeypatch),+ _manual_loop(model, ids, True, monkeypatch),+ )+ finally:+ model.unmerge_adapter()+ assert torch.equal(+ _manual_loop(model, ids, False, monkeypatch), _manual_loop(model, ids, True, monkeypatch)+ )+++@cuda+def test_explicit_positions_past_rope_table(monkeypatch):+ """Positions past the RoPE rows the graphs were sized for run eagerly (the eager step grows the+ table), and the graphs re-capture against the new table."""+ model, tok = _load(MODELS[0], False)+ ids = tok(PROMPTS[:1], return_tensors = "pt").input_ids.to("cuda")+ _manual_loop(model, ids, True, monkeypatch)+ far = 40000+ eager = _manual_loop(model, ids, False, monkeypatch, start = far)+ graph = _manual_loop(model, ids, True, monkeypatch, start = far)+ assert torch.equal(eager, graph)+ assert torch.equal(+ _manual_loop(model, ids, False, monkeypatch), _manual_loop(model, ids, True, monkeypatch)+ )+++@cuda+def test_release_drops_graphs_and_buffers(monkeypatch):+ model, tok = _load(MODELS[0], False, lora = True)+ enc = tok(PROMPTS[:1], return_tensors = "pt").to("cuda")+ _generate(model, tok, enc, True, monkeypatch)+ G.release(model)+ assert G.stats(_causal(model))["graphs"] == 0+ assert not any("_unsloth_graph_buffers" in m.__dict__ for m in model.modules())+ torch.cuda.empty_cache()+ assert torch.equal(+ _generate(model, tok, enc, False, monkeypatch)[1],+ _generate(model, tok, enc, True, monkeypatch)[1],+ )+++@cuda+def test_steps_counter_present():+ # The adapter-change check keys on this counter (bumped by every optimizer step).+ assert isinstance(kernel_utils._OPTIMIZER_STEPS, int)+++def test_graph_statics_are_per_thread():+ """A capture's statics are invisible to another thread's (or model's) eager decode."""+ import threading++ seen = []+ G._TLS.statics = object()+ try:+ t = threading.Thread(target = lambda: seen.append(G.static()))+ t.start()+ t.join()+ finally:+ G._TLS.statics = None+ assert seen == [None] and G.static() is Nonediff --git a/tests/test_decode_kv_direct.py b/tests/test_decode_kv_direct.pynew file mode 100644index 00000000000..44d866022f4--- /dev/null+++ b/tests/test_decode_kv_direct.py@@ -0,0 +1,106 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""Native decode writes K / V projections straight into the step's KV-cache slot (K rotated there in place).+Checked against the copy-based path it replaces."""++import types++import pytest++import unsloth # noqa: F401 (must precede transformers)+import torch++from unsloth.models import llama++N_HEADS, N_KV, HEAD_DIM, HIDDEN = 8, 2, 64, 256++cuda = pytest.mark.skipif(not torch.cuda.is_available(), reason = "CUDA only")+++def _attention(dtype, device):+ torch.manual_seed(0)++ def linear(i, o):+ layer = torch.nn.Linear(i, o, bias = False).to(device = device, dtype = dtype)+ layer.weight.requires_grad_(False)+ return layer++ pos = torch.arange(4096, device = device, dtype = torch.float32)[:, None]+ inv = 1.0 / (10000 ** (torch.arange(0, HEAD_DIM, 2, device = device).float() / HEAD_DIM))+ freqs = torch.cat([pos * inv, pos * inv], dim = -1)+ cos, sin = freqs.cos().to(dtype), freqs.sin().to(dtype)+ config = types.SimpleNamespace(+ num_attention_heads = N_HEADS,+ num_key_value_heads = N_KV,+ hidden_size = HIDDEN,+ sliding_window = None,+ )+ rotary = types.SimpleNamespace(+ extend_rope_embedding = lambda x, n: None, get_cached = lambda n, i: (cos, sin)+ )+ return (+ types.SimpleNamespace(+ config = config,+ num_key_value_groups = N_HEADS // N_KV,+ head_dim = HEAD_DIM,+ q_proj = linear(HIDDEN, N_HEADS * HEAD_DIM),+ k_proj = linear(HIDDEN, N_KV * HEAD_DIM),+ v_proj = linear(HIDDEN, N_KV * HEAD_DIM),+ o_proj = linear(N_HEADS * HEAD_DIM, HIDDEN),+ rotary_emb = rotary,+ ),+ cos,+ sin,+ )+++def _rotate(x, cos, sin, position):+ h = x.shape[-1] // 2+ rh = torch.cat((-x[..., h:], x[..., :h]), dim = -1)+ return x * cos[position] + rh * sin[position]+++@cuda[email protected]("dtype", [torch.float16, torch.bfloat16])[email protected]("bsz", [1, 3])[email protected]("ignore_out", [False, True])+def test_kv_slot_holds_rotated_k_and_raw_v(dtype, bsz, ignore_out, monkeypatch):+ if ignore_out:+ # Quantized projections (NVFP4 / FP8) return a fresh tensor: the slot must still be filled.+ real = llama.fast_linear_forward+ monkeypatch.setattr(+ llama,+ "fast_linear_forward",+ lambda proj, X, temp_lora = None, out = None: real(proj, X).clone(),+ )+ attn, cos, sin = _attention(dtype, "cuda")+ cached = 5+ K1 = torch.randn(bsz, N_KV, cached, HEAD_DIM, dtype = dtype, device = "cuda")+ V1 = torch.randn(bsz, N_KV, cached, HEAD_DIM, dtype = dtype, device = "cuda")+ past = (K1, V1)+ for step in range(4):+ seq_len = cached + step+ X = torch.randn(bsz, 1, HIDDEN, dtype = dtype, device = "cuda")+ out, past = llama.LlamaAttention_fast_forward_inference(+ attn,+ X,+ past,+ torch.full((bsz, 1), seq_len, device = "cuda"),+ attention_mask = None,+ do_prefill = step == 0,+ )+ k = (X @ attn.k_proj.weight.t()).view(bsz, N_KV, HEAD_DIM)+ v = (X @ attn.v_proj.weight.t()).view(bsz, N_KV, HEAD_DIM)+ torch.testing.assert_close(+ attn.paged_attention_K[seq_len], _rotate(k, cos, sin, seq_len), atol = 2e-2, rtol = 2e-2+ )+ torch.testing.assert_close(attn.paged_attention_V[seq_len], v, atol = 0, rtol = 0)+ assert past[0].shape[-2] == seq_len + 1+ torch.testing.assert_close(+ past[0][:, :, -1], attn.paged_attention_K[seq_len], atol = 0, rtol = 0+ )+ # Earlier slots are untouched by later steps.+ torch.testing.assert_close(+ attn.paged_attention_K[:cached].permute(1, 2, 0, 3), K1, atol = 0, rtol = 0+ )diff --git a/unsloth/models/_decode_graphs.py b/unsloth/models/_decode_graphs.pynew file mode 100644index 00000000000..b6e7fa6178e--- /dev/null+++ b/unsloth/models/_decode_graphs.py@@ -0,0 +1,665 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.++"""CUDA-graph replay of the native one-token decode step (opt-in: UNSLOTH_DECODE_CUDA_GRAPHS=1).++The eager decode step is host bound (hundreds of small launches per token). It is captured once per+batch size as n_layers + 1 graph segments split at each layer's attention core, which runs eagerly+between them over the true KV length (_decode_attention_core, the same code the eager step runs), so+a replayed step computes bit for bit what the eager step computes, at every length, with no buckets:++* the per-layer decode buffers (KV cache, projection scratch) persist across generate() calls and are+ sized up front to prompt + max_new_tokens, so the addresses the segments read and write hold;+* the segments write K / V with index_copy_ at a device-side cache slot and index RoPE with a device+ position tensor, so they do not depend on the length either;+* the first decode step after a prefill stays eager (it copies the prefill K / V into the cache), and+ sampling / logits processing stays in transformers.++A graph bakes in every tensor address it reads and every host-side branch taken at capture, so the+graphs are dropped when the model's parameters, LoRA adapters (cast caches included), quant states or+RoPE tables move or change version; this is checked once per generate() call and after every+optimizer step. Anything unsupported runs the eager step.+"""++import math+import os+import threading++import torch++ENABLED = os.environ.get("UNSLOTH_DECODE_CUDA_GRAPHS", "0") == "1"+# KV-cache rows are allocated in multiples of this with graphs on.+BUCKET = 256+# Captured steps (one per batch size / autocast state) kept per model.+MAX_GRAPHS = int(os.environ.get("UNSLOTH_DECODE_GRAPH_MAX", "8"))++__all__ = [+ "ENABLED",+ "static",+ "alloc_decode_buffers",+ "decode_step",+ "invalidate",+ "release",+ "register_attention",+ "start_generation",+ "end_generation",+]++# The graph-mode statics of the step this thread is warming up / capturing (static()); per thread,+# so another thread's (or model's) eager decode never reads them.+_TLS = threading.local()+# Bumped whenever a persistent decode buffer is (re)allocated or resized.+BUFFER_EPOCH = 0+# Bumped per generate() call and per prefill: the next graph step revalidates the fingerprint.+GENERATION_EPOCH = 0+# prompt + max_new_tokens of the running generate() call, 0 when unknown.+MAX_LEN_HINT = 0+_SUPPORTED_ATTENTION = set()+# Tested families only (Qwen2 shares the Llama attention but is untested here).+_SUPPORTED_MODEL_TYPES = {"llama", "qwen3"}+_PROJ_NAMES = ("q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj")+++def static():+ """The _StepStatics of a graph-mode step running on this thread, else None (eager)."""+ return getattr(_TLS, "statics", None)+++def register_attention(fn):+ """Marks a decode attention function as reading static() (device slot, eager core split)."""+ _SUPPORTED_ATTENTION.add(fn)+ return fn+++def _round_up(n, m):+ return -(-n // m) * m+++class _StepStatics:+ """What a graph-mode step reads instead of host state: the device cache slot, the RoPE table+ length, and (per replayed step) the true KV length + mask for the eager attention cores.+ attention(fn) is the split point: eager it runs fn, capturing it closes the current segment."""++ __slots__ = ("rope_len", "slot", "kv_len", "mask", "attention")++ def __init__(self, rope_len, slot):+ self.rope_len = rope_len+ self.slot = slot+ self.kv_len = None+ self.mask = None+ self.attention = _run_now+++def _run_now(fn):+ return fn()+++def _no_inference_tensors():+ # Persistent buffers outlive the generate() inference_mode; inference tensors could not be+ # written by a later decode run under no_grad only.+ import contextlib++ if not torch.is_inference_mode_enabled():+ return contextlib.nullcontext()+ stack = contextlib.ExitStack()+ stack.enter_context(torch.inference_mode(False))+ stack.enter_context(torch.no_grad())+ return stack+++def alloc_decode_buffers(+ attn, K1, V1, bsz, n_heads, n_kv_heads, head_dim, hidden_size, dtype, device, increment+):+ """The decode buffers the first decode step allocates. Eager (graphs off) is unchanged; with+ graphs on they persist per (bsz, dtype, device) and are sized for the whole generate() call."""+ global BUFFER_EPOCH, GENERATION_EPOCH+ seq_len = K1.shape[-2]+ attention_size = n_heads * head_dim+ if not ENABLED:+ attn.paged_attention = torch.empty(+ (increment + seq_len + 1, 2, bsz, n_kv_heads, head_dim),+ dtype = dtype,+ device = device,+ )+ attn.paged_attention_K = attn.paged_attention[:, 0]+ attn.paged_attention_V = attn.paged_attention[:, 1]+ attn.paged_attention_K[:seq_len] = K1.permute(2, 0, 1, 3)+ attn.paged_attention_V[:seq_len] = V1.permute(2, 0, 1, 3)+ attn.temp_QA = torch.empty((2, bsz, 1, attention_size), dtype = dtype, device = device)+ attn.temp_KV = torch.empty((2, bsz, 1, n_kv_heads * head_dim), dtype = dtype, device = device)+ attn.RH_Q = torch.empty((bsz, n_heads, 1, head_dim), dtype = dtype, device = device)+ # Mistral Nemo 12b has weird dimensions.+ if attention_size != hidden_size:+ attn.temp_O = torch.empty((bsz, 1, hidden_size), dtype = dtype, device = device)+ else:+ attn.temp_O = attn.temp_QA[1][:, :, :hidden_size]+ # Row stride kept a multiple of 8 so the grouped decode bmm stays on the fast cuBLAS path.+ attn.attention = torch.empty(+ (bsz, n_heads, 1, increment + seq_len + (-seq_len) % 8),+ dtype = dtype,+ device = device,+ )+ else:+ rows = _round_up(max(increment + seq_len + 1, MAX_LEN_HINT + 1), BUCKET)+ store = attn.__dict__.get("_unsloth_graph_buffers")+ if store is None:+ store = attn._unsloth_graph_buffers = {}+ key = (bsz, dtype, device)+ bufs = store.get(key)+ if bufs is None or bufs[0].shape[0] < rows:+ with _no_inference_tensors():+ # Zeroed: a captured step reads every row of its bucket, and a masked row still+ # multiplies its V by 0, which is NaN for uninitialised NaN / inf bit patterns.+ paged = torch.zeros(+ (rows, 2, bsz, n_kv_heads, head_dim), dtype = dtype, device = device+ )+ temp_QA = torch.empty((2, bsz, 1, attention_size), dtype = dtype, device = device)+ temp_KV = torch.empty(+ (2, bsz, 1, n_kv_heads * head_dim), dtype = dtype, device = device+ )+ RH_Q = torch.empty((bsz, n_heads, 1, head_dim), dtype = dtype, device = device)+ if attention_size != hidden_size:+ temp_O = torch.empty((bsz, 1, hidden_size), dtype = dtype, device = device)+ else:+ temp_O = temp_QA[1][:, :, :hidden_size]+ scores = torch.empty((bsz, n_heads, 1, rows), dtype = dtype, device = device)+ # Graphs reading a replaced / evicted buffer are dropped (BUFFER_EPOCH); a new batch size+ # beside the ones kept leaves their graphs valid.+ if bufs is not None or len(store) >= 2:+ store.clear()+ BUFFER_EPOCH += 1+ bufs = store[key] = (paged, temp_QA, temp_KV, RH_Q, temp_O, scores)+ paged, temp_QA, temp_KV, RH_Q, temp_O, scores = bufs+ # A new sequence: its first graph step re-checks the fingerprint (adapters toggled or+ # merged since the last one, also in decode loops that do not go through generate()).+ GENERATION_EPOCH += 1+ attn.paged_attention = paged+ attn.paged_attention_K = paged[:, 0]+ attn.paged_attention_V = paged[:, 1]+ attn.paged_attention_K[:seq_len] = K1.permute(2, 0, 1, 3)+ attn.paged_attention_V[:seq_len] = V1.permute(2, 0, 1, 3)+ attn.temp_QA, attn.temp_KV, attn.RH_Q, attn.temp_O, attn.attention = (+ temp_QA,+ temp_KV,+ RH_Q,+ temp_O,+ scores,+ )+ attn.scalar = 1.0 / math.sqrt(head_dim) # as the eager code always computed it+ attn.half_head_dim = head_dim // 2+++def resize_decode_buffers(attn, increment, bsz, n_heads, n_kv_heads, head_dim):+ """The eager growth of the KV cache by `increment` rows (new rows zeroed with graphs on)."""+ global BUFFER_EPOCH+ rows = attn.paged_attention.shape[0]+ attn.paged_attention.resize_((rows + increment, 2, bsz, n_kv_heads, head_dim))+ attn.paged_attention_K = attn.paged_attention[:, 0]+ attn.paged_attention_V = attn.paged_attention[:, 1]+ attn.attention.resize_((bsz, n_heads, 1, attn.attention.shape[-1] + increment))+ if ENABLED:+ attn.paged_attention[rows:].zero_()+ BUFFER_EPOCH += 1+++def start_generation(max_len):+ global GENERATION_EPOCH, MAX_LEN_HINT+ GENERATION_EPOCH += 1+ MAX_LEN_HINT = max(0, int(max_len or 0))+++def end_generation():+ global MAX_LEN_HINT+ MAX_LEN_HINT = 0+++class _ModelGraphs:+ def __init__(self):+ self.graphs = {}+ self.statics = {}+ self.pool = None+ self.stream = None+ self.fingerprint = None+ self.generation_epoch = -1+ self.optimizer_steps = -1+ self.buffer_epoch = -1+ self.disabled = None+ self.rope_len = None+ self.captures = 0+ self.replays = 0+ self.eager_steps = 0++ def clear(self):+ self.graphs.clear()+ self.fingerprint = None+ # A pool whose graphs are all freed is released by the allocator: never reuse its handle.+ self.pool = None+++def _state(model):+ state = model.__dict__.get("_unsloth_decode_graphs")+ if state is None:+ state = model.__dict__["_unsloth_decode_graphs"] = _ModelGraphs()+ return state+++_BUFFER_NAMES = (+ "paged_attention",+ "paged_attention_K",+ "paged_attention_V",+ "temp_QA",+ "temp_KV",+ "RH_Q",+ "temp_O",+ "attention",+)+++def _states(model):+ # The state lives on the CausalLM the decode forward runs on, under any PEFT wrapper.+ for module in model.modules():+ state = module.__dict__.get("_unsloth_decode_graphs")+ if state is not None:+ yield state+++def invalidate(model):+ """Drops captured steps (they re-capture on the next decode); buffers stay."""+ global BUFFER_EPOCH+ for state in _states(model):+ state.clear()+ BUFFER_EPOCH += 1+++def release(model):+ """Drops captured steps and the persistent decode buffers (frees their memory)."""+ global BUFFER_EPOCH+ for state in _states(model):+ state.clear()+ state.statics.clear()+ for module in model.modules():+ if "_unsloth_graph_buffers" in module.__dict__:+ del module.__dict__["_unsloth_graph_buffers"]+ for name in _BUFFER_NAMES:+ module.__dict__.pop(name, None)+ BUFFER_EPOCH += 1+++def _version(t):+ if t is None or not isinstance(t, torch.Tensor):+ return None+ if t.is_inference():+ return (t.data_ptr(), None)+ return (t.data_ptr(), t._version)+++def _fingerprint(model, layers, rope_len):+ """Everything a captured step baked in; None when the model cannot be graphed."""+ from ..kernels import utils as K++ items = []+ devices = set()+ for p in model.parameters():+ devices.add(p.device)+ items.append((p.data_ptr(), p.dtype, p.shape))+ if len(devices) != 1 or next(iter(devices)).type != "cuda":+ return None+ compute_dtype = None+ seen_rope = set()+ for layer in layers:+ attn = layer.self_attn+ for owner in (attn, getattr(layer, "mlp", None)):+ for name in _PROJ_NAMES:+ proj = getattr(owner, name, None)+ if proj is None:+ continue+ if (+ K._has_non_lora_adapter(proj)+ or K._has_multiple_active_adapters(proj)+ or K._has_active_dora_adapter(proj)+ or K._has_active_lora_bias(proj)+ or K._has_packed_qweight(proj)+ ):+ return None+ W, W_quant, A, B, S, bias = K.get_lora_parameters_bias(proj)+ absmax = getattr(W_quant, "absmax", None)+ cast = getattr(A, "_fast_lora", None) if A is not None else None+ items.append(+ (+ W.data_ptr(),+ absmax.data_ptr() if isinstance(absmax, torch.Tensor) else id(W_quant),+ _version(A),+ _version(B),+ S,+ bias.data_ptr() if bias is not None else None,+ None if cast is None else (cast[1].data_ptr(), cast[2].data_ptr()),+ )+ )+ rope = getattr(attn, "rotary_emb", None)+ if rope is not None and id(rope) not in seen_rope:+ seen_rope.add(id(rope))+ index = next(iter(devices)).index or 0+ cos, sin = rope.get_cached(rope_len, index)+ items.append((cos.data_ptr(), sin.data_ptr(), cos.shape[0]))+ # Every adapter-aware module outside the projections too (embeddings, lm_head, modules_to_save):+ # their enabled / active / merged state picks the branch a captured step baked in.+ for module in model.modules():+ if hasattr(module, "disable_adapters") or hasattr(module, "active_adapter"):+ try:+ active = (+ module.active_adapters+ if hasattr(module, "active_adapters")+ else module.active_adapter+ )+ items.append(+ (+ id(module),+ getattr(module, "disable_adapters", None),+ tuple(active) if isinstance(active, (list, tuple)) else active,+ getattr(module, "merged", None),+ )+ )+ except Exception:+ return None+ items.append(K._OPTIMIZER_STEPS)+ return tuple(items)+++def _eligible(model, layers):+ config = getattr(model, "config", None)+ if getattr(config, "model_type", None) not in _SUPPORTED_MODEL_TYPES:+ return "model type"+ if getattr(layers, "_unsloth_block_swap", None) is not None:+ return "block swap"+ return None+++def _static_inputs(state, bsz, device, dtype, hidden_size):+ key = (bsz, device)+ s = state.statics.get(key)+ if s is None:+ with _no_inference_tensors():+ s = {+ "ids": torch.zeros((bsz, 1), dtype = torch.long, device = device),+ "pos": torch.zeros((bsz, 1), dtype = torch.long, device = device),+ "slot": torch.zeros((1,), dtype = torch.long, device = device),+ }+ state.statics[key] = s+ like = s.get("like")+ if like is None or like.dtype != dtype:+ # dtype / device stand-in for the hidden states the mask builder reads.+ s["like"] = torch.empty((bsz, 1, hidden_size), dtype = dtype, device = device)+ return s+++class _Step:+ """One captured decode step: graph segments, and the eager attention cores between them."""++ __slots__ = ("segments", "cores", "logits", "statics")++ def __init__(self, statics):+ self.segments = []+ self.cores = []+ self.logits = None+ self.statics = statics++ def replay(self):+ segments, cores = self.segments, self.cores+ for i in range(len(cores)):+ segments[i].replay()+ cores[i]()+ segments[-1].replay()+++def _run_step(model, fast_forward_inference, logits_fn, s, st, past_key_values):+ _TLS.statics = st+ try:+ outputs = fast_forward_inference(+ model,+ s["ids"],+ past_key_values,+ position_ids = s["pos"],+ attention_mask = None,+ )+ return logits_fn(model, outputs[0])+ finally:+ _TLS.statics = None+++def decode_step(+ model,+ fast_forward_inference,+ logits_fn,+ mask_fn,+ input_ids,+ past_key_values,+ position_ids,+ attention_mask,+):+ """Replays (capturing on first use) the decode step; returns (logits, past_key_values) or None+ for the eager path."""+ if torch.is_grad_enabled() or not input_ids.is_cuda:+ return None+ if (+ getattr(fast_forward_inference, "_unsloth_graph_attention", None)+ not in _SUPPORTED_ATTENTION+ ):+ return None+ if position_ids is None or os.environ.get("UNSLOTH_RETURN_HIDDEN_STATES", "0") == "1":+ return None+ if position_ids.numel() != input_ids.shape[0] and position_ids.shape[-1] == 1:+ # Shared / broadcast positions take a different eager RoPE path.+ return None+ if torch.cuda.is_current_stream_capturing() or torch.compiler.is_compiling():+ return None+ state = _state(model)+ if state.disabled is not None:+ return None+ layers = model.model.layers+ attn0 = layers[0].self_attn+ paged = attn0.__dict__.get("paged_attention")+ bsz = input_ids.shape[0]+ if (+ paged is None+ or "_unsloth_graph_buffers" not in attn0.__dict__+ or paged.shape[2] != bsz+ or input_ids.shape[1] != 1+ ):+ # First decode step after a prefill (allocates / fills the cache), or a cache this path+ # did not size.+ state.eager_steps += 1+ return None+ seq_len = past_key_values[0][0].shape[-2]+ kv_len = seq_len + 1+ cap = paged.shape[0]+ if kv_len >= cap:+ # The eager step grows the cache (new addresses: the next step re-captures).+ state.eager_steps += 1+ return None++ from ..kernels import utils as K++ # RoPE rows the captured steps may read: the table is sized (and its address fingerprinted)+ # for this many, so a position at or past it runs eagerly (the eager step grows the table).+ rope_len = max(state.rope_len or 0, cap + 1)+ bound = getattr(position_ids, "_unsloth_position_bound", None)+ if bound is None:+ # Caller-supplied positions can exceed the KV length: one host read, as the eager step does.+ bound = int(position_ids.max()) + 1+ if bound > rope_len:+ state.eager_steps += 1+ state.generation_epoch = -1+ return None+ if (+ state.generation_epoch != GENERATION_EPOCH+ or state.optimizer_steps != K._OPTIMIZER_STEPS+ or state.buffer_epoch != BUFFER_EPOCH+ or state.rope_len != rope_len+ ):+ reason = _eligible(model, layers)+ if reason is not None:+ state.disabled = reason+ return None+ fingerprint = _fingerprint(model, layers, rope_len)+ if fingerprint is None:+ state.disabled = "parameters on several / non-CUDA devices or a non-LoRA adapter"+ return None+ if (+ fingerprint != state.fingerprint+ or state.buffer_epoch != BUFFER_EPOCH+ or state.rope_len != rope_len+ ):+ state.clear()+ state.fingerprint = fingerprint+ state.generation_epoch = GENERATION_EPOCH+ state.optimizer_steps = K._OPTIMIZER_STEPS+ state.buffer_epoch = BUFFER_EPOCH+ state.rope_len = rope_len++ device = input_ids.device+ config = model.config+ s = _static_inputs(state, bsz, device, paged.dtype, config.hidden_size)+ s["ids"].copy_(input_ids)+ last = position_ids[:, -1:] if position_ids.dim() == 2 else position_ids+ if last.numel() != bsz:+ return None+ s["pos"].copy_(last.view(bsz, 1))+ s["slot"].fill_(seq_len)++ autocast = _autocast_state()+ key = (bsz, device, autocast, torch.is_inference_mode_enabled())+ step = state.graphs.get(key)+ if step is None:+ if len(state.graphs) >= MAX_GRAPHS:+ state.graphs.clear()+ state.pool = None+ try:+ step = _capture(+ state, model, fast_forward_inference, logits_fn, mask_fn, s, rope_len,+ past_key_values, autocast, attention_mask, seq_len,+ ) # fmt: skip+ # The warmup may have built LoRA cast caches or RoPE rows the graph now reads.+ state.fingerprint = _fingerprint(model, layers, rope_len)+ except Exception as error:+ state.disabled = f"capture failed: {type(error).__name__}: {error}"+ state.clear()+ if os.environ.get("UNSLOTH_DECODE_CUDA_GRAPHS_STRICT", "0") == "1":+ raise+ return None+ state.graphs[key] = step+ state.captures += 1+ st = step.statics+ st.kv_len = kv_len+ st.mask = mask_fn(model, attention_mask, bsz, 1, s["like"], seq_len)+ try:+ step.replay()+ finally:+ st.mask = None+ state.replays += 1+ next_cache = [+ (+ layer.self_attn.paged_attention_K[:kv_len].permute(1, 2, 0, 3),+ layer.self_attn.paged_attention_V[:kv_len].permute(1, 2, 0, 3),+ )+ for layer in layers+ ]+ return step.logits.clone(), next_cache+++def _autocast_state():+ try:+ enabled = torch.is_autocast_enabled("cuda")+ dtype = torch.get_autocast_dtype("cuda") if enabled else None+ except TypeError:+ enabled = torch.is_autocast_enabled()+ dtype = torch.get_autocast_gpu_dtype() if enabled else None+ return (enabled, dtype)+++def _capture(+ state,+ model,+ fast_forward_inference,+ logits_fn,+ mask_fn,+ s,+ rope_len,+ past_key_values,+ autocast,+ attention_mask,+ seq_len,+):+ if state.pool is None:+ state.pool = torch.cuda.graph_pool_handle()+ if state.stream is None:+ state.stream = torch.cuda.Stream()+ stream = state.stream+ st = _StepStatics(rope_len, s["slot"])+ st.kv_len = seq_len + 1+ st.mask = mask_fn(model, attention_mask, s["ids"].shape[0], 1, s["like"], seq_len)+ enabled, dtype = autocast+ # No autocast weight cache: a cast cached during capture would be freed into the graph's pool.+ no_cache = lambda: torch.autocast("cuda", dtype = dtype, enabled = enabled, cache_enabled = False)+ stream.wait_stream(torch.cuda.current_stream())+ step = _Step(st)+ with torch.cuda.stream(stream), no_cache():+ # Warmup: Triton JIT, cuBLAS workspaces, lazily built RoPE rows / LoRA casts. It writes this+ # step's K / V into its cache slot, the same values the replay writes again.+ _run_step(model, fast_forward_inference, logits_fn, s, st, past_key_values)+ stream.synchronize()++ def split(fn):+ # Close the segment, run the attention core once (its output address is what the next+ # segment reads), open the next segment.+ _end(step.segments[-1])+ # Two calls with the first result alive: the same address means the core writes into a+ # persistent buffer (bsz == 1 writes the query buffer); else (SDPA allocates) its+ # result is copied into a fixed buffer.+ out = fn()+ fixed = out if fn().data_ptr() == out.data_ptr() else torch.empty_like(out)+ step.cores.append(fn if fixed is out else (lambda: fixed.copy_(fn())))+ graph = torch.cuda.CUDAGraph()+ step.segments.append(graph)+ _begin(graph, state.pool)+ return fixed++ graph = torch.cuda.CUDAGraph()+ step.segments.append(graph)+ st.attention = split+ _begin(graph, state.pool)+ try:+ step.logits = _run_step(+ model, fast_forward_inference, logits_fn, s, st, past_key_values+ )+ finally:+ st.attention = _run_now+ if torch.cuda.is_current_stream_capturing():+ _end(step.segments[-1])+ torch.cuda.current_stream().wait_stream(stream)+ st.mask = None+ return step+++def _begin(graph, pool):+ # Outside inference mode: the first capture creates the CUDA RNG's graph state tensors, and+ # inference tensors there make every later capture under no_grad fail (torch 2.10).+ with torch.inference_mode(False):+ graph.capture_begin(pool = pool, capture_error_mode = "thread_local")+++def _end(graph):+ with torch.inference_mode(False):+ graph.capture_end()+++def stats(model):+ state = getattr(model, "__dict__", {}).get("_unsloth_decode_graphs")+ if state is None:+ return {}+ return {+ "captures": state.captures,+ "replays": state.replays,+ "graphs": len(state.graphs),+ "eager_steps": state.eager_steps,+ "disabled": state.disabled,+ }diff --git a/unsloth/models/llama.py b/unsloth/models/llama.pyindex 277e06e8daa..635f62eba02 100644--- a/unsloth/models/llama.py+++ b/unsloth/models/llama.py@@ -157,6 +157,7 @@ from ..kernels.fused_add_rms_layernorm import (fast_add_rms_layernorm_inference_ok,)from ..kernels import fast_lora as _fast_lora+from . import _decode_graphsfrom ..kernels.bnb_override import install_bnb_nf4_override as _install_bnb_nf4_overridefrom ..tokenizer_utils import *from ..context_parallel import get_cp_manager@@ -542,6 +543,9 @@ def _fast_prepare_inputs_for_generation(position_ids = original_attention_mask.long().cumsum(-1) - 1position_ids.masked_fill_(original_attention_mask == 0, 1)position_ids = position_ids[:, -seq_length:]+ if _decode_graphs.ENABLED:+ # Positions from the mask stay below its length: graphed decode skips a host read.+ position_ids._unsloth_position_bound = original_attention_mask.shape[-1]kwargs["position_ids"] = position_idselif kwargs.get("cache_position", None) is not None:cp = kwargs["cache_position"]@@ -698,55 +702,37 @@ def LlamaAttention_fast_forward_inference(hidden_size = self.config.hidden_sizeattention_size = n_heads * head_dim+ # Graph mode (a captured decode step): no host lengths, the cache slot is on the device.+ static = _decode_graphs.static() if _decode_graphs.ENABLED else Noneseq_len = K1.shape[-2]kv_seq_len = seq_len + 1device = hidden_states.deviceif do_prefill:- self.paged_attention = torch.empty(- (KV_CACHE_INCREMENT + seq_len + 1, 2, bsz, n_kv_heads, head_dim),- dtype = dtype,- device = device,- )- self.paged_attention_K = self.paged_attention[:, 0]- self.paged_attention_V = self.paged_attention[:, 1]- self.paged_attention_K[:seq_len] = K1.permute(2, 0, 1, 3)- self.paged_attention_V[:seq_len] = V1.permute(2, 0, 1, 3)- self.temp_QA = torch.empty((2, bsz, 1, attention_size), dtype = dtype, device = device)- self.temp_KV = torch.empty((2, bsz, 1, n_kv_heads * head_dim), dtype = dtype, device = device)- self.RH_Q = torch.empty((bsz, n_heads, 1, head_dim), dtype = dtype, device = device)-- # Mistral Nemo 12b has weird dimensions.- if attention_size != hidden_size:- self.temp_O = torch.empty((bsz, 1, hidden_size), dtype = dtype, device = device)- else:- self.temp_O = self.temp_QA[1][:, :, :hidden_size]-- # Row stride kept a multiple of 8 so the grouped decode bmm stays on the fast cuBLAS path.- self.attention = torch.empty(- (bsz, n_heads, 1, KV_CACHE_INCREMENT + seq_len + (-seq_len) % 8),- dtype = dtype,- device = device,- )- self.scalar = 1.0 / math_sqrt(self.head_dim)- self.half_head_dim = head_dim // 2- elif kv_seq_len >= self.paged_attention.shape[0]:- self.paged_attention.resize_(- (- self.paged_attention.shape[0] + KV_CACHE_INCREMENT,- 2,- bsz,- n_kv_heads,- head_dim,- )+ _decode_graphs.alloc_decode_buffers(+ self, K1, V1, bsz, n_heads, n_kv_heads, head_dim, hidden_size, dtype, device,+ KV_CACHE_INCREMENT,+ ) # fmt: skip+ elif static is None and kv_seq_len >= self.paged_attention.shape[0]:+ _decode_graphs.resize_decode_buffers(+ self, KV_CACHE_INCREMENT, bsz, n_heads, n_kv_heads, head_dim)- self.paged_attention_K = self.paged_attention[:, 0]- self.paged_attention_V = self.paged_attention[:, 1]- self.attention.resize_((bsz, n_heads, 1, self.attention.shape[-1] + KV_CACHE_INCREMENT))Qn = fast_linear_forward(self.q_proj, Xn, out = self.temp_QA[0])- Kn = fast_linear_forward(self.k_proj, Xn, out = self.temp_KV[0])- Vn = fast_linear_forward(self.v_proj, Xn, out = self.temp_KV[1])+ # K / V land straight in this step's cache slot (contiguous [bsz, n_kv_heads, head_dim]); K is rotated+ # there in place, so the two per-layer cache copies below disappear. Graph mode writes scratch,+ # copied into the device-side slot after RoPE.+ if static is None:+ K_slot = self.paged_attention_K[seq_len].view(bsz, 1, n_kv_heads * head_dim)+ V_slot = self.paged_attention_V[seq_len].view(bsz, 1, n_kv_heads * head_dim)+ else:+ K_slot, V_slot = self.temp_KV[0], self.temp_KV[1]+ Kn = fast_linear_forward(self.k_proj, Xn, out = K_slot)+ if Kn.data_ptr() != K_slot.data_ptr():+ Kn = K_slot.copy_(Kn)+ Vn = fast_linear_forward(self.v_proj, Xn, out = V_slot)+ if Vn.data_ptr() != V_slot.data_ptr():+ Vn = V_slot.copy_(Vn)Qn = Qn.view(bsz, 1, n_heads, head_dim).transpose(1, 2)Kn = Kn.view(bsz, 1, n_kv_heads, head_dim).transpose(1, 2)Vn = Vn.view(bsz, 1, n_kv_heads, head_dim).transpose(1, 2)@@ -786,8 +772,21 @@ def LlamaAttention_fast_forward_inference(Kn *= cosKn.addcmul_(RH_K, sin)- self.paged_attention_K[seq_len] = Kn.permute(2, 0, 1, 3)- self.paged_attention_V[seq_len] = Vn.permute(2, 0, 1, 3)+ if static is not None:+ # Captured step: the cache slot is on the device; the attention itself runs eagerly+ # between two captured segments (static.attention), over the true length.+ self.paged_attention_K.index_copy_(+ 0, static.slot, K_slot.view(1, bsz, n_kv_heads, head_dim)+ )+ self.paged_attention_V.index_copy_(+ 0, static.slot, V_slot.view(1, bsz, n_kv_heads, head_dim)+ )+ A = static.attention(+ lambda: _decode_attention_core(self, Qn, static.kv_len, static.mask, bsz, True)[0]+ )+ A = A.transpose(1, 2).reshape(bsz, 1, attention_size)+ return fast_linear_forward(self.o_proj, A, out = self.temp_O), (None, None)+Kn = self.paged_attention_K[:kv_seq_len].permute(1, 2, 0, 3)Vn = self.paged_attention_V[:kv_seq_len].permute(1, 2, 0, 3)@@ -855,6 +854,93 @@ def LlamaAttention_fast_forward_inference(return A, (Kn, Vn)+def _decode_attention_core(self, Qn, kv_seq_len, attention_mask, bsz, sdpa_gqa_with_batch_mask):+ """softmax(Q K^T) V of one decode token over the first kv_seq_len cache rows; returns+ (A [bsz, n_heads, 1, head_dim], K, V). The captured step calls it between its graph segments.+ It is op for op the attention inlined in the Llama / Qwen3 decode functions (kept inline there+ so the eager step pays no extra call), so both compute bit for bit the same.+ sdpa_gqa_with_batch_mask=False (Qwen3) expands K / V instead of SDPA GQA for a masked batch."""+ n_heads = self.__dict__.get("_unsloth_tp_heads", self.config.num_attention_heads)+ n_groups = self.num_key_value_groups+ n_kv_heads = self.__dict__.get("_unsloth_tp_kv_heads", self.config.num_key_value_heads)+ head_dim = self.head_dim+ Kn = self.paged_attention_K[:kv_seq_len].permute(1, 2, 0, 3)+ Vn = self.paged_attention_V[:kv_seq_len].permute(1, 2, 0, 3)++ # Handle sliding windows, from transformers models/mistral/modeling_mistral.py#L193+ sliding_window = getattr(self.config, "sliding_window", None)+ if sliding_window is not None and kv_seq_len > sliding_window:+ start = kv_seq_len - sliding_window+ Knn = Kn[:, :, start:, :]+ Vnn = Vn[:, :, start:, :]+ if attention_mask is not None:+ attention_mask = attention_mask[..., start:]+ else:+ Knn, Vnn = Kn, Vn++ if attention_mask is not None and attention_mask.dim() == 2:+ attention_mask = attention_mask[:, None, None, :].to(torch.bool)+ elif (+ attention_mask is not None+ and attention_mask.dim() == 4+ and attention_mask.dtype != torch.bool+ ):+ # Decode is more stable with boolean keep masks than additive bf16 masks.+ attention_mask = attention_mask.eq(0)++ use_sdpa_gqa = SDPA_HAS_GQA+ if (+ use_sdpa_gqa+ and not sdpa_gqa_with_batch_mask+ and isinstance(attention_mask, torch.Tensor)+ and attention_mask.dim() >= 3+ and attention_mask.shape[0] > 1+ ):+ # Avoid SDPA GQA drift for batched masked decode.+ use_sdpa_gqa = False++ # Grouped query attention; bsz == 1 reads the KV heads in place below.+ _, _, cached_len, _ = Knn.shape+ if bsz != 1 and (not use_sdpa_gqa) and n_groups != 1:+ Knn = Knn[:, :, None, :, :].expand(bsz, n_kv_heads, n_groups, cached_len, head_dim)+ Vnn = Vnn[:, :, None, :, :].expand(bsz, n_kv_heads, n_groups, cached_len, head_dim)+ Knn = Knn.reshape(bsz, n_heads, cached_len, head_dim)+ Vnn = Vnn.reshape(bsz, n_heads, cached_len, head_dim)++ # When qlen == vlen and attn_mask is None, causal attention is the right choice.+ Q_len = Qn.shape[-2]+ K_len = Knn.shape[-2]+ if attention_mask is None and Q_len == K_len:+ is_causal = True+ else:+ is_causal = False+ if bsz == 1:+ Qn *= self.scalar+ # (Q * scalar) @ K beats (Q @ K) * scalar for stopping overflows; see ggerganov/llama.cpp#7805.+ A = _gqa_decode_scores(Qn, Knn, self.attention[:, :, :, :cached_len])+ if attention_mask is not None:+ A.masked_fill_(~attention_mask, float("-inf"))+ A[:] = torch_nn_functional_softmax(A, dim = -1, dtype = torch.float32)+ A = _gqa_decode_values(A, Vnn, Qn)+ else:+ if use_sdpa_gqa:+ A = scaled_dot_product_attention(+ Qn,+ Knn,+ Vnn,+ attn_mask = attention_mask,+ is_causal = is_causal,+ enable_gqa = True,+ )+ else:+ A = scaled_dot_product_attention(+ Qn, Knn, Vnn, attn_mask = attention_mask, is_causal = is_causal+ )+ return A, Kn, Vn+++_decode_graphs.register_attention(LlamaAttention_fast_forward_inference)+torch_nn_functional_silu = torch.nn.functional.silu@@ -1631,6 +1717,23 @@ def LlamaModel_fast_forward(# Ported from transformers models/llama/modeling_llama.py#L825+def _decode_step_mask(self, attention_mask, bsz, q_len, X, seq_len):+ """The decode step's attention mask: None, or a bool keep mask for every layer."""+ if attention_mask is None:+ return None+ attention_mask = _prepare_4d_causal_attention_mask_for_sdpa(+ attention_mask,+ (bsz, q_len),+ X,+ seq_len,+ sliding_window = getattr(self.config, "sliding_window", None),+ )+ # Pre-convert to bool once for all layers, avoiding a per-layer .eq(0).+ if attention_mask is not None and attention_mask.dtype != torch.bool:+ attention_mask = attention_mask.eq(0)+ return attention_mask++def _LlamaModel_fast_forward_inference(attention_fast_forward_inference = LlamaAttention_fast_forward_inference,mlp_fast_forward_inference = fast_swiglu_inference,@@ -1669,7 +1772,11 @@ def _LlamaModel_fast_forward_inference(seq_len = past_key_values[0][0].shape[-2]kv_seq_len = seq_len + 1- if attention_mask is not None:+ static = _decode_graphs.static() if _decode_graphs.ENABLED else None+ if static is not None:+ # Captured step: the runner builds this mask per step (_decode_step_mask), outside the graph.+ attention_mask = None+ elif attention_mask is not None:attention_mask = _prepare_4d_causal_attention_mask_for_sdpa(attention_mask,(bsz, q_len),@@ -1684,7 +1791,10 @@ def _LlamaModel_fast_forward_inference(attention_mask = None# Compute rotary_seq_len once to avoid a per-layer GPU-CPU sync from .item().- rotary_seq_len = max(kv_seq_len, int(position_ids.max().item()) + 1)+ if static is not None:+ rotary_seq_len = static.rope_len+ else:+ rotary_seq_len = max(kv_seq_len, int(position_ids.max().item()) + 1)next_decoder_cache = []block_swap = getattr(self.model.layers, "_unsloth_block_swap", None)@@ -1751,6 +1861,11 @@ def _LlamaModel_fast_forward_inference(attentions = [],)+ if mlp_fast_forward_inference is fast_swiglu_inference:+ # The attention function decides whether a captured (graph-mode) step is possible.+ LlamaModel_fast_forward_inference_custom._unsloth_graph_attention = (+ attention_fast_forward_inference+ )return LlamaModel_fast_forward_inference_custom@@ -1851,6 +1966,19 @@ def _run_backbone(self, **kwargs):model._unsloth_head_reads_final_norm = False+def _decode_logits(self, hidden_states):+ """The decode branch of CausalLM_fast_forward's head (q_len == 1, no labels), for captured steps."""+ lm_head = self.lm_head.weight+ dtype = lm_head.dtype+ if hidden_states.shape[0] == 1:+ logits = torch.mv(lm_head, hidden_states.ravel().to(dtype)).unsqueeze(0).unsqueeze(0)+ else:+ logits = self.lm_head(hidden_states.to(dtype))+ logits = logits.to(_get_dtype(dtype_from_config(self.config)))+ logit_softcapping, logit_scaling = resolve_logit_scaling(self.config)+ return apply_logit_transforms(logits, logit_softcapping, logit_scaling)++def CausalLM_fast_forward(fast_forward_inference):def _CausalLM_fast_forward(self,@@ -1876,6 +2004,19 @@ def CausalLM_fast_forward(fast_forward_inference):if past_key_values is not None and len(past_key_values) == 0:past_key_values = Noneif past_key_values is not None and input_ids is not None and input_ids.shape[1] == 1:+ if _decode_graphs.ENABLED and labels is None and shift_labels is None:+ graphed = _decode_graphs.decode_step(+ self, fast_forward_inference, _decode_logits, _decode_step_mask, input_ids,+ past_key_values, position_ids, attention_mask,+ ) # fmt: skip+ if graphed is not None:+ # The same output the eager decode branch below returns.+ if not return_dict:+ return (graphed[0], graphed[1], [], [])+ return CausalLMOutputWithPast(+ loss = None, logits = graphed[0], past_key_values = graphed[1],+ hidden_states = [], attentions = [],+ ) # fmt: skipoutputs = fast_forward_inference(self,input_ids,@@ -2790,6 +2931,25 @@ class LongRopeRotaryEmbedding(_PerDeviceRopeCache, torch.nn.Module):]+def _generation_max_len(self, args, kwargs):+ """prompt + max_new_tokens of a generate() call (0 if unknown), to size the decode buffers once."""+ try:+ ids = kwargs.get("input_ids", kwargs.get("inputs", args[0] if args else None))+ if ids is None:+ ids = kwargs.get("inputs_embeds", None)+ prompt = ids.shape[1] if hasattr(ids, "shape") and len(ids.shape) >= 2 else 0+ config = kwargs.get("generation_config", None) or getattr(self, "generation_config", None)+ new = kwargs.get("max_new_tokens", None)+ if new is None and kwargs.get("max_length", None) is not None:+ return int(kwargs["max_length"])+ if new is None:+ # A config's default max_length (often the context size) is not a request.+ new = getattr(config, "max_new_tokens", None)+ return prompt + int(new) if new is not None else 0+ except Exception:+ return 0++def unsloth_fast_generate(self, *args, **kwargs):restore_training_mode = self.training# Snapshot the real GC mode (e.g. "unsloth") before for_inference clears it, so the restore@@ -2871,12 +3031,22 @@ def unsloth_fast_generate(self, *args, **kwargs):kwargs["pad_token_id"] = kwargs.pop("pad_token_id", model_eos_token_id)+ if _decode_graphs.ENABLED:+ _decode_graphs.start_generation(_generation_max_len(self, args, kwargs))# Mixed precision autocast.- with (- _get_inference_mode_context_manager(self),- torch.autocast(device_type = DEVICE_TYPE_TORCH, dtype = dtype),- ):- output = self._old_generate(*args, **kwargs)+ try:+ with (+ _get_inference_mode_context_manager(self),+ torch.autocast(device_type = DEVICE_TYPE_TORCH, dtype = dtype),+ ):+ output = self._old_generate(*args, **kwargs)+ finally:+ if _decode_graphs.ENABLED:+ _decode_graphs.end_generation()+ if _decode_graphs.ENABLED and getattr(output, "past_key_values", None) is not None:+ # The returned cache views the persistent decode buffers: hand them over, so a later+ # generate() allocates its own instead of overwriting a cache the caller kept.+ _decode_graphs.release(self)if restore_training_mode:FastLlamaModel.for_training(diff --git a/unsloth/models/qwen3.py b/unsloth/models/qwen3.pyindex 3392c438f63..db63772289b 100644--- a/unsloth/models/qwen3.py+++ b/unsloth/models/qwen3.py@@ -17,6 +17,8 @@ from unsloth_zoo.utils import Version, _get_dtypefrom ..utils.packing import get_packed_info_from_kwargsfrom ..kernels import qk_norm_rope as _qk_norm_ropefrom ..kernels.qk_norm_rope import fast_qk_norm_rope, decode_qk_norm_rope+from . import _decode_graphs+from .llama import _decode_attention_corefrom ..utils.attention_dispatch import (AttentionConfig,AttentionContext,@@ -203,50 +205,21 @@ def Qwen3Attention_fast_forward_inference(hidden_size = self.config.hidden_sizeattention_size = n_heads * head_dim+ # Graph mode (a captured decode step): no host lengths, the cache slot is on the device.+ static = _decode_graphs.static() if _decode_graphs.ENABLED else Noneseq_len = K1.shape[-2]kv_seq_len = seq_len + 1device = hidden_states.deviceif do_prefill:- self.paged_attention = torch.empty(- (KV_CACHE_INCREMENT + seq_len + 1, 2, bsz, n_kv_heads, head_dim),- dtype = dtype,- device = device,- )- self.paged_attention_K = self.paged_attention[:, 0]- self.paged_attention_V = self.paged_attention[:, 1]- self.paged_attention_K[:seq_len] = K1.permute(2, 0, 1, 3)- self.paged_attention_V[:seq_len] = V1.permute(2, 0, 1, 3)- self.temp_QA = torch.empty((2, bsz, 1, attention_size), dtype = dtype, device = device)- self.temp_KV = torch.empty((2, bsz, 1, n_kv_heads * head_dim), dtype = dtype, device = device)- self.RH_Q = torch.empty((bsz, n_heads, 1, head_dim), dtype = dtype, device = device)-- # Mistral Nemo 12b has weird dimensions.- if attention_size != hidden_size:- self.temp_O = torch.empty((bsz, 1, hidden_size), dtype = dtype, device = device)- else:- self.temp_O = self.temp_QA[1][:, :, :hidden_size]-- self.attention = torch.empty(- (bsz, n_heads, 1, KV_CACHE_INCREMENT + seq_len + (-seq_len) % 8),- dtype = dtype,- device = device,+ _decode_graphs.alloc_decode_buffers(+ self, K1, V1, bsz, n_heads, n_kv_heads, head_dim, hidden_size, dtype, device,+ KV_CACHE_INCREMENT,+ ) # fmt: skip+ elif static is None and kv_seq_len >= self.paged_attention.shape[0]:+ _decode_graphs.resize_decode_buffers(+ self, KV_CACHE_INCREMENT, bsz, n_heads, n_kv_heads, head_dim)- self.scalar = 1.0 / math_sqrt(self.head_dim)- self.half_head_dim = head_dim // 2- elif kv_seq_len >= self.paged_attention.shape[0]:- self.paged_attention.resize_(- (- self.paged_attention.shape[0] + KV_CACHE_INCREMENT,- 2,- bsz,- n_kv_heads,- head_dim,- )- )- self.paged_attention_K = self.paged_attention[:, 0]- self.paged_attention_V = self.paged_attention[:, 1]- self.attention.resize_((bsz, n_heads, 1, self.attention.shape[-1] + KV_CACHE_INCREMENT))Qn = fast_linear_forward(self.q_proj, Xn, out = self.temp_QA[0])Kn = fast_linear_forward(self.k_proj, Xn, out = self.temp_KV[0])@@ -295,6 +268,16 @@ def Qwen3Attention_fast_forward_inference(Kn *= cosKn.addcmul_(RH_K, sin)+ if static is not None:+ # Captured step: the cache slot is on the device; the attention itself runs eagerly+ # between two captured segments (static.attention), over the true length.+ self.paged_attention_K.index_copy_(0, static.slot, Kn.permute(2, 0, 1, 3))+ self.paged_attention_V.index_copy_(0, static.slot, Vn.permute(2, 0, 1, 3))+ A = static.attention(+ lambda: _decode_attention_core(self, Qn, static.kv_len, static.mask, bsz, False)[0]+ )+ A = A.transpose(1, 2).reshape(bsz, 1, attention_size)+ return fast_linear_forward(self.o_proj, A, out = self.temp_O), (None, None)self.paged_attention_K[seq_len] = Kn.permute(2, 0, 1, 3)self.paged_attention_V[seq_len] = Vn.permute(2, 0, 1, 3)Kn = self.paged_attention_K[:kv_seq_len].permute(1, 2, 0, 3)@@ -373,6 +356,9 @@ def Qwen3Attention_fast_forward_inference(return A, (Kn, Vn)+_decode_graphs.register_attention(Qwen3Attention_fast_forward_inference)++class FastQwen3Model(FastLlamaModel):@staticmethoddef pre_patch():