Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Decode: write K/V straight into the cache slot, and opt-in CUDA-graph replay of the decode step (#15171)

Daniel Hanaae9f5472386
diff --git a/tests/test_decode_cuda_graphs.py b/tests/test_decode_cuda_graphs.py
new file mode 100644
index 00000000000..687c5a015a3
--- /dev/null
+++ b/tests/test_decode_cuda_graphs.py
@@ -0,0 +1,343 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""CUDA-graph decode (UNSLOTH_DECODE_CUDA_GRAPHS=1): a replayed step is bit for bit the eager step
+(greedy tokens and raw logits, 16-bit and 4-bit, batch 1 and a left-padded batch), graphs are
+re-captured when LoRA weights change between generate() calls, and the KV cache may grow."""
+
+import os
+
+os.environ.setdefault("UNSLOTH_DECODE_CUDA_GRAPHS", "1")
+
+import pytest
+
+import unsloth # noqa: F401 (must precede transformers)
+from unsloth import FastLanguageModel
+import torch
+
+import unsloth.models._decode_graphs as G
+from unsloth.kernels import utils as kernel_utils
+
+cuda = pytest.mark.skipif(not torch.cuda.is_available(), reason = "CUDA only")
+MODELS = (
+ "trl-internal-testing/tiny-LlamaForCausalLM-3.2",
+ "trl-internal-testing/tiny-Qwen3ForCausalLM",
+)
+PROMPTS = [
+ "Once upon a time there was",
+ "The capital of France is Paris and",
+ "def f(x):",
+ "1, 2, 3,",
+]
+
+
+def _load(
+ name,
+ load_in_4bit,
+ lora = False,
+):
+ model, tok = FastLanguageModel.from_pretrained(
+ name, max_seq_length = 2048, load_in_4bit = load_in_4bit
+ )
+ if lora:
+ model = FastLanguageModel.get_peft_model(
+ model, r = 8, lora_alpha = 16, random_state = 0,
+ target_modules = ["q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj"],
+ ) # fmt: skip
+ g = torch.Generator().manual_seed(0)
+ with torch.no_grad():
+ for n, p in model.named_parameters():
+ if "lora_B" in n:
+ p.copy_(torch.randn(p.shape, generator = g) * 0.05)
+ FastLanguageModel.for_inference(model)
+ tok.padding_side = "left"
+ if tok.pad_token is None:
+ tok.pad_token = tok.eos_token
+ return model, tok
+
+
+def _causal(model):
+ base = getattr(model, "base_model", model)
+ return base.model if hasattr(base, "model") and hasattr(base.model, "lm_head") else model
+
+
+def _generate(
+ model,
+ tok,
+ enc,
+ enabled,
+ monkeypatch,
+ new = 40,
+):
+ monkeypatch.setattr(G, "ENABLED", enabled)
+ out = model.generate(
+ **enc, max_new_tokens = new, min_new_tokens = new, do_sample = False, use_cache = True,
+ pad_token_id = tok.pad_token_id, output_logits = True, return_dict_in_generate = True,
+ ) # fmt: skip
+ return out.sequences.cpu(), torch.stack(out.logits).cpu()
+
+
+def _plain(
+ model,
+ tok,
+ enc,
+ new = 20,
+):
+ return model.generate(
+ **enc,
+ max_new_tokens = new,
+ min_new_tokens = new,
+ do_sample = False,
+ pad_token_id = tok.pad_token_id,
+ )
+
+
+def _manual_loop(
+ model,
+ ids,
+ enabled,
+ monkeypatch,
+ steps = 12,
+ start = None,
+):
+ monkeypatch.setattr(G, "ENABLED", enabled)
+ with torch.inference_mode():
+ out = model(ids, use_cache = True, return_dict = True)
+ past, nxt, logits = out.past_key_values, out.logits[:, -1:].argmax(-1), []
+ for i in range(steps):
+ p = (ids.shape[1] if start is None else start) + i
+ pos = torch.full((ids.shape[0], 1), p, device = "cuda", dtype = torch.long)
+ out = model(
+ nxt, past_key_values = past, position_ids = pos, use_cache = True, return_dict = True
+ )
+ past = out.past_key_values
+ logits.append(out.logits[:, -1].float().cpu())
+ nxt = out.logits[:, -1:].argmax(-1)
+ return torch.stack(logits)
+
+
+@cuda
[email protected]("name", MODELS)
[email protected]("load_in_4bit", [False, True])
[email protected]("bsz", [1, 4])
+def test_graph_decode_is_bitwise_eager(name, load_in_4bit, bsz, monkeypatch):
+ model, tok = _load(name, load_in_4bit)
+ enc = tok(PROMPTS[:bsz], return_tensors = "pt", padding = True).to("cuda")
+ eager = _generate(model, tok, enc, False, monkeypatch)
+ before = G.stats(_causal(model)).get("replays", 0)
+ graph = _generate(model, tok, enc, True, monkeypatch)
+ stats = G.stats(_causal(model))
+ assert stats["disabled"] is None, stats
+ # Every step but the first (eager: it fills the cache) was replayed.
+ assert stats["replays"] - before == 40 - 2
+ assert torch.equal(eager[0], graph[0])
+ assert torch.equal(eager[1], graph[1])
+ # generate() calls that return no cache reuse the buffers and the captured step.
+ _plain(model, tok, enc)
+ captures = G.stats(_causal(model))["captures"]
+ _plain(model, tok, enc)
+ assert G.stats(_causal(model))["captures"] == captures
+ again = _generate(model, tok, enc, True, monkeypatch)
+ assert torch.equal(eager[1], again[1])
+
+
+@cuda
+def test_returned_cache_is_not_overwritten(monkeypatch):
+ """A cache handed back by generate(return_dict_in_generate=True) keeps its values through a
+ later generate(): the persistent buffers it views are released to the caller."""
+ model, tok = _load(MODELS[0], False)
+ monkeypatch.setattr(G, "ENABLED", True)
+ enc = tok(PROMPTS[:1], return_tensors = "pt").to("cuda")
+ out = model.generate(
+ **enc,
+ max_new_tokens = 20,
+ do_sample = False,
+ pad_token_id = tok.pad_token_id,
+ return_dict_in_generate = True,
+ )
+ K, V = out.past_key_values[0]
+ kept = K.clone(), V.clone()
+ other = tok(PROMPTS[1:2], return_tensors = "pt").to("cuda")
+ _plain(model, tok, other)
+ assert torch.equal(K, kept[0]) and torch.equal(V, kept[1])
+
+
+@cuda
+def test_shift_labels_decode_loss(monkeypatch):
+ model, tok = _load(MODELS[0], False)
+ ids = tok(PROMPTS[:1], return_tensors = "pt").input_ids.to("cuda")
+
+ def losses(enabled):
+ monkeypatch.setattr(G, "ENABLED", enabled)
+ with torch.inference_mode():
+ out = model(ids, use_cache = True, return_dict = True)
+ past, nxt, res = out.past_key_values, out.logits[:, -1:].argmax(-1), []
+ for i in range(6):
+ pos = torch.full((1, 1), ids.shape[1] + i, device = "cuda", dtype = torch.long)
+ out = model(
+ nxt,
+ past_key_values = past,
+ position_ids = pos,
+ use_cache = True,
+ return_dict = True,
+ shift_labels = torch.full((1, 1), 7, device = "cuda", dtype = torch.long),
+ )
+ past, nxt = out.past_key_values, out.logits[:, -1:].argmax(-1)
+ res.append(out.loss)
+ return res
+
+ eager, graph = losses(False), losses(True)
+ assert all(l is not None for l in graph)
+ assert all(torch.equal(a, b) for a, b in zip(eager, graph))
+
+
+@cuda
[email protected]("name", MODELS)
[email protected]("stale_control", [False, True])
+def test_lora_change_between_generations_recaptures(name, stale_control, monkeypatch):
+ """GRPO: an optimizer step, then an in-place adapter edit, then a disabled adapter between
+ decode runs. stale_control pins the fingerprint, so the old graphs replay: proves the check is
+ what keeps the graph path on the live adapter."""
+ model, tok = _load(name, False, lora = True)
+ enc = tok(PROMPTS[:1], return_tensors = "pt").to("cuda")
+ ids = enc.input_ids
+ _plain(model, tok, enc)
+ _manual_loop(model, ids, True, monkeypatch)
+ if stale_control:
+ pinned = G._state(_causal(model)).fingerprint
+ monkeypatch.setattr(G, "_fingerprint", lambda *a: pinned)
+ params = [p for n, p in model.named_parameters() if "lora_" in n and p.requires_grad]
+ FastLanguageModel.for_training(model)
+ opt = torch.optim.SGD(params, lr = 1.0)
+ for p in params:
+ p.grad = torch.randn_like(p) * 0.05
+ opt.step()
+ opt.zero_grad(set_to_none = True)
+ FastLanguageModel.for_inference(model)
+ _plain(model, tok, enc)
+ eager = _manual_loop(model, ids, False, monkeypatch)
+ graph = _manual_loop(model, ids, True, monkeypatch)
+ if stale_control:
+ assert not torch.equal(eager, graph)
+ return
+ assert torch.equal(eager, graph)
+ with torch.no_grad():
+ for p in params:
+ p.mul_(-1.0)
+ assert torch.equal(
+ _manual_loop(model, ids, False, monkeypatch), _manual_loop(model, ids, True, monkeypatch)
+ )
+ with model.disable_adapter():
+ assert torch.equal(
+ _manual_loop(model, ids, False, monkeypatch),
+ _manual_loop(model, ids, True, monkeypatch),
+ )
+
+
+@cuda
+def test_manual_decode_loop_grows_cache(monkeypatch):
+ """Direct forward calls (no generate() length hint): the cache grows past its first allocation
+ (new addresses), and the graphs re-capture instead of replaying the old buffers."""
+ model, tok = _load(MODELS[0], False)
+ ids = tok(PROMPTS[:1], return_tensors = "pt").input_ids.to("cuda")
+
+ def loop(enabled, steps = 800):
+ monkeypatch.setattr(G, "ENABLED", enabled)
+ with torch.inference_mode():
+ out = model(ids, use_cache = True, return_dict = True)
+ past, nxt, logits = out.past_key_values, out.logits[:, -1:].argmax(-1), []
+ for i in range(steps):
+ pos = torch.full((1, 1), ids.shape[1] + i, device = "cuda", dtype = torch.long)
+ out = model(
+ nxt, past_key_values = past, position_ids = pos, use_cache = True, return_dict = True
+ )
+ past = out.past_key_values
+ logits.append(out.logits[:, -1].float().cpu())
+ nxt = out.logits[:, -1:].argmax(-1)
+ return torch.stack(logits)
+
+ eager = loop(False)
+ stats0 = dict(G.stats(_causal(model)))
+ graph = loop(True)
+ stats = G.stats(_causal(model))
+ assert stats["captures"] - stats0.get("captures", 0) >= 2, stats
+ assert torch.equal(eager, graph)
+
+
+@cuda
+def test_manual_loop_adapter_toggle_and_merge(monkeypatch):
+ """Decode loops outside generate(): each prefill re-checks the fingerprint, so a disabled or
+ merged adapter is never replayed from graphs captured with it active."""
+ model, tok = _load(MODELS[0], False, lora = True)
+ ids = tok(PROMPTS[:1], return_tensors = "pt").input_ids.to("cuda")
+ assert torch.equal(
+ _manual_loop(model, ids, False, monkeypatch), _manual_loop(model, ids, True, monkeypatch)
+ )
+ with model.disable_adapter():
+ assert torch.equal(
+ _manual_loop(model, ids, False, monkeypatch),
+ _manual_loop(model, ids, True, monkeypatch),
+ )
+ model.merge_adapter()
+ try:
+ assert torch.equal(
+ _manual_loop(model, ids, False, monkeypatch),
+ _manual_loop(model, ids, True, monkeypatch),
+ )
+ finally:
+ model.unmerge_adapter()
+ assert torch.equal(
+ _manual_loop(model, ids, False, monkeypatch), _manual_loop(model, ids, True, monkeypatch)
+ )
+
+
+@cuda
+def test_explicit_positions_past_rope_table(monkeypatch):
+ """Positions past the RoPE rows the graphs were sized for run eagerly (the eager step grows the
+ table), and the graphs re-capture against the new table."""
+ model, tok = _load(MODELS[0], False)
+ ids = tok(PROMPTS[:1], return_tensors = "pt").input_ids.to("cuda")
+ _manual_loop(model, ids, True, monkeypatch)
+ far = 40000
+ eager = _manual_loop(model, ids, False, monkeypatch, start = far)
+ graph = _manual_loop(model, ids, True, monkeypatch, start = far)
+ assert torch.equal(eager, graph)
+ assert torch.equal(
+ _manual_loop(model, ids, False, monkeypatch), _manual_loop(model, ids, True, monkeypatch)
+ )
+
+
+@cuda
+def test_release_drops_graphs_and_buffers(monkeypatch):
+ model, tok = _load(MODELS[0], False, lora = True)
+ enc = tok(PROMPTS[:1], return_tensors = "pt").to("cuda")
+ _generate(model, tok, enc, True, monkeypatch)
+ G.release(model)
+ assert G.stats(_causal(model))["graphs"] == 0
+ assert not any("_unsloth_graph_buffers" in m.__dict__ for m in model.modules())
+ torch.cuda.empty_cache()
+ assert torch.equal(
+ _generate(model, tok, enc, False, monkeypatch)[1],
+ _generate(model, tok, enc, True, monkeypatch)[1],
+ )
+
+
+@cuda
+def test_steps_counter_present():
+ # The adapter-change check keys on this counter (bumped by every optimizer step).
+ assert isinstance(kernel_utils._OPTIMIZER_STEPS, int)
+
+
+def test_graph_statics_are_per_thread():
+ """A capture's statics are invisible to another thread's (or model's) eager decode."""
+ import threading
+
+ seen = []
+ G._TLS.statics = object()
+ try:
+ t = threading.Thread(target = lambda: seen.append(G.static()))
+ t.start()
+ t.join()
+ finally:
+ G._TLS.statics = None
+ assert seen == [None] and G.static() is None
diff --git a/tests/test_decode_kv_direct.py b/tests/test_decode_kv_direct.py
new file mode 100644
index 00000000000..44d866022f4
--- /dev/null
+++ b/tests/test_decode_kv_direct.py
@@ -0,0 +1,106 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""Native decode writes K / V projections straight into the step's KV-cache slot (K rotated there in place).
+Checked against the copy-based path it replaces."""
+
+import types
+
+import pytest
+
+import unsloth # noqa: F401 (must precede transformers)
+import torch
+
+from unsloth.models import llama
+
+N_HEADS, N_KV, HEAD_DIM, HIDDEN = 8, 2, 64, 256
+
+cuda = pytest.mark.skipif(not torch.cuda.is_available(), reason = "CUDA only")
+
+
+def _attention(dtype, device):
+ torch.manual_seed(0)
+
+ def linear(i, o):
+ layer = torch.nn.Linear(i, o, bias = False).to(device = device, dtype = dtype)
+ layer.weight.requires_grad_(False)
+ return layer
+
+ pos = torch.arange(4096, device = device, dtype = torch.float32)[:, None]
+ inv = 1.0 / (10000 ** (torch.arange(0, HEAD_DIM, 2, device = device).float() / HEAD_DIM))
+ freqs = torch.cat([pos * inv, pos * inv], dim = -1)
+ cos, sin = freqs.cos().to(dtype), freqs.sin().to(dtype)
+ config = types.SimpleNamespace(
+ num_attention_heads = N_HEADS,
+ num_key_value_heads = N_KV,
+ hidden_size = HIDDEN,
+ sliding_window = None,
+ )
+ rotary = types.SimpleNamespace(
+ extend_rope_embedding = lambda x, n: None, get_cached = lambda n, i: (cos, sin)
+ )
+ return (
+ types.SimpleNamespace(
+ config = config,
+ num_key_value_groups = N_HEADS // N_KV,
+ head_dim = HEAD_DIM,
+ q_proj = linear(HIDDEN, N_HEADS * HEAD_DIM),
+ k_proj = linear(HIDDEN, N_KV * HEAD_DIM),
+ v_proj = linear(HIDDEN, N_KV * HEAD_DIM),
+ o_proj = linear(N_HEADS * HEAD_DIM, HIDDEN),
+ rotary_emb = rotary,
+ ),
+ cos,
+ sin,
+ )
+
+
+def _rotate(x, cos, sin, position):
+ h = x.shape[-1] // 2
+ rh = torch.cat((-x[..., h:], x[..., :h]), dim = -1)
+ return x * cos[position] + rh * sin[position]
+
+
+@cuda
[email protected]("dtype", [torch.float16, torch.bfloat16])
[email protected]("bsz", [1, 3])
[email protected]("ignore_out", [False, True])
+def test_kv_slot_holds_rotated_k_and_raw_v(dtype, bsz, ignore_out, monkeypatch):
+ if ignore_out:
+ # Quantized projections (NVFP4 / FP8) return a fresh tensor: the slot must still be filled.
+ real = llama.fast_linear_forward
+ monkeypatch.setattr(
+ llama,
+ "fast_linear_forward",
+ lambda proj, X, temp_lora = None, out = None: real(proj, X).clone(),
+ )
+ attn, cos, sin = _attention(dtype, "cuda")
+ cached = 5
+ K1 = torch.randn(bsz, N_KV, cached, HEAD_DIM, dtype = dtype, device = "cuda")
+ V1 = torch.randn(bsz, N_KV, cached, HEAD_DIM, dtype = dtype, device = "cuda")
+ past = (K1, V1)
+ for step in range(4):
+ seq_len = cached + step
+ X = torch.randn(bsz, 1, HIDDEN, dtype = dtype, device = "cuda")
+ out, past = llama.LlamaAttention_fast_forward_inference(
+ attn,
+ X,
+ past,
+ torch.full((bsz, 1), seq_len, device = "cuda"),
+ attention_mask = None,
+ do_prefill = step == 0,
+ )
+ k = (X @ attn.k_proj.weight.t()).view(bsz, N_KV, HEAD_DIM)
+ v = (X @ attn.v_proj.weight.t()).view(bsz, N_KV, HEAD_DIM)
+ torch.testing.assert_close(
+ attn.paged_attention_K[seq_len], _rotate(k, cos, sin, seq_len), atol = 2e-2, rtol = 2e-2
+ )
+ torch.testing.assert_close(attn.paged_attention_V[seq_len], v, atol = 0, rtol = 0)
+ assert past[0].shape[-2] == seq_len + 1
+ torch.testing.assert_close(
+ past[0][:, :, -1], attn.paged_attention_K[seq_len], atol = 0, rtol = 0
+ )
+ # Earlier slots are untouched by later steps.
+ torch.testing.assert_close(
+ attn.paged_attention_K[:cached].permute(1, 2, 0, 3), K1, atol = 0, rtol = 0
+ )
diff --git a/unsloth/models/_decode_graphs.py b/unsloth/models/_decode_graphs.py
new file mode 100644
index 00000000000..b6e7fa6178e
--- /dev/null
+++ b/unsloth/models/_decode_graphs.py
@@ -0,0 +1,665 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
+
+"""CUDA-graph replay of the native one-token decode step (opt-in: UNSLOTH_DECODE_CUDA_GRAPHS=1).
+
+The eager decode step is host bound (hundreds of small launches per token). It is captured once per
+batch size as n_layers + 1 graph segments split at each layer's attention core, which runs eagerly
+between them over the true KV length (_decode_attention_core, the same code the eager step runs), so
+a replayed step computes bit for bit what the eager step computes, at every length, with no buckets:
+
+* the per-layer decode buffers (KV cache, projection scratch) persist across generate() calls and are
+ sized up front to prompt + max_new_tokens, so the addresses the segments read and write hold;
+* the segments write K / V with index_copy_ at a device-side cache slot and index RoPE with a device
+ position tensor, so they do not depend on the length either;
+* the first decode step after a prefill stays eager (it copies the prefill K / V into the cache), and
+ sampling / logits processing stays in transformers.
+
+A graph bakes in every tensor address it reads and every host-side branch taken at capture, so the
+graphs are dropped when the model's parameters, LoRA adapters (cast caches included), quant states or
+RoPE tables move or change version; this is checked once per generate() call and after every
+optimizer step. Anything unsupported runs the eager step.
+"""
+
+import math
+import os
+import threading
+
+import torch
+
+ENABLED = os.environ.get("UNSLOTH_DECODE_CUDA_GRAPHS", "0") == "1"
+# KV-cache rows are allocated in multiples of this with graphs on.
+BUCKET = 256
+# Captured steps (one per batch size / autocast state) kept per model.
+MAX_GRAPHS = int(os.environ.get("UNSLOTH_DECODE_GRAPH_MAX", "8"))
+
+__all__ = [
+ "ENABLED",
+ "static",
+ "alloc_decode_buffers",
+ "decode_step",
+ "invalidate",
+ "release",
+ "register_attention",
+ "start_generation",
+ "end_generation",
+]
+
+# The graph-mode statics of the step this thread is warming up / capturing (static()); per thread,
+# so another thread's (or model's) eager decode never reads them.
+_TLS = threading.local()
+# Bumped whenever a persistent decode buffer is (re)allocated or resized.
+BUFFER_EPOCH = 0
+# Bumped per generate() call and per prefill: the next graph step revalidates the fingerprint.
+GENERATION_EPOCH = 0
+# prompt + max_new_tokens of the running generate() call, 0 when unknown.
+MAX_LEN_HINT = 0
+_SUPPORTED_ATTENTION = set()
+# Tested families only (Qwen2 shares the Llama attention but is untested here).
+_SUPPORTED_MODEL_TYPES = {"llama", "qwen3"}
+_PROJ_NAMES = ("q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj")
+
+
+def static():
+ """The _StepStatics of a graph-mode step running on this thread, else None (eager)."""
+ return getattr(_TLS, "statics", None)
+
+
+def register_attention(fn):
+ """Marks a decode attention function as reading static() (device slot, eager core split)."""
+ _SUPPORTED_ATTENTION.add(fn)
+ return fn
+
+
+def _round_up(n, m):
+ return -(-n // m) * m
+
+
+class _StepStatics:
+ """What a graph-mode step reads instead of host state: the device cache slot, the RoPE table
+ length, and (per replayed step) the true KV length + mask for the eager attention cores.
+ attention(fn) is the split point: eager it runs fn, capturing it closes the current segment."""
+
+ __slots__ = ("rope_len", "slot", "kv_len", "mask", "attention")
+
+ def __init__(self, rope_len, slot):
+ self.rope_len = rope_len
+ self.slot = slot
+ self.kv_len = None
+ self.mask = None
+ self.attention = _run_now
+
+
+def _run_now(fn):
+ return fn()
+
+
+def _no_inference_tensors():
+ # Persistent buffers outlive the generate() inference_mode; inference tensors could not be
+ # written by a later decode run under no_grad only.
+ import contextlib
+
+ if not torch.is_inference_mode_enabled():
+ return contextlib.nullcontext()
+ stack = contextlib.ExitStack()
+ stack.enter_context(torch.inference_mode(False))
+ stack.enter_context(torch.no_grad())
+ return stack
+
+
+def alloc_decode_buffers(
+ attn, K1, V1, bsz, n_heads, n_kv_heads, head_dim, hidden_size, dtype, device, increment
+):
+ """The decode buffers the first decode step allocates. Eager (graphs off) is unchanged; with
+ graphs on they persist per (bsz, dtype, device) and are sized for the whole generate() call."""
+ global BUFFER_EPOCH, GENERATION_EPOCH
+ seq_len = K1.shape[-2]
+ attention_size = n_heads * head_dim
+ if not ENABLED:
+ attn.paged_attention = torch.empty(
+ (increment + seq_len + 1, 2, bsz, n_kv_heads, head_dim),
+ dtype = dtype,
+ device = device,
+ )
+ attn.paged_attention_K = attn.paged_attention[:, 0]
+ attn.paged_attention_V = attn.paged_attention[:, 1]
+ attn.paged_attention_K[:seq_len] = K1.permute(2, 0, 1, 3)
+ attn.paged_attention_V[:seq_len] = V1.permute(2, 0, 1, 3)
+ attn.temp_QA = torch.empty((2, bsz, 1, attention_size), dtype = dtype, device = device)
+ attn.temp_KV = torch.empty((2, bsz, 1, n_kv_heads * head_dim), dtype = dtype, device = device)
+ attn.RH_Q = torch.empty((bsz, n_heads, 1, head_dim), dtype = dtype, device = device)
+ # Mistral Nemo 12b has weird dimensions.
+ if attention_size != hidden_size:
+ attn.temp_O = torch.empty((bsz, 1, hidden_size), dtype = dtype, device = device)
+ else:
+ attn.temp_O = attn.temp_QA[1][:, :, :hidden_size]
+ # Row stride kept a multiple of 8 so the grouped decode bmm stays on the fast cuBLAS path.
+ attn.attention = torch.empty(
+ (bsz, n_heads, 1, increment + seq_len + (-seq_len) % 8),
+ dtype = dtype,
+ device = device,
+ )
+ else:
+ rows = _round_up(max(increment + seq_len + 1, MAX_LEN_HINT + 1), BUCKET)
+ store = attn.__dict__.get("_unsloth_graph_buffers")
+ if store is None:
+ store = attn._unsloth_graph_buffers = {}
+ key = (bsz, dtype, device)
+ bufs = store.get(key)
+ if bufs is None or bufs[0].shape[0] < rows:
+ with _no_inference_tensors():
+ # Zeroed: a captured step reads every row of its bucket, and a masked row still
+ # multiplies its V by 0, which is NaN for uninitialised NaN / inf bit patterns.
+ paged = torch.zeros(
+ (rows, 2, bsz, n_kv_heads, head_dim), dtype = dtype, device = device
+ )
+ temp_QA = torch.empty((2, bsz, 1, attention_size), dtype = dtype, device = device)
+ temp_KV = torch.empty(
+ (2, bsz, 1, n_kv_heads * head_dim), dtype = dtype, device = device
+ )
+ RH_Q = torch.empty((bsz, n_heads, 1, head_dim), dtype = dtype, device = device)
+ if attention_size != hidden_size:
+ temp_O = torch.empty((bsz, 1, hidden_size), dtype = dtype, device = device)
+ else:
+ temp_O = temp_QA[1][:, :, :hidden_size]
+ scores = torch.empty((bsz, n_heads, 1, rows), dtype = dtype, device = device)
+ # Graphs reading a replaced / evicted buffer are dropped (BUFFER_EPOCH); a new batch size
+ # beside the ones kept leaves their graphs valid.
+ if bufs is not None or len(store) >= 2:
+ store.clear()
+ BUFFER_EPOCH += 1
+ bufs = store[key] = (paged, temp_QA, temp_KV, RH_Q, temp_O, scores)
+ paged, temp_QA, temp_KV, RH_Q, temp_O, scores = bufs
+ # A new sequence: its first graph step re-checks the fingerprint (adapters toggled or
+ # merged since the last one, also in decode loops that do not go through generate()).
+ GENERATION_EPOCH += 1
+ attn.paged_attention = paged
+ attn.paged_attention_K = paged[:, 0]
+ attn.paged_attention_V = paged[:, 1]
+ attn.paged_attention_K[:seq_len] = K1.permute(2, 0, 1, 3)
+ attn.paged_attention_V[:seq_len] = V1.permute(2, 0, 1, 3)
+ attn.temp_QA, attn.temp_KV, attn.RH_Q, attn.temp_O, attn.attention = (
+ temp_QA,
+ temp_KV,
+ RH_Q,
+ temp_O,
+ scores,
+ )
+ attn.scalar = 1.0 / math.sqrt(head_dim) # as the eager code always computed it
+ attn.half_head_dim = head_dim // 2
+
+
+def resize_decode_buffers(attn, increment, bsz, n_heads, n_kv_heads, head_dim):
+ """The eager growth of the KV cache by `increment` rows (new rows zeroed with graphs on)."""
+ global BUFFER_EPOCH
+ rows = attn.paged_attention.shape[0]
+ attn.paged_attention.resize_((rows + increment, 2, bsz, n_kv_heads, head_dim))
+ attn.paged_attention_K = attn.paged_attention[:, 0]
+ attn.paged_attention_V = attn.paged_attention[:, 1]
+ attn.attention.resize_((bsz, n_heads, 1, attn.attention.shape[-1] + increment))
+ if ENABLED:
+ attn.paged_attention[rows:].zero_()
+ BUFFER_EPOCH += 1
+
+
+def start_generation(max_len):
+ global GENERATION_EPOCH, MAX_LEN_HINT
+ GENERATION_EPOCH += 1
+ MAX_LEN_HINT = max(0, int(max_len or 0))
+
+
+def end_generation():
+ global MAX_LEN_HINT
+ MAX_LEN_HINT = 0
+
+
+class _ModelGraphs:
+ def __init__(self):
+ self.graphs = {}
+ self.statics = {}
+ self.pool = None
+ self.stream = None
+ self.fingerprint = None
+ self.generation_epoch = -1
+ self.optimizer_steps = -1
+ self.buffer_epoch = -1
+ self.disabled = None
+ self.rope_len = None
+ self.captures = 0
+ self.replays = 0
+ self.eager_steps = 0
+
+ def clear(self):
+ self.graphs.clear()
+ self.fingerprint = None
+ # A pool whose graphs are all freed is released by the allocator: never reuse its handle.
+ self.pool = None
+
+
+def _state(model):
+ state = model.__dict__.get("_unsloth_decode_graphs")
+ if state is None:
+ state = model.__dict__["_unsloth_decode_graphs"] = _ModelGraphs()
+ return state
+
+
+_BUFFER_NAMES = (
+ "paged_attention",
+ "paged_attention_K",
+ "paged_attention_V",
+ "temp_QA",
+ "temp_KV",
+ "RH_Q",
+ "temp_O",
+ "attention",
+)
+
+
+def _states(model):
+ # The state lives on the CausalLM the decode forward runs on, under any PEFT wrapper.
+ for module in model.modules():
+ state = module.__dict__.get("_unsloth_decode_graphs")
+ if state is not None:
+ yield state
+
+
+def invalidate(model):
+ """Drops captured steps (they re-capture on the next decode); buffers stay."""
+ global BUFFER_EPOCH
+ for state in _states(model):
+ state.clear()
+ BUFFER_EPOCH += 1
+
+
+def release(model):
+ """Drops captured steps and the persistent decode buffers (frees their memory)."""
+ global BUFFER_EPOCH
+ for state in _states(model):
+ state.clear()
+ state.statics.clear()
+ for module in model.modules():
+ if "_unsloth_graph_buffers" in module.__dict__:
+ del module.__dict__["_unsloth_graph_buffers"]
+ for name in _BUFFER_NAMES:
+ module.__dict__.pop(name, None)
+ BUFFER_EPOCH += 1
+
+
+def _version(t):
+ if t is None or not isinstance(t, torch.Tensor):
+ return None
+ if t.is_inference():
+ return (t.data_ptr(), None)
+ return (t.data_ptr(), t._version)
+
+
+def _fingerprint(model, layers, rope_len):
+ """Everything a captured step baked in; None when the model cannot be graphed."""
+ from ..kernels import utils as K
+
+ items = []
+ devices = set()
+ for p in model.parameters():
+ devices.add(p.device)
+ items.append((p.data_ptr(), p.dtype, p.shape))
+ if len(devices) != 1 or next(iter(devices)).type != "cuda":
+ return None
+ compute_dtype = None
+ seen_rope = set()
+ for layer in layers:
+ attn = layer.self_attn
+ for owner in (attn, getattr(layer, "mlp", None)):
+ for name in _PROJ_NAMES:
+ proj = getattr(owner, name, None)
+ if proj is None:
+ continue
+ if (
+ K._has_non_lora_adapter(proj)
+ or K._has_multiple_active_adapters(proj)
+ or K._has_active_dora_adapter(proj)
+ or K._has_active_lora_bias(proj)
+ or K._has_packed_qweight(proj)
+ ):
+ return None
+ W, W_quant, A, B, S, bias = K.get_lora_parameters_bias(proj)
+ absmax = getattr(W_quant, "absmax", None)
+ cast = getattr(A, "_fast_lora", None) if A is not None else None
+ items.append(
+ (
+ W.data_ptr(),
+ absmax.data_ptr() if isinstance(absmax, torch.Tensor) else id(W_quant),
+ _version(A),
+ _version(B),
+ S,
+ bias.data_ptr() if bias is not None else None,
+ None if cast is None else (cast[1].data_ptr(), cast[2].data_ptr()),
+ )
+ )
+ rope = getattr(attn, "rotary_emb", None)
+ if rope is not None and id(rope) not in seen_rope:
+ seen_rope.add(id(rope))
+ index = next(iter(devices)).index or 0
+ cos, sin = rope.get_cached(rope_len, index)
+ items.append((cos.data_ptr(), sin.data_ptr(), cos.shape[0]))
+ # Every adapter-aware module outside the projections too (embeddings, lm_head, modules_to_save):
+ # their enabled / active / merged state picks the branch a captured step baked in.
+ for module in model.modules():
+ if hasattr(module, "disable_adapters") or hasattr(module, "active_adapter"):
+ try:
+ active = (
+ module.active_adapters
+ if hasattr(module, "active_adapters")
+ else module.active_adapter
+ )
+ items.append(
+ (
+ id(module),
+ getattr(module, "disable_adapters", None),
+ tuple(active) if isinstance(active, (list, tuple)) else active,
+ getattr(module, "merged", None),
+ )
+ )
+ except Exception:
+ return None
+ items.append(K._OPTIMIZER_STEPS)
+ return tuple(items)
+
+
+def _eligible(model, layers):
+ config = getattr(model, "config", None)
+ if getattr(config, "model_type", None) not in _SUPPORTED_MODEL_TYPES:
+ return "model type"
+ if getattr(layers, "_unsloth_block_swap", None) is not None:
+ return "block swap"
+ return None
+
+
+def _static_inputs(state, bsz, device, dtype, hidden_size):
+ key = (bsz, device)
+ s = state.statics.get(key)
+ if s is None:
+ with _no_inference_tensors():
+ s = {
+ "ids": torch.zeros((bsz, 1), dtype = torch.long, device = device),
+ "pos": torch.zeros((bsz, 1), dtype = torch.long, device = device),
+ "slot": torch.zeros((1,), dtype = torch.long, device = device),
+ }
+ state.statics[key] = s
+ like = s.get("like")
+ if like is None or like.dtype != dtype:
+ # dtype / device stand-in for the hidden states the mask builder reads.
+ s["like"] = torch.empty((bsz, 1, hidden_size), dtype = dtype, device = device)
+ return s
+
+
+class _Step:
+ """One captured decode step: graph segments, and the eager attention cores between them."""
+
+ __slots__ = ("segments", "cores", "logits", "statics")
+
+ def __init__(self, statics):
+ self.segments = []
+ self.cores = []
+ self.logits = None
+ self.statics = statics
+
+ def replay(self):
+ segments, cores = self.segments, self.cores
+ for i in range(len(cores)):
+ segments[i].replay()
+ cores[i]()
+ segments[-1].replay()
+
+
+def _run_step(model, fast_forward_inference, logits_fn, s, st, past_key_values):
+ _TLS.statics = st
+ try:
+ outputs = fast_forward_inference(
+ model,
+ s["ids"],
+ past_key_values,
+ position_ids = s["pos"],
+ attention_mask = None,
+ )
+ return logits_fn(model, outputs[0])
+ finally:
+ _TLS.statics = None
+
+
+def decode_step(
+ model,
+ fast_forward_inference,
+ logits_fn,
+ mask_fn,
+ input_ids,
+ past_key_values,
+ position_ids,
+ attention_mask,
+):
+ """Replays (capturing on first use) the decode step; returns (logits, past_key_values) or None
+ for the eager path."""
+ if torch.is_grad_enabled() or not input_ids.is_cuda:
+ return None
+ if (
+ getattr(fast_forward_inference, "_unsloth_graph_attention", None)
+ not in _SUPPORTED_ATTENTION
+ ):
+ return None
+ if position_ids is None or os.environ.get("UNSLOTH_RETURN_HIDDEN_STATES", "0") == "1":
+ return None
+ if position_ids.numel() != input_ids.shape[0] and position_ids.shape[-1] == 1:
+ # Shared / broadcast positions take a different eager RoPE path.
+ return None
+ if torch.cuda.is_current_stream_capturing() or torch.compiler.is_compiling():
+ return None
+ state = _state(model)
+ if state.disabled is not None:
+ return None
+ layers = model.model.layers
+ attn0 = layers[0].self_attn
+ paged = attn0.__dict__.get("paged_attention")
+ bsz = input_ids.shape[0]
+ if (
+ paged is None
+ or "_unsloth_graph_buffers" not in attn0.__dict__
+ or paged.shape[2] != bsz
+ or input_ids.shape[1] != 1
+ ):
+ # First decode step after a prefill (allocates / fills the cache), or a cache this path
+ # did not size.
+ state.eager_steps += 1
+ return None
+ seq_len = past_key_values[0][0].shape[-2]
+ kv_len = seq_len + 1
+ cap = paged.shape[0]
+ if kv_len >= cap:
+ # The eager step grows the cache (new addresses: the next step re-captures).
+ state.eager_steps += 1
+ return None
+
+ from ..kernels import utils as K
+
+ # RoPE rows the captured steps may read: the table is sized (and its address fingerprinted)
+ # for this many, so a position at or past it runs eagerly (the eager step grows the table).
+ rope_len = max(state.rope_len or 0, cap + 1)
+ bound = getattr(position_ids, "_unsloth_position_bound", None)
+ if bound is None:
+ # Caller-supplied positions can exceed the KV length: one host read, as the eager step does.
+ bound = int(position_ids.max()) + 1
+ if bound > rope_len:
+ state.eager_steps += 1
+ state.generation_epoch = -1
+ return None
+ if (
+ state.generation_epoch != GENERATION_EPOCH
+ or state.optimizer_steps != K._OPTIMIZER_STEPS
+ or state.buffer_epoch != BUFFER_EPOCH
+ or state.rope_len != rope_len
+ ):
+ reason = _eligible(model, layers)
+ if reason is not None:
+ state.disabled = reason
+ return None
+ fingerprint = _fingerprint(model, layers, rope_len)
+ if fingerprint is None:
+ state.disabled = "parameters on several / non-CUDA devices or a non-LoRA adapter"
+ return None
+ if (
+ fingerprint != state.fingerprint
+ or state.buffer_epoch != BUFFER_EPOCH
+ or state.rope_len != rope_len
+ ):
+ state.clear()
+ state.fingerprint = fingerprint
+ state.generation_epoch = GENERATION_EPOCH
+ state.optimizer_steps = K._OPTIMIZER_STEPS
+ state.buffer_epoch = BUFFER_EPOCH
+ state.rope_len = rope_len
+
+ device = input_ids.device
+ config = model.config
+ s = _static_inputs(state, bsz, device, paged.dtype, config.hidden_size)
+ s["ids"].copy_(input_ids)
+ last = position_ids[:, -1:] if position_ids.dim() == 2 else position_ids
+ if last.numel() != bsz:
+ return None
+ s["pos"].copy_(last.view(bsz, 1))
+ s["slot"].fill_(seq_len)
+
+ autocast = _autocast_state()
+ key = (bsz, device, autocast, torch.is_inference_mode_enabled())
+ step = state.graphs.get(key)
+ if step is None:
+ if len(state.graphs) >= MAX_GRAPHS:
+ state.graphs.clear()
+ state.pool = None
+ try:
+ step = _capture(
+ state, model, fast_forward_inference, logits_fn, mask_fn, s, rope_len,
+ past_key_values, autocast, attention_mask, seq_len,
+ ) # fmt: skip
+ # The warmup may have built LoRA cast caches or RoPE rows the graph now reads.
+ state.fingerprint = _fingerprint(model, layers, rope_len)
+ except Exception as error:
+ state.disabled = f"capture failed: {type(error).__name__}: {error}"
+ state.clear()
+ if os.environ.get("UNSLOTH_DECODE_CUDA_GRAPHS_STRICT", "0") == "1":
+ raise
+ return None
+ state.graphs[key] = step
+ state.captures += 1
+ st = step.statics
+ st.kv_len = kv_len
+ st.mask = mask_fn(model, attention_mask, bsz, 1, s["like"], seq_len)
+ try:
+ step.replay()
+ finally:
+ st.mask = None
+ state.replays += 1
+ next_cache = [
+ (
+ layer.self_attn.paged_attention_K[:kv_len].permute(1, 2, 0, 3),
+ layer.self_attn.paged_attention_V[:kv_len].permute(1, 2, 0, 3),
+ )
+ for layer in layers
+ ]
+ return step.logits.clone(), next_cache
+
+
+def _autocast_state():
+ try:
+ enabled = torch.is_autocast_enabled("cuda")
+ dtype = torch.get_autocast_dtype("cuda") if enabled else None
+ except TypeError:
+ enabled = torch.is_autocast_enabled()
+ dtype = torch.get_autocast_gpu_dtype() if enabled else None
+ return (enabled, dtype)
+
+
+def _capture(
+ state,
+ model,
+ fast_forward_inference,
+ logits_fn,
+ mask_fn,
+ s,
+ rope_len,
+ past_key_values,
+ autocast,
+ attention_mask,
+ seq_len,
+):
+ if state.pool is None:
+ state.pool = torch.cuda.graph_pool_handle()
+ if state.stream is None:
+ state.stream = torch.cuda.Stream()
+ stream = state.stream
+ st = _StepStatics(rope_len, s["slot"])
+ st.kv_len = seq_len + 1
+ st.mask = mask_fn(model, attention_mask, s["ids"].shape[0], 1, s["like"], seq_len)
+ enabled, dtype = autocast
+ # No autocast weight cache: a cast cached during capture would be freed into the graph's pool.
+ no_cache = lambda: torch.autocast("cuda", dtype = dtype, enabled = enabled, cache_enabled = False)
+ stream.wait_stream(torch.cuda.current_stream())
+ step = _Step(st)
+ with torch.cuda.stream(stream), no_cache():
+ # Warmup: Triton JIT, cuBLAS workspaces, lazily built RoPE rows / LoRA casts. It writes this
+ # step's K / V into its cache slot, the same values the replay writes again.
+ _run_step(model, fast_forward_inference, logits_fn, s, st, past_key_values)
+ stream.synchronize()
+
+ def split(fn):
+ # Close the segment, run the attention core once (its output address is what the next
+ # segment reads), open the next segment.
+ _end(step.segments[-1])
+ # Two calls with the first result alive: the same address means the core writes into a
+ # persistent buffer (bsz == 1 writes the query buffer); else (SDPA allocates) its
+ # result is copied into a fixed buffer.
+ out = fn()
+ fixed = out if fn().data_ptr() == out.data_ptr() else torch.empty_like(out)
+ step.cores.append(fn if fixed is out else (lambda: fixed.copy_(fn())))
+ graph = torch.cuda.CUDAGraph()
+ step.segments.append(graph)
+ _begin(graph, state.pool)
+ return fixed
+
+ graph = torch.cuda.CUDAGraph()
+ step.segments.append(graph)
+ st.attention = split
+ _begin(graph, state.pool)
+ try:
+ step.logits = _run_step(
+ model, fast_forward_inference, logits_fn, s, st, past_key_values
+ )
+ finally:
+ st.attention = _run_now
+ if torch.cuda.is_current_stream_capturing():
+ _end(step.segments[-1])
+ torch.cuda.current_stream().wait_stream(stream)
+ st.mask = None
+ return step
+
+
+def _begin(graph, pool):
+ # Outside inference mode: the first capture creates the CUDA RNG's graph state tensors, and
+ # inference tensors there make every later capture under no_grad fail (torch 2.10).
+ with torch.inference_mode(False):
+ graph.capture_begin(pool = pool, capture_error_mode = "thread_local")
+
+
+def _end(graph):
+ with torch.inference_mode(False):
+ graph.capture_end()
+
+
+def stats(model):
+ state = getattr(model, "__dict__", {}).get("_unsloth_decode_graphs")
+ if state is None:
+ return {}
+ return {
+ "captures": state.captures,
+ "replays": state.replays,
+ "graphs": len(state.graphs),
+ "eager_steps": state.eager_steps,
+ "disabled": state.disabled,
+ }
diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py
index 277e06e8daa..635f62eba02 100644
--- a/unsloth/models/llama.py
+++ b/unsloth/models/llama.py
@@ -157,6 +157,7 @@ from ..kernels.fused_add_rms_layernorm import (
fast_add_rms_layernorm_inference_ok,
)
from ..kernels import fast_lora as _fast_lora
+from . import _decode_graphs
from ..kernels.bnb_override import install_bnb_nf4_override as _install_bnb_nf4_override
from ..tokenizer_utils import *
from ..context_parallel import get_cp_manager
@@ -542,6 +543,9 @@ def _fast_prepare_inputs_for_generation(
position_ids = original_attention_mask.long().cumsum(-1) - 1
position_ids.masked_fill_(original_attention_mask == 0, 1)
position_ids = position_ids[:, -seq_length:]
+ if _decode_graphs.ENABLED:
+ # Positions from the mask stay below its length: graphed decode skips a host read.
+ position_ids._unsloth_position_bound = original_attention_mask.shape[-1]
kwargs["position_ids"] = position_ids
elif kwargs.get("cache_position", None) is not None:
cp = kwargs["cache_position"]
@@ -698,55 +702,37 @@ def LlamaAttention_fast_forward_inference(
hidden_size = self.config.hidden_size
attention_size = n_heads * head_dim
+ # Graph mode (a captured decode step): no host lengths, the cache slot is on the device.
+ static = _decode_graphs.static() if _decode_graphs.ENABLED else None
seq_len = K1.shape[-2]
kv_seq_len = seq_len + 1
device = hidden_states.device
if do_prefill:
- self.paged_attention = torch.empty(
- (KV_CACHE_INCREMENT + seq_len + 1, 2, bsz, n_kv_heads, head_dim),
- dtype = dtype,
- device = device,
- )
- self.paged_attention_K = self.paged_attention[:, 0]
- self.paged_attention_V = self.paged_attention[:, 1]
- self.paged_attention_K[:seq_len] = K1.permute(2, 0, 1, 3)
- self.paged_attention_V[:seq_len] = V1.permute(2, 0, 1, 3)
- self.temp_QA = torch.empty((2, bsz, 1, attention_size), dtype = dtype, device = device)
- self.temp_KV = torch.empty((2, bsz, 1, n_kv_heads * head_dim), dtype = dtype, device = device)
- self.RH_Q = torch.empty((bsz, n_heads, 1, head_dim), dtype = dtype, device = device)
-
- # Mistral Nemo 12b has weird dimensions.
- if attention_size != hidden_size:
- self.temp_O = torch.empty((bsz, 1, hidden_size), dtype = dtype, device = device)
- else:
- self.temp_O = self.temp_QA[1][:, :, :hidden_size]
-
- # Row stride kept a multiple of 8 so the grouped decode bmm stays on the fast cuBLAS path.
- self.attention = torch.empty(
- (bsz, n_heads, 1, KV_CACHE_INCREMENT + seq_len + (-seq_len) % 8),
- dtype = dtype,
- device = device,
- )
- self.scalar = 1.0 / math_sqrt(self.head_dim)
- self.half_head_dim = head_dim // 2
- elif kv_seq_len >= self.paged_attention.shape[0]:
- self.paged_attention.resize_(
- (
- self.paged_attention.shape[0] + KV_CACHE_INCREMENT,
- 2,
- bsz,
- n_kv_heads,
- head_dim,
- )
+ _decode_graphs.alloc_decode_buffers(
+ self, K1, V1, bsz, n_heads, n_kv_heads, head_dim, hidden_size, dtype, device,
+ KV_CACHE_INCREMENT,
+ ) # fmt: skip
+ elif static is None and kv_seq_len >= self.paged_attention.shape[0]:
+ _decode_graphs.resize_decode_buffers(
+ self, KV_CACHE_INCREMENT, bsz, n_heads, n_kv_heads, head_dim
)
- self.paged_attention_K = self.paged_attention[:, 0]
- self.paged_attention_V = self.paged_attention[:, 1]
- self.attention.resize_((bsz, n_heads, 1, self.attention.shape[-1] + KV_CACHE_INCREMENT))
Qn = fast_linear_forward(self.q_proj, Xn, out = self.temp_QA[0])
- Kn = fast_linear_forward(self.k_proj, Xn, out = self.temp_KV[0])
- Vn = fast_linear_forward(self.v_proj, Xn, out = self.temp_KV[1])
+ # K / V land straight in this step's cache slot (contiguous [bsz, n_kv_heads, head_dim]); K is rotated
+ # there in place, so the two per-layer cache copies below disappear. Graph mode writes scratch,
+ # copied into the device-side slot after RoPE.
+ if static is None:
+ K_slot = self.paged_attention_K[seq_len].view(bsz, 1, n_kv_heads * head_dim)
+ V_slot = self.paged_attention_V[seq_len].view(bsz, 1, n_kv_heads * head_dim)
+ else:
+ K_slot, V_slot = self.temp_KV[0], self.temp_KV[1]
+ Kn = fast_linear_forward(self.k_proj, Xn, out = K_slot)
+ if Kn.data_ptr() != K_slot.data_ptr():
+ Kn = K_slot.copy_(Kn)
+ Vn = fast_linear_forward(self.v_proj, Xn, out = V_slot)
+ if Vn.data_ptr() != V_slot.data_ptr():
+ Vn = V_slot.copy_(Vn)
Qn = Qn.view(bsz, 1, n_heads, head_dim).transpose(1, 2)
Kn = Kn.view(bsz, 1, n_kv_heads, head_dim).transpose(1, 2)
Vn = Vn.view(bsz, 1, n_kv_heads, head_dim).transpose(1, 2)
@@ -786,8 +772,21 @@ def LlamaAttention_fast_forward_inference(
Kn *= cos
Kn.addcmul_(RH_K, sin)
- self.paged_attention_K[seq_len] = Kn.permute(2, 0, 1, 3)
- self.paged_attention_V[seq_len] = Vn.permute(2, 0, 1, 3)
+ if static is not None:
+ # Captured step: the cache slot is on the device; the attention itself runs eagerly
+ # between two captured segments (static.attention), over the true length.
+ self.paged_attention_K.index_copy_(
+ 0, static.slot, K_slot.view(1, bsz, n_kv_heads, head_dim)
+ )
+ self.paged_attention_V.index_copy_(
+ 0, static.slot, V_slot.view(1, bsz, n_kv_heads, head_dim)
+ )
+ A = static.attention(
+ lambda: _decode_attention_core(self, Qn, static.kv_len, static.mask, bsz, True)[0]
+ )
+ A = A.transpose(1, 2).reshape(bsz, 1, attention_size)
+ return fast_linear_forward(self.o_proj, A, out = self.temp_O), (None, None)
+
Kn = self.paged_attention_K[:kv_seq_len].permute(1, 2, 0, 3)
Vn = self.paged_attention_V[:kv_seq_len].permute(1, 2, 0, 3)
@@ -855,6 +854,93 @@ def LlamaAttention_fast_forward_inference(
return A, (Kn, Vn)
+def _decode_attention_core(self, Qn, kv_seq_len, attention_mask, bsz, sdpa_gqa_with_batch_mask):
+ """softmax(Q K^T) V of one decode token over the first kv_seq_len cache rows; returns
+ (A [bsz, n_heads, 1, head_dim], K, V). The captured step calls it between its graph segments.
+ It is op for op the attention inlined in the Llama / Qwen3 decode functions (kept inline there
+ so the eager step pays no extra call), so both compute bit for bit the same.
+ sdpa_gqa_with_batch_mask=False (Qwen3) expands K / V instead of SDPA GQA for a masked batch."""
+ n_heads = self.__dict__.get("_unsloth_tp_heads", self.config.num_attention_heads)
+ n_groups = self.num_key_value_groups
+ n_kv_heads = self.__dict__.get("_unsloth_tp_kv_heads", self.config.num_key_value_heads)
+ head_dim = self.head_dim
+ Kn = self.paged_attention_K[:kv_seq_len].permute(1, 2, 0, 3)
+ Vn = self.paged_attention_V[:kv_seq_len].permute(1, 2, 0, 3)
+
+ # Handle sliding windows, from transformers models/mistral/modeling_mistral.py#L193
+ sliding_window = getattr(self.config, "sliding_window", None)
+ if sliding_window is not None and kv_seq_len > sliding_window:
+ start = kv_seq_len - sliding_window
+ Knn = Kn[:, :, start:, :]
+ Vnn = Vn[:, :, start:, :]
+ if attention_mask is not None:
+ attention_mask = attention_mask[..., start:]
+ else:
+ Knn, Vnn = Kn, Vn
+
+ if attention_mask is not None and attention_mask.dim() == 2:
+ attention_mask = attention_mask[:, None, None, :].to(torch.bool)
+ elif (
+ attention_mask is not None
+ and attention_mask.dim() == 4
+ and attention_mask.dtype != torch.bool
+ ):
+ # Decode is more stable with boolean keep masks than additive bf16 masks.
+ attention_mask = attention_mask.eq(0)
+
+ use_sdpa_gqa = SDPA_HAS_GQA
+ if (
+ use_sdpa_gqa
+ and not sdpa_gqa_with_batch_mask
+ and isinstance(attention_mask, torch.Tensor)
+ and attention_mask.dim() >= 3
+ and attention_mask.shape[0] > 1
+ ):
+ # Avoid SDPA GQA drift for batched masked decode.
+ use_sdpa_gqa = False
+
+ # Grouped query attention; bsz == 1 reads the KV heads in place below.
+ _, _, cached_len, _ = Knn.shape
+ if bsz != 1 and (not use_sdpa_gqa) and n_groups != 1:
+ Knn = Knn[:, :, None, :, :].expand(bsz, n_kv_heads, n_groups, cached_len, head_dim)
+ Vnn = Vnn[:, :, None, :, :].expand(bsz, n_kv_heads, n_groups, cached_len, head_dim)
+ Knn = Knn.reshape(bsz, n_heads, cached_len, head_dim)
+ Vnn = Vnn.reshape(bsz, n_heads, cached_len, head_dim)
+
+ # When qlen == vlen and attn_mask is None, causal attention is the right choice.
+ Q_len = Qn.shape[-2]
+ K_len = Knn.shape[-2]
+ if attention_mask is None and Q_len == K_len:
+ is_causal = True
+ else:
+ is_causal = False
+ if bsz == 1:
+ Qn *= self.scalar
+ # (Q * scalar) @ K beats (Q @ K) * scalar for stopping overflows; see ggerganov/llama.cpp#7805.
+ A = _gqa_decode_scores(Qn, Knn, self.attention[:, :, :, :cached_len])
+ if attention_mask is not None:
+ A.masked_fill_(~attention_mask, float("-inf"))
+ A[:] = torch_nn_functional_softmax(A, dim = -1, dtype = torch.float32)
+ A = _gqa_decode_values(A, Vnn, Qn)
+ else:
+ if use_sdpa_gqa:
+ A = scaled_dot_product_attention(
+ Qn,
+ Knn,
+ Vnn,
+ attn_mask = attention_mask,
+ is_causal = is_causal,
+ enable_gqa = True,
+ )
+ else:
+ A = scaled_dot_product_attention(
+ Qn, Knn, Vnn, attn_mask = attention_mask, is_causal = is_causal
+ )
+ return A, Kn, Vn
+
+
+_decode_graphs.register_attention(LlamaAttention_fast_forward_inference)
+
torch_nn_functional_silu = torch.nn.functional.silu
@@ -1631,6 +1717,23 @@ def LlamaModel_fast_forward(
# Ported from transformers models/llama/modeling_llama.py#L825
+def _decode_step_mask(self, attention_mask, bsz, q_len, X, seq_len):
+ """The decode step's attention mask: None, or a bool keep mask for every layer."""
+ if attention_mask is None:
+ return None
+ attention_mask = _prepare_4d_causal_attention_mask_for_sdpa(
+ attention_mask,
+ (bsz, q_len),
+ X,
+ seq_len,
+ sliding_window = getattr(self.config, "sliding_window", None),
+ )
+ # Pre-convert to bool once for all layers, avoiding a per-layer .eq(0).
+ if attention_mask is not None and attention_mask.dtype != torch.bool:
+ attention_mask = attention_mask.eq(0)
+ return attention_mask
+
+
def _LlamaModel_fast_forward_inference(
attention_fast_forward_inference = LlamaAttention_fast_forward_inference,
mlp_fast_forward_inference = fast_swiglu_inference,
@@ -1669,7 +1772,11 @@ def _LlamaModel_fast_forward_inference(
seq_len = past_key_values[0][0].shape[-2]
kv_seq_len = seq_len + 1
- if attention_mask is not None:
+ static = _decode_graphs.static() if _decode_graphs.ENABLED else None
+ if static is not None:
+ # Captured step: the runner builds this mask per step (_decode_step_mask), outside the graph.
+ attention_mask = None
+ elif attention_mask is not None:
attention_mask = _prepare_4d_causal_attention_mask_for_sdpa(
attention_mask,
(bsz, q_len),
@@ -1684,7 +1791,10 @@ def _LlamaModel_fast_forward_inference(
attention_mask = None
# Compute rotary_seq_len once to avoid a per-layer GPU-CPU sync from .item().
- rotary_seq_len = max(kv_seq_len, int(position_ids.max().item()) + 1)
+ if static is not None:
+ rotary_seq_len = static.rope_len
+ else:
+ rotary_seq_len = max(kv_seq_len, int(position_ids.max().item()) + 1)
next_decoder_cache = []
block_swap = getattr(self.model.layers, "_unsloth_block_swap", None)
@@ -1751,6 +1861,11 @@ def _LlamaModel_fast_forward_inference(
attentions = [],
)
+ if mlp_fast_forward_inference is fast_swiglu_inference:
+ # The attention function decides whether a captured (graph-mode) step is possible.
+ LlamaModel_fast_forward_inference_custom._unsloth_graph_attention = (
+ attention_fast_forward_inference
+ )
return LlamaModel_fast_forward_inference_custom
@@ -1851,6 +1966,19 @@ def _run_backbone(self, **kwargs):
model._unsloth_head_reads_final_norm = False
+def _decode_logits(self, hidden_states):
+ """The decode branch of CausalLM_fast_forward's head (q_len == 1, no labels), for captured steps."""
+ lm_head = self.lm_head.weight
+ dtype = lm_head.dtype
+ if hidden_states.shape[0] == 1:
+ logits = torch.mv(lm_head, hidden_states.ravel().to(dtype)).unsqueeze(0).unsqueeze(0)
+ else:
+ logits = self.lm_head(hidden_states.to(dtype))
+ logits = logits.to(_get_dtype(dtype_from_config(self.config)))
+ logit_softcapping, logit_scaling = resolve_logit_scaling(self.config)
+ return apply_logit_transforms(logits, logit_softcapping, logit_scaling)
+
+
def CausalLM_fast_forward(fast_forward_inference):
def _CausalLM_fast_forward(
self,
@@ -1876,6 +2004,19 @@ def CausalLM_fast_forward(fast_forward_inference):
if past_key_values is not None and len(past_key_values) == 0:
past_key_values = None
if past_key_values is not None and input_ids is not None and input_ids.shape[1] == 1:
+ if _decode_graphs.ENABLED and labels is None and shift_labels is None:
+ graphed = _decode_graphs.decode_step(
+ self, fast_forward_inference, _decode_logits, _decode_step_mask, input_ids,
+ past_key_values, position_ids, attention_mask,
+ ) # fmt: skip
+ if graphed is not None:
+ # The same output the eager decode branch below returns.
+ if not return_dict:
+ return (graphed[0], graphed[1], [], [])
+ return CausalLMOutputWithPast(
+ loss = None, logits = graphed[0], past_key_values = graphed[1],
+ hidden_states = [], attentions = [],
+ ) # fmt: skip
outputs = fast_forward_inference(
self,
input_ids,
@@ -2790,6 +2931,25 @@ class LongRopeRotaryEmbedding(_PerDeviceRopeCache, torch.nn.Module):
]
+def _generation_max_len(self, args, kwargs):
+ """prompt + max_new_tokens of a generate() call (0 if unknown), to size the decode buffers once."""
+ try:
+ ids = kwargs.get("input_ids", kwargs.get("inputs", args[0] if args else None))
+ if ids is None:
+ ids = kwargs.get("inputs_embeds", None)
+ prompt = ids.shape[1] if hasattr(ids, "shape") and len(ids.shape) >= 2 else 0
+ config = kwargs.get("generation_config", None) or getattr(self, "generation_config", None)
+ new = kwargs.get("max_new_tokens", None)
+ if new is None and kwargs.get("max_length", None) is not None:
+ return int(kwargs["max_length"])
+ if new is None:
+ # A config's default max_length (often the context size) is not a request.
+ new = getattr(config, "max_new_tokens", None)
+ return prompt + int(new) if new is not None else 0
+ except Exception:
+ return 0
+
+
def unsloth_fast_generate(self, *args, **kwargs):
restore_training_mode = self.training
# Snapshot the real GC mode (e.g. "unsloth") before for_inference clears it, so the restore
@@ -2871,12 +3031,22 @@ def unsloth_fast_generate(self, *args, **kwargs):
kwargs["pad_token_id"] = kwargs.pop("pad_token_id", model_eos_token_id)
+ if _decode_graphs.ENABLED:
+ _decode_graphs.start_generation(_generation_max_len(self, args, kwargs))
# Mixed precision autocast.
- with (
- _get_inference_mode_context_manager(self),
- torch.autocast(device_type = DEVICE_TYPE_TORCH, dtype = dtype),
- ):
- output = self._old_generate(*args, **kwargs)
+ try:
+ with (
+ _get_inference_mode_context_manager(self),
+ torch.autocast(device_type = DEVICE_TYPE_TORCH, dtype = dtype),
+ ):
+ output = self._old_generate(*args, **kwargs)
+ finally:
+ if _decode_graphs.ENABLED:
+ _decode_graphs.end_generation()
+ if _decode_graphs.ENABLED and getattr(output, "past_key_values", None) is not None:
+ # The returned cache views the persistent decode buffers: hand them over, so a later
+ # generate() allocates its own instead of overwriting a cache the caller kept.
+ _decode_graphs.release(self)
if restore_training_mode:
FastLlamaModel.for_training(
diff --git a/unsloth/models/qwen3.py b/unsloth/models/qwen3.py
index 3392c438f63..db63772289b 100644
--- a/unsloth/models/qwen3.py
+++ b/unsloth/models/qwen3.py
@@ -17,6 +17,8 @@ from unsloth_zoo.utils import Version, _get_dtype
from ..utils.packing import get_packed_info_from_kwargs
from ..kernels import qk_norm_rope as _qk_norm_rope
from ..kernels.qk_norm_rope import fast_qk_norm_rope, decode_qk_norm_rope
+from . import _decode_graphs
+from .llama import _decode_attention_core
from ..utils.attention_dispatch import (
AttentionConfig,
AttentionContext,
@@ -203,50 +205,21 @@ def Qwen3Attention_fast_forward_inference(
hidden_size = self.config.hidden_size
attention_size = n_heads * head_dim
+ # Graph mode (a captured decode step): no host lengths, the cache slot is on the device.
+ static = _decode_graphs.static() if _decode_graphs.ENABLED else None
seq_len = K1.shape[-2]
kv_seq_len = seq_len + 1
device = hidden_states.device
if do_prefill:
- self.paged_attention = torch.empty(
- (KV_CACHE_INCREMENT + seq_len + 1, 2, bsz, n_kv_heads, head_dim),
- dtype = dtype,
- device = device,
- )
- self.paged_attention_K = self.paged_attention[:, 0]
- self.paged_attention_V = self.paged_attention[:, 1]
- self.paged_attention_K[:seq_len] = K1.permute(2, 0, 1, 3)
- self.paged_attention_V[:seq_len] = V1.permute(2, 0, 1, 3)
- self.temp_QA = torch.empty((2, bsz, 1, attention_size), dtype = dtype, device = device)
- self.temp_KV = torch.empty((2, bsz, 1, n_kv_heads * head_dim), dtype = dtype, device = device)
- self.RH_Q = torch.empty((bsz, n_heads, 1, head_dim), dtype = dtype, device = device)
-
- # Mistral Nemo 12b has weird dimensions.
- if attention_size != hidden_size:
- self.temp_O = torch.empty((bsz, 1, hidden_size), dtype = dtype, device = device)
- else:
- self.temp_O = self.temp_QA[1][:, :, :hidden_size]
-
- self.attention = torch.empty(
- (bsz, n_heads, 1, KV_CACHE_INCREMENT + seq_len + (-seq_len) % 8),
- dtype = dtype,
- device = device,
+ _decode_graphs.alloc_decode_buffers(
+ self, K1, V1, bsz, n_heads, n_kv_heads, head_dim, hidden_size, dtype, device,
+ KV_CACHE_INCREMENT,
+ ) # fmt: skip
+ elif static is None and kv_seq_len >= self.paged_attention.shape[0]:
+ _decode_graphs.resize_decode_buffers(
+ self, KV_CACHE_INCREMENT, bsz, n_heads, n_kv_heads, head_dim
)
- self.scalar = 1.0 / math_sqrt(self.head_dim)
- self.half_head_dim = head_dim // 2
- elif kv_seq_len >= self.paged_attention.shape[0]:
- self.paged_attention.resize_(
- (
- self.paged_attention.shape[0] + KV_CACHE_INCREMENT,
- 2,
- bsz,
- n_kv_heads,
- head_dim,
- )
- )
- self.paged_attention_K = self.paged_attention[:, 0]
- self.paged_attention_V = self.paged_attention[:, 1]
- self.attention.resize_((bsz, n_heads, 1, self.attention.shape[-1] + KV_CACHE_INCREMENT))
Qn = fast_linear_forward(self.q_proj, Xn, out = self.temp_QA[0])
Kn = fast_linear_forward(self.k_proj, Xn, out = self.temp_KV[0])
@@ -295,6 +268,16 @@ def Qwen3Attention_fast_forward_inference(
Kn *= cos
Kn.addcmul_(RH_K, sin)
+ if static is not None:
+ # Captured step: the cache slot is on the device; the attention itself runs eagerly
+ # between two captured segments (static.attention), over the true length.
+ self.paged_attention_K.index_copy_(0, static.slot, Kn.permute(2, 0, 1, 3))
+ self.paged_attention_V.index_copy_(0, static.slot, Vn.permute(2, 0, 1, 3))
+ A = static.attention(
+ lambda: _decode_attention_core(self, Qn, static.kv_len, static.mask, bsz, False)[0]
+ )
+ A = A.transpose(1, 2).reshape(bsz, 1, attention_size)
+ return fast_linear_forward(self.o_proj, A, out = self.temp_O), (None, None)
self.paged_attention_K[seq_len] = Kn.permute(2, 0, 1, 3)
self.paged_attention_V[seq_len] = Vn.permute(2, 0, 1, 3)
Kn = self.paged_attention_K[:kv_seq_len].permute(1, 2, 0, 3)
@@ -373,6 +356,9 @@ def Qwen3Attention_fast_forward_inference(
return A, (Kn, Vn)
+_decode_graphs.register_attention(Qwen3Attention_fast_forward_inference)
+
+
class FastQwen3Model(FastLlamaModel):
@staticmethod
def pre_patch():