Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Decode graphs: grow the KV cache instead of reserving max_new_tokens, and capture on the model's GPU (#15237)

Daniel Handc55427e2813
diff --git a/tests/test_decode_graphs_kv_growth_device.py b/tests/test_decode_graphs_kv_growth_device.py
new file mode 100644
index 00000000000..f9de0b32520
--- /dev/null
+++ b/tests/test_decode_graphs_kv_growth_device.py
@@ -0,0 +1,139 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""Graph decode KV growth and non-current-GPU capture, each bitwise equal to eager."""
+
+import pytest
+
+import unsloth # noqa: F401 (must precede transformers)
+from unsloth import FastLanguageModel
+import torch
+from transformers import StoppingCriteria, StoppingCriteriaList
+
+import unsloth.models._decode_graphs as G
+
+cuda = pytest.mark.skipif(not torch.cuda.is_available(), reason = "CUDA only")
+MODEL = "trl-internal-testing/tiny-Qwen3ForCausalLM"
+PROMPTS = ["Once upon a time there was", "The capital of France is Paris and"] * 4
+
+
+class _StopAfter(StoppingCriteria):
+ def __init__(self, n):
+ self.n, self.k = n, 0
+
+ def __call__(self, input_ids, scores, **kwargs):
+ self.k += 1
+ return torch.full(
+ (input_ids.shape[0],), self.k >= self.n, dtype = torch.bool, device = input_ids.device
+ )
+
+
+def _load(device = 0):
+ model, tok = FastLanguageModel.from_pretrained(
+ MODEL, max_seq_length = 8192, load_in_4bit = False, device_map = {"": device}
+ )
+ FastLanguageModel.for_inference(model)
+ tok.padding_side = "left"
+ if tok.pad_token is None:
+ tok.pad_token = tok.eos_token
+ return model, tok
+
+
+def _run(
+ model,
+ tok,
+ enc,
+ graphs,
+ monkeypatch,
+ max_new,
+ stop = None,
+ min_new = 0,
+):
+ monkeypatch.setattr(G, "ENABLED", None)
+ monkeypatch.delenv("UNSLOTH_DECODE_CUDA_GRAPHS", raising = False)
+ criteria = StoppingCriteriaList([_StopAfter(stop)]) if stop else None
+
+ def gen():
+ return model.generate(
+ **enc, max_new_tokens = max_new, min_new_tokens = min_new, do_sample = False,
+ pad_token_id = tok.pad_token_id, output_logits = True, return_dict_in_generate = True,
+ stopping_criteria = criteria,
+ ) # fmt: skip
+
+ seen = [0]
+ for name in ("alloc_decode_buffers", "resize_decode_buffers"):
+ fn = getattr(G, name)
+
+ def spy(
+ attn,
+ *args,
+ _fn = fn,
+ **kwargs,
+ ):
+ out = _fn(attn, *args, **kwargs)
+ seen[0] = max(seen[0], attn.paged_attention.shape[0])
+ return out
+
+ monkeypatch.setattr(G, name, spy)
+ before = dict(G.stats(model))
+ if graphs:
+ with G.rl_generation():
+ out = gen()
+ else:
+ monkeypatch.setattr(G, "ENABLED", False)
+ out = gen()
+ rows = seen[0] or None
+ after = G.stats(model)
+ delta = {k: after.get(k, 0) - before.get(k, 0) for k in ("captures", "replays")}
+ G.release(model)
+ return out.sequences.cpu(), torch.stack(out.logits).cpu(), rows, delta
+
+
+@cuda
+def test_early_stop_does_not_hold_max_new_tokens(monkeypatch):
+ model, tok = _load()
+ enc = tok(PROMPTS, return_tensors = "pt", padding = True).to("cuda")
+ ids_e, logits_e, _, _ = _run(model, tok, enc, False, monkeypatch, 32768, stop = 20)
+ ids_g, logits_g, rows, delta = _run(model, tok, enc, True, monkeypatch, 32768, stop = 20)
+ assert torch.equal(ids_e, ids_g) and torch.equal(logits_e, logits_g)
+ assert delta["replays"] > 0, delta
+ # prompt + one 512 increment, rounded to the 256-row bucket; not 32768 + prompt
+ assert rows is not None and rows <= 1024, rows
+
+
+@cuda
+def test_long_rollout_grows_and_stays_bitwise(monkeypatch):
+ model, tok = _load()
+ enc = tok(PROMPTS, return_tensors = "pt", padding = True).to("cuda")
+ new = 1500
+ ids_e, logits_e, _, _ = _run(model, tok, enc, False, monkeypatch, new, min_new = new)
+ ids_g, logits_g, _, delta = _run(model, tok, enc, True, monkeypatch, new, min_new = new)
+ assert torch.equal(ids_e, ids_g) and torch.equal(logits_e, logits_g)
+ # 768 -> 1536 -> hint: a few re-captures, most steps replayed
+ assert 2 <= delta["captures"] <= 4, delta
+ assert delta["replays"] >= new - 10, delta
+
+
+@cuda
+def test_prealloc_kill_switch(monkeypatch):
+ model, tok = _load()
+ enc = tok(PROMPTS, return_tensors = "pt", padding = True).to("cuda")
+ monkeypatch.setattr(G, "PREALLOC", True)
+ _, _, rows, _ = _run(model, tok, enc, True, monkeypatch, 4096, stop = 5)
+ assert rows >= 4096, rows
+
+
+ not torch.cuda.is_available() or torch.cuda.device_count() < 2, reason = "needs 2 CUDA GPUs"
+)
+def test_model_on_a_non_current_gpu_replays(monkeypatch):
+ torch.cuda.set_device(0)
+ model, tok = _load(device = 1)
+ enc = tok(PROMPTS, return_tensors = "pt", padding = True).to("cuda:1")
+ assert torch.cuda.current_device() == 0
+ ids_e, logits_e, _, _ = _run(model, tok, enc, False, monkeypatch, 48, min_new = 48)
+ ids_g, logits_g, _, delta = _run(model, tok, enc, True, monkeypatch, 48, min_new = 48)
+ assert torch.cuda.current_device() == 0
+ assert G._state(model).disabled is None, G._state(model).disabled
+ assert delta["captures"] >= 1 and delta["replays"] > 0, delta
+ assert torch.equal(ids_e, ids_g) and torch.equal(logits_e, logits_g)
diff --git a/unsloth/models/_decode_graphs.py b/unsloth/models/_decode_graphs.py
index ceedcaf26bc..3dc8decdd08 100644
--- a/unsloth/models/_decode_graphs.py
+++ b/unsloth/models/_decode_graphs.py
@@ -66,6 +66,8 @@ BUFFER_EPOCH = 0
GENERATION_EPOCH = 0
# prompt + max_new_tokens of the running generate() call, 0 when unknown.
MAX_LEN_HINT = 0
+# 1: size the graph KV cache for prompt + max_new_tokens up front (early stops still hold it all).
+PREALLOC = os.environ.get("UNSLOTH_DECODE_GRAPH_PREALLOC", "0") == "1"
_SUPPORTED_ATTENTION = set()
# Tested families only (Qwen2 shares the Llama attention but is untested here).
_SUPPORTED_MODEL_TYPES = {"llama", "qwen3"}
@@ -197,6 +199,9 @@ def alloc_decode_buffers(
)
else:
rows = _round_up(max(increment + seq_len + 1, MAX_LEN_HINT + 1), BUCKET)
+ if not PREALLOC:
+ # Start at the eager size; resize_decode_buffers grows it.
+ rows = min(rows, _round_up(increment + seq_len + 1, BUCKET))
store = attn.__dict__.get("_unsloth_graph_buffers")
if store is None:
store = attn._unsloth_graph_buffers = {}
@@ -249,6 +254,12 @@ def resize_decode_buffers(attn, increment, bsz, n_heads, n_kv_heads, head_dim):
"""The eager growth of the KV cache by `increment` rows (new rows zeroed with graphs on)."""
global BUFFER_EPOCH
rows = attn.paged_attention.shape[0]
+ if "_unsloth_graph_buffers" in attn.__dict__:
+ # Each growth re-captures: double, capped at the hint, so O(log n) re-captures.
+ grown = max(rows + increment, 2 * rows)
+ if MAX_LEN_HINT + 1 > rows:
+ grown = min(grown, MAX_LEN_HINT + 1)
+ increment = _round_up(grown, BUCKET) - rows
attn.paged_attention.resize_((rows + increment, 2, bsz, n_kv_heads, head_dim))
attn.paged_attention_K = attn.paged_attention[:, 0]
attn.paged_attention_V = attn.paged_attention[:, 1]
@@ -525,6 +536,14 @@ def decode_step(
return None
if torch.cuda.is_current_stream_capturing() or torch.compiler.is_compiling():
return None
+ index = input_ids.device.index
+ if index is not None and index != torch.cuda.current_device():
+ # Capture on the model's GPU, else the capture fails and decode stays eager.
+ with torch.cuda.device(index):
+ return decode_step(
+ model, fast_forward_inference, logits_fn, mask_fn, input_ids, past_key_values,
+ position_ids, attention_mask,
+ ) # fmt: skip
state = _state(model)
if state.disabled is not None:
return None
@@ -662,6 +681,9 @@ def _capture(
attention_mask,
seq_len,
):
+ if state.stream is not None and state.stream.device.index != torch.cuda.current_device():
+ state.stream = None
+ state.pool = None
if state.pool is None:
state.pool = torch.cuda.graph_pool_handle()
if state.stream is None:
diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py
index 100133cdee2..08caf5ea346 100644
--- a/unsloth/models/llama.py
+++ b/unsloth/models/llama.py
@@ -1757,17 +1757,13 @@ def _LlamaModel_fast_forward_inference(
X = X.to(_get_dtype(dtype_from_config(self.config)))
bsz, q_len, hd = X.shape
assert q_len == 1
- residual = torch.empty(
- (bsz, q_len, hd), dtype = torch.float32, device = f"{DEVICE_TYPE_TORCH}:0"
- )
- _XX = torch.empty((2, bsz, q_len, hd), dtype = torch.float32, device = f"{DEVICE_TYPE_TORCH}:0")
+ # Not device 0: a cross-device copy per step cannot be graph captured.
+ scratch_device = X.device
+ residual = torch.empty((bsz, q_len, hd), dtype = torch.float32, device = scratch_device)
+ _XX = torch.empty((2, bsz, q_len, hd), dtype = torch.float32, device = scratch_device)
XX, XX2 = _XX[0], _XX[1]
- variance = torch.empty(
- (bsz, q_len, 1), dtype = torch.float32, device = f"{DEVICE_TYPE_TORCH}:0"
- )
- temp_mlp = torch.empty(
- (2, bsz, 1, mlp_size), dtype = X.dtype, device = f"{DEVICE_TYPE_TORCH}:0"
- )
+ variance = torch.empty((bsz, q_len, 1), dtype = torch.float32, device = scratch_device)
+ temp_mlp = torch.empty((2, bsz, 1, mlp_size), dtype = X.dtype, device = scratch_device)
temp_gates = _LazyDeviceScratch(DEVICE_COUNT, temp_mlp[0].to)
temp_ups = _LazyDeviceScratch(DEVICE_COUNT, temp_mlp[1].to)