Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Decode graphs: grow the KV cache instead of reserving max_new_tokens, and capture on the model's GPU (#15237)
Daniel Handc55427e2813
diff --git a/tests/test_decode_graphs_kv_growth_device.py b/tests/test_decode_graphs_kv_growth_device.pynew file mode 100644index 00000000000..f9de0b32520--- /dev/null+++ b/tests/test_decode_graphs_kv_growth_device.py@@ -0,0 +1,139 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""Graph decode KV growth and non-current-GPU capture, each bitwise equal to eager."""++import pytest++import unsloth # noqa: F401 (must precede transformers)+from unsloth import FastLanguageModel+import torch+from transformers import StoppingCriteria, StoppingCriteriaList++import unsloth.models._decode_graphs as G++cuda = pytest.mark.skipif(not torch.cuda.is_available(), reason = "CUDA only")+MODEL = "trl-internal-testing/tiny-Qwen3ForCausalLM"+PROMPTS = ["Once upon a time there was", "The capital of France is Paris and"] * 4+++class _StopAfter(StoppingCriteria):+ def __init__(self, n):+ self.n, self.k = n, 0++ def __call__(self, input_ids, scores, **kwargs):+ self.k += 1+ return torch.full(+ (input_ids.shape[0],), self.k >= self.n, dtype = torch.bool, device = input_ids.device+ )+++def _load(device = 0):+ model, tok = FastLanguageModel.from_pretrained(+ MODEL, max_seq_length = 8192, load_in_4bit = False, device_map = {"": device}+ )+ FastLanguageModel.for_inference(model)+ tok.padding_side = "left"+ if tok.pad_token is None:+ tok.pad_token = tok.eos_token+ return model, tok+++def _run(+ model,+ tok,+ enc,+ graphs,+ monkeypatch,+ max_new,+ stop = None,+ min_new = 0,+):+ monkeypatch.setattr(G, "ENABLED", None)+ monkeypatch.delenv("UNSLOTH_DECODE_CUDA_GRAPHS", raising = False)+ criteria = StoppingCriteriaList([_StopAfter(stop)]) if stop else None++ def gen():+ return model.generate(+ **enc, max_new_tokens = max_new, min_new_tokens = min_new, do_sample = False,+ pad_token_id = tok.pad_token_id, output_logits = True, return_dict_in_generate = True,+ stopping_criteria = criteria,+ ) # fmt: skip++ seen = [0]+ for name in ("alloc_decode_buffers", "resize_decode_buffers"):+ fn = getattr(G, name)++ def spy(+ attn,+ *args,+ _fn = fn,+ **kwargs,+ ):+ out = _fn(attn, *args, **kwargs)+ seen[0] = max(seen[0], attn.paged_attention.shape[0])+ return out++ monkeypatch.setattr(G, name, spy)+ before = dict(G.stats(model))+ if graphs:+ with G.rl_generation():+ out = gen()+ else:+ monkeypatch.setattr(G, "ENABLED", False)+ out = gen()+ rows = seen[0] or None+ after = G.stats(model)+ delta = {k: after.get(k, 0) - before.get(k, 0) for k in ("captures", "replays")}+ G.release(model)+ return out.sequences.cpu(), torch.stack(out.logits).cpu(), rows, delta+++@cuda+def test_early_stop_does_not_hold_max_new_tokens(monkeypatch):+ model, tok = _load()+ enc = tok(PROMPTS, return_tensors = "pt", padding = True).to("cuda")+ ids_e, logits_e, _, _ = _run(model, tok, enc, False, monkeypatch, 32768, stop = 20)+ ids_g, logits_g, rows, delta = _run(model, tok, enc, True, monkeypatch, 32768, stop = 20)+ assert torch.equal(ids_e, ids_g) and torch.equal(logits_e, logits_g)+ assert delta["replays"] > 0, delta+ # prompt + one 512 increment, rounded to the 256-row bucket; not 32768 + prompt+ assert rows is not None and rows <= 1024, rows+++@cuda+def test_long_rollout_grows_and_stays_bitwise(monkeypatch):+ model, tok = _load()+ enc = tok(PROMPTS, return_tensors = "pt", padding = True).to("cuda")+ new = 1500+ ids_e, logits_e, _, _ = _run(model, tok, enc, False, monkeypatch, new, min_new = new)+ ids_g, logits_g, _, delta = _run(model, tok, enc, True, monkeypatch, new, min_new = new)+ assert torch.equal(ids_e, ids_g) and torch.equal(logits_e, logits_g)+ # 768 -> 1536 -> hint: a few re-captures, most steps replayed+ assert 2 <= delta["captures"] <= 4, delta+ assert delta["replays"] >= new - 10, delta+++@cuda+def test_prealloc_kill_switch(monkeypatch):+ model, tok = _load()+ enc = tok(PROMPTS, return_tensors = "pt", padding = True).to("cuda")+ monkeypatch.setattr(G, "PREALLOC", True)+ _, _, rows, _ = _run(model, tok, enc, True, monkeypatch, 4096, stop = 5)+ assert rows >= 4096, rows+++ not torch.cuda.is_available() or torch.cuda.device_count() < 2, reason = "needs 2 CUDA GPUs"+)+def test_model_on_a_non_current_gpu_replays(monkeypatch):+ torch.cuda.set_device(0)+ model, tok = _load(device = 1)+ enc = tok(PROMPTS, return_tensors = "pt", padding = True).to("cuda:1")+ assert torch.cuda.current_device() == 0+ ids_e, logits_e, _, _ = _run(model, tok, enc, False, monkeypatch, 48, min_new = 48)+ ids_g, logits_g, _, delta = _run(model, tok, enc, True, monkeypatch, 48, min_new = 48)+ assert torch.cuda.current_device() == 0+ assert G._state(model).disabled is None, G._state(model).disabled+ assert delta["captures"] >= 1 and delta["replays"] > 0, delta+ assert torch.equal(ids_e, ids_g) and torch.equal(logits_e, logits_g)diff --git a/unsloth/models/_decode_graphs.py b/unsloth/models/_decode_graphs.pyindex ceedcaf26bc..3dc8decdd08 100644--- a/unsloth/models/_decode_graphs.py+++ b/unsloth/models/_decode_graphs.py@@ -66,6 +66,8 @@ BUFFER_EPOCH = 0GENERATION_EPOCH = 0# prompt + max_new_tokens of the running generate() call, 0 when unknown.MAX_LEN_HINT = 0+# 1: size the graph KV cache for prompt + max_new_tokens up front (early stops still hold it all).+PREALLOC = os.environ.get("UNSLOTH_DECODE_GRAPH_PREALLOC", "0") == "1"_SUPPORTED_ATTENTION = set()# Tested families only (Qwen2 shares the Llama attention but is untested here)._SUPPORTED_MODEL_TYPES = {"llama", "qwen3"}@@ -197,6 +199,9 @@ def alloc_decode_buffers()else:rows = _round_up(max(increment + seq_len + 1, MAX_LEN_HINT + 1), BUCKET)+ if not PREALLOC:+ # Start at the eager size; resize_decode_buffers grows it.+ rows = min(rows, _round_up(increment + seq_len + 1, BUCKET))store = attn.__dict__.get("_unsloth_graph_buffers")if store is None:store = attn._unsloth_graph_buffers = {}@@ -249,6 +254,12 @@ def resize_decode_buffers(attn, increment, bsz, n_heads, n_kv_heads, head_dim):"""The eager growth of the KV cache by `increment` rows (new rows zeroed with graphs on)."""global BUFFER_EPOCHrows = attn.paged_attention.shape[0]+ if "_unsloth_graph_buffers" in attn.__dict__:+ # Each growth re-captures: double, capped at the hint, so O(log n) re-captures.+ grown = max(rows + increment, 2 * rows)+ if MAX_LEN_HINT + 1 > rows:+ grown = min(grown, MAX_LEN_HINT + 1)+ increment = _round_up(grown, BUCKET) - rowsattn.paged_attention.resize_((rows + increment, 2, bsz, n_kv_heads, head_dim))attn.paged_attention_K = attn.paged_attention[:, 0]attn.paged_attention_V = attn.paged_attention[:, 1]@@ -525,6 +536,14 @@ def decode_step(return Noneif torch.cuda.is_current_stream_capturing() or torch.compiler.is_compiling():return None+ index = input_ids.device.index+ if index is not None and index != torch.cuda.current_device():+ # Capture on the model's GPU, else the capture fails and decode stays eager.+ with torch.cuda.device(index):+ return decode_step(+ model, fast_forward_inference, logits_fn, mask_fn, input_ids, past_key_values,+ position_ids, attention_mask,+ ) # fmt: skipstate = _state(model)if state.disabled is not None:return None@@ -662,6 +681,9 @@ def _capture(attention_mask,seq_len,):+ if state.stream is not None and state.stream.device.index != torch.cuda.current_device():+ state.stream = None+ state.pool = Noneif state.pool is None:state.pool = torch.cuda.graph_pool_handle()if state.stream is None:diff --git a/unsloth/models/llama.py b/unsloth/models/llama.pyindex 100133cdee2..08caf5ea346 100644--- a/unsloth/models/llama.py+++ b/unsloth/models/llama.py@@ -1757,17 +1757,13 @@ def _LlamaModel_fast_forward_inference(X = X.to(_get_dtype(dtype_from_config(self.config)))bsz, q_len, hd = X.shapeassert q_len == 1- residual = torch.empty(- (bsz, q_len, hd), dtype = torch.float32, device = f"{DEVICE_TYPE_TORCH}:0"- )- _XX = torch.empty((2, bsz, q_len, hd), dtype = torch.float32, device = f"{DEVICE_TYPE_TORCH}:0")+ # Not device 0: a cross-device copy per step cannot be graph captured.+ scratch_device = X.device+ residual = torch.empty((bsz, q_len, hd), dtype = torch.float32, device = scratch_device)+ _XX = torch.empty((2, bsz, q_len, hd), dtype = torch.float32, device = scratch_device)XX, XX2 = _XX[0], _XX[1]- variance = torch.empty(- (bsz, q_len, 1), dtype = torch.float32, device = f"{DEVICE_TYPE_TORCH}:0"- )- temp_mlp = torch.empty(- (2, bsz, 1, mlp_size), dtype = X.dtype, device = f"{DEVICE_TYPE_TORCH}:0"- )+ variance = torch.empty((bsz, q_len, 1), dtype = torch.float32, device = scratch_device)+ temp_mlp = torch.empty((2, bsz, 1, mlp_size), dtype = X.dtype, device = scratch_device)temp_gates = _LazyDeviceScratch(DEVICE_COUNT, temp_mlp[0].to)temp_ups = _LazyDeviceScratch(DEVICE_COUNT, temp_mlp[1].to)