Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Decode with CUDA graphs by default in plain generate(), with memory release and an eager fallback (#16691)

Daniel Handb82c05511c1
diff --git a/tests/test_decode_cuda_graphs.py b/tests/test_decode_cuda_graphs.py
index b898bfc67ac..c6decab0e9f 100644
--- a/tests/test_decode_cuda_graphs.py
+++ b/tests/test_decode_cuda_graphs.py
@@ -203,6 +203,8 @@ def test_lora_change_between_generations_recaptures(name, stale_control, monkeyp
pinned = G._state(_causal(model)).fingerprint
monkeypatch.setattr(G, "_fingerprint", lambda *a: pinned)
params = [p for n, p in model.named_parameters() if "lora_" in n and p.requires_grad]
+ # for_training() frees the graphs; keep them here, so the fingerprint alone must catch the edits.
+ monkeypatch.setattr(G, "release_for_training", lambda model: None)
FastLanguageModel.for_training(model)
opt = torch.optim.SGD(params, lr = 1.0)
for p in params:
diff --git a/tests/test_decode_cuda_graphs_default.py b/tests/test_decode_cuda_graphs_default.py
index 40b58682180..4c834ddd8b0 100644
--- a/tests/test_decode_cuda_graphs_default.py
+++ b/tests/test_decode_cuda_graphs_default.py
@@ -1,10 +1,11 @@
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
-"""When CUDA-graph decode runs (decided per call): UNSLOTH_DECODE_CUDA_GRAPHS unset = off for plain
-model.generate(), on inside an RL trainer's generation (rl_generation(), entered by rl.py's
-unwrap_model_for_generation); =1 on everywhere, =0 off everywhere. Returning to training releases
-the graphs and every decode buffer."""
+"""When CUDA-graph decode runs (decided per call): UNSLOTH_DECODE_CUDA_GRAPHS unset = on inside
+Unsloth's generate() (generate_scope()) and an RL trainer's generation (rl_generation(), entered by
+rl.py's unwrap_model_for_generation), off for a hand-written decode loop outside both; =1 on
+everywhere, =0 off everywhere. Returning to training (or for_training) releases the graphs and every
+decode buffer."""
import pytest
@@ -129,39 +130,61 @@ def _gen(
@rl_default
-def test_plain_generate_off_rl_generate_on_and_released(runtime):
+def test_plain_generate_on_by_default_bitwise_and_released(runtime):
+ if not G._STUDIO_DEFAULT_OK:
+ pytest.skip("the plain generate() default stays eager on native Windows")
model, tok = _load()
FastLanguageModel.for_inference(model)
+ runtime.setenv(ENV, "0")
eager = _gen(model, tok)
assert _Stats(model)().get("replays", 0) == 0
assert _graph_buffers(model) == 0
- with G.rl_generation():
- graph = _gen(model, tok)
+ runtime.delenv(ENV)
+ graph = _gen(model, tok)
stats = _Stats(model)()
assert stats["disabled"] is None and stats["replays"] == 24 - 2, stats
assert torch.equal(eager[0], graph[0]) and torch.equal(eager[1], graph[1])
+ with G.rl_generation():
+ assert torch.equal(_gen(model, tok)[1], eager[1])
- # =0 wins inside the RL scope; =1 turns plain generate on.
runtime.setenv(ENV, "0")
replays = _Stats(model)()["replays"]
with G.rl_generation():
assert torch.equal(_gen(model, tok)[1], eager[1])
assert _Stats(model)()["replays"] == replays
- runtime.setenv(ENV, "1")
- assert torch.equal(_gen(model, tok)[1], eager[1])
- assert _Stats(model)()["replays"] > replays
- # generate() called on a model in training mode returns to training: everything released.
+ # for_training frees what an inference-mode generate() kept (a returned cache would release them).
runtime.delenv(ENV)
+ enc = tok(PROMPTS, return_tensors = "pt", padding = True).to("cuda")
+ model.generate(**enc, max_new_tokens = 8, do_sample = False, pad_token_id = tok.pad_token_id)
+ assert _graph_buffers(model) > 0
FastLanguageModel.for_training(model)
- with G.rl_generation():
- _gen(model, tok)
+ assert _graph_buffers(model) == 0 and _decode_buffers(model) == 0
+ assert _Stats(model)()["graphs"] == 0
+
+ # generate() called on a model in training mode returns to training: everything released.
+ _gen(model, tok)
assert model.training
assert _graph_buffers(model) == 0 and _decode_buffers(model) == 0
assert _Stats(model)()["graphs"] == 0
+def test_generate_scope_turns_graphs_on(runtime):
+ runtime.setattr(G, "_DEFAULT_OK", True)
+ runtime.setattr(G, "_STUDIO_DEFAULT_OK", True)
+ assert G.enabled() is False
+ with G.generate_scope():
+ assert G.enabled() is True
+ runtime.setenv(ENV, "0")
+ assert G.enabled() is False
+ runtime.delenv(ENV)
+ assert G.enabled() is False
+ runtime.setattr(G, "_STUDIO_DEFAULT_OK", False) # native Windows
+ with G.generate_scope():
+ assert G.enabled() is False
+
+
@rl_default
def test_grpo_generation_uses_graphs_and_releases(runtime, tmp_path):
"""A real GRPOTrainer step: its generate() runs graphed with the variable unset, completions
@@ -203,3 +226,106 @@ def test_grpo_generation_uses_graphs_and_releases(runtime, tmp_path):
assert len(seen["replays"]) == 2 and all(r > 0 for r in seen["replays"]), seen
assert seen["buffers_at_loss"] == [(0, 0), (0, 0)], seen
assert _Stats(model)()["disabled"] is None
+
+
+@rl_default
+def test_out_of_memory_in_the_graph_path_redoes_the_call_eagerly(runtime):
+ """A real CUDA allocator OOM raised only while graphs are on (the first graph-mode buffer
+ allocation asks for 1 PiB): generate() frees the graphs, redoes the call eagerly once, counts it,
+ warns once, and returns the eager tokens and logits."""
+ if not G._STUDIO_DEFAULT_OK:
+ pytest.skip("the plain generate() default stays eager on native Windows")
+ model, tok = _load()
+ FastLanguageModel.for_inference(model)
+ runtime.setenv(ENV, "0")
+ eager = _gen(model, tok)
+ runtime.delenv(ENV)
+ original = G.alloc_decode_buffers
+
+ def allocate(*args, **kwargs):
+ if G.enabled():
+ torch.empty(1 << 50, dtype = torch.uint8, device = "cuda")
+ return original(*args, **kwargs)
+
+ runtime.setattr(G, "alloc_decode_buffers", allocate)
+ runtime.setattr(G, "_OOM_WARNED", False)
+ redos = G.EAGER_REDOS
+ with pytest.warns(UserWarning, match = "ran out of GPU memory"):
+ redone = _gen(model, tok)
+ assert G.EAGER_REDOS == redos + 1
+ assert torch.equal(eager[0], redone[0]) and torch.equal(eager[1], redone[1])
+ assert _graph_buffers(model) == 0
+ # Without the forced OOM the next call is graphed again.
+ runtime.setattr(G, "alloc_decode_buffers", original)
+ replays = _Stats(model)().get("replays", 0)
+ assert torch.equal(_gen(model, tok)[1], eager[1])
+ assert _Stats(model)()["replays"] > replays and G.EAGER_REDOS == redos + 1
+
+
+@rl_default
+def test_training_forward_without_for_training_frees_generate_memory(runtime):
+ """Sampling in a trainer callback: generate() on an inference-mode model, then training resumes
+ with model.train() only. The first training forward frees the graphs and decode caches."""
+ if not G._STUDIO_DEFAULT_OK:
+ pytest.skip("the plain generate() default stays eager on native Windows")
+ model, tok = _load()
+ FastLanguageModel.for_inference(model)
+ enc = tok(PROMPTS, return_tensors = "pt", padding = True).to("cuda")
+ model.generate(**enc, max_new_tokens = 8, do_sample = False, pad_token_id = tok.pad_token_id)
+ assert _graph_buffers(model) > 0
+ model.train()
+ out = model(**enc, labels = enc.input_ids)
+ out.loss.backward()
+ assert _graph_buffers(model) == 0 and _decode_buffers(model) == 0
+ assert _Stats(model)()["graphs"] == 0
+
+
+def _gen_tokens(model, tok, prompts, new):
+ enc = tok(prompts, return_tensors = "pt", padding = True).to("cuda")
+ out = model.generate(
+ **enc, max_new_tokens = new, min_new_tokens = new, do_sample = False,
+ pad_token_id = tok.pad_token_id,
+ ) # fmt: skip
+ return out.cpu()
+
+
+@rl_default
+def test_short_calls_decode_eagerly_and_keep_the_kept_graph(runtime):
+ """Short calls decode eagerly unless their batch is kept; tokens equal eager."""
+ if not G._STUDIO_DEFAULT_OK:
+ pytest.skip("the plain generate() default stays eager on native Windows")
+ model, tok = _load()
+ FastLanguageModel.for_inference(model)
+ one, two = PROMPTS[:1], PROMPTS
+ runtime.setenv(ENV, "0")
+ eager = {
+ (p, n): _gen_tokens(model, tok, ps, n)
+ for p, ps in (("one", one), ("two", two))
+ for n in (4, 24)
+ }
+ runtime.delenv(ENV)
+
+ def check(p, ps, n):
+ assert torch.equal(_gen_tokens(model, tok, ps, n), eager[(p, n)]), (p, n)
+
+ def batches():
+ return {k[0] for m in model.modules() for k in m.__dict__.get("_unsloth_graph_buffers", {})}
+
+ check("two", two, 24)
+ before = _Stats(model)()
+ assert before["captures"] > 0 and batches() == {2}, (before, batches())
+ check("one", one, 4)
+ after = _Stats(model)()
+ assert after["captures"] == before["captures"] and after["replays"] == before["replays"], (
+ before,
+ after,
+ )
+ assert batches() == {2}, batches()
+ check("two", two, 4)
+ short = _Stats(model)()
+ assert short["captures"] == before["captures"] and short["replays"] > before["replays"], (
+ before,
+ short,
+ )
+ check("two", two, 24)
+ assert _Stats(model)()["captures"] == before["captures"], (before, _Stats(model)())
diff --git a/tests/test_decode_cuda_graphs_studio.py b/tests/test_decode_cuda_graphs_studio.py
index 21d272721c7..778ca4eb24d 100644
--- a/tests/test_decode_cuda_graphs_studio.py
+++ b/tests/test_decode_cuda_graphs_studio.py
@@ -93,7 +93,7 @@ def test_studio_scope_is_off_on_native_windows(G, monkeypatch):
@pytest.mark.skipif(not has_real_cuda(), reason = "CUDA only")
-def test_studio_generate_graphed_plain_generate_eager():
+def test_studio_generate_graphed_plain_generate_eager(monkeypatch):
import unsloth # noqa: F401
from unsloth import FastLanguageModel
import unsloth.models._decode_graphs as G
@@ -118,13 +118,16 @@ def test_studio_generate_graphed_plain_generate_eager():
def replays():
return G.stats(model).get("replays", 0)
+ # Plain generate() graphs by default too (generate_scope); =0 is the eager reference.
+ monkeypatch.setenv("UNSLOTH_DECODE_CUDA_GRAPHS", "0")
eager = gen()
assert replays() == 0
+ monkeypatch.delenv("UNSLOTH_DECODE_CUDA_GRAPHS")
with G.studio_generation():
graphed = gen()
assert replays() == 24 - 2 and G.stats(model)["disabled"] is None
assert torch.equal(eager[0], graphed[0]) and torch.equal(eager[1], graphed[1])
- assert torch.equal(gen()[1], eager[1]) and replays() == 24 - 2
+ assert torch.equal(gen()[1], eager[1]) and replays() == 2 * (24 - 2)
G.release_for_training(model)
assert G.stats(model)["graphs"] == 0
assert not any("paged_attention" in m.__dict__ for m in model.modules())
diff --git a/tests/test_decode_graphs_memory.py b/tests/test_decode_graphs_memory.py
new file mode 100644
index 00000000000..fa465139b81
--- /dev/null
+++ b/tests/test_decode_graphs_memory.py
@@ -0,0 +1,687 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""Graph decode memory planning and OOM fallback; CPU only, CUDA memory queries mocked."""
+
+import importlib.util
+import types
+import warnings
+from pathlib import Path
+
+import pytest
+import torch
+
+ENV = "UNSLOTH_DECODE_CUDA_GRAPHS"
+_SOURCE = Path(__file__).resolve().parents[1] / "unsloth" / "models" / "_decode_graphs.py"
+CUDA0 = torch.device("cuda", 0)
+GB = 1 << 30
+
+
+def G(monkeypatch):
+ spec = importlib.util.spec_from_file_location("_unsloth_decode_graphs_memory", _SOURCE)
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+ module.ENABLED = None
+ module._DEFAULT_OK = True
+ module._STUDIO_DEFAULT_OK = True
+ monkeypatch.delenv(ENV, raising = False)
+ module.emptied = 0
+
+ def empty():
+ module.emptied += 1
+
+ module._empty_cache = empty
+ return module
+
+
+class _Attn(torch.nn.Module):
+ pass
+
+
+class _Model(torch.nn.Module):
+ """Qwen3-4B-like dims; layer attention modules carry the graph-mode buffer store."""
+
+ def __init__(self, layers = 2):
+ super().__init__()
+ self.config = types.SimpleNamespace(
+ num_hidden_layers = 36, hidden_size = 2560, num_attention_heads = 32,
+ num_key_value_heads = 8, head_dim = 128, intermediate_size = 9728, vocab_size = 151936,
+ model_type = "qwen3",
+ ) # fmt: skip
+ self.attns = torch.nn.ModuleList(_Attn() for _ in range(layers))
+
+ def keep(
+ self,
+ bsz,
+ rows,
+ dtype = torch.float16,
+ ):
+ for attn in self.attns:
+ paged = torch.zeros((rows, 2, 1, 1, 1), dtype = dtype)
+ store = attn.__dict__.setdefault("_unsloth_graph_buffers", {})
+ store[(bsz, dtype, CUDA0)] = (paged, None, None, None, None, None)
+ attn.paged_attention = paged
+
+ def held(self):
+ return len(
+ {key for a in self.attns for key in a.__dict__.get("_unsloth_graph_buffers", {})}
+ )
+
+
+def _ids(bsz, prompt):
+ return types.SimpleNamespace(shape = (bsz, prompt), device = CUDA0)
+
+
+def _need(
+ G,
+ bsz,
+ prompt,
+ max_len = 0,
+):
+ rows, kv, work = G._call_bytes(_Model().config, bsz, prompt, torch.float16, 512, max_len)
+ return G.CALIB_WORK * (kv + work) + G.CALIB_GRAPH_KV * kv
+
+
+def _plan(
+ G,
+ model,
+ bsz,
+ prompt,
+ max_len = 0,
+):
+ return G.plan_call(model, _ids(bsz, prompt), torch.float16, 512, max_len)
+
+
+def test_plan_frees_kept_buffers_when_the_call_does_not_fit_beside_them(G):
+ model = _Model()
+ model.keep(1, 2816)
+ model.keep(4, 2816)
+ need = _need(G, 16, 2048, 2081)
+ assert 11 * GB < need < 13 * GB
+ free = iter([int(0.9 * need), int(1.1 * need)])
+ G._free_bytes = lambda device: next(free)
+ plan, token = _plan(G, model, 16, 2048, 2081)
+ assert plan == "graph" and token is not None
+ assert model.held() == 0 and G.emptied == 1 and G.EAGER_UNFIT == 0
+ assert not any("paged_attention" in a.__dict__ for a in model.attns)
+
+
+def test_plan_holds_no_buffer_reference_when_it_empties_the_cache(G):
+ import weakref
+
+ model = _Model()
+ model.keep(1, 2816)
+ model.keep(4, 2816)
+ model.keep(16, 64) # too small to reuse, still scanned
+ refs = [
+ weakref.ref(bufs[0])
+ for a in model.attns
+ for bufs in a.__dict__["_unsloth_graph_buffers"].values()
+ ]
+ alive_at_empty = []
+
+ def empty():
+ G.emptied += 1
+ alive_at_empty.append(sum(r() is not None for r in refs))
+
+ G._empty_cache = empty
+ need = _need(G, 16, 2048, 2081)
+ free = iter([int(0.9 * need), int(1.1 * need)])
+ G._free_bytes = lambda device: next(free)
+ assert _plan(G, model, 16, 2048, 2081)[0] == "graph"
+ assert alive_at_empty == [0]
+
+
+def test_plan_runs_a_call_that_cannot_fit_eagerly_from_the_start(G):
+ model = _Model()
+ model.keep(4, 2816)
+ G._free_bytes = lambda device: int(0.95 * _need(G, 16, 2048, 2081))
+ plan, token = _plan(G, model, 16, 2048, 2081)
+ assert (plan, token) == ("eager", None)
+ # Kept state was freed first (the eager call needs the memory too), then the call was counted.
+ assert model.held() == 0 and G.EAGER_UNFIT == 1
+ assert _plan(G, model, 16, 2048, 2081)[0] == "eager" and G.EAGER_UNFIT == 2 and G.emptied == 1
+
+
+def test_plan_keeps_buffers_when_there_is_room(G):
+ model = _Model()
+ model.keep(1, 2816)
+ G._free_bytes = lambda device: 20 * GB
+ assert _plan(G, model, 16, 2048)[0] == "graph"
+ assert model.held() == 1 and G.emptied == 0 and G.EAGER_UNFIT == 0
+
+
+def test_plan_counts_a_reusable_buffer_of_the_same_batch_size(G):
+ model = _Model(layers = 1)
+ rows, kv, work = G._call_bytes(model.config, 4, 128, torch.float16, 512)
+ model.keep(4, rows)
+ reused = model.attns[0]._unsloth_graph_buffers[(4, torch.float16, CUDA0)][0]
+ G._free_bytes = (
+ lambda device: int(_need(G, 4, 128) - reused.numel() * reused.element_size()) + 1
+ )
+ assert _plan(G, model, 4, 128)[0] == "graph"
+ assert model.held() == 1
+
+
+def test_plan_lets_non_cuda_or_shapeless_inputs_through(G):
+ model = _Model()
+ G._free_bytes = lambda device: 0
+ cpu_ids = types.SimpleNamespace(shape = (16, 2048), device = torch.device("cpu"))
+ assert G.plan_call(model, cpu_ids, torch.float16, 512) == ("graph", None)
+ assert G.plan_call(model, None, torch.float16, 512) == ("graph", None)
+ assert G.EAGER_UNFIT == 0
+
+
+def test_measured_peak_can_only_raise_the_estimate(G):
+ model = _Model()
+ est = _need(G, 4, 2048, 2081)
+ stats = {"alloc": 3 * GB, "peak": 3 * GB}
+ G._alloc_stats = lambda device: (stats["alloc"], stats["peak"])
+ G._free_bytes = lambda device: 20 * GB
+ plan, token = _plan(G, model, 4, 2048, 2081)
+ assert plan == "graph"
+ stats["peak"] = 5 * GB # below the estimate
+ G.record_need(model, token)
+ key = G._shape_key(4, 2048, 2081)
+ assert model.__dict__[G.NEEDS][key] == 2 * GB < est
+ # A measurement can only raise the estimate.
+ G._free_bytes = lambda device: int(est) - 1
+ assert _plan(G, model, 4, 2048, 2081)[0] == "eager"
+ G._free_bytes = lambda device: int(est) + 1
+ assert _plan(G, model, 4, 2048, 2081)[0] == "graph"
+ G._free_bytes = lambda device: 20 * GB
+ plan, token = _plan(G, model, 4, 2048, 2081)
+ stats["peak"] = 3 * GB + 6 * GB
+ G.record_need(model, token)
+ assert model.__dict__[G.NEEDS][key] == 6 * GB > est
+ G._free_bytes = lambda device: int(G.MEASURED_MARGIN * 6 * GB) - 1
+ assert _plan(G, model, 4, 2048, 2081)[0] == "eager"
+ G._free_bytes = lambda device: int(G.MEASURED_MARGIN * 6 * GB) + 1
+ assert _plan(G, model, 4, 2048, 2081)[0] == "graph"
+ G.record_need(model, _plan(G, model, 1, 128)[1])
+ assert G._shape_key(1, 128, 0) not in model.__dict__[G.NEEDS]
+
+
+def test_prompt_length_is_part_of_the_shape(G):
+ # Same total-length bucket, different prompts: the key includes the prompt bucket.
+ assert G._shape_key(4, 128, 4096) != G._shape_key(4, 2048, 4096)
+ model = _Model()
+ stats = {"alloc": 3 * GB, "peak": 3 * GB}
+ G._alloc_stats = lambda device: (stats["alloc"], stats["peak"])
+ G._free_bytes = lambda device: 30 * GB
+ plan, token = _plan(G, model, 4, 128, 4096)
+ stats["peak"] = 3 * GB + GB // 2
+ G.record_need(model, token)
+ long_need = _need(G, 4, 2048, 4096)
+ G._free_bytes = lambda device: int(long_need) - 1
+ assert _plan(G, model, 4, 2048, 4096)[0] == "eager"
+
+
+def test_an_early_stop_measurement_never_admits_a_full_length_call(G):
+ model = _Model()
+ est = _need(G, 4, 2048, 4096)
+ stats = {"alloc": 3 * GB, "peak": 3 * GB}
+ G._alloc_stats = lambda device: (stats["alloc"], stats["peak"])
+ G._free_bytes = lambda device: 30 * GB
+ plan, token = _plan(G, model, 4, 2048, 4096)
+ stats["peak"] = 3 * GB + GB
+ G.record_need(model, token)
+ G._free_bytes = lambda device: int(G.MEASURED_MARGIN * GB) + 1
+ assert _plan(G, model, 4, 2048, 4096)[0] == "eager"
+ G._free_bytes = lambda device: int(est) + 1
+ assert _plan(G, model, 4, 2048, 4096)[0] == "graph"
+
+
+def test_release_keeps_the_measured_needs(G):
+ model = _Model()
+ model.__dict__[G.NEEDS] = {(4, 2304, 2304): GB}
+ model.keep(4, 2816)
+ G.release_for_training(model)
+ assert model.__dict__[G.NEEDS] == {(4, 2304, 2304): GB}
+
+
+def _oom_while_graphed(G, seen):
+ def generate(*args, **kwargs):
+ on = G.enabled()
+ seen.append(on)
+ if on:
+ raise torch.cuda.OutOfMemoryError("CUDA out of memory (stand-in)")
+ return "eager-output"
+
+ return generate
+
+
+def test_oom_with_graphs_redoes_the_call_once_eagerly_and_warns_once(G):
+ model = _Model()
+ seen = []
+ for _ in range(2):
+ model.keep(1, 2816)
+ with G.generate_scope():
+ graphs = G.enabled()
+ with warnings.catch_warnings(record = True) as caught:
+ warnings.simplefilter("always")
+ out = G.generate_with_fallback(model, _oom_while_graphed(G, seen), graphs, (), {})
+ assert out == "eager-output"
+ assert model.held() == 0
+ with G.generate_scope():
+ assert G.enabled() is True
+ assert seen == [True, False, True, False]
+ assert G._OOM_WARNED is True
+
+
+def test_oom_warning_is_emitted_once_per_process(G):
+ model = _Model()
+ messages = []
+ for _ in range(3):
+ with G.generate_scope():
+ with warnings.catch_warnings(record = True) as caught:
+ warnings.simplefilter("always")
+ G.generate_with_fallback(model, _oom_while_graphed(G, []), True, (), {})
+ messages += [
+ str(w.message) for w in caught if "ran out of GPU memory" in str(w.message)
+ ]
+ assert len(messages) == 1
+
+
+ "graphs, kwargs",
+ [
+ (False, {}),
+ (True, {"streamer": object()}),
+ (True, {"past_key_values": object()}),
+ (True, {"stopping_criteria": [object()]}),
+ (True, {"logits_processor": [object()]}),
+ ],
+)
+def test_oom_is_raised_when_a_redo_is_not_safe(G, graphs, kwargs):
+ model = _Model()
+ model.keep(1, 2816)
+
+ def generate(*args, **kw):
+ raise torch.cuda.OutOfMemoryError("CUDA out of memory (stand-in)")
+
+ with pytest.raises(torch.cuda.OutOfMemoryError):
+ G.generate_with_fallback(model, generate, graphs, (), kwargs)
+ assert model.held() == 1
+
+
+def test_oom_in_the_eager_redo_propagates(G):
+ model = _Model()
+ calls = []
+
+ def generate(*args, **kw):
+ calls.append(G.enabled())
+ raise torch.cuda.OutOfMemoryError("CUDA out of memory (stand-in)")
+
+ with G.generate_scope():
+ with pytest.raises(torch.cuda.OutOfMemoryError):
+ G.generate_with_fallback(model, generate, True, (), {})
+ assert calls == [True, False]
+
+
+def test_eager_scope_beats_the_variable_and_the_scopes(G, monkeypatch):
+ monkeypatch.setenv(ENV, "1")
+ with G.rl_generation(), G.eager_scope():
+ assert G.enabled() is False
+ assert G.enabled() is True
+
+
+def _with_graphs(G, model, *bszs):
+ state = G._state(model)
+ for bsz in bszs:
+ model.keep(bsz, 2816)
+ state.graphs[(bsz, CUDA0, (False, None), True, None)] = object()
+ state.pool = object()
+ return state
+
+
+def test_keep_only_drops_other_batch_sizes_and_keeps_this_ones_graphs(G):
+ model = _Model()
+ state = _with_graphs(G, model, 1, 4)
+ epoch = G.BUFFER_EPOCH
+ assert G.keep_only(model, 4) is True
+ assert {k for a in model.attns for k in a._unsloth_graph_buffers} == {(4, torch.float16, CUDA0)}
+ assert [k[0] for k in state.graphs] == [4] and state.pool is not None
+ # The kept graphs still read valid buffers: no epoch bump, so the next call of this shape replays.
+ assert G.BUFFER_EPOCH == epoch
+ assert all(
+ a.paged_attention is a._unsloth_graph_buffers[(4, torch.float16, CUDA0)][0]
+ for a in model.attns
+ )
+ assert G.keep_only(model, 4) is False
+
+
+def test_keep_only_never_evicts_the_cache_decode_just_used(G):
+ model = _Model()
+ _with_graphs(G, model, 4, 1) # decode ran at bs 1
+ G.keep_only(model, 4)
+ assert {k for a in model.attns for k in a._unsloth_graph_buffers} == {(1, torch.float16, CUDA0)}
+ assert all(
+ a.paged_attention is a._unsloth_graph_buffers[(1, torch.float16, CUDA0)][0]
+ for a in model.attns
+ )
+
+
+def test_training_forward_releases_what_a_graphed_generate_kept(G):
+ model = _Model()
+ _with_graphs(G, model, 4)
+ assert G.release_if_training(model) is False
+ assert model.held() == 1
+ model.__dict__[G.HELD] = True
+ assert G.release_if_training(model) is True
+ assert model.held() == 0 and G.HELD not in model.__dict__
+ assert G._state(model).graphs == {}
+ assert G.release_if_training(model) is False
+
+
+def test_release_clears_the_held_mark(G):
+ model = _Model()
+ _with_graphs(G, model, 1)
+ model.__dict__[G.HELD] = True
+ G.release_for_training(model)
+ assert G.HELD not in model.__dict__
+
+
+def test_eager_redo_is_counted(G):
+ model = _Model()
+ assert G.EAGER_REDOS == 0
+ with G.generate_scope():
+ G.generate_with_fallback(model, _oom_while_graphed(G, []), True, (), {})
+ G.generate_with_fallback(model, lambda: "ok", True, (), {})
+ assert G.EAGER_REDOS == 1
+
+
+def test_causal_lm_forward_calls_the_training_release():
+ """The hook lives in llama.py's _CausalLM_fast_forward (importing unsloth needs an accelerator,
+ so this reads the source): a training forward with grad on releases what generate() kept."""
+ import ast
+
+ source = (_SOURCE.parent / "llama.py").read_text(encoding = "utf-8")
+ tree = ast.parse(source)
+ fwd = next(
+ n
+ for n in ast.walk(tree)
+ if isinstance(n, ast.FunctionDef) and n.name == "_CausalLM_fast_forward"
+ )
+ hooks = [
+ n for n in ast.walk(fwd)
+ if isinstance(n, ast.If) and "release_if_training" in ast.unparse(n)
+ and "self.training" in ast.unparse(n.test) and "is_grad_enabled" in ast.unparse(n.test)
+ ] # fmt: skip
+ assert hooks, "training forward does not free graphs a generate() kept"
+ gen = next(
+ n
+ for n in ast.walk(tree)
+ if isinstance(n, ast.FunctionDef) and n.name == "unsloth_fast_generate"
+ )
+ text = ast.unparse(gen)
+ assert "keep_only" in text and "HELD" in text
+ # A call the plan says cannot fit graphed runs eagerly from the start; graphed calls record their peak.
+ assert "plan_call" in text and "eager_scope" in text and "record_need" in text
+ # Plan and keep_only use the expanded decode batch.
+ assert text.count("decode_batch(") >= 2
+
+
+def test_measured_need_includes_a_reused_cache_and_is_discounted_once(G):
+ # The stored footprint includes a reused cache, so the reuse is subtracted once.
+ model = _Model(layers = 1)
+ rows, kv, work = G._call_bytes(model.config, 4, 2048, torch.float16, 512, 2081)
+ model.keep(4, rows)
+ reused = model.attns[0]._unsloth_graph_buffers[(4, torch.float16, CUDA0)][0]
+ reused_bytes = reused.numel() * reused.element_size()
+ del reused
+ stats = {"alloc": 3 * GB, "peak": 3 * GB}
+ G._alloc_stats = lambda device: (stats["alloc"], stats["peak"])
+ G._free_bytes = lambda device: 20 * GB
+ plan, token = _plan(G, model, 4, 2048, 2081)
+ assert plan == "graph"
+ stats["peak"] = 11 * GB
+ G.record_need(model, token)
+ key = G._shape_key(4, 2048, 2081)
+ total = 8 * GB + reused_bytes
+ assert model.__dict__[G.NEEDS][key] == total
+ assert G.MEASURED_MARGIN * total > _need(G, 4, 2048, 2081)
+ need = G.MEASURED_MARGIN * total - reused_bytes
+ G._free_bytes = lambda device: int(need) + 1
+ assert _plan(G, model, 4, 2048, 2081)[0] == "graph" and model.held() == 1
+ free = iter([int(need) - 1, int(G.MEASURED_MARGIN * total) - 1])
+ G._free_bytes = lambda device: next(free)
+ assert _plan(G, model, 4, 2048, 2081)[0] == "eager" and model.held() == 0
+
+
+def test_decode_batch_follows_generate_expansion(G):
+ cfg = types.SimpleNamespace(num_beams = 1, num_return_sequences = 1)
+ assert G.decode_batch(3, {}, cfg) == 3
+ assert G.decode_batch(3, {"num_return_sequences": 2, "do_sample": True}, cfg) == 6
+ assert G.decode_batch(3, {"num_beams": 4, "num_return_sequences": 2}, cfg) == 12
+ beams = types.SimpleNamespace(num_beams = 2, num_return_sequences = 1)
+ assert G.decode_batch(3, {}, beams) == 6
+ assert G.decode_batch(3, {"generation_config": beams}, cfg) == 6
+ assert G.decode_batch(3, {}, None) == 3
+
+
+def test_plan_sizes_the_expanded_batch(G):
+ model = _Model()
+ need = _need(G, 4, 2048, 2081)
+ G._free_bytes = lambda device: int(need) - 1
+ # 2 prompts x num_return_sequences 2 decode at bs 4.
+ assert G.plan_call(model, _ids(2, 2048), torch.float16, 512, 2081, bsz = 4)[0] == "eager"
+ G._free_bytes = lambda device: int(need) + 1
+ plan, token = G.plan_call(model, _ids(2, 2048), torch.float16, 512, 2081, bsz = 4)
+ assert plan == "graph" and token[0] == G._shape_key(4, 2048, 2081)
+
+
+def test_keep_only_keeps_the_batch_size_decode_used(G):
+ model = _Model()
+ state = _with_graphs(G, model, 1, 4) # decoded at bs 4
+ assert G.decode_batch_used(model) == 4
+ G.keep_only(model, None)
+ assert {k for a in model.attns for k in a._unsloth_graph_buffers} == {(4, torch.float16, CUDA0)}
+ assert [k[0] for k in state.graphs] == [4]
+
+
[email protected]("model_type", ["mistral", "qwen2", "gemma", "gemma2"])
+def test_a_model_type_that_never_graphs_keeps_eager_oom_and_planning(G, model_type):
+ model = _Model()
+ model.config.model_type = model_type
+ model.keep(1, 2816)
+ calls = []
+
+ def generate(*args, **kw):
+ calls.append(G.enabled())
+ raise torch.cuda.OutOfMemoryError("CUDA out of memory (stand-in)")
+
+ with G.generate_scope():
+ with warnings.catch_warnings(record = True) as caught:
+ warnings.simplefilter("always")
+ with pytest.raises(torch.cuda.OutOfMemoryError):
+ G.generate_with_fallback(model, generate, G.enabled(), (), {})
+ assert calls == [True]
+ assert not [w for w in caught if "ran out of GPU memory" in str(w.message)]
+ assert G.EAGER_REDOS == 0 and model.held() == 1
+ G._free_bytes = lambda device: 0
+ assert G.plan_call(model, _ids(16, 2048), torch.float16, 256, 4096) == ("graph", None)
+ assert G.EAGER_UNFIT == 0 and model.held() == 1 and G.emptied == 0
+
+
+ "max_len, prompt, replays",
+ [(0, 32, True), (33, 32, False), (34, 32, False), (35, 32, True), (288, 32, True)],
+)
+def test_a_call_too_short_to_replay_a_step_is_known(G, max_len, prompt, replays):
+ assert G.can_replay(max_len, prompt) is replays
+
+
+def _generate_fn():
+ import ast
+ tree = ast.parse((_SOURCE.parent / "llama.py").read_text(encoding = "utf-8"))
+ return next(
+ n
+ for n in ast.walk(tree)
+ if isinstance(n, ast.FunctionDef) and n.name == "unsloth_fast_generate"
+ )
+
+
+def test_generate_runs_a_short_call_eagerly_when_nothing_is_kept():
+ """Reads llama.py: importing unsloth needs an accelerator."""
+ import ast
+
+ gen = _generate_fn()
+ assert "short_call" in ast.unparse(gen)
+ guards = [
+ n for n in ast.walk(gen)
+ if isinstance(n, ast.If) and "short" in ast.unparse(n.test) and "HELD" in ast.unparse(n.test)
+ ] # fmt: skip
+ assert guards, "a short call is not routed eagerly"
+ body = ast.unparse(guards[0].body[0]) if guards[0].body else ""
+ assert "'eager'" in body or '"eager"' in body
+ assert "plan_call" not in ast.unparse(guards[0].body[0])
+
+
+def test_generate_runs_a_short_call_beside_kept_graphs_eagerly():
+ import ast
+
+ text = ast.unparse(_generate_fn())
+ assert "eager=short" in text
+ assert "if short:\n" in text and text.index("eager=short") < text.index("if short:\n")
+
+
+def _keep_graph(model, bsz):
+ state = model.__dict__.setdefault("_unsloth_decode_graphs", types.SimpleNamespace(graphs = {}))
+ state.graphs[(bsz, CUDA0, None, True, None)] = object()
+
+
+ "new, kept, short",
+ [
+ (1, None, True), (2, None, True), (2, 1, True), (3, None, True), (6, None, True),
+ (6, 4, True), (7, None, False), (8, None, False), (32, None, False), (3, 1, False),
+ (6, 1, False),
+ ],
+) # fmt: skip
+def test_a_short_call_decodes_eagerly_unless_a_graph_of_its_batch_is_kept(G, new, kept, short):
+ model = _Model()
+ if kept is not None:
+ _keep_graph(model, kept)
+ assert G.CAPTURE_MIN_STEPS == 6
+ assert G.short_call(model, 32 + new, 32, 1) is short
+ assert G.short_call(model, 0, 32, 1) is False
+
+
+def test_a_short_call_matches_the_capture_break_even(G):
+ model = _Model()
+ for new in range(3, 40):
+ G.start_generation(32 + new)
+ try:
+ assert G.short_call(model, 32 + new, 32, 1) is (not G.worth_capturing(32 + 2))
+ finally:
+ G.end_generation()
+
+
+def test_a_short_call_beside_kept_graphs_only_makes_room(G):
+ model = _Model()
+ model.keep(4, 2816)
+ G._free_bytes = lambda device: 100 * GB
+ assert G.plan_call(model, _ids(1, 32), torch.float16, 512, 36, eager = True) == ("eager", None)
+ assert model.held() == 1 and G.EAGER_UNFIT == 0
+ G._free_bytes = lambda device: 0
+ assert G.plan_call(model, _ids(1, 32), torch.float16, 512, 36, eager = True) == ("eager", None)
+ assert model.held() == 0 and G.emptied == 1 and G.EAGER_UNFIT == 0
+
+
+def test_capture_only_when_enough_steps_are_left(G):
+ G.start_generation(32 + 8)
+ try:
+ assert G.CAPTURE_MIN_STEPS == 6
+ assert G.worth_capturing(35) is True
+ assert G.worth_capturing(36) is False
+ G.start_generation(32 + 4)
+ assert not any(G.worth_capturing(kv) for kv in range(33, 37))
+ finally:
+ G.end_generation()
+ assert G.worth_capturing(10_000) is True
+
+
+def test_decode_step_skips_only_a_new_capture():
+ import ast
+
+ tree = ast.parse(_SOURCE.read_text(encoding = "utf-8"))
+ fn = next(
+ n for n in ast.walk(tree) if isinstance(n, ast.FunctionDef) and n.name == "decode_step"
+ )
+ guards = [
+ n for n in ast.walk(fn)
+ if isinstance(n, ast.If) and "worth_capturing" in ast.unparse(n.test)
+ ] # fmt: skip
+ assert guards and all("step is None" in ast.unparse(g.test) for g in guards)
+ text = ast.unparse(fn)
+ assert text.index("worth_capturing") < text.index("_capture(")
+
+
+def test_an_eager_short_call_gets_no_credit_for_a_kept_graph_cache(G):
+ model = _Model()
+ model.keep(16, 8192)
+ rows, kv, work = G._call_bytes(model.config, 16, 64, torch.float16, 512, 66)
+ G._kept_buffers = lambda model, key, rows: (True, int(4 * (kv + work)))
+ G._free_bytes = lambda device: int(0.5 * G.CALIB_WORK * (kv + work))
+ assert G.plan_call(model, _ids(16, 64), torch.float16, 512, 66, eager = True) == ("eager", None)
+ assert model.held() == 0 and G.emptied == 1 and G.EAGER_UNFIT == 0
+
+
+def test_an_oversized_kept_cache_is_credited_only_up_to_this_call_kv(G):
+ model = _Model()
+ model.keep(4, 8192)
+ rows, kv, work = G._call_bytes(model.config, 4, 128, torch.float16, 512, 160)
+ G._kept_buffers = lambda model, key, rows: (True, int(20 * (kv + work)))
+ G._free_bytes = lambda device: 0
+ assert G.plan_call(model, _ids(4, 128), torch.float16, 512, 160) == ("eager", None)
+ assert model.held() == 0 and G.emptied == 1 and G.EAGER_UNFIT == 1
+
+
+def _llama_helpers(*names):
+ import ast
+
+ path = Path(_SOURCE).with_name("llama.py")
+ tree = ast.parse(path.read_text(encoding = "utf-8"))
+ fns = [n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name in names]
+ namespace = {}
+ exec(compile(ast.Module(body = fns, type_ignores = []), str(path), "exec"), namespace)
+ return namespace
+
+
+def test_planning_honours_an_explicit_max_length_in_the_caller_generation_config():
+ transformers = pytest.importorskip("transformers")
+ ns = _llama_helpers("_generation_max_len", "_generation_plan_len")
+ model = types.SimpleNamespace(generation_config = transformers.GenerationConfig(max_length = 40960))
+ ids = torch.zeros((1, 128), dtype = torch.long)
+ config = transformers.GenerationConfig(max_length = 4096)
+ assert ns["_generation_max_len"](model, (ids,), {"generation_config": config}) == 0
+ assert ns["_generation_plan_len"](model, (ids,), {"generation_config": config}) == 4096
+ # The model's default config is not a request, and max_new_tokens still wins.
+ assert ns["_generation_plan_len"](model, (ids,), {}) == 0
+ assert (
+ ns["_generation_plan_len"](
+ model, (ids,), {"generation_config": config, "max_new_tokens": 32}
+ )
+ == 160
+ )
+
+
+def test_cache_growth_cap_uses_the_planned_length():
+ # The planner budgets plan_len rows; start_generation's hint caps the cache growth, so both must get it.
+ import ast
+
+ path = Path(_SOURCE).with_name("llama.py")
+ tree = ast.parse(path.read_text(encoding = "utf-8"))
+ fn = next(
+ n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name == "unsloth_fast_generate"
+ )
+ args = {}
+ for node in ast.walk(fn):
+ if isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute):
+ if node.func.attr in ("start_generation", "short_call", "plan_call"):
+ args[node.func.attr] = node.args
+ assert args["start_generation"][0].id == "plan_len"
+ assert args["short_call"][1].id == "plan_len"
+ assert args["plan_call"][4].id == "plan_len"
diff --git a/tests/test_for_training_generation_flag.py b/tests/test_for_training_generation_flag.py
index d7fc5e85c18..7204bf238e1 100644
--- a/tests/test_for_training_generation_flag.py
+++ b/tests/test_for_training_generation_flag.py
@@ -95,6 +95,8 @@ def _for_training(module, class_name):
# No tensor parallel world in these tests (for_training calls the TP guard first since #15368).
namespace["_tensor_parallel_for_training"] = lambda model: None
namespace["enable_fp8_lora"] = lambda model: []
+ # No decode graphs held in these tests (for_training frees them first since decode graphs became the default).
+ namespace["_decode_graphs"] = types.SimpleNamespace(release_for_training = lambda model: None)
exec(compile(ast.fix_missing_locations(compiled), str(path), "exec"), namespace)
return namespace["for_training"]
diff --git a/tests/test_return_logits_survives_for_training.py b/tests/test_return_logits_survives_for_training.py
index 36fd4625573..31f32d76a7c 100644
--- a/tests/test_return_logits_survives_for_training.py
+++ b/tests/test_return_logits_survives_for_training.py
@@ -179,6 +179,9 @@ def _fast_llama_for_training():
set_module_gradient_checkpointing = lambda module, flag: None,
_tensor_parallel_for_training = lambda model: None, # tp_replica.py: no tp model here
enable_fp8_lora = lambda model: [],
+ _decode_graphs = types.SimpleNamespace(
+ release_for_training = lambda model: None
+ ), # no graphs held here
)
module = ast.fix_missing_locations(ast.Module(body = [fn], type_ignores = []))
exec(compile(module, str(MODELS / "llama.py"), "exec"), namespace)
diff --git a/unsloth/models/_decode_graphs.py b/unsloth/models/_decode_graphs.py
index d5d3e6f9506..08f1dde31ed 100644
--- a/unsloth/models/_decode_graphs.py
+++ b/unsloth/models/_decode_graphs.py
@@ -1,9 +1,8 @@
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
-"""CUDA-graph decode; UNSLOTH_DECODE_CUDA_GRAPHS=1 forces, =0 disables; unset, only RL-trainer
-generate and Studio's chat worker on NVIDIA CUDA; graphs dropped when params, LoRA, quant or RoPE
-change.
+"""CUDA-graph decode; UNSLOTH_DECODE_CUDA_GRAPHS=1 forces, =0 disables; unset, on for every
+Unsloth generate() on NVIDIA CUDA outside native Windows for the model types below.
UNSLOTH_DECODE_CUDA_GRAPHS_FULL=1 also captures the attention cores: each step is then ONE graph that
attends over a fixed span of cache rows (the next power of two >= the KV length, at least BUCKET, at
@@ -29,6 +28,8 @@ FULL = os.environ.get("UNSLOTH_DECODE_CUDA_GRAPHS_FULL", "0") == "1"
FULL_MAX_BSZ = int(os.environ.get("UNSLOTH_DECODE_FULL_GRAPH_MAX_BSZ", "1"))
# KV-cache rows are allocated in multiples of this with graphs on.
BUCKET = 256
+# T4 Qwen3-0.6B: 3-6 new tokens ran 5-58% slower graphed, 8 tokens 1.25x faster.
+CAPTURE_MIN_STEPS = int(os.environ.get("UNSLOTH_DECODE_GRAPH_CAPTURE_MIN_STEPS", "6"))
# Captured steps (one per batch size / autocast state) kept per model.
MAX_GRAPHS = int(os.environ.get("UNSLOTH_DECODE_GRAPH_MAX", "8"))
# Whole-step graphs keep one graph per span too (log2(cache / BUCKET) + 1 per batch size).
@@ -39,6 +40,21 @@ __all__ = [
"enabled",
"rl_generation",
"studio_generation",
+ "generate_scope",
+ "eager_scope",
+ "graphable",
+ "can_replay",
+ "short_call",
+ "kept_graph",
+ "plan_call",
+ "record_need",
+ "decode_batch",
+ "decode_batch_used",
+ "EAGER_UNFIT",
+ "generate_with_fallback",
+ "keep_only",
+ "release_if_training",
+ "EAGER_REDOS",
"static",
"alloc_decode_buffers",
"decode_step",
@@ -69,6 +85,8 @@ _PROJ_NAMES = ("q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "
def enabled():
"""Whether a decode step / prefill on this thread runs (and sizes its buffers) for graph replay."""
+ if getattr(_TLS, "eager", 0):
+ return False
if ENABLED is not None:
return bool(ENABLED)
if torch.compiler.is_compiling():
@@ -125,6 +143,308 @@ class studio_generation(rl_generation):
return super().__exit__(*exc) if self._entered else False
+class generate_scope(studio_generation):
+ """Per-thread scope of plain model.generate(): graphs default on, as in Studio."""
+
+ __slots__ = ()
+
+
+class eager_scope:
+ """Per-thread: force eager decode (the OOM redo in generate())."""
+
+ __slots__ = ()
+
+ def __enter__(self):
+ _TLS.eager = getattr(_TLS, "eager", 0) + 1
+ return self
+
+ def __exit__(self, *exc):
+ _TLS.eager -= 1
+ return False
+
+
+# Need = CALIB_WORK x (KV + prefill work) + CALIB_GRAPH_KV x KV. Calibrated on Qwen3-4B QLoRA bs16 x 2048,
+# T4: eager peaked 1.14x the estimate; graph capture OOMed at ~12 GiB free (~0.26 x KV extra).
+CALIB_WORK = float(os.environ.get("UNSLOTH_DECODE_GRAPH_CALIB_WORK", "1.15"))
+CALIB_GRAPH_KV = float(os.environ.get("UNSLOTH_DECODE_GRAPH_CALIB_KV", "0.30"))
+# A shape's largest measured footprint x this raises (never lowers) the estimate.
+MEASURED_MARGIN = 1.05
+EAGER_UNFIT = 0
+NEEDS = "_unsloth_decode_needs"
+
+
+def _call_bytes(
+ config,
+ bsz,
+ prompt,
+ dtype,
+ increment,
+ max_len = 0,
+):
+ """(cache rows, KV bytes, prefill work bytes), sized like alloc_decode_buffers."""
+ try:
+ layers = int(config.num_hidden_layers)
+ hidden = int(config.hidden_size)
+ heads = int(config.num_attention_heads)
+ kv_heads = int(getattr(config, "num_key_value_heads", None) or heads)
+ head_dim = int(getattr(config, "head_dim", None) or hidden // heads)
+ inter = int(getattr(config, "intermediate_size", None) or 4 * hidden)
+ vocab = int(getattr(config, "vocab_size", 0) or 0)
+ except (AttributeError, TypeError, ValueError):
+ return None, None, None
+ size = torch.empty((), dtype = dtype).element_size()
+ rows = _round_up(max(prompt + increment + 1, max_len + 1), BUCKET)
+ kv = layers * rows * 2 * bsz * kv_heads * head_dim * size
+ tokens = bsz * prompt
+ work = tokens * (4 * hidden + 3 * inter + 2 * heads * head_dim) * size + bsz * vocab * 4
+ return rows, kv, work
+
+
+def _allocator(device):
+ """(allocated, reserved, peak allocated) from one stats read."""
+ try:
+ stats = torch.cuda.memory_stats_as_nested_dict(device)
+ return (
+ stats["allocated_bytes"]["all"]["current"],
+ stats["reserved_bytes"]["all"]["current"],
+ stats["allocated_bytes"]["all"]["peak"],
+ )
+ except (KeyError, TypeError):
+ return (
+ torch.cuda.memory_allocated(device),
+ torch.cuda.memory_reserved(device),
+ torch.cuda.max_memory_allocated(device),
+ )
+
+
+def _free_bytes(device):
+ free, _ = torch.cuda.mem_get_info(device)
+ allocated, reserved, _ = _allocator(device)
+ return free + reserved - allocated
+
+
+def _empty_cache():
+ torch.cuda.empty_cache()
+
+
+def _alloc_stats(device):
+ # Peak is never reset: other code may rely on max_memory_allocated.
+ allocated, _, peak = _allocator(device)
+ return allocated, peak
+
+
+def _shape_key(bsz, prompt, max_len):
+ # Includes the prompt bucket: a longer prompt has a larger prefill.
+ return (bsz, _round_up(prompt + 1, BUCKET), _round_up(max(prompt, max_len) + 1, BUCKET))
+
+
+def _kept_buffers(model, key, rows):
+ """(any buffers kept, bytes of a kept cache for `key` with >= rows); no buffer reference outlives release()."""
+ kept, reused = False, 0
+ for m in model.modules():
+ store = m.__dict__.get("_unsloth_graph_buffers")
+ if not store:
+ continue
+ kept = True
+ bufs = store.get(key)
+ if bufs is not None and bufs[0].shape[0] >= rows:
+ reused += bufs[0].numel() * bufs[0].element_size()
+ return kept, reused
+
+
+def can_replay(max_len, prompt):
+ """The first decode step is always eager, so <= 2 new tokens never replay; 0 (unknown) may."""
+ return not max_len or max_len - prompt > 2
+
+
+def kept_graph(model, bsz):
+ return any(key[0] == bsz for state in _states(model) for key in state.graphs)
+
+
+def short_call(model, max_len, prompt, bsz):
+ """Too short to win back a capture unless a graph of its batch is kept: decode eagerly."""
+ if not max_len:
+ return False
+ left = max_len - prompt - 1
+ return left < 2 or (left < CAPTURE_MIN_STEPS and not kept_graph(model, bsz))
+
+
+def graphable(model):
+ """Other model types always decode eagerly, so planning them only costs."""
+ return getattr(getattr(model, "config", None), "model_type", None) in _SUPPORTED_MODEL_TYPES
+
+
+def plan_call(
+ model,
+ input_ids,
+ dtype,
+ increment,
+ max_len = 0,
+ bsz = None,
+ eager = False,
+):
+ """("graph", token) or ("eager", None). Releases kept graphs / buffers first if the call would not
+ fit beside them; still too big runs eagerly up front rather than relying on the OOM redo. `bsz` is
+ the expanded decode batch (decode_batch()). eager: only makes room, uncounted."""
+ global EAGER_UNFIT
+ device = getattr(input_ids, "device", None)
+ if (
+ not graphable(model)
+ or not hasattr(input_ids, "shape")
+ or len(input_ids.shape) < 2
+ or getattr(device, "type", None) != "cuda"
+ ):
+ return "graph", None
+ bsz, prompt = int(bsz or input_ids.shape[0]), int(input_ids.shape[1])
+ rows, kv, work = _call_bytes(model.config, bsz, prompt, dtype, increment, max_len)
+ if rows is None:
+ return "graph", None
+ key = _shape_key(bsz, prompt, max_len)
+ measured = model.__dict__.get(NEEDS, {}).get(key)
+ # A measurement can only raise the estimate; it includes a reused cache, so reuse is subtracted once.
+ estimate = CALIB_WORK * (kv + work) + CALIB_GRAPH_KV * kv
+ full = estimate if measured is None else max(estimate, MEASURED_MARGIN * measured)
+ kept, reused = _kept_buffers(model, (bsz, dtype, device), rows)
+ if eager:
+ # Eager decode uses fresh buffers: no graph extra, no kept-cache credit.
+ full, reused = CALIB_WORK * (kv + work), 0
+ # Cap credit at this call's KV: a larger kept cache cannot also cover prefill.
+ reused = min(reused, kv)
+ need = full - reused
+ free = _free_bytes(device)
+ if free < need and (kept or any(state.graphs for state in _states(model))):
+ release(model)
+ _empty_cache()
+ free = _free_bytes(device)
+ reused = 0
+ need = full
+ if eager:
+ return "eager", None
+ if free < need:
+ EAGER_UNFIT += 1
+ return "eager", None
+ return "graph", (key, device, *_alloc_stats(device), reused)
+
+
+def record_need(model, token):
+ """Store the largest graphed footprint per shape (peak delta + reused kept cache). The peak is never
+ reset, so a call that did not raise it is skipped."""
+ if token is None:
+ return
+ key, device, start, peak_before, reused = token
+ peak = _alloc_stats(device)[1]
+ if peak > peak_before:
+ needs = model.__dict__.setdefault(NEEDS, {})
+ needs[key] = max(needs.get(key, 0), peak - start + reused)
+
+
+def decode_batch(
+ bsz,
+ kwargs,
+ generation_config = None,
+):
+ """Batch size generate() decodes at: prompts x num_beams, else x num_return_sequences; kwargs win."""
+ config = kwargs.get("generation_config", None) or generation_config
+
+ def get(name):
+ value = kwargs.get(name, None)
+ if value is None:
+ value = getattr(config, name, None)
+ try:
+ return max(1, int(value or 1))
+ except (TypeError, ValueError):
+ return 1
+
+ beams = get("num_beams")
+ return bsz * (beams if beams > 1 else get("num_return_sequences"))
+
+
+def decode_batch_used(model):
+ """Batch size of the cache the last graph-mode decode ran on, or None."""
+ for module in model.modules():
+ d = module.__dict__
+ store = d.get("_unsloth_graph_buffers")
+ paged = d.get("paged_attention")
+ if store and paged is not None:
+ for key, bufs in store.items():
+ if bufs[0] is paged:
+ return key[0]
+ return None
+
+
+_OOM_WARNED = False
+EAGER_REDOS = 0
+HELD = "_unsloth_decode_held"
+
+
+def generate_with_fallback(model, generate, graphs, args, kwargs):
+ """On a graph-mode OOM, free graphs and redo once eagerly; caller streamer / past_key_values /
+ stopping_criteria / logits_processor may hold failed-attempt state, so those re-raise."""
+ global _OOM_WARNED, EAGER_REDOS
+ try:
+ return generate(*args, **kwargs)
+ except torch.cuda.OutOfMemoryError:
+ if (
+ not graphs
+ or not graphable(model)
+ or kwargs.get("streamer", None) is not None
+ or kwargs.get("past_key_values", None) is not None
+ or kwargs.get("stopping_criteria", None)
+ or kwargs.get("logits_processor", None)
+ ):
+ raise
+ if not _OOM_WARNED:
+ _OOM_WARNED = True
+ import warnings
+ warnings.warn(
+ "Unsloth: generate() ran out of GPU memory with CUDA-graph decode; freed the decode graphs "
+ "and buffers and decoded this call eagerly (UNSLOTH_DECODE_CUDA_GRAPHS=0 always decodes eagerly).",
+ stacklevel = 3,
+ )
+ release_for_training(model)
+ _empty_cache()
+ EAGER_REDOS += 1
+ with eager_scope():
+ return generate(*args, **kwargs)
+
+
+def keep_only(model, bsz = None):
+ """Drop decode buffers and graphs of every other batch size; this one's graphs stay valid. The batch
+ size decode actually ran at wins over `bsz`."""
+ used = decode_batch_used(model)
+ if used is not None:
+ bsz = used
+ if bsz is None:
+ return False
+ dropped = False
+ for module in model.modules():
+ store = module.__dict__.get("_unsloth_graph_buffers")
+ if store:
+ for key in [k for k in store if k[0] != bsz]:
+ bufs = store.pop(key)
+ dropped = True
+ if module.__dict__.get("paged_attention") is bufs[0]:
+ for name in _BUFFER_NAMES:
+ module.__dict__.pop(name, None)
+ state = module.__dict__.get("_unsloth_decode_graphs")
+ if state is not None:
+ for key in [k for k in state.graphs if k[0] != bsz]:
+ del state.graphs[key]
+ dropped = True
+ if not state.graphs:
+ state.pool = None
+ return dropped
+
+
+def release_if_training(model):
+ """Run from the training forward: training can resume without for_training() (e.g. sampling in a
+ trainer callback), so this is the only place guaranteed to free kept graphs."""
+ if model.__dict__.pop(HELD, False):
+ release_for_training(model)
+ return True
+ return False
+
+
def static():
"""The _StepStatics of a graph-mode step running on this thread, else None (eager)."""
if torch.compiler.is_compiling():
@@ -230,6 +550,13 @@ def alloc_decode_buffers(
key = (bsz, dtype, device)
bufs = store.get(key)
if bufs is None or bufs[0].shape[0] < rows:
+ # Evict before allocating so old and new caches are never both held (peak memory).
+ if bufs is not None or len(store) >= 2:
+ store.clear()
+ for name in _BUFFER_NAMES:
+ attn.__dict__.pop(name, None)
+ bufs = None
+ BUFFER_EPOCH += 1
with _no_inference_tensors():
# Zeroed: a captured step reads every row of its bucket, and a masked row still
# multiplies its V by 0, which is NaN for uninitialised NaN / inf bit patterns.
@@ -246,11 +573,6 @@ def alloc_decode_buffers(
else:
temp_O = temp_QA[1][:, :, :hidden_size]
scores = torch.empty((bsz, n_heads, 1, rows), dtype = dtype, device = device)
- # Graphs reading a replaced / evicted buffer are dropped (BUFFER_EPOCH); a new batch size
- # beside the ones kept leaves their graphs valid.
- if bufs is not None or len(store) >= 2:
- store.clear()
- BUFFER_EPOCH += 1
bufs = store[key] = (paged, temp_QA, temp_KV, RH_Q, temp_O, scores)
elif FULL:
# A reused buffer still holds the last generation's rows: a whole-step graph reads them
@@ -303,6 +625,13 @@ def start_generation(max_len):
MAX_LEN_HINT = max(0, int(max_len or 0))
+def worth_capturing(kv_len):
+ """CAPTURE_MIN_STEPS steps left at kv_len; unknown length (0): yes."""
+ if MAX_LEN_HINT <= 0:
+ return True
+ return MAX_LEN_HINT - kv_len + 1 >= CAPTURE_MIN_STEPS
+
+
def end_generation():
global MAX_LEN_HINT
MAX_LEN_HINT = 0
@@ -324,6 +653,7 @@ class _ModelGraphs:
self.captures = 0
self.replays = 0
self.eager_steps = 0
+ self.short_skips = 0
def clear(self):
self.graphs.clear()
@@ -373,6 +703,7 @@ def release(model):
found = False
for module in model.modules():
d = module.__dict__
+ d.pop(HELD, None)
state = d.get("_unsloth_decode_graphs")
if state is not None:
found = True
@@ -702,6 +1033,10 @@ def decode_step(
autocast = _autocast_state()
key = (bsz, device, autocast, torch.is_inference_mode_enabled(), span)
step = state.graphs.get(key)
+ if step is None and not worth_capturing(kv_len):
+ state.eager_steps += 1
+ state.short_skips += 1
+ return None
if step is None:
if len(state.graphs) >= (MAX_GRAPHS if span is None else MAX_FULL_GRAPHS):
state.graphs.clear()
@@ -844,5 +1179,6 @@ def stats(model):
"replays": state.replays,
"graphs": len(state.graphs),
"eager_steps": state.eager_steps,
+ "short_skips": state.short_skips,
"disabled": state.disabled,
}
diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py
index 65e7e1aeef2..fdf7b6e43cc 100644
--- a/unsloth/models/llama.py
+++ b/unsloth/models/llama.py
@@ -2018,6 +2018,9 @@ def CausalLM_fast_forward(fast_forward_inference):
) -> Union[Tuple, CausalLMOutputWithPast]:
# Not a named parameter: HF's find_labels would make it a required eval label.
shift_labels = kwargs.pop("shift_labels", None)
+ # Training may resume without for_training() (trainer callback sampling): free kept graphs here.
+ if _decode_graphs.HELD in self.__dict__ and self.training and torch.is_grad_enabled():
+ _decode_graphs.release_if_training(self)
past_key_values = _cache_as_legacy_tuple(past_key_values)
if past_key_values is not None and len(past_key_values) == 0:
past_key_values = None
@@ -2986,6 +2989,30 @@ def _generation_max_len(self, args, kwargs):
return 0
+def _generation_plan_len(self, args, kwargs):
+ """_generation_max_len, or a max_length the caller's own GenerationConfig sets explicitly (the call can grow
+ to it; the model's default config is not a request). Sizes both the plan and the cache growth cap."""
+ max_len = _generation_max_len(self, args, kwargs)
+ if max_len or kwargs.get("max_new_tokens", None) is not None:
+ return max_len
+ try:
+ config = kwargs.get("generation_config", None)
+ explicit = config.to_diff_dict() if config is not None else {}
+ if (
+ explicit.get("max_new_tokens", None) is None
+ and explicit.get("max_length", None) is not None
+ ):
+ return int(explicit["max_length"])
+ except Exception:
+ pass
+ return max_len
+
+
+def _generation_input_ids(args, kwargs):
+ ids = kwargs.get("input_ids", kwargs.get("inputs", args[0] if args else None))
+ return ids if ids is not None else kwargs.get("inputs_embeds", None)
+
+
def unsloth_fast_generate(self, *args, **kwargs):
restore_training_mode = self.training
# Snapshot the real GC mode (e.g. unsloth) before for_inference clears it, so restore does not collapse it to a bool
@@ -3064,20 +3091,48 @@ def unsloth_fast_generate(self, *args, **kwargs):
kwargs["pad_token_id"] = kwargs.pop("pad_token_id", model_eos_token_id)
- # Decided once per call (env / RL generation scope), so the prefill and every decode step agree.
- graphs = _decode_graphs.enabled()
- if graphs:
- _decode_graphs.start_generation(_generation_max_len(self, args, kwargs))
- # Mixed precision autocast.
- try:
- with (
- _get_inference_mode_context_manager(self),
- torch.autocast(device_type = DEVICE_TYPE_TORCH, dtype = dtype),
- ):
- output = self._old_generate(*args, **kwargs)
- finally:
+ # Decided once per call so the prefill and every decode step agree.
+ with _decode_graphs.generate_scope():
+ graphs = _decode_graphs.enabled()
+ plan, token = "graph", None
if graphs:
- _decode_graphs.end_generation()
+ plan_len = _generation_plan_len(self, args, kwargs)
+ try:
+ ids = _generation_input_ids(args, kwargs)
+ bsz = _decode_graphs.decode_batch(
+ int(ids.shape[0]), kwargs, getattr(self, "generation_config", None)
+ )
+ short = _decode_graphs.short_call(self, plan_len, int(ids.shape[1]), bsz)
+ if short and _decode_graphs.HELD not in self.__dict__:
+ plan, token = "eager", None
+ else:
+ plan, token = _decode_graphs.plan_call(
+ self, ids, dtype, KV_CACHE_INCREMENT, plan_len, bsz = bsz, eager = short,
+ ) # fmt: skip
+ if short:
+ plan, token = "eager", None
+ except Exception:
+ plan, token = "graph", None
+ if plan == "eager":
+ graphs = False
+ else:
+ _decode_graphs.start_generation(plan_len)
+ redos = _decode_graphs.EAGER_REDOS
+ # Mixed precision autocast.
+ try:
+ with (
+ _decode_graphs.eager_scope() if plan == "eager" else contextlib.nullcontext(),
+ _get_inference_mode_context_manager(self),
+ torch.autocast(device_type = DEVICE_TYPE_TORCH, dtype = dtype),
+ ):
+ output = _decode_graphs.generate_with_fallback(
+ self, self._old_generate, graphs, args, kwargs
+ )
+ finally:
+ if graphs:
+ _decode_graphs.end_generation()
+ if graphs and _decode_graphs.EAGER_REDOS == redos:
+ _decode_graphs.record_need(self, token)
if graphs and restore_training_mode:
# Back to training: the next optimizer step invalidates the graphs anyway; free them and the
# decode buffers for training.
@@ -3086,6 +3141,16 @@ def unsloth_fast_generate(self, *args, **kwargs):
# The returned cache views the persistent decode buffers: hand them over, so a later
# generate() allocates its own instead of overwriting a cache the caller kept.
_decode_graphs.release(self)
+ elif graphs:
+ # keep_only uses the expanded decode batch.
+ ids = _generation_input_ids(args, kwargs)
+ expected = None
+ if hasattr(ids, "shape") and len(ids.shape) >= 2:
+ expected = _decode_graphs.decode_batch(
+ int(ids.shape[0]), kwargs, getattr(self, "generation_config", None)
+ )
+ _decode_graphs.keep_only(self, expected)
+ self.__dict__[_decode_graphs.HELD] = True
if restore_training_mode:
FastLlamaModel.for_training(
@@ -5645,6 +5710,7 @@ class FastLlamaModel:
"Unsloth: I think you're passing a tokenizer, not the model to for_training!"
)
_tensor_parallel_for_training(model)
+ _decode_graphs.release_for_training(model)
# Mode last, as model.train() was: a custom train() keeps the final say over the chain below.
checkpointing, apply_training_mode = switch_training_mode(model, True, defer = True)
use_gradient_checkpointing = resolve_training_gradient_checkpointing(