Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Decode with CUDA graphs by default in plain generate(), with memory release and an eager fallback (#16691)
Daniel Handb82c05511c1
diff --git a/tests/test_decode_cuda_graphs.py b/tests/test_decode_cuda_graphs.pyindex b898bfc67ac..c6decab0e9f 100644--- a/tests/test_decode_cuda_graphs.py+++ b/tests/test_decode_cuda_graphs.py@@ -203,6 +203,8 @@ def test_lora_change_between_generations_recaptures(name, stale_control, monkeyppinned = G._state(_causal(model)).fingerprintmonkeypatch.setattr(G, "_fingerprint", lambda *a: pinned)params = [p for n, p in model.named_parameters() if "lora_" in n and p.requires_grad]+ # for_training() frees the graphs; keep them here, so the fingerprint alone must catch the edits.+ monkeypatch.setattr(G, "release_for_training", lambda model: None)FastLanguageModel.for_training(model)opt = torch.optim.SGD(params, lr = 1.0)for p in params:diff --git a/tests/test_decode_cuda_graphs_default.py b/tests/test_decode_cuda_graphs_default.pyindex 40b58682180..4c834ddd8b0 100644--- a/tests/test_decode_cuda_graphs_default.py+++ b/tests/test_decode_cuda_graphs_default.py@@ -1,10 +1,11 @@# SPDX-License-Identifier: AGPL-3.0-only# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.-"""When CUDA-graph decode runs (decided per call): UNSLOTH_DECODE_CUDA_GRAPHS unset = off for plain-model.generate(), on inside an RL trainer's generation (rl_generation(), entered by rl.py's-unwrap_model_for_generation); =1 on everywhere, =0 off everywhere. Returning to training releases-the graphs and every decode buffer."""+"""When CUDA-graph decode runs (decided per call): UNSLOTH_DECODE_CUDA_GRAPHS unset = on inside+Unsloth's generate() (generate_scope()) and an RL trainer's generation (rl_generation(), entered by+rl.py's unwrap_model_for_generation), off for a hand-written decode loop outside both; =1 on+everywhere, =0 off everywhere. Returning to training (or for_training) releases the graphs and every+decode buffer."""import pytest@@ -129,39 +130,61 @@ def _gen(@rl_default-def test_plain_generate_off_rl_generate_on_and_released(runtime):+def test_plain_generate_on_by_default_bitwise_and_released(runtime):+ if not G._STUDIO_DEFAULT_OK:+ pytest.skip("the plain generate() default stays eager on native Windows")model, tok = _load()FastLanguageModel.for_inference(model)+ runtime.setenv(ENV, "0")eager = _gen(model, tok)assert _Stats(model)().get("replays", 0) == 0assert _graph_buffers(model) == 0- with G.rl_generation():- graph = _gen(model, tok)+ runtime.delenv(ENV)+ graph = _gen(model, tok)stats = _Stats(model)()assert stats["disabled"] is None and stats["replays"] == 24 - 2, statsassert torch.equal(eager[0], graph[0]) and torch.equal(eager[1], graph[1])+ with G.rl_generation():+ assert torch.equal(_gen(model, tok)[1], eager[1])- # =0 wins inside the RL scope; =1 turns plain generate on.runtime.setenv(ENV, "0")replays = _Stats(model)()["replays"]with G.rl_generation():assert torch.equal(_gen(model, tok)[1], eager[1])assert _Stats(model)()["replays"] == replays- runtime.setenv(ENV, "1")- assert torch.equal(_gen(model, tok)[1], eager[1])- assert _Stats(model)()["replays"] > replays- # generate() called on a model in training mode returns to training: everything released.+ # for_training frees what an inference-mode generate() kept (a returned cache would release them).runtime.delenv(ENV)+ enc = tok(PROMPTS, return_tensors = "pt", padding = True).to("cuda")+ model.generate(**enc, max_new_tokens = 8, do_sample = False, pad_token_id = tok.pad_token_id)+ assert _graph_buffers(model) > 0FastLanguageModel.for_training(model)- with G.rl_generation():- _gen(model, tok)+ assert _graph_buffers(model) == 0 and _decode_buffers(model) == 0+ assert _Stats(model)()["graphs"] == 0++ # generate() called on a model in training mode returns to training: everything released.+ _gen(model, tok)assert model.trainingassert _graph_buffers(model) == 0 and _decode_buffers(model) == 0assert _Stats(model)()["graphs"] == 0+def test_generate_scope_turns_graphs_on(runtime):+ runtime.setattr(G, "_DEFAULT_OK", True)+ runtime.setattr(G, "_STUDIO_DEFAULT_OK", True)+ assert G.enabled() is False+ with G.generate_scope():+ assert G.enabled() is True+ runtime.setenv(ENV, "0")+ assert G.enabled() is False+ runtime.delenv(ENV)+ assert G.enabled() is False+ runtime.setattr(G, "_STUDIO_DEFAULT_OK", False) # native Windows+ with G.generate_scope():+ assert G.enabled() is False++@rl_defaultdef test_grpo_generation_uses_graphs_and_releases(runtime, tmp_path):"""A real GRPOTrainer step: its generate() runs graphed with the variable unset, completions@@ -203,3 +226,106 @@ def test_grpo_generation_uses_graphs_and_releases(runtime, tmp_path):assert len(seen["replays"]) == 2 and all(r > 0 for r in seen["replays"]), seenassert seen["buffers_at_loss"] == [(0, 0), (0, 0)], seenassert _Stats(model)()["disabled"] is None+++@rl_default+def test_out_of_memory_in_the_graph_path_redoes_the_call_eagerly(runtime):+ """A real CUDA allocator OOM raised only while graphs are on (the first graph-mode buffer+ allocation asks for 1 PiB): generate() frees the graphs, redoes the call eagerly once, counts it,+ warns once, and returns the eager tokens and logits."""+ if not G._STUDIO_DEFAULT_OK:+ pytest.skip("the plain generate() default stays eager on native Windows")+ model, tok = _load()+ FastLanguageModel.for_inference(model)+ runtime.setenv(ENV, "0")+ eager = _gen(model, tok)+ runtime.delenv(ENV)+ original = G.alloc_decode_buffers++ def allocate(*args, **kwargs):+ if G.enabled():+ torch.empty(1 << 50, dtype = torch.uint8, device = "cuda")+ return original(*args, **kwargs)++ runtime.setattr(G, "alloc_decode_buffers", allocate)+ runtime.setattr(G, "_OOM_WARNED", False)+ redos = G.EAGER_REDOS+ with pytest.warns(UserWarning, match = "ran out of GPU memory"):+ redone = _gen(model, tok)+ assert G.EAGER_REDOS == redos + 1+ assert torch.equal(eager[0], redone[0]) and torch.equal(eager[1], redone[1])+ assert _graph_buffers(model) == 0+ # Without the forced OOM the next call is graphed again.+ runtime.setattr(G, "alloc_decode_buffers", original)+ replays = _Stats(model)().get("replays", 0)+ assert torch.equal(_gen(model, tok)[1], eager[1])+ assert _Stats(model)()["replays"] > replays and G.EAGER_REDOS == redos + 1+++@rl_default+def test_training_forward_without_for_training_frees_generate_memory(runtime):+ """Sampling in a trainer callback: generate() on an inference-mode model, then training resumes+ with model.train() only. The first training forward frees the graphs and decode caches."""+ if not G._STUDIO_DEFAULT_OK:+ pytest.skip("the plain generate() default stays eager on native Windows")+ model, tok = _load()+ FastLanguageModel.for_inference(model)+ enc = tok(PROMPTS, return_tensors = "pt", padding = True).to("cuda")+ model.generate(**enc, max_new_tokens = 8, do_sample = False, pad_token_id = tok.pad_token_id)+ assert _graph_buffers(model) > 0+ model.train()+ out = model(**enc, labels = enc.input_ids)+ out.loss.backward()+ assert _graph_buffers(model) == 0 and _decode_buffers(model) == 0+ assert _Stats(model)()["graphs"] == 0+++def _gen_tokens(model, tok, prompts, new):+ enc = tok(prompts, return_tensors = "pt", padding = True).to("cuda")+ out = model.generate(+ **enc, max_new_tokens = new, min_new_tokens = new, do_sample = False,+ pad_token_id = tok.pad_token_id,+ ) # fmt: skip+ return out.cpu()+++@rl_default+def test_short_calls_decode_eagerly_and_keep_the_kept_graph(runtime):+ """Short calls decode eagerly unless their batch is kept; tokens equal eager."""+ if not G._STUDIO_DEFAULT_OK:+ pytest.skip("the plain generate() default stays eager on native Windows")+ model, tok = _load()+ FastLanguageModel.for_inference(model)+ one, two = PROMPTS[:1], PROMPTS+ runtime.setenv(ENV, "0")+ eager = {+ (p, n): _gen_tokens(model, tok, ps, n)+ for p, ps in (("one", one), ("two", two))+ for n in (4, 24)+ }+ runtime.delenv(ENV)++ def check(p, ps, n):+ assert torch.equal(_gen_tokens(model, tok, ps, n), eager[(p, n)]), (p, n)++ def batches():+ return {k[0] for m in model.modules() for k in m.__dict__.get("_unsloth_graph_buffers", {})}++ check("two", two, 24)+ before = _Stats(model)()+ assert before["captures"] > 0 and batches() == {2}, (before, batches())+ check("one", one, 4)+ after = _Stats(model)()+ assert after["captures"] == before["captures"] and after["replays"] == before["replays"], (+ before,+ after,+ )+ assert batches() == {2}, batches()+ check("two", two, 4)+ short = _Stats(model)()+ assert short["captures"] == before["captures"] and short["replays"] > before["replays"], (+ before,+ short,+ )+ check("two", two, 24)+ assert _Stats(model)()["captures"] == before["captures"], (before, _Stats(model)())diff --git a/tests/test_decode_cuda_graphs_studio.py b/tests/test_decode_cuda_graphs_studio.pyindex 21d272721c7..778ca4eb24d 100644--- a/tests/test_decode_cuda_graphs_studio.py+++ b/tests/test_decode_cuda_graphs_studio.py@@ -93,7 +93,7 @@ def test_studio_scope_is_off_on_native_windows(G, monkeypatch):@pytest.mark.skipif(not has_real_cuda(), reason = "CUDA only")-def test_studio_generate_graphed_plain_generate_eager():+def test_studio_generate_graphed_plain_generate_eager(monkeypatch):import unsloth # noqa: F401from unsloth import FastLanguageModelimport unsloth.models._decode_graphs as G@@ -118,13 +118,16 @@ def test_studio_generate_graphed_plain_generate_eager():def replays():return G.stats(model).get("replays", 0)+ # Plain generate() graphs by default too (generate_scope); =0 is the eager reference.+ monkeypatch.setenv("UNSLOTH_DECODE_CUDA_GRAPHS", "0")eager = gen()assert replays() == 0+ monkeypatch.delenv("UNSLOTH_DECODE_CUDA_GRAPHS")with G.studio_generation():graphed = gen()assert replays() == 24 - 2 and G.stats(model)["disabled"] is Noneassert torch.equal(eager[0], graphed[0]) and torch.equal(eager[1], graphed[1])- assert torch.equal(gen()[1], eager[1]) and replays() == 24 - 2+ assert torch.equal(gen()[1], eager[1]) and replays() == 2 * (24 - 2)G.release_for_training(model)assert G.stats(model)["graphs"] == 0assert not any("paged_attention" in m.__dict__ for m in model.modules())diff --git a/tests/test_decode_graphs_memory.py b/tests/test_decode_graphs_memory.pynew file mode 100644index 00000000000..fa465139b81--- /dev/null+++ b/tests/test_decode_graphs_memory.py@@ -0,0 +1,687 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""Graph decode memory planning and OOM fallback; CPU only, CUDA memory queries mocked."""++import importlib.util+import types+import warnings+from pathlib import Path++import pytest+import torch++ENV = "UNSLOTH_DECODE_CUDA_GRAPHS"+_SOURCE = Path(__file__).resolve().parents[1] / "unsloth" / "models" / "_decode_graphs.py"+CUDA0 = torch.device("cuda", 0)+GB = 1 << 30+++def G(monkeypatch):+ spec = importlib.util.spec_from_file_location("_unsloth_decode_graphs_memory", _SOURCE)+ module = importlib.util.module_from_spec(spec)+ spec.loader.exec_module(module)+ module.ENABLED = None+ module._DEFAULT_OK = True+ module._STUDIO_DEFAULT_OK = True+ monkeypatch.delenv(ENV, raising = False)+ module.emptied = 0++ def empty():+ module.emptied += 1++ module._empty_cache = empty+ return module+++class _Attn(torch.nn.Module):+ pass+++class _Model(torch.nn.Module):+ """Qwen3-4B-like dims; layer attention modules carry the graph-mode buffer store."""++ def __init__(self, layers = 2):+ super().__init__()+ self.config = types.SimpleNamespace(+ num_hidden_layers = 36, hidden_size = 2560, num_attention_heads = 32,+ num_key_value_heads = 8, head_dim = 128, intermediate_size = 9728, vocab_size = 151936,+ model_type = "qwen3",+ ) # fmt: skip+ self.attns = torch.nn.ModuleList(_Attn() for _ in range(layers))++ def keep(+ self,+ bsz,+ rows,+ dtype = torch.float16,+ ):+ for attn in self.attns:+ paged = torch.zeros((rows, 2, 1, 1, 1), dtype = dtype)+ store = attn.__dict__.setdefault("_unsloth_graph_buffers", {})+ store[(bsz, dtype, CUDA0)] = (paged, None, None, None, None, None)+ attn.paged_attention = paged++ def held(self):+ return len(+ {key for a in self.attns for key in a.__dict__.get("_unsloth_graph_buffers", {})}+ )+++def _ids(bsz, prompt):+ return types.SimpleNamespace(shape = (bsz, prompt), device = CUDA0)+++def _need(+ G,+ bsz,+ prompt,+ max_len = 0,+):+ rows, kv, work = G._call_bytes(_Model().config, bsz, prompt, torch.float16, 512, max_len)+ return G.CALIB_WORK * (kv + work) + G.CALIB_GRAPH_KV * kv+++def _plan(+ G,+ model,+ bsz,+ prompt,+ max_len = 0,+):+ return G.plan_call(model, _ids(bsz, prompt), torch.float16, 512, max_len)+++def test_plan_frees_kept_buffers_when_the_call_does_not_fit_beside_them(G):+ model = _Model()+ model.keep(1, 2816)+ model.keep(4, 2816)+ need = _need(G, 16, 2048, 2081)+ assert 11 * GB < need < 13 * GB+ free = iter([int(0.9 * need), int(1.1 * need)])+ G._free_bytes = lambda device: next(free)+ plan, token = _plan(G, model, 16, 2048, 2081)+ assert plan == "graph" and token is not None+ assert model.held() == 0 and G.emptied == 1 and G.EAGER_UNFIT == 0+ assert not any("paged_attention" in a.__dict__ for a in model.attns)+++def test_plan_holds_no_buffer_reference_when_it_empties_the_cache(G):+ import weakref++ model = _Model()+ model.keep(1, 2816)+ model.keep(4, 2816)+ model.keep(16, 64) # too small to reuse, still scanned+ refs = [+ weakref.ref(bufs[0])+ for a in model.attns+ for bufs in a.__dict__["_unsloth_graph_buffers"].values()+ ]+ alive_at_empty = []++ def empty():+ G.emptied += 1+ alive_at_empty.append(sum(r() is not None for r in refs))++ G._empty_cache = empty+ need = _need(G, 16, 2048, 2081)+ free = iter([int(0.9 * need), int(1.1 * need)])+ G._free_bytes = lambda device: next(free)+ assert _plan(G, model, 16, 2048, 2081)[0] == "graph"+ assert alive_at_empty == [0]+++def test_plan_runs_a_call_that_cannot_fit_eagerly_from_the_start(G):+ model = _Model()+ model.keep(4, 2816)+ G._free_bytes = lambda device: int(0.95 * _need(G, 16, 2048, 2081))+ plan, token = _plan(G, model, 16, 2048, 2081)+ assert (plan, token) == ("eager", None)+ # Kept state was freed first (the eager call needs the memory too), then the call was counted.+ assert model.held() == 0 and G.EAGER_UNFIT == 1+ assert _plan(G, model, 16, 2048, 2081)[0] == "eager" and G.EAGER_UNFIT == 2 and G.emptied == 1+++def test_plan_keeps_buffers_when_there_is_room(G):+ model = _Model()+ model.keep(1, 2816)+ G._free_bytes = lambda device: 20 * GB+ assert _plan(G, model, 16, 2048)[0] == "graph"+ assert model.held() == 1 and G.emptied == 0 and G.EAGER_UNFIT == 0+++def test_plan_counts_a_reusable_buffer_of_the_same_batch_size(G):+ model = _Model(layers = 1)+ rows, kv, work = G._call_bytes(model.config, 4, 128, torch.float16, 512)+ model.keep(4, rows)+ reused = model.attns[0]._unsloth_graph_buffers[(4, torch.float16, CUDA0)][0]+ G._free_bytes = (+ lambda device: int(_need(G, 4, 128) - reused.numel() * reused.element_size()) + 1+ )+ assert _plan(G, model, 4, 128)[0] == "graph"+ assert model.held() == 1+++def test_plan_lets_non_cuda_or_shapeless_inputs_through(G):+ model = _Model()+ G._free_bytes = lambda device: 0+ cpu_ids = types.SimpleNamespace(shape = (16, 2048), device = torch.device("cpu"))+ assert G.plan_call(model, cpu_ids, torch.float16, 512) == ("graph", None)+ assert G.plan_call(model, None, torch.float16, 512) == ("graph", None)+ assert G.EAGER_UNFIT == 0+++def test_measured_peak_can_only_raise_the_estimate(G):+ model = _Model()+ est = _need(G, 4, 2048, 2081)+ stats = {"alloc": 3 * GB, "peak": 3 * GB}+ G._alloc_stats = lambda device: (stats["alloc"], stats["peak"])+ G._free_bytes = lambda device: 20 * GB+ plan, token = _plan(G, model, 4, 2048, 2081)+ assert plan == "graph"+ stats["peak"] = 5 * GB # below the estimate+ G.record_need(model, token)+ key = G._shape_key(4, 2048, 2081)+ assert model.__dict__[G.NEEDS][key] == 2 * GB < est+ # A measurement can only raise the estimate.+ G._free_bytes = lambda device: int(est) - 1+ assert _plan(G, model, 4, 2048, 2081)[0] == "eager"+ G._free_bytes = lambda device: int(est) + 1+ assert _plan(G, model, 4, 2048, 2081)[0] == "graph"+ G._free_bytes = lambda device: 20 * GB+ plan, token = _plan(G, model, 4, 2048, 2081)+ stats["peak"] = 3 * GB + 6 * GB+ G.record_need(model, token)+ assert model.__dict__[G.NEEDS][key] == 6 * GB > est+ G._free_bytes = lambda device: int(G.MEASURED_MARGIN * 6 * GB) - 1+ assert _plan(G, model, 4, 2048, 2081)[0] == "eager"+ G._free_bytes = lambda device: int(G.MEASURED_MARGIN * 6 * GB) + 1+ assert _plan(G, model, 4, 2048, 2081)[0] == "graph"+ G.record_need(model, _plan(G, model, 1, 128)[1])+ assert G._shape_key(1, 128, 0) not in model.__dict__[G.NEEDS]+++def test_prompt_length_is_part_of_the_shape(G):+ # Same total-length bucket, different prompts: the key includes the prompt bucket.+ assert G._shape_key(4, 128, 4096) != G._shape_key(4, 2048, 4096)+ model = _Model()+ stats = {"alloc": 3 * GB, "peak": 3 * GB}+ G._alloc_stats = lambda device: (stats["alloc"], stats["peak"])+ G._free_bytes = lambda device: 30 * GB+ plan, token = _plan(G, model, 4, 128, 4096)+ stats["peak"] = 3 * GB + GB // 2+ G.record_need(model, token)+ long_need = _need(G, 4, 2048, 4096)+ G._free_bytes = lambda device: int(long_need) - 1+ assert _plan(G, model, 4, 2048, 4096)[0] == "eager"+++def test_an_early_stop_measurement_never_admits_a_full_length_call(G):+ model = _Model()+ est = _need(G, 4, 2048, 4096)+ stats = {"alloc": 3 * GB, "peak": 3 * GB}+ G._alloc_stats = lambda device: (stats["alloc"], stats["peak"])+ G._free_bytes = lambda device: 30 * GB+ plan, token = _plan(G, model, 4, 2048, 4096)+ stats["peak"] = 3 * GB + GB+ G.record_need(model, token)+ G._free_bytes = lambda device: int(G.MEASURED_MARGIN * GB) + 1+ assert _plan(G, model, 4, 2048, 4096)[0] == "eager"+ G._free_bytes = lambda device: int(est) + 1+ assert _plan(G, model, 4, 2048, 4096)[0] == "graph"+++def test_release_keeps_the_measured_needs(G):+ model = _Model()+ model.__dict__[G.NEEDS] = {(4, 2304, 2304): GB}+ model.keep(4, 2816)+ G.release_for_training(model)+ assert model.__dict__[G.NEEDS] == {(4, 2304, 2304): GB}+++def _oom_while_graphed(G, seen):+ def generate(*args, **kwargs):+ on = G.enabled()+ seen.append(on)+ if on:+ raise torch.cuda.OutOfMemoryError("CUDA out of memory (stand-in)")+ return "eager-output"++ return generate+++def test_oom_with_graphs_redoes_the_call_once_eagerly_and_warns_once(G):+ model = _Model()+ seen = []+ for _ in range(2):+ model.keep(1, 2816)+ with G.generate_scope():+ graphs = G.enabled()+ with warnings.catch_warnings(record = True) as caught:+ warnings.simplefilter("always")+ out = G.generate_with_fallback(model, _oom_while_graphed(G, seen), graphs, (), {})+ assert out == "eager-output"+ assert model.held() == 0+ with G.generate_scope():+ assert G.enabled() is True+ assert seen == [True, False, True, False]+ assert G._OOM_WARNED is True+++def test_oom_warning_is_emitted_once_per_process(G):+ model = _Model()+ messages = []+ for _ in range(3):+ with G.generate_scope():+ with warnings.catch_warnings(record = True) as caught:+ warnings.simplefilter("always")+ G.generate_with_fallback(model, _oom_while_graphed(G, []), True, (), {})+ messages += [+ str(w.message) for w in caught if "ran out of GPU memory" in str(w.message)+ ]+ assert len(messages) == 1+++ "graphs, kwargs",+ [+ (False, {}),+ (True, {"streamer": object()}),+ (True, {"past_key_values": object()}),+ (True, {"stopping_criteria": [object()]}),+ (True, {"logits_processor": [object()]}),+ ],+)+def test_oom_is_raised_when_a_redo_is_not_safe(G, graphs, kwargs):+ model = _Model()+ model.keep(1, 2816)++ def generate(*args, **kw):+ raise torch.cuda.OutOfMemoryError("CUDA out of memory (stand-in)")++ with pytest.raises(torch.cuda.OutOfMemoryError):+ G.generate_with_fallback(model, generate, graphs, (), kwargs)+ assert model.held() == 1+++def test_oom_in_the_eager_redo_propagates(G):+ model = _Model()+ calls = []++ def generate(*args, **kw):+ calls.append(G.enabled())+ raise torch.cuda.OutOfMemoryError("CUDA out of memory (stand-in)")++ with G.generate_scope():+ with pytest.raises(torch.cuda.OutOfMemoryError):+ G.generate_with_fallback(model, generate, True, (), {})+ assert calls == [True, False]+++def test_eager_scope_beats_the_variable_and_the_scopes(G, monkeypatch):+ monkeypatch.setenv(ENV, "1")+ with G.rl_generation(), G.eager_scope():+ assert G.enabled() is False+ assert G.enabled() is True+++def _with_graphs(G, model, *bszs):+ state = G._state(model)+ for bsz in bszs:+ model.keep(bsz, 2816)+ state.graphs[(bsz, CUDA0, (False, None), True, None)] = object()+ state.pool = object()+ return state+++def test_keep_only_drops_other_batch_sizes_and_keeps_this_ones_graphs(G):+ model = _Model()+ state = _with_graphs(G, model, 1, 4)+ epoch = G.BUFFER_EPOCH+ assert G.keep_only(model, 4) is True+ assert {k for a in model.attns for k in a._unsloth_graph_buffers} == {(4, torch.float16, CUDA0)}+ assert [k[0] for k in state.graphs] == [4] and state.pool is not None+ # The kept graphs still read valid buffers: no epoch bump, so the next call of this shape replays.+ assert G.BUFFER_EPOCH == epoch+ assert all(+ a.paged_attention is a._unsloth_graph_buffers[(4, torch.float16, CUDA0)][0]+ for a in model.attns+ )+ assert G.keep_only(model, 4) is False+++def test_keep_only_never_evicts_the_cache_decode_just_used(G):+ model = _Model()+ _with_graphs(G, model, 4, 1) # decode ran at bs 1+ G.keep_only(model, 4)+ assert {k for a in model.attns for k in a._unsloth_graph_buffers} == {(1, torch.float16, CUDA0)}+ assert all(+ a.paged_attention is a._unsloth_graph_buffers[(1, torch.float16, CUDA0)][0]+ for a in model.attns+ )+++def test_training_forward_releases_what_a_graphed_generate_kept(G):+ model = _Model()+ _with_graphs(G, model, 4)+ assert G.release_if_training(model) is False+ assert model.held() == 1+ model.__dict__[G.HELD] = True+ assert G.release_if_training(model) is True+ assert model.held() == 0 and G.HELD not in model.__dict__+ assert G._state(model).graphs == {}+ assert G.release_if_training(model) is False+++def test_release_clears_the_held_mark(G):+ model = _Model()+ _with_graphs(G, model, 1)+ model.__dict__[G.HELD] = True+ G.release_for_training(model)+ assert G.HELD not in model.__dict__+++def test_eager_redo_is_counted(G):+ model = _Model()+ assert G.EAGER_REDOS == 0+ with G.generate_scope():+ G.generate_with_fallback(model, _oom_while_graphed(G, []), True, (), {})+ G.generate_with_fallback(model, lambda: "ok", True, (), {})+ assert G.EAGER_REDOS == 1+++def test_causal_lm_forward_calls_the_training_release():+ """The hook lives in llama.py's _CausalLM_fast_forward (importing unsloth needs an accelerator,+ so this reads the source): a training forward with grad on releases what generate() kept."""+ import ast++ source = (_SOURCE.parent / "llama.py").read_text(encoding = "utf-8")+ tree = ast.parse(source)+ fwd = next(+ n+ for n in ast.walk(tree)+ if isinstance(n, ast.FunctionDef) and n.name == "_CausalLM_fast_forward"+ )+ hooks = [+ n for n in ast.walk(fwd)+ if isinstance(n, ast.If) and "release_if_training" in ast.unparse(n)+ and "self.training" in ast.unparse(n.test) and "is_grad_enabled" in ast.unparse(n.test)+ ] # fmt: skip+ assert hooks, "training forward does not free graphs a generate() kept"+ gen = next(+ n+ for n in ast.walk(tree)+ if isinstance(n, ast.FunctionDef) and n.name == "unsloth_fast_generate"+ )+ text = ast.unparse(gen)+ assert "keep_only" in text and "HELD" in text+ # A call the plan says cannot fit graphed runs eagerly from the start; graphed calls record their peak.+ assert "plan_call" in text and "eager_scope" in text and "record_need" in text+ # Plan and keep_only use the expanded decode batch.+ assert text.count("decode_batch(") >= 2+++def test_measured_need_includes_a_reused_cache_and_is_discounted_once(G):+ # The stored footprint includes a reused cache, so the reuse is subtracted once.+ model = _Model(layers = 1)+ rows, kv, work = G._call_bytes(model.config, 4, 2048, torch.float16, 512, 2081)+ model.keep(4, rows)+ reused = model.attns[0]._unsloth_graph_buffers[(4, torch.float16, CUDA0)][0]+ reused_bytes = reused.numel() * reused.element_size()+ del reused+ stats = {"alloc": 3 * GB, "peak": 3 * GB}+ G._alloc_stats = lambda device: (stats["alloc"], stats["peak"])+ G._free_bytes = lambda device: 20 * GB+ plan, token = _plan(G, model, 4, 2048, 2081)+ assert plan == "graph"+ stats["peak"] = 11 * GB+ G.record_need(model, token)+ key = G._shape_key(4, 2048, 2081)+ total = 8 * GB + reused_bytes+ assert model.__dict__[G.NEEDS][key] == total+ assert G.MEASURED_MARGIN * total > _need(G, 4, 2048, 2081)+ need = G.MEASURED_MARGIN * total - reused_bytes+ G._free_bytes = lambda device: int(need) + 1+ assert _plan(G, model, 4, 2048, 2081)[0] == "graph" and model.held() == 1+ free = iter([int(need) - 1, int(G.MEASURED_MARGIN * total) - 1])+ G._free_bytes = lambda device: next(free)+ assert _plan(G, model, 4, 2048, 2081)[0] == "eager" and model.held() == 0+++def test_decode_batch_follows_generate_expansion(G):+ cfg = types.SimpleNamespace(num_beams = 1, num_return_sequences = 1)+ assert G.decode_batch(3, {}, cfg) == 3+ assert G.decode_batch(3, {"num_return_sequences": 2, "do_sample": True}, cfg) == 6+ assert G.decode_batch(3, {"num_beams": 4, "num_return_sequences": 2}, cfg) == 12+ beams = types.SimpleNamespace(num_beams = 2, num_return_sequences = 1)+ assert G.decode_batch(3, {}, beams) == 6+ assert G.decode_batch(3, {"generation_config": beams}, cfg) == 6+ assert G.decode_batch(3, {}, None) == 3+++def test_plan_sizes_the_expanded_batch(G):+ model = _Model()+ need = _need(G, 4, 2048, 2081)+ G._free_bytes = lambda device: int(need) - 1+ # 2 prompts x num_return_sequences 2 decode at bs 4.+ assert G.plan_call(model, _ids(2, 2048), torch.float16, 512, 2081, bsz = 4)[0] == "eager"+ G._free_bytes = lambda device: int(need) + 1+ plan, token = G.plan_call(model, _ids(2, 2048), torch.float16, 512, 2081, bsz = 4)+ assert plan == "graph" and token[0] == G._shape_key(4, 2048, 2081)+++def test_keep_only_keeps_the_batch_size_decode_used(G):+ model = _Model()+ state = _with_graphs(G, model, 1, 4) # decoded at bs 4+ assert G.decode_batch_used(model) == 4+ G.keep_only(model, None)+ assert {k for a in model.attns for k in a._unsloth_graph_buffers} == {(4, torch.float16, CUDA0)}+ assert [k[0] for k in state.graphs] == [4]++[email protected]("model_type", ["mistral", "qwen2", "gemma", "gemma2"])+def test_a_model_type_that_never_graphs_keeps_eager_oom_and_planning(G, model_type):+ model = _Model()+ model.config.model_type = model_type+ model.keep(1, 2816)+ calls = []++ def generate(*args, **kw):+ calls.append(G.enabled())+ raise torch.cuda.OutOfMemoryError("CUDA out of memory (stand-in)")++ with G.generate_scope():+ with warnings.catch_warnings(record = True) as caught:+ warnings.simplefilter("always")+ with pytest.raises(torch.cuda.OutOfMemoryError):+ G.generate_with_fallback(model, generate, G.enabled(), (), {})+ assert calls == [True]+ assert not [w for w in caught if "ran out of GPU memory" in str(w.message)]+ assert G.EAGER_REDOS == 0 and model.held() == 1+ G._free_bytes = lambda device: 0+ assert G.plan_call(model, _ids(16, 2048), torch.float16, 256, 4096) == ("graph", None)+ assert G.EAGER_UNFIT == 0 and model.held() == 1 and G.emptied == 0+++ "max_len, prompt, replays",+ [(0, 32, True), (33, 32, False), (34, 32, False), (35, 32, True), (288, 32, True)],+)+def test_a_call_too_short_to_replay_a_step_is_known(G, max_len, prompt, replays):+ assert G.can_replay(max_len, prompt) is replays+++def _generate_fn():+ import ast+ tree = ast.parse((_SOURCE.parent / "llama.py").read_text(encoding = "utf-8"))+ return next(+ n+ for n in ast.walk(tree)+ if isinstance(n, ast.FunctionDef) and n.name == "unsloth_fast_generate"+ )+++def test_generate_runs_a_short_call_eagerly_when_nothing_is_kept():+ """Reads llama.py: importing unsloth needs an accelerator."""+ import ast++ gen = _generate_fn()+ assert "short_call" in ast.unparse(gen)+ guards = [+ n for n in ast.walk(gen)+ if isinstance(n, ast.If) and "short" in ast.unparse(n.test) and "HELD" in ast.unparse(n.test)+ ] # fmt: skip+ assert guards, "a short call is not routed eagerly"+ body = ast.unparse(guards[0].body[0]) if guards[0].body else ""+ assert "'eager'" in body or '"eager"' in body+ assert "plan_call" not in ast.unparse(guards[0].body[0])+++def test_generate_runs_a_short_call_beside_kept_graphs_eagerly():+ import ast++ text = ast.unparse(_generate_fn())+ assert "eager=short" in text+ assert "if short:\n" in text and text.index("eager=short") < text.index("if short:\n")+++def _keep_graph(model, bsz):+ state = model.__dict__.setdefault("_unsloth_decode_graphs", types.SimpleNamespace(graphs = {}))+ state.graphs[(bsz, CUDA0, None, True, None)] = object()+++ "new, kept, short",+ [+ (1, None, True), (2, None, True), (2, 1, True), (3, None, True), (6, None, True),+ (6, 4, True), (7, None, False), (8, None, False), (32, None, False), (3, 1, False),+ (6, 1, False),+ ],+) # fmt: skip+def test_a_short_call_decodes_eagerly_unless_a_graph_of_its_batch_is_kept(G, new, kept, short):+ model = _Model()+ if kept is not None:+ _keep_graph(model, kept)+ assert G.CAPTURE_MIN_STEPS == 6+ assert G.short_call(model, 32 + new, 32, 1) is short+ assert G.short_call(model, 0, 32, 1) is False+++def test_a_short_call_matches_the_capture_break_even(G):+ model = _Model()+ for new in range(3, 40):+ G.start_generation(32 + new)+ try:+ assert G.short_call(model, 32 + new, 32, 1) is (not G.worth_capturing(32 + 2))+ finally:+ G.end_generation()+++def test_a_short_call_beside_kept_graphs_only_makes_room(G):+ model = _Model()+ model.keep(4, 2816)+ G._free_bytes = lambda device: 100 * GB+ assert G.plan_call(model, _ids(1, 32), torch.float16, 512, 36, eager = True) == ("eager", None)+ assert model.held() == 1 and G.EAGER_UNFIT == 0+ G._free_bytes = lambda device: 0+ assert G.plan_call(model, _ids(1, 32), torch.float16, 512, 36, eager = True) == ("eager", None)+ assert model.held() == 0 and G.emptied == 1 and G.EAGER_UNFIT == 0+++def test_capture_only_when_enough_steps_are_left(G):+ G.start_generation(32 + 8)+ try:+ assert G.CAPTURE_MIN_STEPS == 6+ assert G.worth_capturing(35) is True+ assert G.worth_capturing(36) is False+ G.start_generation(32 + 4)+ assert not any(G.worth_capturing(kv) for kv in range(33, 37))+ finally:+ G.end_generation()+ assert G.worth_capturing(10_000) is True+++def test_decode_step_skips_only_a_new_capture():+ import ast++ tree = ast.parse(_SOURCE.read_text(encoding = "utf-8"))+ fn = next(+ n for n in ast.walk(tree) if isinstance(n, ast.FunctionDef) and n.name == "decode_step"+ )+ guards = [+ n for n in ast.walk(fn)+ if isinstance(n, ast.If) and "worth_capturing" in ast.unparse(n.test)+ ] # fmt: skip+ assert guards and all("step is None" in ast.unparse(g.test) for g in guards)+ text = ast.unparse(fn)+ assert text.index("worth_capturing") < text.index("_capture(")+++def test_an_eager_short_call_gets_no_credit_for_a_kept_graph_cache(G):+ model = _Model()+ model.keep(16, 8192)+ rows, kv, work = G._call_bytes(model.config, 16, 64, torch.float16, 512, 66)+ G._kept_buffers = lambda model, key, rows: (True, int(4 * (kv + work)))+ G._free_bytes = lambda device: int(0.5 * G.CALIB_WORK * (kv + work))+ assert G.plan_call(model, _ids(16, 64), torch.float16, 512, 66, eager = True) == ("eager", None)+ assert model.held() == 0 and G.emptied == 1 and G.EAGER_UNFIT == 0+++def test_an_oversized_kept_cache_is_credited_only_up_to_this_call_kv(G):+ model = _Model()+ model.keep(4, 8192)+ rows, kv, work = G._call_bytes(model.config, 4, 128, torch.float16, 512, 160)+ G._kept_buffers = lambda model, key, rows: (True, int(20 * (kv + work)))+ G._free_bytes = lambda device: 0+ assert G.plan_call(model, _ids(4, 128), torch.float16, 512, 160) == ("eager", None)+ assert model.held() == 0 and G.emptied == 1 and G.EAGER_UNFIT == 1+++def _llama_helpers(*names):+ import ast++ path = Path(_SOURCE).with_name("llama.py")+ tree = ast.parse(path.read_text(encoding = "utf-8"))+ fns = [n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name in names]+ namespace = {}+ exec(compile(ast.Module(body = fns, type_ignores = []), str(path), "exec"), namespace)+ return namespace+++def test_planning_honours_an_explicit_max_length_in_the_caller_generation_config():+ transformers = pytest.importorskip("transformers")+ ns = _llama_helpers("_generation_max_len", "_generation_plan_len")+ model = types.SimpleNamespace(generation_config = transformers.GenerationConfig(max_length = 40960))+ ids = torch.zeros((1, 128), dtype = torch.long)+ config = transformers.GenerationConfig(max_length = 4096)+ assert ns["_generation_max_len"](model, (ids,), {"generation_config": config}) == 0+ assert ns["_generation_plan_len"](model, (ids,), {"generation_config": config}) == 4096+ # The model's default config is not a request, and max_new_tokens still wins.+ assert ns["_generation_plan_len"](model, (ids,), {}) == 0+ assert (+ ns["_generation_plan_len"](+ model, (ids,), {"generation_config": config, "max_new_tokens": 32}+ )+ == 160+ )+++def test_cache_growth_cap_uses_the_planned_length():+ # The planner budgets plan_len rows; start_generation's hint caps the cache growth, so both must get it.+ import ast++ path = Path(_SOURCE).with_name("llama.py")+ tree = ast.parse(path.read_text(encoding = "utf-8"))+ fn = next(+ n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name == "unsloth_fast_generate"+ )+ args = {}+ for node in ast.walk(fn):+ if isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute):+ if node.func.attr in ("start_generation", "short_call", "plan_call"):+ args[node.func.attr] = node.args+ assert args["start_generation"][0].id == "plan_len"+ assert args["short_call"][1].id == "plan_len"+ assert args["plan_call"][4].id == "plan_len"diff --git a/tests/test_for_training_generation_flag.py b/tests/test_for_training_generation_flag.pyindex d7fc5e85c18..7204bf238e1 100644--- a/tests/test_for_training_generation_flag.py+++ b/tests/test_for_training_generation_flag.py@@ -95,6 +95,8 @@ def _for_training(module, class_name):# No tensor parallel world in these tests (for_training calls the TP guard first since #15368).namespace["_tensor_parallel_for_training"] = lambda model: Nonenamespace["enable_fp8_lora"] = lambda model: []+ # No decode graphs held in these tests (for_training frees them first since decode graphs became the default).+ namespace["_decode_graphs"] = types.SimpleNamespace(release_for_training = lambda model: None)exec(compile(ast.fix_missing_locations(compiled), str(path), "exec"), namespace)return namespace["for_training"]diff --git a/tests/test_return_logits_survives_for_training.py b/tests/test_return_logits_survives_for_training.pyindex 36fd4625573..31f32d76a7c 100644--- a/tests/test_return_logits_survives_for_training.py+++ b/tests/test_return_logits_survives_for_training.py@@ -179,6 +179,9 @@ def _fast_llama_for_training():set_module_gradient_checkpointing = lambda module, flag: None,_tensor_parallel_for_training = lambda model: None, # tp_replica.py: no tp model hereenable_fp8_lora = lambda model: [],+ _decode_graphs = types.SimpleNamespace(+ release_for_training = lambda model: None+ ), # no graphs held here)module = ast.fix_missing_locations(ast.Module(body = [fn], type_ignores = []))exec(compile(module, str(MODELS / "llama.py"), "exec"), namespace)diff --git a/unsloth/models/_decode_graphs.py b/unsloth/models/_decode_graphs.pyindex d5d3e6f9506..08f1dde31ed 100644--- a/unsloth/models/_decode_graphs.py+++ b/unsloth/models/_decode_graphs.py@@ -1,9 +1,8 @@# SPDX-License-Identifier: AGPL-3.0-only# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.-"""CUDA-graph decode; UNSLOTH_DECODE_CUDA_GRAPHS=1 forces, =0 disables; unset, only RL-trainer-generate and Studio's chat worker on NVIDIA CUDA; graphs dropped when params, LoRA, quant or RoPE-change.+"""CUDA-graph decode; UNSLOTH_DECODE_CUDA_GRAPHS=1 forces, =0 disables; unset, on for every+Unsloth generate() on NVIDIA CUDA outside native Windows for the model types below.UNSLOTH_DECODE_CUDA_GRAPHS_FULL=1 also captures the attention cores: each step is then ONE graph thatattends over a fixed span of cache rows (the next power of two >= the KV length, at least BUCKET, at@@ -29,6 +28,8 @@ FULL = os.environ.get("UNSLOTH_DECODE_CUDA_GRAPHS_FULL", "0") == "1"FULL_MAX_BSZ = int(os.environ.get("UNSLOTH_DECODE_FULL_GRAPH_MAX_BSZ", "1"))# KV-cache rows are allocated in multiples of this with graphs on.BUCKET = 256+# T4 Qwen3-0.6B: 3-6 new tokens ran 5-58% slower graphed, 8 tokens 1.25x faster.+CAPTURE_MIN_STEPS = int(os.environ.get("UNSLOTH_DECODE_GRAPH_CAPTURE_MIN_STEPS", "6"))# Captured steps (one per batch size / autocast state) kept per model.MAX_GRAPHS = int(os.environ.get("UNSLOTH_DECODE_GRAPH_MAX", "8"))# Whole-step graphs keep one graph per span too (log2(cache / BUCKET) + 1 per batch size).@@ -39,6 +40,21 @@ __all__ = ["enabled","rl_generation","studio_generation",+ "generate_scope",+ "eager_scope",+ "graphable",+ "can_replay",+ "short_call",+ "kept_graph",+ "plan_call",+ "record_need",+ "decode_batch",+ "decode_batch_used",+ "EAGER_UNFIT",+ "generate_with_fallback",+ "keep_only",+ "release_if_training",+ "EAGER_REDOS","static","alloc_decode_buffers","decode_step",@@ -69,6 +85,8 @@ _PROJ_NAMES = ("q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "def enabled():"""Whether a decode step / prefill on this thread runs (and sizes its buffers) for graph replay."""+ if getattr(_TLS, "eager", 0):+ return Falseif ENABLED is not None:return bool(ENABLED)if torch.compiler.is_compiling():@@ -125,6 +143,308 @@ class studio_generation(rl_generation):return super().__exit__(*exc) if self._entered else False+class generate_scope(studio_generation):+ """Per-thread scope of plain model.generate(): graphs default on, as in Studio."""++ __slots__ = ()+++class eager_scope:+ """Per-thread: force eager decode (the OOM redo in generate())."""++ __slots__ = ()++ def __enter__(self):+ _TLS.eager = getattr(_TLS, "eager", 0) + 1+ return self++ def __exit__(self, *exc):+ _TLS.eager -= 1+ return False+++# Need = CALIB_WORK x (KV + prefill work) + CALIB_GRAPH_KV x KV. Calibrated on Qwen3-4B QLoRA bs16 x 2048,+# T4: eager peaked 1.14x the estimate; graph capture OOMed at ~12 GiB free (~0.26 x KV extra).+CALIB_WORK = float(os.environ.get("UNSLOTH_DECODE_GRAPH_CALIB_WORK", "1.15"))+CALIB_GRAPH_KV = float(os.environ.get("UNSLOTH_DECODE_GRAPH_CALIB_KV", "0.30"))+# A shape's largest measured footprint x this raises (never lowers) the estimate.+MEASURED_MARGIN = 1.05+EAGER_UNFIT = 0+NEEDS = "_unsloth_decode_needs"+++def _call_bytes(+ config,+ bsz,+ prompt,+ dtype,+ increment,+ max_len = 0,+):+ """(cache rows, KV bytes, prefill work bytes), sized like alloc_decode_buffers."""+ try:+ layers = int(config.num_hidden_layers)+ hidden = int(config.hidden_size)+ heads = int(config.num_attention_heads)+ kv_heads = int(getattr(config, "num_key_value_heads", None) or heads)+ head_dim = int(getattr(config, "head_dim", None) or hidden // heads)+ inter = int(getattr(config, "intermediate_size", None) or 4 * hidden)+ vocab = int(getattr(config, "vocab_size", 0) or 0)+ except (AttributeError, TypeError, ValueError):+ return None, None, None+ size = torch.empty((), dtype = dtype).element_size()+ rows = _round_up(max(prompt + increment + 1, max_len + 1), BUCKET)+ kv = layers * rows * 2 * bsz * kv_heads * head_dim * size+ tokens = bsz * prompt+ work = tokens * (4 * hidden + 3 * inter + 2 * heads * head_dim) * size + bsz * vocab * 4+ return rows, kv, work+++def _allocator(device):+ """(allocated, reserved, peak allocated) from one stats read."""+ try:+ stats = torch.cuda.memory_stats_as_nested_dict(device)+ return (+ stats["allocated_bytes"]["all"]["current"],+ stats["reserved_bytes"]["all"]["current"],+ stats["allocated_bytes"]["all"]["peak"],+ )+ except (KeyError, TypeError):+ return (+ torch.cuda.memory_allocated(device),+ torch.cuda.memory_reserved(device),+ torch.cuda.max_memory_allocated(device),+ )+++def _free_bytes(device):+ free, _ = torch.cuda.mem_get_info(device)+ allocated, reserved, _ = _allocator(device)+ return free + reserved - allocated+++def _empty_cache():+ torch.cuda.empty_cache()+++def _alloc_stats(device):+ # Peak is never reset: other code may rely on max_memory_allocated.+ allocated, _, peak = _allocator(device)+ return allocated, peak+++def _shape_key(bsz, prompt, max_len):+ # Includes the prompt bucket: a longer prompt has a larger prefill.+ return (bsz, _round_up(prompt + 1, BUCKET), _round_up(max(prompt, max_len) + 1, BUCKET))+++def _kept_buffers(model, key, rows):+ """(any buffers kept, bytes of a kept cache for `key` with >= rows); no buffer reference outlives release()."""+ kept, reused = False, 0+ for m in model.modules():+ store = m.__dict__.get("_unsloth_graph_buffers")+ if not store:+ continue+ kept = True+ bufs = store.get(key)+ if bufs is not None and bufs[0].shape[0] >= rows:+ reused += bufs[0].numel() * bufs[0].element_size()+ return kept, reused+++def can_replay(max_len, prompt):+ """The first decode step is always eager, so <= 2 new tokens never replay; 0 (unknown) may."""+ return not max_len or max_len - prompt > 2+++def kept_graph(model, bsz):+ return any(key[0] == bsz for state in _states(model) for key in state.graphs)+++def short_call(model, max_len, prompt, bsz):+ """Too short to win back a capture unless a graph of its batch is kept: decode eagerly."""+ if not max_len:+ return False+ left = max_len - prompt - 1+ return left < 2 or (left < CAPTURE_MIN_STEPS and not kept_graph(model, bsz))+++def graphable(model):+ """Other model types always decode eagerly, so planning them only costs."""+ return getattr(getattr(model, "config", None), "model_type", None) in _SUPPORTED_MODEL_TYPES+++def plan_call(+ model,+ input_ids,+ dtype,+ increment,+ max_len = 0,+ bsz = None,+ eager = False,+):+ """("graph", token) or ("eager", None). Releases kept graphs / buffers first if the call would not+ fit beside them; still too big runs eagerly up front rather than relying on the OOM redo. `bsz` is+ the expanded decode batch (decode_batch()). eager: only makes room, uncounted."""+ global EAGER_UNFIT+ device = getattr(input_ids, "device", None)+ if (+ not graphable(model)+ or not hasattr(input_ids, "shape")+ or len(input_ids.shape) < 2+ or getattr(device, "type", None) != "cuda"+ ):+ return "graph", None+ bsz, prompt = int(bsz or input_ids.shape[0]), int(input_ids.shape[1])+ rows, kv, work = _call_bytes(model.config, bsz, prompt, dtype, increment, max_len)+ if rows is None:+ return "graph", None+ key = _shape_key(bsz, prompt, max_len)+ measured = model.__dict__.get(NEEDS, {}).get(key)+ # A measurement can only raise the estimate; it includes a reused cache, so reuse is subtracted once.+ estimate = CALIB_WORK * (kv + work) + CALIB_GRAPH_KV * kv+ full = estimate if measured is None else max(estimate, MEASURED_MARGIN * measured)+ kept, reused = _kept_buffers(model, (bsz, dtype, device), rows)+ if eager:+ # Eager decode uses fresh buffers: no graph extra, no kept-cache credit.+ full, reused = CALIB_WORK * (kv + work), 0+ # Cap credit at this call's KV: a larger kept cache cannot also cover prefill.+ reused = min(reused, kv)+ need = full - reused+ free = _free_bytes(device)+ if free < need and (kept or any(state.graphs for state in _states(model))):+ release(model)+ _empty_cache()+ free = _free_bytes(device)+ reused = 0+ need = full+ if eager:+ return "eager", None+ if free < need:+ EAGER_UNFIT += 1+ return "eager", None+ return "graph", (key, device, *_alloc_stats(device), reused)+++def record_need(model, token):+ """Store the largest graphed footprint per shape (peak delta + reused kept cache). The peak is never+ reset, so a call that did not raise it is skipped."""+ if token is None:+ return+ key, device, start, peak_before, reused = token+ peak = _alloc_stats(device)[1]+ if peak > peak_before:+ needs = model.__dict__.setdefault(NEEDS, {})+ needs[key] = max(needs.get(key, 0), peak - start + reused)+++def decode_batch(+ bsz,+ kwargs,+ generation_config = None,+):+ """Batch size generate() decodes at: prompts x num_beams, else x num_return_sequences; kwargs win."""+ config = kwargs.get("generation_config", None) or generation_config++ def get(name):+ value = kwargs.get(name, None)+ if value is None:+ value = getattr(config, name, None)+ try:+ return max(1, int(value or 1))+ except (TypeError, ValueError):+ return 1++ beams = get("num_beams")+ return bsz * (beams if beams > 1 else get("num_return_sequences"))+++def decode_batch_used(model):+ """Batch size of the cache the last graph-mode decode ran on, or None."""+ for module in model.modules():+ d = module.__dict__+ store = d.get("_unsloth_graph_buffers")+ paged = d.get("paged_attention")+ if store and paged is not None:+ for key, bufs in store.items():+ if bufs[0] is paged:+ return key[0]+ return None+++_OOM_WARNED = False+EAGER_REDOS = 0+HELD = "_unsloth_decode_held"+++def generate_with_fallback(model, generate, graphs, args, kwargs):+ """On a graph-mode OOM, free graphs and redo once eagerly; caller streamer / past_key_values /+ stopping_criteria / logits_processor may hold failed-attempt state, so those re-raise."""+ global _OOM_WARNED, EAGER_REDOS+ try:+ return generate(*args, **kwargs)+ except torch.cuda.OutOfMemoryError:+ if (+ not graphs+ or not graphable(model)+ or kwargs.get("streamer", None) is not None+ or kwargs.get("past_key_values", None) is not None+ or kwargs.get("stopping_criteria", None)+ or kwargs.get("logits_processor", None)+ ):+ raise+ if not _OOM_WARNED:+ _OOM_WARNED = True+ import warnings+ warnings.warn(+ "Unsloth: generate() ran out of GPU memory with CUDA-graph decode; freed the decode graphs "+ "and buffers and decoded this call eagerly (UNSLOTH_DECODE_CUDA_GRAPHS=0 always decodes eagerly).",+ stacklevel = 3,+ )+ release_for_training(model)+ _empty_cache()+ EAGER_REDOS += 1+ with eager_scope():+ return generate(*args, **kwargs)+++def keep_only(model, bsz = None):+ """Drop decode buffers and graphs of every other batch size; this one's graphs stay valid. The batch+ size decode actually ran at wins over `bsz`."""+ used = decode_batch_used(model)+ if used is not None:+ bsz = used+ if bsz is None:+ return False+ dropped = False+ for module in model.modules():+ store = module.__dict__.get("_unsloth_graph_buffers")+ if store:+ for key in [k for k in store if k[0] != bsz]:+ bufs = store.pop(key)+ dropped = True+ if module.__dict__.get("paged_attention") is bufs[0]:+ for name in _BUFFER_NAMES:+ module.__dict__.pop(name, None)+ state = module.__dict__.get("_unsloth_decode_graphs")+ if state is not None:+ for key in [k for k in state.graphs if k[0] != bsz]:+ del state.graphs[key]+ dropped = True+ if not state.graphs:+ state.pool = None+ return dropped+++def release_if_training(model):+ """Run from the training forward: training can resume without for_training() (e.g. sampling in a+ trainer callback), so this is the only place guaranteed to free kept graphs."""+ if model.__dict__.pop(HELD, False):+ release_for_training(model)+ return True+ return False++def static():"""The _StepStatics of a graph-mode step running on this thread, else None (eager)."""if torch.compiler.is_compiling():@@ -230,6 +550,13 @@ def alloc_decode_buffers(key = (bsz, dtype, device)bufs = store.get(key)if bufs is None or bufs[0].shape[0] < rows:+ # Evict before allocating so old and new caches are never both held (peak memory).+ if bufs is not None or len(store) >= 2:+ store.clear()+ for name in _BUFFER_NAMES:+ attn.__dict__.pop(name, None)+ bufs = None+ BUFFER_EPOCH += 1with _no_inference_tensors():# Zeroed: a captured step reads every row of its bucket, and a masked row still# multiplies its V by 0, which is NaN for uninitialised NaN / inf bit patterns.@@ -246,11 +573,6 @@ def alloc_decode_buffers(else:temp_O = temp_QA[1][:, :, :hidden_size]scores = torch.empty((bsz, n_heads, 1, rows), dtype = dtype, device = device)- # Graphs reading a replaced / evicted buffer are dropped (BUFFER_EPOCH); a new batch size- # beside the ones kept leaves their graphs valid.- if bufs is not None or len(store) >= 2:- store.clear()- BUFFER_EPOCH += 1bufs = store[key] = (paged, temp_QA, temp_KV, RH_Q, temp_O, scores)elif FULL:# A reused buffer still holds the last generation's rows: a whole-step graph reads them@@ -303,6 +625,13 @@ def start_generation(max_len):MAX_LEN_HINT = max(0, int(max_len or 0))+def worth_capturing(kv_len):+ """CAPTURE_MIN_STEPS steps left at kv_len; unknown length (0): yes."""+ if MAX_LEN_HINT <= 0:+ return True+ return MAX_LEN_HINT - kv_len + 1 >= CAPTURE_MIN_STEPS++def end_generation():global MAX_LEN_HINTMAX_LEN_HINT = 0@@ -324,6 +653,7 @@ class _ModelGraphs:self.captures = 0self.replays = 0self.eager_steps = 0+ self.short_skips = 0def clear(self):self.graphs.clear()@@ -373,6 +703,7 @@ def release(model):found = Falsefor module in model.modules():d = module.__dict__+ d.pop(HELD, None)state = d.get("_unsloth_decode_graphs")if state is not None:found = True@@ -702,6 +1033,10 @@ def decode_step(autocast = _autocast_state()key = (bsz, device, autocast, torch.is_inference_mode_enabled(), span)step = state.graphs.get(key)+ if step is None and not worth_capturing(kv_len):+ state.eager_steps += 1+ state.short_skips += 1+ return Noneif step is None:if len(state.graphs) >= (MAX_GRAPHS if span is None else MAX_FULL_GRAPHS):state.graphs.clear()@@ -844,5 +1179,6 @@ def stats(model):"replays": state.replays,"graphs": len(state.graphs),"eager_steps": state.eager_steps,+ "short_skips": state.short_skips,"disabled": state.disabled,}diff --git a/unsloth/models/llama.py b/unsloth/models/llama.pyindex 65e7e1aeef2..fdf7b6e43cc 100644--- a/unsloth/models/llama.py+++ b/unsloth/models/llama.py@@ -2018,6 +2018,9 @@ def CausalLM_fast_forward(fast_forward_inference):) -> Union[Tuple, CausalLMOutputWithPast]:# Not a named parameter: HF's find_labels would make it a required eval label.shift_labels = kwargs.pop("shift_labels", None)+ # Training may resume without for_training() (trainer callback sampling): free kept graphs here.+ if _decode_graphs.HELD in self.__dict__ and self.training and torch.is_grad_enabled():+ _decode_graphs.release_if_training(self)past_key_values = _cache_as_legacy_tuple(past_key_values)if past_key_values is not None and len(past_key_values) == 0:past_key_values = None@@ -2986,6 +2989,30 @@ def _generation_max_len(self, args, kwargs):return 0+def _generation_plan_len(self, args, kwargs):+ """_generation_max_len, or a max_length the caller's own GenerationConfig sets explicitly (the call can grow+ to it; the model's default config is not a request). Sizes both the plan and the cache growth cap."""+ max_len = _generation_max_len(self, args, kwargs)+ if max_len or kwargs.get("max_new_tokens", None) is not None:+ return max_len+ try:+ config = kwargs.get("generation_config", None)+ explicit = config.to_diff_dict() if config is not None else {}+ if (+ explicit.get("max_new_tokens", None) is None+ and explicit.get("max_length", None) is not None+ ):+ return int(explicit["max_length"])+ except Exception:+ pass+ return max_len+++def _generation_input_ids(args, kwargs):+ ids = kwargs.get("input_ids", kwargs.get("inputs", args[0] if args else None))+ return ids if ids is not None else kwargs.get("inputs_embeds", None)++def unsloth_fast_generate(self, *args, **kwargs):restore_training_mode = self.training# Snapshot the real GC mode (e.g. unsloth) before for_inference clears it, so restore does not collapse it to a bool@@ -3064,20 +3091,48 @@ def unsloth_fast_generate(self, *args, **kwargs):kwargs["pad_token_id"] = kwargs.pop("pad_token_id", model_eos_token_id)- # Decided once per call (env / RL generation scope), so the prefill and every decode step agree.- graphs = _decode_graphs.enabled()- if graphs:- _decode_graphs.start_generation(_generation_max_len(self, args, kwargs))- # Mixed precision autocast.- try:- with (- _get_inference_mode_context_manager(self),- torch.autocast(device_type = DEVICE_TYPE_TORCH, dtype = dtype),- ):- output = self._old_generate(*args, **kwargs)- finally:+ # Decided once per call so the prefill and every decode step agree.+ with _decode_graphs.generate_scope():+ graphs = _decode_graphs.enabled()+ plan, token = "graph", Noneif graphs:- _decode_graphs.end_generation()+ plan_len = _generation_plan_len(self, args, kwargs)+ try:+ ids = _generation_input_ids(args, kwargs)+ bsz = _decode_graphs.decode_batch(+ int(ids.shape[0]), kwargs, getattr(self, "generation_config", None)+ )+ short = _decode_graphs.short_call(self, plan_len, int(ids.shape[1]), bsz)+ if short and _decode_graphs.HELD not in self.__dict__:+ plan, token = "eager", None+ else:+ plan, token = _decode_graphs.plan_call(+ self, ids, dtype, KV_CACHE_INCREMENT, plan_len, bsz = bsz, eager = short,+ ) # fmt: skip+ if short:+ plan, token = "eager", None+ except Exception:+ plan, token = "graph", None+ if plan == "eager":+ graphs = False+ else:+ _decode_graphs.start_generation(plan_len)+ redos = _decode_graphs.EAGER_REDOS+ # Mixed precision autocast.+ try:+ with (+ _decode_graphs.eager_scope() if plan == "eager" else contextlib.nullcontext(),+ _get_inference_mode_context_manager(self),+ torch.autocast(device_type = DEVICE_TYPE_TORCH, dtype = dtype),+ ):+ output = _decode_graphs.generate_with_fallback(+ self, self._old_generate, graphs, args, kwargs+ )+ finally:+ if graphs:+ _decode_graphs.end_generation()+ if graphs and _decode_graphs.EAGER_REDOS == redos:+ _decode_graphs.record_need(self, token)if graphs and restore_training_mode:# Back to training: the next optimizer step invalidates the graphs anyway; free them and the# decode buffers for training.@@ -3086,6 +3141,16 @@ def unsloth_fast_generate(self, *args, **kwargs):# The returned cache views the persistent decode buffers: hand them over, so a later# generate() allocates its own instead of overwriting a cache the caller kept._decode_graphs.release(self)+ elif graphs:+ # keep_only uses the expanded decode batch.+ ids = _generation_input_ids(args, kwargs)+ expected = None+ if hasattr(ids, "shape") and len(ids.shape) >= 2:+ expected = _decode_graphs.decode_batch(+ int(ids.shape[0]), kwargs, getattr(self, "generation_config", None)+ )+ _decode_graphs.keep_only(self, expected)+ self.__dict__[_decode_graphs.HELD] = Trueif restore_training_mode:FastLlamaModel.for_training(@@ -5645,6 +5710,7 @@ class FastLlamaModel:"Unsloth: I think you're passing a tokenizer, not the model to for_training!")_tensor_parallel_for_training(model)+ _decode_graphs.release_for_training(model)# Mode last, as model.train() was: a custom train() keeps the final say over the chain below.checkpointing, apply_training_mode = switch_training_mode(model, True, defer = True)use_gradient_checkpointing = resolve_training_gradient_checkpointing(