Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Block CUDA graphs: decide before recording, release the pool, cap FSDP2 kept weights, drop unneeded copies (#15352)

Daniel Han2bcde87ffb81
diff --git a/tests/test_block_cudagraphs_clone.py b/tests/test_block_cudagraphs_clone.py
index b498a429e61..a98adb7da36 100644
--- a/tests/test_block_cudagraphs_clone.py
+++ b/tests/test_block_cudagraphs_clone.py
@@ -12,6 +12,8 @@ import importlib.util
import os
import sys
+import types
+
import pytest
import torch
@@ -40,6 +42,13 @@ def _block_env(monkeypatch):
# the eager path, so the graph-cap test would see no compiled call at all.
for key in ("UNSLOTH_BLOCK_COMPILE", "UNSLOTH_COMPILE_DISABLE", "UNSLOTH_FORCE_FLOAT32"):
monkeypatch.delenv(key, raising = False)
+ # Graphs go off process wide; every test starts with them on.
+ monkeypatch.setattr(
+ bc,
+ "_CUDAGRAPH_STATE",
+ getattr(bc, "_new_cudagraph_state", dict)(),
+ raising = False,
+ )
def test_clone_grad_is_identity_with_copied_gradient():
@@ -132,7 +141,10 @@ def test_cap_reads_the_environment(monkeypatch):
monkeypatch.setenv("UNSLOTH_BLOCK_CUDAGRAPHS_MAX_SHAPES", "3")
assert _load()._CUDAGRAPH_MAX_SHAPES == 3
monkeypatch.setenv("UNSLOTH_BLOCK_CUDAGRAPHS_MAX_SHAPES", "junk")
- assert _load()._CUDAGRAPH_MAX_SHAPES == 8
+ assert _load()._CUDAGRAPH_MAX_SHAPES == 2
+ monkeypatch.delenv("UNSLOTH_BLOCK_CUDAGRAPHS_MAX_SHAPES")
+ # One fixed length plus a short last batch; T4 variable lengths paid 15-65 s for 8.
+ assert _load()._CUDAGRAPH_MAX_SHAPES == 2
finally:
monkeypatch.delenv("UNSLOTH_BLOCK_CUDAGRAPHS_MAX_SHAPES", raising = False)
_load()
@@ -279,6 +291,450 @@ def test_nonreentrant_checkpoint_runs_without_graphs(monkeypatch):
assert not bc._in_nonreentrant_checkpoint()
+def _tagging_compile(monkeypatch, calls):
+ def fake_compile(fn, **kwargs):
+ tag = "graphs" if kwargs.get("options", {}).get("triton.cudagraphs") else "plain"
+
+ def run(*args, **kw):
+ calls.append(tag)
+ return fn(*args, **kw)
+
+ return run
+
+ monkeypatch.setattr(torch, "compile", fake_compile)
+
+
+def _linear_layer_class(name, module = None):
+ def __init__(self):
+ torch.nn.Module.__init__(self)
+ self.proj = torch.nn.Linear(8, 8)
+
+ def forward(self, hidden_states, **kwargs):
+ return self.proj(hidden_states)
+
+ if module is not None:
+ forward.__module__ = module
+ return type(name, (torch.nn.Module,), {"__init__": __init__, "forward": forward})
+
+
+def test_past_the_cap_every_block_class_goes_off_and_the_pool_is_released(monkeypatch):
+ """One cudagraph tree (one pool) per device serves every block class: graphs go off for all of
+ them together, so the release can free the whole pool."""
+ calls, released = [], []
+ _tagging_compile(monkeypatch, calls)
+ monkeypatch.setattr(bc, "_CUDAGRAPH_MAX_SHAPES", 1)
+ monkeypatch.setattr(
+ bc,
+ "_release_graph_pool",
+ lambda: released.append(1) or bc._CUDAGRAPH_STATE.update(release = "done"),
+ )
+ A, B = _linear_layer_class("ADecoderLayer"), _linear_layer_class("BDecoderLayer")
+ for cls in (A, B):
+ bc._wrap(cls, {**bc._COMPILE_OPTIONS, "triton.cudagraphs": True})
+ a, b = A().train(), B().train()
+
+ def step(L):
+ calls.clear()
+ x = torch.randn(2, L, 8, requires_grad = True)
+ b(a(x)).sum().backward()
+ return list(calls)
+
+ assert step(16) == ["graphs", "graphs"]
+ assert step(16) == ["graphs", "graphs"]
+ assert not released
+ assert step(24) == ["plain", "plain"] # A saw the new length first: B is off too
+ assert bc._CUDAGRAPH_STATE["off"] == "shapes"
+ assert released == [1]
+ assert step(16) == ["plain", "plain"]
+ assert released == [1]
+
+
+def test_cap_notice_is_printed_once(monkeypatch, capsys):
+ calls = []
+ _tagging_compile(monkeypatch, calls)
+ monkeypatch.setattr(bc, "_CUDAGRAPH_MAX_SHAPES", 1)
+ A = _linear_layer_class("NoteDecoderLayer")
+ bc._wrap(A, {**bc._COMPILE_OPTIONS, "triton.cudagraphs": True})
+ layer = A().train()
+ for L in (8, 16, 24, 32):
+ layer(torch.randn(2, L, 8, requires_grad = True)).sum().backward()
+ out = capsys.readouterr().out
+ assert out.count("block CUDA graphs turned off") == 1
+ assert "UNSLOTH_BLOCK_CUDAGRAPHS_MAX_SHAPES" in out
+
+
[email protected]("key", ["packed_seq_lengths", "cu_seq_lens_q"])
+def test_packed_rows_turn_graphs_off_before_any_recording(monkeypatch, capsys, key):
+ """Packed / padding-free rows change total length every step: decide before recording."""
+ calls = []
+ _tagging_compile(monkeypatch, calls)
+ A = _linear_layer_class("PackedDecoderLayer")
+ bc._wrap(A, {**bc._COMPILE_OPTIONS, "triton.cudagraphs": True})
+ layer = A().train()
+ lengths = torch.tensor([5, 11], dtype = torch.int32)
+ layer(torch.randn(1, 16, 8, requires_grad = True), **{key: lengths}).sum().backward()
+ assert calls == ["plain"]
+ assert bc._CUDAGRAPH_STATE["off"] == "packed"
+ calls.clear()
+ layer(torch.randn(2, 16, 8, requires_grad = True)).sum().backward() # later padded rows too
+ assert calls == ["plain"]
+ assert capsys.readouterr().out.count("packed / padding-free rows") == 1
+
+
+def test_packed_rows_on_unsloth_forward_say_graphs_do_not_apply(monkeypatch, capsys):
+ # Unsloth's own forwards run packed rows stock (no compiled call at all): say so once.
+ calls = []
+ _tagging_compile(monkeypatch, calls)
+ A = _linear_layer_class("UPackedDecoderLayer", module = "unsloth.models.fake")
+ bc._wrap(A, {**bc._COMPILE_OPTIONS, "triton.cudagraphs": True})
+ layer = A().train()
+ lengths = torch.tensor([5, 11], dtype = torch.int32)
+ layer(torch.randn(1, 16, 8, requires_grad = True), packed_seq_lengths = lengths).sum().backward()
+ assert calls == []
+ assert bc._CUDAGRAPH_STATE["off"] == "packed"
+ assert "packed / padding-free rows" in capsys.readouterr().out
+
+
+def test_flag_unset_ignores_packed_rows(monkeypatch, capsys):
+ calls = []
+ _tagging_compile(monkeypatch, calls)
+ A = _linear_layer_class("NoFlagDecoderLayer")
+ bc._wrap(A, {**bc._COMPILE_OPTIONS, "triton.cudagraphs": False})
+ layer = A().train()
+ lengths = torch.tensor([5, 11], dtype = torch.int32)
+ layer(torch.randn(1, 16, 8, requires_grad = True), packed_seq_lengths = lengths).sum().backward()
+ assert calls == ["plain"]
+ assert bc._CUDAGRAPH_STATE["off"] is None
+ assert capsys.readouterr().out == ""
+
+
+def _counting_clones(monkeypatch):
+ seen = {"outputs": 0, "grad": 0}
+ clone_outputs = bc._clone_outputs
+
+ def counted(out):
+ seen["outputs"] += 1
+ return clone_outputs(out)
+
+ class Counted(bc._CloneGrad):
+ @staticmethod
+ def forward(ctx, x):
+ seen["grad"] += 1
+ return x.view_as(x)
+
+ monkeypatch.setattr(bc, "_clone_outputs", counted)
+ monkeypatch.setattr(bc, "_CloneGrad", Counted)
+ return seen
+
+
[email protected]("trainable_input", [True, False])
+def test_no_pool_copies_outside_checkpointing(monkeypatch, trainable_input):
+ """A grad-mode block call outside backward leaves a forward pending, so cudagraph trees keep
+ the next block and the backward in the same generation: its outputs need no copy."""
+ monkeypatch.setattr(torch, "compile", lambda fn, **kwargs: fn)
+ seen = _counting_clones(monkeypatch)
+ A = _linear_layer_class("FreeDecoderLayer")
+ bc._wrap(A, {**bc._COMPILE_OPTIONS, "triton.cudagraphs": True})
+ layers = [A().train() for _ in range(3)]
+ x = torch.randn(2, 4, 8, requires_grad = trainable_input)
+ y = x
+ for layer in layers:
+ y = layer(y)
+ y.sum().backward()
+ # Only a leaf input's gradient is copied: its .grad outlives the generation.
+ assert seen == {"outputs": 0, "grad": 1 if trainable_input else 0}
+ assert all(layer.proj.weight.grad is not None for layer in layers)
+ assert (x.grad is not None) is trainable_input
+
+
+ "through", ["view", "view_add", "cat", "addmm_input", "addmm_bias", "embedding"]
+)
+def test_input_viewing_a_trainable_param_copies_its_gradient(monkeypatch, through):
+ """inputs_embeds = param.view(...): the view's backward passes the pool gradient straight to
+ AccumulateGrad, so the parameter's first .grad would alias the pool. An embedding lookup writes
+ a new gradient, so it needs no copy."""
+ monkeypatch.setattr(torch, "compile", lambda fn, **kwargs: fn)
+ seen = _counting_clones(monkeypatch)
+ A = _linear_layer_class("ViewInputDecoderLayer")
+ bc._wrap(A, {**bc._COMPILE_OPTIONS, "triton.cudagraphs": True})
+ layer = A().train()
+ param = torch.nn.Parameter(torch.randn(8, 8))
+ if through == "embedding":
+ x = torch.nn.functional.embedding(torch.tensor([[0, 1, 2, 3], [4, 5, 6, 7]]), param)
+ else:
+ x = param.view(2, 4, 8)
+ if through == "view_add":
+ x = x + torch.zeros(2, 4, 8)
+ elif through in ("addmm_input", "addmm_bias"):
+ # addmm's backward passes `grad` itself to its additive input; a broadcast bias is summed
+ a, b = torch.randn(8, 8), torch.randn(8, 8)
+ if through == "addmm_bias":
+ param = torch.nn.Parameter(torch.randn(8))
+ x = torch.addmm(param, a, b).view(2, 4, 8)
+ elif through == "cat":
+ # cat's backward hands each input a narrowed view of the incoming gradient
+ param = torch.nn.Parameter(torch.randn(4, 8))
+ x = torch.cat([param.view(1, 4, 8), torch.zeros(1, 4, 8)], dim = 0)
+ layer(x).sum().backward()
+ assert seen["grad"] == (0 if through in ("embedding", "addmm_bias") else 1), seen
+ assert param.grad is not None
+
+
[email protected]("source", ["block", "foreign"])
+def test_only_graphed_block_regions_stop_the_input_gradient_walk(monkeypatch, source):
+ """A previous graphed block decided its own input gradient, so the walk stops at its compiled
+ node. Any other compiled function (a compiled prompt preprocessor) can hand the incoming
+ gradient straight to a trainable leaf, so its first .grad would alias the pool: copy it."""
+ real_compile = torch.compile
+ torch._dynamo.reset()
+ # aot_eager gives the real CompiledFunctionBackward nodes on CPU; graphs themselves are faked.
+ monkeypatch.setattr(
+ torch, "compile", lambda fn, **kwargs: real_compile(fn, backend = "aot_eager")
+ )
+ seen = _counting_clones(monkeypatch)
+ A = _linear_layer_class("CompiledRegionDecoderLayer")
+ bc._wrap(A, {**bc._COMPILE_OPTIONS, "triton.cudagraphs": True})
+ layers = [A().train() for _ in range(2)]
+ if source == "foreign":
+ prompt = torch.nn.Parameter(torch.randn(2, 4, 8))
+ x = real_compile(lambda p: p.clone(), backend = "aot_eager")(prompt)
+ assert type(x.grad_fn).__name__.startswith("CompiledFunction")
+ layers[0](x).sum().backward()
+ assert seen["grad"] == 1, seen
+ assert prompt.grad is not None
+ else:
+ # block-to-block hand-off: no copy, the memory win stays
+ y = layers[0](torch.randn(2, 4, 8))
+ assert type(y.grad_fn).__name__.startswith("CompiledFunction")
+ layers[1](y).sum().backward()
+ assert seen["grad"] == 0, seen
+ assert layers[0].proj.weight.grad is not None
+ torch._dynamo.reset()
+
+
[email protected]("exposed", ["kwarg", "config", None])
+def test_returned_hidden_states_keep_their_pool_copies(monkeypatch, exposed):
+ """output_hidden_states hands every block output to the caller, who may keep it past the
+ generation: those calls copy out of the pool as before; other calls still skip the copies."""
+ monkeypatch.setattr(torch, "compile", lambda fn, **kwargs: fn)
+ monkeypatch.setitem(bc._COMPILE_OPTIONS, "triton.cudagraphs", True)
+ seen = _counting_clones(monkeypatch)
+ A = _linear_layer_class("ExposedDecoderLayer")
+ bc._wrap(A, {**bc._COMPILE_OPTIONS, "triton.cudagraphs": True})
+
+ class Inner(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.config = types.SimpleNamespace(output_hidden_states = exposed == "config")
+ self.layers = torch.nn.ModuleList([A().train() for _ in range(2)])
+
+ def forward(
+ self,
+ x,
+ output_hidden_states = None,
+ ):
+ hidden = []
+ for layer in self.layers:
+ x = layer(x)
+ hidden.append(x)
+ return x, hidden
+
+ inner = Inner()
+ bc._watch_hidden_states(inner)
+ x = torch.randn(2, 4, 8)
+ kwargs = {"output_hidden_states": True} if exposed == "kwarg" else {}
+ y, _ = inner(x, **kwargs)
+ y.sum().backward()
+ assert seen["outputs"] == (0 if exposed is None else 2), seen
+ bc._HIDDEN_STATES_EXPOSED[0] = False
+
+
[email protected]("use_reentrant", [True])
+def test_block_after_a_checkpointed_block_copies_its_input_gradient(monkeypatch, use_reentrant):
+ """Checkpointed blocks then an uncheckpointed one: the checkpoint's backward recomputes (a new
+ torch.compile invocation, a new generation) before it reads the gradient handed to it."""
+ from torch.utils.checkpoint import checkpoint
+
+ monkeypatch.setattr(torch, "compile", lambda fn, **kwargs: fn)
+ seen = _counting_clones(monkeypatch)
+ A = _linear_layer_class("MixedDecoderLayer")
+ bc._wrap(A, {**bc._COMPILE_OPTIONS, "triton.cudagraphs": True})
+ layers = [A().train() for _ in range(3)]
+ x = torch.randn(2, 4, 8, requires_grad = True)
+ y = x
+ for layer in layers[:2]:
+ y = checkpoint(layer, y, use_reentrant = use_reentrant)
+ before = dict(seen)
+ out = layers[2](y)
+ # the last block (grad mode, outside backward) still routes its input gradient out of the pool
+ assert seen == {"outputs": before["outputs"], "grad": before["grad"] + 1}
+ out.sum().backward()
+ assert x.grad is not None
+
+
[email protected]("reentrant_kind", ["checkpoint", "no_grad"])
+def test_pool_copies_kept_where_a_block_call_starts_a_generation(monkeypatch, reentrant_kind):
+ from torch.utils.checkpoint import checkpoint
+
+ monkeypatch.setattr(torch, "compile", lambda fn, **kwargs: fn)
+ seen = _counting_clones(monkeypatch)
+ A = _linear_layer_class("CopyDecoderLayer")
+ bc._wrap(A, {**bc._COMPILE_OPTIONS, "triton.cudagraphs": True})
+ layers = [A().train() for _ in range(3)]
+ x = torch.randn(2, 4, 8, requires_grad = True)
+ if reentrant_kind == "no_grad":
+ with torch.no_grad():
+ for layer in layers:
+ x = layer(x)
+ assert seen == {"outputs": 6, "grad": 3} # each first call runs a grad-mode trial too
+ return
+ y = x
+ for layer in layers:
+ y = checkpoint(layer, y, use_reentrant = True)
+ y.sum().backward()
+ # Trial + no-grad forward + recompute inside backward: every call copies (main did the same).
+ assert seen == {"outputs": 9, "grad": 6}
+
+
+def test_frozen_block_without_grad_input_still_copies(monkeypatch):
+ # Nothing requires grad: an inference graph, which may start a new generation per call.
+ monkeypatch.setattr(torch, "compile", lambda fn, **kwargs: fn)
+ seen = _counting_clones(monkeypatch)
+ A = _linear_layer_class("FrozenDecoderLayer")
+ bc._wrap(A, {**bc._COMPILE_OPTIONS, "triton.cudagraphs": True})
+ layer = A().train().requires_grad_(False)
+ layer(torch.randn(2, 4, 8))
+ assert seen == {"outputs": 1, "grad": 0}
+
+
+class _FakeManager:
+ def __init__(
+ self,
+ compile_ids,
+ pending = False,
+ live = (),
+ ):
+ self.id_to_compile_id = dict(enumerate(compile_ids))
+ self.ids_to_funcs = dict.fromkeys(self.id_to_compile_id)
+ self.running_forwards_with_pending_backwards = pending
+ self._live = list(live)
+
+ def live_cudagraph_pool_storages_in_curr_execution(self):
+ return self._live
+
+
+def test_pool_release_verdict():
+ ours = {"a", "b"}
+ assert bc._pool_release_verdict([_FakeManager(["a", "b", "a"])], ours) == "ok"
+ assert bc._pool_release_verdict([_FakeManager(["a", "c"])], ours) == "foreign"
+ assert bc._pool_release_verdict([_FakeManager(["a", None])], ours) == "foreign"
+ assert bc._pool_release_verdict([_FakeManager(["a"], pending = True)], ours) == "pending"
+ assert bc._pool_release_verdict([_FakeManager(["a"], live = [object()])], ours) == "pending"
+
+
[email protected]("ids", [["a", "a"], ["a", "x"]])
+def test_release_resets_the_trees_only_when_every_function_is_ours(monkeypatch, capsys, ids):
+ from types import SimpleNamespace
+
+ from torch._inductor import cudagraph_trees
+
+ manager = _FakeManager(ids)
+ resets = []
+ containers = {0: SimpleNamespace(tree_manager = manager)}
+ monkeypatch.setattr(cudagraph_trees, "get_obj", lambda local, name: containers)
+ monkeypatch.setattr(cudagraph_trees, "reset_cudagraph_trees", lambda: resets.append(1))
+ reserved = [6 * 2**30]
+ monkeypatch.setattr(torch.cuda, "memory_reserved", lambda d = None: reserved[0])
+ monkeypatch.setattr(torch.cuda, "empty_cache", lambda: reserved.__setitem__(0, 2 * 2**30))
+ bc._CUDAGRAPH_STATE["ours"].add("a")
+ bc._CUDAGRAPH_STATE["managers"][0] = manager
+ bc._release_graph_pool()
+ out = capsys.readouterr().out
+ if ids == ["a", "a"]:
+ assert resets == [1] and bc._CUDAGRAPH_STATE["release"] == "done"
+ assert "4.00 GiB reserved freed" in out
+ else:
+ assert resets == [] and bc._CUDAGRAPH_STATE["release"] == "kept"
+ assert "stays allocated" in out
+ bc._release_graph_pool() # decided: never again
+ assert len(resets) == (1 if ids == ["a", "a"] else 0)
+
+
+def test_release_waits_for_a_pending_backward(monkeypatch):
+ from types import SimpleNamespace
+
+ from torch._inductor import cudagraph_trees
+
+ manager = _FakeManager(["a"], pending = True)
+ resets = []
+ monkeypatch.setattr(
+ cudagraph_trees, "get_obj", lambda local, name: {0: SimpleNamespace(tree_manager = manager)}
+ )
+ monkeypatch.setattr(cudagraph_trees, "reset_cudagraph_trees", lambda: resets.append(1))
+ monkeypatch.setattr(torch.cuda, "memory_reserved", lambda d = None: 0)
+ monkeypatch.setattr(torch.cuda, "empty_cache", lambda: None)
+ bc._CUDAGRAPH_STATE["ours"].add("a")
+ bc._release_graph_pool()
+ assert resets == [] and bc._CUDAGRAPH_STATE["release"] is None
+ manager.running_forwards_with_pending_backwards = False
+ bc._release_graph_pool()
+ assert resets == [1] and bc._CUDAGRAPH_STATE["release"] == "done"
+
+
+def test_block_marked_by_fsdp2_runs_without_graphs(monkeypatch):
+ calls = []
+ _tagging_compile(monkeypatch, calls)
+ A = _linear_layer_class("MarkedDecoderLayer")
+ bc._wrap(A, {**bc._COMPILE_OPTIONS, "triton.cudagraphs": True})
+ kept, marked = A().train(), A().train()
+ marked.__dict__["_unsloth_block_graphs_off"] = True
+ marked(kept(torch.randn(2, 4, 8, requires_grad = True))).sum().backward()
+ assert calls == ["graphs", "plain"]
+ assert bc._CUDAGRAPH_STATE["off"] is None
+
+
+def test_graphed_call_records_the_compile_ids_it_created(monkeypatch):
+ # The first call creates the tree (and its manager); later calls read the cached manager only.
+ manager = _FakeManager([])
+ lookups = []
+
+ def tree_manager(device):
+ lookups.append(device)
+ return manager
+
+ def compiled(n, cid):
+ for _ in range(n):
+ i = len(manager.ids_to_funcs)
+ manager.ids_to_funcs[i] = None
+ manager.id_to_compile_id[i] = cid
+ return n
+
+ monkeypatch.setattr(bc, "_tree_manager", tree_manager)
+ manager.ids_to_funcs[0], manager.id_to_compile_id[0] = None, "foreign" # made before us
+ assert bc._graphed_call(compiled, 0, 0, "x") == 0
+ bc._graphed_call(compiled, 0, 2, "a")
+ bc._graphed_call(compiled, 0, 0, "y")
+ bc._graphed_call(compiled, 0, 1, "b")
+ assert lookups == [0]
+ assert bc._CUDAGRAPH_STATE["ours"] == {"a", "b"}
+ assert bc._pool_release_verdict([manager], bc._CUDAGRAPH_STATE["ours"]) == "foreign"
+
+
+def test_grad_trial_of_a_checkpointed_block_takes_the_recompute_path(monkeypatch):
+ # The first no-grad call (checkpoint forward) runs a grad-mode trial first; it stands in for the
+ # recompute, so it copies like the recompute does.
+ monkeypatch.setattr(torch, "compile", lambda fn, **kwargs: fn)
+ seen = _counting_clones(monkeypatch)
+ A = _linear_layer_class("TrialDecoderLayer")
+ bc._wrap(A, {**bc._COMPILE_OPTIONS, "triton.cudagraphs": True})
+ layer = A().train()
+ with torch.no_grad():
+ layer(torch.randn(2, 4, 8, requires_grad = True))
+ assert seen == {"outputs": 2, "grad": 1} # trial (outputs + input grad) and the no-grad call
+ assert bc._IN_TRIAL == [False]
+
+
class FakeMoeExperts(torch.nn.Module):
def __init__(self):
super().__init__()
diff --git a/tests/test_block_cudagraphs_gpu.py b/tests/test_block_cudagraphs_gpu.py
new file mode 100644
index 00000000000..e223ea87839
--- /dev/null
+++ b/tests/test_block_cudagraphs_gpu.py
@@ -0,0 +1,204 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
+
+"""Block CUDA graphs (UNSLOTH_BLOCK_CUDAGRAPHS=1) on a real GPU with real cudagraph trees.
+
+Fixed shape without checkpointing: replays, no copies out of the pool, weights match eager over
+gradient accumulation. Reentrant checkpointing: still copies, still matches. Variable lengths: graphs
+go off past the cap and the graph pool is released (it stayed reserved: +3 to 4.75 GB on a T4)."""
+
+import importlib.util
+import os
+import sys
+
+import pytest
+import torch
+
+pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason = "needs a CUDA GPU")
+
+_DIR = os.path.join(
+ os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "unsloth", "models"
+)
+H = 512
+
+
+def _load():
+ spec = importlib.util.spec_from_file_location(
+ "_ublock_compile_cg_gpu", os.path.join(_DIR, "_block_compile.py")
+ )
+ mod = importlib.util.module_from_spec(spec)
+ sys.modules[spec.name] = mod
+ spec.loader.exec_module(mod)
+ return mod
+
+
+bc = _load()
+
+
+def _forward(self, hidden_states, **kwargs):
+ x = self.norm(hidden_states)
+ return hidden_states + self.down(torch.nn.functional.silu(self.up(x)))
+
+
+def _init(self):
+ torch.nn.Module.__init__(self)
+ self.norm = torch.nn.LayerNorm(H)
+ self.up = torch.nn.Linear(H, 4 * H)
+ self.down = torch.nn.Linear(4 * H, H)
+
+
+class EagerBlock(torch.nn.Module):
+ __init__ = _init
+ forward = _forward
+
+
+class GpuCgDecoderLayer(torch.nn.Module):
+ __init__ = _init
+ forward = _forward
+
+
+bc._wrap(GpuCgDecoderLayer, {**bc._COMPILE_OPTIONS, "triton.cudagraphs": True})
+
+
+class Counters:
+ def __init__(self, monkeypatch):
+ from torch._inductor import cudagraph_trees as T
+
+ self.record = self.replay = self.clones = 0
+ manager = T.CUDAGraphTreeManager
+ record, execute = manager.record_function, manager.execute_node
+
+ def counted_record(this, *a, **k):
+ self.record += 1
+ return record(this, *a, **k)
+
+ def counted_execute(this, *a, **k):
+ self.replay += 1
+ return execute(this, *a, **k)
+
+ monkeypatch.setattr(manager, "record_function", counted_record)
+ monkeypatch.setattr(manager, "execute_node", counted_execute)
+ clone = bc._clone_outputs
+
+ def counted_clone(out):
+ self.clones += 1
+ return clone(out)
+
+ monkeypatch.setattr(bc, "_clone_outputs", counted_clone)
+
+
[email protected](autouse = True)
+def _fresh(monkeypatch):
+ for key in ("UNSLOTH_BLOCK_COMPILE", "UNSLOTH_COMPILE_DISABLE", "UNSLOTH_FORCE_FLOAT32"):
+ monkeypatch.delenv(key, raising = False)
+ torch._dynamo.reset() # also resets the cudagraph trees
+ if hasattr(bc, "_CUDAGRAPH_STATE"):
+ monkeypatch.setattr(
+ bc,
+ "_CUDAGRAPH_STATE",
+ getattr(bc, "_new_cudagraph_state", dict)(),
+ )
+ yield
+ torch._dynamo.reset()
+ torch.cuda.empty_cache()
+
+
+def _models(n = 3):
+ torch.manual_seed(0)
+ graphed = torch.nn.ModuleList(GpuCgDecoderLayer() for _ in range(n)).cuda().train()
+ eager = torch.nn.ModuleList(EagerBlock() for _ in range(n)).cuda().train()
+ eager.load_state_dict(graphed.state_dict())
+ return graphed, eager
+
+
+def _step(layers, xs, opt, checkpoint):
+ from torch.utils.checkpoint import checkpoint as ckpt
+
+ losses = []
+ for x in xs: # gradient accumulation: the first .grad of a micro step lives across generations
+ y = x
+ for layer in layers:
+ y = ckpt(layer, y, use_reentrant = True) if checkpoint else layer(y)
+ loss = y.float().square().mean()
+ loss.backward()
+ losses.append(loss.item())
+ opt.step()
+ opt.zero_grad(set_to_none = True)
+ return losses
+
+
+def _train(
+ lengths,
+ checkpoint,
+ micro = 2,
+ steps = None,
+):
+ graphed, eager = _models()
+ opts = [torch.optim.SGD(m.parameters(), lr = 0.1) for m in (graphed, eager)]
+ out = {"graphed": [], "eager": []}
+ for step, L in enumerate(lengths):
+ g = torch.Generator(device = "cuda").manual_seed(step)
+ xs = [
+ torch.randn(2, L, H, device = "cuda", generator = g).requires_grad_(checkpoint)
+ for _ in range(micro)
+ ]
+ out["graphed"].append(_step(graphed, xs, opts[0], checkpoint))
+ out["eager"].append(
+ _step(eager, [x.detach().requires_grad_(checkpoint) for x in xs], opts[1], checkpoint)
+ )
+ if steps is not None:
+ steps(step)
+ return graphed, eager, out
+
+
+def _assert_same(graphed, eager, out):
+ torch.testing.assert_close(out["graphed"], out["eager"], rtol = 2e-3, atol = 2e-4)
+ for (name, p), q in zip(graphed.named_parameters(), eager.parameters()):
+ torch.testing.assert_close(p, q, rtol = 2e-3, atol = 2e-4, msg = name)
+
+
+def test_fixed_shape_replays_without_pool_copies(monkeypatch):
+ c = Counters(monkeypatch)
+ seen = []
+ graphed, eager, out = _train(
+ [64] * 6, checkpoint = False, steps = lambda s: seen.append((c.record, c.replay))
+ )
+ _assert_same(graphed, eager, out)
+ assert seen[-1][1] > seen[2][1], seen # replays keep coming
+ assert seen[-1][0] == seen[2][0], seen # and nothing re-records after warm-up
+ assert c.clones == 0 # outputs stay in the pool inside one generation
+
+
+def test_reentrant_checkpoint_still_copies_and_matches(monkeypatch):
+ c = Counters(monkeypatch)
+ graphed, eager, out = _train([64] * 5, checkpoint = True)
+ _assert_same(graphed, eager, out)
+ assert c.replay > 0 and c.clones > 0
+
+
+def test_variable_lengths_turn_graphs_off_and_release_the_pool(monkeypatch):
+ monkeypatch.setattr(bc, "_CUDAGRAPH_MAX_SHAPES", 2)
+ c = Counters(monkeypatch)
+ records = []
+
+ def after(step):
+ torch.cuda.synchronize()
+ records.append(c.record)
+
+ lengths = [256, 256, 384, 384, 512, 256, 640, 768]
+ torch.cuda.empty_cache()
+ base = torch.cuda.memory_reserved()
+ graphed, eager, out = _train(lengths, checkpoint = False, steps = after)
+ _assert_same(graphed, eager, out)
+ assert records[3] > 0, records
+ # The third distinct length (step 4) turns graphs off for good: nothing recorded afterwards.
+ assert records[-1] == records[3], records
+ del graphed, eager, out
+ torch.cuda.synchronize()
+ torch.cuda.empty_cache()
+ left = torch.cuda.memory_reserved() - base
+ # The graph pool is gone: what stays reserved is at most allocator slack.
+ assert left < 64 * 2**20, left / 2**20
+ state = getattr(bc, "_CUDAGRAPH_STATE", None)
+ if state is not None:
+ assert state["off"] == "shapes" and state["release"] == "done", state
diff --git a/tests/test_fsdp2_native_block_cudagraphs_gloo.py b/tests/test_fsdp2_native_block_cudagraphs_gloo.py
index 40604358107..52a1ce7f463 100644
--- a/tests/test_fsdp2_native_block_cudagraphs_gloo.py
+++ b/tests/test_fsdp2_native_block_cudagraphs_gloo.py
@@ -112,7 +112,8 @@ def _train(native, model, layout, rank):
opt.zero_grad(set_to_none = True)
losses.append(float(loss.detach()))
gathered.append(_gathered(model))
- return {"losses": losses, "gathered": gathered}
+ graphs_off = sum(bool(m.__dict__.get("_unsloth_block_graphs_off")) for m in model.modules())
+ return {"losses": losses, "gathered": gathered, "graphs_off_blocks": graphs_off}
def _worker(rank, world, port, out, path):
@@ -125,6 +126,11 @@ def _worker(rank, world, port, out, path):
for graphs in (False, True)
}
res["full", "no_blocks"] = _run(native, "full", True, blocks = False)
+ os.environ["UNSLOTH_BLOCK_CUDAGRAPHS_FSDP2_KEEP_GB"] = "0"
+ try:
+ res["full", "over_budget"] = _run(native, "full", True)
+ finally:
+ os.environ.pop("UNSLOTH_BLOCK_CUDAGRAPHS_FSDP2_KEEP_GB", None)
torch.save(res, f"{path}.rank{rank}")
out.put("ok")
except Exception as exc:
@@ -181,3 +187,20 @@ def test_without_graphs_the_gathered_weights_are_freed(ranks, layout):
for res in ranks:
snap = res[layout, False]["gathered"][-1]
assert any(size == 0 for _, size, _ in snap), snap
+
+
+def test_over_the_keep_budget_nothing_is_kept_and_those_blocks_skip_graphs(ranks):
+ """A sharded full finetune kept every block weight gathered in fp32 (T4 Qwen3-0.6B: 1.64 GiB
+ per rank) for graphs that measured no faster: over the budget keep nothing, run no graphs."""
+ for res in ranks:
+ run = res["full", "over_budget"]
+ blocks = [g for g in run["gathered"][-1] if g[0]]
+ assert blocks and any(size == 0 for _, size, _ in blocks), blocks
+ assert run["graphs_off_blocks"] > 0
+ assert run["losses"] == res["full", False]["losses"]
+
+
[email protected]("layout", ["lora", "full"])
+def test_within_the_keep_budget_blocks_keep_their_graphs(ranks, layout):
+ for res in ranks:
+ assert res[layout, True]["graphs_off_blocks"] == 0
diff --git a/unsloth/models/_block_compile.py b/unsloth/models/_block_compile.py
index ce842becd41..19f6414e877 100644
--- a/unsloth/models/_block_compile.py
+++ b/unsloth/models/_block_compile.py
@@ -27,11 +27,280 @@ _COMPILE_OPTIONS = {
if os.environ.get("UNSLOTH_BLOCK_CUDAGRAPHS", "0") == "1":
_COMPILE_OPTIONS["triton.cudagraphs"] = True
# Shape cap: variable lengths recorded a graph set per length (3.5x slower, pool grew every step).
-# Past the cap graphs go off for the whole layer kind; admitted shapes alone still grew the pool.
+# Two shapes = one fixed length plus a short last batch. Past the cap graphs go off for every block
+# and the graph pool is released (T4: 8 shapes cost 15-65 s over 40 steps and kept 3-4.75 GB).
try:
- _CUDAGRAPH_MAX_SHAPES = max(0, int(os.environ.get("UNSLOTH_BLOCK_CUDAGRAPHS_MAX_SHAPES", "8")))
+ _CUDAGRAPH_MAX_SHAPES = max(0, int(os.environ.get("UNSLOTH_BLOCK_CUDAGRAPHS_MAX_SHAPES", "2")))
except ValueError:
- _CUDAGRAPH_MAX_SHAPES = 8
+ _CUDAGRAPH_MAX_SHAPES = 2
+
+
+# Process wide: every block class shares one cudagraph tree (one pool per device).
+# off: why graphs went off (None = on); release: None, "done" or "kept"; ours: compile ids of the
+# tree functions block calls created; managers: CUDA index -> the tree manager block graphs ran on.
+def _new_cudagraph_state():
+ return {"off": None, "release": None, "ours": set(), "managers": {}, "notes": set()}
+
+
+_CUDAGRAPH_STATE = _new_cudagraph_state()
+
+
+_PACKED_NOTE = (
+ "Unsloth: UNSLOTH_BLOCK_CUDAGRAPHS=1 does not apply to packed / padding-free rows (their total "
+ "length changes every step, so a graph would be recorded per step); blocks run without CUDA graphs."
+)
+_SHAPES_NOTE = (
+ "Unsloth: block CUDA graphs turned off: more than {} distinct batch shapes (variable padded "
+ "lengths). Pad every batch to one length (padding='max_length') to keep them, or raise "
+ "UNSLOTH_BLOCK_CUDAGRAPHS_MAX_SHAPES."
+)
+
+
+def _rank0_note(key, message):
+ state = _CUDAGRAPH_STATE
+ if key in state["notes"]:
+ return
+ state["notes"].add(key)
+ try:
+ import torch.distributed as dist
+ if dist.is_available() and dist.is_initialized() and dist.get_rank() != 0:
+ return
+ except Exception:
+ pass
+ print(message, flush = True)
+
+
+def _graphs_off(reason, message):
+ """Every block runs its graphs-off copy from now on; the pool is released at a later call."""
+ if _CUDAGRAPH_STATE["off"] is None:
+ _CUDAGRAPH_STATE["off"] = reason
+ _rank0_note("off", message)
+
+
+def _tree_manager(device):
+ if device is None:
+ return None
+ try:
+ from torch._inductor import cudagraph_trees
+ return cudagraph_trees.get_manager(device, create_if_none_exists = False)
+ except Exception:
+ return None
+
+
+def _graphed_call(compiled, device, *args, **kwargs):
+ """Call the graphs-on region; note the compile ids of tree functions it created (forward here,
+ the backward of the same compile later), so the release can tell its pool is only ours."""
+ managers = _CUDAGRAPH_STATE["managers"]
+ manager = managers.get(device)
+ if manager is None:
+ # The manager may predate the first block call (another region made it).
+ manager = _tree_manager(device)
+ if manager is not None:
+ managers[device] = manager
+ before = len(manager.ids_to_funcs) if manager is not None else 0
+ out = _mark_block_outputs(compiled(*args, **kwargs))
+ if manager is None:
+ manager = _tree_manager(device)
+ if manager is None:
+ return out
+ managers[device] = manager
+ if len(manager.ids_to_funcs) > before:
+ ids = list(manager.ids_to_funcs)[before:]
+ _CUDAGRAPH_STATE["ours"].update(manager.id_to_compile_id.get(i) for i in ids)
+ return out
+
+
+_BLOCK_NODE = "_unsloth_graphed_block"
+
+
+def _mark_block_outputs(out):
+ """Tag the autograd nodes of a graphed block call, so a later block's input check can stop there:
+ that call decided its own input gradient. Other compiled functions may pass gradients through."""
+ import torch.utils._pytree as pytree
+
+ for x in pytree.tree_leaves(out):
+ if torch.is_tensor(x) and x.grad_fn is not None:
+ x.grad_fn.metadata[_BLOCK_NODE] = True
+ return out
+
+
+def _pool_release_verdict(managers, ours):
+ """ "ok" when every tree function belongs to a block compile and nothing in the pool is alive
+ (no pending backward), "pending" to retry at a later call, "foreign" when another torch.compile
+ region also records into this tree: resetting would break its next call."""
+ for manager in managers:
+ if any(cid is None or cid not in ours for cid in manager.id_to_compile_id.values()):
+ return "foreign"
+ for manager in managers:
+ if manager.running_forwards_with_pending_backwards:
+ return "pending"
+ if manager.live_cudagraph_pool_storages_in_curr_execution():
+ return "pending"
+ return "ok"
+
+
+def _release_graph_pool():
+ """Free the cudagraph tree pool via reset_cudagraph_trees(), only when every tree function is ours:
+ a foreign one called afterwards fails ("Running CUDAGraph after shutdown")."""
+ state = _CUDAGRAPH_STATE
+ if state["release"] is not None:
+ return
+ try:
+ from torch._inductor import cudagraph_trees
+ containers = cudagraph_trees.get_obj(cudagraph_trees.local, "tree_manager_containers")
+ managers = [c.tree_manager for c in list(containers.values()) if c.tree_manager is not None]
+ except Exception:
+ state["release"] = "kept"
+ return
+ if not managers:
+ state["release"] = "done" # nothing was recorded
+ return
+ verdict = _pool_release_verdict(managers, state["ours"])
+ if verdict == "pending":
+ return
+ if verdict == "foreign":
+ state["release"] = "kept"
+ _rank0_note(
+ "kept",
+ "Unsloth: block CUDA graphs are off; their memory pool stays allocated because another "
+ "compiled region records into the same CUDA graph tree.",
+ )
+ return
+ devices = sorted(state["managers"])
+ before = sum(torch.cuda.memory_reserved(d) for d in devices)
+ cudagraph_trees.reset_cudagraph_trees()
+ torch.cuda.empty_cache()
+ freed = before - sum(torch.cuda.memory_reserved(d) for d in devices)
+ state["release"] = "done"
+ state["managers"].clear()
+ _rank0_note(
+ "released",
+ f"Unsloth: released the block CUDA graph pool ({max(freed, 0) / 2**30:.2f} GiB reserved freed).",
+ )
+
+
+def _packed_kwargs(kwargs):
+ get = kwargs.get
+ return (
+ get("packed_seq_lengths") is not None
+ or get("cu_seq_lens_q") is not None
+ or get("cu_seq_lens_k") is not None
+ or get("cu_seqlens") is not None
+ )
+
+
+def _trainable(layer):
+ params = layer.__dict__.get("_unsloth_block_params")
+ if params is None:
+ params = layer.__dict__["_unsloth_block_params"] = list(layer.parameters())
+ return any(p.requires_grad for p in params)
+
+
+# The grad-mode trial before a checkpointed block's first no-grad call stands in for the recompute.
+_IN_TRIAL = [False]
+
+
+# output_hidden_states hands block outputs to the caller, who may keep them past the generation.
+_HIDDEN_STATES_EXPOSED = [False]
+
+
+def _note_hidden_states_exposed(module, args, kwargs):
+ want = kwargs.get("output_hidden_states")
+ if want is None:
+ want = getattr(getattr(module, "config", None), "output_hidden_states", False)
+ _HIDDEN_STATES_EXPOSED[0] = bool(want)
+
+
+def _watch_hidden_states(model):
+ """Hook the model and each decoder list's owner to record whether hidden states are returned."""
+ modules = dict(model.named_modules())
+ lists = {id(layer_list) for layer_list in _layer_lists(model)}
+ owners = [model]
+ for name, module in modules.items():
+ if id(module) in lists and "." in name:
+ owners.append(modules[name.rsplit(".", 1)[0]])
+ for owner in owners:
+ if not owner.__dict__.get("_unsloth_watch_hidden_states", False):
+ owner.__dict__["_unsloth_watch_hidden_states"] = True
+ owner.register_forward_pre_hook(_note_hidden_states_exposed, with_kwargs = True)
+
+
+def _pool_outputs_stay_valid(layer, hidden_states):
+ """A grad-mode call outside backward leaves a forward pending, so cudagraph trees start no new
+ generation (can_start_new_generation) and its outputs need no copy out of the pool. Checkpointed
+ calls (no-grad forward, recompute in backward, and the trial standing in for it) may start one."""
+ return (
+ not _IN_TRIAL[0]
+ and not _HIDDEN_STATES_EXPOSED[0]
+ and torch.is_grad_enabled()
+ and torch._C._current_graph_task_id() == -1
+ and (hidden_states.requires_grad or _trainable(layer))
+ )
+
+
+def _input_grad_must_leave_pool(hidden_states):
+ """True when the input gradient outlives the generation: a leaf's .grad (kept live, so the pool is
+ never released) or a checkpoint's backward, which recomputes (a new generation) before reading it."""
+ if not hidden_states.requires_grad:
+ return False
+ node = hidden_states.grad_fn
+ if node is None:
+ return True # leaf: its .grad would be pool memory
+ # (node, fresh): fresh once the path passed an op whose backward allocates its input gradient.
+ stack, seen = [(node, False)], 0
+ while stack:
+ node, fresh = stack.pop()
+ seen += 1
+ if seen > 32:
+ return True
+ name = type(node).__name__
+ if name.startswith("CompiledFunction") and node.metadata.get(_BLOCK_NODE, False):
+ continue # the previous block's region: its own input is its own call's decision
+ if isinstance(node, torch.autograd.function.BackwardCFunction):
+ forward_cls = getattr(node, "_forward_cls", None)
+ if not (isinstance(forward_cls, type) and issubclass(forward_cls, _CloneGrad)):
+ return True # a checkpoint (or other Python autograd Function) may recompute first
+ if name == "AccumulateGrad" and not fresh:
+ return True # a view / add of a leaf (inputs_embeds = param.view(...)) keeps the pool gradient
+ allocates = name.startswith(_ALLOCATING_BACKWARD)
+ for i, (n, _) in enumerate(node.next_functions):
+ if n is None:
+ continue
+ edge_fresh = fresh or allocates
+ if allocates and i == 0 and name.startswith(_ADDITIVE_BACKWARD):
+ # addmm / baddbmm hand their additive input `grad` itself (beta = 1), unless it was
+ # broadcast (a bias), which the engine sums to its shape in new storage
+ full = 3 if name.startswith("Baddbmm") else 2
+ variable = getattr(n, "variable", None)
+ edge_fresh = fresh or (variable is not None and variable.dim() < full)
+ stack.append((n, edge_fresh))
+ return False
+
+
+_ADDITIVE_BACKWARD = ("AddmmBackward", "BaddbmmBackward")
+
+
+# Backward nodes that write their input gradient to new storage (a view, add, clone or cat passes
+# on the incoming gradient or a narrowed view of it).
+_ALLOCATING_BACKWARD = (
+ "MmBackward",
+ "AddmmBackward",
+ "BmmBackward",
+ "BaddbmmBackward",
+ "MulBackward",
+ "DivBackward",
+ "EmbeddingBackward",
+ "EmbeddingDenseBackward",
+ "NativeLayerNormBackward",
+ "FusedRmsNormBackward",
+ "SoftmaxBackward",
+ "SiluBackward",
+ "GeluBackward",
+ "IndexBackward",
+ "SliceBackward",
+ "SelectBackward",
+ "ConvolutionBackward",
+)
class _CloneGrad(torch.autograd.Function):
@@ -95,32 +364,27 @@ def _graphs_unsafe_moe(layer):
for m in layer.modules()
)
layer.__dict__["_unsloth_block_graphs_moe"] = moe
- if moe and not _GRAPHS_MOE_LOGGED[0]:
- _GRAPHS_MOE_LOGGED[0] = True
- print("Unsloth: block CUDA graphs skip MoE layers; they run compiled without graphs.")
+ if moe:
+ _rank0_note(
+ "moe",
+ "Unsloth: block CUDA graphs skip MoE layers; they run compiled without graphs.",
+ )
return moe
-_GRAPHS_MOE_LOGGED = [False]
-
-
def _flex_block_mask_call(args, kwargs):
"""BlockMask index tensors are static graph inputs rebuilt every step, so the graph re-records
each call (StaticInputIdxMismatch is outside the re-record limit); Gemma3 was slower with graphs."""
for value in (*args, *kwargs.values()):
if type(value).__name__ == "BlockMask":
- if not _GRAPHS_FLEX_LOGGED[0]:
- _GRAPHS_FLEX_LOGGED[0] = True
- print(
- "Unsloth: block CUDA graphs skip flex attention blocks; they run compiled without graphs."
- )
+ _rank0_note(
+ "flex",
+ "Unsloth: block CUDA graphs skip flex attention blocks; they run compiled without graphs.",
+ )
return True
return False
-_GRAPHS_FLEX_LOGGED = [False]
-
-
def _ddp_optimizer_mode():
# optimize_ddp also takes "python_reducer" / "no_optimization" strings: only "ddp_optimizer"
# (or True) makes dynamo split graphs (convert_frame.py checks == "ddp_optimizer").
@@ -904,7 +1168,6 @@ def _wrap(
**isolate,
)
graph_shapes = set()
- graphs_off = [False]
def run(self, hidden_states, args, kwargs, mode):
previous = inline_flag[0]
@@ -926,23 +1189,39 @@ def _wrap(
kwargs = {**kwargs, packed_module.BLOCK_VARLEN_KWARG: varlen}
# Capture hooks (router_logits) append to empty scratch lists: no guard on the layer index.
with capture_scratch(), _no_ddp_graph_split():
- if graphs and not graphs_off[0]:
+ if graphs and _CUDAGRAPH_STATE["off"] is None:
shape = tuple(hidden_states.shape)
- if shape not in graph_shapes:
- graph_shapes.add(shape)
- graphs_off[0] = len(graph_shapes) > _CUDAGRAPH_MAX_SHAPES
+ if _packed_kwargs(kwargs):
+ _graphs_off("packed", _PACKED_NOTE)
+ elif shape not in graph_shapes:
+ if len(graph_shapes) >= _CUDAGRAPH_MAX_SHAPES:
+ _graphs_off("shapes", _SHAPES_NOTE.format(_CUDAGRAPH_MAX_SHAPES))
+ else:
+ graph_shapes.add(shape)
if graphs and (
- graphs_off[0]
+ _CUDAGRAPH_STATE["off"] is not None
+ or self.__dict__.get("_unsloth_block_graphs_off", False)
or _in_nonreentrant_checkpoint()
or _graphs_unsafe_moe(self)
or _flex_block_mask_call(args, kwargs)
):
+ if _CUDAGRAPH_STATE["off"] is not None and _CUDAGRAPH_STATE["release"] is None:
+ _release_graph_pool()
return plain(self, hidden_states, *args, **kwargs)
if graphs:
_param_grads_out_of_pool(self)
+ device = hidden_states.device.index
+ if _pool_outputs_stay_valid(self, hidden_states):
+ if _input_grad_must_leave_pool(hidden_states):
+ # A leaf's .grad outlives the generation (trainable inputs_embeds); a
+ # checkpoint's backward recomputes (a new generation) before reading it.
+ hidden_states = _CloneGrad.apply(hidden_states)
+ return _graphed_call(compiled, device, self, hidden_states, *args, **kwargs)
if hidden_states.requires_grad and torch.is_grad_enabled():
hidden_states = _CloneGrad.apply(hidden_states)
- return _clone_outputs(compiled(self, hidden_states, *args, **kwargs))
+ return _clone_outputs(
+ _graphed_call(compiled, device, self, hidden_states, *args, **kwargs)
+ )
return compiled(self, hidden_states, *args, **kwargs)
finally:
inline_flag[0] = previous
@@ -980,6 +1259,13 @@ def _wrap(
# PrefixGrouper's int segment table would recompile at every new GRPO length.
or (unsloth_forward and kwargs.get("prefix_seg_info") is not None)
):
+ if (
+ graphs
+ and self.training
+ and _CUDAGRAPH_STATE["off"] is None
+ and _packed_kwargs(kwargs)
+ ):
+ _graphs_off("packed", _PACKED_NOTE)
return original(self, hidden_states, *args, **kwargs)
if tracing:
# Inside a user's torch.compile the enclosing graph owns the layer: no dynamic marking
@@ -1002,7 +1288,11 @@ def _wrap(
# Grad-mode trial first (RNG forked): a checkpoint recompute must not demote below this call.
devices = [hidden_states.device.index] if hidden_states.is_cuda else []
with torch.random.fork_rng(devices = devices), torch.enable_grad():
- forward(self, hidden_states.detach().requires_grad_(), *args, **kwargs)
+ _IN_TRIAL[0] = True
+ try:
+ forward(self, hidden_states.detach().requires_grad_(), *args, **kwargs)
+ finally:
+ _IN_TRIAL[0] = False
mode = self.__dict__["_unsloth_block_mode"]
if mode == _OFF:
return original(self, hidden_states, *args, **kwargs)
@@ -1315,6 +1605,8 @@ def patch_block_compile(model):
continue
_wrap(cls, _COMPILE_OPTIONS, len(kinds.get(cls, ())) or 1)
wrapped.append(cls.__name__)
+ if _COMPILE_OPTIONS.get("triton.cudagraphs"):
+ _watch_hidden_states(model)
if wrapped:
print(f"Unsloth: compiling each transformer block as one region ({', '.join(wrapped)}).")
return wrapped
diff --git a/unsloth/models/fsdp2_native.py b/unsloth/models/fsdp2_native.py
index e2612099d23..350c1333aea 100644
--- a/unsloth/models/fsdp2_native.py
+++ b/unsloth/models/fsdp2_native.py
@@ -1010,8 +1010,9 @@ def _skip_free_unsharded_param():
return None
-def _compiled_block_modules(model):
- """ids of every module inside a decoder block _block_compile wrapped (and did not turn off)."""
+def _compiled_block_modules(model, roots = None):
+ """ids of every module inside a decoder block _block_compile wrapped (and did not turn off);
+ `roots` (a dict) collects id -> block module."""
inside = set()
for module in model.modules():
if id(module) in inside or not getattr(
@@ -1020,21 +1021,41 @@ def _compiled_block_modules(model):
continue
if module.__dict__.get("_unsloth_block_mode", 1) == 0:
continue
- inside.update(id(m) for m in module.modules())
+ for m in module.modules():
+ inside.add(id(m))
+ if roots is not None:
+ roots[id(m)] = module
return inside
+def _block_cudagraphs_keep_budget():
+ """Per rank bytes of gathered block weights kept for block CUDA graphs. LoRA keeps only the
+ adapters (T4 Qwen3-0.6B: 0.04 GiB); a sharded full finetune keeps every block weight in fp32
+ (1.64 GiB per rank there) for graphs that measured no faster."""
+ try:
+ return float(os.environ.get("UNSLOTH_BLOCK_CUDAGRAPHS_FSDP2_KEEP_GB", "0.5")) * 2**30
+ except ValueError:
+ return 0.5 * 2**30
+
+
+def _gib(nbytes):
+ # LoRA adapters are tens of MiB: "0.00 GiB" read as nothing kept.
+ return f"{nbytes / 2**30:.2f} GiB" if nbytes >= 2**30 / 10 else f"{nbytes / 2**20:.1f} MiB"
+
+
def _keep_unsharded_for_block_cudagraphs(model):
"""FSDP2 frees gathered params at reshard, moving their addresses so block CUDA graphs re-record every
step (2 x T4 Qwen3 LoRA: 6 records, 0 replays). Keep the storage; the next all-gather refills it in place.
- Only weights inside compiled blocks: the rest (embeddings, head, final norm) never enter a block graph."""
+ Only weights inside compiled blocks: the rest (embeddings, head, final norm) never enter a block graph.
+ Over the budget nothing is kept and those blocks run without graphs (never recorded)."""
if not _block_cudagraphs_requested():
return 0
- blocks = _compiled_block_modules(model)
+ roots = {}
+ blocks = _compiled_block_modules(model, roots)
if not blocks:
return 0
FSDPModule = _fsdp_module_class()
- kept, nbytes = 0, 0
+ candidates, nbytes = [], 0
for module in model.modules():
if not isinstance(module, FSDPModule):
continue
@@ -1046,25 +1067,44 @@ def _keep_unsharded_for_block_cudagraphs(model):
if getattr(fsdp_param, "_unsloth_keep_unsharded", False):
continue
info = getattr(fsdp_param, "_module_info", None)
- if id(getattr(info, "module", None)) not in blocks:
+ owner = id(getattr(info, "module", None))
+ if owner not in blocks:
continue
if not callable(getattr(type(fsdp_param), "free_unsharded_param", None)):
- return kept # a torch without this FSDP2 internal: graphs re-record as before
- fsdp_param.free_unsharded_param = _skip_free_unsharded_param
- fsdp_param._unsloth_keep_unsharded = True
- kept += 1
+ return 0 # a torch without this FSDP2 internal: graphs re-record as before
+ candidates.append((fsdp_param, roots[owner]))
try:
- dtype = getattr(fsdp_param, "param_dtype", None) or fsdp_param.orig_dtype
+ # param_dtype / orig_dtype are set at FSDP2's lazy init (first forward), after this.
+ dtype = (
+ getattr(fsdp_param, "param_dtype", None)
+ or getattr(getattr(group, "mp_policy", None), "param_dtype", None)
+ or fsdp_param.sharded_param.dtype
+ )
nbytes += fsdp_param._orig_size.numel() * dtype.itemsize
except Exception:
pass
- if kept:
+ if not candidates:
+ return 0
+ budget = _block_cudagraphs_keep_budget()
+ if nbytes > budget:
+ layers = {id(block): block for _, block in candidates}
+ for block in layers.values():
+ block.__dict__["_unsloth_block_graphs_off"] = True
_print_rank0(
- f"Unsloth: UNSLOTH_BLOCK_CUDAGRAPHS=1 under native FSDP2 keeps the gathered weights of "
- f"{kept} parameters allocated ({nbytes / 2**30:.2f} GiB per rank) so the block CUDA graphs "
- "replay instead of re-recording every step."
+ f"Unsloth: UNSLOTH_BLOCK_CUDAGRAPHS=1 under native FSDP2 would keep {_gib(nbytes)} "
+ f"per rank of gathered block weights allocated (budget {_gib(budget)}, "
+ f"UNSLOTH_BLOCK_CUDAGRAPHS_FSDP2_KEEP_GB); {len(layers)} blocks run without CUDA graphs."
)
- return kept
+ return 0
+ for fsdp_param, _ in candidates:
+ fsdp_param.free_unsharded_param = _skip_free_unsharded_param
+ fsdp_param._unsloth_keep_unsharded = True
+ _print_rank0(
+ f"Unsloth: UNSLOTH_BLOCK_CUDAGRAPHS=1 under native FSDP2 keeps the gathered weights of "
+ f"{len(candidates)} parameters allocated ({_gib(nbytes)} per rank) so the block CUDA graphs "
+ "replay instead of re-recording every step."
+ )
+ return len(candidates)
def _post_shard(