Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Block CUDA graphs: decide before recording, release the pool, cap FSDP2 kept weights, drop unneeded copies (#15352)
Daniel Han2bcde87ffb81
diff --git a/tests/test_block_cudagraphs_clone.py b/tests/test_block_cudagraphs_clone.pyindex b498a429e61..a98adb7da36 100644--- a/tests/test_block_cudagraphs_clone.py+++ b/tests/test_block_cudagraphs_clone.py@@ -12,6 +12,8 @@ import importlib.utilimport osimport sys+import types+import pytestimport torch@@ -40,6 +42,13 @@ def _block_env(monkeypatch):# the eager path, so the graph-cap test would see no compiled call at all.for key in ("UNSLOTH_BLOCK_COMPILE", "UNSLOTH_COMPILE_DISABLE", "UNSLOTH_FORCE_FLOAT32"):monkeypatch.delenv(key, raising = False)+ # Graphs go off process wide; every test starts with them on.+ monkeypatch.setattr(+ bc,+ "_CUDAGRAPH_STATE",+ getattr(bc, "_new_cudagraph_state", dict)(),+ raising = False,+ )def test_clone_grad_is_identity_with_copied_gradient():@@ -132,7 +141,10 @@ def test_cap_reads_the_environment(monkeypatch):monkeypatch.setenv("UNSLOTH_BLOCK_CUDAGRAPHS_MAX_SHAPES", "3")assert _load()._CUDAGRAPH_MAX_SHAPES == 3monkeypatch.setenv("UNSLOTH_BLOCK_CUDAGRAPHS_MAX_SHAPES", "junk")- assert _load()._CUDAGRAPH_MAX_SHAPES == 8+ assert _load()._CUDAGRAPH_MAX_SHAPES == 2+ monkeypatch.delenv("UNSLOTH_BLOCK_CUDAGRAPHS_MAX_SHAPES")+ # One fixed length plus a short last batch; T4 variable lengths paid 15-65 s for 8.+ assert _load()._CUDAGRAPH_MAX_SHAPES == 2finally:monkeypatch.delenv("UNSLOTH_BLOCK_CUDAGRAPHS_MAX_SHAPES", raising = False)_load()@@ -279,6 +291,450 @@ def test_nonreentrant_checkpoint_runs_without_graphs(monkeypatch):assert not bc._in_nonreentrant_checkpoint()+def _tagging_compile(monkeypatch, calls):+ def fake_compile(fn, **kwargs):+ tag = "graphs" if kwargs.get("options", {}).get("triton.cudagraphs") else "plain"++ def run(*args, **kw):+ calls.append(tag)+ return fn(*args, **kw)++ return run++ monkeypatch.setattr(torch, "compile", fake_compile)+++def _linear_layer_class(name, module = None):+ def __init__(self):+ torch.nn.Module.__init__(self)+ self.proj = torch.nn.Linear(8, 8)++ def forward(self, hidden_states, **kwargs):+ return self.proj(hidden_states)++ if module is not None:+ forward.__module__ = module+ return type(name, (torch.nn.Module,), {"__init__": __init__, "forward": forward})+++def test_past_the_cap_every_block_class_goes_off_and_the_pool_is_released(monkeypatch):+ """One cudagraph tree (one pool) per device serves every block class: graphs go off for all of+ them together, so the release can free the whole pool."""+ calls, released = [], []+ _tagging_compile(monkeypatch, calls)+ monkeypatch.setattr(bc, "_CUDAGRAPH_MAX_SHAPES", 1)+ monkeypatch.setattr(+ bc,+ "_release_graph_pool",+ lambda: released.append(1) or bc._CUDAGRAPH_STATE.update(release = "done"),+ )+ A, B = _linear_layer_class("ADecoderLayer"), _linear_layer_class("BDecoderLayer")+ for cls in (A, B):+ bc._wrap(cls, {**bc._COMPILE_OPTIONS, "triton.cudagraphs": True})+ a, b = A().train(), B().train()++ def step(L):+ calls.clear()+ x = torch.randn(2, L, 8, requires_grad = True)+ b(a(x)).sum().backward()+ return list(calls)++ assert step(16) == ["graphs", "graphs"]+ assert step(16) == ["graphs", "graphs"]+ assert not released+ assert step(24) == ["plain", "plain"] # A saw the new length first: B is off too+ assert bc._CUDAGRAPH_STATE["off"] == "shapes"+ assert released == [1]+ assert step(16) == ["plain", "plain"]+ assert released == [1]+++def test_cap_notice_is_printed_once(monkeypatch, capsys):+ calls = []+ _tagging_compile(monkeypatch, calls)+ monkeypatch.setattr(bc, "_CUDAGRAPH_MAX_SHAPES", 1)+ A = _linear_layer_class("NoteDecoderLayer")+ bc._wrap(A, {**bc._COMPILE_OPTIONS, "triton.cudagraphs": True})+ layer = A().train()+ for L in (8, 16, 24, 32):+ layer(torch.randn(2, L, 8, requires_grad = True)).sum().backward()+ out = capsys.readouterr().out+ assert out.count("block CUDA graphs turned off") == 1+ assert "UNSLOTH_BLOCK_CUDAGRAPHS_MAX_SHAPES" in out++[email protected]("key", ["packed_seq_lengths", "cu_seq_lens_q"])+def test_packed_rows_turn_graphs_off_before_any_recording(monkeypatch, capsys, key):+ """Packed / padding-free rows change total length every step: decide before recording."""+ calls = []+ _tagging_compile(monkeypatch, calls)+ A = _linear_layer_class("PackedDecoderLayer")+ bc._wrap(A, {**bc._COMPILE_OPTIONS, "triton.cudagraphs": True})+ layer = A().train()+ lengths = torch.tensor([5, 11], dtype = torch.int32)+ layer(torch.randn(1, 16, 8, requires_grad = True), **{key: lengths}).sum().backward()+ assert calls == ["plain"]+ assert bc._CUDAGRAPH_STATE["off"] == "packed"+ calls.clear()+ layer(torch.randn(2, 16, 8, requires_grad = True)).sum().backward() # later padded rows too+ assert calls == ["plain"]+ assert capsys.readouterr().out.count("packed / padding-free rows") == 1+++def test_packed_rows_on_unsloth_forward_say_graphs_do_not_apply(monkeypatch, capsys):+ # Unsloth's own forwards run packed rows stock (no compiled call at all): say so once.+ calls = []+ _tagging_compile(monkeypatch, calls)+ A = _linear_layer_class("UPackedDecoderLayer", module = "unsloth.models.fake")+ bc._wrap(A, {**bc._COMPILE_OPTIONS, "triton.cudagraphs": True})+ layer = A().train()+ lengths = torch.tensor([5, 11], dtype = torch.int32)+ layer(torch.randn(1, 16, 8, requires_grad = True), packed_seq_lengths = lengths).sum().backward()+ assert calls == []+ assert bc._CUDAGRAPH_STATE["off"] == "packed"+ assert "packed / padding-free rows" in capsys.readouterr().out+++def test_flag_unset_ignores_packed_rows(monkeypatch, capsys):+ calls = []+ _tagging_compile(monkeypatch, calls)+ A = _linear_layer_class("NoFlagDecoderLayer")+ bc._wrap(A, {**bc._COMPILE_OPTIONS, "triton.cudagraphs": False})+ layer = A().train()+ lengths = torch.tensor([5, 11], dtype = torch.int32)+ layer(torch.randn(1, 16, 8, requires_grad = True), packed_seq_lengths = lengths).sum().backward()+ assert calls == ["plain"]+ assert bc._CUDAGRAPH_STATE["off"] is None+ assert capsys.readouterr().out == ""+++def _counting_clones(monkeypatch):+ seen = {"outputs": 0, "grad": 0}+ clone_outputs = bc._clone_outputs++ def counted(out):+ seen["outputs"] += 1+ return clone_outputs(out)++ class Counted(bc._CloneGrad):+ @staticmethod+ def forward(ctx, x):+ seen["grad"] += 1+ return x.view_as(x)++ monkeypatch.setattr(bc, "_clone_outputs", counted)+ monkeypatch.setattr(bc, "_CloneGrad", Counted)+ return seen++[email protected]("trainable_input", [True, False])+def test_no_pool_copies_outside_checkpointing(monkeypatch, trainable_input):+ """A grad-mode block call outside backward leaves a forward pending, so cudagraph trees keep+ the next block and the backward in the same generation: its outputs need no copy."""+ monkeypatch.setattr(torch, "compile", lambda fn, **kwargs: fn)+ seen = _counting_clones(monkeypatch)+ A = _linear_layer_class("FreeDecoderLayer")+ bc._wrap(A, {**bc._COMPILE_OPTIONS, "triton.cudagraphs": True})+ layers = [A().train() for _ in range(3)]+ x = torch.randn(2, 4, 8, requires_grad = trainable_input)+ y = x+ for layer in layers:+ y = layer(y)+ y.sum().backward()+ # Only a leaf input's gradient is copied: its .grad outlives the generation.+ assert seen == {"outputs": 0, "grad": 1 if trainable_input else 0}+ assert all(layer.proj.weight.grad is not None for layer in layers)+ assert (x.grad is not None) is trainable_input+++ "through", ["view", "view_add", "cat", "addmm_input", "addmm_bias", "embedding"]+)+def test_input_viewing_a_trainable_param_copies_its_gradient(monkeypatch, through):+ """inputs_embeds = param.view(...): the view's backward passes the pool gradient straight to+ AccumulateGrad, so the parameter's first .grad would alias the pool. An embedding lookup writes+ a new gradient, so it needs no copy."""+ monkeypatch.setattr(torch, "compile", lambda fn, **kwargs: fn)+ seen = _counting_clones(monkeypatch)+ A = _linear_layer_class("ViewInputDecoderLayer")+ bc._wrap(A, {**bc._COMPILE_OPTIONS, "triton.cudagraphs": True})+ layer = A().train()+ param = torch.nn.Parameter(torch.randn(8, 8))+ if through == "embedding":+ x = torch.nn.functional.embedding(torch.tensor([[0, 1, 2, 3], [4, 5, 6, 7]]), param)+ else:+ x = param.view(2, 4, 8)+ if through == "view_add":+ x = x + torch.zeros(2, 4, 8)+ elif through in ("addmm_input", "addmm_bias"):+ # addmm's backward passes `grad` itself to its additive input; a broadcast bias is summed+ a, b = torch.randn(8, 8), torch.randn(8, 8)+ if through == "addmm_bias":+ param = torch.nn.Parameter(torch.randn(8))+ x = torch.addmm(param, a, b).view(2, 4, 8)+ elif through == "cat":+ # cat's backward hands each input a narrowed view of the incoming gradient+ param = torch.nn.Parameter(torch.randn(4, 8))+ x = torch.cat([param.view(1, 4, 8), torch.zeros(1, 4, 8)], dim = 0)+ layer(x).sum().backward()+ assert seen["grad"] == (0 if through in ("embedding", "addmm_bias") else 1), seen+ assert param.grad is not None++[email protected]("source", ["block", "foreign"])+def test_only_graphed_block_regions_stop_the_input_gradient_walk(monkeypatch, source):+ """A previous graphed block decided its own input gradient, so the walk stops at its compiled+ node. Any other compiled function (a compiled prompt preprocessor) can hand the incoming+ gradient straight to a trainable leaf, so its first .grad would alias the pool: copy it."""+ real_compile = torch.compile+ torch._dynamo.reset()+ # aot_eager gives the real CompiledFunctionBackward nodes on CPU; graphs themselves are faked.+ monkeypatch.setattr(+ torch, "compile", lambda fn, **kwargs: real_compile(fn, backend = "aot_eager")+ )+ seen = _counting_clones(monkeypatch)+ A = _linear_layer_class("CompiledRegionDecoderLayer")+ bc._wrap(A, {**bc._COMPILE_OPTIONS, "triton.cudagraphs": True})+ layers = [A().train() for _ in range(2)]+ if source == "foreign":+ prompt = torch.nn.Parameter(torch.randn(2, 4, 8))+ x = real_compile(lambda p: p.clone(), backend = "aot_eager")(prompt)+ assert type(x.grad_fn).__name__.startswith("CompiledFunction")+ layers[0](x).sum().backward()+ assert seen["grad"] == 1, seen+ assert prompt.grad is not None+ else:+ # block-to-block hand-off: no copy, the memory win stays+ y = layers[0](torch.randn(2, 4, 8))+ assert type(y.grad_fn).__name__.startswith("CompiledFunction")+ layers[1](y).sum().backward()+ assert seen["grad"] == 0, seen+ assert layers[0].proj.weight.grad is not None+ torch._dynamo.reset()++[email protected]("exposed", ["kwarg", "config", None])+def test_returned_hidden_states_keep_their_pool_copies(monkeypatch, exposed):+ """output_hidden_states hands every block output to the caller, who may keep it past the+ generation: those calls copy out of the pool as before; other calls still skip the copies."""+ monkeypatch.setattr(torch, "compile", lambda fn, **kwargs: fn)+ monkeypatch.setitem(bc._COMPILE_OPTIONS, "triton.cudagraphs", True)+ seen = _counting_clones(monkeypatch)+ A = _linear_layer_class("ExposedDecoderLayer")+ bc._wrap(A, {**bc._COMPILE_OPTIONS, "triton.cudagraphs": True})++ class Inner(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.config = types.SimpleNamespace(output_hidden_states = exposed == "config")+ self.layers = torch.nn.ModuleList([A().train() for _ in range(2)])++ def forward(+ self,+ x,+ output_hidden_states = None,+ ):+ hidden = []+ for layer in self.layers:+ x = layer(x)+ hidden.append(x)+ return x, hidden++ inner = Inner()+ bc._watch_hidden_states(inner)+ x = torch.randn(2, 4, 8)+ kwargs = {"output_hidden_states": True} if exposed == "kwarg" else {}+ y, _ = inner(x, **kwargs)+ y.sum().backward()+ assert seen["outputs"] == (0 if exposed is None else 2), seen+ bc._HIDDEN_STATES_EXPOSED[0] = False++[email protected]("use_reentrant", [True])+def test_block_after_a_checkpointed_block_copies_its_input_gradient(monkeypatch, use_reentrant):+ """Checkpointed blocks then an uncheckpointed one: the checkpoint's backward recomputes (a new+ torch.compile invocation, a new generation) before it reads the gradient handed to it."""+ from torch.utils.checkpoint import checkpoint++ monkeypatch.setattr(torch, "compile", lambda fn, **kwargs: fn)+ seen = _counting_clones(monkeypatch)+ A = _linear_layer_class("MixedDecoderLayer")+ bc._wrap(A, {**bc._COMPILE_OPTIONS, "triton.cudagraphs": True})+ layers = [A().train() for _ in range(3)]+ x = torch.randn(2, 4, 8, requires_grad = True)+ y = x+ for layer in layers[:2]:+ y = checkpoint(layer, y, use_reentrant = use_reentrant)+ before = dict(seen)+ out = layers[2](y)+ # the last block (grad mode, outside backward) still routes its input gradient out of the pool+ assert seen == {"outputs": before["outputs"], "grad": before["grad"] + 1}+ out.sum().backward()+ assert x.grad is not None++[email protected]("reentrant_kind", ["checkpoint", "no_grad"])+def test_pool_copies_kept_where_a_block_call_starts_a_generation(monkeypatch, reentrant_kind):+ from torch.utils.checkpoint import checkpoint++ monkeypatch.setattr(torch, "compile", lambda fn, **kwargs: fn)+ seen = _counting_clones(monkeypatch)+ A = _linear_layer_class("CopyDecoderLayer")+ bc._wrap(A, {**bc._COMPILE_OPTIONS, "triton.cudagraphs": True})+ layers = [A().train() for _ in range(3)]+ x = torch.randn(2, 4, 8, requires_grad = True)+ if reentrant_kind == "no_grad":+ with torch.no_grad():+ for layer in layers:+ x = layer(x)+ assert seen == {"outputs": 6, "grad": 3} # each first call runs a grad-mode trial too+ return+ y = x+ for layer in layers:+ y = checkpoint(layer, y, use_reentrant = True)+ y.sum().backward()+ # Trial + no-grad forward + recompute inside backward: every call copies (main did the same).+ assert seen == {"outputs": 9, "grad": 6}+++def test_frozen_block_without_grad_input_still_copies(monkeypatch):+ # Nothing requires grad: an inference graph, which may start a new generation per call.+ monkeypatch.setattr(torch, "compile", lambda fn, **kwargs: fn)+ seen = _counting_clones(monkeypatch)+ A = _linear_layer_class("FrozenDecoderLayer")+ bc._wrap(A, {**bc._COMPILE_OPTIONS, "triton.cudagraphs": True})+ layer = A().train().requires_grad_(False)+ layer(torch.randn(2, 4, 8))+ assert seen == {"outputs": 1, "grad": 0}+++class _FakeManager:+ def __init__(+ self,+ compile_ids,+ pending = False,+ live = (),+ ):+ self.id_to_compile_id = dict(enumerate(compile_ids))+ self.ids_to_funcs = dict.fromkeys(self.id_to_compile_id)+ self.running_forwards_with_pending_backwards = pending+ self._live = list(live)++ def live_cudagraph_pool_storages_in_curr_execution(self):+ return self._live+++def test_pool_release_verdict():+ ours = {"a", "b"}+ assert bc._pool_release_verdict([_FakeManager(["a", "b", "a"])], ours) == "ok"+ assert bc._pool_release_verdict([_FakeManager(["a", "c"])], ours) == "foreign"+ assert bc._pool_release_verdict([_FakeManager(["a", None])], ours) == "foreign"+ assert bc._pool_release_verdict([_FakeManager(["a"], pending = True)], ours) == "pending"+ assert bc._pool_release_verdict([_FakeManager(["a"], live = [object()])], ours) == "pending"++[email protected]("ids", [["a", "a"], ["a", "x"]])+def test_release_resets_the_trees_only_when_every_function_is_ours(monkeypatch, capsys, ids):+ from types import SimpleNamespace++ from torch._inductor import cudagraph_trees++ manager = _FakeManager(ids)+ resets = []+ containers = {0: SimpleNamespace(tree_manager = manager)}+ monkeypatch.setattr(cudagraph_trees, "get_obj", lambda local, name: containers)+ monkeypatch.setattr(cudagraph_trees, "reset_cudagraph_trees", lambda: resets.append(1))+ reserved = [6 * 2**30]+ monkeypatch.setattr(torch.cuda, "memory_reserved", lambda d = None: reserved[0])+ monkeypatch.setattr(torch.cuda, "empty_cache", lambda: reserved.__setitem__(0, 2 * 2**30))+ bc._CUDAGRAPH_STATE["ours"].add("a")+ bc._CUDAGRAPH_STATE["managers"][0] = manager+ bc._release_graph_pool()+ out = capsys.readouterr().out+ if ids == ["a", "a"]:+ assert resets == [1] and bc._CUDAGRAPH_STATE["release"] == "done"+ assert "4.00 GiB reserved freed" in out+ else:+ assert resets == [] and bc._CUDAGRAPH_STATE["release"] == "kept"+ assert "stays allocated" in out+ bc._release_graph_pool() # decided: never again+ assert len(resets) == (1 if ids == ["a", "a"] else 0)+++def test_release_waits_for_a_pending_backward(monkeypatch):+ from types import SimpleNamespace++ from torch._inductor import cudagraph_trees++ manager = _FakeManager(["a"], pending = True)+ resets = []+ monkeypatch.setattr(+ cudagraph_trees, "get_obj", lambda local, name: {0: SimpleNamespace(tree_manager = manager)}+ )+ monkeypatch.setattr(cudagraph_trees, "reset_cudagraph_trees", lambda: resets.append(1))+ monkeypatch.setattr(torch.cuda, "memory_reserved", lambda d = None: 0)+ monkeypatch.setattr(torch.cuda, "empty_cache", lambda: None)+ bc._CUDAGRAPH_STATE["ours"].add("a")+ bc._release_graph_pool()+ assert resets == [] and bc._CUDAGRAPH_STATE["release"] is None+ manager.running_forwards_with_pending_backwards = False+ bc._release_graph_pool()+ assert resets == [1] and bc._CUDAGRAPH_STATE["release"] == "done"+++def test_block_marked_by_fsdp2_runs_without_graphs(monkeypatch):+ calls = []+ _tagging_compile(monkeypatch, calls)+ A = _linear_layer_class("MarkedDecoderLayer")+ bc._wrap(A, {**bc._COMPILE_OPTIONS, "triton.cudagraphs": True})+ kept, marked = A().train(), A().train()+ marked.__dict__["_unsloth_block_graphs_off"] = True+ marked(kept(torch.randn(2, 4, 8, requires_grad = True))).sum().backward()+ assert calls == ["graphs", "plain"]+ assert bc._CUDAGRAPH_STATE["off"] is None+++def test_graphed_call_records_the_compile_ids_it_created(monkeypatch):+ # The first call creates the tree (and its manager); later calls read the cached manager only.+ manager = _FakeManager([])+ lookups = []++ def tree_manager(device):+ lookups.append(device)+ return manager++ def compiled(n, cid):+ for _ in range(n):+ i = len(manager.ids_to_funcs)+ manager.ids_to_funcs[i] = None+ manager.id_to_compile_id[i] = cid+ return n++ monkeypatch.setattr(bc, "_tree_manager", tree_manager)+ manager.ids_to_funcs[0], manager.id_to_compile_id[0] = None, "foreign" # made before us+ assert bc._graphed_call(compiled, 0, 0, "x") == 0+ bc._graphed_call(compiled, 0, 2, "a")+ bc._graphed_call(compiled, 0, 0, "y")+ bc._graphed_call(compiled, 0, 1, "b")+ assert lookups == [0]+ assert bc._CUDAGRAPH_STATE["ours"] == {"a", "b"}+ assert bc._pool_release_verdict([manager], bc._CUDAGRAPH_STATE["ours"]) == "foreign"+++def test_grad_trial_of_a_checkpointed_block_takes_the_recompute_path(monkeypatch):+ # The first no-grad call (checkpoint forward) runs a grad-mode trial first; it stands in for the+ # recompute, so it copies like the recompute does.+ monkeypatch.setattr(torch, "compile", lambda fn, **kwargs: fn)+ seen = _counting_clones(monkeypatch)+ A = _linear_layer_class("TrialDecoderLayer")+ bc._wrap(A, {**bc._COMPILE_OPTIONS, "triton.cudagraphs": True})+ layer = A().train()+ with torch.no_grad():+ layer(torch.randn(2, 4, 8, requires_grad = True))+ assert seen == {"outputs": 2, "grad": 1} # trial (outputs + input grad) and the no-grad call+ assert bc._IN_TRIAL == [False]++class FakeMoeExperts(torch.nn.Module):def __init__(self):super().__init__()diff --git a/tests/test_block_cudagraphs_gpu.py b/tests/test_block_cudagraphs_gpu.pynew file mode 100644index 00000000000..e223ea87839--- /dev/null+++ b/tests/test_block_cudagraphs_gpu.py@@ -0,0 +1,204 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.++"""Block CUDA graphs (UNSLOTH_BLOCK_CUDAGRAPHS=1) on a real GPU with real cudagraph trees.++Fixed shape without checkpointing: replays, no copies out of the pool, weights match eager over+gradient accumulation. Reentrant checkpointing: still copies, still matches. Variable lengths: graphs+go off past the cap and the graph pool is released (it stayed reserved: +3 to 4.75 GB on a T4)."""++import importlib.util+import os+import sys++import pytest+import torch++pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason = "needs a CUDA GPU")++_DIR = os.path.join(+ os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "unsloth", "models"+)+H = 512+++def _load():+ spec = importlib.util.spec_from_file_location(+ "_ublock_compile_cg_gpu", os.path.join(_DIR, "_block_compile.py")+ )+ mod = importlib.util.module_from_spec(spec)+ sys.modules[spec.name] = mod+ spec.loader.exec_module(mod)+ return mod+++bc = _load()+++def _forward(self, hidden_states, **kwargs):+ x = self.norm(hidden_states)+ return hidden_states + self.down(torch.nn.functional.silu(self.up(x)))+++def _init(self):+ torch.nn.Module.__init__(self)+ self.norm = torch.nn.LayerNorm(H)+ self.up = torch.nn.Linear(H, 4 * H)+ self.down = torch.nn.Linear(4 * H, H)+++class EagerBlock(torch.nn.Module):+ __init__ = _init+ forward = _forward+++class GpuCgDecoderLayer(torch.nn.Module):+ __init__ = _init+ forward = _forward+++bc._wrap(GpuCgDecoderLayer, {**bc._COMPILE_OPTIONS, "triton.cudagraphs": True})+++class Counters:+ def __init__(self, monkeypatch):+ from torch._inductor import cudagraph_trees as T++ self.record = self.replay = self.clones = 0+ manager = T.CUDAGraphTreeManager+ record, execute = manager.record_function, manager.execute_node++ def counted_record(this, *a, **k):+ self.record += 1+ return record(this, *a, **k)++ def counted_execute(this, *a, **k):+ self.replay += 1+ return execute(this, *a, **k)++ monkeypatch.setattr(manager, "record_function", counted_record)+ monkeypatch.setattr(manager, "execute_node", counted_execute)+ clone = bc._clone_outputs++ def counted_clone(out):+ self.clones += 1+ return clone(out)++ monkeypatch.setattr(bc, "_clone_outputs", counted_clone)++[email protected](autouse = True)+def _fresh(monkeypatch):+ for key in ("UNSLOTH_BLOCK_COMPILE", "UNSLOTH_COMPILE_DISABLE", "UNSLOTH_FORCE_FLOAT32"):+ monkeypatch.delenv(key, raising = False)+ torch._dynamo.reset() # also resets the cudagraph trees+ if hasattr(bc, "_CUDAGRAPH_STATE"):+ monkeypatch.setattr(+ bc,+ "_CUDAGRAPH_STATE",+ getattr(bc, "_new_cudagraph_state", dict)(),+ )+ yield+ torch._dynamo.reset()+ torch.cuda.empty_cache()+++def _models(n = 3):+ torch.manual_seed(0)+ graphed = torch.nn.ModuleList(GpuCgDecoderLayer() for _ in range(n)).cuda().train()+ eager = torch.nn.ModuleList(EagerBlock() for _ in range(n)).cuda().train()+ eager.load_state_dict(graphed.state_dict())+ return graphed, eager+++def _step(layers, xs, opt, checkpoint):+ from torch.utils.checkpoint import checkpoint as ckpt++ losses = []+ for x in xs: # gradient accumulation: the first .grad of a micro step lives across generations+ y = x+ for layer in layers:+ y = ckpt(layer, y, use_reentrant = True) if checkpoint else layer(y)+ loss = y.float().square().mean()+ loss.backward()+ losses.append(loss.item())+ opt.step()+ opt.zero_grad(set_to_none = True)+ return losses+++def _train(+ lengths,+ checkpoint,+ micro = 2,+ steps = None,+):+ graphed, eager = _models()+ opts = [torch.optim.SGD(m.parameters(), lr = 0.1) for m in (graphed, eager)]+ out = {"graphed": [], "eager": []}+ for step, L in enumerate(lengths):+ g = torch.Generator(device = "cuda").manual_seed(step)+ xs = [+ torch.randn(2, L, H, device = "cuda", generator = g).requires_grad_(checkpoint)+ for _ in range(micro)+ ]+ out["graphed"].append(_step(graphed, xs, opts[0], checkpoint))+ out["eager"].append(+ _step(eager, [x.detach().requires_grad_(checkpoint) for x in xs], opts[1], checkpoint)+ )+ if steps is not None:+ steps(step)+ return graphed, eager, out+++def _assert_same(graphed, eager, out):+ torch.testing.assert_close(out["graphed"], out["eager"], rtol = 2e-3, atol = 2e-4)+ for (name, p), q in zip(graphed.named_parameters(), eager.parameters()):+ torch.testing.assert_close(p, q, rtol = 2e-3, atol = 2e-4, msg = name)+++def test_fixed_shape_replays_without_pool_copies(monkeypatch):+ c = Counters(monkeypatch)+ seen = []+ graphed, eager, out = _train(+ [64] * 6, checkpoint = False, steps = lambda s: seen.append((c.record, c.replay))+ )+ _assert_same(graphed, eager, out)+ assert seen[-1][1] > seen[2][1], seen # replays keep coming+ assert seen[-1][0] == seen[2][0], seen # and nothing re-records after warm-up+ assert c.clones == 0 # outputs stay in the pool inside one generation+++def test_reentrant_checkpoint_still_copies_and_matches(monkeypatch):+ c = Counters(monkeypatch)+ graphed, eager, out = _train([64] * 5, checkpoint = True)+ _assert_same(graphed, eager, out)+ assert c.replay > 0 and c.clones > 0+++def test_variable_lengths_turn_graphs_off_and_release_the_pool(monkeypatch):+ monkeypatch.setattr(bc, "_CUDAGRAPH_MAX_SHAPES", 2)+ c = Counters(monkeypatch)+ records = []++ def after(step):+ torch.cuda.synchronize()+ records.append(c.record)++ lengths = [256, 256, 384, 384, 512, 256, 640, 768]+ torch.cuda.empty_cache()+ base = torch.cuda.memory_reserved()+ graphed, eager, out = _train(lengths, checkpoint = False, steps = after)+ _assert_same(graphed, eager, out)+ assert records[3] > 0, records+ # The third distinct length (step 4) turns graphs off for good: nothing recorded afterwards.+ assert records[-1] == records[3], records+ del graphed, eager, out+ torch.cuda.synchronize()+ torch.cuda.empty_cache()+ left = torch.cuda.memory_reserved() - base+ # The graph pool is gone: what stays reserved is at most allocator slack.+ assert left < 64 * 2**20, left / 2**20+ state = getattr(bc, "_CUDAGRAPH_STATE", None)+ if state is not None:+ assert state["off"] == "shapes" and state["release"] == "done", statediff --git a/tests/test_fsdp2_native_block_cudagraphs_gloo.py b/tests/test_fsdp2_native_block_cudagraphs_gloo.pyindex 40604358107..52a1ce7f463 100644--- a/tests/test_fsdp2_native_block_cudagraphs_gloo.py+++ b/tests/test_fsdp2_native_block_cudagraphs_gloo.py@@ -112,7 +112,8 @@ def _train(native, model, layout, rank):opt.zero_grad(set_to_none = True)losses.append(float(loss.detach()))gathered.append(_gathered(model))- return {"losses": losses, "gathered": gathered}+ graphs_off = sum(bool(m.__dict__.get("_unsloth_block_graphs_off")) for m in model.modules())+ return {"losses": losses, "gathered": gathered, "graphs_off_blocks": graphs_off}def _worker(rank, world, port, out, path):@@ -125,6 +126,11 @@ def _worker(rank, world, port, out, path):for graphs in (False, True)}res["full", "no_blocks"] = _run(native, "full", True, blocks = False)+ os.environ["UNSLOTH_BLOCK_CUDAGRAPHS_FSDP2_KEEP_GB"] = "0"+ try:+ res["full", "over_budget"] = _run(native, "full", True)+ finally:+ os.environ.pop("UNSLOTH_BLOCK_CUDAGRAPHS_FSDP2_KEEP_GB", None)torch.save(res, f"{path}.rank{rank}")out.put("ok")except Exception as exc:@@ -181,3 +187,20 @@ def test_without_graphs_the_gathered_weights_are_freed(ranks, layout):for res in ranks:snap = res[layout, False]["gathered"][-1]assert any(size == 0 for _, size, _ in snap), snap+++def test_over_the_keep_budget_nothing_is_kept_and_those_blocks_skip_graphs(ranks):+ """A sharded full finetune kept every block weight gathered in fp32 (T4 Qwen3-0.6B: 1.64 GiB+ per rank) for graphs that measured no faster: over the budget keep nothing, run no graphs."""+ for res in ranks:+ run = res["full", "over_budget"]+ blocks = [g for g in run["gathered"][-1] if g[0]]+ assert blocks and any(size == 0 for _, size, _ in blocks), blocks+ assert run["graphs_off_blocks"] > 0+ assert run["losses"] == res["full", False]["losses"]++[email protected]("layout", ["lora", "full"])+def test_within_the_keep_budget_blocks_keep_their_graphs(ranks, layout):+ for res in ranks:+ assert res[layout, True]["graphs_off_blocks"] == 0diff --git a/unsloth/models/_block_compile.py b/unsloth/models/_block_compile.pyindex ce842becd41..19f6414e877 100644--- a/unsloth/models/_block_compile.py+++ b/unsloth/models/_block_compile.py@@ -27,11 +27,280 @@ _COMPILE_OPTIONS = {if os.environ.get("UNSLOTH_BLOCK_CUDAGRAPHS", "0") == "1":_COMPILE_OPTIONS["triton.cudagraphs"] = True# Shape cap: variable lengths recorded a graph set per length (3.5x slower, pool grew every step).-# Past the cap graphs go off for the whole layer kind; admitted shapes alone still grew the pool.+# Two shapes = one fixed length plus a short last batch. Past the cap graphs go off for every block+# and the graph pool is released (T4: 8 shapes cost 15-65 s over 40 steps and kept 3-4.75 GB).try:- _CUDAGRAPH_MAX_SHAPES = max(0, int(os.environ.get("UNSLOTH_BLOCK_CUDAGRAPHS_MAX_SHAPES", "8")))+ _CUDAGRAPH_MAX_SHAPES = max(0, int(os.environ.get("UNSLOTH_BLOCK_CUDAGRAPHS_MAX_SHAPES", "2")))except ValueError:- _CUDAGRAPH_MAX_SHAPES = 8+ _CUDAGRAPH_MAX_SHAPES = 2+++# Process wide: every block class shares one cudagraph tree (one pool per device).+# off: why graphs went off (None = on); release: None, "done" or "kept"; ours: compile ids of the+# tree functions block calls created; managers: CUDA index -> the tree manager block graphs ran on.+def _new_cudagraph_state():+ return {"off": None, "release": None, "ours": set(), "managers": {}, "notes": set()}+++_CUDAGRAPH_STATE = _new_cudagraph_state()+++_PACKED_NOTE = (+ "Unsloth: UNSLOTH_BLOCK_CUDAGRAPHS=1 does not apply to packed / padding-free rows (their total "+ "length changes every step, so a graph would be recorded per step); blocks run without CUDA graphs."+)+_SHAPES_NOTE = (+ "Unsloth: block CUDA graphs turned off: more than {} distinct batch shapes (variable padded "+ "lengths). Pad every batch to one length (padding='max_length') to keep them, or raise "+ "UNSLOTH_BLOCK_CUDAGRAPHS_MAX_SHAPES."+)+++def _rank0_note(key, message):+ state = _CUDAGRAPH_STATE+ if key in state["notes"]:+ return+ state["notes"].add(key)+ try:+ import torch.distributed as dist+ if dist.is_available() and dist.is_initialized() and dist.get_rank() != 0:+ return+ except Exception:+ pass+ print(message, flush = True)+++def _graphs_off(reason, message):+ """Every block runs its graphs-off copy from now on; the pool is released at a later call."""+ if _CUDAGRAPH_STATE["off"] is None:+ _CUDAGRAPH_STATE["off"] = reason+ _rank0_note("off", message)+++def _tree_manager(device):+ if device is None:+ return None+ try:+ from torch._inductor import cudagraph_trees+ return cudagraph_trees.get_manager(device, create_if_none_exists = False)+ except Exception:+ return None+++def _graphed_call(compiled, device, *args, **kwargs):+ """Call the graphs-on region; note the compile ids of tree functions it created (forward here,+ the backward of the same compile later), so the release can tell its pool is only ours."""+ managers = _CUDAGRAPH_STATE["managers"]+ manager = managers.get(device)+ if manager is None:+ # The manager may predate the first block call (another region made it).+ manager = _tree_manager(device)+ if manager is not None:+ managers[device] = manager+ before = len(manager.ids_to_funcs) if manager is not None else 0+ out = _mark_block_outputs(compiled(*args, **kwargs))+ if manager is None:+ manager = _tree_manager(device)+ if manager is None:+ return out+ managers[device] = manager+ if len(manager.ids_to_funcs) > before:+ ids = list(manager.ids_to_funcs)[before:]+ _CUDAGRAPH_STATE["ours"].update(manager.id_to_compile_id.get(i) for i in ids)+ return out+++_BLOCK_NODE = "_unsloth_graphed_block"+++def _mark_block_outputs(out):+ """Tag the autograd nodes of a graphed block call, so a later block's input check can stop there:+ that call decided its own input gradient. Other compiled functions may pass gradients through."""+ import torch.utils._pytree as pytree++ for x in pytree.tree_leaves(out):+ if torch.is_tensor(x) and x.grad_fn is not None:+ x.grad_fn.metadata[_BLOCK_NODE] = True+ return out+++def _pool_release_verdict(managers, ours):+ """ "ok" when every tree function belongs to a block compile and nothing in the pool is alive+ (no pending backward), "pending" to retry at a later call, "foreign" when another torch.compile+ region also records into this tree: resetting would break its next call."""+ for manager in managers:+ if any(cid is None or cid not in ours for cid in manager.id_to_compile_id.values()):+ return "foreign"+ for manager in managers:+ if manager.running_forwards_with_pending_backwards:+ return "pending"+ if manager.live_cudagraph_pool_storages_in_curr_execution():+ return "pending"+ return "ok"+++def _release_graph_pool():+ """Free the cudagraph tree pool via reset_cudagraph_trees(), only when every tree function is ours:+ a foreign one called afterwards fails ("Running CUDAGraph after shutdown")."""+ state = _CUDAGRAPH_STATE+ if state["release"] is not None:+ return+ try:+ from torch._inductor import cudagraph_trees+ containers = cudagraph_trees.get_obj(cudagraph_trees.local, "tree_manager_containers")+ managers = [c.tree_manager for c in list(containers.values()) if c.tree_manager is not None]+ except Exception:+ state["release"] = "kept"+ return+ if not managers:+ state["release"] = "done" # nothing was recorded+ return+ verdict = _pool_release_verdict(managers, state["ours"])+ if verdict == "pending":+ return+ if verdict == "foreign":+ state["release"] = "kept"+ _rank0_note(+ "kept",+ "Unsloth: block CUDA graphs are off; their memory pool stays allocated because another "+ "compiled region records into the same CUDA graph tree.",+ )+ return+ devices = sorted(state["managers"])+ before = sum(torch.cuda.memory_reserved(d) for d in devices)+ cudagraph_trees.reset_cudagraph_trees()+ torch.cuda.empty_cache()+ freed = before - sum(torch.cuda.memory_reserved(d) for d in devices)+ state["release"] = "done"+ state["managers"].clear()+ _rank0_note(+ "released",+ f"Unsloth: released the block CUDA graph pool ({max(freed, 0) / 2**30:.2f} GiB reserved freed).",+ )+++def _packed_kwargs(kwargs):+ get = kwargs.get+ return (+ get("packed_seq_lengths") is not None+ or get("cu_seq_lens_q") is not None+ or get("cu_seq_lens_k") is not None+ or get("cu_seqlens") is not None+ )+++def _trainable(layer):+ params = layer.__dict__.get("_unsloth_block_params")+ if params is None:+ params = layer.__dict__["_unsloth_block_params"] = list(layer.parameters())+ return any(p.requires_grad for p in params)+++# The grad-mode trial before a checkpointed block's first no-grad call stands in for the recompute.+_IN_TRIAL = [False]+++# output_hidden_states hands block outputs to the caller, who may keep them past the generation.+_HIDDEN_STATES_EXPOSED = [False]+++def _note_hidden_states_exposed(module, args, kwargs):+ want = kwargs.get("output_hidden_states")+ if want is None:+ want = getattr(getattr(module, "config", None), "output_hidden_states", False)+ _HIDDEN_STATES_EXPOSED[0] = bool(want)+++def _watch_hidden_states(model):+ """Hook the model and each decoder list's owner to record whether hidden states are returned."""+ modules = dict(model.named_modules())+ lists = {id(layer_list) for layer_list in _layer_lists(model)}+ owners = [model]+ for name, module in modules.items():+ if id(module) in lists and "." in name:+ owners.append(modules[name.rsplit(".", 1)[0]])+ for owner in owners:+ if not owner.__dict__.get("_unsloth_watch_hidden_states", False):+ owner.__dict__["_unsloth_watch_hidden_states"] = True+ owner.register_forward_pre_hook(_note_hidden_states_exposed, with_kwargs = True)+++def _pool_outputs_stay_valid(layer, hidden_states):+ """A grad-mode call outside backward leaves a forward pending, so cudagraph trees start no new+ generation (can_start_new_generation) and its outputs need no copy out of the pool. Checkpointed+ calls (no-grad forward, recompute in backward, and the trial standing in for it) may start one."""+ return (+ not _IN_TRIAL[0]+ and not _HIDDEN_STATES_EXPOSED[0]+ and torch.is_grad_enabled()+ and torch._C._current_graph_task_id() == -1+ and (hidden_states.requires_grad or _trainable(layer))+ )+++def _input_grad_must_leave_pool(hidden_states):+ """True when the input gradient outlives the generation: a leaf's .grad (kept live, so the pool is+ never released) or a checkpoint's backward, which recomputes (a new generation) before reading it."""+ if not hidden_states.requires_grad:+ return False+ node = hidden_states.grad_fn+ if node is None:+ return True # leaf: its .grad would be pool memory+ # (node, fresh): fresh once the path passed an op whose backward allocates its input gradient.+ stack, seen = [(node, False)], 0+ while stack:+ node, fresh = stack.pop()+ seen += 1+ if seen > 32:+ return True+ name = type(node).__name__+ if name.startswith("CompiledFunction") and node.metadata.get(_BLOCK_NODE, False):+ continue # the previous block's region: its own input is its own call's decision+ if isinstance(node, torch.autograd.function.BackwardCFunction):+ forward_cls = getattr(node, "_forward_cls", None)+ if not (isinstance(forward_cls, type) and issubclass(forward_cls, _CloneGrad)):+ return True # a checkpoint (or other Python autograd Function) may recompute first+ if name == "AccumulateGrad" and not fresh:+ return True # a view / add of a leaf (inputs_embeds = param.view(...)) keeps the pool gradient+ allocates = name.startswith(_ALLOCATING_BACKWARD)+ for i, (n, _) in enumerate(node.next_functions):+ if n is None:+ continue+ edge_fresh = fresh or allocates+ if allocates and i == 0 and name.startswith(_ADDITIVE_BACKWARD):+ # addmm / baddbmm hand their additive input `grad` itself (beta = 1), unless it was+ # broadcast (a bias), which the engine sums to its shape in new storage+ full = 3 if name.startswith("Baddbmm") else 2+ variable = getattr(n, "variable", None)+ edge_fresh = fresh or (variable is not None and variable.dim() < full)+ stack.append((n, edge_fresh))+ return False+++_ADDITIVE_BACKWARD = ("AddmmBackward", "BaddbmmBackward")+++# Backward nodes that write their input gradient to new storage (a view, add, clone or cat passes+# on the incoming gradient or a narrowed view of it).+_ALLOCATING_BACKWARD = (+ "MmBackward",+ "AddmmBackward",+ "BmmBackward",+ "BaddbmmBackward",+ "MulBackward",+ "DivBackward",+ "EmbeddingBackward",+ "EmbeddingDenseBackward",+ "NativeLayerNormBackward",+ "FusedRmsNormBackward",+ "SoftmaxBackward",+ "SiluBackward",+ "GeluBackward",+ "IndexBackward",+ "SliceBackward",+ "SelectBackward",+ "ConvolutionBackward",+)class _CloneGrad(torch.autograd.Function):@@ -95,32 +364,27 @@ def _graphs_unsafe_moe(layer):for m in layer.modules())layer.__dict__["_unsloth_block_graphs_moe"] = moe- if moe and not _GRAPHS_MOE_LOGGED[0]:- _GRAPHS_MOE_LOGGED[0] = True- print("Unsloth: block CUDA graphs skip MoE layers; they run compiled without graphs.")+ if moe:+ _rank0_note(+ "moe",+ "Unsloth: block CUDA graphs skip MoE layers; they run compiled without graphs.",+ )return moe-_GRAPHS_MOE_LOGGED = [False]--def _flex_block_mask_call(args, kwargs):"""BlockMask index tensors are static graph inputs rebuilt every step, so the graph re-recordseach call (StaticInputIdxMismatch is outside the re-record limit); Gemma3 was slower with graphs."""for value in (*args, *kwargs.values()):if type(value).__name__ == "BlockMask":- if not _GRAPHS_FLEX_LOGGED[0]:- _GRAPHS_FLEX_LOGGED[0] = True- print(- "Unsloth: block CUDA graphs skip flex attention blocks; they run compiled without graphs."- )+ _rank0_note(+ "flex",+ "Unsloth: block CUDA graphs skip flex attention blocks; they run compiled without graphs.",+ )return Truereturn False-_GRAPHS_FLEX_LOGGED = [False]--def _ddp_optimizer_mode():# optimize_ddp also takes "python_reducer" / "no_optimization" strings: only "ddp_optimizer"# (or True) makes dynamo split graphs (convert_frame.py checks == "ddp_optimizer").@@ -904,7 +1168,6 @@ def _wrap(**isolate,)graph_shapes = set()- graphs_off = [False]def run(self, hidden_states, args, kwargs, mode):previous = inline_flag[0]@@ -926,23 +1189,39 @@ def _wrap(kwargs = {**kwargs, packed_module.BLOCK_VARLEN_KWARG: varlen}# Capture hooks (router_logits) append to empty scratch lists: no guard on the layer index.with capture_scratch(), _no_ddp_graph_split():- if graphs and not graphs_off[0]:+ if graphs and _CUDAGRAPH_STATE["off"] is None:shape = tuple(hidden_states.shape)- if shape not in graph_shapes:- graph_shapes.add(shape)- graphs_off[0] = len(graph_shapes) > _CUDAGRAPH_MAX_SHAPES+ if _packed_kwargs(kwargs):+ _graphs_off("packed", _PACKED_NOTE)+ elif shape not in graph_shapes:+ if len(graph_shapes) >= _CUDAGRAPH_MAX_SHAPES:+ _graphs_off("shapes", _SHAPES_NOTE.format(_CUDAGRAPH_MAX_SHAPES))+ else:+ graph_shapes.add(shape)if graphs and (- graphs_off[0]+ _CUDAGRAPH_STATE["off"] is not None+ or self.__dict__.get("_unsloth_block_graphs_off", False)or _in_nonreentrant_checkpoint()or _graphs_unsafe_moe(self)or _flex_block_mask_call(args, kwargs)):+ if _CUDAGRAPH_STATE["off"] is not None and _CUDAGRAPH_STATE["release"] is None:+ _release_graph_pool()return plain(self, hidden_states, *args, **kwargs)if graphs:_param_grads_out_of_pool(self)+ device = hidden_states.device.index+ if _pool_outputs_stay_valid(self, hidden_states):+ if _input_grad_must_leave_pool(hidden_states):+ # A leaf's .grad outlives the generation (trainable inputs_embeds); a+ # checkpoint's backward recomputes (a new generation) before reading it.+ hidden_states = _CloneGrad.apply(hidden_states)+ return _graphed_call(compiled, device, self, hidden_states, *args, **kwargs)if hidden_states.requires_grad and torch.is_grad_enabled():hidden_states = _CloneGrad.apply(hidden_states)- return _clone_outputs(compiled(self, hidden_states, *args, **kwargs))+ return _clone_outputs(+ _graphed_call(compiled, device, self, hidden_states, *args, **kwargs)+ )return compiled(self, hidden_states, *args, **kwargs)finally:inline_flag[0] = previous@@ -980,6 +1259,13 @@ def _wrap(# PrefixGrouper's int segment table would recompile at every new GRPO length.or (unsloth_forward and kwargs.get("prefix_seg_info") is not None)):+ if (+ graphs+ and self.training+ and _CUDAGRAPH_STATE["off"] is None+ and _packed_kwargs(kwargs)+ ):+ _graphs_off("packed", _PACKED_NOTE)return original(self, hidden_states, *args, **kwargs)if tracing:# Inside a user's torch.compile the enclosing graph owns the layer: no dynamic marking@@ -1002,7 +1288,11 @@ def _wrap(# Grad-mode trial first (RNG forked): a checkpoint recompute must not demote below this call.devices = [hidden_states.device.index] if hidden_states.is_cuda else []with torch.random.fork_rng(devices = devices), torch.enable_grad():- forward(self, hidden_states.detach().requires_grad_(), *args, **kwargs)+ _IN_TRIAL[0] = True+ try:+ forward(self, hidden_states.detach().requires_grad_(), *args, **kwargs)+ finally:+ _IN_TRIAL[0] = Falsemode = self.__dict__["_unsloth_block_mode"]if mode == _OFF:return original(self, hidden_states, *args, **kwargs)@@ -1315,6 +1605,8 @@ def patch_block_compile(model):continue_wrap(cls, _COMPILE_OPTIONS, len(kinds.get(cls, ())) or 1)wrapped.append(cls.__name__)+ if _COMPILE_OPTIONS.get("triton.cudagraphs"):+ _watch_hidden_states(model)if wrapped:print(f"Unsloth: compiling each transformer block as one region ({', '.join(wrapped)}).")return wrappeddiff --git a/unsloth/models/fsdp2_native.py b/unsloth/models/fsdp2_native.pyindex e2612099d23..350c1333aea 100644--- a/unsloth/models/fsdp2_native.py+++ b/unsloth/models/fsdp2_native.py@@ -1010,8 +1010,9 @@ def _skip_free_unsharded_param():return None-def _compiled_block_modules(model):- """ids of every module inside a decoder block _block_compile wrapped (and did not turn off)."""+def _compiled_block_modules(model, roots = None):+ """ids of every module inside a decoder block _block_compile wrapped (and did not turn off);+ `roots` (a dict) collects id -> block module."""inside = set()for module in model.modules():if id(module) in inside or not getattr(@@ -1020,21 +1021,41 @@ def _compiled_block_modules(model):continueif module.__dict__.get("_unsloth_block_mode", 1) == 0:continue- inside.update(id(m) for m in module.modules())+ for m in module.modules():+ inside.add(id(m))+ if roots is not None:+ roots[id(m)] = modulereturn inside+def _block_cudagraphs_keep_budget():+ """Per rank bytes of gathered block weights kept for block CUDA graphs. LoRA keeps only the+ adapters (T4 Qwen3-0.6B: 0.04 GiB); a sharded full finetune keeps every block weight in fp32+ (1.64 GiB per rank there) for graphs that measured no faster."""+ try:+ return float(os.environ.get("UNSLOTH_BLOCK_CUDAGRAPHS_FSDP2_KEEP_GB", "0.5")) * 2**30+ except ValueError:+ return 0.5 * 2**30+++def _gib(nbytes):+ # LoRA adapters are tens of MiB: "0.00 GiB" read as nothing kept.+ return f"{nbytes / 2**30:.2f} GiB" if nbytes >= 2**30 / 10 else f"{nbytes / 2**20:.1f} MiB"++def _keep_unsharded_for_block_cudagraphs(model):"""FSDP2 frees gathered params at reshard, moving their addresses so block CUDA graphs re-record everystep (2 x T4 Qwen3 LoRA: 6 records, 0 replays). Keep the storage; the next all-gather refills it in place.- Only weights inside compiled blocks: the rest (embeddings, head, final norm) never enter a block graph."""+ Only weights inside compiled blocks: the rest (embeddings, head, final norm) never enter a block graph.+ Over the budget nothing is kept and those blocks run without graphs (never recorded)."""if not _block_cudagraphs_requested():return 0- blocks = _compiled_block_modules(model)+ roots = {}+ blocks = _compiled_block_modules(model, roots)if not blocks:return 0FSDPModule = _fsdp_module_class()- kept, nbytes = 0, 0+ candidates, nbytes = [], 0for module in model.modules():if not isinstance(module, FSDPModule):continue@@ -1046,25 +1067,44 @@ def _keep_unsharded_for_block_cudagraphs(model):if getattr(fsdp_param, "_unsloth_keep_unsharded", False):continueinfo = getattr(fsdp_param, "_module_info", None)- if id(getattr(info, "module", None)) not in blocks:+ owner = id(getattr(info, "module", None))+ if owner not in blocks:continueif not callable(getattr(type(fsdp_param), "free_unsharded_param", None)):- return kept # a torch without this FSDP2 internal: graphs re-record as before- fsdp_param.free_unsharded_param = _skip_free_unsharded_param- fsdp_param._unsloth_keep_unsharded = True- kept += 1+ return 0 # a torch without this FSDP2 internal: graphs re-record as before+ candidates.append((fsdp_param, roots[owner]))try:- dtype = getattr(fsdp_param, "param_dtype", None) or fsdp_param.orig_dtype+ # param_dtype / orig_dtype are set at FSDP2's lazy init (first forward), after this.+ dtype = (+ getattr(fsdp_param, "param_dtype", None)+ or getattr(getattr(group, "mp_policy", None), "param_dtype", None)+ or fsdp_param.sharded_param.dtype+ )nbytes += fsdp_param._orig_size.numel() * dtype.itemsizeexcept Exception:pass- if kept:+ if not candidates:+ return 0+ budget = _block_cudagraphs_keep_budget()+ if nbytes > budget:+ layers = {id(block): block for _, block in candidates}+ for block in layers.values():+ block.__dict__["_unsloth_block_graphs_off"] = True_print_rank0(- f"Unsloth: UNSLOTH_BLOCK_CUDAGRAPHS=1 under native FSDP2 keeps the gathered weights of "- f"{kept} parameters allocated ({nbytes / 2**30:.2f} GiB per rank) so the block CUDA graphs "- "replay instead of re-recording every step."+ f"Unsloth: UNSLOTH_BLOCK_CUDAGRAPHS=1 under native FSDP2 would keep {_gib(nbytes)} "+ f"per rank of gathered block weights allocated (budget {_gib(budget)}, "+ f"UNSLOTH_BLOCK_CUDAGRAPHS_FSDP2_KEEP_GB); {len(layers)} blocks run without CUDA graphs.")- return kept+ return 0+ for fsdp_param, _ in candidates:+ fsdp_param.free_unsharded_param = _skip_free_unsharded_param+ fsdp_param._unsloth_keep_unsharded = True+ _print_rank0(+ f"Unsloth: UNSLOTH_BLOCK_CUDAGRAPHS=1 under native FSDP2 keeps the gathered weights of "+ f"{len(candidates)} parameters allocated ({_gib(nbytes)} per rank) so the block CUDA graphs "+ "replay instead of re-recording every step."+ )+ return len(candidates)def _post_shard(