Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Opt-in CUDA graphs for compiled blocks under gradient checkpointing (UNSLOTH_BLOCK_CUDAGRAPHS=1) (#15154)

Daniel Han8feaae2a138a
diff --git a/tests/test_block_cudagraphs_clone.py b/tests/test_block_cudagraphs_clone.py
new file mode 100644
index 00000000000..5a1456abe9b
--- /dev/null
+++ b/tests/test_block_cudagraphs_clone.py
@@ -0,0 +1,124 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
+
+"""Block CUDA graphs (opt-in) must copy block outputs and input gradients out of the graph pool.
+
+Under Unsloth checkpointing every block call starts a new cudagraph tree generation, so a
+tensor the checkpoint keeps (the saved block input, the gradient handed to the previous
+block) would be overwritten by the next replay.
+"""
+
+import importlib.util
+import os
+import sys
+
+import torch
+
+_DIR = os.path.join(
+ os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "unsloth", "models"
+)
+
+
+def _load():
+ # By file path: importing the unsloth package needs an accelerator.
+ spec = importlib.util.spec_from_file_location(
+ "_ublock_compile_cg", os.path.join(_DIR, "_block_compile.py")
+ )
+ mod = importlib.util.module_from_spec(spec)
+ sys.modules[spec.name] = mod
+ spec.loader.exec_module(mod)
+ return mod
+
+
+bc = _load()
+
+
+def test_clone_grad_is_identity_with_copied_gradient():
+ x = torch.randn(3, 4, requires_grad = True)
+ seen = {}
+
+ def keep(grad):
+ seen["ptr"] = grad.data_ptr()
+ return grad
+
+ y = bc._CloneGrad.apply(x)
+ y.register_hook(keep)
+ assert torch.equal(y, x)
+ (y * 2).sum().backward()
+ assert torch.equal(x.grad, torch.full_like(x, 2.0))
+ assert x.grad.data_ptr() != seen["ptr"]
+
+
+def test_clone_outputs_copies_every_tensor():
+ a, b = torch.randn(2), torch.randn(3)
+ out = bc._clone_outputs((a, None, b))
+ assert out[1] is None
+ assert torch.equal(out[0], a) and out[0].data_ptr() != a.data_ptr()
+ assert torch.equal(out[2], b) and out[2].data_ptr() != b.data_ptr()
+ single = bc._clone_outputs(a)
+ assert torch.equal(single, a) and single.data_ptr() != a.data_ptr()
+
+
+def test_cudagraphs_opt_in_flag(monkeypatch):
+ try:
+ monkeypatch.setenv("UNSLOTH_BLOCK_CUDAGRAPHS", "1")
+ assert _load()._COMPILE_OPTIONS["triton.cudagraphs"] is True
+ monkeypatch.setenv("UNSLOTH_BLOCK_CUDAGRAPHS", "0")
+ assert _load()._COMPILE_OPTIONS["triton.cudagraphs"] is False
+ finally:
+ monkeypatch.delenv("UNSLOTH_BLOCK_CUDAGRAPHS", raising = False)
+ _load()
+
+
+def test_new_shapes_past_the_cap_skip_graphs(monkeypatch):
+ """Variable lengths recorded a graph set per new length: 3.5x slower steps and a pool that
+ grew every step on an L4. One shape past the cap runs every later call without graphs."""
+ calls = []
+
+ def fake_compile(fn, **kwargs):
+ tag = "graphs" if kwargs.get("options", {}).get("triton.cudagraphs") else "plain"
+
+ def run(*args, **kw):
+ calls.append(tag)
+ return fn(*args, **kw)
+
+ return run
+
+ monkeypatch.setattr(torch, "compile", fake_compile)
+ monkeypatch.setattr(bc, "_CUDAGRAPH_MAX_SHAPES", 2)
+
+ class CapDecoderLayer(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.proj = torch.nn.Linear(8, 8)
+
+ def forward(self, hidden_states, **kwargs):
+ return self.proj(hidden_states)
+
+ bc._wrap(CapDecoderLayer, {**bc._COMPILE_OPTIONS, "triton.cudagraphs": True})
+ layer = CapDecoderLayer().train()
+
+ def step(L):
+ calls.clear()
+ x = torch.randn(2, L, 8, requires_grad = True)
+ layer(x).sum().backward()
+ assert x.grad is not None and x.grad.shape == x.shape
+ return set(calls)
+
+ assert step(16) == {"graphs"}
+ assert step(24) == {"graphs"}
+ assert step(16) == {"graphs"}
+ assert step(32) == {"plain"} # third shape: over the cap
+ assert step(16) == {"plain"} # graphs stay off, also for admitted shapes
+ assert step(40) == {"plain"}
+
+
+def test_cap_reads_the_environment(monkeypatch):
+ try:
+ monkeypatch.setenv("UNSLOTH_BLOCK_CUDAGRAPHS_MAX_SHAPES", "3")
+ assert _load()._CUDAGRAPH_MAX_SHAPES == 3
+ monkeypatch.setenv("UNSLOTH_BLOCK_CUDAGRAPHS_MAX_SHAPES", "junk")
+ assert _load()._CUDAGRAPH_MAX_SHAPES == 8
+ finally:
+ monkeypatch.delenv("UNSLOTH_BLOCK_CUDAGRAPHS_MAX_SHAPES", raising = False)
+ _load()
diff --git a/unsloth/models/_block_compile.py b/unsloth/models/_block_compile.py
index f3cecd8df4e..e31622e3cbc 100644
--- a/unsloth/models/_block_compile.py
+++ b/unsloth/models/_block_compile.py
@@ -21,6 +21,32 @@ _COMPILE_OPTIONS = {
"shape_padding": True,
"triton.cudagraphs": False,
}
+# Opt-in per-block CUDA graphs; only pays off for static shapes.
+if os.environ.get("UNSLOTH_BLOCK_CUDAGRAPHS", "0") == "1":
+ _COMPILE_OPTIONS["triton.cudagraphs"] = True
+# Shape cap: variable lengths recorded a graph set per length (3.5x slower, pool grew every step).
+# Past the cap graphs go off for the whole layer kind; admitted shapes alone still grew the pool.
+try:
+ _CUDAGRAPH_MAX_SHAPES = max(0, int(os.environ.get("UNSLOTH_BLOCK_CUDAGRAPHS_MAX_SHAPES", "8")))
+except ValueError:
+ _CUDAGRAPH_MAX_SHAPES = 8
+
+
+class _CloneGrad(torch.autograd.Function):
+ """Identity; backward copies the input gradient out of the CUDA graph pool."""
+
+ @staticmethod
+ def forward(ctx, x):
+ return x.view_as(x)
+
+ @staticmethod
+ def backward(ctx, grad):
+ return grad.clone()
+
+
+def _clone_outputs(out):
+ import torch.utils._pytree as pytree
+ return pytree.tree_map(lambda x: x.clone() if torch.is_tensor(x) else x, out)
# gpt-oss builds its flex mask inside attention: the block broke and recompiled per length.
@@ -719,6 +745,18 @@ def _wrap(cls, compile_options):
# then the stock forward. The mode is per layer (a checkpoint recompute must not switch graph);
# a failure is also the starting mode for later layers of the same kind.
state = {"reported": set(), "start": {}}
+ # Under Unsloth checkpointing each block call is a new cudagraph generation: the next replay
+ # overwrites this block's outputs and input gradient, so both are cloned out of the pool.
+ graphs = bool(compile_options.get("triton.cudagraphs"))
+ if graphs:
+ plain = torch.compile(
+ original,
+ fullgraph = False,
+ dynamic = None,
+ options = {**compile_options, "triton.cudagraphs": False},
+ )
+ graph_shapes = set()
+ graphs_off = [False]
def run(self, hidden_states, args, kwargs, mode):
previous = inline_flag[0]
@@ -727,6 +765,17 @@ def _wrap(cls, compile_options):
try:
# Capture hooks (router_logits) append to empty scratch lists: no guard on the layer index.
with capture_scratch():
+ if graphs and not graphs_off[0]:
+ shape = tuple(hidden_states.shape)
+ if shape not in graph_shapes:
+ graph_shapes.add(shape)
+ graphs_off[0] = len(graph_shapes) > _CUDAGRAPH_MAX_SHAPES
+ if graphs and graphs_off[0]:
+ return plain(self, hidden_states, *args, **kwargs)
+ if graphs:
+ if hidden_states.requires_grad and torch.is_grad_enabled():
+ hidden_states = _CloneGrad.apply(hidden_states)
+ return _clone_outputs(compiled(self, hidden_states, *args, **kwargs))
return compiled(self, hidden_states, *args, **kwargs)
finally:
inline_flag[0] = previous