Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Opt-in CUDA graphs for compiled blocks under gradient checkpointing (UNSLOTH_BLOCK_CUDAGRAPHS=1) (#15154)
Daniel Han8feaae2a138a
diff --git a/tests/test_block_cudagraphs_clone.py b/tests/test_block_cudagraphs_clone.pynew file mode 100644index 00000000000..5a1456abe9b--- /dev/null+++ b/tests/test_block_cudagraphs_clone.py@@ -0,0 +1,124 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.++"""Block CUDA graphs (opt-in) must copy block outputs and input gradients out of the graph pool.++Under Unsloth checkpointing every block call starts a new cudagraph tree generation, so a+tensor the checkpoint keeps (the saved block input, the gradient handed to the previous+block) would be overwritten by the next replay.+"""++import importlib.util+import os+import sys++import torch++_DIR = os.path.join(+ os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "unsloth", "models"+)+++def _load():+ # By file path: importing the unsloth package needs an accelerator.+ spec = importlib.util.spec_from_file_location(+ "_ublock_compile_cg", os.path.join(_DIR, "_block_compile.py")+ )+ mod = importlib.util.module_from_spec(spec)+ sys.modules[spec.name] = mod+ spec.loader.exec_module(mod)+ return mod+++bc = _load()+++def test_clone_grad_is_identity_with_copied_gradient():+ x = torch.randn(3, 4, requires_grad = True)+ seen = {}++ def keep(grad):+ seen["ptr"] = grad.data_ptr()+ return grad++ y = bc._CloneGrad.apply(x)+ y.register_hook(keep)+ assert torch.equal(y, x)+ (y * 2).sum().backward()+ assert torch.equal(x.grad, torch.full_like(x, 2.0))+ assert x.grad.data_ptr() != seen["ptr"]+++def test_clone_outputs_copies_every_tensor():+ a, b = torch.randn(2), torch.randn(3)+ out = bc._clone_outputs((a, None, b))+ assert out[1] is None+ assert torch.equal(out[0], a) and out[0].data_ptr() != a.data_ptr()+ assert torch.equal(out[2], b) and out[2].data_ptr() != b.data_ptr()+ single = bc._clone_outputs(a)+ assert torch.equal(single, a) and single.data_ptr() != a.data_ptr()+++def test_cudagraphs_opt_in_flag(monkeypatch):+ try:+ monkeypatch.setenv("UNSLOTH_BLOCK_CUDAGRAPHS", "1")+ assert _load()._COMPILE_OPTIONS["triton.cudagraphs"] is True+ monkeypatch.setenv("UNSLOTH_BLOCK_CUDAGRAPHS", "0")+ assert _load()._COMPILE_OPTIONS["triton.cudagraphs"] is False+ finally:+ monkeypatch.delenv("UNSLOTH_BLOCK_CUDAGRAPHS", raising = False)+ _load()+++def test_new_shapes_past_the_cap_skip_graphs(monkeypatch):+ """Variable lengths recorded a graph set per new length: 3.5x slower steps and a pool that+ grew every step on an L4. One shape past the cap runs every later call without graphs."""+ calls = []++ def fake_compile(fn, **kwargs):+ tag = "graphs" if kwargs.get("options", {}).get("triton.cudagraphs") else "plain"++ def run(*args, **kw):+ calls.append(tag)+ return fn(*args, **kw)++ return run++ monkeypatch.setattr(torch, "compile", fake_compile)+ monkeypatch.setattr(bc, "_CUDAGRAPH_MAX_SHAPES", 2)++ class CapDecoderLayer(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.proj = torch.nn.Linear(8, 8)++ def forward(self, hidden_states, **kwargs):+ return self.proj(hidden_states)++ bc._wrap(CapDecoderLayer, {**bc._COMPILE_OPTIONS, "triton.cudagraphs": True})+ layer = CapDecoderLayer().train()++ def step(L):+ calls.clear()+ x = torch.randn(2, L, 8, requires_grad = True)+ layer(x).sum().backward()+ assert x.grad is not None and x.grad.shape == x.shape+ return set(calls)++ assert step(16) == {"graphs"}+ assert step(24) == {"graphs"}+ assert step(16) == {"graphs"}+ assert step(32) == {"plain"} # third shape: over the cap+ assert step(16) == {"plain"} # graphs stay off, also for admitted shapes+ assert step(40) == {"plain"}+++def test_cap_reads_the_environment(monkeypatch):+ try:+ monkeypatch.setenv("UNSLOTH_BLOCK_CUDAGRAPHS_MAX_SHAPES", "3")+ assert _load()._CUDAGRAPH_MAX_SHAPES == 3+ monkeypatch.setenv("UNSLOTH_BLOCK_CUDAGRAPHS_MAX_SHAPES", "junk")+ assert _load()._CUDAGRAPH_MAX_SHAPES == 8+ finally:+ monkeypatch.delenv("UNSLOTH_BLOCK_CUDAGRAPHS_MAX_SHAPES", raising = False)+ _load()diff --git a/unsloth/models/_block_compile.py b/unsloth/models/_block_compile.pyindex f3cecd8df4e..e31622e3cbc 100644--- a/unsloth/models/_block_compile.py+++ b/unsloth/models/_block_compile.py@@ -21,6 +21,32 @@ _COMPILE_OPTIONS = {"shape_padding": True,"triton.cudagraphs": False,}+# Opt-in per-block CUDA graphs; only pays off for static shapes.+if os.environ.get("UNSLOTH_BLOCK_CUDAGRAPHS", "0") == "1":+ _COMPILE_OPTIONS["triton.cudagraphs"] = True+# Shape cap: variable lengths recorded a graph set per length (3.5x slower, pool grew every step).+# Past the cap graphs go off for the whole layer kind; admitted shapes alone still grew the pool.+try:+ _CUDAGRAPH_MAX_SHAPES = max(0, int(os.environ.get("UNSLOTH_BLOCK_CUDAGRAPHS_MAX_SHAPES", "8")))+except ValueError:+ _CUDAGRAPH_MAX_SHAPES = 8+++class _CloneGrad(torch.autograd.Function):+ """Identity; backward copies the input gradient out of the CUDA graph pool."""++ @staticmethod+ def forward(ctx, x):+ return x.view_as(x)++ @staticmethod+ def backward(ctx, grad):+ return grad.clone()+++def _clone_outputs(out):+ import torch.utils._pytree as pytree+ return pytree.tree_map(lambda x: x.clone() if torch.is_tensor(x) else x, out)# gpt-oss builds its flex mask inside attention: the block broke and recompiled per length.@@ -719,6 +745,18 @@ def _wrap(cls, compile_options):# then the stock forward. The mode is per layer (a checkpoint recompute must not switch graph);# a failure is also the starting mode for later layers of the same kind.state = {"reported": set(), "start": {}}+ # Under Unsloth checkpointing each block call is a new cudagraph generation: the next replay+ # overwrites this block's outputs and input gradient, so both are cloned out of the pool.+ graphs = bool(compile_options.get("triton.cudagraphs"))+ if graphs:+ plain = torch.compile(+ original,+ fullgraph = False,+ dynamic = None,+ options = {**compile_options, "triton.cudagraphs": False},+ )+ graph_shapes = set()+ graphs_off = [False]def run(self, hidden_states, args, kwargs, mode):previous = inline_flag[0]@@ -727,6 +765,17 @@ def _wrap(cls, compile_options):try:# Capture hooks (router_logits) append to empty scratch lists: no guard on the layer index.with capture_scratch():+ if graphs and not graphs_off[0]:+ shape = tuple(hidden_states.shape)+ if shape not in graph_shapes:+ graph_shapes.add(shape)+ graphs_off[0] = len(graph_shapes) > _CUDAGRAPH_MAX_SHAPES+ if graphs and graphs_off[0]:+ return plain(self, hidden_states, *args, **kwargs)+ if graphs:+ if hidden_states.requires_grad and torch.is_grad_enabled():+ hidden_states = _CloneGrad.apply(hidden_states)+ return _clone_outputs(compiled(self, hidden_states, *args, **kwargs))return compiled(self, hidden_states, *args, **kwargs)finally:inline_flag[0] = previous