Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Block CUDA graphs: turn on automatically for fixed-shape, launch-bound LoRA runs (#15768)

Daniel Han04d378863712
diff --git a/tests/test_block_cudagraphs_auto.py b/tests/test_block_cudagraphs_auto.py
new file mode 100644
index 00000000000..e14fc21bf7b
--- /dev/null
+++ b/tests/test_block_cudagraphs_auto.py
@@ -0,0 +1,355 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
+
+"""Block CUDA graphs auto mode (UNSLOTH_BLOCK_CUDAGRAPHS unset): graphs turn on only after the first
+forward passes all had one batch shape and a timed block launched slower than the GPU ran it.
+"1" still forces them on, "0" (or any other value) keeps them off. Timing and the device checks are
+stubbed, so this runs on CPU."""
+
+import importlib.util
+import os
+import sys
+
+import pytest
+import torch
+
+_DIR = os.path.join(
+ os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "unsloth", "models"
+)
+
+
+def _load():
+ spec = importlib.util.spec_from_file_location(
+ "_ublock_compile_cg_auto", os.path.join(_DIR, "_block_compile.py")
+ )
+ mod = importlib.util.module_from_spec(spec)
+ sys.modules[spec.name] = mod
+ spec.loader.exec_module(mod)
+ return mod
+
+
+bc = _load()
+
+
[email protected](autouse = True)
+def _block_env(monkeypatch):
+ for key in (
+ "UNSLOTH_BLOCK_COMPILE",
+ "UNSLOTH_COMPILE_DISABLE",
+ "UNSLOTH_FORCE_FLOAT32",
+ "UNSLOTH_BLOCK_CUDAGRAPHS",
+ ):
+ monkeypatch.delenv(key, raising = False)
+ monkeypatch.setattr(bc, "_CUDAGRAPH_STATE", bc._new_cudagraph_state())
+ monkeypatch.setattr(bc, "_CUDAGRAPH_MODE", "auto")
+
+
+ "value, mode",
+ [(None, "auto"), ("", "auto"), ("auto", "auto"), ("1", "on"), ("0", "off"), ("true", "off")],
+)
+def test_mode_from_environment(monkeypatch, value, mode):
+ """Unset is auto now; "1" forces graphs, every other value keeps the old meaning (off)."""
+ try:
+ if value is None:
+ monkeypatch.delenv("UNSLOTH_BLOCK_CUDAGRAPHS", raising = False)
+ else:
+ monkeypatch.setenv("UNSLOTH_BLOCK_CUDAGRAPHS", value)
+ mod = _load()
+ assert mod._CUDAGRAPH_MODE == mode
+ # Only a forced "1" records graphs from the first call; auto starts graphs-off.
+ assert mod._COMPILE_OPTIONS["triton.cudagraphs"] is (mode == "on")
+ finally:
+ monkeypatch.delenv("UNSLOTH_BLOCK_CUDAGRAPHS", raising = False)
+ _load()
+
+
+def _harness(
+ monkeypatch,
+ host = 3e-3,
+ gpu = 1e-3,
+ free_gb = 40.0,
+ static = None,
+):
+ calls = []
+ compiles = []
+
+ def fake_compile(fn, **kwargs):
+ tag = "graphs" if kwargs.get("options", {}).get("triton.cudagraphs") else "plain"
+ compiles.append(tag)
+
+ def run(*args, **kw):
+ calls.append(tag)
+ return fn(*args, **kw)
+
+ return run
+
+ monkeypatch.setattr(torch, "compile", fake_compile)
+ monkeypatch.setattr(bc, "_auto_static_reason", lambda layer, hs: static)
+ timed = []
+
+ def fake_time(call, device):
+ timed.append(1)
+ return call(), host, gpu
+
+ monkeypatch.setattr(bc, "_auto_time", fake_time)
+ monkeypatch.setattr(
+ torch.cuda, "mem_get_info", lambda device = None: (int(free_gb * 2**30), 80 * 2**30)
+ )
+
+ class AutoDecoderLayer(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.proj = torch.nn.Linear(8, 8)
+
+ def forward(self, hidden_states, **kwargs):
+ return self.proj(hidden_states)
+
+ bc._wrap(AutoDecoderLayer, dict(bc._COMPILE_OPTIONS))
+ layers = torch.nn.ModuleList([AutoDecoderLayer(), AutoDecoderLayer()]).train()
+ return layers, calls, compiles, timed
+
+
+def _step(
+ layers,
+ calls,
+ L = 16,
+ checkpoint = False,
+ **kwargs,
+):
+ calls.clear()
+ x = torch.randn(2, L, 8, requires_grad = True)
+ h = x
+ for layer in layers:
+ if checkpoint:
+ h = torch.utils.checkpoint.checkpoint(
+ lambda t, layer = layer: layer(t, **kwargs), h, use_reentrant = True
+ )
+ else:
+ h = layer(h, **kwargs)
+ h.sum().backward()
+ assert x.grad is not None and x.grad.shape == x.shape
+ return list(calls)
+
+
+def test_auto_turns_graphs_on_for_fixed_shape_launch_bound(monkeypatch):
+ layers, calls, compiles, timed = _harness(monkeypatch)
+ assert compiles == ["plain"] # the graphed region is compiled only once graphs turn on
+ for _ in range(bc._AUTO_WARMUP_PASSES):
+ assert _step(layers, calls) == ["plain", "plain"]
+ assert _step(layers, calls) == ["plain", "plain"] # pass 3: timed, still graphs-off
+ assert len(timed) == 2
+ _step(layers, calls) # pass 4 times the remaining calls, then decides
+ assert bc._CUDAGRAPH_STATE["auto"]["decision"] == "on"
+ assert _step(layers, calls) == ["graphs", "graphs"]
+ assert compiles == ["plain", "graphs"]
+ assert bc._CUDAGRAPH_STATE["auto"]["phase"] == "on"
+ assert _step(layers, calls) == ["graphs", "graphs"]
+ assert len(timed) == bc._AUTO_MEASURE_CALLS
+
+
+def test_auto_keeps_graphs_off_when_gpu_bound(monkeypatch):
+ layers, calls, compiles, _ = _harness(monkeypatch, host = 1e-3, gpu = 2e-3)
+ for _ in range(8):
+ assert _step(layers, calls) == ["plain", "plain"]
+ auto = bc._CUDAGRAPH_STATE["auto"]
+ assert auto["phase"] == "off" and auto["why"].startswith("GPU-bound")
+ assert compiles == ["plain"]
+
+
+def test_auto_ratio_threshold(monkeypatch):
+ """Launching must be clearly slower than running: A100 ratios 0.55-0.92 gained nothing with graphs
+ forced on (-1.2% to +1.0%), 1.26 gained 12.6%."""
+ layers, calls, _, _ = _harness(monkeypatch, host = 1.1e-3, gpu = 1e-3)
+ for _ in range(6):
+ assert _step(layers, calls) == ["plain", "plain"]
+ assert bc._CUDAGRAPH_STATE["auto"]["phase"] == "off"
+
+
+def test_auto_shape_change_while_probing_keeps_graphs_off(monkeypatch):
+ layers, calls, _, timed = _harness(monkeypatch)
+ _step(layers, calls, L = 16)
+ _step(layers, calls, L = 24) # padded to the longest row: variable lengths
+ auto = bc._CUDAGRAPH_STATE["auto"]
+ assert auto["phase"] == "off" and auto["why"] == "batch shape changed"
+ for _ in range(6):
+ assert _step(layers, calls, L = 16) == ["plain", "plain"]
+ assert not timed
+
+
+def test_auto_after_on_new_shapes_past_the_cap_turn_graphs_off(monkeypatch):
+ monkeypatch.setattr(bc, "_CUDAGRAPH_MAX_SHAPES", 2)
+ released = []
+ monkeypatch.setattr(bc, "_release_graph_pool", lambda: released.append(1))
+ layers, calls, _, _ = _harness(monkeypatch)
+ for _ in range(5):
+ _step(layers, calls)
+ assert _step(layers, calls) == ["graphs", "graphs"]
+ assert _step(layers, calls, L = 8) == ["graphs", "graphs"] # short last batch: second shape
+ assert _step(layers, calls, L = 24) == ["plain", "plain"] # third: over the cap
+ assert bc._CUDAGRAPH_STATE["off"] == "shapes"
+ assert released
+ assert _step(layers, calls) == ["plain", "plain"]
+
+
+def test_auto_packed_rows_keep_graphs_off(monkeypatch):
+ layers, calls, _, timed = _harness(monkeypatch)
+ lengths = torch.tensor([8, 8])
+ for _ in range(6):
+ assert _step(layers, calls, packed_seq_lengths = lengths) == ["plain", "plain"]
+ auto = bc._CUDAGRAPH_STATE["auto"]
+ assert auto["phase"] == "off" and "packed" in auto["why"]
+ assert not timed
+
+
+def test_auto_low_free_memory_keeps_graphs_off(monkeypatch):
+ layers, calls, _, _ = _harness(monkeypatch, free_gb = 1.0)
+ for _ in range(6):
+ assert _step(layers, calls) == ["plain", "plain"]
+ assert bc._CUDAGRAPH_STATE["auto"]["why"].endswith("GiB free")
+
+
+def test_auto_static_reason_keeps_graphs_off(monkeypatch):
+ layers, calls, _, timed = _harness(monkeypatch, static = "distributed")
+ for _ in range(6):
+ assert _step(layers, calls) == ["plain", "plain"]
+ assert bc._CUDAGRAPH_STATE["auto"]["why"] == "distributed"
+ assert not timed
+
+
+def test_auto_counts_passes_not_recomputes(monkeypatch):
+ """Reentrant checkpointing calls each block again inside backward: those calls are not forward
+ passes, so graphs still start after the same number of steps, at a pass boundary."""
+ layers, calls, _, timed = _harness(monkeypatch)
+ for _ in range(bc._AUTO_WARMUP_PASSES):
+ # the first step also runs each block's grad-mode trial before its first no-grad call
+ assert set(_step(layers, calls, checkpoint = True)) == {"plain"}
+ _step(layers, calls, checkpoint = True)
+ _step(layers, calls, checkpoint = True)
+ assert len(timed) == bc._AUTO_MEASURE_CALLS # forward calls only, never a recompute
+ assert _step(layers, calls, checkpoint = True) == ["graphs"] * 4
+
+
+def test_explicit_off_never_probes(monkeypatch):
+ monkeypatch.setattr(bc, "_CUDAGRAPH_MODE", "off")
+ layers, calls, compiles, timed = _harness(monkeypatch)
+ for _ in range(6):
+ assert _step(layers, calls) == ["plain", "plain"]
+ assert not timed and compiles == ["plain"]
+ assert bc._CUDAGRAPH_STATE["auto"]["phase"] == "probe" # untouched
+
+
+def test_static_reason_rules():
+ layer = torch.nn.Linear(4, 4)
+ assert bc._auto_static_reason(layer, torch.zeros(1)) == "not a CUDA tensor"
+
+
+def test_static_reason_full_finetuning(monkeypatch):
+ class FakeCuda:
+ is_cuda = True
+
+ layer = torch.nn.Linear(4, 4) # every weight trainable
+ monkeypatch.setattr(torch.version, "hip", None, raising = False)
+ monkeypatch.setattr(torch, "__version__", "2.14.1")
+ assert bc._auto_static_reason(layer, FakeCuda()) == "full finetuning"
+ for p in layer.parameters():
+ p.requires_grad_(False)
+ assert bc._auto_static_reason(layer, FakeCuda()) is None
+ monkeypatch.setattr(torch, "__version__", "2.12.1+cu128")
+ assert bc._auto_static_reason(layer, FakeCuda()) == "torch < 2.13"
+
+
+def test_a_new_model_decides_again(monkeypatch):
+ """Graphs on for one (launch-bound) model must not carry over to the next model loaded."""
+ layers, calls, _, _ = _harness(monkeypatch)
+ for _ in range(5):
+ _step(layers, calls)
+ assert _step(layers, calls) == ["graphs", "graphs"]
+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
+ monkeypatch.setattr(bc, "_torch_supported", lambda: True)
+
+ class NextDecoderLayer(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.proj = torch.nn.Linear(8, 8)
+
+ def forward(self, hidden_states, **kwargs):
+ return self.proj(hidden_states)
+
+ model = torch.nn.Module()
+ model.layers = torch.nn.ModuleList([NextDecoderLayer(), NextDecoderLayer()])
+ assert bc.patch_block_compile(model) == ["NextDecoderLayer"]
+ assert bc._CUDAGRAPH_STATE["auto"]["phase"] == "probe"
+ model.train()
+ assert _step(model.layers, calls) == ["plain", "plain"]
+
+
+ "version, hip, hooked",
+ [("2.14.1", None, True), ("2.12.1+cu128", None, False), ("2.14.1", "7.0", False)],
+)
+def test_auto_hooks_the_model_only_where_graphs_can_turn_on(monkeypatch, version, hip, hooked):
+ """The hidden-states watcher only serves graphs: no forward hook where auto can never use them."""
+ monkeypatch.setattr(bc, "_CUDAGRAPH_MODE", "auto")
+ monkeypatch.setattr(torch, "__version__", version)
+ monkeypatch.setattr(torch.version, "hip", hip, raising = False)
+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
+ monkeypatch.setattr(bc, "_torch_supported", lambda: True)
+
+ class HookDecoderLayer(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.proj = torch.nn.Linear(8, 8)
+
+ def forward(self, hidden_states, **kwargs):
+ return self.proj(hidden_states)
+
+ model = torch.nn.Module()
+ model.layers = torch.nn.ModuleList([HookDecoderLayer(), HookDecoderLayer()])
+ assert bc.patch_block_compile(model) == ["HookDecoderLayer"]
+ assert model.__dict__.get("_unsloth_watch_hidden_states", False) is hooked
+
+
+def test_probe_times_on_the_blocks_device(monkeypatch):
+ """A block on cuda:1 while cuda:0 is current: sleep and both events must run on cuda:1."""
+ current = ["cuda:0"]
+ log = []
+
+ class Guard:
+ def __init__(self, device):
+ self.device = str(device)
+
+ def __enter__(self):
+ self.prev, current[0] = current[0], self.device
+
+ def __exit__(self, *exc):
+ current[0] = self.prev
+
+ class Event:
+ def __init__(self, enable_timing = False):
+ pass
+
+ def record(self):
+ log.append(("record", current[0]))
+
+ def synchronize(self):
+ pass
+
+ def elapsed_time(self, other):
+ return 2.0
+
+ monkeypatch.setattr(torch.cuda, "device", Guard)
+ monkeypatch.setattr(torch.cuda, "Event", Event)
+ monkeypatch.setattr(torch.cuda, "synchronize", lambda device = None: None)
+ monkeypatch.setattr(torch.cuda, "_sleep", lambda cycles: log.append(("sleep", current[0])))
+ out, host, gpu = bc._auto_time(
+ lambda: log.append(("call", current[0])) or "out", torch.device("cuda", 1)
+ )
+ assert out == "out" and gpu == 2.0 / 1e3 and host is not None
+ assert log == [
+ ("sleep", "cuda:1"),
+ ("record", "cuda:1"),
+ ("call", "cuda:1"),
+ ("record", "cuda:1"),
+ ]
+ assert current[0] == "cuda:0"
diff --git a/unsloth/models/_block_compile.py b/unsloth/models/_block_compile.py
index ae5ca2781fa..76c13b79749 100644
--- a/unsloth/models/_block_compile.py
+++ b/unsloth/models/_block_compile.py
@@ -9,6 +9,7 @@ import functools
import inspect
import os
import sys
+import time
import types
import weakref
@@ -23,8 +24,20 @@ _COMPILE_OPTIONS = {
"shape_padding": True,
"triton.cudagraphs": False,
}
-# Opt-in per-block CUDA graphs; only pays off for static shapes.
-if os.environ.get("UNSLOTH_BLOCK_CUDAGRAPHS", "0") == "1":
+
+
+def _cudagraph_mode():
+ """UNSLOTH_BLOCK_CUDAGRAPHS: "1" always on, unset or "auto" decided per run (see _auto_observe),
+ anything else off (before auto, every value but "1" meant off)."""
+ value = os.environ.get("UNSLOTH_BLOCK_CUDAGRAPHS", "auto").strip().lower()
+ if value == "1":
+ return "on"
+ return "auto" if value in ("", "auto") else "off"
+
+
+# Per-block CUDA graphs only pay off for static shapes on a launch-bound step.
+_CUDAGRAPH_MODE = _cudagraph_mode()
+if _CUDAGRAPH_MODE == "on":
_COMPILE_OPTIONS["triton.cudagraphs"] = True
# Shape cap: variable lengths recorded a graph set per length (3.5x slower, pool grew every step).
# Two shapes = one fixed length plus a short last batch. Past the cap graphs go off for every block
@@ -39,7 +52,29 @@ except ValueError:
# off: why graphs went off (None = on); release: None, "done" or "kept"; ours: compile ids of the
# tree functions block calls created; managers: CUDA index -> the tree manager block graphs ran on.
def _new_cudagraph_state():
- return {"off": None, "release": None, "ours": set(), "managers": {}, "notes": set()}
+ return {
+ "off": None,
+ "release": None,
+ "ours": set(),
+ "managers": {},
+ "notes": set(),
+ "auto": _new_auto_state(),
+ }
+
+
+# Auto mode: phase "probe" (graphs off, watching), "on" or "off" (final). first: the layer whose
+# call starts a forward pass; passes: forward passes seen, all with batch shape `shape`.
+def _new_auto_state():
+ return {
+ "phase": "probe",
+ "first": None,
+ "shape": None,
+ "passes": 0,
+ "host": [],
+ "gpu": [],
+ "decision": None,
+ "why": None,
+ }
_CUDAGRAPH_STATE = _new_cudagraph_state()
@@ -351,7 +386,7 @@ def _in_nonreentrant_checkpoint():
return bool(hooks) and getattr(hooks[0], "__module__", None) == "torch.utils.checkpoint"
-def _graphs_unsafe_moe(layer):
+def _graphs_unsafe_moe(layer, note = True):
"""MoE routing / grouped GEMMs read token counts the capture cannot hold (Qwen3-MoE: host copy
during capture, routing outputs overwritten by the next replay)."""
moe = layer.__dict__.get("_unsloth_block_graphs_moe")
@@ -364,27 +399,217 @@ def _graphs_unsafe_moe(layer):
for m in layer.modules()
)
layer.__dict__["_unsloth_block_graphs_moe"] = moe
- if moe:
- _rank0_note(
- "moe",
- "Unsloth: block CUDA graphs skip MoE layers; they run compiled without graphs.",
- )
+ if moe and note:
+ _rank0_note(
+ "moe",
+ "Unsloth: block CUDA graphs skip MoE layers; they run compiled without graphs.",
+ )
return moe
-def _flex_block_mask_call(args, kwargs):
+def _flex_block_mask_call(
+ args,
+ kwargs,
+ note = True,
+):
"""BlockMask index tensors are static graph inputs rebuilt every step, so the graph re-records
each call (StaticInputIdxMismatch is outside the re-record limit); Gemma3 was slower with graphs."""
for value in (*args, *kwargs.values()):
if type(value).__name__ == "BlockMask":
- _rank0_note(
- "flex",
- "Unsloth: block CUDA graphs skip flex attention blocks; they run compiled without graphs.",
- )
+ if note:
+ _rank0_note(
+ "flex",
+ "Unsloth: block CUDA graphs skip flex attention blocks; they run compiled without graphs.",
+ )
return True
return False
+# Auto mode. Graphs remove launch overhead only: they pay off when the host takes longer to launch a
+# block than the GPU takes to run it, and cost a recording per batch shape. Forward passes 1-2 run
+# without graphs (compile); pass 3 times a few block calls; graphs start at the next pass when every
+# call so far had one batch shape and launching was slower than running by _AUTO_RATIO.
+_AUTO_WARMUP_PASSES = 2
+_AUTO_MEASURE_CALLS = 4
+_AUTO_SLEEP_CYCLES = 20_000_000 # ~10-15 ms: the block's launches queue behind it
+try:
+ _AUTO_RATIO = float(os.environ.get("UNSLOTH_BLOCK_CUDAGRAPHS_AUTO_RATIO", "1.2"))
+except ValueError:
+ _AUTO_RATIO = 1.2
+# 2.13+: the graphed region gets its own recompile bucket (isolate_recompiles) instead of sharing
+# the stock region's cache limit. Before 2.11 FastLanguageModel LoRA blocks are not compiled at all.
+_AUTO_MIN_TORCH = "2.13"
+# The graph pool adds 0.5-0.8 GB reserved (Qwen3-0.6B LoRA): keep graphs off without this much free.
+_AUTO_FREE_GB = 2.0
+_AUTO_FREE_FRACTION = 0.10
+
+
+def _auto_note(message):
+ if os.environ.get("UNSLOTH_ENABLE_LOGGING", "0") == "1":
+ _rank0_note("auto", message)
+
+
+def _auto_off(auto, why):
+ auto["phase"] = "off"
+ auto["why"] = why
+ auto["first"] = None
+ _auto_note(f"Unsloth: block CUDA graphs stay off ({why}).")
+
+
+def _auto_platform_reason():
+ """Why auto mode can never turn graphs on in this process (None: it may)."""
+ if getattr(torch.version, "hip", None):
+ return "ROCm"
+ from packaging.version import Version
+
+ if Version(torch.__version__.split("+")[0]) < Version(_AUTO_MIN_TORCH):
+ return f"torch < {_AUTO_MIN_TORCH}"
+ return None
+
+
+def _auto_static_reason(layer, hidden_states):
+ """Why auto mode never turns graphs on for this run (None: it may)."""
+ if not hidden_states.is_cuda:
+ return "not a CUDA tensor"
+ why = _auto_platform_reason()
+ if why is not None:
+ return why
+ try:
+ import torch.distributed as dist
+ if dist.is_available() and dist.is_initialized() and dist.get_world_size() > 1:
+ return "distributed"
+ except Exception:
+ return "distributed"
+ total = trainable = 0
+ for p in layer.parameters():
+ total += p.numel()
+ if p.requires_grad:
+ trainable += p.numel()
+ if total and trainable * 2 > total:
+ # Measured only for LoRA / QLoRA: full finetuning copies every block weight grad per step.
+ return "full finetuning"
+ return None
+
+
+def _auto_call_reason(layer, hidden_states, args, kwargs):
+ """Why this probed call rules graphs out (None: it does not)."""
+ if _packed_kwargs(kwargs):
+ return "packed / padding-free rows"
+ if _in_nonreentrant_checkpoint():
+ return "non-reentrant checkpointing"
+ if _graphs_unsafe_moe(layer, note = False):
+ return "MoE layer"
+ if _flex_block_mask_call(args, kwargs, note = False):
+ return "flex attention"
+ return None
+
+
+def _auto_observe(layer, hidden_states, args, kwargs):
+ """Auto mode, per block call while undecided: "graphs", "plain", or "measure" (plain, timed)."""
+ auto = _CUDAGRAPH_STATE["auto"]
+ if auto["phase"] == "off":
+ return "plain"
+ if auto["phase"] == "on":
+ return "graphs"
+ outside = _IN_TRIAL[0] or torch._C._current_graph_task_id() != -1
+ if auto["first"] is None:
+ if outside:
+ return "plain" # the trial / a recompute before any forward pass was seen
+ why = _auto_static_reason(layer, hidden_states)
+ if why is not None:
+ _auto_off(auto, why)
+ return "plain"
+ auto["first"] = weakref.ref(layer)
+ auto["shape"] = tuple(hidden_states.shape)
+ why = _auto_call_reason(layer, hidden_states, args, kwargs)
+ if why is None and tuple(hidden_states.shape) != auto["shape"]:
+ why = "batch shape changed"
+ if why is not None:
+ _auto_off(auto, why)
+ return "plain"
+ if outside:
+ return "plain"
+ if auto["first"]() is layer: # a new forward pass
+ if auto["decision"] == "on":
+ auto["phase"] = "on"
+ auto["first"] = None
+ return "graphs"
+ auto["passes"] += 1
+ if auto["passes"] > _AUTO_WARMUP_PASSES and len(auto["host"]) < _AUTO_MEASURE_CALLS:
+ return "measure"
+ return "plain"
+
+
+def _auto_decide(auto, device):
+ host, gpu = sum(auto["host"]), sum(auto["gpu"])
+ ratio = host / max(gpu, 1e-9)
+ detail = f"block launch {host * 1e3 / len(auto['host']):.2f} ms vs GPU {gpu * 1e3 / len(auto['gpu']):.2f} ms"
+ auto["ratio"] = ratio
+ if ratio < _AUTO_RATIO:
+ _auto_off(auto, f"GPU-bound: {detail}")
+ return
+ try:
+ free, total = torch.cuda.mem_get_info(device)
+ except Exception:
+ _auto_off(auto, "free memory unknown")
+ return
+ need = max(_AUTO_FREE_GB * 2**30, _AUTO_FREE_FRACTION * total)
+ if free < need:
+ _auto_off(auto, f"{free / 2**30:.1f} GiB free")
+ return
+ auto["decision"] = "on"
+ _rank0_note(
+ "auto",
+ f"Unsloth: block CUDA graphs on (one batch shape, {detail}). "
+ "UNSLOTH_BLOCK_CUDAGRAPHS=0 keeps them off.",
+ )
+
+
+def _auto_time(call, device):
+ """(output, host seconds to launch one block call, GPU seconds to run it). The launches queue
+ behind a GPU sleep, so the event pair after the sleep sees the block's kernels back to back; if
+ launching outlasts the sleep the GPU time includes gaps, which can only understate the ratio.
+ Sleep and events run on the block's device (cuda:1 or a device_map shard need not be current).
+ Host / GPU are None when timing is unavailable."""
+ try:
+ guard = torch.cuda.device(device)
+ except Exception:
+ return call(), None, None
+ with guard:
+ try:
+ torch.cuda.synchronize(device)
+ start = torch.cuda.Event(enable_timing = True)
+ end = torch.cuda.Event(enable_timing = True)
+ torch.cuda._sleep(_AUTO_SLEEP_CYCLES)
+ start.record()
+ except Exception:
+ return call(), None, None
+ t0 = time.perf_counter()
+ out = call()
+ host = time.perf_counter() - t0
+ try:
+ end.record()
+ end.synchronize()
+ return out, host, start.elapsed_time(end) / 1e3
+ except Exception:
+ return out, None, None
+
+
+def _auto_measure(call, hidden_states):
+ auto = _CUDAGRAPH_STATE["auto"]
+ out, host, gpu = _auto_time(call, hidden_states.device)
+ if auto["phase"] != "probe" or auto["decision"] is not None:
+ return out
+ if host is None or gpu is None:
+ _auto_off(auto, "timing unavailable")
+ return out
+ auto["host"].append(host)
+ auto["gpu"].append(gpu)
+ if len(auto["host"]) >= _AUTO_MEASURE_CALLS:
+ _auto_decide(auto, hidden_states.device)
+ return out
+
+
def _note_block_mask_call(layer, args, kwargs):
"""_flex_block_mask_call, also recording `_unsloth_block_flex_only`; the observer runs on each change."""
flex = _flex_block_mask_call(args, kwargs)
@@ -1173,6 +1398,8 @@ def _wrap(
# Under Unsloth checkpointing each block call is a new cudagraph generation: the next replay
# overwrites this block's outputs and gradients, so they are cloned out of the pool.
graphs = bool(compile_options.get("triton.cudagraphs"))
+ # Auto: `compiled` stays the graphs-off region; the graphed one is compiled once graphs turn on.
+ auto = not graphs and _CUDAGRAPH_MODE == "auto"
if graphs:
plain = torch.compile(
original,
@@ -1181,6 +1408,21 @@ def _wrap(
options = {**compile_options, "triton.cudagraphs": False},
**isolate,
)
+ else:
+ plain = compiled
+ graphed_box = [compiled if graphs else None]
+
+ def graphed():
+ if graphed_box[0] is None:
+ graphed_box[0] = torch.compile(
+ original,
+ fullgraph = False,
+ dynamic = None,
+ options = {**compile_options, "triton.cudagraphs": True},
+ **isolate,
+ )
+ return graphed_box[0]
+
graph_shapes = set()
def run(self, hidden_states, args, kwargs, mode):
@@ -1203,7 +1445,15 @@ def _wrap(
kwargs = {**kwargs, packed_module.BLOCK_VARLEN_KWARG: varlen}
# Capture hooks (router_logits) append to empty scratch lists: no guard on the layer index.
with capture_scratch(), _no_ddp_graph_split():
- if graphs and _CUDAGRAPH_STATE["off"] is None:
+ use_graphs = graphs
+ if auto and _CUDAGRAPH_STATE["auto"]["phase"] != "off":
+ verdict = _auto_observe(self, hidden_states, args, kwargs)
+ if verdict == "measure":
+ return _auto_measure(
+ lambda: compiled(self, hidden_states, *args, **kwargs), hidden_states
+ )
+ use_graphs = verdict == "graphs"
+ if use_graphs and _CUDAGRAPH_STATE["off"] is None:
shape = tuple(hidden_states.shape)
if _packed_kwargs(kwargs):
_graphs_off("packed", _PACKED_NOTE)
@@ -1213,8 +1463,8 @@ def _wrap(
else:
graph_shapes.add(shape)
# Before the graphs_off read: the observer may set or clear it for this block.
- flex = graphs and _note_block_mask_call(self, args, kwargs)
- if graphs and (
+ flex = use_graphs and _note_block_mask_call(self, args, kwargs)
+ if use_graphs and (
_CUDAGRAPH_STATE["off"] is not None
or self.__dict__.get("_unsloth_block_graphs_off", False)
or _in_nonreentrant_checkpoint()
@@ -1224,7 +1474,7 @@ def _wrap(
if _CUDAGRAPH_STATE["off"] is not None and _CUDAGRAPH_STATE["release"] is None:
_release_graph_pool()
return plain(self, hidden_states, *args, **kwargs)
- if graphs:
+ if use_graphs:
_param_grads_out_of_pool(self)
device = hidden_states.device.index
if _pool_outputs_stay_valid(self, hidden_states):
@@ -1232,11 +1482,13 @@ def _wrap(
# A leaf's .grad outlives the generation (trainable inputs_embeds); a
# checkpoint's backward recomputes (a new generation) before reading it.
hidden_states = _CloneGrad.apply(hidden_states)
- return _graphed_call(compiled, device, self, hidden_states, *args, **kwargs)
+ return _graphed_call(
+ graphed(), device, self, hidden_states, *args, **kwargs
+ )
if hidden_states.requires_grad and torch.is_grad_enabled():
hidden_states = _CloneGrad.apply(hidden_states)
return _clone_outputs(
- _graphed_call(compiled, device, self, hidden_states, *args, **kwargs)
+ _graphed_call(graphed(), device, self, hidden_states, *args, **kwargs)
)
return compiled(self, hidden_states, *args, **kwargs)
finally:
@@ -1275,13 +1527,16 @@ def _wrap(
# PrefixGrouper's int segment table would recompile at every new GRPO length.
or (unsloth_forward and kwargs.get("prefix_seg_info") is not None)
):
+ phase = _CUDAGRAPH_STATE["auto"]["phase"] if auto else None
if (
- graphs
+ (graphs or phase == "on")
and self.training
and _CUDAGRAPH_STATE["off"] is None
and _packed_kwargs(kwargs)
):
_graphs_off("packed", _PACKED_NOTE)
+ elif phase == "probe" and self.training and _packed_kwargs(kwargs):
+ _auto_off(_CUDAGRAPH_STATE["auto"], "packed / padding-free rows")
return original(self, hidden_states, *args, **kwargs)
if tracing:
# Inside a user's torch.compile the enclosing graph owns the layer: no dynamic marking
@@ -1621,8 +1876,14 @@ def patch_block_compile(model):
continue
_wrap(cls, _COMPILE_OPTIONS, len(kinds.get(cls, ())) or 1)
wrapped.append(cls.__name__)
- if _COMPILE_OPTIONS.get("triton.cudagraphs"):
+ if _COMPILE_OPTIONS.get("triton.cudagraphs") or (
+ _CUDAGRAPH_MODE == "auto" and _auto_platform_reason() is None
+ ):
_watch_hidden_states(model)
+ if _CUDAGRAPH_MODE == "auto":
+ # A new model decides again: a launch-bound model loaded first must not turn graphs on for
+ # a GPU-bound one. Graphs that went off for shape churn stay off.
+ _CUDAGRAPH_STATE["auto"] = _new_auto_state()
if wrapped:
print(f"Unsloth: compiling each transformer block as one region ({', '.join(wrapped)}).")
return wrapped