Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Block CUDA graphs: turn on automatically for fixed-shape, launch-bound LoRA runs (#15768)
Daniel Han04d378863712
diff --git a/tests/test_block_cudagraphs_auto.py b/tests/test_block_cudagraphs_auto.pynew file mode 100644index 00000000000..e14fc21bf7b--- /dev/null+++ b/tests/test_block_cudagraphs_auto.py@@ -0,0 +1,355 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.++"""Block CUDA graphs auto mode (UNSLOTH_BLOCK_CUDAGRAPHS unset): graphs turn on only after the first+forward passes all had one batch shape and a timed block launched slower than the GPU ran it.+"1" still forces them on, "0" (or any other value) keeps them off. Timing and the device checks are+stubbed, so this runs on CPU."""++import importlib.util+import os+import sys++import pytest+import torch++_DIR = os.path.join(+ os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "unsloth", "models"+)+++def _load():+ spec = importlib.util.spec_from_file_location(+ "_ublock_compile_cg_auto", os.path.join(_DIR, "_block_compile.py")+ )+ mod = importlib.util.module_from_spec(spec)+ sys.modules[spec.name] = mod+ spec.loader.exec_module(mod)+ return mod+++bc = _load()++[email protected](autouse = True)+def _block_env(monkeypatch):+ for key in (+ "UNSLOTH_BLOCK_COMPILE",+ "UNSLOTH_COMPILE_DISABLE",+ "UNSLOTH_FORCE_FLOAT32",+ "UNSLOTH_BLOCK_CUDAGRAPHS",+ ):+ monkeypatch.delenv(key, raising = False)+ monkeypatch.setattr(bc, "_CUDAGRAPH_STATE", bc._new_cudagraph_state())+ monkeypatch.setattr(bc, "_CUDAGRAPH_MODE", "auto")+++ "value, mode",+ [(None, "auto"), ("", "auto"), ("auto", "auto"), ("1", "on"), ("0", "off"), ("true", "off")],+)+def test_mode_from_environment(monkeypatch, value, mode):+ """Unset is auto now; "1" forces graphs, every other value keeps the old meaning (off)."""+ try:+ if value is None:+ monkeypatch.delenv("UNSLOTH_BLOCK_CUDAGRAPHS", raising = False)+ else:+ monkeypatch.setenv("UNSLOTH_BLOCK_CUDAGRAPHS", value)+ mod = _load()+ assert mod._CUDAGRAPH_MODE == mode+ # Only a forced "1" records graphs from the first call; auto starts graphs-off.+ assert mod._COMPILE_OPTIONS["triton.cudagraphs"] is (mode == "on")+ finally:+ monkeypatch.delenv("UNSLOTH_BLOCK_CUDAGRAPHS", raising = False)+ _load()+++def _harness(+ monkeypatch,+ host = 3e-3,+ gpu = 1e-3,+ free_gb = 40.0,+ static = None,+):+ calls = []+ compiles = []++ def fake_compile(fn, **kwargs):+ tag = "graphs" if kwargs.get("options", {}).get("triton.cudagraphs") else "plain"+ compiles.append(tag)++ def run(*args, **kw):+ calls.append(tag)+ return fn(*args, **kw)++ return run++ monkeypatch.setattr(torch, "compile", fake_compile)+ monkeypatch.setattr(bc, "_auto_static_reason", lambda layer, hs: static)+ timed = []++ def fake_time(call, device):+ timed.append(1)+ return call(), host, gpu++ monkeypatch.setattr(bc, "_auto_time", fake_time)+ monkeypatch.setattr(+ torch.cuda, "mem_get_info", lambda device = None: (int(free_gb * 2**30), 80 * 2**30)+ )++ class AutoDecoderLayer(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.proj = torch.nn.Linear(8, 8)++ def forward(self, hidden_states, **kwargs):+ return self.proj(hidden_states)++ bc._wrap(AutoDecoderLayer, dict(bc._COMPILE_OPTIONS))+ layers = torch.nn.ModuleList([AutoDecoderLayer(), AutoDecoderLayer()]).train()+ return layers, calls, compiles, timed+++def _step(+ layers,+ calls,+ L = 16,+ checkpoint = False,+ **kwargs,+):+ calls.clear()+ x = torch.randn(2, L, 8, requires_grad = True)+ h = x+ for layer in layers:+ if checkpoint:+ h = torch.utils.checkpoint.checkpoint(+ lambda t, layer = layer: layer(t, **kwargs), h, use_reentrant = True+ )+ else:+ h = layer(h, **kwargs)+ h.sum().backward()+ assert x.grad is not None and x.grad.shape == x.shape+ return list(calls)+++def test_auto_turns_graphs_on_for_fixed_shape_launch_bound(monkeypatch):+ layers, calls, compiles, timed = _harness(monkeypatch)+ assert compiles == ["plain"] # the graphed region is compiled only once graphs turn on+ for _ in range(bc._AUTO_WARMUP_PASSES):+ assert _step(layers, calls) == ["plain", "plain"]+ assert _step(layers, calls) == ["plain", "plain"] # pass 3: timed, still graphs-off+ assert len(timed) == 2+ _step(layers, calls) # pass 4 times the remaining calls, then decides+ assert bc._CUDAGRAPH_STATE["auto"]["decision"] == "on"+ assert _step(layers, calls) == ["graphs", "graphs"]+ assert compiles == ["plain", "graphs"]+ assert bc._CUDAGRAPH_STATE["auto"]["phase"] == "on"+ assert _step(layers, calls) == ["graphs", "graphs"]+ assert len(timed) == bc._AUTO_MEASURE_CALLS+++def test_auto_keeps_graphs_off_when_gpu_bound(monkeypatch):+ layers, calls, compiles, _ = _harness(monkeypatch, host = 1e-3, gpu = 2e-3)+ for _ in range(8):+ assert _step(layers, calls) == ["plain", "plain"]+ auto = bc._CUDAGRAPH_STATE["auto"]+ assert auto["phase"] == "off" and auto["why"].startswith("GPU-bound")+ assert compiles == ["plain"]+++def test_auto_ratio_threshold(monkeypatch):+ """Launching must be clearly slower than running: A100 ratios 0.55-0.92 gained nothing with graphs+ forced on (-1.2% to +1.0%), 1.26 gained 12.6%."""+ layers, calls, _, _ = _harness(monkeypatch, host = 1.1e-3, gpu = 1e-3)+ for _ in range(6):+ assert _step(layers, calls) == ["plain", "plain"]+ assert bc._CUDAGRAPH_STATE["auto"]["phase"] == "off"+++def test_auto_shape_change_while_probing_keeps_graphs_off(monkeypatch):+ layers, calls, _, timed = _harness(monkeypatch)+ _step(layers, calls, L = 16)+ _step(layers, calls, L = 24) # padded to the longest row: variable lengths+ auto = bc._CUDAGRAPH_STATE["auto"]+ assert auto["phase"] == "off" and auto["why"] == "batch shape changed"+ for _ in range(6):+ assert _step(layers, calls, L = 16) == ["plain", "plain"]+ assert not timed+++def test_auto_after_on_new_shapes_past_the_cap_turn_graphs_off(monkeypatch):+ monkeypatch.setattr(bc, "_CUDAGRAPH_MAX_SHAPES", 2)+ released = []+ monkeypatch.setattr(bc, "_release_graph_pool", lambda: released.append(1))+ layers, calls, _, _ = _harness(monkeypatch)+ for _ in range(5):+ _step(layers, calls)+ assert _step(layers, calls) == ["graphs", "graphs"]+ assert _step(layers, calls, L = 8) == ["graphs", "graphs"] # short last batch: second shape+ assert _step(layers, calls, L = 24) == ["plain", "plain"] # third: over the cap+ assert bc._CUDAGRAPH_STATE["off"] == "shapes"+ assert released+ assert _step(layers, calls) == ["plain", "plain"]+++def test_auto_packed_rows_keep_graphs_off(monkeypatch):+ layers, calls, _, timed = _harness(monkeypatch)+ lengths = torch.tensor([8, 8])+ for _ in range(6):+ assert _step(layers, calls, packed_seq_lengths = lengths) == ["plain", "plain"]+ auto = bc._CUDAGRAPH_STATE["auto"]+ assert auto["phase"] == "off" and "packed" in auto["why"]+ assert not timed+++def test_auto_low_free_memory_keeps_graphs_off(monkeypatch):+ layers, calls, _, _ = _harness(monkeypatch, free_gb = 1.0)+ for _ in range(6):+ assert _step(layers, calls) == ["plain", "plain"]+ assert bc._CUDAGRAPH_STATE["auto"]["why"].endswith("GiB free")+++def test_auto_static_reason_keeps_graphs_off(monkeypatch):+ layers, calls, _, timed = _harness(monkeypatch, static = "distributed")+ for _ in range(6):+ assert _step(layers, calls) == ["plain", "plain"]+ assert bc._CUDAGRAPH_STATE["auto"]["why"] == "distributed"+ assert not timed+++def test_auto_counts_passes_not_recomputes(monkeypatch):+ """Reentrant checkpointing calls each block again inside backward: those calls are not forward+ passes, so graphs still start after the same number of steps, at a pass boundary."""+ layers, calls, _, timed = _harness(monkeypatch)+ for _ in range(bc._AUTO_WARMUP_PASSES):+ # the first step also runs each block's grad-mode trial before its first no-grad call+ assert set(_step(layers, calls, checkpoint = True)) == {"plain"}+ _step(layers, calls, checkpoint = True)+ _step(layers, calls, checkpoint = True)+ assert len(timed) == bc._AUTO_MEASURE_CALLS # forward calls only, never a recompute+ assert _step(layers, calls, checkpoint = True) == ["graphs"] * 4+++def test_explicit_off_never_probes(monkeypatch):+ monkeypatch.setattr(bc, "_CUDAGRAPH_MODE", "off")+ layers, calls, compiles, timed = _harness(monkeypatch)+ for _ in range(6):+ assert _step(layers, calls) == ["plain", "plain"]+ assert not timed and compiles == ["plain"]+ assert bc._CUDAGRAPH_STATE["auto"]["phase"] == "probe" # untouched+++def test_static_reason_rules():+ layer = torch.nn.Linear(4, 4)+ assert bc._auto_static_reason(layer, torch.zeros(1)) == "not a CUDA tensor"+++def test_static_reason_full_finetuning(monkeypatch):+ class FakeCuda:+ is_cuda = True++ layer = torch.nn.Linear(4, 4) # every weight trainable+ monkeypatch.setattr(torch.version, "hip", None, raising = False)+ monkeypatch.setattr(torch, "__version__", "2.14.1")+ assert bc._auto_static_reason(layer, FakeCuda()) == "full finetuning"+ for p in layer.parameters():+ p.requires_grad_(False)+ assert bc._auto_static_reason(layer, FakeCuda()) is None+ monkeypatch.setattr(torch, "__version__", "2.12.1+cu128")+ assert bc._auto_static_reason(layer, FakeCuda()) == "torch < 2.13"+++def test_a_new_model_decides_again(monkeypatch):+ """Graphs on for one (launch-bound) model must not carry over to the next model loaded."""+ layers, calls, _, _ = _harness(monkeypatch)+ for _ in range(5):+ _step(layers, calls)+ assert _step(layers, calls) == ["graphs", "graphs"]+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)+ monkeypatch.setattr(bc, "_torch_supported", lambda: True)++ class NextDecoderLayer(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.proj = torch.nn.Linear(8, 8)++ def forward(self, hidden_states, **kwargs):+ return self.proj(hidden_states)++ model = torch.nn.Module()+ model.layers = torch.nn.ModuleList([NextDecoderLayer(), NextDecoderLayer()])+ assert bc.patch_block_compile(model) == ["NextDecoderLayer"]+ assert bc._CUDAGRAPH_STATE["auto"]["phase"] == "probe"+ model.train()+ assert _step(model.layers, calls) == ["plain", "plain"]+++ "version, hip, hooked",+ [("2.14.1", None, True), ("2.12.1+cu128", None, False), ("2.14.1", "7.0", False)],+)+def test_auto_hooks_the_model_only_where_graphs_can_turn_on(monkeypatch, version, hip, hooked):+ """The hidden-states watcher only serves graphs: no forward hook where auto can never use them."""+ monkeypatch.setattr(bc, "_CUDAGRAPH_MODE", "auto")+ monkeypatch.setattr(torch, "__version__", version)+ monkeypatch.setattr(torch.version, "hip", hip, raising = False)+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)+ monkeypatch.setattr(bc, "_torch_supported", lambda: True)++ class HookDecoderLayer(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.proj = torch.nn.Linear(8, 8)++ def forward(self, hidden_states, **kwargs):+ return self.proj(hidden_states)++ model = torch.nn.Module()+ model.layers = torch.nn.ModuleList([HookDecoderLayer(), HookDecoderLayer()])+ assert bc.patch_block_compile(model) == ["HookDecoderLayer"]+ assert model.__dict__.get("_unsloth_watch_hidden_states", False) is hooked+++def test_probe_times_on_the_blocks_device(monkeypatch):+ """A block on cuda:1 while cuda:0 is current: sleep and both events must run on cuda:1."""+ current = ["cuda:0"]+ log = []++ class Guard:+ def __init__(self, device):+ self.device = str(device)++ def __enter__(self):+ self.prev, current[0] = current[0], self.device++ def __exit__(self, *exc):+ current[0] = self.prev++ class Event:+ def __init__(self, enable_timing = False):+ pass++ def record(self):+ log.append(("record", current[0]))++ def synchronize(self):+ pass++ def elapsed_time(self, other):+ return 2.0++ monkeypatch.setattr(torch.cuda, "device", Guard)+ monkeypatch.setattr(torch.cuda, "Event", Event)+ monkeypatch.setattr(torch.cuda, "synchronize", lambda device = None: None)+ monkeypatch.setattr(torch.cuda, "_sleep", lambda cycles: log.append(("sleep", current[0])))+ out, host, gpu = bc._auto_time(+ lambda: log.append(("call", current[0])) or "out", torch.device("cuda", 1)+ )+ assert out == "out" and gpu == 2.0 / 1e3 and host is not None+ assert log == [+ ("sleep", "cuda:1"),+ ("record", "cuda:1"),+ ("call", "cuda:1"),+ ("record", "cuda:1"),+ ]+ assert current[0] == "cuda:0"diff --git a/unsloth/models/_block_compile.py b/unsloth/models/_block_compile.pyindex ae5ca2781fa..76c13b79749 100644--- a/unsloth/models/_block_compile.py+++ b/unsloth/models/_block_compile.py@@ -9,6 +9,7 @@ import functoolsimport inspectimport osimport sys+import timeimport typesimport weakref@@ -23,8 +24,20 @@ _COMPILE_OPTIONS = {"shape_padding": True,"triton.cudagraphs": False,}-# Opt-in per-block CUDA graphs; only pays off for static shapes.-if os.environ.get("UNSLOTH_BLOCK_CUDAGRAPHS", "0") == "1":+++def _cudagraph_mode():+ """UNSLOTH_BLOCK_CUDAGRAPHS: "1" always on, unset or "auto" decided per run (see _auto_observe),+ anything else off (before auto, every value but "1" meant off)."""+ value = os.environ.get("UNSLOTH_BLOCK_CUDAGRAPHS", "auto").strip().lower()+ if value == "1":+ return "on"+ return "auto" if value in ("", "auto") else "off"+++# Per-block CUDA graphs only pay off for static shapes on a launch-bound step.+_CUDAGRAPH_MODE = _cudagraph_mode()+if _CUDAGRAPH_MODE == "on":_COMPILE_OPTIONS["triton.cudagraphs"] = True# Shape cap: variable lengths recorded a graph set per length (3.5x slower, pool grew every step).# Two shapes = one fixed length plus a short last batch. Past the cap graphs go off for every block@@ -39,7 +52,29 @@ except ValueError:# off: why graphs went off (None = on); release: None, "done" or "kept"; ours: compile ids of the# tree functions block calls created; managers: CUDA index -> the tree manager block graphs ran on.def _new_cudagraph_state():- return {"off": None, "release": None, "ours": set(), "managers": {}, "notes": set()}+ return {+ "off": None,+ "release": None,+ "ours": set(),+ "managers": {},+ "notes": set(),+ "auto": _new_auto_state(),+ }+++# Auto mode: phase "probe" (graphs off, watching), "on" or "off" (final). first: the layer whose+# call starts a forward pass; passes: forward passes seen, all with batch shape `shape`.+def _new_auto_state():+ return {+ "phase": "probe",+ "first": None,+ "shape": None,+ "passes": 0,+ "host": [],+ "gpu": [],+ "decision": None,+ "why": None,+ }_CUDAGRAPH_STATE = _new_cudagraph_state()@@ -351,7 +386,7 @@ def _in_nonreentrant_checkpoint():return bool(hooks) and getattr(hooks[0], "__module__", None) == "torch.utils.checkpoint"-def _graphs_unsafe_moe(layer):+def _graphs_unsafe_moe(layer, note = True):"""MoE routing / grouped GEMMs read token counts the capture cannot hold (Qwen3-MoE: host copyduring capture, routing outputs overwritten by the next replay)."""moe = layer.__dict__.get("_unsloth_block_graphs_moe")@@ -364,27 +399,217 @@ def _graphs_unsafe_moe(layer):for m in layer.modules())layer.__dict__["_unsloth_block_graphs_moe"] = moe- if moe:- _rank0_note(- "moe",- "Unsloth: block CUDA graphs skip MoE layers; they run compiled without graphs.",- )+ if moe and note:+ _rank0_note(+ "moe",+ "Unsloth: block CUDA graphs skip MoE layers; they run compiled without graphs.",+ )return moe-def _flex_block_mask_call(args, kwargs):+def _flex_block_mask_call(+ args,+ kwargs,+ note = True,+):"""BlockMask index tensors are static graph inputs rebuilt every step, so the graph re-recordseach call (StaticInputIdxMismatch is outside the re-record limit); Gemma3 was slower with graphs."""for value in (*args, *kwargs.values()):if type(value).__name__ == "BlockMask":- _rank0_note(- "flex",- "Unsloth: block CUDA graphs skip flex attention blocks; they run compiled without graphs.",- )+ if note:+ _rank0_note(+ "flex",+ "Unsloth: block CUDA graphs skip flex attention blocks; they run compiled without graphs.",+ )return Truereturn False+# Auto mode. Graphs remove launch overhead only: they pay off when the host takes longer to launch a+# block than the GPU takes to run it, and cost a recording per batch shape. Forward passes 1-2 run+# without graphs (compile); pass 3 times a few block calls; graphs start at the next pass when every+# call so far had one batch shape and launching was slower than running by _AUTO_RATIO.+_AUTO_WARMUP_PASSES = 2+_AUTO_MEASURE_CALLS = 4+_AUTO_SLEEP_CYCLES = 20_000_000 # ~10-15 ms: the block's launches queue behind it+try:+ _AUTO_RATIO = float(os.environ.get("UNSLOTH_BLOCK_CUDAGRAPHS_AUTO_RATIO", "1.2"))+except ValueError:+ _AUTO_RATIO = 1.2+# 2.13+: the graphed region gets its own recompile bucket (isolate_recompiles) instead of sharing+# the stock region's cache limit. Before 2.11 FastLanguageModel LoRA blocks are not compiled at all.+_AUTO_MIN_TORCH = "2.13"+# The graph pool adds 0.5-0.8 GB reserved (Qwen3-0.6B LoRA): keep graphs off without this much free.+_AUTO_FREE_GB = 2.0+_AUTO_FREE_FRACTION = 0.10+++def _auto_note(message):+ if os.environ.get("UNSLOTH_ENABLE_LOGGING", "0") == "1":+ _rank0_note("auto", message)+++def _auto_off(auto, why):+ auto["phase"] = "off"+ auto["why"] = why+ auto["first"] = None+ _auto_note(f"Unsloth: block CUDA graphs stay off ({why}).")+++def _auto_platform_reason():+ """Why auto mode can never turn graphs on in this process (None: it may)."""+ if getattr(torch.version, "hip", None):+ return "ROCm"+ from packaging.version import Version++ if Version(torch.__version__.split("+")[0]) < Version(_AUTO_MIN_TORCH):+ return f"torch < {_AUTO_MIN_TORCH}"+ return None+++def _auto_static_reason(layer, hidden_states):+ """Why auto mode never turns graphs on for this run (None: it may)."""+ if not hidden_states.is_cuda:+ return "not a CUDA tensor"+ why = _auto_platform_reason()+ if why is not None:+ return why+ try:+ import torch.distributed as dist+ if dist.is_available() and dist.is_initialized() and dist.get_world_size() > 1:+ return "distributed"+ except Exception:+ return "distributed"+ total = trainable = 0+ for p in layer.parameters():+ total += p.numel()+ if p.requires_grad:+ trainable += p.numel()+ if total and trainable * 2 > total:+ # Measured only for LoRA / QLoRA: full finetuning copies every block weight grad per step.+ return "full finetuning"+ return None+++def _auto_call_reason(layer, hidden_states, args, kwargs):+ """Why this probed call rules graphs out (None: it does not)."""+ if _packed_kwargs(kwargs):+ return "packed / padding-free rows"+ if _in_nonreentrant_checkpoint():+ return "non-reentrant checkpointing"+ if _graphs_unsafe_moe(layer, note = False):+ return "MoE layer"+ if _flex_block_mask_call(args, kwargs, note = False):+ return "flex attention"+ return None+++def _auto_observe(layer, hidden_states, args, kwargs):+ """Auto mode, per block call while undecided: "graphs", "plain", or "measure" (plain, timed)."""+ auto = _CUDAGRAPH_STATE["auto"]+ if auto["phase"] == "off":+ return "plain"+ if auto["phase"] == "on":+ return "graphs"+ outside = _IN_TRIAL[0] or torch._C._current_graph_task_id() != -1+ if auto["first"] is None:+ if outside:+ return "plain" # the trial / a recompute before any forward pass was seen+ why = _auto_static_reason(layer, hidden_states)+ if why is not None:+ _auto_off(auto, why)+ return "plain"+ auto["first"] = weakref.ref(layer)+ auto["shape"] = tuple(hidden_states.shape)+ why = _auto_call_reason(layer, hidden_states, args, kwargs)+ if why is None and tuple(hidden_states.shape) != auto["shape"]:+ why = "batch shape changed"+ if why is not None:+ _auto_off(auto, why)+ return "plain"+ if outside:+ return "plain"+ if auto["first"]() is layer: # a new forward pass+ if auto["decision"] == "on":+ auto["phase"] = "on"+ auto["first"] = None+ return "graphs"+ auto["passes"] += 1+ if auto["passes"] > _AUTO_WARMUP_PASSES and len(auto["host"]) < _AUTO_MEASURE_CALLS:+ return "measure"+ return "plain"+++def _auto_decide(auto, device):+ host, gpu = sum(auto["host"]), sum(auto["gpu"])+ ratio = host / max(gpu, 1e-9)+ detail = f"block launch {host * 1e3 / len(auto['host']):.2f} ms vs GPU {gpu * 1e3 / len(auto['gpu']):.2f} ms"+ auto["ratio"] = ratio+ if ratio < _AUTO_RATIO:+ _auto_off(auto, f"GPU-bound: {detail}")+ return+ try:+ free, total = torch.cuda.mem_get_info(device)+ except Exception:+ _auto_off(auto, "free memory unknown")+ return+ need = max(_AUTO_FREE_GB * 2**30, _AUTO_FREE_FRACTION * total)+ if free < need:+ _auto_off(auto, f"{free / 2**30:.1f} GiB free")+ return+ auto["decision"] = "on"+ _rank0_note(+ "auto",+ f"Unsloth: block CUDA graphs on (one batch shape, {detail}). "+ "UNSLOTH_BLOCK_CUDAGRAPHS=0 keeps them off.",+ )+++def _auto_time(call, device):+ """(output, host seconds to launch one block call, GPU seconds to run it). The launches queue+ behind a GPU sleep, so the event pair after the sleep sees the block's kernels back to back; if+ launching outlasts the sleep the GPU time includes gaps, which can only understate the ratio.+ Sleep and events run on the block's device (cuda:1 or a device_map shard need not be current).+ Host / GPU are None when timing is unavailable."""+ try:+ guard = torch.cuda.device(device)+ except Exception:+ return call(), None, None+ with guard:+ try:+ torch.cuda.synchronize(device)+ start = torch.cuda.Event(enable_timing = True)+ end = torch.cuda.Event(enable_timing = True)+ torch.cuda._sleep(_AUTO_SLEEP_CYCLES)+ start.record()+ except Exception:+ return call(), None, None+ t0 = time.perf_counter()+ out = call()+ host = time.perf_counter() - t0+ try:+ end.record()+ end.synchronize()+ return out, host, start.elapsed_time(end) / 1e3+ except Exception:+ return out, None, None+++def _auto_measure(call, hidden_states):+ auto = _CUDAGRAPH_STATE["auto"]+ out, host, gpu = _auto_time(call, hidden_states.device)+ if auto["phase"] != "probe" or auto["decision"] is not None:+ return out+ if host is None or gpu is None:+ _auto_off(auto, "timing unavailable")+ return out+ auto["host"].append(host)+ auto["gpu"].append(gpu)+ if len(auto["host"]) >= _AUTO_MEASURE_CALLS:+ _auto_decide(auto, hidden_states.device)+ return out++def _note_block_mask_call(layer, args, kwargs):"""_flex_block_mask_call, also recording `_unsloth_block_flex_only`; the observer runs on each change."""flex = _flex_block_mask_call(args, kwargs)@@ -1173,6 +1398,8 @@ def _wrap(# Under Unsloth checkpointing each block call is a new cudagraph generation: the next replay# overwrites this block's outputs and gradients, so they are cloned out of the pool.graphs = bool(compile_options.get("triton.cudagraphs"))+ # Auto: `compiled` stays the graphs-off region; the graphed one is compiled once graphs turn on.+ auto = not graphs and _CUDAGRAPH_MODE == "auto"if graphs:plain = torch.compile(original,@@ -1181,6 +1408,21 @@ def _wrap(options = {**compile_options, "triton.cudagraphs": False},**isolate,)+ else:+ plain = compiled+ graphed_box = [compiled if graphs else None]++ def graphed():+ if graphed_box[0] is None:+ graphed_box[0] = torch.compile(+ original,+ fullgraph = False,+ dynamic = None,+ options = {**compile_options, "triton.cudagraphs": True},+ **isolate,+ )+ return graphed_box[0]+graph_shapes = set()def run(self, hidden_states, args, kwargs, mode):@@ -1203,7 +1445,15 @@ def _wrap(kwargs = {**kwargs, packed_module.BLOCK_VARLEN_KWARG: varlen}# Capture hooks (router_logits) append to empty scratch lists: no guard on the layer index.with capture_scratch(), _no_ddp_graph_split():- if graphs and _CUDAGRAPH_STATE["off"] is None:+ use_graphs = graphs+ if auto and _CUDAGRAPH_STATE["auto"]["phase"] != "off":+ verdict = _auto_observe(self, hidden_states, args, kwargs)+ if verdict == "measure":+ return _auto_measure(+ lambda: compiled(self, hidden_states, *args, **kwargs), hidden_states+ )+ use_graphs = verdict == "graphs"+ if use_graphs and _CUDAGRAPH_STATE["off"] is None:shape = tuple(hidden_states.shape)if _packed_kwargs(kwargs):_graphs_off("packed", _PACKED_NOTE)@@ -1213,8 +1463,8 @@ def _wrap(else:graph_shapes.add(shape)# Before the graphs_off read: the observer may set or clear it for this block.- flex = graphs and _note_block_mask_call(self, args, kwargs)- if graphs and (+ flex = use_graphs and _note_block_mask_call(self, args, kwargs)+ if use_graphs and (_CUDAGRAPH_STATE["off"] is not Noneor self.__dict__.get("_unsloth_block_graphs_off", False)or _in_nonreentrant_checkpoint()@@ -1224,7 +1474,7 @@ def _wrap(if _CUDAGRAPH_STATE["off"] is not None and _CUDAGRAPH_STATE["release"] is None:_release_graph_pool()return plain(self, hidden_states, *args, **kwargs)- if graphs:+ if use_graphs:_param_grads_out_of_pool(self)device = hidden_states.device.indexif _pool_outputs_stay_valid(self, hidden_states):@@ -1232,11 +1482,13 @@ def _wrap(# A leaf's .grad outlives the generation (trainable inputs_embeds); a# checkpoint's backward recomputes (a new generation) before reading it.hidden_states = _CloneGrad.apply(hidden_states)- return _graphed_call(compiled, device, self, hidden_states, *args, **kwargs)+ return _graphed_call(+ graphed(), device, self, hidden_states, *args, **kwargs+ )if hidden_states.requires_grad and torch.is_grad_enabled():hidden_states = _CloneGrad.apply(hidden_states)return _clone_outputs(- _graphed_call(compiled, device, self, hidden_states, *args, **kwargs)+ _graphed_call(graphed(), device, self, hidden_states, *args, **kwargs))return compiled(self, hidden_states, *args, **kwargs)finally:@@ -1275,13 +1527,16 @@ def _wrap(# PrefixGrouper's int segment table would recompile at every new GRPO length.or (unsloth_forward and kwargs.get("prefix_seg_info") is not None)):+ phase = _CUDAGRAPH_STATE["auto"]["phase"] if auto else Noneif (- graphs+ (graphs or phase == "on")and self.trainingand _CUDAGRAPH_STATE["off"] is Noneand _packed_kwargs(kwargs)):_graphs_off("packed", _PACKED_NOTE)+ elif phase == "probe" and self.training and _packed_kwargs(kwargs):+ _auto_off(_CUDAGRAPH_STATE["auto"], "packed / padding-free rows")return original(self, hidden_states, *args, **kwargs)if tracing:# Inside a user's torch.compile the enclosing graph owns the layer: no dynamic marking@@ -1621,8 +1876,14 @@ def patch_block_compile(model):continue_wrap(cls, _COMPILE_OPTIONS, len(kinds.get(cls, ())) or 1)wrapped.append(cls.__name__)- if _COMPILE_OPTIONS.get("triton.cudagraphs"):+ if _COMPILE_OPTIONS.get("triton.cudagraphs") or (+ _CUDAGRAPH_MODE == "auto" and _auto_platform_reason() is None+ ):_watch_hidden_states(model)+ if _CUDAGRAPH_MODE == "auto":+ # A new model decides again: a launch-bound model loaded first must not turn graphs on for+ # a GPU-bound one. Graphs that went off for shape churn stay off.+ _CUDAGRAPH_STATE["auto"] = _new_auto_state()if wrapped:print(f"Unsloth: compiling each transformer block as one region ({', '.join(wrapped)}).")return wrapped