Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Block CUDA graphs: auto mode stays off when decoder blocks span several devices (#15931)

Daniel Hane039edb2b647
diff --git a/tests/test_block_cudagraphs_auto.py b/tests/test_block_cudagraphs_auto.py
index f481ee71189..bdc51418aaf 100644
--- a/tests/test_block_cudagraphs_auto.py
+++ b/tests/test_block_cudagraphs_auto.py
@@ -400,3 +400,18 @@ def test_probe_times_on_the_blocks_device(monkeypatch):
("record", "cuda:1"),
]
assert current[0] == "cuda:0"
+
+
+def test_auto_blocks_on_several_devices_keep_graphs_off(monkeypatch):
+ # device_map split: the first pass already visits both shards, before any timing or memory read.
+ layers, calls, compiles, timed = _harness(monkeypatch)
+ shard = {id(layers[0]): "cuda:0", id(layers[1]): "cuda:1"}
+ current = []
+ for layer in layers:
+ layer.register_forward_pre_hook(lambda m, a, layer = layer: current.append(shard[id(layer)]))
+ monkeypatch.setattr(bc, "_auto_block_device", lambda hs: current[-1], raising = False)
+ for _ in range(6):
+ assert _step(layers, calls) == ["plain", "plain"]
+ auto = bc._CUDAGRAPH_STATE["auto"]
+ assert auto["phase"] == "off" and auto["why"] == "blocks on several devices"
+ assert compiles == ["plain"] and not timed
diff --git a/unsloth/models/_block_compile.py b/unsloth/models/_block_compile.py
index f41dd1f900c..f696490ee85 100644
--- a/unsloth/models/_block_compile.py
+++ b/unsloth/models/_block_compile.py
@@ -65,12 +65,13 @@ def _new_cudagraph_state():
# Auto mode: phase "probe" (graphs off, watching), "on" or "off" (final). first: the layer whose
-# call starts a forward pass; passes: forward passes seen, all with batch shape `shape`.
+# call starts a forward pass; passes: forward passes seen, all with batch shape `shape` on `device`.
def _new_auto_state():
return {
"phase": "probe",
"first": None,
"shape": None,
+ "device": None,
"passes": 0,
"host": [],
"gpu": [],
@@ -529,6 +530,10 @@ def _auto_call_reason(layer, hidden_states, args, kwargs):
return None
+def _auto_block_device(hidden_states):
+ return hidden_states.device
+
+
def _auto_observe(layer, hidden_states, args, kwargs):
"""Auto mode, per block call while undecided: "graphs", "plain", or "measure" (plain, timed)."""
auto = _CUDAGRAPH_STATE["auto"]
@@ -546,9 +551,13 @@ def _auto_observe(layer, hidden_states, args, kwargs):
return "plain"
auto["first"] = weakref.ref(layer)
auto["shape"] = tuple(hidden_states.shape)
+ auto["device"] = _auto_block_device(hidden_states)
why = _auto_call_reason(layer, hidden_states, args, kwargs)
if why is None and tuple(hidden_states.shape) != auto["shape"]:
why = "batch shape changed"
+ if why is None and _auto_block_device(hidden_states) != auto["device"]:
+ # A device_map split: timing and free memory are only read on one device.
+ why = "blocks on several devices"
if why is not None:
_auto_off(auto, why)
return "plain"