Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Block CUDA graphs: auto mode stays off when decoder blocks span several devices (#15931)
Daniel Hane039edb2b647
diff --git a/tests/test_block_cudagraphs_auto.py b/tests/test_block_cudagraphs_auto.pyindex f481ee71189..bdc51418aaf 100644--- a/tests/test_block_cudagraphs_auto.py+++ b/tests/test_block_cudagraphs_auto.py@@ -400,3 +400,18 @@ def test_probe_times_on_the_blocks_device(monkeypatch):("record", "cuda:1"),]assert current[0] == "cuda:0"+++def test_auto_blocks_on_several_devices_keep_graphs_off(monkeypatch):+ # device_map split: the first pass already visits both shards, before any timing or memory read.+ layers, calls, compiles, timed = _harness(monkeypatch)+ shard = {id(layers[0]): "cuda:0", id(layers[1]): "cuda:1"}+ current = []+ for layer in layers:+ layer.register_forward_pre_hook(lambda m, a, layer = layer: current.append(shard[id(layer)]))+ monkeypatch.setattr(bc, "_auto_block_device", lambda hs: current[-1], raising = False)+ for _ in range(6):+ assert _step(layers, calls) == ["plain", "plain"]+ auto = bc._CUDAGRAPH_STATE["auto"]+ assert auto["phase"] == "off" and auto["why"] == "blocks on several devices"+ assert compiles == ["plain"] and not timeddiff --git a/unsloth/models/_block_compile.py b/unsloth/models/_block_compile.pyindex f41dd1f900c..f696490ee85 100644--- a/unsloth/models/_block_compile.py+++ b/unsloth/models/_block_compile.py@@ -65,12 +65,13 @@ def _new_cudagraph_state():# Auto mode: phase "probe" (graphs off, watching), "on" or "off" (final). first: the layer whose-# call starts a forward pass; passes: forward passes seen, all with batch shape `shape`.+# call starts a forward pass; passes: forward passes seen, all with batch shape `shape` on `device`.def _new_auto_state():return {"phase": "probe","first": None,"shape": None,+ "device": None,"passes": 0,"host": [],"gpu": [],@@ -529,6 +530,10 @@ def _auto_call_reason(layer, hidden_states, args, kwargs):return None+def _auto_block_device(hidden_states):+ return hidden_states.device++def _auto_observe(layer, hidden_states, args, kwargs):"""Auto mode, per block call while undecided: "graphs", "plain", or "measure" (plain, timed)."""auto = _CUDAGRAPH_STATE["auto"]@@ -546,9 +551,13 @@ def _auto_observe(layer, hidden_states, args, kwargs):return "plain"auto["first"] = weakref.ref(layer)auto["shape"] = tuple(hidden_states.shape)+ auto["device"] = _auto_block_device(hidden_states)why = _auto_call_reason(layer, hidden_states, args, kwargs)if why is None and tuple(hidden_states.shape) != auto["shape"]:why = "batch shape changed"+ if why is None and _auto_block_device(hidden_states) != auto["device"]:+ # A device_map split: timing and free memory are only read on one device.+ why = "blocks on several devices"if why is not None:_auto_off(auto, why)return "plain"