Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Block CUDA graphs: run MoE layers and flex attention blocks compiled without graphs (#15553)

Daniel Hanf442e809304e
diff --git a/tests/test_block_cudagraphs_clone.py b/tests/test_block_cudagraphs_clone.py
index 7b5bfb41b96..b498a429e61 100644
--- a/tests/test_block_cudagraphs_clone.py
+++ b/tests/test_block_cudagraphs_clone.py
@@ -277,3 +277,118 @@ def test_nonreentrant_checkpoint_runs_without_graphs(monkeypatch):
assert len(seen) == 2 and set(seen) == want
assert x.grad is not None
assert not bc._in_nonreentrant_checkpoint()
+
+
+class FakeMoeExperts(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.weight = torch.nn.Parameter(torch.randn(4, 8, 8) / 8)
+
+ def forward(self, x):
+ return torch.einsum("btd,ed->bte", x, self.weight.sum(0))
+
+
+class FakeSparseMoeBlock(torch.nn.Module):
+ # transformers 4.57: ModuleList experts + num_experts.
+ def __init__(self):
+ super().__init__()
+ self.num_experts = 2
+ self.experts = torch.nn.ModuleList([torch.nn.Linear(8, 8) for _ in range(2)])
+
+ def forward(self, x):
+ return sum(e(x) for e in self.experts)
+
+
+class FakeRoutedMoe(torch.nn.Module):
+ # transformers 4.57 DeepSeek-V2 / V3, GLM4-MoE: n_routed_experts, num_experts only on the gate.
+ def __init__(self):
+ super().__init__()
+ self.n_routed_experts = 2
+ self.gate = torch.nn.Linear(8, 2)
+ self.gate.num_experts = 2
+ self.experts = torch.nn.ModuleList([torch.nn.Linear(8, 8) for _ in range(2)])
+ self.shared_experts = torch.nn.Linear(8, 8)
+
+ def forward(self, x):
+ return sum(e(x) for e in self.experts) + self.shared_experts(x)
+
+
[email protected]("moe", [None, FakeMoeExperts, FakeSparseMoeBlock, FakeRoutedMoe])
+def test_moe_layers_run_without_graphs(monkeypatch, moe):
+ """Qwen3-MoE broke under block CUDA graphs (capture host copy / replay-overwritten routing)."""
+ seen = []
+
+ def fake_compile(fn, **kwargs):
+ tag = "graphs" if kwargs.get("options", {}).get("triton.cudagraphs") else "plain"
+
+ def inner(*args, **kw):
+ seen.append(tag)
+ return fn(*args, **kw)
+
+ return inner
+
+ monkeypatch.setattr(torch, "compile", fake_compile)
+
+ class MoeDecoderLayer(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.proj = torch.nn.Linear(8, 8)
+ self.mlp = moe() if moe is not None else torch.nn.Linear(8, 8)
+
+ def forward(self, hidden_states, **kwargs):
+ return self.mlp(self.proj(hidden_states))
+
+ bc._wrap(MoeDecoderLayer, {**bc._COMPILE_OPTIONS, "triton.cudagraphs": True})
+ layer = MoeDecoderLayer().train()
+ for _ in range(2):
+ x = torch.randn(2, 16, 8, requires_grad = True)
+ layer(x).sum().backward()
+ assert x.grad is not None
+ assert set(seen) == ({"graphs"} if moe is None else {"plain"})
+ assert not layer.__dict__.get("_unsloth_block_grad_hooks") or moe is None
+
+
+class BlockMask:
+ """Stand-in for torch.nn.attention.flex_attention.BlockMask (matched by class name)."""
+
+ def __init__(self):
+ self.kv_num_blocks = torch.ones(1, 1, 1, dtype = torch.int32)
+
+
[email protected]("mask", [None, "tensor", "block_mask"])
+def test_flex_block_mask_calls_run_without_graphs(monkeypatch, mask):
+ """Flex BlockMask calls re-record block graphs every step."""
+ seen = []
+
+ def fake_compile(fn, **kwargs):
+ tag = "graphs" if kwargs.get("options", {}).get("triton.cudagraphs") else "plain"
+
+ def inner(*args, **kw):
+ seen.append(tag)
+ return fn(*args, **kw)
+
+ return inner
+
+ monkeypatch.setattr(torch, "compile", fake_compile)
+
+ class FlexDecoderLayer(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.proj = torch.nn.Linear(8, 8)
+
+ def forward(
+ self,
+ hidden_states,
+ attention_mask = None,
+ **kwargs,
+ ):
+ return self.proj(hidden_states)
+
+ bc._wrap(FlexDecoderLayer, {**bc._COMPILE_OPTIONS, "triton.cudagraphs": True})
+ layer = FlexDecoderLayer().train()
+ value = {None: None, "tensor": torch.ones(2, 1, 16, 16), "block_mask": BlockMask()}[mask]
+ for _ in range(2):
+ x = torch.randn(2, 16, 8, requires_grad = True)
+ layer(x, attention_mask = value).sum().backward()
+ assert x.grad is not None
+ assert set(seen) == ({"plain"} if mask == "block_mask" else {"graphs"})
diff --git a/unsloth/models/_block_compile.py b/unsloth/models/_block_compile.py
index e76e202171d..5bdedf725b9 100644
--- a/unsloth/models/_block_compile.py
+++ b/unsloth/models/_block_compile.py
@@ -82,6 +82,45 @@ def _in_nonreentrant_checkpoint():
return bool(hooks) and getattr(hooks[0], "__module__", None) == "torch.utils.checkpoint"
+def _graphs_unsafe_moe(layer):
+ """MoE routing / grouped GEMMs read token counts the capture cannot hold (Qwen3-MoE: host copy
+ during capture, routing outputs overwritten by the next replay)."""
+ moe = layer.__dict__.get("_unsloth_block_graphs_moe")
+ if moe is None:
+ # "experts" child: transformers 4.57 DeepSeek-V2 / V3, GLM4-MoE have no num_experts.
+ moe = any(
+ _experts_of(m) is not None
+ or m._modules.get("experts") is not None
+ or (hasattr(m, "num_experts") and hasattr(m, "experts"))
+ for m in layer.modules()
+ )
+ layer.__dict__["_unsloth_block_graphs_moe"] = moe
+ if moe and not _GRAPHS_MOE_LOGGED[0]:
+ _GRAPHS_MOE_LOGGED[0] = True
+ print("Unsloth: block CUDA graphs skip MoE layers; they run compiled without graphs.")
+ return moe
+
+
+_GRAPHS_MOE_LOGGED = [False]
+
+
+def _flex_block_mask_call(args, kwargs):
+ """BlockMask index tensors are static graph inputs rebuilt every step, so the graph re-records
+ each call (StaticInputIdxMismatch is outside the re-record limit); Gemma3 was slower with graphs."""
+ for value in (*args, *kwargs.values()):
+ if type(value).__name__ == "BlockMask":
+ if not _GRAPHS_FLEX_LOGGED[0]:
+ _GRAPHS_FLEX_LOGGED[0] = True
+ print(
+ "Unsloth: block CUDA graphs skip flex attention blocks; they run compiled without graphs."
+ )
+ return True
+ return False
+
+
+_GRAPHS_FLEX_LOGGED = [False]
+
+
def _ddp_optimizer_mode():
# optimize_ddp also takes "python_reducer" / "no_optimization" strings: only "ddp_optimizer"
# (or True) makes dynamo split graphs (convert_frame.py checks == "ddp_optimizer").
@@ -892,7 +931,12 @@ def _wrap(
if shape not in graph_shapes:
graph_shapes.add(shape)
graphs_off[0] = len(graph_shapes) > _CUDAGRAPH_MAX_SHAPES
- if graphs and (graphs_off[0] or _in_nonreentrant_checkpoint()):
+ if graphs and (
+ graphs_off[0]
+ or _in_nonreentrant_checkpoint()
+ or _graphs_unsafe_moe(self)
+ or _flex_block_mask_call(args, kwargs)
+ ):
return plain(self, hidden_states, *args, **kwargs)
if graphs:
_param_grads_out_of_pool(self)