Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Block CUDA graphs: run MoE layers and flex attention blocks compiled without graphs (#15553)
Daniel Hanf442e809304e
diff --git a/tests/test_block_cudagraphs_clone.py b/tests/test_block_cudagraphs_clone.pyindex 7b5bfb41b96..b498a429e61 100644--- a/tests/test_block_cudagraphs_clone.py+++ b/tests/test_block_cudagraphs_clone.py@@ -277,3 +277,118 @@ def test_nonreentrant_checkpoint_runs_without_graphs(monkeypatch):assert len(seen) == 2 and set(seen) == wantassert x.grad is not Noneassert not bc._in_nonreentrant_checkpoint()+++class FakeMoeExperts(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.weight = torch.nn.Parameter(torch.randn(4, 8, 8) / 8)++ def forward(self, x):+ return torch.einsum("btd,ed->bte", x, self.weight.sum(0))+++class FakeSparseMoeBlock(torch.nn.Module):+ # transformers 4.57: ModuleList experts + num_experts.+ def __init__(self):+ super().__init__()+ self.num_experts = 2+ self.experts = torch.nn.ModuleList([torch.nn.Linear(8, 8) for _ in range(2)])++ def forward(self, x):+ return sum(e(x) for e in self.experts)+++class FakeRoutedMoe(torch.nn.Module):+ # transformers 4.57 DeepSeek-V2 / V3, GLM4-MoE: n_routed_experts, num_experts only on the gate.+ def __init__(self):+ super().__init__()+ self.n_routed_experts = 2+ self.gate = torch.nn.Linear(8, 2)+ self.gate.num_experts = 2+ self.experts = torch.nn.ModuleList([torch.nn.Linear(8, 8) for _ in range(2)])+ self.shared_experts = torch.nn.Linear(8, 8)++ def forward(self, x):+ return sum(e(x) for e in self.experts) + self.shared_experts(x)++[email protected]("moe", [None, FakeMoeExperts, FakeSparseMoeBlock, FakeRoutedMoe])+def test_moe_layers_run_without_graphs(monkeypatch, moe):+ """Qwen3-MoE broke under block CUDA graphs (capture host copy / replay-overwritten routing)."""+ seen = []++ def fake_compile(fn, **kwargs):+ tag = "graphs" if kwargs.get("options", {}).get("triton.cudagraphs") else "plain"++ def inner(*args, **kw):+ seen.append(tag)+ return fn(*args, **kw)++ return inner++ monkeypatch.setattr(torch, "compile", fake_compile)++ class MoeDecoderLayer(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.proj = torch.nn.Linear(8, 8)+ self.mlp = moe() if moe is not None else torch.nn.Linear(8, 8)++ def forward(self, hidden_states, **kwargs):+ return self.mlp(self.proj(hidden_states))++ bc._wrap(MoeDecoderLayer, {**bc._COMPILE_OPTIONS, "triton.cudagraphs": True})+ layer = MoeDecoderLayer().train()+ for _ in range(2):+ x = torch.randn(2, 16, 8, requires_grad = True)+ layer(x).sum().backward()+ assert x.grad is not None+ assert set(seen) == ({"graphs"} if moe is None else {"plain"})+ assert not layer.__dict__.get("_unsloth_block_grad_hooks") or moe is None+++class BlockMask:+ """Stand-in for torch.nn.attention.flex_attention.BlockMask (matched by class name)."""++ def __init__(self):+ self.kv_num_blocks = torch.ones(1, 1, 1, dtype = torch.int32)++[email protected]("mask", [None, "tensor", "block_mask"])+def test_flex_block_mask_calls_run_without_graphs(monkeypatch, mask):+ """Flex BlockMask calls re-record block graphs every step."""+ seen = []++ def fake_compile(fn, **kwargs):+ tag = "graphs" if kwargs.get("options", {}).get("triton.cudagraphs") else "plain"++ def inner(*args, **kw):+ seen.append(tag)+ return fn(*args, **kw)++ return inner++ monkeypatch.setattr(torch, "compile", fake_compile)++ class FlexDecoderLayer(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.proj = torch.nn.Linear(8, 8)++ def forward(+ self,+ hidden_states,+ attention_mask = None,+ **kwargs,+ ):+ return self.proj(hidden_states)++ bc._wrap(FlexDecoderLayer, {**bc._COMPILE_OPTIONS, "triton.cudagraphs": True})+ layer = FlexDecoderLayer().train()+ value = {None: None, "tensor": torch.ones(2, 1, 16, 16), "block_mask": BlockMask()}[mask]+ for _ in range(2):+ x = torch.randn(2, 16, 8, requires_grad = True)+ layer(x, attention_mask = value).sum().backward()+ assert x.grad is not None+ assert set(seen) == ({"plain"} if mask == "block_mask" else {"graphs"})diff --git a/unsloth/models/_block_compile.py b/unsloth/models/_block_compile.pyindex e76e202171d..5bdedf725b9 100644--- a/unsloth/models/_block_compile.py+++ b/unsloth/models/_block_compile.py@@ -82,6 +82,45 @@ def _in_nonreentrant_checkpoint():return bool(hooks) and getattr(hooks[0], "__module__", None) == "torch.utils.checkpoint"+def _graphs_unsafe_moe(layer):+ """MoE routing / grouped GEMMs read token counts the capture cannot hold (Qwen3-MoE: host copy+ during capture, routing outputs overwritten by the next replay)."""+ moe = layer.__dict__.get("_unsloth_block_graphs_moe")+ if moe is None:+ # "experts" child: transformers 4.57 DeepSeek-V2 / V3, GLM4-MoE have no num_experts.+ moe = any(+ _experts_of(m) is not None+ or m._modules.get("experts") is not None+ or (hasattr(m, "num_experts") and hasattr(m, "experts"))+ for m in layer.modules()+ )+ layer.__dict__["_unsloth_block_graphs_moe"] = moe+ if moe and not _GRAPHS_MOE_LOGGED[0]:+ _GRAPHS_MOE_LOGGED[0] = True+ print("Unsloth: block CUDA graphs skip MoE layers; they run compiled without graphs.")+ return moe+++_GRAPHS_MOE_LOGGED = [False]+++def _flex_block_mask_call(args, kwargs):+ """BlockMask index tensors are static graph inputs rebuilt every step, so the graph re-records+ each call (StaticInputIdxMismatch is outside the re-record limit); Gemma3 was slower with graphs."""+ for value in (*args, *kwargs.values()):+ if type(value).__name__ == "BlockMask":+ if not _GRAPHS_FLEX_LOGGED[0]:+ _GRAPHS_FLEX_LOGGED[0] = True+ print(+ "Unsloth: block CUDA graphs skip flex attention blocks; they run compiled without graphs."+ )+ return True+ return False+++_GRAPHS_FLEX_LOGGED = [False]++def _ddp_optimizer_mode():# optimize_ddp also takes "python_reducer" / "no_optimization" strings: only "ddp_optimizer"# (or True) makes dynamo split graphs (convert_frame.py checks == "ddp_optimizer").@@ -892,7 +931,12 @@ def _wrap(if shape not in graph_shapes:graph_shapes.add(shape)graphs_off[0] = len(graph_shapes) > _CUDAGRAPH_MAX_SHAPES- if graphs and (graphs_off[0] or _in_nonreentrant_checkpoint()):+ if graphs and (+ graphs_off[0]+ or _in_nonreentrant_checkpoint()+ or _graphs_unsafe_moe(self)+ or _flex_block_mask_call(args, kwargs)+ ):return plain(self, hidden_states, *args, **kwargs)if graphs:_param_grads_out_of_pool(self)