Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Block compile: router_logits capture no longer recompiles every layer (#14998)
Daniel Han012192af3374
diff --git a/tests/test_block_compile.py b/tests/test_block_compile.pyindex 9c4e41801a6..ec1361a8466 100644--- a/tests/test_block_compile.py+++ b/tests/test_block_compile.py@@ -1395,3 +1395,71 @@ def test_zero3_partitioned_training_keeps_the_stock_forward(monkeypatch):assert calls == []model.layers[0](x).sum().backward()assert calls+++def test_router_logits_capture_compiles_one_graph_for_every_layer(monkeypatch):+ """A capture hook inside the block appends to an empty scratch list on every call.++ Before: the hook appended to the collector's list and Dynamo guarded on its length (the layer+ index), so each of 10 layers compiled its own graph. Now <= 2 graphs (static, then dynamic) across+ 10 layers at 10 sequence lengths, and the collected logits match the stock forward in order.+ """+ output_capturing = pytest.importorskip("transformers.utils.output_capturing")+ misc = pytest.importorskip("unsloth_zoo.temporary_patches.misc")+ if not hasattr(misc, "output_capture_scratch"):+ pytest.skip("needs an unsloth_zoo with output_capture_scratch (unsloth-zoo#2041)")+ misc.patch_output_collector_for_compiled_submodules()+ from torch._dynamo.testing import CompileCounter++ M = _load()+ cc = CompileCounter()+ real_compile = torch.compile+ monkeypatch.setattr(+ torch,+ "compile",+ lambda fn, **kw: real_compile(fn, backend = cc, fullgraph = False, dynamic = None),+ )+ var = output_capturing._active_collector++ class RouterDecoderLayer(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.proj = torch.nn.Linear(8, 8)+ self.router = torch.nn.Linear(8, 4)++ def hook(+ module, args, output+ ): # transformers' capture hook for a non hidden_states key+ collected = var.get()+ if collected is None or "router_logits" not in collected.keys():+ return+ collected["router_logits"].append(output)++ self.router.register_forward_hook(hook)++ def forward(+ self,+ hidden_states,+ attention_mask = None,+ ):+ return self.proj(hidden_states) + self.router(hidden_states).sum(-1, keepdim = True)++ M._wrap(RouterDecoderLayer, {})+ lengths = [5, 9, 12, 17, 23, 31, 40, 47, 58, 64]+ layers = [RouterDecoderLayer() for _ in lengths]+ xs = [torch.randn(2, n, 8) for n in lengths]+ with torch.no_grad():+ want = [layer.router(x) for layer, x in zip(layers, xs)]+ torch._dynamo.reset()+ collected = {"router_logits": []}+ token = var.set(collected)+ try:+ for layer, x in zip(layers, xs):+ layer(x.requires_grad_())+ finally:+ var.reset(token)+ assert cc.frame_count <= 2, f"{cc.frame_count} graphs for {len(lengths)} layers"+ got = collected["router_logits"]+ assert len(got) == len(want)+ for g, w in zip(got, want):+ torch.testing.assert_close(g.detach(), w)diff --git a/unsloth/models/_block_compile.py b/unsloth/models/_block_compile.pyindex 4fae6116fbe..8047ab0229e 100644--- a/unsloth/models/_block_compile.py+++ b/unsloth/models/_block_compile.py@@ -4,6 +4,7 @@"""One torch.compile region per decoder layer for training (stock forward: attention, MLP or MoE,norms, LoRA, NF4). UNSLOTH_BLOCK_COMPILE=0 or UNSLOTH_COMPILE_DISABLE turns it off."""+import contextlibimport functoolsimport osimport sys@@ -693,6 +694,10 @@ def _wrap(cls, compile_options):from unsloth_zoo.temporary_patches.utils import UNSLOTH_DECODE_COMPILE as inline_flagexcept Exception:inline_flag = [False]+ try:+ from unsloth_zoo.temporary_patches.misc import output_capture_scratch as capture_scratch+ except Exception:+ capture_scratch = contextlib.nullcontext# First call per grad mode compiles with errors raised: on failure retry with flex outside,# then the stock forward. The mode is per layer (a checkpoint recompute must not switch graph);# a failure is also the starting mode for later layers of the same kind.@@ -703,7 +708,9 @@ def _wrap(cls, compile_options):inline_flag[0] = True_IN_BLOCK[0] = modetry:- return compiled(self, hidden_states, *args, **kwargs)+ # Capture hooks (router_logits) append to empty scratch lists: no guard on the layer index.+ with capture_scratch():+ return compiled(self, hidden_states, *args, **kwargs)finally:inline_flag[0] = previous_IN_BLOCK[0] = _OFF