Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Block compile: mark the batch dim dynamic so an uneven last batch does not recompile every layer (#15106)
Daniel Han7a0d86c0fcc7
diff --git a/tests/test_block_compile.py b/tests/test_block_compile.pyindex 9110d7ee8e0..f522b12ce49 100644--- a/tests/test_block_compile.py+++ b/tests/test_block_compile.py@@ -53,6 +53,74 @@ def test_mark_seq_dynamic_skips_widths(monkeypatch):assert sorted(marked) == [(2, 1), (3, 1), (3, 2), (4, 2), (4, 3)]+def test_mark_batch_dynamic_marks_dim0_only(monkeypatch):+ M = _load()+ marked = []+ monkeypatch.setattr(+ torch._dynamo, "maybe_mark_dynamic", lambda t, d: marked.append((t.dim(), d))+ )+ B, L = 4, 16+ hidden = torch.zeros(B, L, 8)+ mask4d = torch.zeros(B, 1, L, L)+ pos = torch.zeros(1, L, dtype = torch.long) # broadcast over the batch: not marked+ scale = torch.zeros(B) # 1D: not marked+ M._mark_batch_dynamic(B, (hidden, {"m": mask4d}, [pos, scale, "x", None]))+ assert sorted(marked) == [(3, 0), (4, 0)]++[email protected]("dynamic_batch", ["1", "0"])+def test_batch_change_does_not_recompile(monkeypatch, dynamic_batch):+ """An uneven last batch recompiled every compiled block (+8 graphs, 40 s on a 4-bit Qwen3-0.6B, B200)."""+ monkeypatch.setenv("UNSLOTH_BLOCK_COMPILE_DYNAMIC_BATCH", dynamic_batch)+ M = _load()+ from torch._dynamo.testing import CompileCounter++ counter = CompileCounter()+ real_compile = torch.compile+ monkeypatch.setattr(+ torch,+ "compile",+ lambda fn, **kwargs: real_compile(fn, backend = counter, fullgraph = False, dynamic = None),+ )+ torch._dynamo.reset()++ class BatchDecoderLayer(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.proj = torch.nn.Linear(8, 8)++ def forward(+ self,+ hidden_states,+ attention_mask = None,+ position_ids = None,+ **kwargs,+ ):+ out = self.proj(hidden_states) + position_ids[..., None].to(hidden_states.dtype)+ return out * attention_mask[:, 0, :, :1]++ M._wrap(BatchDecoderLayer, {})+ layer = BatchDecoderLayer().train()++ def step(B, L):+ x = torch.randn(B, L, 8, requires_grad = True)+ mask = torch.ones(B, 1, L, L)+ pos = torch.arange(L)[None].expand(B, -1)+ layer(x, attention_mask = mask, position_ids = pos).sum().backward()++ step(4, 16)+ step(4, 24) # sequence dim is dynamic: no recompile+ before = counter.frame_count+ step(3, 20) # uneven last batch+ step(6, 12)+ extra = counter.frame_count - before+ torch._dynamo.reset()+ if dynamic_batch == "1":+ assert extra == 0+ else:+ assert extra >= 1 # the control: without the batch mark the batch change recompiles++def test_rebind_globals_resolves():M = _load()ns = {"__name__": "not_a_registered_module", "OFFSET": 3}diff --git a/unsloth/models/_block_compile.py b/unsloth/models/_block_compile.pyindex 84bcb34720f..f3cecd8df4e 100644--- a/unsloth/models/_block_compile.py+++ b/unsloth/models/_block_compile.py@@ -143,6 +143,22 @@ def _mark_seq_dynamic(seq_len, obj):_mark_seq_dynamic(seq_len, o)+def _mark_batch_dynamic(batch, obj):+ # Batch dim too: an uneven last training batch (size >= 2) otherwise recompiles every graph.+ if isinstance(obj, torch.Tensor):+ if obj.dim() >= 2 and obj.shape[0] == batch:+ torch._dynamo.maybe_mark_dynamic(obj, 0)+ elif isinstance(obj, (tuple, list)):+ for o in obj:+ _mark_batch_dynamic(batch, o)+ elif isinstance(obj, dict):+ for o in obj.values():+ _mark_batch_dynamic(batch, o)+++_DYNAMIC_BATCH = os.environ.get("UNSLOTH_BLOCK_COMPILE_DYNAMIC_BATCH", "1") != "0"++# fla's opaque op state: None until a GatedDeltaNet layer ran its self-check, then True / False._GDN_OP = [None]# Packed batches (cu_seqlens) through the op: set eagerly with _GDN_OP from the op's self-check.@@ -759,6 +775,8 @@ def _wrap(cls, compile_options):_prepare_gdn(self, hidden_states)if hidden_states.dim() == 3 and hidden_states.shape[1] > 1:_mark_seq_dynamic(hidden_states.shape[1], (hidden_states, args, kwargs))+ if _DYNAMIC_BATCH and hidden_states.shape[0] > 1:+ _mark_batch_dynamic(hidden_states.shape[0], (hidden_states, args, kwargs))_precheck_band(self, hidden_states.shape[1], args, kwargs)bit = 2 if torch.is_grad_enabled() else 1checked = self.__dict__.get("_unsloth_block_checked", 0)