Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Block compile: mark the batch dim dynamic so an uneven last batch does not recompile every layer (#15106)

Daniel Han7a0d86c0fcc7
diff --git a/tests/test_block_compile.py b/tests/test_block_compile.py
index 9110d7ee8e0..f522b12ce49 100644
--- a/tests/test_block_compile.py
+++ b/tests/test_block_compile.py
@@ -53,6 +53,74 @@ def test_mark_seq_dynamic_skips_widths(monkeypatch):
assert sorted(marked) == [(2, 1), (3, 1), (3, 2), (4, 2), (4, 3)]
+def test_mark_batch_dynamic_marks_dim0_only(monkeypatch):
+ M = _load()
+ marked = []
+ monkeypatch.setattr(
+ torch._dynamo, "maybe_mark_dynamic", lambda t, d: marked.append((t.dim(), d))
+ )
+ B, L = 4, 16
+ hidden = torch.zeros(B, L, 8)
+ mask4d = torch.zeros(B, 1, L, L)
+ pos = torch.zeros(1, L, dtype = torch.long) # broadcast over the batch: not marked
+ scale = torch.zeros(B) # 1D: not marked
+ M._mark_batch_dynamic(B, (hidden, {"m": mask4d}, [pos, scale, "x", None]))
+ assert sorted(marked) == [(3, 0), (4, 0)]
+
+
[email protected]("dynamic_batch", ["1", "0"])
+def test_batch_change_does_not_recompile(monkeypatch, dynamic_batch):
+ """An uneven last batch recompiled every compiled block (+8 graphs, 40 s on a 4-bit Qwen3-0.6B, B200)."""
+ monkeypatch.setenv("UNSLOTH_BLOCK_COMPILE_DYNAMIC_BATCH", dynamic_batch)
+ M = _load()
+ from torch._dynamo.testing import CompileCounter
+
+ counter = CompileCounter()
+ real_compile = torch.compile
+ monkeypatch.setattr(
+ torch,
+ "compile",
+ lambda fn, **kwargs: real_compile(fn, backend = counter, fullgraph = False, dynamic = None),
+ )
+ torch._dynamo.reset()
+
+ class BatchDecoderLayer(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.proj = torch.nn.Linear(8, 8)
+
+ def forward(
+ self,
+ hidden_states,
+ attention_mask = None,
+ position_ids = None,
+ **kwargs,
+ ):
+ out = self.proj(hidden_states) + position_ids[..., None].to(hidden_states.dtype)
+ return out * attention_mask[:, 0, :, :1]
+
+ M._wrap(BatchDecoderLayer, {})
+ layer = BatchDecoderLayer().train()
+
+ def step(B, L):
+ x = torch.randn(B, L, 8, requires_grad = True)
+ mask = torch.ones(B, 1, L, L)
+ pos = torch.arange(L)[None].expand(B, -1)
+ layer(x, attention_mask = mask, position_ids = pos).sum().backward()
+
+ step(4, 16)
+ step(4, 24) # sequence dim is dynamic: no recompile
+ before = counter.frame_count
+ step(3, 20) # uneven last batch
+ step(6, 12)
+ extra = counter.frame_count - before
+ torch._dynamo.reset()
+ if dynamic_batch == "1":
+ assert extra == 0
+ else:
+ assert extra >= 1 # the control: without the batch mark the batch change recompiles
+
+
def test_rebind_globals_resolves():
M = _load()
ns = {"__name__": "not_a_registered_module", "OFFSET": 3}
diff --git a/unsloth/models/_block_compile.py b/unsloth/models/_block_compile.py
index 84bcb34720f..f3cecd8df4e 100644
--- a/unsloth/models/_block_compile.py
+++ b/unsloth/models/_block_compile.py
@@ -143,6 +143,22 @@ def _mark_seq_dynamic(seq_len, obj):
_mark_seq_dynamic(seq_len, o)
+def _mark_batch_dynamic(batch, obj):
+ # Batch dim too: an uneven last training batch (size >= 2) otherwise recompiles every graph.
+ if isinstance(obj, torch.Tensor):
+ if obj.dim() >= 2 and obj.shape[0] == batch:
+ torch._dynamo.maybe_mark_dynamic(obj, 0)
+ elif isinstance(obj, (tuple, list)):
+ for o in obj:
+ _mark_batch_dynamic(batch, o)
+ elif isinstance(obj, dict):
+ for o in obj.values():
+ _mark_batch_dynamic(batch, o)
+
+
+_DYNAMIC_BATCH = os.environ.get("UNSLOTH_BLOCK_COMPILE_DYNAMIC_BATCH", "1") != "0"
+
+
# fla's opaque op state: None until a GatedDeltaNet layer ran its self-check, then True / False.
_GDN_OP = [None]
# Packed batches (cu_seqlens) through the op: set eagerly with _GDN_OP from the op's self-check.
@@ -759,6 +775,8 @@ def _wrap(cls, compile_options):
_prepare_gdn(self, hidden_states)
if hidden_states.dim() == 3 and hidden_states.shape[1] > 1:
_mark_seq_dynamic(hidden_states.shape[1], (hidden_states, args, kwargs))
+ if _DYNAMIC_BATCH and hidden_states.shape[0] > 1:
+ _mark_batch_dynamic(hidden_states.shape[0], (hidden_states, args, kwargs))
_precheck_band(self, hidden_states.shape[1], args, kwargs)
bit = 2 if torch.is_grad_enabled() else 1
checked = self.__dict__.get("_unsloth_block_checked", 0)