Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Block compile: say when packed rows run each block uncompiled (#16272)

Daniel Han3d9b1cd07bd5
diff --git a/tests/test_block_compile.py b/tests/test_block_compile.py
index 64ba540209d..521ba55821a 100644
--- a/tests/test_block_compile.py
+++ b/tests/test_block_compile.py
@@ -584,6 +584,31 @@ def test_unsloth_host_metadata_calls_keep_the_stock_forward(monkeypatch, key):
assert calls # no host metadata: compiled
+def test_packed_stock_forward_says_blocks_run_uncompiled(monkeypatch, capsys):
+ M = _load()
+ monkeypatch.setattr(torch, "compile", lambda fn, **kwargs: fn)
+
+ class PackedDecoderLayer(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.proj = torch.nn.Linear(8, 8)
+
+ def forward(self, hidden_states, **kwargs):
+ return self.proj(hidden_states)
+
+ PackedDecoderLayer.forward.__module__ = "unsloth.models.llama"
+ M._wrap(PackedDecoderLayer, {})
+ layer = PackedDecoderLayer().train()
+ x = torch.randn(1, 4, 8)
+ layer(x)
+ layer(x, packed_seq_lengths = None)
+ assert "uncompiled" not in capsys.readouterr().out
+ layer(x, packed_seq_lengths = torch.tensor([2, 2]))
+ layer(x, packed_seq_lengths = torch.tensor([1, 3]))
+ out = capsys.readouterr().out
+ assert out.count("packed / padding-free rows run each transformer block uncompiled") == 1
+
+
def test_compiled_block_flags_packed_attention(monkeypatch):
"""hf_packed_attention runs packed rows outside the graph only while a block's compiled call
runs: set there, restored after, never on the stock forward."""
diff --git a/unsloth/models/_block_compile.py b/unsloth/models/_block_compile.py
index 2242911aa29..b46911975e2 100644
--- a/unsloth/models/_block_compile.py
+++ b/unsloth/models/_block_compile.py
@@ -94,6 +94,12 @@ _SHAPES_NOTE = (
)
+_PACKED_STOCK_NOTE = (
+ "Unsloth: packed / padding-free rows run each transformer block uncompiled; only unpacked "
+ "batches use the compiled blocks."
+)
+
+
def _rank0_note(key, message):
state = _CUDAGRAPH_STATE
if key in state["notes"]:
@@ -1577,6 +1583,13 @@ def _wrap(
_graphs_off("packed", _PACKED_NOTE)
elif phase == "probe" and self.training and _packed_kwargs(kwargs):
_auto_off(_CUDAGRAPH_STATE["auto"], "packed / padding-free rows")
+ if (
+ unsloth_forward
+ and self.training
+ and not tracing
+ and kwargs.get("packed_seq_lengths") is not None
+ ):
+ _rank0_note("packed_stock", _PACKED_STOCK_NOTE)
return original(self, hidden_states, *args, **kwargs)
if tracing:
# Inside a user's torch.compile the enclosing graph owns the layer: no dynamic marking