Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Block compile: say when packed rows run each block uncompiled (#16272)
Daniel Han3d9b1cd07bd5
diff --git a/tests/test_block_compile.py b/tests/test_block_compile.pyindex 64ba540209d..521ba55821a 100644--- a/tests/test_block_compile.py+++ b/tests/test_block_compile.py@@ -584,6 +584,31 @@ def test_unsloth_host_metadata_calls_keep_the_stock_forward(monkeypatch, key):assert calls # no host metadata: compiled+def test_packed_stock_forward_says_blocks_run_uncompiled(monkeypatch, capsys):+ M = _load()+ monkeypatch.setattr(torch, "compile", lambda fn, **kwargs: fn)++ class PackedDecoderLayer(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.proj = torch.nn.Linear(8, 8)++ def forward(self, hidden_states, **kwargs):+ return self.proj(hidden_states)++ PackedDecoderLayer.forward.__module__ = "unsloth.models.llama"+ M._wrap(PackedDecoderLayer, {})+ layer = PackedDecoderLayer().train()+ x = torch.randn(1, 4, 8)+ layer(x)+ layer(x, packed_seq_lengths = None)+ assert "uncompiled" not in capsys.readouterr().out+ layer(x, packed_seq_lengths = torch.tensor([2, 2]))+ layer(x, packed_seq_lengths = torch.tensor([1, 3]))+ out = capsys.readouterr().out+ assert out.count("packed / padding-free rows run each transformer block uncompiled") == 1++def test_compiled_block_flags_packed_attention(monkeypatch):"""hf_packed_attention runs packed rows outside the graph only while a block's compiled callruns: set there, restored after, never on the stock forward."""diff --git a/unsloth/models/_block_compile.py b/unsloth/models/_block_compile.pyindex 2242911aa29..b46911975e2 100644--- a/unsloth/models/_block_compile.py+++ b/unsloth/models/_block_compile.py@@ -94,6 +94,12 @@ _SHAPES_NOTE = ()+_PACKED_STOCK_NOTE = (+ "Unsloth: packed / padding-free rows run each transformer block uncompiled; only unpacked "+ "batches use the compiled blocks."+)++def _rank0_note(key, message):state = _CUDAGRAPH_STATEif key in state["notes"]:@@ -1577,6 +1583,13 @@ def _wrap(_graphs_off("packed", _PACKED_NOTE)elif phase == "probe" and self.training and _packed_kwargs(kwargs):_auto_off(_CUDAGRAPH_STATE["auto"], "packed / padding-free rows")+ if (+ unsloth_forward+ and self.training+ and not tracing+ and kwargs.get("packed_seq_lengths") is not None+ ):+ _rank0_note("packed_stock", _PACKED_STOCK_NOTE)return original(self, hidden_states, *args, **kwargs)if tracing:# Inside a user's torch.compile the enclosing graph owns the layer: no dynamic marking