Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Compiled blocks: packed rows run varlen attention outside the graph (#14808)

Daniel Hane6f72a0198e1
diff --git a/tests/test_block_compile.py b/tests/test_block_compile.py
index 6526ab63d59..ce2a27c54b7 100644
--- a/tests/test_block_compile.py
+++ b/tests/test_block_compile.py
@@ -544,6 +544,42 @@ def test_unsloth_host_metadata_calls_keep_the_stock_forward(monkeypatch, key):
assert calls # no host metadata: compiled
+def test_compiled_block_flags_packed_attention(monkeypatch):
+ """hf_packed_attention runs packed rows outside the graph only while a block's compiled call
+ runs: set there, restored after, never on the stock forward."""
+ M = _load()
+ packed_mod = types.ModuleType(M._PACKED_MODULE)
+ packed_mod.IN_COMPILED_BLOCK = [False]
+ monkeypatch.setitem(sys.modules, M._PACKED_MODULE, packed_mod)
+ seen = []
+
+ def fake_compile(fn, **kwargs):
+ def compiled(*args, **kw):
+ seen.append(packed_mod.IN_COMPILED_BLOCK[0])
+ return fn(*args, **kw)
+
+ return compiled
+
+ monkeypatch.setattr(torch, "compile", fake_compile)
+ cls = _toy_class()
+ M._wrap(cls, {})
+ layer = cls()
+ x = torch.randn(1, 4, 8)
+ with torch.no_grad():
+ layer(x)
+ assert seen and all(seen)
+ assert packed_mod.IN_COMPILED_BLOCK == [False]
+ seen.clear()
+ layer.eval()(x) # stock forward
+ assert seen == [] and packed_mod.IN_COMPILED_BLOCK == [False]
+ # Without the module (never imported) the block still runs.
+ monkeypatch.delitem(sys.modules, M._PACKED_MODULE)
+ layer.train()
+ with torch.no_grad():
+ layer(x)
+ assert M._NO_PACKED == [False]
+
+
def test_gemma4_band_fast_path_in_blocks(monkeypatch):
M = _load()
from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS
diff --git a/tests/test_hf_packed_attention.py b/tests/test_hf_packed_attention.py
index 2ddb796fe47..cce2c231577 100644
--- a/tests/test_hf_packed_attention.py
+++ b/tests/test_hf_packed_attention.py
@@ -403,6 +403,141 @@ def test_rocm_keeps_windowed_rows_off_flash_attn(monkeypatch, window):
assert seen and not dispatched
+def _block_flag(
+ monkeypatch,
+ value,
+ leaves = None,
+):
+ monkeypatch.setattr(hpa, "IN_COMPILED_BLOCK", [value])
+ if leaves is not None:
+ monkeypatch.setattr(hpa, "_leaves_block", lambda *args: leaves)
+
+
[email protected]("in_block,leaves", [(True, True), (True, None), (False, True)])
+def test_packed_rows_leave_a_compiled_block_for_the_eager_wrapper(monkeypatch, in_block, leaves):
+ """Inside an Unsloth compiled block a packed call the predicate sends out runs the wrapper
+ eagerly (one graph break). A CPU row (real predicate) and a user's own compile (flag off) keep
+ the wrapped sdpa in the graph."""
+ hpa._ORIG_SDPA[0] = _plain_sdpa
+ monkeypatch.delenv("UNSLOTH_HF_PACKED_VARLEN", raising = False)
+ _block_flag(monkeypatch, in_block, leaves)
+ leaving = in_block and leaves is True
+ lengths = [3, 3]
+ module = types.SimpleNamespace(is_causal = True, num_key_value_groups = 1, training = False)
+ q = torch.randn(1, 4, 6, 16)
+ mask, psl = _block_causal(lengths), torch.tensor(lengths, dtype = torch.int32)
+ torch._dynamo.reset()
+ torch._dynamo.utils.counters.clear()
+ fn = torch.compile(
+ lambda q: hpa._sdpa_packed_varlen(module, q, q, q, mask, packed_seq_lengths = psl)[0] * 2,
+ backend = "eager",
+ )
+ before = dict(hpa.HF_PACKED_ATTENTION_STATS)
+ out = fn(q)
+ torch.testing.assert_close(out, _plain_sdpa(module, q, q, q, mask)[0] * 2)
+ breaks = sum(torch._dynamo.utils.counters["graph_break"].values())
+ # CPU rows take the eager wrapper's own fallback, counted only outside a trace.
+ if leaving:
+ assert hpa.HF_PACKED_ATTENTION_STATS["fallback"] == before["fallback"] + 1
+ assert breaks >= 1
+ else:
+ assert hpa.HF_PACKED_ATTENTION_STATS == before
+ assert breaks == 0
+ # Unpacked rows stay in the graph either way.
+ torch._dynamo.reset()
+ torch._dynamo.utils.counters.clear()
+ plain = torch.compile(
+ lambda q: hpa._sdpa_packed_varlen(module, q, q, q, mask)[0], backend = "eager", fullgraph = True
+ )
+ plain(q)
+ assert hpa.HF_PACKED_ATTENTION_STATS["fallback"] == before["fallback"] + int(leaving)
+
+
+ "args",
+ [
+ ("cpu", None, torch.bfloat16, 64, None),
+ ("cuda", 0, torch.bfloat16, 64, -1), # a window varlen cannot take
+ ("cuda", 0, torch.bfloat16, 512, None), # past flash
+ ("cuda", 0, torch.bfloat16, 60, None),
+ ],
+)
+def test_compiled_block_keeps_rows_without_a_varlen_win(monkeypatch, args):
+ monkeypatch.setattr(hpa, "_LEAVES_BLOCK", {})
+ assert hpa._leaves_block_eager(*args) is False
+
+
+def test_compiled_block_keeps_rows_on_torch_with_cheap_cudnn_plans(monkeypatch):
+ monkeypatch.setattr(hpa, "_SLOW_CUDNN_PLANS", False)
+ monkeypatch.setattr(hpa, "_LEAVES_BLOCK", {})
+ hpa._LEAVES_BLOCK.clear()
+ assert hpa._leaves_block_eager("cuda", 0, torch.bfloat16, 64, None) is False
+
+
+def test_compiled_block_verdict_is_computed_eagerly_while_tracing(monkeypatch):
+ """The probe runs real tensors once per key; traced instead (an lru_cache wrapper is traced
+ through __wrapped__) it saw fake tensors and kept every row in the graph."""
+ calls = []
+
+ def probe(*key):
+ calls.append(type(torch.zeros(1)) is torch.Tensor)
+ return True
+
+ monkeypatch.setattr(hpa, "_leaves_block_probe", probe)
+ monkeypatch.setattr(hpa, "_LEAVES_BLOCK", {})
+ torch._dynamo.reset()
+ fn = torch.compile(
+ lambda x: x * 2 if hpa._leaves_block("cuda", 0, torch.bfloat16, 64, None) else x,
+ backend = "eager",
+ fullgraph = True,
+ )
+ torch.testing.assert_close(fn(torch.ones(3)), torch.full((3,), 2.0))
+ fn(torch.ones(3))
+ assert calls == [True]
+
+
+@_CUDA
+@_TORCH_VARLEN_HOST
+def test_packed_rows_in_a_compiled_block_take_varlen(monkeypatch):
+ """Real predicate, evaluated while tracing: it must agree with the eager verdict, and a row it
+ sends out takes the varlen kernel."""
+ if hpa._torch_varlen(torch.device("cuda", 0)) is None:
+ pytest.skip(
+ reason = "varlen kernel needs torch's varlen_attn with window_size (CUDA torch >= 2.11)"
+ )
+ from transformers.integrations.sdpa_attention import sdpa_attention_forward
+
+ hpa._ORIG_SDPA[0] = sdpa_attention_forward
+ monkeypatch.delenv("UNSLOTH_HF_PACKED_VARLEN", raising = False)
+ _block_flag(monkeypatch, True)
+ monkeypatch.setattr(hpa, "_LEAVES_BLOCK", {})
+ torch.manual_seed(0)
+ lengths = [37, 100, 2, 5, 230]
+ T, H, n_kv, D = sum(lengths), 8, 2, 64
+ module = types.SimpleNamespace(is_causal = True, num_key_value_groups = H // n_kv, training = True)
+ q = torch.randn(1, H, T, D, device = "cuda", dtype = torch.bfloat16, requires_grad = True)
+ k = torch.randn(1, n_kv, T, D, device = "cuda", dtype = torch.bfloat16, requires_grad = True)
+ v = torch.randn(1, n_kv, T, D, device = "cuda", dtype = torch.bfloat16, requires_grad = True)
+ mask, psl = _block_causal(lengths, "cuda"), torch.tensor(lengths, dtype = torch.int32)
+ torch._dynamo.reset()
+ fn = torch.compile(
+ lambda q, k, v: hpa._sdpa_packed_varlen(
+ module, q, k, v, mask, scaling = 0.11, packed_seq_lengths = psl
+ )[0],
+ )
+ for t, dims in ((q, (2,)), (k, (2,)), (v, (2,)), (mask, (2, 3))):
+ torch._dynamo.mark_dynamic(t, dims) # as _block_compile marks the sequence
+ before = hpa.HF_PACKED_ATTENTION_STATS["fast"]
+ fast = fn(q, k, v)
+ expected = hpa._leaves_block_probe("cuda", 0, torch.bfloat16, D, None)
+ assert hpa._LEAVES_BLOCK[("cuda", 0, torch.bfloat16, D, None)] is expected
+ assert hpa.HF_PACKED_ATTENTION_STATS["fast"] == before + int(expected)
+ if hpa._SLOW_CUDNN_PLANS and torch.cuda.get_device_capability(0)[0] == 10:
+ assert expected # the case this exists for: SM100 picks cuDNN for the dense mask
+ ref, _ = sdpa_attention_forward(module, q, k, v, mask, scaling = 0.11)
+ torch.testing.assert_close(fast.float(), ref.float(), atol = 2e-2, rtol = 2e-2)
+
+
@_CUDA
@pytest.mark.parametrize("window", [None, 3, 4, 128])
@pytest.mark.parametrize("route", ["torch", "torch_2_11", "dispatch"])
diff --git a/unsloth/models/_block_compile.py b/unsloth/models/_block_compile.py
index d62e077facf..a1828fe9747 100644
--- a/unsloth/models/_block_compile.py
+++ b/unsloth/models/_block_compile.py
@@ -661,6 +661,8 @@ def _precheck_band(layer, seq_len, args, kwargs):
# Mode of the block being traced (_SPLIT: flex_attention outside); dynamo guards on it.
_IN_BLOCK = [0]
+_PACKED_MODULE = "unsloth.utils.hf_packed_attention"
+_NO_PACKED = [False]
_OFF, _FULL, _SPLIT = 0, 1, 2
_DEMOTION = (_FULL, _SPLIT, _OFF)
_FIRST_MODE = _SPLIT if os.environ.get("UNSLOTH_BLOCK_FLEX", "inside") == "outside" else _FULL
@@ -828,6 +830,10 @@ def _wrap(
previous = inline_flag[0]
inline_flag[0] = True
_IN_BLOCK[0] = mode
+ # Packed rows run hf_packed_attention's varlen kernel outside the graph (see there).
+ packed = getattr(sys.modules.get(_PACKED_MODULE), "IN_COMPILED_BLOCK", _NO_PACKED)
+ packed_previous = packed[0]
+ packed[0] = True
try:
# Capture hooks (router_logits) append to empty scratch lists: no guard on the layer index.
with capture_scratch():
@@ -847,6 +853,7 @@ def _wrap(
finally:
inline_flag[0] = previous
_IN_BLOCK[0] = _OFF
+ packed[0] = packed_previous
def _inline(self, hidden_states, args, kwargs, mode):
previous = inline_flag[0]
diff --git a/unsloth/utils/hf_packed_attention.py b/unsloth/utils/hf_packed_attention.py
index 63cf90bd5ab..af850f6ff38 100644
--- a/unsloth/utils/hf_packed_attention.py
+++ b/unsloth/utils/hf_packed_attention.py
@@ -17,6 +17,8 @@ __all__ = ["enable_hf_packed_attention", "HF_PACKED_ATTENTION_STATS"]
_FLAG = "_unsloth_hf_packed_varlen"
_ENV = "UNSLOTH_HF_PACKED_VARLEN"
_ORIG_SDPA = [None]
+# True inside _block_compile's own compiled blocks (not a user's outer compile); see _leaves_block.
+IN_COMPILED_BLOCK = [False]
# efficient: the subset of fast calls that ran SDPA's memory-efficient kernel over cu_seqlens.
HF_PACKED_ATTENTION_STATS = {"fast": 0, "fallback": 0, "mask_checks": 0, "efficient": 0}
# Layers of one window share a mask object: window -> [weakref, (cu_seqlens, versions), verdict].
@@ -330,6 +332,31 @@ def _sdpa_packed_varlen(
HF_PACKED_ATTENTION_STATS["fallback"] += 1
return orig(module, query, key, value, attention_mask, *args, **named, **kwargs)
# Before any counter while tracing: Dynamo would guard on it and recompile every forward.
+ if (
+ IN_COMPILED_BLOCK[0]
+ and torch.compiler.is_compiling()
+ and kwargs.get("packed_seq_lengths") is not None
+ and not _disabled()
+ and _leaves_block(
+ query.device.type,
+ query.device.index,
+ query.dtype,
+ # int(): a symbolic head dim (dynamic=True) cannot be folded and would trace the probe.
+ int(query.shape[-1]),
+ _window_key(module, kwargs),
+ )
+ ):
+ return _sdpa_packed_varlen_eager(
+ module,
+ query,
+ key,
+ value,
+ attention_mask,
+ dropout = dropout,
+ scaling = scaling,
+ is_causal = is_causal,
+ **kwargs,
+ )
if kwargs.get("packed_seq_lengths") is None or _disabled() or torch.compiler.is_compiling():
return orig(
module,
@@ -533,6 +560,57 @@ def _sdpa_packed_varlen(
return out.contiguous(), None
+_sdpa_packed_varlen_eager = torch.compiler.disable(_sdpa_packed_varlen)
+
+# Compiled cuDNN SDPA over a dense mask builds a plan per new length, slowly before torch 2.13.
+_SLOW_CUDNN_PLANS = tuple(int(x) for x in torch.__version__.split("+")[0].split(".")[:2]) < (2, 13)
+_LEAVES_BLOCK = {}
+
+
+def _window_key(module, kwargs):
+ window = _sliding_window(module, kwargs)
+ return -1 if window is _NO_WINDOW else window
+
+
+def _leaves_block_eager(device_type, device_index, dtype, head_dim, window):
+ """Packed rows leave a compiled block only to dodge those plans, and only when the eager wrapper
+ has a varlen kernel for them; elsewhere the graph break is pure cost."""
+ key = (device_type, device_index, dtype, head_dim, window)
+ if key not in _LEAVES_BLOCK:
+ _LEAVES_BLOCK[key] = _leaves_block_probe(*key)
+ return _LEAVES_BLOCK[key]
+
+
+def _leaves_block_probe(device_type, device_index, dtype, head_dim, window):
+ if (
+ not _SLOW_CUDNN_PLANS
+ or device_type != "cuda"
+ or window == -1
+ or head_dim % 8
+ or head_dim > 256
+ ):
+ return False
+ try:
+ from .flash_impl import flash_impl_allows
+ from .attention_dispatch import FLASH_VARLEN, XFORMERS, select_attention_backend
+ from torch.nn.attention import SDPBackend
+
+ device = torch.device("cuda", device_index)
+ kernel = _torch_varlen(device) is not None and flash_impl_allows(head_dim, window, device)
+ if not kernel and select_attention_backend(use_varlen = True) not in (FLASH_VARLEN, XFORMERS):
+ return False
+ q = torch.zeros(1, 2, 64, head_dim, device = device, dtype = dtype, requires_grad = True)
+ mask = torch.ones(64, 64, dtype = torch.bool, device = device).tril()[None, None]
+ choice = torch._fused_sdp_choice(q, q, q, attn_mask = mask, dropout_p = 0.0, is_causal = False)
+ return choice == int(SDPBackend.CUDNN_ATTENTION)
+ except Exception:
+ return False
+
+
+# Baked into the graph at trace time. Not an lru_cache: Dynamo traces those via __wrapped__ (fake tensors).
+_leaves_block = torch._dynamo.assume_constant_result(_leaves_block_eager)
+
+
def enable_hf_packed_attention() -> bool:
"""Wrap transformers' "sdpa" attention entry once. True when the wrapper is installed."""
if _disabled():