Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Compiled blocks: packed rows run varlen attention outside the graph (#14808)
Daniel Hane6f72a0198e1
diff --git a/tests/test_block_compile.py b/tests/test_block_compile.pyindex 6526ab63d59..ce2a27c54b7 100644--- a/tests/test_block_compile.py+++ b/tests/test_block_compile.py@@ -544,6 +544,42 @@ def test_unsloth_host_metadata_calls_keep_the_stock_forward(monkeypatch, key):assert calls # no host metadata: compiled+def test_compiled_block_flags_packed_attention(monkeypatch):+ """hf_packed_attention runs packed rows outside the graph only while a block's compiled call+ runs: set there, restored after, never on the stock forward."""+ M = _load()+ packed_mod = types.ModuleType(M._PACKED_MODULE)+ packed_mod.IN_COMPILED_BLOCK = [False]+ monkeypatch.setitem(sys.modules, M._PACKED_MODULE, packed_mod)+ seen = []++ def fake_compile(fn, **kwargs):+ def compiled(*args, **kw):+ seen.append(packed_mod.IN_COMPILED_BLOCK[0])+ return fn(*args, **kw)++ return compiled++ monkeypatch.setattr(torch, "compile", fake_compile)+ cls = _toy_class()+ M._wrap(cls, {})+ layer = cls()+ x = torch.randn(1, 4, 8)+ with torch.no_grad():+ layer(x)+ assert seen and all(seen)+ assert packed_mod.IN_COMPILED_BLOCK == [False]+ seen.clear()+ layer.eval()(x) # stock forward+ assert seen == [] and packed_mod.IN_COMPILED_BLOCK == [False]+ # Without the module (never imported) the block still runs.+ monkeypatch.delitem(sys.modules, M._PACKED_MODULE)+ layer.train()+ with torch.no_grad():+ layer(x)+ assert M._NO_PACKED == [False]++def test_gemma4_band_fast_path_in_blocks(monkeypatch):M = _load()from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONSdiff --git a/tests/test_hf_packed_attention.py b/tests/test_hf_packed_attention.pyindex 2ddb796fe47..cce2c231577 100644--- a/tests/test_hf_packed_attention.py+++ b/tests/test_hf_packed_attention.py@@ -403,6 +403,141 @@ def test_rocm_keeps_windowed_rows_off_flash_attn(monkeypatch, window):assert seen and not dispatched+def _block_flag(+ monkeypatch,+ value,+ leaves = None,+):+ monkeypatch.setattr(hpa, "IN_COMPILED_BLOCK", [value])+ if leaves is not None:+ monkeypatch.setattr(hpa, "_leaves_block", lambda *args: leaves)++[email protected]("in_block,leaves", [(True, True), (True, None), (False, True)])+def test_packed_rows_leave_a_compiled_block_for_the_eager_wrapper(monkeypatch, in_block, leaves):+ """Inside an Unsloth compiled block a packed call the predicate sends out runs the wrapper+ eagerly (one graph break). A CPU row (real predicate) and a user's own compile (flag off) keep+ the wrapped sdpa in the graph."""+ hpa._ORIG_SDPA[0] = _plain_sdpa+ monkeypatch.delenv("UNSLOTH_HF_PACKED_VARLEN", raising = False)+ _block_flag(monkeypatch, in_block, leaves)+ leaving = in_block and leaves is True+ lengths = [3, 3]+ module = types.SimpleNamespace(is_causal = True, num_key_value_groups = 1, training = False)+ q = torch.randn(1, 4, 6, 16)+ mask, psl = _block_causal(lengths), torch.tensor(lengths, dtype = torch.int32)+ torch._dynamo.reset()+ torch._dynamo.utils.counters.clear()+ fn = torch.compile(+ lambda q: hpa._sdpa_packed_varlen(module, q, q, q, mask, packed_seq_lengths = psl)[0] * 2,+ backend = "eager",+ )+ before = dict(hpa.HF_PACKED_ATTENTION_STATS)+ out = fn(q)+ torch.testing.assert_close(out, _plain_sdpa(module, q, q, q, mask)[0] * 2)+ breaks = sum(torch._dynamo.utils.counters["graph_break"].values())+ # CPU rows take the eager wrapper's own fallback, counted only outside a trace.+ if leaving:+ assert hpa.HF_PACKED_ATTENTION_STATS["fallback"] == before["fallback"] + 1+ assert breaks >= 1+ else:+ assert hpa.HF_PACKED_ATTENTION_STATS == before+ assert breaks == 0+ # Unpacked rows stay in the graph either way.+ torch._dynamo.reset()+ torch._dynamo.utils.counters.clear()+ plain = torch.compile(+ lambda q: hpa._sdpa_packed_varlen(module, q, q, q, mask)[0], backend = "eager", fullgraph = True+ )+ plain(q)+ assert hpa.HF_PACKED_ATTENTION_STATS["fallback"] == before["fallback"] + int(leaving)+++ "args",+ [+ ("cpu", None, torch.bfloat16, 64, None),+ ("cuda", 0, torch.bfloat16, 64, -1), # a window varlen cannot take+ ("cuda", 0, torch.bfloat16, 512, None), # past flash+ ("cuda", 0, torch.bfloat16, 60, None),+ ],+)+def test_compiled_block_keeps_rows_without_a_varlen_win(monkeypatch, args):+ monkeypatch.setattr(hpa, "_LEAVES_BLOCK", {})+ assert hpa._leaves_block_eager(*args) is False+++def test_compiled_block_keeps_rows_on_torch_with_cheap_cudnn_plans(monkeypatch):+ monkeypatch.setattr(hpa, "_SLOW_CUDNN_PLANS", False)+ monkeypatch.setattr(hpa, "_LEAVES_BLOCK", {})+ hpa._LEAVES_BLOCK.clear()+ assert hpa._leaves_block_eager("cuda", 0, torch.bfloat16, 64, None) is False+++def test_compiled_block_verdict_is_computed_eagerly_while_tracing(monkeypatch):+ """The probe runs real tensors once per key; traced instead (an lru_cache wrapper is traced+ through __wrapped__) it saw fake tensors and kept every row in the graph."""+ calls = []++ def probe(*key):+ calls.append(type(torch.zeros(1)) is torch.Tensor)+ return True++ monkeypatch.setattr(hpa, "_leaves_block_probe", probe)+ monkeypatch.setattr(hpa, "_LEAVES_BLOCK", {})+ torch._dynamo.reset()+ fn = torch.compile(+ lambda x: x * 2 if hpa._leaves_block("cuda", 0, torch.bfloat16, 64, None) else x,+ backend = "eager",+ fullgraph = True,+ )+ torch.testing.assert_close(fn(torch.ones(3)), torch.full((3,), 2.0))+ fn(torch.ones(3))+ assert calls == [True]+++@_CUDA+@_TORCH_VARLEN_HOST+def test_packed_rows_in_a_compiled_block_take_varlen(monkeypatch):+ """Real predicate, evaluated while tracing: it must agree with the eager verdict, and a row it+ sends out takes the varlen kernel."""+ if hpa._torch_varlen(torch.device("cuda", 0)) is None:+ pytest.skip(+ reason = "varlen kernel needs torch's varlen_attn with window_size (CUDA torch >= 2.11)"+ )+ from transformers.integrations.sdpa_attention import sdpa_attention_forward++ hpa._ORIG_SDPA[0] = sdpa_attention_forward+ monkeypatch.delenv("UNSLOTH_HF_PACKED_VARLEN", raising = False)+ _block_flag(monkeypatch, True)+ monkeypatch.setattr(hpa, "_LEAVES_BLOCK", {})+ torch.manual_seed(0)+ lengths = [37, 100, 2, 5, 230]+ T, H, n_kv, D = sum(lengths), 8, 2, 64+ module = types.SimpleNamespace(is_causal = True, num_key_value_groups = H // n_kv, training = True)+ q = torch.randn(1, H, T, D, device = "cuda", dtype = torch.bfloat16, requires_grad = True)+ k = torch.randn(1, n_kv, T, D, device = "cuda", dtype = torch.bfloat16, requires_grad = True)+ v = torch.randn(1, n_kv, T, D, device = "cuda", dtype = torch.bfloat16, requires_grad = True)+ mask, psl = _block_causal(lengths, "cuda"), torch.tensor(lengths, dtype = torch.int32)+ torch._dynamo.reset()+ fn = torch.compile(+ lambda q, k, v: hpa._sdpa_packed_varlen(+ module, q, k, v, mask, scaling = 0.11, packed_seq_lengths = psl+ )[0],+ )+ for t, dims in ((q, (2,)), (k, (2,)), (v, (2,)), (mask, (2, 3))):+ torch._dynamo.mark_dynamic(t, dims) # as _block_compile marks the sequence+ before = hpa.HF_PACKED_ATTENTION_STATS["fast"]+ fast = fn(q, k, v)+ expected = hpa._leaves_block_probe("cuda", 0, torch.bfloat16, D, None)+ assert hpa._LEAVES_BLOCK[("cuda", 0, torch.bfloat16, D, None)] is expected+ assert hpa.HF_PACKED_ATTENTION_STATS["fast"] == before + int(expected)+ if hpa._SLOW_CUDNN_PLANS and torch.cuda.get_device_capability(0)[0] == 10:+ assert expected # the case this exists for: SM100 picks cuDNN for the dense mask+ ref, _ = sdpa_attention_forward(module, q, k, v, mask, scaling = 0.11)+ torch.testing.assert_close(fast.float(), ref.float(), atol = 2e-2, rtol = 2e-2)++@_CUDA@pytest.mark.parametrize("window", [None, 3, 4, 128])@pytest.mark.parametrize("route", ["torch", "torch_2_11", "dispatch"])diff --git a/unsloth/models/_block_compile.py b/unsloth/models/_block_compile.pyindex d62e077facf..a1828fe9747 100644--- a/unsloth/models/_block_compile.py+++ b/unsloth/models/_block_compile.py@@ -661,6 +661,8 @@ def _precheck_band(layer, seq_len, args, kwargs):# Mode of the block being traced (_SPLIT: flex_attention outside); dynamo guards on it._IN_BLOCK = [0]+_PACKED_MODULE = "unsloth.utils.hf_packed_attention"+_NO_PACKED = [False]_OFF, _FULL, _SPLIT = 0, 1, 2_DEMOTION = (_FULL, _SPLIT, _OFF)_FIRST_MODE = _SPLIT if os.environ.get("UNSLOTH_BLOCK_FLEX", "inside") == "outside" else _FULL@@ -828,6 +830,10 @@ def _wrap(previous = inline_flag[0]inline_flag[0] = True_IN_BLOCK[0] = mode+ # Packed rows run hf_packed_attention's varlen kernel outside the graph (see there).+ packed = getattr(sys.modules.get(_PACKED_MODULE), "IN_COMPILED_BLOCK", _NO_PACKED)+ packed_previous = packed[0]+ packed[0] = Truetry:# Capture hooks (router_logits) append to empty scratch lists: no guard on the layer index.with capture_scratch():@@ -847,6 +853,7 @@ def _wrap(finally:inline_flag[0] = previous_IN_BLOCK[0] = _OFF+ packed[0] = packed_previousdef _inline(self, hidden_states, args, kwargs, mode):previous = inline_flag[0]diff --git a/unsloth/utils/hf_packed_attention.py b/unsloth/utils/hf_packed_attention.pyindex 63cf90bd5ab..af850f6ff38 100644--- a/unsloth/utils/hf_packed_attention.py+++ b/unsloth/utils/hf_packed_attention.py@@ -17,6 +17,8 @@ __all__ = ["enable_hf_packed_attention", "HF_PACKED_ATTENTION_STATS"]_FLAG = "_unsloth_hf_packed_varlen"_ENV = "UNSLOTH_HF_PACKED_VARLEN"_ORIG_SDPA = [None]+# True inside _block_compile's own compiled blocks (not a user's outer compile); see _leaves_block.+IN_COMPILED_BLOCK = [False]# efficient: the subset of fast calls that ran SDPA's memory-efficient kernel over cu_seqlens.HF_PACKED_ATTENTION_STATS = {"fast": 0, "fallback": 0, "mask_checks": 0, "efficient": 0}# Layers of one window share a mask object: window -> [weakref, (cu_seqlens, versions), verdict].@@ -330,6 +332,31 @@ def _sdpa_packed_varlen(HF_PACKED_ATTENTION_STATS["fallback"] += 1return orig(module, query, key, value, attention_mask, *args, **named, **kwargs)# Before any counter while tracing: Dynamo would guard on it and recompile every forward.+ if (+ IN_COMPILED_BLOCK[0]+ and torch.compiler.is_compiling()+ and kwargs.get("packed_seq_lengths") is not None+ and not _disabled()+ and _leaves_block(+ query.device.type,+ query.device.index,+ query.dtype,+ # int(): a symbolic head dim (dynamic=True) cannot be folded and would trace the probe.+ int(query.shape[-1]),+ _window_key(module, kwargs),+ )+ ):+ return _sdpa_packed_varlen_eager(+ module,+ query,+ key,+ value,+ attention_mask,+ dropout = dropout,+ scaling = scaling,+ is_causal = is_causal,+ **kwargs,+ )if kwargs.get("packed_seq_lengths") is None or _disabled() or torch.compiler.is_compiling():return orig(module,@@ -533,6 +560,57 @@ def _sdpa_packed_varlen(return out.contiguous(), None+_sdpa_packed_varlen_eager = torch.compiler.disable(_sdpa_packed_varlen)++# Compiled cuDNN SDPA over a dense mask builds a plan per new length, slowly before torch 2.13.+_SLOW_CUDNN_PLANS = tuple(int(x) for x in torch.__version__.split("+")[0].split(".")[:2]) < (2, 13)+_LEAVES_BLOCK = {}+++def _window_key(module, kwargs):+ window = _sliding_window(module, kwargs)+ return -1 if window is _NO_WINDOW else window+++def _leaves_block_eager(device_type, device_index, dtype, head_dim, window):+ """Packed rows leave a compiled block only to dodge those plans, and only when the eager wrapper+ has a varlen kernel for them; elsewhere the graph break is pure cost."""+ key = (device_type, device_index, dtype, head_dim, window)+ if key not in _LEAVES_BLOCK:+ _LEAVES_BLOCK[key] = _leaves_block_probe(*key)+ return _LEAVES_BLOCK[key]+++def _leaves_block_probe(device_type, device_index, dtype, head_dim, window):+ if (+ not _SLOW_CUDNN_PLANS+ or device_type != "cuda"+ or window == -1+ or head_dim % 8+ or head_dim > 256+ ):+ return False+ try:+ from .flash_impl import flash_impl_allows+ from .attention_dispatch import FLASH_VARLEN, XFORMERS, select_attention_backend+ from torch.nn.attention import SDPBackend++ device = torch.device("cuda", device_index)+ kernel = _torch_varlen(device) is not None and flash_impl_allows(head_dim, window, device)+ if not kernel and select_attention_backend(use_varlen = True) not in (FLASH_VARLEN, XFORMERS):+ return False+ q = torch.zeros(1, 2, 64, head_dim, device = device, dtype = dtype, requires_grad = True)+ mask = torch.ones(64, 64, dtype = torch.bool, device = device).tril()[None, None]+ choice = torch._fused_sdp_choice(q, q, q, attn_mask = mask, dropout_p = 0.0, is_causal = False)+ return choice == int(SDPBackend.CUDNN_ATTENTION)+ except Exception:+ return False+++# Baked into the graph at trace time. Not an lru_cache: Dynamo traces those via __wrapped__ (fake tensors).+_leaves_block = torch._dynamo.assume_constant_result(_leaves_block_eager)++def enable_hf_packed_attention() -> bool:"""Wrap transformers' "sdpa" attention entry once. True when the wrapper is installed."""if _disabled():