Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Packed SDPA rows: one torch varlen_attn call instead of one SDPA call per segment (#13221)
Daniel Han2ea28ddd7d29
diff --git a/tests/utils/test_torch_varlen_packed.py b/tests/utils/test_torch_varlen_packed.pynew file mode 100644index 00000000000..763e7fde0b1--- /dev/null+++ b/tests/utils/test_torch_varlen_packed.py@@ -0,0 +1,259 @@+# SPDX-License-Identifier: AGPL-3.0-only+"""Packed SDPA rows via torch varlen_attn match the per-segment fallback."""++import types++import pytest+import torch+from torch.nn.attention import SDPBackend, sdpa_kernel+from real_accelerator import has_real_cuda # tests/_shared, on sys.path via tests/conftest.py++import unsloth # noqa: F401+from unsloth.utils import attention_dispatch as ad+from unsloth.utils import packing++_REAL_VARLEN = ad._TORCH_VARLEN_ATTN+needs_varlen = pytest.mark.skipif(+ not has_real_cuda()+ or ad._TORCH_VARLEN_ATTN is None+ or torch.version.hip is not None+ or torch.cuda.get_device_capability()[0] < 8,+ reason = "needs sm80+ CUDA (not ROCm) and torch varlen_attn with window_size + enable_gqa (2.12+)",+)+++def _seq_info(lengths, device):+ lengths = torch.tensor(lengths, dtype = torch.int32, device = device)+ cu = torch.cat((torch.zeros(1, dtype = torch.int32, device = device), lengths.cumsum(0).int()))+ return lengths, cu, int(lengths.max())+++def _run(+ monkeypatch,+ varlen,+ lengths,+ total,+ n_heads,+ n_kv,+ qkv,+ is_causal = True,+ window = None,+ sdpa_kwargs = None,+):+ calls = []+ if varlen:++ def counted(*args, **kwargs):+ calls.append(kwargs["window_size"])+ return _REAL_VARLEN(*args, **kwargs)++ monkeypatch.setattr(ad, "_TORCH_VARLEN_ATTN", counted)+ else:+ monkeypatch.setattr(ad, "_TORCH_VARLEN_ATTN", None)+ packing.clear_packed_caches()+ config = ad.AttentionConfig(+ backend = ad.SDPA, n_kv_heads = n_kv, n_groups = n_heads // n_kv, sdpa_kwargs = sdpa_kwargs+ )+ context = ad.AttentionContext(+ bsz = 1,+ q_len = total,+ kv_seq_len = total,+ n_heads = n_heads,+ head_dim = qkv[0].shape[-1],+ requires_grad = True,+ seq_info = _seq_info(lengths, qkv[0].device),+ attention_mask = None,+ causal_mask = None,+ is_causal = is_causal,+ sliding_window = window,+ )+ Q, K, V = (x.detach().clone().requires_grad_() for x in qkv)+ out = ad.run_attention(config = config, context = context, Q = Q, K = K, V = V)+ out = out.reshape(1, total, n_heads, -1)+ (+ out.float() * torch.linspace(-1, 1, out.numel(), device = out.device).view_as(out)+ ).sum().backward()+ return (out.detach(), Q.grad, K.grad, V.grad), calls+++CASES = [+ # lengths, total (> sum = pad tail), heads, kv heads, causal, window, expected window_size+ ((5, 3, 4), 12, 4, 4, True, None, (-1, 0)),+ ((5, 3, 4), 12, 4, 2, True, None, (-1, 0)),+ ((4, 4, 4), 12, 4, 1, True, None, None), # one run of equal lengths: one batched SDPA call+ ((7,), 10, 4, 2, True, None, (-1, 0)),+ ((6, 2, 5), 13, 4, 2, True, 3, (2, 0)),+ ((6, 2, 5), 13, 4, 2, False, None, (-1, -1)),+ ((1, 1, 9), 11, 2, 1, True, 4, (3, 0)),+ ((300, 700, 24), 1100, 8, 2, True, None, (-1, 0)),+ ((300, 700, 24), 1100, 8, 2, True, 128, (127, 0)),+]+++@needs_varlen[email protected]("dtype", [torch.bfloat16, torch.float16])[email protected]("head_dim", [64, 128, 256])[email protected]("lengths, total, n_heads, n_kv, is_causal, window, window_size", CASES)+def test_varlen_matches_segments(+ monkeypatch, dtype, head_dim, lengths, total, n_heads, n_kv, is_causal, window, window_size+):+ g = torch.Generator().manual_seed(3407)+ shapes = ((1, n_heads, total, head_dim), (1, n_kv, total, head_dim), (1, n_kv, total, head_dim))+ qkv = [torch.randn(s, generator = g).to("cuda", dtype) for s in shapes]+ args = (lengths, total, n_heads, n_kv, qkv, is_causal, window)+ segs, seg_calls = _run(monkeypatch, False, *args)+ varlen, calls = _run(monkeypatch, True, *args)+ assert seg_calls == [] and calls == ([] if window_size is None else [window_size])+ for name, a, b in zip(("out", "dQ", "dK", "dV"), segs, varlen):+ assert torch.isfinite(b).all(), name+ torch.testing.assert_close(b.float(), a.float(), atol = 2e-2, rtol = 2e-2, msg = name)+++@needs_varlen+def test_bidirectional_mha_head_dim_256_runs(monkeypatch):+ g = torch.Generator().manual_seed(2)+ qkv = [torch.randn(1, 8, 1024, 256, generator = g).cuda().bfloat16() for _ in range(3)]+ out, calls = _run(monkeypatch, True, (300, 500, 224), 1024, 8, 8, qkv, is_causal = False)+ assert calls == [] and all(torch.isfinite(x).all() for x in out)+++@needs_varlen+ "backend, lengths, total, expect_varlen",+ [+ ("CUDNN_ATTENTION", (300, 700, 24), 1100, True), # mean 275: launches dominate+ ("CUDNN_ATTENTION", (1500, 2000, 1800), 5300, False), # mean 1767: cuDNN per segment wins+ ("FLASH_ATTENTION", (1500, 2000, 1800), 5300, True), # same kernel, fewer launches+ ("FLASH_ATTENTION", (2048, 2048), 4096, False), # one run: one batched call already+ ("MATH", (300, 700, 24), 1100, False), # caller's priority picked math+ ("EFFICIENT_ATTENTION", (300, 700, 24), 1100, False),+ ],+)+def test_varlen_only_where_it_beats_segments(monkeypatch, backend, lengths, total, expect_varlen):+ from torch.nn.attention import SDPBackend as B++ monkeypatch.setattr(torch, "_fused_sdp_choice", lambda *a, **k: int(getattr(B, backend)))+ ad._sdpa_choice.cache_clear()+ qkv = [torch.zeros(1, 4, total, 64, device = "cuda", dtype = torch.bfloat16) for _ in range(3)]+ try:+ _, calls = _run(monkeypatch, True, lengths, total, 4, 4, qkv)+ finally:+ ad._sdpa_choice.cache_clear()+ assert calls == ([(-1, 0)] if expect_varlen else [])+++@needs_varlen+def test_scale_is_forwarded(monkeypatch):+ g = torch.Generator().manual_seed(0)+ qkv = [torch.randn(1, 4, 40, 64, generator = g).cuda().bfloat16() for _ in range(3)]+ kwargs = dict(sdpa_kwargs = {"scale": 0.3, "dropout_p": 0.0})+ segs, _ = _run(monkeypatch, False, (25, 15), 40, 4, 4, qkv, **kwargs)+ varlen, calls = _run(monkeypatch, True, (25, 15), 40, 4, 4, qkv, **kwargs)+ assert calls == [(-1, 0)]+ torch.testing.assert_close(varlen[0].float(), segs[0].float(), atol = 2e-2, rtol = 2e-2)+++@needs_varlen+ "case",+ [+ "fp32",+ "dropout",+ "attn_mask",+ "bidirectional_window",+ "bidirectional_mha",+ "deterministic",+ "int32_overflow",+ "flash_sdp_disabled",+ "math_priority",+ ],+)+def test_ineligible_calls_keep_the_segment_path(monkeypatch, case):+ dtype = torch.float32 if case == "fp32" else torch.bfloat16+ g = torch.Generator().manual_seed(1)+ qkv = [torch.randn(1, 4, 24, 64, generator = g).to("cuda", dtype) for _ in range(3)]+ kwargs = {}+ if case == "dropout":+ kwargs["sdpa_kwargs"] = {"dropout_p": 0.1}+ torch.manual_seed(0)+ elif case == "attn_mask":+ kwargs["sdpa_kwargs"] = {"attn_mask": None, "scale": None}+ elif case == "bidirectional_window":+ kwargs.update(is_causal = False, window = 4)+ elif case == "bidirectional_mha":+ kwargs.update(is_causal = False)+ elif case == "deterministic":+ monkeypatch.setattr(torch, "are_deterministic_algorithms_enabled", lambda: True)+ elif case == "int32_overflow":+ monkeypatch.setattr(ad, "_varlen_backward_overflows_int32", lambda *a: True)+ if case == "flash_sdp_disabled":+ with sdpa_kernel([SDPBackend.EFFICIENT_ATTENTION, SDPBackend.MATH]):+ _, calls = _run(monkeypatch, True, (10, 14), 24, 4, 4, qkv, **kwargs)+ elif case == "math_priority":+ ad._sdpa_choice.cache_clear()+ with sdpa_kernel([SDPBackend.MATH, SDPBackend.FLASH_ATTENTION], set_priority = True):+ _, calls = _run(monkeypatch, True, (10, 14), 24, 4, 4, qkv, **kwargs)+ ad._sdpa_choice.cache_clear()+ else:+ _, calls = _run(monkeypatch, True, (10, 14), 24, 4, 4, qkv, **kwargs)+ assert calls == []+++@needs_varlen+def test_lengths_overrunning_the_row_keep_the_segment_path(monkeypatch):+ # Malformed lengths (sum 30 > 24 tokens) must raise as before, not index past the row in the kernel.+ calls = []++ def stub(*args, **kwargs):+ calls.append(1)+ raise AssertionError("varlen_attn called with cu_seqlens past the row")++ monkeypatch.setattr(ad, "_TORCH_VARLEN_ATTN", stub)+ packing.clear_packed_caches()+ g = torch.Generator().manual_seed(2)+ Q, K, V = (torch.randn(1, 4, 24, 64, generator = g).to("cuda", torch.bfloat16) for _ in range(3))+ config = ad.AttentionConfig(backend = ad.SDPA, n_kv_heads = 4, n_groups = 1)+ context = ad.AttentionContext(+ bsz = 1,+ q_len = 24,+ kv_seq_len = 24,+ n_heads = 4,+ head_dim = 64,+ requires_grad = False,+ seq_info = _seq_info((10, 14, 6), "cuda"),+ attention_mask = None,+ causal_mask = None,+ is_causal = True,+ sliding_window = None,+ )+ with pytest.raises(RuntimeError):+ ad.run_attention(config = config, context = context, Q = Q, K = K, V = V)+ assert calls == []+++def test_loader_needs_window_and_gqa(monkeypatch):+ import sys++ def old_varlen_attn(+ query,+ key,+ value,+ cu_seq_q,+ cu_seq_k,+ max_q,+ max_k,+ *,+ is_causal = False,+ ):+ raise AssertionError("not called")++ fake = types.ModuleType("torch.nn.attention.varlen")+ fake.varlen_attn = old_varlen_attn+ monkeypatch.setitem(sys.modules, "torch.nn.attention.varlen", fake)+ monkeypatch.delenv("UNSLOTH_TORCH_VARLEN", raising = False)+ assert ad._load_torch_varlen_attn() is None+++def test_kill_switch(monkeypatch):+ monkeypatch.setenv("UNSLOTH_TORCH_VARLEN", "0")+ assert ad._load_torch_varlen_attn() is Nonediff --git a/unsloth/utils/attention_dispatch.py b/unsloth/utils/attention_dispatch.pyindex 4fdccd2979a..cc225bd001a 100644--- a/unsloth/utils/attention_dispatch.py+++ b/unsloth/utils/attention_dispatch.py@@ -14,6 +14,7 @@from __future__ import annotations+import functoolsimport osfrom dataclasses import dataclassfrom typing import Any, Optional, Tuple@@ -87,6 +88,162 @@ _SDPA_PACKED_SEGMENTS = os.environ.get("UNSLOTH_SDPA_PACKED_SEGMENTS", "1").lowe)+def _load_torch_varlen_attn():+ # Needs window_size (torch 2.11) and enable_gqa (2.12); older builds keep the per-segment path.+ if os.environ.get("UNSLOTH_TORCH_VARLEN", "1").lower() in ("0", "false", "no", "off"):+ return None+ try:+ import inspect+ from torch.nn.attention.varlen import varlen_attn+ except Exception:+ return None+ params = inspect.signature(varlen_attn).parameters+ if "window_size" not in params or "enable_gqa" not in params:+ return None+ # Causal varlen always uses torch's bundled flash kernel, which some source builds omit.+ if not getattr(torch.backends.cuda, "is_flash_attention_available", lambda: True)():+ return None+ return varlen_attn+++_TORCH_VARLEN_ATTN = _load_torch_varlen_attn()+++def _torch_varlen_takes(+ Q: Tensor,+ K: Tensor,+ V: Tensor,+ sdpa_kwargs: dict,+ is_causal: bool,+ sliding_window: Optional[int],+) -> bool:+ """True when torch's varlen kernel computes exactly what the per-segment SDPA path would."""+ if _TORCH_VARLEN_ATTN is None or not Q.is_cuda or torch.version.hip is not None:+ return False+ if Q.dtype not in (torch.float16, torch.bfloat16) or K.dtype != Q.dtype or V.dtype != Q.dtype:+ return False+ if (+ Q.shape[-1] > 256+ or Q.shape[-1] % 8 != 0+ or torch.cuda.get_device_capability(Q.device)[0] < 8+ ):+ return False+ # A bidirectional window has no (left, right) spelling matching packed_block_mask.+ if set(sdpa_kwargs) - {"scale", "dropout_p"} or sdpa_kwargs.get("dropout_p"):+ return False+ if sliding_window is not None and not is_causal:+ return False+ # torch 2.14 routes bidirectional MHA to cuDNN ragged on sm90/sm100: no head_dim 256 plan.+ if not is_causal and Q.shape[1] == K.shape[1]:+ return False+ return not torch.are_deterministic_algorithms_enabled()+++# Above this mean segment length cuDNN's per-segment SDPA beats torch's varlen flash kernel (B200 sweep).+_VARLEN_CUDNN_MAX_MEAN_LEN = 1024++[email protected]_cache(maxsize = 64)+def _sdpa_choice(device, dtype, head_dim, n_heads, n_kv_heads, is_causal, enable_gqa, backends):+ try:+ q = torch.empty(1, n_heads, 128, head_dim, device = device, dtype = dtype)+ k = torch.empty(1, n_kv_heads, 128, head_dim, device = device, dtype = dtype)+ return int(torch._fused_sdp_choice(q, k, k, is_causal = is_causal, enable_gqa = enable_gqa))+ except Exception:+ return -1+++def _sdpa_backend_state(device, dtype, head_dim, n_heads, n_kv_heads, is_causal, enable_gqa):+ """(flash SDPA enabled, SDPBackend SDPA would pick) for these shapes. Both reads break a dynamo graph."""+ cuda = torch.backends.cuda+ backends = (+ cuda.flash_sdp_enabled(),+ cuda.cudnn_sdp_enabled(),+ cuda.mem_efficient_sdp_enabled(),+ cuda.math_sdp_enabled(),+ tuple(getattr(torch._C, "_get_sdp_priority_order", tuple)()),+ )+ choice = _sdpa_choice(+ device, dtype, head_dim, n_heads, n_kv_heads, is_causal, enable_gqa, backends+ )+ return backends[0], choice+++if hasattr(torch.compiler, "assume_constant_result"):+ _sdpa_backend_state = torch.compiler.assume_constant_result(_sdpa_backend_state)+++def _torch_varlen_beats_segments(+ Q: Tensor, K: Tensor, lengths: Tuple[int, ...], n_groups: int, is_causal: bool+) -> bool:+ """Varlen saves one SDPA launch per run of equal lengths, but its flash kernel is slower per token+ than cuDNN on long segments; with flash already chosen, the kernel is the same and only launches drop."""+ lengths = [length for length in lengths if length > 0]+ # Lengths overrunning the row would send cu_seqlens past the tensor; leave them to the old path's error.+ if sum(lengths) != Q.shape[-2]:+ return False+ runs = sum(1 for i, length in enumerate(lengths) if i == 0 or lengths[i - 1] != length)+ if runs <= 1:+ return False+ flash, choice = _sdpa_backend_state(+ str(Q.device),+ Q.dtype,+ Q.shape[-1],+ Q.shape[1],+ K.shape[1],+ is_causal,+ n_groups != 1 and _sdpa_flash_takes_gqa(Q),+ )+ # varlen_attn calls flash directly: honour a disabled flash backend and math / efficient-first priorities.+ from torch.nn.attention import SDPBackend++ if not flash:+ return False+ if choice == int(SDPBackend.FLASH_ATTENTION):+ return True+ if choice == int(SDPBackend.CUDNN_ATTENTION):+ return sum(lengths) / len(lengths) <= _VARLEN_CUDNN_MAX_MEAN_LEN+ return False+++def _torch_varlen_packed(+ Q: Tensor,+ K: Tensor,+ V: Tensor,+ seq_info,+ *,+ n_groups: int,+ is_causal: bool,+ sliding_window: Optional[int],+ scale: Optional[float],+) -> Tensor:+ """Q (1, H, T, D), K / V (1, H_kv, T, D) -> (1, T, H, D) over the row's segments, pad tail included."""+ total = Q.shape[-2]+ cu_seqlens, max_seqlen = cover_padded_cu_seqlens(seq_info, total)+ # seq_info may live on the CPU.+ cu_seqlens = cu_seqlens.to(device = Q.device, dtype = torch.int32, non_blocking = True)+ if not is_causal:+ window_size = (-1, -1)+ elif sliding_window is not None:+ window_size = (sliding_window - 1, 0) # packed_block_mask keeps k >= q - (window - 1)+ else:+ window_size = (-1, 0)+ Q_f, K_f, V_f = (x[0].transpose(0, 1).contiguous() for x in (Q, K, V))+ out = _TORCH_VARLEN_ATTN(+ Q_f,+ K_f,+ V_f,+ cu_seqlens,+ cu_seqlens,+ max_seqlen,+ max_seqlen,+ scale = scale,+ window_size = window_size,+ enable_gqa = n_groups != 1,+ )+ return out.unsqueeze(0)++def _sdpa_flash_takes_gqa(Q: Tensor) -> bool:# Only flash / math take enable_gqa: expand K/V elsewhere (pre-sm80, fp32, wide heads, ROCm).return (@@ -590,11 +747,36 @@ def run_attention(and Q.shape[0] == 1and Q.shape[-2] == K.shape[-2]):+ lengths = packed_segment_lengths(context.seq_info, K.shape[-2])+ if (+ _torch_varlen_takes(Q, K, V, sdpa_kwargs, context.is_causal, sliding_window)+ and _torch_varlen_beats_segments(Q, K, lengths, config.n_groups, context.is_causal)+ ) and not (+ # Varlen backward may share flash-attn 2's int32 dq_accum limit.+ requires_grad+ and not _VARLEN_INT32_GUARD_DISABLED+ and _varlen_backward_overflows_int32(+ cover_padded_cu_seqlens(context.seq_info, Q.shape[-2])[0].numel() - 1,+ Q.shape[-2],+ n_heads,+ head_dim,+ )+ ):+ return _torch_varlen_packed(+ Q,+ K,+ V,+ context.seq_info,+ n_groups = config.n_groups,+ is_causal = context.is_causal,+ sliding_window = sliding_window,+ scale = sdpa_kwargs.get("scale"),+ )out = _sdpa_packed_segments(Q,K,V,- packed_segment_lengths(context.seq_info, K.shape[-2]),+ lengths,n_groups = config.n_groups,is_causal = context.is_causal,sliding_window = sliding_window,