Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Packed SDPA rows: one torch varlen_attn call instead of one SDPA call per segment (#13221)

Daniel Han2ea28ddd7d29
diff --git a/tests/utils/test_torch_varlen_packed.py b/tests/utils/test_torch_varlen_packed.py
new file mode 100644
index 00000000000..763e7fde0b1
--- /dev/null
+++ b/tests/utils/test_torch_varlen_packed.py
@@ -0,0 +1,259 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+"""Packed SDPA rows via torch varlen_attn match the per-segment fallback."""
+
+import types
+
+import pytest
+import torch
+from torch.nn.attention import SDPBackend, sdpa_kernel
+from real_accelerator import has_real_cuda # tests/_shared, on sys.path via tests/conftest.py
+
+import unsloth # noqa: F401
+from unsloth.utils import attention_dispatch as ad
+from unsloth.utils import packing
+
+_REAL_VARLEN = ad._TORCH_VARLEN_ATTN
+needs_varlen = pytest.mark.skipif(
+ not has_real_cuda()
+ or ad._TORCH_VARLEN_ATTN is None
+ or torch.version.hip is not None
+ or torch.cuda.get_device_capability()[0] < 8,
+ reason = "needs sm80+ CUDA (not ROCm) and torch varlen_attn with window_size + enable_gqa (2.12+)",
+)
+
+
+def _seq_info(lengths, device):
+ lengths = torch.tensor(lengths, dtype = torch.int32, device = device)
+ cu = torch.cat((torch.zeros(1, dtype = torch.int32, device = device), lengths.cumsum(0).int()))
+ return lengths, cu, int(lengths.max())
+
+
+def _run(
+ monkeypatch,
+ varlen,
+ lengths,
+ total,
+ n_heads,
+ n_kv,
+ qkv,
+ is_causal = True,
+ window = None,
+ sdpa_kwargs = None,
+):
+ calls = []
+ if varlen:
+
+ def counted(*args, **kwargs):
+ calls.append(kwargs["window_size"])
+ return _REAL_VARLEN(*args, **kwargs)
+
+ monkeypatch.setattr(ad, "_TORCH_VARLEN_ATTN", counted)
+ else:
+ monkeypatch.setattr(ad, "_TORCH_VARLEN_ATTN", None)
+ packing.clear_packed_caches()
+ config = ad.AttentionConfig(
+ backend = ad.SDPA, n_kv_heads = n_kv, n_groups = n_heads // n_kv, sdpa_kwargs = sdpa_kwargs
+ )
+ context = ad.AttentionContext(
+ bsz = 1,
+ q_len = total,
+ kv_seq_len = total,
+ n_heads = n_heads,
+ head_dim = qkv[0].shape[-1],
+ requires_grad = True,
+ seq_info = _seq_info(lengths, qkv[0].device),
+ attention_mask = None,
+ causal_mask = None,
+ is_causal = is_causal,
+ sliding_window = window,
+ )
+ Q, K, V = (x.detach().clone().requires_grad_() for x in qkv)
+ out = ad.run_attention(config = config, context = context, Q = Q, K = K, V = V)
+ out = out.reshape(1, total, n_heads, -1)
+ (
+ out.float() * torch.linspace(-1, 1, out.numel(), device = out.device).view_as(out)
+ ).sum().backward()
+ return (out.detach(), Q.grad, K.grad, V.grad), calls
+
+
+CASES = [
+ # lengths, total (> sum = pad tail), heads, kv heads, causal, window, expected window_size
+ ((5, 3, 4), 12, 4, 4, True, None, (-1, 0)),
+ ((5, 3, 4), 12, 4, 2, True, None, (-1, 0)),
+ ((4, 4, 4), 12, 4, 1, True, None, None), # one run of equal lengths: one batched SDPA call
+ ((7,), 10, 4, 2, True, None, (-1, 0)),
+ ((6, 2, 5), 13, 4, 2, True, 3, (2, 0)),
+ ((6, 2, 5), 13, 4, 2, False, None, (-1, -1)),
+ ((1, 1, 9), 11, 2, 1, True, 4, (3, 0)),
+ ((300, 700, 24), 1100, 8, 2, True, None, (-1, 0)),
+ ((300, 700, 24), 1100, 8, 2, True, 128, (127, 0)),
+]
+
+
+@needs_varlen
[email protected]("dtype", [torch.bfloat16, torch.float16])
[email protected]("head_dim", [64, 128, 256])
[email protected]("lengths, total, n_heads, n_kv, is_causal, window, window_size", CASES)
+def test_varlen_matches_segments(
+ monkeypatch, dtype, head_dim, lengths, total, n_heads, n_kv, is_causal, window, window_size
+):
+ g = torch.Generator().manual_seed(3407)
+ shapes = ((1, n_heads, total, head_dim), (1, n_kv, total, head_dim), (1, n_kv, total, head_dim))
+ qkv = [torch.randn(s, generator = g).to("cuda", dtype) for s in shapes]
+ args = (lengths, total, n_heads, n_kv, qkv, is_causal, window)
+ segs, seg_calls = _run(monkeypatch, False, *args)
+ varlen, calls = _run(monkeypatch, True, *args)
+ assert seg_calls == [] and calls == ([] if window_size is None else [window_size])
+ for name, a, b in zip(("out", "dQ", "dK", "dV"), segs, varlen):
+ assert torch.isfinite(b).all(), name
+ torch.testing.assert_close(b.float(), a.float(), atol = 2e-2, rtol = 2e-2, msg = name)
+
+
+@needs_varlen
+def test_bidirectional_mha_head_dim_256_runs(monkeypatch):
+ g = torch.Generator().manual_seed(2)
+ qkv = [torch.randn(1, 8, 1024, 256, generator = g).cuda().bfloat16() for _ in range(3)]
+ out, calls = _run(monkeypatch, True, (300, 500, 224), 1024, 8, 8, qkv, is_causal = False)
+ assert calls == [] and all(torch.isfinite(x).all() for x in out)
+
+
+@needs_varlen
+ "backend, lengths, total, expect_varlen",
+ [
+ ("CUDNN_ATTENTION", (300, 700, 24), 1100, True), # mean 275: launches dominate
+ ("CUDNN_ATTENTION", (1500, 2000, 1800), 5300, False), # mean 1767: cuDNN per segment wins
+ ("FLASH_ATTENTION", (1500, 2000, 1800), 5300, True), # same kernel, fewer launches
+ ("FLASH_ATTENTION", (2048, 2048), 4096, False), # one run: one batched call already
+ ("MATH", (300, 700, 24), 1100, False), # caller's priority picked math
+ ("EFFICIENT_ATTENTION", (300, 700, 24), 1100, False),
+ ],
+)
+def test_varlen_only_where_it_beats_segments(monkeypatch, backend, lengths, total, expect_varlen):
+ from torch.nn.attention import SDPBackend as B
+
+ monkeypatch.setattr(torch, "_fused_sdp_choice", lambda *a, **k: int(getattr(B, backend)))
+ ad._sdpa_choice.cache_clear()
+ qkv = [torch.zeros(1, 4, total, 64, device = "cuda", dtype = torch.bfloat16) for _ in range(3)]
+ try:
+ _, calls = _run(monkeypatch, True, lengths, total, 4, 4, qkv)
+ finally:
+ ad._sdpa_choice.cache_clear()
+ assert calls == ([(-1, 0)] if expect_varlen else [])
+
+
+@needs_varlen
+def test_scale_is_forwarded(monkeypatch):
+ g = torch.Generator().manual_seed(0)
+ qkv = [torch.randn(1, 4, 40, 64, generator = g).cuda().bfloat16() for _ in range(3)]
+ kwargs = dict(sdpa_kwargs = {"scale": 0.3, "dropout_p": 0.0})
+ segs, _ = _run(monkeypatch, False, (25, 15), 40, 4, 4, qkv, **kwargs)
+ varlen, calls = _run(monkeypatch, True, (25, 15), 40, 4, 4, qkv, **kwargs)
+ assert calls == [(-1, 0)]
+ torch.testing.assert_close(varlen[0].float(), segs[0].float(), atol = 2e-2, rtol = 2e-2)
+
+
+@needs_varlen
+ "case",
+ [
+ "fp32",
+ "dropout",
+ "attn_mask",
+ "bidirectional_window",
+ "bidirectional_mha",
+ "deterministic",
+ "int32_overflow",
+ "flash_sdp_disabled",
+ "math_priority",
+ ],
+)
+def test_ineligible_calls_keep_the_segment_path(monkeypatch, case):
+ dtype = torch.float32 if case == "fp32" else torch.bfloat16
+ g = torch.Generator().manual_seed(1)
+ qkv = [torch.randn(1, 4, 24, 64, generator = g).to("cuda", dtype) for _ in range(3)]
+ kwargs = {}
+ if case == "dropout":
+ kwargs["sdpa_kwargs"] = {"dropout_p": 0.1}
+ torch.manual_seed(0)
+ elif case == "attn_mask":
+ kwargs["sdpa_kwargs"] = {"attn_mask": None, "scale": None}
+ elif case == "bidirectional_window":
+ kwargs.update(is_causal = False, window = 4)
+ elif case == "bidirectional_mha":
+ kwargs.update(is_causal = False)
+ elif case == "deterministic":
+ monkeypatch.setattr(torch, "are_deterministic_algorithms_enabled", lambda: True)
+ elif case == "int32_overflow":
+ monkeypatch.setattr(ad, "_varlen_backward_overflows_int32", lambda *a: True)
+ if case == "flash_sdp_disabled":
+ with sdpa_kernel([SDPBackend.EFFICIENT_ATTENTION, SDPBackend.MATH]):
+ _, calls = _run(monkeypatch, True, (10, 14), 24, 4, 4, qkv, **kwargs)
+ elif case == "math_priority":
+ ad._sdpa_choice.cache_clear()
+ with sdpa_kernel([SDPBackend.MATH, SDPBackend.FLASH_ATTENTION], set_priority = True):
+ _, calls = _run(monkeypatch, True, (10, 14), 24, 4, 4, qkv, **kwargs)
+ ad._sdpa_choice.cache_clear()
+ else:
+ _, calls = _run(monkeypatch, True, (10, 14), 24, 4, 4, qkv, **kwargs)
+ assert calls == []
+
+
+@needs_varlen
+def test_lengths_overrunning_the_row_keep_the_segment_path(monkeypatch):
+ # Malformed lengths (sum 30 > 24 tokens) must raise as before, not index past the row in the kernel.
+ calls = []
+
+ def stub(*args, **kwargs):
+ calls.append(1)
+ raise AssertionError("varlen_attn called with cu_seqlens past the row")
+
+ monkeypatch.setattr(ad, "_TORCH_VARLEN_ATTN", stub)
+ packing.clear_packed_caches()
+ g = torch.Generator().manual_seed(2)
+ Q, K, V = (torch.randn(1, 4, 24, 64, generator = g).to("cuda", torch.bfloat16) for _ in range(3))
+ config = ad.AttentionConfig(backend = ad.SDPA, n_kv_heads = 4, n_groups = 1)
+ context = ad.AttentionContext(
+ bsz = 1,
+ q_len = 24,
+ kv_seq_len = 24,
+ n_heads = 4,
+ head_dim = 64,
+ requires_grad = False,
+ seq_info = _seq_info((10, 14, 6), "cuda"),
+ attention_mask = None,
+ causal_mask = None,
+ is_causal = True,
+ sliding_window = None,
+ )
+ with pytest.raises(RuntimeError):
+ ad.run_attention(config = config, context = context, Q = Q, K = K, V = V)
+ assert calls == []
+
+
+def test_loader_needs_window_and_gqa(monkeypatch):
+ import sys
+
+ def old_varlen_attn(
+ query,
+ key,
+ value,
+ cu_seq_q,
+ cu_seq_k,
+ max_q,
+ max_k,
+ *,
+ is_causal = False,
+ ):
+ raise AssertionError("not called")
+
+ fake = types.ModuleType("torch.nn.attention.varlen")
+ fake.varlen_attn = old_varlen_attn
+ monkeypatch.setitem(sys.modules, "torch.nn.attention.varlen", fake)
+ monkeypatch.delenv("UNSLOTH_TORCH_VARLEN", raising = False)
+ assert ad._load_torch_varlen_attn() is None
+
+
+def test_kill_switch(monkeypatch):
+ monkeypatch.setenv("UNSLOTH_TORCH_VARLEN", "0")
+ assert ad._load_torch_varlen_attn() is None
diff --git a/unsloth/utils/attention_dispatch.py b/unsloth/utils/attention_dispatch.py
index 4fdccd2979a..cc225bd001a 100644
--- a/unsloth/utils/attention_dispatch.py
+++ b/unsloth/utils/attention_dispatch.py
@@ -14,6 +14,7 @@
from __future__ import annotations
+import functools
import os
from dataclasses import dataclass
from typing import Any, Optional, Tuple
@@ -87,6 +88,162 @@ _SDPA_PACKED_SEGMENTS = os.environ.get("UNSLOTH_SDPA_PACKED_SEGMENTS", "1").lowe
)
+def _load_torch_varlen_attn():
+ # Needs window_size (torch 2.11) and enable_gqa (2.12); older builds keep the per-segment path.
+ if os.environ.get("UNSLOTH_TORCH_VARLEN", "1").lower() in ("0", "false", "no", "off"):
+ return None
+ try:
+ import inspect
+ from torch.nn.attention.varlen import varlen_attn
+ except Exception:
+ return None
+ params = inspect.signature(varlen_attn).parameters
+ if "window_size" not in params or "enable_gqa" not in params:
+ return None
+ # Causal varlen always uses torch's bundled flash kernel, which some source builds omit.
+ if not getattr(torch.backends.cuda, "is_flash_attention_available", lambda: True)():
+ return None
+ return varlen_attn
+
+
+_TORCH_VARLEN_ATTN = _load_torch_varlen_attn()
+
+
+def _torch_varlen_takes(
+ Q: Tensor,
+ K: Tensor,
+ V: Tensor,
+ sdpa_kwargs: dict,
+ is_causal: bool,
+ sliding_window: Optional[int],
+) -> bool:
+ """True when torch's varlen kernel computes exactly what the per-segment SDPA path would."""
+ if _TORCH_VARLEN_ATTN is None or not Q.is_cuda or torch.version.hip is not None:
+ return False
+ if Q.dtype not in (torch.float16, torch.bfloat16) or K.dtype != Q.dtype or V.dtype != Q.dtype:
+ return False
+ if (
+ Q.shape[-1] > 256
+ or Q.shape[-1] % 8 != 0
+ or torch.cuda.get_device_capability(Q.device)[0] < 8
+ ):
+ return False
+ # A bidirectional window has no (left, right) spelling matching packed_block_mask.
+ if set(sdpa_kwargs) - {"scale", "dropout_p"} or sdpa_kwargs.get("dropout_p"):
+ return False
+ if sliding_window is not None and not is_causal:
+ return False
+ # torch 2.14 routes bidirectional MHA to cuDNN ragged on sm90/sm100: no head_dim 256 plan.
+ if not is_causal and Q.shape[1] == K.shape[1]:
+ return False
+ return not torch.are_deterministic_algorithms_enabled()
+
+
+# Above this mean segment length cuDNN's per-segment SDPA beats torch's varlen flash kernel (B200 sweep).
+_VARLEN_CUDNN_MAX_MEAN_LEN = 1024
+
+
[email protected]_cache(maxsize = 64)
+def _sdpa_choice(device, dtype, head_dim, n_heads, n_kv_heads, is_causal, enable_gqa, backends):
+ try:
+ q = torch.empty(1, n_heads, 128, head_dim, device = device, dtype = dtype)
+ k = torch.empty(1, n_kv_heads, 128, head_dim, device = device, dtype = dtype)
+ return int(torch._fused_sdp_choice(q, k, k, is_causal = is_causal, enable_gqa = enable_gqa))
+ except Exception:
+ return -1
+
+
+def _sdpa_backend_state(device, dtype, head_dim, n_heads, n_kv_heads, is_causal, enable_gqa):
+ """(flash SDPA enabled, SDPBackend SDPA would pick) for these shapes. Both reads break a dynamo graph."""
+ cuda = torch.backends.cuda
+ backends = (
+ cuda.flash_sdp_enabled(),
+ cuda.cudnn_sdp_enabled(),
+ cuda.mem_efficient_sdp_enabled(),
+ cuda.math_sdp_enabled(),
+ tuple(getattr(torch._C, "_get_sdp_priority_order", tuple)()),
+ )
+ choice = _sdpa_choice(
+ device, dtype, head_dim, n_heads, n_kv_heads, is_causal, enable_gqa, backends
+ )
+ return backends[0], choice
+
+
+if hasattr(torch.compiler, "assume_constant_result"):
+ _sdpa_backend_state = torch.compiler.assume_constant_result(_sdpa_backend_state)
+
+
+def _torch_varlen_beats_segments(
+ Q: Tensor, K: Tensor, lengths: Tuple[int, ...], n_groups: int, is_causal: bool
+) -> bool:
+ """Varlen saves one SDPA launch per run of equal lengths, but its flash kernel is slower per token
+ than cuDNN on long segments; with flash already chosen, the kernel is the same and only launches drop."""
+ lengths = [length for length in lengths if length > 0]
+ # Lengths overrunning the row would send cu_seqlens past the tensor; leave them to the old path's error.
+ if sum(lengths) != Q.shape[-2]:
+ return False
+ runs = sum(1 for i, length in enumerate(lengths) if i == 0 or lengths[i - 1] != length)
+ if runs <= 1:
+ return False
+ flash, choice = _sdpa_backend_state(
+ str(Q.device),
+ Q.dtype,
+ Q.shape[-1],
+ Q.shape[1],
+ K.shape[1],
+ is_causal,
+ n_groups != 1 and _sdpa_flash_takes_gqa(Q),
+ )
+ # varlen_attn calls flash directly: honour a disabled flash backend and math / efficient-first priorities.
+ from torch.nn.attention import SDPBackend
+
+ if not flash:
+ return False
+ if choice == int(SDPBackend.FLASH_ATTENTION):
+ return True
+ if choice == int(SDPBackend.CUDNN_ATTENTION):
+ return sum(lengths) / len(lengths) <= _VARLEN_CUDNN_MAX_MEAN_LEN
+ return False
+
+
+def _torch_varlen_packed(
+ Q: Tensor,
+ K: Tensor,
+ V: Tensor,
+ seq_info,
+ *,
+ n_groups: int,
+ is_causal: bool,
+ sliding_window: Optional[int],
+ scale: Optional[float],
+) -> Tensor:
+ """Q (1, H, T, D), K / V (1, H_kv, T, D) -> (1, T, H, D) over the row's segments, pad tail included."""
+ total = Q.shape[-2]
+ cu_seqlens, max_seqlen = cover_padded_cu_seqlens(seq_info, total)
+ # seq_info may live on the CPU.
+ cu_seqlens = cu_seqlens.to(device = Q.device, dtype = torch.int32, non_blocking = True)
+ if not is_causal:
+ window_size = (-1, -1)
+ elif sliding_window is not None:
+ window_size = (sliding_window - 1, 0) # packed_block_mask keeps k >= q - (window - 1)
+ else:
+ window_size = (-1, 0)
+ Q_f, K_f, V_f = (x[0].transpose(0, 1).contiguous() for x in (Q, K, V))
+ out = _TORCH_VARLEN_ATTN(
+ Q_f,
+ K_f,
+ V_f,
+ cu_seqlens,
+ cu_seqlens,
+ max_seqlen,
+ max_seqlen,
+ scale = scale,
+ window_size = window_size,
+ enable_gqa = n_groups != 1,
+ )
+ return out.unsqueeze(0)
+
+
def _sdpa_flash_takes_gqa(Q: Tensor) -> bool:
# Only flash / math take enable_gqa: expand K/V elsewhere (pre-sm80, fp32, wide heads, ROCm).
return (
@@ -590,11 +747,36 @@ def run_attention(
and Q.shape[0] == 1
and Q.shape[-2] == K.shape[-2]
):
+ lengths = packed_segment_lengths(context.seq_info, K.shape[-2])
+ if (
+ _torch_varlen_takes(Q, K, V, sdpa_kwargs, context.is_causal, sliding_window)
+ and _torch_varlen_beats_segments(Q, K, lengths, config.n_groups, context.is_causal)
+ ) and not (
+ # Varlen backward may share flash-attn 2's int32 dq_accum limit.
+ requires_grad
+ and not _VARLEN_INT32_GUARD_DISABLED
+ and _varlen_backward_overflows_int32(
+ cover_padded_cu_seqlens(context.seq_info, Q.shape[-2])[0].numel() - 1,
+ Q.shape[-2],
+ n_heads,
+ head_dim,
+ )
+ ):
+ return _torch_varlen_packed(
+ Q,
+ K,
+ V,
+ context.seq_info,
+ n_groups = config.n_groups,
+ is_causal = context.is_causal,
+ sliding_window = sliding_window,
+ scale = sdpa_kwargs.get("scale"),
+ )
out = _sdpa_packed_segments(
Q,
K,
V,
- packed_segment_lengths(context.seq_info, K.shape[-2]),
+ lengths,
n_groups = config.n_groups,
is_causal = context.is_causal,
sliding_window = sliding_window,