Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Padding-free: call flash varlen through a plain autograd.Function in eager (#15160)

Daniel Han71f705bf64d9
diff --git a/tests/test_lean_flash_varlen.py b/tests/test_lean_flash_varlen.py
new file mode 100644
index 00000000000..9ffdc48f7c5
--- /dev/null
+++ b/tests/test_lean_flash_varlen.py
@@ -0,0 +1,85 @@
+# Unsloth Zoo - Utilities for Unsloth
+# Copyright 2023-present Daniel Han-Chen, Michael Han-Chen & the Unsloth team. All rights reserved.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+"""The lean eager varlen path must match torch's varlen_attn: same aten flash kernels, outputs and grads."""
+
+import pytest
+import torch
+
+ad = pytest.importorskip("unsloth.utils.attention_dispatch")
+
+needs_lean = pytest.mark.skipif(
+ not torch.cuda.is_available() or not getattr(ad, "_LEAN_VARLEN", False),
+ reason = "needs CUDA and torch varlen_attn",
+)
+
+
+def _run(fn, q, k, v, cu, mx, window, gqa, g):
+ q, k, v = (x.detach().clone().requires_grad_(True) for x in (q, k, v))
+ out = fn(q, k, v, cu, mx, window, gqa)
+ out.backward(g)
+ return out.detach(), q.grad, k.grad, v.grad
+
+
+def _torch(q, k, v, cu, mx, window, gqa):
+ return ad._TORCH_VARLEN_ATTN(
+ q, k, v, cu, cu, mx, mx, scale = None, window_size = window, enable_gqa = gqa
+ )
+
+
+def _lean(q, k, v, cu, mx, window, gqa):
+ return ad._LeanFlashVarlen.apply(q, k, v, cu, mx, window == (-1, 0), None, window[0], window[1])
+
+
+@needs_lean
[email protected]("dtype", [torch.bfloat16, torch.float16])
[email protected]("hq,hkv", [(8, 8), (16, 8)])
[email protected]("window", [(-1, 0), (31, 0)])
+def test_lean_matches_torch_varlen(dtype, hq, hkv, window):
+ if dtype == torch.bfloat16 and not torch.cuda.is_bf16_supported():
+ pytest.skip("no bf16")
+ torch.manual_seed(0)
+ lens = [37, 128, 5, 200, 64] # the last segment stands in for the collator's pad tail
+ T, D = sum(lens), 128
+ cu = torch.tensor([0] + list(torch.tensor(lens).cumsum(0)), dtype = torch.int32, device = "cuda")
+ q = torch.randn(T, hq, D, device = "cuda", dtype = dtype)
+ k = torch.randn(T, hkv, D, device = "cuda", dtype = dtype)
+ v = torch.randn(T, hkv, D, device = "cuda", dtype = dtype)
+ g = torch.randn(T, hq, D, device = "cuda", dtype = dtype)
+ ref = _run(_torch, q, k, v, cu, max(lens), window, hq != hkv, g)
+ ref2 = _run(_torch, q, k, v, cu, max(lens), window, hq != hkv, g)
+ got = _run(_lean, q, k, v, cu, max(lens), window, hq != hkv, g)
+ assert torch.equal(got[0], ref[0]) # forward: identical kernel, identical bits
+ for a, b, c in zip(got[1:], ref[1:], ref2[1:]):
+ noise = (
+ (b.float() - c.float()).abs().max().item()
+ ) # flash backward accumulates dq atomically
+ assert (a.float() - b.float()).abs().max().item() <= max(noise, 1e-6) * 2 + 1e-3
+
+
+@needs_lean
+def test_dispatch_uses_lean_in_eager_and_torch_when_compiling(monkeypatch):
+ calls = []
+ real = ad._LeanFlashVarlen.apply
+ monkeypatch.setattr(ad._LeanFlashVarlen, "apply", lambda *a: calls.append(1) or real(*a))
+ q = torch.randn(64, 4, 64, device = "cuda", dtype = torch.float16)
+ cu = torch.tensor([0, 30, 64], dtype = torch.int32, device = "cuda")
+ ad._varlen_attn_call(q, q, q, cu, 34, scale = None, window_size = (-1, 0), enable_gqa = False)
+ expect = 0 if ad._varlen_can_use_cudnn(q, q, q, 34, [-1, 0], False) else 1
+ assert len(calls) == expect
+ monkeypatch.setattr(torch.compiler, "is_compiling", lambda: True)
+ ad._varlen_attn_call(q, q, q, cu, 34, scale = None, window_size = (-1, 0), enable_gqa = False)
+ assert len(calls) == expect
diff --git a/unsloth/utils/attention_dispatch.py b/unsloth/utils/attention_dispatch.py
index d39c8eb99c6..304a832aade 100644
--- a/unsloth/utils/attention_dispatch.py
+++ b/unsloth/utils/attention_dispatch.py
@@ -323,6 +323,116 @@ def _load_torch_varlen_attn():
_TORCH_VARLEN_ATTN = _load_torch_varlen_attn()
+# varlen_attn is a torch.library custom op: per call it pays Python custom-op dispatch twice (forward
+# and backward), a cuDNN eligibility check and a zeros launch for its rng state. Padding-free training
+# runs it once per layer per micro-batch, which on a launch-bound step costs more than the padding it
+# saves. Eager calls that torch would send to flash anyway take the same aten flash kernels through
+# a plain autograd.Function. UNSLOTH_LEAN_VARLEN=0 restores varlen_attn everywhere.
+_LEAN_VARLEN = _TORCH_VARLEN_ATTN is not None and os.environ.get(
+ "UNSLOTH_LEAN_VARLEN", "1"
+).lower() not in ("0", "false", "no", "off")
+if _LEAN_VARLEN:
+ try:
+ from torch.nn.attention.varlen import _can_use_cudnn as _varlen_can_use_cudnn
+ except Exception:
+ # torch 2.12 / 2.13 decide per device only (_should_use_cudnn, False on every GPU in 2.13).
+ try:
+ from torch.nn.attention.varlen import _should_use_cudnn as _varlen_should_use_cudnn
+ def _varlen_can_use_cudnn(
+ query,
+ key,
+ value,
+ max_q,
+ window_size,
+ enable_gqa = False,
+ ):
+ return query.is_cuda and _varlen_should_use_cudnn(query.device.index)
+
+ except Exception:
+ _LEAN_VARLEN = False
+
+
+class _LeanFlashVarlen(torch.autograd.Function):
+ @staticmethod
+ def forward(
+ ctx, query, key, value, cu_seqlens, max_seqlen, is_causal, scale, window_left, window_right
+ ):
+ out, lse, philox_seed, philox_offset, _ = torch.ops.aten._flash_attention_forward(
+ query,
+ key,
+ value,
+ cu_seqlens,
+ cu_seqlens,
+ max_seqlen,
+ max_seqlen,
+ 0.0,
+ is_causal,
+ return_debug_mask = False,
+ scale = scale,
+ window_size_left = window_left,
+ window_size_right = window_right,
+ )
+ ctx.save_for_backward(query, key, value, cu_seqlens, out, lse, philox_seed, philox_offset)
+ ctx.max_seqlen, ctx.is_causal, ctx.scale = max_seqlen, is_causal, scale
+ ctx.window = (window_left, window_right)
+ return out
+
+ @staticmethod
+ def backward(ctx, grad_out):
+ query, key, value, cu_seqlens, out, lse, philox_seed, philox_offset = ctx.saved_tensors
+ dq, dk, dv = torch.ops.aten._flash_attention_backward(
+ grad_out.contiguous(),
+ query,
+ key,
+ value,
+ out,
+ lse,
+ cu_seqlens,
+ cu_seqlens,
+ ctx.max_seqlen,
+ ctx.max_seqlen,
+ 0.0,
+ ctx.is_causal,
+ philox_seed,
+ philox_offset,
+ scale = ctx.scale,
+ window_size_left = ctx.window[0],
+ window_size_right = ctx.window[1],
+ )
+ return dq, dk, dv, None, None, None, None, None, None
+
+
+def _varlen_attn_call(Q_f, K_f, V_f, cu_seqlens, max_seqlen, *, scale, window_size, enable_gqa):
+ if (
+ _LEAN_VARLEN
+ and not torch.compiler.is_compiling()
+ and not _varlen_can_use_cudnn(Q_f, K_f, V_f, max_seqlen, list(window_size), enable_gqa)
+ ):
+ return _LeanFlashVarlen.apply(
+ Q_f,
+ K_f,
+ V_f,
+ cu_seqlens,
+ max_seqlen,
+ window_size == (-1, 0),
+ scale,
+ window_size[0],
+ window_size[1],
+ )
+ return _TORCH_VARLEN_ATTN(
+ Q_f,
+ K_f,
+ V_f,
+ cu_seqlens,
+ cu_seqlens,
+ max_seqlen,
+ max_seqlen,
+ scale = scale,
+ window_size = window_size,
+ enable_gqa = enable_gqa,
+ )
+
+
from .flash_impl import flash_impl_allows # noqa: E402
@@ -638,14 +748,13 @@ def _torch_varlen_packed(
window_size = (sliding_window - 1, 0) # packed_block_mask keeps k >= q - (window - 1)
else:
window_size = (-1, 0)
- Q_f, K_f, V_f = (x[0].transpose(0, 1).contiguous() for x in (Q, K, V))
- out = _TORCH_VARLEN_ATTN(
+ # squeeze, not [0]: select's backward allocates and zero-fills a full gradient per tensor.
+ Q_f, K_f, V_f = (x.squeeze(0).transpose(0, 1).contiguous() for x in (Q, K, V))
+ out = _varlen_attn_call(
Q_f,
K_f,
V_f,
cu_seqlens,
- cu_seqlens,
- max_seqlen,
max_seqlen,
scale = scale,
window_size = window_size,