Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Padding-free: call flash varlen through a plain autograd.Function in eager (#15160)
Daniel Han71f705bf64d9
diff --git a/tests/test_lean_flash_varlen.py b/tests/test_lean_flash_varlen.pynew file mode 100644index 00000000000..9ffdc48f7c5--- /dev/null+++ b/tests/test_lean_flash_varlen.py@@ -0,0 +1,85 @@+# Unsloth Zoo - Utilities for Unsloth+# Copyright 2023-present Daniel Han-Chen, Michael Han-Chen & the Unsloth team. All rights reserved.+#+# This program is free software: you can redistribute it and/or modify+# it under the terms of the GNU Affero General Public License as published+# by the Free Software Foundation, either version 3 of the License, or+# (at your option) any later version.+#+# This program is distributed in the hope that it will be useful,+# but WITHOUT ANY WARRANTY; without even the implied warranty of+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+# GNU Affero General Public License for more details.+#+# You should have received a copy of the GNU Affero General Public License+# along with this program. If not, see <https://www.gnu.org/licenses/>.++"""The lean eager varlen path must match torch's varlen_attn: same aten flash kernels, outputs and grads."""++import pytest+import torch++ad = pytest.importorskip("unsloth.utils.attention_dispatch")++needs_lean = pytest.mark.skipif(+ not torch.cuda.is_available() or not getattr(ad, "_LEAN_VARLEN", False),+ reason = "needs CUDA and torch varlen_attn",+)+++def _run(fn, q, k, v, cu, mx, window, gqa, g):+ q, k, v = (x.detach().clone().requires_grad_(True) for x in (q, k, v))+ out = fn(q, k, v, cu, mx, window, gqa)+ out.backward(g)+ return out.detach(), q.grad, k.grad, v.grad+++def _torch(q, k, v, cu, mx, window, gqa):+ return ad._TORCH_VARLEN_ATTN(+ q, k, v, cu, cu, mx, mx, scale = None, window_size = window, enable_gqa = gqa+ )+++def _lean(q, k, v, cu, mx, window, gqa):+ return ad._LeanFlashVarlen.apply(q, k, v, cu, mx, window == (-1, 0), None, window[0], window[1])+++@needs_lean[email protected]("dtype", [torch.bfloat16, torch.float16])[email protected]("hq,hkv", [(8, 8), (16, 8)])[email protected]("window", [(-1, 0), (31, 0)])+def test_lean_matches_torch_varlen(dtype, hq, hkv, window):+ if dtype == torch.bfloat16 and not torch.cuda.is_bf16_supported():+ pytest.skip("no bf16")+ torch.manual_seed(0)+ lens = [37, 128, 5, 200, 64] # the last segment stands in for the collator's pad tail+ T, D = sum(lens), 128+ cu = torch.tensor([0] + list(torch.tensor(lens).cumsum(0)), dtype = torch.int32, device = "cuda")+ q = torch.randn(T, hq, D, device = "cuda", dtype = dtype)+ k = torch.randn(T, hkv, D, device = "cuda", dtype = dtype)+ v = torch.randn(T, hkv, D, device = "cuda", dtype = dtype)+ g = torch.randn(T, hq, D, device = "cuda", dtype = dtype)+ ref = _run(_torch, q, k, v, cu, max(lens), window, hq != hkv, g)+ ref2 = _run(_torch, q, k, v, cu, max(lens), window, hq != hkv, g)+ got = _run(_lean, q, k, v, cu, max(lens), window, hq != hkv, g)+ assert torch.equal(got[0], ref[0]) # forward: identical kernel, identical bits+ for a, b, c in zip(got[1:], ref[1:], ref2[1:]):+ noise = (+ (b.float() - c.float()).abs().max().item()+ ) # flash backward accumulates dq atomically+ assert (a.float() - b.float()).abs().max().item() <= max(noise, 1e-6) * 2 + 1e-3+++@needs_lean+def test_dispatch_uses_lean_in_eager_and_torch_when_compiling(monkeypatch):+ calls = []+ real = ad._LeanFlashVarlen.apply+ monkeypatch.setattr(ad._LeanFlashVarlen, "apply", lambda *a: calls.append(1) or real(*a))+ q = torch.randn(64, 4, 64, device = "cuda", dtype = torch.float16)+ cu = torch.tensor([0, 30, 64], dtype = torch.int32, device = "cuda")+ ad._varlen_attn_call(q, q, q, cu, 34, scale = None, window_size = (-1, 0), enable_gqa = False)+ expect = 0 if ad._varlen_can_use_cudnn(q, q, q, 34, [-1, 0], False) else 1+ assert len(calls) == expect+ monkeypatch.setattr(torch.compiler, "is_compiling", lambda: True)+ ad._varlen_attn_call(q, q, q, cu, 34, scale = None, window_size = (-1, 0), enable_gqa = False)+ assert len(calls) == expectdiff --git a/unsloth/utils/attention_dispatch.py b/unsloth/utils/attention_dispatch.pyindex d39c8eb99c6..304a832aade 100644--- a/unsloth/utils/attention_dispatch.py+++ b/unsloth/utils/attention_dispatch.py@@ -323,6 +323,116 @@ def _load_torch_varlen_attn():_TORCH_VARLEN_ATTN = _load_torch_varlen_attn()+# varlen_attn is a torch.library custom op: per call it pays Python custom-op dispatch twice (forward+# and backward), a cuDNN eligibility check and a zeros launch for its rng state. Padding-free training+# runs it once per layer per micro-batch, which on a launch-bound step costs more than the padding it+# saves. Eager calls that torch would send to flash anyway take the same aten flash kernels through+# a plain autograd.Function. UNSLOTH_LEAN_VARLEN=0 restores varlen_attn everywhere.+_LEAN_VARLEN = _TORCH_VARLEN_ATTN is not None and os.environ.get(+ "UNSLOTH_LEAN_VARLEN", "1"+).lower() not in ("0", "false", "no", "off")+if _LEAN_VARLEN:+ try:+ from torch.nn.attention.varlen import _can_use_cudnn as _varlen_can_use_cudnn+ except Exception:+ # torch 2.12 / 2.13 decide per device only (_should_use_cudnn, False on every GPU in 2.13).+ try:+ from torch.nn.attention.varlen import _should_use_cudnn as _varlen_should_use_cudnn+ def _varlen_can_use_cudnn(+ query,+ key,+ value,+ max_q,+ window_size,+ enable_gqa = False,+ ):+ return query.is_cuda and _varlen_should_use_cudnn(query.device.index)++ except Exception:+ _LEAN_VARLEN = False+++class _LeanFlashVarlen(torch.autograd.Function):+ @staticmethod+ def forward(+ ctx, query, key, value, cu_seqlens, max_seqlen, is_causal, scale, window_left, window_right+ ):+ out, lse, philox_seed, philox_offset, _ = torch.ops.aten._flash_attention_forward(+ query,+ key,+ value,+ cu_seqlens,+ cu_seqlens,+ max_seqlen,+ max_seqlen,+ 0.0,+ is_causal,+ return_debug_mask = False,+ scale = scale,+ window_size_left = window_left,+ window_size_right = window_right,+ )+ ctx.save_for_backward(query, key, value, cu_seqlens, out, lse, philox_seed, philox_offset)+ ctx.max_seqlen, ctx.is_causal, ctx.scale = max_seqlen, is_causal, scale+ ctx.window = (window_left, window_right)+ return out++ @staticmethod+ def backward(ctx, grad_out):+ query, key, value, cu_seqlens, out, lse, philox_seed, philox_offset = ctx.saved_tensors+ dq, dk, dv = torch.ops.aten._flash_attention_backward(+ grad_out.contiguous(),+ query,+ key,+ value,+ out,+ lse,+ cu_seqlens,+ cu_seqlens,+ ctx.max_seqlen,+ ctx.max_seqlen,+ 0.0,+ ctx.is_causal,+ philox_seed,+ philox_offset,+ scale = ctx.scale,+ window_size_left = ctx.window[0],+ window_size_right = ctx.window[1],+ )+ return dq, dk, dv, None, None, None, None, None, None+++def _varlen_attn_call(Q_f, K_f, V_f, cu_seqlens, max_seqlen, *, scale, window_size, enable_gqa):+ if (+ _LEAN_VARLEN+ and not torch.compiler.is_compiling()+ and not _varlen_can_use_cudnn(Q_f, K_f, V_f, max_seqlen, list(window_size), enable_gqa)+ ):+ return _LeanFlashVarlen.apply(+ Q_f,+ K_f,+ V_f,+ cu_seqlens,+ max_seqlen,+ window_size == (-1, 0),+ scale,+ window_size[0],+ window_size[1],+ )+ return _TORCH_VARLEN_ATTN(+ Q_f,+ K_f,+ V_f,+ cu_seqlens,+ cu_seqlens,+ max_seqlen,+ max_seqlen,+ scale = scale,+ window_size = window_size,+ enable_gqa = enable_gqa,+ )++from .flash_impl import flash_impl_allows # noqa: E402@@ -638,14 +748,13 @@ def _torch_varlen_packed(window_size = (sliding_window - 1, 0) # packed_block_mask keeps k >= q - (window - 1)else:window_size = (-1, 0)- Q_f, K_f, V_f = (x[0].transpose(0, 1).contiguous() for x in (Q, K, V))- out = _TORCH_VARLEN_ATTN(+ # squeeze, not [0]: select's backward allocates and zero-fills a full gradient per tensor.+ Q_f, K_f, V_f = (x.squeeze(0).transpose(0, 1).contiguous() for x in (Q, K, V))+ out = _varlen_attn_call(Q_f,K_f,V_f,cu_seqlens,- cu_seqlens,- max_seqlen,max_seqlen,scale = scale,window_size = window_size,