Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Run packed rows on the transformers modeling path through varlen flash attention (#13234)

Daniel Hanc554fbfe22c7
diff --git a/tests/test_hf_packed_attention.py b/tests/test_hf_packed_attention.py
new file mode 100644
index 00000000000..398acd779cf
--- /dev/null
+++ b/tests/test_hf_packed_attention.py
@@ -0,0 +1,508 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+"""Packed rows on the transformers modeling path run varlen attention, and nothing else changes."""
+
+import inspect
+import types
+
+import pytest
+import torch
+from real_accelerator import has_real_cuda # tests/_shared, on sys.path via tests/conftest.py
+
+from unsloth.utils import hf_packed_attention as hpa
+
+
+def _block_causal(lengths, device = "cpu"):
+ segment = torch.repeat_interleave(torch.arange(len(lengths)), torch.tensor(lengths)).to(device)
+ return ((segment[:, None] == segment[None, :]).tril())[None, None]
+
+
+def _cu(lengths, device = "cpu"):
+ return torch.tensor(
+ [0] + list(torch.tensor(lengths).cumsum(0)), dtype = torch.int32, device = device
+ )
+
+
[email protected](autouse = True)
+def _fresh_state():
+ saved = hpa._ORIG_SDPA[0]
+ hpa._MASK_CHECK[:] = [None, None, None]
+ yield
+ hpa._MASK_CHECK[:] = [None, None, None]
+ hpa._ORIG_SDPA[0] = saved
+
+
+def test_block_causal_mask_is_recognised():
+ lengths = [3, 5, 2]
+ assert hpa._mask_is_block_causal(_block_causal(lengths), _cu(lengths), 10)
+
+
+ "edit",
+ ["cross_document", "bidirectional", "window", "plain_causal", "pad_tail", "float"],
+)
+def test_any_other_mask_keeps_the_masked_path(edit):
+ lengths = [3, 5, 2]
+ mask, cu, total = _block_causal(lengths), _cu(lengths), 10
+ if edit == "cross_document":
+ mask = mask.clone()
+ mask[0, 0, 4, 1] = True
+ elif edit == "bidirectional":
+ mask = mask.clone()
+ mask[0, 0, 3, 4] = True
+ elif edit == "window":
+ mask = mask.clone()
+ mask[0, 0, 7, 3] = False
+ elif edit == "plain_causal":
+ mask = torch.ones(10, 10, dtype = torch.bool).tril()[None, None]
+ elif edit == "pad_tail":
+ cu = _cu([3, 5])
+ elif edit == "float":
+ mask = torch.where(mask, 0.0, float("-inf"))
+ assert not hpa._mask_is_block_causal(mask, cu, total)
+
+
+def test_mask_verdict_is_cached_per_mask_object():
+ lengths = [4, 4]
+ mask, cu = _block_causal(lengths), _cu(lengths)
+ assert hpa._mask_is_block_causal(mask, cu, 8)
+ # Same object: the cached verdict stands. A new object is checked again.
+ assert hpa._mask_is_block_causal(mask, cu, 8)
+ other = torch.ones(8, 8, dtype = torch.bool).tril()[None, None]
+ assert not hpa._mask_is_block_causal(other, cu, 8)
+
+
+def _calls():
+ seen = []
+
+ def orig(
+ module,
+ q,
+ k,
+ v,
+ mask,
+ dropout = 0.0,
+ scaling = None,
+ is_causal = None,
+ **kwargs,
+ ):
+ seen.append(kwargs)
+ return "orig", None
+
+ return seen, orig
+
+
+def test_unpacked_calls_go_straight_through(monkeypatch):
+ seen, orig = _calls()
+ hpa._ORIG_SDPA[0] = orig
+ q = torch.zeros(2, 4, 6, 8)
+ out = hpa._sdpa_packed_varlen(types.SimpleNamespace(), q, q, q, None, scaling = 0.3, foo = 1)
+ assert out == ("orig", None) and seen == [{"foo": 1}]
+
+
+_CUDA = pytest.mark.skipif(not has_real_cuda(), reason = "needs a CUDA device")
+
+
+@_CUDA
+ "why",
+ [
+ "none",
+ "sliding",
+ "softcap",
+ "sinks",
+ "fp32",
+ "dropout",
+ "batched",
+ "cpu",
+ "head_dim_512",
+ "mla_value_width",
+ "autocast_dtype",
+ "paged_cache",
+ "no_mask",
+ "module_softcap",
+ "not_causal",
+ "positional",
+ "kv_batched",
+ ],
+)
+def test_packed_calls_that_varlen_cannot_express_fall_back(monkeypatch, why):
+ """Each reason alone sends the call to the wrapped sdpa; "none" is the control that takes varlen."""
+ from unsloth.utils.attention_dispatch import HAS_FLASH_ATTENTION, HAS_XFORMERS
+
+ if not (HAS_FLASH_ATTENTION or HAS_XFORMERS or hpa._torch_varlen(torch.device("cuda", 0))):
+ pytest.skip("needs a varlen kernel")
+ seen, orig = _calls()
+ hpa._ORIG_SDPA[0] = orig
+ monkeypatch.delenv("UNSLOTH_HF_PACKED_VARLEN", raising = False)
+ lengths, D = [3, 3], 64
+ module = types.SimpleNamespace(is_causal = True)
+ device, dtype, B = "cuda", torch.bfloat16, 1
+ kwargs = {"packed_seq_lengths": torch.tensor(lengths, dtype = torch.int32)}
+ dropout, mask, v_width = 0.0, _block_causal(lengths, "cuda"), D
+ if why == "sliding":
+ module.sliding_window = 4
+ elif why == "softcap":
+ module.config = types.SimpleNamespace(attn_logit_softcapping = 50.0)
+ elif why == "module_softcap":
+ module.attn_logit_softcapping = 30.0
+ elif why == "sinks":
+ module.sinks = torch.zeros(4)
+ elif why == "fp32":
+ dtype = torch.float32
+ elif why == "dropout":
+ dropout = 0.1
+ elif why == "batched":
+ B = 2
+ elif why == "cpu":
+ device, mask = "cpu", _block_causal(lengths)
+ elif why == "head_dim_512":
+ D = v_width = 512
+ elif why == "mla_value_width":
+ D, v_width = 192, 128
+ elif why == "paged_cache":
+ kwargs["cache"] = object()
+ elif why == "no_mask":
+ mask = None
+ q = torch.randn(B, 4, 6, D, device = device, dtype = dtype)
+ v = torch.randn(B, 4, 6, v_width, device = device, dtype = dtype)
+ k = q
+ if why == "kv_batched":
+ k, v = q.expand(2, -1, -1, -1), v.expand(2, -1, -1, -1)
+ before = dict(hpa.HF_PACKED_ATTENTION_STATS)
+ if why == "autocast_dtype":
+ with torch.autocast("cuda", dtype = torch.float16):
+ out = hpa._sdpa_packed_varlen(module, q, k, v, mask, dropout = dropout, **kwargs)
+ elif why == "not_causal":
+ out = hpa._sdpa_packed_varlen(module, q, k, v, mask, is_causal = False, **kwargs)
+ elif why == "positional":
+ out = hpa._sdpa_packed_varlen(module, q, k, v, mask, 0.0, **kwargs)
+ else:
+ out = hpa._sdpa_packed_varlen(module, q, k, v, mask, dropout = dropout, **kwargs)
+ if why == "none":
+ assert out[0].shape == (1, 6, 4, D) and seen == []
+ assert hpa.HF_PACKED_ATTENTION_STATS["fast"] == before["fast"] + 1
+ else:
+ assert out == ("orig", None)
+ assert hpa.HF_PACKED_ATTENTION_STATS["fallback"] == before["fallback"] + 1
+
+
+def test_an_edited_mask_is_checked_again():
+ lengths = [3, 3]
+ mask, cu = _block_causal(lengths).clone(), _cu(lengths)
+ assert hpa._mask_is_block_causal(mask, cu, 6)
+ mask[0, 0, 3, 0] = True
+ assert not hpa._mask_is_block_causal(mask, cu, 6)
+ with torch.inference_mode():
+ frozen, frozen_cu = _block_causal(lengths), _cu(lengths)
+ assert hpa._mask_is_block_causal(frozen, frozen_cu, 6)
+
+
+def test_a_missing_mask_is_not_block_causal():
+ assert not hpa._mask_is_block_causal(None, _cu([6]), 6)
+
+
+def _plain_sdpa(
+ module,
+ q,
+ k,
+ v,
+ mask,
+ dropout = 0.0,
+ scaling = None,
+ is_causal = None,
+ **kwargs,
+):
+ # Not transformers' sdpa_attention_forward: compiling that after importing unsloth trips a Dynamo
+ # guard bug on torch <= 2.13, with or without this wrapper.
+ out = torch.nn.functional.scaled_dot_product_attention(q, k, v, attn_mask = mask, scale = scaling)
+ return out.transpose(1, 2), None
+
+
+@_CUDA
+def test_fullgraph_compile_keeps_the_wrapped_sdpa(monkeypatch):
+ sdpa_attention_forward = _plain_sdpa
+ hpa._ORIG_SDPA[0] = sdpa_attention_forward
+ monkeypatch.delenv("UNSLOTH_HF_PACKED_VARLEN", raising = False)
+ lengths = [3, 3]
+ module = types.SimpleNamespace(is_causal = True, num_key_value_groups = 1, training = False)
+ q = torch.randn(1, 4, 6, 64, device = "cuda", dtype = torch.bfloat16)
+ mask, psl = _block_causal(lengths, "cuda"), torch.tensor(lengths, dtype = torch.int32)
+ fn = torch.compile(
+ lambda q: hpa._sdpa_packed_varlen(module, q, q, q, mask, packed_seq_lengths = psl)[0],
+ backend = "eager",
+ fullgraph = True,
+ )
+ torch.testing.assert_close(fn(q), sdpa_attention_forward(module, q, q, q, mask)[0])
+
+
+@_CUDA
[email protected]("route", ["torch", "dispatch"])
+def test_reentrant_checkpoint_of_grouped_heads(monkeypatch, route):
+ from unsloth.utils.attention_dispatch import HAS_FLASH_ATTENTION, HAS_XFORMERS
+
+ if route == "torch" and hpa._torch_varlen(torch.device("cuda", 0)) is None:
+ pytest.skip("this torch has no window_size / enable_gqa varlen_attn")
+ if route == "dispatch":
+ if not (HAS_FLASH_ATTENTION or HAS_XFORMERS):
+ pytest.skip("needs xformers or flash-attn")
+ monkeypatch.setattr(hpa, "_torch_varlen", lambda device: None)
+ from torch.utils.checkpoint import checkpoint
+ from transformers.integrations.sdpa_attention import sdpa_attention_forward
+
+ hpa._ORIG_SDPA[0] = sdpa_attention_forward
+ monkeypatch.delenv("UNSLOTH_HF_PACKED_VARLEN", raising = False)
+ lengths = [40, 24]
+ module = types.SimpleNamespace(is_causal = True, num_key_value_groups = 4, training = True)
+ q = torch.randn(1, 8, 64, 64, device = "cuda", dtype = torch.bfloat16, requires_grad = True)
+ k = torch.randn(1, 2, 64, 64, device = "cuda", dtype = torch.bfloat16, requires_grad = True)
+ v = torch.randn(1, 2, 64, 64, device = "cuda", dtype = torch.bfloat16, requires_grad = True)
+ mask, psl = _block_causal(lengths, "cuda"), torch.tensor(lengths, dtype = torch.int32)
+ fn = lambda q, k, v: hpa._sdpa_packed_varlen(module, q, k, v, mask, packed_seq_lengths = psl)[0]
+ out = checkpoint(fn, q, k, v, use_reentrant = True)
+ out.float().sum().backward()
+ ref = sdpa_attention_forward(module, q, k, v, mask)[0]
+ torch.testing.assert_close(out.float(), ref.float(), atol = 2e-2, rtol = 2e-2)
+
+
+def test_kill_switch(monkeypatch):
+ seen, orig = _calls()
+ hpa._ORIG_SDPA[0] = orig
+ monkeypatch.setenv("UNSLOTH_HF_PACKED_VARLEN", "0")
+ q = torch.zeros(1, 4, 6, 8, dtype = torch.bfloat16)
+ lengths = torch.tensor([3, 3], dtype = torch.int32)
+ out = hpa._sdpa_packed_varlen(
+ types.SimpleNamespace(), q, q, q, _block_causal([3, 3]), packed_seq_lengths = lengths
+ )
+ assert out == ("orig", None)
+ assert not hpa.enable_hf_packed_attention()
+
+
+def test_install_forwards_router_sentinels_and_runs_once(monkeypatch):
+ mapping_mod = pytest.importorskip("transformers.modeling_utils")
+ mapping = mapping_mod.ALL_ATTENTION_FUNCTIONS
+
+ def router(*args, **kwargs):
+ return None
+
+ router._unsloth_gemma4_flash = True
+ monkeypatch.setitem(mapping, "sdpa", router)
+ hpa._ORIG_SDPA[0] = None
+ monkeypatch.delenv("UNSLOTH_HF_PACKED_VARLEN", raising = False)
+ assert hpa.enable_hf_packed_attention()
+ installed = mapping["sdpa"]
+ assert installed is hpa._sdpa_packed_varlen and hpa._ORIG_SDPA[0] is router
+ # A zoo router re-running on the next load sees its own sentinel and leaves the chain alone.
+ assert getattr(installed, "_unsloth_gemma4_flash", False)
+ # A router stacked above later never gets wrapped a second time.
+ monkeypatch.setitem(mapping, "sdpa", router)
+ assert hpa.enable_hf_packed_attention()
+ assert mapping["sdpa"] is router and hpa._ORIG_SDPA[0] is router
+
+
+@_CUDA
[email protected]("route", ["torch", "torch_2_11", "dispatch"])
[email protected]("n_kv", [8, 2])
+def test_varlen_matches_sdpa_over_the_dense_mask(monkeypatch, n_kv, route):
+ from unsloth.utils.attention_dispatch import HAS_FLASH_ATTENTION, HAS_XFORMERS
+
+ if route != "dispatch" and hpa._torch_varlen(torch.device("cuda", 0)) is None:
+ pytest.skip("this torch has no window_size varlen_attn")
+ if route == "torch_2_11":
+ # torch 2.11's varlen_attn has scale / window_size but no enable_gqa.
+ native = hpa._TORCH_VARLEN[0]
+
+ def without_gqa(*args, **kwargs):
+ assert "enable_gqa" not in kwargs
+ return native(*args, **kwargs)
+
+ monkeypatch.setattr(hpa, "_TORCH_VARLEN", [without_gqa])
+ monkeypatch.setattr(hpa, "_TORCH_VARLEN_GQA", [False])
+ if route == "dispatch":
+ if not (HAS_FLASH_ATTENTION or HAS_XFORMERS):
+ pytest.skip("needs xformers or flash-attn")
+ monkeypatch.setattr(hpa, "_torch_varlen", lambda device: None)
+ from transformers.integrations.sdpa_attention import sdpa_attention_forward
+
+ hpa._ORIG_SDPA[0] = sdpa_attention_forward
+ monkeypatch.delenv("UNSLOTH_HF_PACKED_VARLEN", raising = False)
+ torch.manual_seed(0)
+ lengths = [37, 100, 5, 114]
+ T, H, D = sum(lengths), 8, 64
+ module = types.SimpleNamespace(is_causal = True, num_key_value_groups = H // n_kv, training = True)
+ q = torch.randn(1, H, T, D, device = "cuda", dtype = torch.bfloat16, requires_grad = True)
+ k = torch.randn(1, n_kv, T, D, device = "cuda", dtype = torch.bfloat16, requires_grad = True)
+ v = torch.randn(1, n_kv, T, D, device = "cuda", dtype = torch.bfloat16, requires_grad = True)
+ mask = _block_causal(lengths, "cuda")
+ psl = torch.tensor(lengths, dtype = torch.int32)
+ before = hpa.HF_PACKED_ATTENTION_STATS["fast"]
+ fast, _ = hpa._sdpa_packed_varlen(module, q, k, v, mask, scaling = 0.11, packed_seq_lengths = psl)
+ assert hpa.HF_PACKED_ATTENTION_STATS["fast"] == before + 1
+ ref, _ = sdpa_attention_forward(module, q, k, v, mask, scaling = 0.11)
+ assert fast.shape == ref.shape
+ torch.testing.assert_close(fast.float(), ref.float(), atol = 2e-2, rtol = 2e-2)
+ g = torch.randn_like(ref)
+ grads_fast = torch.autograd.grad(fast, (q, k, v), g)
+ grads_ref = torch.autograd.grad(ref, (q, k, v), g)
+ for a, b in zip(grads_fast, grads_ref):
+ torch.testing.assert_close(a.float(), b.float(), atol = 5e-2, rtol = 5e-2)
+
+
+def test_new_lengths_tensor_with_a_cached_mask_is_checked_again():
+ lengths = [3, 3]
+ mask = _block_causal(lengths)
+ assert hpa._mask_is_block_causal(mask, _cu(lengths), 6)
+ # A different cu_seqlens object must not be compared elementwise against the cached one.
+ assert hpa._mask_is_block_causal(mask, _cu(lengths), 6)
+
+
+def test_long_masks_are_checked_in_row_blocks(monkeypatch):
+ monkeypatch.setattr(hpa, "_MASK_CHECK_ROWS", 4)
+ lengths = [5, 7, 6]
+ mask = _block_causal(lengths).clone()
+ assert hpa._mask_is_block_causal(mask, _cu(lengths), 18)
+ mask[0, 0, 17, 0] = True
+ assert not hpa._mask_is_block_causal(mask, _cu(lengths), 18)
+
+
+@_CUDA
+def test_compiling_leaves_the_counters_alone(monkeypatch):
+ hpa._ORIG_SDPA[0] = _plain_sdpa
+ monkeypatch.delenv("UNSLOTH_HF_PACKED_VARLEN", raising = False)
+ lengths = [3, 3]
+ module = types.SimpleNamespace(is_causal = True, num_key_value_groups = 1, training = False)
+ q = torch.randn(1, 4, 6, 64, device = "cuda", dtype = torch.bfloat16)
+ mask, psl = _block_causal(lengths, "cuda"), torch.tensor(lengths, dtype = torch.int32)
+ fn = torch.compile(
+ lambda q: hpa._sdpa_packed_varlen(module, q, q, q, mask, packed_seq_lengths = psl)[0],
+ backend = "eager",
+ fullgraph = True,
+ )
+ positional = torch.compile(
+ lambda q: hpa._sdpa_packed_varlen(module, q, q, q, mask, 0.0, packed_seq_lengths = psl)[0],
+ backend = "eager",
+ fullgraph = True,
+ )
+ before = dict(hpa.HF_PACKED_ATTENTION_STATS)
+ fn(q)
+ positional(q)
+ assert hpa.HF_PACKED_ATTENTION_STATS == before
+
+
+@_CUDA
[email protected]("mode", ["training_no_grad", "eval_grad", "eval_no_grad"])
+def test_int32_guard_follows_training_or_grad_mode(monkeypatch, mode):
+ """A reentrant checkpoint's no_grad forward (training) and its recompute both fall back; a pure
+ no_grad eval forward allocates no backward buffer and keeps varlen."""
+ import unsloth.utils.attention_dispatch as ad
+
+ seen, orig = _calls()
+ hpa._ORIG_SDPA[0] = orig
+ monkeypatch.delenv("UNSLOTH_HF_PACKED_VARLEN", raising = False)
+ monkeypatch.setattr(ad, "_varlen_backward_overflows_int32", lambda *a: True)
+ lengths = [3, 3]
+ module = types.SimpleNamespace(is_causal = True, training = mode == "training_no_grad")
+ q = torch.randn(1, 4, 6, 64, device = "cuda", dtype = torch.bfloat16)
+ psl = torch.tensor(lengths, dtype = torch.int32)
+ with torch.set_grad_enabled(mode == "eval_grad"):
+ out = hpa._sdpa_packed_varlen(
+ module, q, q, q, _block_causal(lengths, "cuda"), packed_seq_lengths = psl
+ )
+ if mode == "eval_no_grad":
+ if hpa._torch_varlen(q.device) is None and not (ad.HAS_FLASH_ATTENTION or ad.HAS_XFORMERS):
+ pytest.skip("needs a varlen kernel")
+ assert out[0].shape == (1, 6, 4, 64)
+ else:
+ assert out == ("orig", None)
+
+
+@_CUDA
+def test_a_refusing_torch_varlen_hands_the_call_back(monkeypatch):
+ seen, orig = _calls()
+ hpa._ORIG_SDPA[0] = orig
+ monkeypatch.delenv("UNSLOTH_HF_PACKED_VARLEN", raising = False)
+
+ def refuse(*args, **kwargs):
+ raise RuntimeError("GQA is not supported with the cuDNN backend.")
+
+ monkeypatch.setattr(hpa, "_TORCH_VARLEN", [refuse])
+ monkeypatch.setattr(hpa, "_TORCH_VARLEN_DEVICES", {})
+ lengths = [3, 3]
+ q = torch.randn(1, 4, 6, 64, device = "cuda", dtype = torch.bfloat16)
+ psl = torch.tensor(lengths, dtype = torch.int32)
+ out = hpa._sdpa_packed_varlen(
+ types.SimpleNamespace(is_causal = True),
+ q,
+ q,
+ q,
+ _block_causal(lengths, "cuda"),
+ packed_seq_lengths = psl,
+ )
+ assert out == ("orig", None)
+ assert hpa._TORCH_VARLEN == [None]
+
+
+def test_torch_varlen_kill_switch(monkeypatch):
+ monkeypatch.setenv("UNSLOTH_TORCH_VARLEN", "0")
+ monkeypatch.setattr(hpa, "_TORCH_VARLEN", [])
+ monkeypatch.setattr(hpa, "_TORCH_VARLEN_GQA", [])
+ assert hpa._torch_varlen(torch.device("cuda", 0)) is None
+ assert hpa._TORCH_VARLEN == [None]
+
+
+@_CUDA
[email protected]("policy", ["math_only", "flash_disabled", "math_first", "deterministic"])
+def test_an_sdpa_policy_keeps_every_varlen_kernel_off(monkeypatch, policy):
+ """The wrapped sdpa runs, not torch varlen nor the xformers / flash-attn dispatch; outside, varlen runs."""
+ import contextlib
+ from torch.nn.attention import SDPBackend, sdpa_kernel
+ from unsloth.utils import attention_dispatch as ad
+
+ if policy == "math_first" and "set_priority" not in inspect.signature(sdpa_kernel).parameters:
+ pytest.skip("this torch has no sdpa_kernel(set_priority = True)")
+ calls = []
+
+ def spy(q, *args, **kwargs):
+ calls.append(1)
+ return torch.zeros_like(q)
+
+ monkeypatch.setattr(hpa, "_TORCH_VARLEN", [spy])
+ monkeypatch.setattr(hpa, "_TORCH_VARLEN_DEVICES", {})
+ monkeypatch.setattr(ad, "run_attention", lambda **kwargs: calls.append(2) or kwargs["Q"])
+ seen, orig = _calls()
+ hpa._ORIG_SDPA[0] = orig
+ monkeypatch.delenv("UNSLOTH_HF_PACKED_VARLEN", raising = False)
+ lengths = [3, 3]
+ q = torch.randn(1, 4, 6, 64, device = "cuda", dtype = torch.bfloat16)
+ k = torch.randn(1, 2, 6, 64, device = "cuda", dtype = torch.bfloat16)
+ module = types.SimpleNamespace(is_causal = True)
+
+ def run():
+ return hpa._sdpa_packed_varlen(
+ module,
+ q,
+ k,
+ k,
+ _block_causal(lengths, "cuda"),
+ packed_seq_lengths = torch.tensor(lengths, dtype = torch.int32),
+ )
+
+ if policy == "math_only":
+ context = sdpa_kernel([SDPBackend.MATH])
+ elif policy == "flash_disabled":
+ context = sdpa_kernel([SDPBackend.EFFICIENT_ATTENTION, SDPBackend.MATH])
+ elif policy == "math_first":
+ context = sdpa_kernel([SDPBackend.MATH, SDPBackend.FLASH_ATTENTION], set_priority = True)
+ else:
+ context = contextlib.nullcontext()
+ deterministic = torch.are_deterministic_algorithms_enabled()
+ torch.use_deterministic_algorithms(True)
+ try:
+ with context:
+ assert run() == ("orig", None)
+ finally:
+ if policy == "deterministic":
+ torch.use_deterministic_algorithms(deterministic)
+ assert calls == [] and len(seen) == 1
+ run()
+ assert calls == [1]
diff --git a/unsloth/utils/hf_packed_attention.py b/unsloth/utils/hf_packed_attention.py
new file mode 100644
index 00000000000..236ba5692b6
--- /dev/null
+++ b/unsloth/utils/hf_packed_attention.py
@@ -0,0 +1,353 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+"""Packed rows on the transformers modeling path run a varlen flash kernel (torch's, else xformers /
+flash-attn) instead of SDPA over a dense block-causal mask; anything else reaches the wrapped "sdpa"."""
+
+import inspect
+import logging
+import os
+import weakref
+
+import torch
+
+__all__ = ["enable_hf_packed_attention", "HF_PACKED_ATTENTION_STATS"]
+
+_FLAG = "_unsloth_hf_packed_varlen"
+_ENV = "UNSLOTH_HF_PACKED_VARLEN"
+_ORIG_SDPA = [None]
+HF_PACKED_ATTENTION_STATS = {"fast": 0, "fallback": 0, "mask_checks": 0}
+# All layers of a forward share one mask object: weakref, (cu_seqlens, versions), verdict.
+_MASK_CHECK = [None, None, None]
+_MASK_CHECK_ROWS = 1024
+
+
+def _disabled() -> bool:
+ return os.environ.get(_ENV, "1").lower() in ("0", "false", "no", "off")
+
+
+def _mask_is_block_causal(attention_mask, cu_seqlens, total) -> bool:
+ """Exactly causal inside each packed segment and closed across them; windows, bidirectional
+ spans, pad tails and None (one document, already is_causal) keep the wrapped path."""
+ if not (
+ isinstance(attention_mask, torch.Tensor)
+ and attention_mask.dtype == torch.bool
+ and attention_mask.dim() == 4
+ and attention_mask.shape[0] == 1
+ and attention_mask.shape[1] == 1
+ and attention_mask.shape[-2:] == (total, total)
+ ):
+ return False
+ # Versions catch in-place edits; inference tensors have none, so are always rechecked.
+ cacheable = not (torch.is_inference(attention_mask) or torch.is_inference(cu_seqlens))
+ key = (attention_mask._version, cu_seqlens._version, total) if cacheable else None
+ ref, checked, verdict = _MASK_CHECK
+ if (
+ cacheable
+ and ref is not None
+ and ref() is attention_mask
+ and checked[0] is cu_seqlens
+ and checked[1] == key
+ ):
+ return verdict
+ HF_PACKED_ATTENTION_STATS["mask_checks"] += 1
+ verdict = False
+ if int(cu_seqlens[-1].item()) == total:
+ lengths = (cu_seqlens[1:] - cu_seqlens[:-1]).to(torch.int64)
+ segment = torch.repeat_interleave(
+ torch.arange(lengths.numel(), device = attention_mask.device),
+ lengths.to(attention_mask.device),
+ )
+ position = torch.arange(total, device = attention_mask.device)
+ # Row blocks keep the reference to _MASK_CHECK_ROWS x T, not a second T x T mask.
+ verdict = True
+ for start in range(0, total, _MASK_CHECK_ROWS):
+ rows = slice(start, min(start + _MASK_CHECK_ROWS, total))
+ expected = (segment[rows, None] == segment[None, :]) & (
+ position[None, :] <= position[rows, None]
+ )
+ if not torch.equal(attention_mask[0, 0, rows], expected):
+ verdict = False
+ break
+ if cacheable:
+ _MASK_CHECK[:] = [weakref.ref(attention_mask), (cu_seqlens, key), verdict]
+ return verdict
+
+
+def _autocast_dtype_differs(query) -> bool:
+ try:
+ enabled = torch.is_autocast_enabled(query.device.type)
+ dtype = torch.get_autocast_dtype(query.device.type) if enabled else None
+ except (AttributeError, TypeError):
+ enabled = torch.is_autocast_enabled()
+ dtype = torch.get_autocast_gpu_dtype() if enabled else None
+ return enabled and dtype != query.dtype
+
+
+def _sliding_or_softcapped(module, kwargs) -> bool:
+ if kwargs.get("sliding_window") or getattr(module, "sliding_window", None):
+ return True
+ if kwargs.get("softcap") or getattr(module, "sinks", None) is not None:
+ return True
+ if getattr(module, "attn_logit_softcapping", None):
+ return True
+ config = getattr(module, "config", None)
+ return bool(getattr(config, "attn_logit_softcapping", None))
+
+
+_TORCH_VARLEN = []
+_TORCH_VARLEN_GQA = []
+_TORCH_VARLEN_DEVICES = {}
+
+
+def _torch_varlen(device):
+ """torch's own flash varlen (a third of xformers' host cost per call), else None."""
+ if not _TORCH_VARLEN:
+ fn, native_gqa = None, False
+ try:
+ # attention_dispatch's kill switch for torch's varlen_attn covers this path too.
+ if os.environ.get("UNSLOTH_TORCH_VARLEN", "1").lower() in ("0", "false", "no", "off"):
+ raise ImportError
+ from torch.nn.attention.varlen import varlen_attn
+
+ # 2.11 added scale / window_size, 2.12 enable_gqa; 2.10 only had is_causal and is not used.
+ parameters = inspect.signature(varlen_attn).parameters.keys()
+ if {"scale", "window_size"} <= parameters:
+ fn, native_gqa = varlen_attn, "enable_gqa" in parameters
+ except Exception:
+ fn = None
+ _TORCH_VARLEN.append(fn)
+ _TORCH_VARLEN_GQA[:] = [native_gqa]
+ fn = _TORCH_VARLEN[0]
+ if fn is None or torch.version.hip is not None:
+ return None
+ ok = _TORCH_VARLEN_DEVICES.get(device.index)
+ if ok is None:
+ ok = torch.cuda.get_device_capability(device)[0] >= 8
+ _TORCH_VARLEN_DEVICES[device.index] = ok
+ return fn if ok else None
+
+
+_SDPA_MATH = []
+
+
+def _sdpa_requested(query, key) -> bool:
+ """Deterministic mode, a disabled flash SDPA backend or a math-first sdpa_kernel priority asks for
+ SDPA itself, so no varlen kernel (torch's, xformers or flash-attn) may replace it."""
+ if torch.are_deterministic_algorithms_enabled():
+ return True
+ if not _SDPA_MATH:
+ from torch.nn.attention import SDPBackend
+ from .attention_dispatch import _sdpa_backend_state
+ _SDPA_MATH[:] = [int(SDPBackend.MATH), _sdpa_backend_state]
+ math, backend_state = _SDPA_MATH
+ n_heads, n_kv_heads = query.shape[1], key.shape[1]
+ flash, choice = backend_state(
+ str(query.device),
+ query.dtype,
+ query.shape[-1],
+ n_heads,
+ n_kv_heads,
+ True,
+ n_heads != n_kv_heads,
+ )
+ return not flash or choice == math
+
+
+def _sdpa_packed_varlen(
+ module,
+ query,
+ key,
+ value,
+ attention_mask,
+ *args,
+ dropout = 0.0,
+ scaling = None,
+ is_causal = None,
+ **kwargs,
+):
+ orig = _ORIG_SDPA[0]
+ if args:
+ # Positional dropout / scaling / is_causal (and later parameters) go back exactly as given.
+ named = {"dropout": dropout, "scaling": scaling, "is_causal": is_causal}
+ for name in list(named)[: len(args)]:
+ named.pop(name)
+ if kwargs.get("packed_seq_lengths") is not None and not torch.compiler.is_compiling():
+ HF_PACKED_ATTENTION_STATS["fallback"] += 1
+ return orig(module, query, key, value, attention_mask, *args, **named, **kwargs)
+ # Before any counter while tracing: Dynamo would guard on it and recompile every forward.
+ if kwargs.get("packed_seq_lengths") is None or _disabled() or torch.compiler.is_compiling():
+ return orig(
+ module,
+ query,
+ key,
+ value,
+ attention_mask,
+ dropout = dropout,
+ scaling = scaling,
+ is_causal = is_causal,
+ **kwargs,
+ )
+ from .attention_dispatch import (
+ _VARLEN_INT32_GUARD_DISABLED,
+ FLASH_VARLEN,
+ XFORMERS,
+ _varlen_backward_overflows_int32,
+ select_attention_backend,
+ )
+ from .packing import get_packed_info_from_kwargs
+
+ bsz, n_heads, q_len, head_dim = query.shape
+ seq_info = torch_varlen = backend = None
+ if (
+ bsz == 1
+ and q_len > 0
+ and query.is_cuda
+ and query.device == key.device == value.device
+ and key.shape[0] == value.shape[0] == 1
+ and key.shape[1] == value.shape[1]
+ and key.shape[2] == value.shape[2] == q_len
+ # Not MLA (192 / 128) nor Gemma-4 global (512).
+ and key.shape[-1] == value.shape[-1] == head_dim
+ and head_dim <= 256
+ and head_dim % 8 == 0
+ and query.dtype in (torch.float16, torch.bfloat16)
+ and query.dtype == key.dtype == value.dtype
+ and not _autocast_dtype_differs(query)
+ and not dropout
+ and is_causal is not False
+ and getattr(module, "is_causal", True)
+ and kwargs.get("position_bias") is None
+ and kwargs.get("cache") is None
+ and kwargs.get("past_key_values") is None
+ and n_heads % key.shape[1] == 0
+ and not _sliding_or_softcapped(module, kwargs)
+ and not _sdpa_requested(query, key)
+ ):
+ torch_varlen = _torch_varlen(query.device)
+ backend = select_attention_backend(use_varlen = True)
+ # Flash-2 varlen backward faults past int32 indexing. Keyed on training / grad mode, not
+ # requires_grad, so a reentrant checkpoint's no_grad forward and its recompute agree.
+ if (
+ not _VARLEN_INT32_GUARD_DISABLED
+ and (getattr(module, "training", False) or torch.is_grad_enabled())
+ and _varlen_backward_overflows_int32(
+ kwargs["packed_seq_lengths"].numel(), q_len, n_heads, head_dim
+ )
+ ):
+ torch_varlen, backend = None, None
+ if torch_varlen is not None or backend in (FLASH_VARLEN, XFORMERS):
+ seq_info = get_packed_info_from_kwargs(kwargs, query.device)
+ if seq_info is None or not _mask_is_block_causal(attention_mask, seq_info[1], q_len):
+ HF_PACKED_ATTENTION_STATS["fallback"] += 1
+ return orig(
+ module,
+ query,
+ key,
+ value,
+ attention_mask,
+ dropout = dropout,
+ scaling = scaling,
+ is_causal = is_causal,
+ **kwargs,
+ )
+ HF_PACKED_ATTENTION_STATS["fast"] += 1
+ scale = head_dim**-0.5 if scaling is None else scaling
+ n_kv_heads = key.shape[1]
+ _, cu_seqlens, max_seqlen = seq_info
+ if torch_varlen is not None:
+ gqa, k_in, v_in = {}, key, value
+ if n_kv_heads != n_heads:
+ if not _TORCH_VARLEN_GQA or _TORCH_VARLEN_GQA[0]:
+ gqa = {"enable_gqa": True}
+ else:
+ # torch 2.11: no enable_gqa, so repeat the K/V heads as transformers' sdpa path does.
+ k_in = key.repeat_interleave(n_heads // n_kv_heads, dim = 1)
+ v_in = value.repeat_interleave(n_heads // n_kv_heads, dim = 1)
+ try:
+ out = torch_varlen(
+ query[0].transpose(0, 1),
+ k_in[0].transpose(0, 1),
+ v_in[0].transpose(0, 1),
+ cu_seqlens,
+ cu_seqlens,
+ max_seqlen,
+ max_seqlen,
+ scale = scale,
+ window_size = (-1, 0),
+ **gqa,
+ )
+ except torch.OutOfMemoryError:
+ raise
+ except RuntimeError as exc:
+ # A build whose varlen picks a backend without causal GQA (cuDNN) refuses at forward: stop
+ # using it and give this call to the wrapped sdpa.
+ _TORCH_VARLEN[:] = [None]
+ logging.getLogger(__name__).info(f"Unsloth: torch varlen_attn unavailable ({exc})")
+ HF_PACKED_ATTENTION_STATS["fast"] -= 1
+ HF_PACKED_ATTENTION_STATS["fallback"] += 1
+ return orig(
+ module,
+ query,
+ key,
+ value,
+ attention_mask,
+ dropout = dropout,
+ scaling = scaling,
+ is_causal = is_causal,
+ **kwargs,
+ )
+ return out.unsqueeze(0), None
+ from .attention_dispatch import AttentionConfig, AttentionContext, run_attention
+
+ config = AttentionConfig(
+ backend = backend,
+ n_kv_heads = n_kv_heads,
+ n_groups = n_heads // n_kv_heads,
+ flash_varlen_kwargs = {"dropout_p": 0.0, "causal": True, "softmax_scale": scale},
+ sdpa_kwargs = {"scale": scale},
+ xformers_kwargs = {"scale": scale},
+ )
+ context = AttentionContext(
+ bsz = 1,
+ q_len = q_len,
+ kv_seq_len = q_len,
+ n_heads = n_heads,
+ head_dim = head_dim,
+ # Not grad mode: xformers needs a backward op for grad tensors even under no_grad (reentrant
+ # checkpointing), and that op rejects the grouped 5D layout.
+ requires_grad = query.requires_grad or key.requires_grad or value.requires_grad,
+ seq_info = seq_info,
+ attention_mask = None,
+ causal_mask = None,
+ )
+ out = run_attention(config = config, context = context, Q = query, K = key, V = value)
+ return out.contiguous(), None
+
+
+def enable_hf_packed_attention() -> bool:
+ """Wrap transformers' "sdpa" attention entry once. True when the wrapper is installed."""
+ if _disabled():
+ return False
+ try:
+ from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS
+ except Exception:
+ return False
+ current = ALL_ATTENTION_FUNCTIONS.get("sdpa", None)
+ if current is None:
+ return False
+ # Once per process: gemma-4's router stacked above does not forward our sentinel, and re-wrapping
+ # it would loop through its boxed original.
+ if getattr(current, _FLAG, False) or _ORIG_SDPA[0] is not None:
+ return True
+ _ORIG_SDPA[0] = current
+ # Zoo sdpa routers re-run per load and check their sentinel on the installed entry: forward them.
+ try:
+ for name, value in vars(current).items():
+ if name.startswith(("_unsloth_", "__unsloth_")):
+ setattr(_sdpa_packed_varlen, name, value)
+ except Exception:
+ pass
+ setattr(_sdpa_packed_varlen, _FLAG, True)
+ # Direct assignment: register() does not update the mapping layers read.
+ ALL_ATTENTION_FUNCTIONS["sdpa"] = _sdpa_packed_varlen
+ return True
diff --git a/unsloth/utils/packing.py b/unsloth/utils/packing.py
index 08525ad9345..afc585ae1f2 100644
--- a/unsloth/utils/packing.py
+++ b/unsloth/utils/packing.py
@@ -194,6 +194,15 @@ def configure_padding_free(config):
setattr(config, "remove_unused_columns", False)
+def _enable_hf_packed_attention() -> None:
+ # Packed rows on the transformers modeling path run varlen instead of SDPA over a dense mask.
+ try:
+ from .hf_packed_attention import enable_hf_packed_attention
+ enable_hf_packed_attention()
+ except Exception as exc:
+ logging.getLogger(__name__).info(f"Unsloth: packed varlen attention not enabled ({exc})")
+
+
def enable_sample_packing(
model,
trainer,
@@ -212,6 +221,7 @@ def enable_sample_packing(
collator = getattr(trainer, "data_collator", None)
if collator is None or not hasattr(collator, "torch_call"):
return
+ _enable_hf_packed_attention()
if getattr(collator, "_unsloth_packing_wrapped", False):
return
@@ -263,6 +273,7 @@ def enable_padding_free_metadata(model, trainer):
return
mark_allow_overlength(model)
+ _enable_hf_packed_attention()
if hasattr(collator, "return_position_ids"):
collator.return_position_ids = True
if hasattr(trainer, "args") and hasattr(trainer.args, "remove_unused_columns"):