Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Run packed rows on the transformers modeling path through varlen flash attention (#13234)
Daniel Hanc554fbfe22c7
diff --git a/tests/test_hf_packed_attention.py b/tests/test_hf_packed_attention.pynew file mode 100644index 00000000000..398acd779cf--- /dev/null+++ b/tests/test_hf_packed_attention.py@@ -0,0 +1,508 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.+"""Packed rows on the transformers modeling path run varlen attention, and nothing else changes."""++import inspect+import types++import pytest+import torch+from real_accelerator import has_real_cuda # tests/_shared, on sys.path via tests/conftest.py++from unsloth.utils import hf_packed_attention as hpa+++def _block_causal(lengths, device = "cpu"):+ segment = torch.repeat_interleave(torch.arange(len(lengths)), torch.tensor(lengths)).to(device)+ return ((segment[:, None] == segment[None, :]).tril())[None, None]+++def _cu(lengths, device = "cpu"):+ return torch.tensor(+ [0] + list(torch.tensor(lengths).cumsum(0)), dtype = torch.int32, device = device+ )++[email protected](autouse = True)+def _fresh_state():+ saved = hpa._ORIG_SDPA[0]+ hpa._MASK_CHECK[:] = [None, None, None]+ yield+ hpa._MASK_CHECK[:] = [None, None, None]+ hpa._ORIG_SDPA[0] = saved+++def test_block_causal_mask_is_recognised():+ lengths = [3, 5, 2]+ assert hpa._mask_is_block_causal(_block_causal(lengths), _cu(lengths), 10)+++ "edit",+ ["cross_document", "bidirectional", "window", "plain_causal", "pad_tail", "float"],+)+def test_any_other_mask_keeps_the_masked_path(edit):+ lengths = [3, 5, 2]+ mask, cu, total = _block_causal(lengths), _cu(lengths), 10+ if edit == "cross_document":+ mask = mask.clone()+ mask[0, 0, 4, 1] = True+ elif edit == "bidirectional":+ mask = mask.clone()+ mask[0, 0, 3, 4] = True+ elif edit == "window":+ mask = mask.clone()+ mask[0, 0, 7, 3] = False+ elif edit == "plain_causal":+ mask = torch.ones(10, 10, dtype = torch.bool).tril()[None, None]+ elif edit == "pad_tail":+ cu = _cu([3, 5])+ elif edit == "float":+ mask = torch.where(mask, 0.0, float("-inf"))+ assert not hpa._mask_is_block_causal(mask, cu, total)+++def test_mask_verdict_is_cached_per_mask_object():+ lengths = [4, 4]+ mask, cu = _block_causal(lengths), _cu(lengths)+ assert hpa._mask_is_block_causal(mask, cu, 8)+ # Same object: the cached verdict stands. A new object is checked again.+ assert hpa._mask_is_block_causal(mask, cu, 8)+ other = torch.ones(8, 8, dtype = torch.bool).tril()[None, None]+ assert not hpa._mask_is_block_causal(other, cu, 8)+++def _calls():+ seen = []++ def orig(+ module,+ q,+ k,+ v,+ mask,+ dropout = 0.0,+ scaling = None,+ is_causal = None,+ **kwargs,+ ):+ seen.append(kwargs)+ return "orig", None++ return seen, orig+++def test_unpacked_calls_go_straight_through(monkeypatch):+ seen, orig = _calls()+ hpa._ORIG_SDPA[0] = orig+ q = torch.zeros(2, 4, 6, 8)+ out = hpa._sdpa_packed_varlen(types.SimpleNamespace(), q, q, q, None, scaling = 0.3, foo = 1)+ assert out == ("orig", None) and seen == [{"foo": 1}]+++_CUDA = pytest.mark.skipif(not has_real_cuda(), reason = "needs a CUDA device")+++@_CUDA+ "why",+ [+ "none",+ "sliding",+ "softcap",+ "sinks",+ "fp32",+ "dropout",+ "batched",+ "cpu",+ "head_dim_512",+ "mla_value_width",+ "autocast_dtype",+ "paged_cache",+ "no_mask",+ "module_softcap",+ "not_causal",+ "positional",+ "kv_batched",+ ],+)+def test_packed_calls_that_varlen_cannot_express_fall_back(monkeypatch, why):+ """Each reason alone sends the call to the wrapped sdpa; "none" is the control that takes varlen."""+ from unsloth.utils.attention_dispatch import HAS_FLASH_ATTENTION, HAS_XFORMERS++ if not (HAS_FLASH_ATTENTION or HAS_XFORMERS or hpa._torch_varlen(torch.device("cuda", 0))):+ pytest.skip("needs a varlen kernel")+ seen, orig = _calls()+ hpa._ORIG_SDPA[0] = orig+ monkeypatch.delenv("UNSLOTH_HF_PACKED_VARLEN", raising = False)+ lengths, D = [3, 3], 64+ module = types.SimpleNamespace(is_causal = True)+ device, dtype, B = "cuda", torch.bfloat16, 1+ kwargs = {"packed_seq_lengths": torch.tensor(lengths, dtype = torch.int32)}+ dropout, mask, v_width = 0.0, _block_causal(lengths, "cuda"), D+ if why == "sliding":+ module.sliding_window = 4+ elif why == "softcap":+ module.config = types.SimpleNamespace(attn_logit_softcapping = 50.0)+ elif why == "module_softcap":+ module.attn_logit_softcapping = 30.0+ elif why == "sinks":+ module.sinks = torch.zeros(4)+ elif why == "fp32":+ dtype = torch.float32+ elif why == "dropout":+ dropout = 0.1+ elif why == "batched":+ B = 2+ elif why == "cpu":+ device, mask = "cpu", _block_causal(lengths)+ elif why == "head_dim_512":+ D = v_width = 512+ elif why == "mla_value_width":+ D, v_width = 192, 128+ elif why == "paged_cache":+ kwargs["cache"] = object()+ elif why == "no_mask":+ mask = None+ q = torch.randn(B, 4, 6, D, device = device, dtype = dtype)+ v = torch.randn(B, 4, 6, v_width, device = device, dtype = dtype)+ k = q+ if why == "kv_batched":+ k, v = q.expand(2, -1, -1, -1), v.expand(2, -1, -1, -1)+ before = dict(hpa.HF_PACKED_ATTENTION_STATS)+ if why == "autocast_dtype":+ with torch.autocast("cuda", dtype = torch.float16):+ out = hpa._sdpa_packed_varlen(module, q, k, v, mask, dropout = dropout, **kwargs)+ elif why == "not_causal":+ out = hpa._sdpa_packed_varlen(module, q, k, v, mask, is_causal = False, **kwargs)+ elif why == "positional":+ out = hpa._sdpa_packed_varlen(module, q, k, v, mask, 0.0, **kwargs)+ else:+ out = hpa._sdpa_packed_varlen(module, q, k, v, mask, dropout = dropout, **kwargs)+ if why == "none":+ assert out[0].shape == (1, 6, 4, D) and seen == []+ assert hpa.HF_PACKED_ATTENTION_STATS["fast"] == before["fast"] + 1+ else:+ assert out == ("orig", None)+ assert hpa.HF_PACKED_ATTENTION_STATS["fallback"] == before["fallback"] + 1+++def test_an_edited_mask_is_checked_again():+ lengths = [3, 3]+ mask, cu = _block_causal(lengths).clone(), _cu(lengths)+ assert hpa._mask_is_block_causal(mask, cu, 6)+ mask[0, 0, 3, 0] = True+ assert not hpa._mask_is_block_causal(mask, cu, 6)+ with torch.inference_mode():+ frozen, frozen_cu = _block_causal(lengths), _cu(lengths)+ assert hpa._mask_is_block_causal(frozen, frozen_cu, 6)+++def test_a_missing_mask_is_not_block_causal():+ assert not hpa._mask_is_block_causal(None, _cu([6]), 6)+++def _plain_sdpa(+ module,+ q,+ k,+ v,+ mask,+ dropout = 0.0,+ scaling = None,+ is_causal = None,+ **kwargs,+):+ # Not transformers' sdpa_attention_forward: compiling that after importing unsloth trips a Dynamo+ # guard bug on torch <= 2.13, with or without this wrapper.+ out = torch.nn.functional.scaled_dot_product_attention(q, k, v, attn_mask = mask, scale = scaling)+ return out.transpose(1, 2), None+++@_CUDA+def test_fullgraph_compile_keeps_the_wrapped_sdpa(monkeypatch):+ sdpa_attention_forward = _plain_sdpa+ hpa._ORIG_SDPA[0] = sdpa_attention_forward+ monkeypatch.delenv("UNSLOTH_HF_PACKED_VARLEN", raising = False)+ lengths = [3, 3]+ module = types.SimpleNamespace(is_causal = True, num_key_value_groups = 1, training = False)+ q = torch.randn(1, 4, 6, 64, device = "cuda", dtype = torch.bfloat16)+ mask, psl = _block_causal(lengths, "cuda"), torch.tensor(lengths, dtype = torch.int32)+ fn = torch.compile(+ lambda q: hpa._sdpa_packed_varlen(module, q, q, q, mask, packed_seq_lengths = psl)[0],+ backend = "eager",+ fullgraph = True,+ )+ torch.testing.assert_close(fn(q), sdpa_attention_forward(module, q, q, q, mask)[0])+++@_CUDA[email protected]("route", ["torch", "dispatch"])+def test_reentrant_checkpoint_of_grouped_heads(monkeypatch, route):+ from unsloth.utils.attention_dispatch import HAS_FLASH_ATTENTION, HAS_XFORMERS++ if route == "torch" and hpa._torch_varlen(torch.device("cuda", 0)) is None:+ pytest.skip("this torch has no window_size / enable_gqa varlen_attn")+ if route == "dispatch":+ if not (HAS_FLASH_ATTENTION or HAS_XFORMERS):+ pytest.skip("needs xformers or flash-attn")+ monkeypatch.setattr(hpa, "_torch_varlen", lambda device: None)+ from torch.utils.checkpoint import checkpoint+ from transformers.integrations.sdpa_attention import sdpa_attention_forward++ hpa._ORIG_SDPA[0] = sdpa_attention_forward+ monkeypatch.delenv("UNSLOTH_HF_PACKED_VARLEN", raising = False)+ lengths = [40, 24]+ module = types.SimpleNamespace(is_causal = True, num_key_value_groups = 4, training = True)+ q = torch.randn(1, 8, 64, 64, device = "cuda", dtype = torch.bfloat16, requires_grad = True)+ k = torch.randn(1, 2, 64, 64, device = "cuda", dtype = torch.bfloat16, requires_grad = True)+ v = torch.randn(1, 2, 64, 64, device = "cuda", dtype = torch.bfloat16, requires_grad = True)+ mask, psl = _block_causal(lengths, "cuda"), torch.tensor(lengths, dtype = torch.int32)+ fn = lambda q, k, v: hpa._sdpa_packed_varlen(module, q, k, v, mask, packed_seq_lengths = psl)[0]+ out = checkpoint(fn, q, k, v, use_reentrant = True)+ out.float().sum().backward()+ ref = sdpa_attention_forward(module, q, k, v, mask)[0]+ torch.testing.assert_close(out.float(), ref.float(), atol = 2e-2, rtol = 2e-2)+++def test_kill_switch(monkeypatch):+ seen, orig = _calls()+ hpa._ORIG_SDPA[0] = orig+ monkeypatch.setenv("UNSLOTH_HF_PACKED_VARLEN", "0")+ q = torch.zeros(1, 4, 6, 8, dtype = torch.bfloat16)+ lengths = torch.tensor([3, 3], dtype = torch.int32)+ out = hpa._sdpa_packed_varlen(+ types.SimpleNamespace(), q, q, q, _block_causal([3, 3]), packed_seq_lengths = lengths+ )+ assert out == ("orig", None)+ assert not hpa.enable_hf_packed_attention()+++def test_install_forwards_router_sentinels_and_runs_once(monkeypatch):+ mapping_mod = pytest.importorskip("transformers.modeling_utils")+ mapping = mapping_mod.ALL_ATTENTION_FUNCTIONS++ def router(*args, **kwargs):+ return None++ router._unsloth_gemma4_flash = True+ monkeypatch.setitem(mapping, "sdpa", router)+ hpa._ORIG_SDPA[0] = None+ monkeypatch.delenv("UNSLOTH_HF_PACKED_VARLEN", raising = False)+ assert hpa.enable_hf_packed_attention()+ installed = mapping["sdpa"]+ assert installed is hpa._sdpa_packed_varlen and hpa._ORIG_SDPA[0] is router+ # A zoo router re-running on the next load sees its own sentinel and leaves the chain alone.+ assert getattr(installed, "_unsloth_gemma4_flash", False)+ # A router stacked above later never gets wrapped a second time.+ monkeypatch.setitem(mapping, "sdpa", router)+ assert hpa.enable_hf_packed_attention()+ assert mapping["sdpa"] is router and hpa._ORIG_SDPA[0] is router+++@_CUDA[email protected]("route", ["torch", "torch_2_11", "dispatch"])[email protected]("n_kv", [8, 2])+def test_varlen_matches_sdpa_over_the_dense_mask(monkeypatch, n_kv, route):+ from unsloth.utils.attention_dispatch import HAS_FLASH_ATTENTION, HAS_XFORMERS++ if route != "dispatch" and hpa._torch_varlen(torch.device("cuda", 0)) is None:+ pytest.skip("this torch has no window_size varlen_attn")+ if route == "torch_2_11":+ # torch 2.11's varlen_attn has scale / window_size but no enable_gqa.+ native = hpa._TORCH_VARLEN[0]++ def without_gqa(*args, **kwargs):+ assert "enable_gqa" not in kwargs+ return native(*args, **kwargs)++ monkeypatch.setattr(hpa, "_TORCH_VARLEN", [without_gqa])+ monkeypatch.setattr(hpa, "_TORCH_VARLEN_GQA", [False])+ if route == "dispatch":+ if not (HAS_FLASH_ATTENTION or HAS_XFORMERS):+ pytest.skip("needs xformers or flash-attn")+ monkeypatch.setattr(hpa, "_torch_varlen", lambda device: None)+ from transformers.integrations.sdpa_attention import sdpa_attention_forward++ hpa._ORIG_SDPA[0] = sdpa_attention_forward+ monkeypatch.delenv("UNSLOTH_HF_PACKED_VARLEN", raising = False)+ torch.manual_seed(0)+ lengths = [37, 100, 5, 114]+ T, H, D = sum(lengths), 8, 64+ module = types.SimpleNamespace(is_causal = True, num_key_value_groups = H // n_kv, training = True)+ q = torch.randn(1, H, T, D, device = "cuda", dtype = torch.bfloat16, requires_grad = True)+ k = torch.randn(1, n_kv, T, D, device = "cuda", dtype = torch.bfloat16, requires_grad = True)+ v = torch.randn(1, n_kv, T, D, device = "cuda", dtype = torch.bfloat16, requires_grad = True)+ mask = _block_causal(lengths, "cuda")+ psl = torch.tensor(lengths, dtype = torch.int32)+ before = hpa.HF_PACKED_ATTENTION_STATS["fast"]+ fast, _ = hpa._sdpa_packed_varlen(module, q, k, v, mask, scaling = 0.11, packed_seq_lengths = psl)+ assert hpa.HF_PACKED_ATTENTION_STATS["fast"] == before + 1+ ref, _ = sdpa_attention_forward(module, q, k, v, mask, scaling = 0.11)+ assert fast.shape == ref.shape+ torch.testing.assert_close(fast.float(), ref.float(), atol = 2e-2, rtol = 2e-2)+ g = torch.randn_like(ref)+ grads_fast = torch.autograd.grad(fast, (q, k, v), g)+ grads_ref = torch.autograd.grad(ref, (q, k, v), g)+ for a, b in zip(grads_fast, grads_ref):+ torch.testing.assert_close(a.float(), b.float(), atol = 5e-2, rtol = 5e-2)+++def test_new_lengths_tensor_with_a_cached_mask_is_checked_again():+ lengths = [3, 3]+ mask = _block_causal(lengths)+ assert hpa._mask_is_block_causal(mask, _cu(lengths), 6)+ # A different cu_seqlens object must not be compared elementwise against the cached one.+ assert hpa._mask_is_block_causal(mask, _cu(lengths), 6)+++def test_long_masks_are_checked_in_row_blocks(monkeypatch):+ monkeypatch.setattr(hpa, "_MASK_CHECK_ROWS", 4)+ lengths = [5, 7, 6]+ mask = _block_causal(lengths).clone()+ assert hpa._mask_is_block_causal(mask, _cu(lengths), 18)+ mask[0, 0, 17, 0] = True+ assert not hpa._mask_is_block_causal(mask, _cu(lengths), 18)+++@_CUDA+def test_compiling_leaves_the_counters_alone(monkeypatch):+ hpa._ORIG_SDPA[0] = _plain_sdpa+ monkeypatch.delenv("UNSLOTH_HF_PACKED_VARLEN", raising = False)+ lengths = [3, 3]+ module = types.SimpleNamespace(is_causal = True, num_key_value_groups = 1, training = False)+ q = torch.randn(1, 4, 6, 64, device = "cuda", dtype = torch.bfloat16)+ mask, psl = _block_causal(lengths, "cuda"), torch.tensor(lengths, dtype = torch.int32)+ fn = torch.compile(+ lambda q: hpa._sdpa_packed_varlen(module, q, q, q, mask, packed_seq_lengths = psl)[0],+ backend = "eager",+ fullgraph = True,+ )+ positional = torch.compile(+ lambda q: hpa._sdpa_packed_varlen(module, q, q, q, mask, 0.0, packed_seq_lengths = psl)[0],+ backend = "eager",+ fullgraph = True,+ )+ before = dict(hpa.HF_PACKED_ATTENTION_STATS)+ fn(q)+ positional(q)+ assert hpa.HF_PACKED_ATTENTION_STATS == before+++@_CUDA[email protected]("mode", ["training_no_grad", "eval_grad", "eval_no_grad"])+def test_int32_guard_follows_training_or_grad_mode(monkeypatch, mode):+ """A reentrant checkpoint's no_grad forward (training) and its recompute both fall back; a pure+ no_grad eval forward allocates no backward buffer and keeps varlen."""+ import unsloth.utils.attention_dispatch as ad++ seen, orig = _calls()+ hpa._ORIG_SDPA[0] = orig+ monkeypatch.delenv("UNSLOTH_HF_PACKED_VARLEN", raising = False)+ monkeypatch.setattr(ad, "_varlen_backward_overflows_int32", lambda *a: True)+ lengths = [3, 3]+ module = types.SimpleNamespace(is_causal = True, training = mode == "training_no_grad")+ q = torch.randn(1, 4, 6, 64, device = "cuda", dtype = torch.bfloat16)+ psl = torch.tensor(lengths, dtype = torch.int32)+ with torch.set_grad_enabled(mode == "eval_grad"):+ out = hpa._sdpa_packed_varlen(+ module, q, q, q, _block_causal(lengths, "cuda"), packed_seq_lengths = psl+ )+ if mode == "eval_no_grad":+ if hpa._torch_varlen(q.device) is None and not (ad.HAS_FLASH_ATTENTION or ad.HAS_XFORMERS):+ pytest.skip("needs a varlen kernel")+ assert out[0].shape == (1, 6, 4, 64)+ else:+ assert out == ("orig", None)+++@_CUDA+def test_a_refusing_torch_varlen_hands_the_call_back(monkeypatch):+ seen, orig = _calls()+ hpa._ORIG_SDPA[0] = orig+ monkeypatch.delenv("UNSLOTH_HF_PACKED_VARLEN", raising = False)++ def refuse(*args, **kwargs):+ raise RuntimeError("GQA is not supported with the cuDNN backend.")++ monkeypatch.setattr(hpa, "_TORCH_VARLEN", [refuse])+ monkeypatch.setattr(hpa, "_TORCH_VARLEN_DEVICES", {})+ lengths = [3, 3]+ q = torch.randn(1, 4, 6, 64, device = "cuda", dtype = torch.bfloat16)+ psl = torch.tensor(lengths, dtype = torch.int32)+ out = hpa._sdpa_packed_varlen(+ types.SimpleNamespace(is_causal = True),+ q,+ q,+ q,+ _block_causal(lengths, "cuda"),+ packed_seq_lengths = psl,+ )+ assert out == ("orig", None)+ assert hpa._TORCH_VARLEN == [None]+++def test_torch_varlen_kill_switch(monkeypatch):+ monkeypatch.setenv("UNSLOTH_TORCH_VARLEN", "0")+ monkeypatch.setattr(hpa, "_TORCH_VARLEN", [])+ monkeypatch.setattr(hpa, "_TORCH_VARLEN_GQA", [])+ assert hpa._torch_varlen(torch.device("cuda", 0)) is None+ assert hpa._TORCH_VARLEN == [None]+++@_CUDA[email protected]("policy", ["math_only", "flash_disabled", "math_first", "deterministic"])+def test_an_sdpa_policy_keeps_every_varlen_kernel_off(monkeypatch, policy):+ """The wrapped sdpa runs, not torch varlen nor the xformers / flash-attn dispatch; outside, varlen runs."""+ import contextlib+ from torch.nn.attention import SDPBackend, sdpa_kernel+ from unsloth.utils import attention_dispatch as ad++ if policy == "math_first" and "set_priority" not in inspect.signature(sdpa_kernel).parameters:+ pytest.skip("this torch has no sdpa_kernel(set_priority = True)")+ calls = []++ def spy(q, *args, **kwargs):+ calls.append(1)+ return torch.zeros_like(q)++ monkeypatch.setattr(hpa, "_TORCH_VARLEN", [spy])+ monkeypatch.setattr(hpa, "_TORCH_VARLEN_DEVICES", {})+ monkeypatch.setattr(ad, "run_attention", lambda **kwargs: calls.append(2) or kwargs["Q"])+ seen, orig = _calls()+ hpa._ORIG_SDPA[0] = orig+ monkeypatch.delenv("UNSLOTH_HF_PACKED_VARLEN", raising = False)+ lengths = [3, 3]+ q = torch.randn(1, 4, 6, 64, device = "cuda", dtype = torch.bfloat16)+ k = torch.randn(1, 2, 6, 64, device = "cuda", dtype = torch.bfloat16)+ module = types.SimpleNamespace(is_causal = True)++ def run():+ return hpa._sdpa_packed_varlen(+ module,+ q,+ k,+ k,+ _block_causal(lengths, "cuda"),+ packed_seq_lengths = torch.tensor(lengths, dtype = torch.int32),+ )++ if policy == "math_only":+ context = sdpa_kernel([SDPBackend.MATH])+ elif policy == "flash_disabled":+ context = sdpa_kernel([SDPBackend.EFFICIENT_ATTENTION, SDPBackend.MATH])+ elif policy == "math_first":+ context = sdpa_kernel([SDPBackend.MATH, SDPBackend.FLASH_ATTENTION], set_priority = True)+ else:+ context = contextlib.nullcontext()+ deterministic = torch.are_deterministic_algorithms_enabled()+ torch.use_deterministic_algorithms(True)+ try:+ with context:+ assert run() == ("orig", None)+ finally:+ if policy == "deterministic":+ torch.use_deterministic_algorithms(deterministic)+ assert calls == [] and len(seen) == 1+ run()+ assert calls == [1]diff --git a/unsloth/utils/hf_packed_attention.py b/unsloth/utils/hf_packed_attention.pynew file mode 100644index 00000000000..236ba5692b6--- /dev/null+++ b/unsloth/utils/hf_packed_attention.py@@ -0,0 +1,353 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.+"""Packed rows on the transformers modeling path run a varlen flash kernel (torch's, else xformers /+flash-attn) instead of SDPA over a dense block-causal mask; anything else reaches the wrapped "sdpa"."""++import inspect+import logging+import os+import weakref++import torch++__all__ = ["enable_hf_packed_attention", "HF_PACKED_ATTENTION_STATS"]++_FLAG = "_unsloth_hf_packed_varlen"+_ENV = "UNSLOTH_HF_PACKED_VARLEN"+_ORIG_SDPA = [None]+HF_PACKED_ATTENTION_STATS = {"fast": 0, "fallback": 0, "mask_checks": 0}+# All layers of a forward share one mask object: weakref, (cu_seqlens, versions), verdict.+_MASK_CHECK = [None, None, None]+_MASK_CHECK_ROWS = 1024+++def _disabled() -> bool:+ return os.environ.get(_ENV, "1").lower() in ("0", "false", "no", "off")+++def _mask_is_block_causal(attention_mask, cu_seqlens, total) -> bool:+ """Exactly causal inside each packed segment and closed across them; windows, bidirectional+ spans, pad tails and None (one document, already is_causal) keep the wrapped path."""+ if not (+ isinstance(attention_mask, torch.Tensor)+ and attention_mask.dtype == torch.bool+ and attention_mask.dim() == 4+ and attention_mask.shape[0] == 1+ and attention_mask.shape[1] == 1+ and attention_mask.shape[-2:] == (total, total)+ ):+ return False+ # Versions catch in-place edits; inference tensors have none, so are always rechecked.+ cacheable = not (torch.is_inference(attention_mask) or torch.is_inference(cu_seqlens))+ key = (attention_mask._version, cu_seqlens._version, total) if cacheable else None+ ref, checked, verdict = _MASK_CHECK+ if (+ cacheable+ and ref is not None+ and ref() is attention_mask+ and checked[0] is cu_seqlens+ and checked[1] == key+ ):+ return verdict+ HF_PACKED_ATTENTION_STATS["mask_checks"] += 1+ verdict = False+ if int(cu_seqlens[-1].item()) == total:+ lengths = (cu_seqlens[1:] - cu_seqlens[:-1]).to(torch.int64)+ segment = torch.repeat_interleave(+ torch.arange(lengths.numel(), device = attention_mask.device),+ lengths.to(attention_mask.device),+ )+ position = torch.arange(total, device = attention_mask.device)+ # Row blocks keep the reference to _MASK_CHECK_ROWS x T, not a second T x T mask.+ verdict = True+ for start in range(0, total, _MASK_CHECK_ROWS):+ rows = slice(start, min(start + _MASK_CHECK_ROWS, total))+ expected = (segment[rows, None] == segment[None, :]) & (+ position[None, :] <= position[rows, None]+ )+ if not torch.equal(attention_mask[0, 0, rows], expected):+ verdict = False+ break+ if cacheable:+ _MASK_CHECK[:] = [weakref.ref(attention_mask), (cu_seqlens, key), verdict]+ return verdict+++def _autocast_dtype_differs(query) -> bool:+ try:+ enabled = torch.is_autocast_enabled(query.device.type)+ dtype = torch.get_autocast_dtype(query.device.type) if enabled else None+ except (AttributeError, TypeError):+ enabled = torch.is_autocast_enabled()+ dtype = torch.get_autocast_gpu_dtype() if enabled else None+ return enabled and dtype != query.dtype+++def _sliding_or_softcapped(module, kwargs) -> bool:+ if kwargs.get("sliding_window") or getattr(module, "sliding_window", None):+ return True+ if kwargs.get("softcap") or getattr(module, "sinks", None) is not None:+ return True+ if getattr(module, "attn_logit_softcapping", None):+ return True+ config = getattr(module, "config", None)+ return bool(getattr(config, "attn_logit_softcapping", None))+++_TORCH_VARLEN = []+_TORCH_VARLEN_GQA = []+_TORCH_VARLEN_DEVICES = {}+++def _torch_varlen(device):+ """torch's own flash varlen (a third of xformers' host cost per call), else None."""+ if not _TORCH_VARLEN:+ fn, native_gqa = None, False+ try:+ # attention_dispatch's kill switch for torch's varlen_attn covers this path too.+ if os.environ.get("UNSLOTH_TORCH_VARLEN", "1").lower() in ("0", "false", "no", "off"):+ raise ImportError+ from torch.nn.attention.varlen import varlen_attn++ # 2.11 added scale / window_size, 2.12 enable_gqa; 2.10 only had is_causal and is not used.+ parameters = inspect.signature(varlen_attn).parameters.keys()+ if {"scale", "window_size"} <= parameters:+ fn, native_gqa = varlen_attn, "enable_gqa" in parameters+ except Exception:+ fn = None+ _TORCH_VARLEN.append(fn)+ _TORCH_VARLEN_GQA[:] = [native_gqa]+ fn = _TORCH_VARLEN[0]+ if fn is None or torch.version.hip is not None:+ return None+ ok = _TORCH_VARLEN_DEVICES.get(device.index)+ if ok is None:+ ok = torch.cuda.get_device_capability(device)[0] >= 8+ _TORCH_VARLEN_DEVICES[device.index] = ok+ return fn if ok else None+++_SDPA_MATH = []+++def _sdpa_requested(query, key) -> bool:+ """Deterministic mode, a disabled flash SDPA backend or a math-first sdpa_kernel priority asks for+ SDPA itself, so no varlen kernel (torch's, xformers or flash-attn) may replace it."""+ if torch.are_deterministic_algorithms_enabled():+ return True+ if not _SDPA_MATH:+ from torch.nn.attention import SDPBackend+ from .attention_dispatch import _sdpa_backend_state+ _SDPA_MATH[:] = [int(SDPBackend.MATH), _sdpa_backend_state]+ math, backend_state = _SDPA_MATH+ n_heads, n_kv_heads = query.shape[1], key.shape[1]+ flash, choice = backend_state(+ str(query.device),+ query.dtype,+ query.shape[-1],+ n_heads,+ n_kv_heads,+ True,+ n_heads != n_kv_heads,+ )+ return not flash or choice == math+++def _sdpa_packed_varlen(+ module,+ query,+ key,+ value,+ attention_mask,+ *args,+ dropout = 0.0,+ scaling = None,+ is_causal = None,+ **kwargs,+):+ orig = _ORIG_SDPA[0]+ if args:+ # Positional dropout / scaling / is_causal (and later parameters) go back exactly as given.+ named = {"dropout": dropout, "scaling": scaling, "is_causal": is_causal}+ for name in list(named)[: len(args)]:+ named.pop(name)+ if kwargs.get("packed_seq_lengths") is not None and not torch.compiler.is_compiling():+ HF_PACKED_ATTENTION_STATS["fallback"] += 1+ return orig(module, query, key, value, attention_mask, *args, **named, **kwargs)+ # Before any counter while tracing: Dynamo would guard on it and recompile every forward.+ if kwargs.get("packed_seq_lengths") is None or _disabled() or torch.compiler.is_compiling():+ return orig(+ module,+ query,+ key,+ value,+ attention_mask,+ dropout = dropout,+ scaling = scaling,+ is_causal = is_causal,+ **kwargs,+ )+ from .attention_dispatch import (+ _VARLEN_INT32_GUARD_DISABLED,+ FLASH_VARLEN,+ XFORMERS,+ _varlen_backward_overflows_int32,+ select_attention_backend,+ )+ from .packing import get_packed_info_from_kwargs++ bsz, n_heads, q_len, head_dim = query.shape+ seq_info = torch_varlen = backend = None+ if (+ bsz == 1+ and q_len > 0+ and query.is_cuda+ and query.device == key.device == value.device+ and key.shape[0] == value.shape[0] == 1+ and key.shape[1] == value.shape[1]+ and key.shape[2] == value.shape[2] == q_len+ # Not MLA (192 / 128) nor Gemma-4 global (512).+ and key.shape[-1] == value.shape[-1] == head_dim+ and head_dim <= 256+ and head_dim % 8 == 0+ and query.dtype in (torch.float16, torch.bfloat16)+ and query.dtype == key.dtype == value.dtype+ and not _autocast_dtype_differs(query)+ and not dropout+ and is_causal is not False+ and getattr(module, "is_causal", True)+ and kwargs.get("position_bias") is None+ and kwargs.get("cache") is None+ and kwargs.get("past_key_values") is None+ and n_heads % key.shape[1] == 0+ and not _sliding_or_softcapped(module, kwargs)+ and not _sdpa_requested(query, key)+ ):+ torch_varlen = _torch_varlen(query.device)+ backend = select_attention_backend(use_varlen = True)+ # Flash-2 varlen backward faults past int32 indexing. Keyed on training / grad mode, not+ # requires_grad, so a reentrant checkpoint's no_grad forward and its recompute agree.+ if (+ not _VARLEN_INT32_GUARD_DISABLED+ and (getattr(module, "training", False) or torch.is_grad_enabled())+ and _varlen_backward_overflows_int32(+ kwargs["packed_seq_lengths"].numel(), q_len, n_heads, head_dim+ )+ ):+ torch_varlen, backend = None, None+ if torch_varlen is not None or backend in (FLASH_VARLEN, XFORMERS):+ seq_info = get_packed_info_from_kwargs(kwargs, query.device)+ if seq_info is None or not _mask_is_block_causal(attention_mask, seq_info[1], q_len):+ HF_PACKED_ATTENTION_STATS["fallback"] += 1+ return orig(+ module,+ query,+ key,+ value,+ attention_mask,+ dropout = dropout,+ scaling = scaling,+ is_causal = is_causal,+ **kwargs,+ )+ HF_PACKED_ATTENTION_STATS["fast"] += 1+ scale = head_dim**-0.5 if scaling is None else scaling+ n_kv_heads = key.shape[1]+ _, cu_seqlens, max_seqlen = seq_info+ if torch_varlen is not None:+ gqa, k_in, v_in = {}, key, value+ if n_kv_heads != n_heads:+ if not _TORCH_VARLEN_GQA or _TORCH_VARLEN_GQA[0]:+ gqa = {"enable_gqa": True}+ else:+ # torch 2.11: no enable_gqa, so repeat the K/V heads as transformers' sdpa path does.+ k_in = key.repeat_interleave(n_heads // n_kv_heads, dim = 1)+ v_in = value.repeat_interleave(n_heads // n_kv_heads, dim = 1)+ try:+ out = torch_varlen(+ query[0].transpose(0, 1),+ k_in[0].transpose(0, 1),+ v_in[0].transpose(0, 1),+ cu_seqlens,+ cu_seqlens,+ max_seqlen,+ max_seqlen,+ scale = scale,+ window_size = (-1, 0),+ **gqa,+ )+ except torch.OutOfMemoryError:+ raise+ except RuntimeError as exc:+ # A build whose varlen picks a backend without causal GQA (cuDNN) refuses at forward: stop+ # using it and give this call to the wrapped sdpa.+ _TORCH_VARLEN[:] = [None]+ logging.getLogger(__name__).info(f"Unsloth: torch varlen_attn unavailable ({exc})")+ HF_PACKED_ATTENTION_STATS["fast"] -= 1+ HF_PACKED_ATTENTION_STATS["fallback"] += 1+ return orig(+ module,+ query,+ key,+ value,+ attention_mask,+ dropout = dropout,+ scaling = scaling,+ is_causal = is_causal,+ **kwargs,+ )+ return out.unsqueeze(0), None+ from .attention_dispatch import AttentionConfig, AttentionContext, run_attention++ config = AttentionConfig(+ backend = backend,+ n_kv_heads = n_kv_heads,+ n_groups = n_heads // n_kv_heads,+ flash_varlen_kwargs = {"dropout_p": 0.0, "causal": True, "softmax_scale": scale},+ sdpa_kwargs = {"scale": scale},+ xformers_kwargs = {"scale": scale},+ )+ context = AttentionContext(+ bsz = 1,+ q_len = q_len,+ kv_seq_len = q_len,+ n_heads = n_heads,+ head_dim = head_dim,+ # Not grad mode: xformers needs a backward op for grad tensors even under no_grad (reentrant+ # checkpointing), and that op rejects the grouped 5D layout.+ requires_grad = query.requires_grad or key.requires_grad or value.requires_grad,+ seq_info = seq_info,+ attention_mask = None,+ causal_mask = None,+ )+ out = run_attention(config = config, context = context, Q = query, K = key, V = value)+ return out.contiguous(), None+++def enable_hf_packed_attention() -> bool:+ """Wrap transformers' "sdpa" attention entry once. True when the wrapper is installed."""+ if _disabled():+ return False+ try:+ from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS+ except Exception:+ return False+ current = ALL_ATTENTION_FUNCTIONS.get("sdpa", None)+ if current is None:+ return False+ # Once per process: gemma-4's router stacked above does not forward our sentinel, and re-wrapping+ # it would loop through its boxed original.+ if getattr(current, _FLAG, False) or _ORIG_SDPA[0] is not None:+ return True+ _ORIG_SDPA[0] = current+ # Zoo sdpa routers re-run per load and check their sentinel on the installed entry: forward them.+ try:+ for name, value in vars(current).items():+ if name.startswith(("_unsloth_", "__unsloth_")):+ setattr(_sdpa_packed_varlen, name, value)+ except Exception:+ pass+ setattr(_sdpa_packed_varlen, _FLAG, True)+ # Direct assignment: register() does not update the mapping layers read.+ ALL_ATTENTION_FUNCTIONS["sdpa"] = _sdpa_packed_varlen+ return Truediff --git a/unsloth/utils/packing.py b/unsloth/utils/packing.pyindex 08525ad9345..afc585ae1f2 100644--- a/unsloth/utils/packing.py+++ b/unsloth/utils/packing.py@@ -194,6 +194,15 @@ def configure_padding_free(config):setattr(config, "remove_unused_columns", False)+def _enable_hf_packed_attention() -> None:+ # Packed rows on the transformers modeling path run varlen instead of SDPA over a dense mask.+ try:+ from .hf_packed_attention import enable_hf_packed_attention+ enable_hf_packed_attention()+ except Exception as exc:+ logging.getLogger(__name__).info(f"Unsloth: packed varlen attention not enabled ({exc})")++def enable_sample_packing(model,trainer,@@ -212,6 +221,7 @@ def enable_sample_packing(collator = getattr(trainer, "data_collator", None)if collator is None or not hasattr(collator, "torch_call"):return+ _enable_hf_packed_attention()if getattr(collator, "_unsloth_packing_wrapped", False):return@@ -263,6 +273,7 @@ def enable_padding_free_metadata(model, trainer):returnmark_allow_overlength(model)+ _enable_hf_packed_attention()if hasattr(collator, "return_position_ids"):collator.return_position_ids = Trueif hasattr(trainer, "args") and hasattr(trainer.args, "remove_unused_columns"):