Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

86.3 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
"""Packed rows on the transformers modeling path run varlen attention, and nothing else changes."""

import contextlib
import inspect
import types

import pytest
import torch
from real_accelerator import has_real_cuda  # tests/_shared, on sys.path via tests/conftest.py

from unsloth.utils import hf_packed_attention as hpa


def _block_causal(lengths, device = "cpu"):
    segment = torch.repeat_interleave(torch.arange(len(lengths)), torch.tensor(lengths)).to(device)
    return ((segment[:, None] == segment[None, :]).tril())[None, None]


def _hf_window_mask(
    lengths,
    window,
    device = "cpu",
):
    """The mask transformers builds for a packed row on a sliding layer: its own mask functions."""
    from transformers.masking_utils import (
        and_masks,
        find_packed_sequence_indices,
        packed_sequence_mask_function,
        sliding_window_causal_mask_function,
    )

    position_ids = torch.cat([torch.arange(n) for n in lengths])[None].to(device)
    packed = find_packed_sequence_indices(position_ids)
    fn = and_masks(
        sliding_window_causal_mask_function(window), packed_sequence_mask_function(packed)
    )
    idx = torch.arange(sum(lengths), device = device)
    zero = torch.zeros((), dtype = torch.long, device = device)
    return fn(zero, zero, idx[:, None], idx[None, :])[None, None]


def _cu(lengths, device = "cpu"):
    return torch.tensor(
        [0] + list(torch.tensor(lengths).cumsum(0)), dtype = torch.int32, device = device
    )


@pytest.fixture(autouse = True)
def _fresh_state():
    saved = hpa._ORIG_SDPA[0]
    hpa._MASK_CHECK.clear()
    yield
    hpa._MASK_CHECK.clear()
    hpa._ORIG_SDPA[0] = saved


def test_block_causal_mask_is_recognised():
    lengths = [3, 5, 2]
    assert hpa._mask_is_block_causal(_block_causal(lengths), _cu(lengths), 10)


@pytest.mark.parametrize(
    "edit",
    ["cross_document", "bidirectional", "window", "plain_causal", "pad_tail", "float"],
)
def test_any_other_mask_keeps_the_masked_path(edit):
    lengths = [3, 5, 2]
    mask, cu, total = _block_causal(lengths), _cu(lengths), 10
    if edit == "cross_document":
        mask = mask.clone()
        mask[0, 0, 4, 1] = True
    elif edit == "bidirectional":
        mask = mask.clone()
        mask[0, 0, 3, 4] = True
    elif edit == "window":
        mask = mask.clone()
        mask[0, 0, 7, 3] = False
    elif edit == "plain_causal":
        mask = torch.ones(10, 10, dtype = torch.bool).tril()[None, None]
    elif edit == "pad_tail":
        cu = _cu([3, 5])
    elif edit == "float":
        mask = torch.where(mask, 0.0, float("-inf"))
    assert not hpa._mask_is_block_causal(mask, cu, total)


@pytest.mark.parametrize("window", [1, 3, 4, 128])
def test_windowed_mask_is_recognised_only_with_its_window(window):
    lengths = [2, 7, 130, 4]
    mask, cu, total = _hf_window_mask(lengths, window), _cu(lengths), sum(lengths)
    assert hpa._mask_is_block_causal(mask, cu, total, window)
    # One off either way, or no window at all, is a different band for the 130-token document.
    for wrong in (window - 1 or None, window + 1, None):
        hpa._MASK_CHECK.clear()
        assert not hpa._mask_is_block_causal(mask, cu, total, wrong)


def test_documents_inside_the_window_are_plain_block_causal():
    lengths = [3, 4, 2]
    mask = _hf_window_mask(lengths, 4)
    assert torch.equal(mask, _block_causal(lengths))
    assert hpa._mask_is_block_causal(mask, _cu(lengths), 9, 4)


def test_sliding_and_full_layers_keep_their_own_verdicts():
    lengths = [6, 9]
    cu = _cu(lengths)
    full, sliding = _block_causal(lengths), _hf_window_mask(lengths, 3)
    for _ in range(3):
        assert hpa._mask_is_block_causal(full, cu, 15)
        assert hpa._mask_is_block_causal(sliding, cu, 15, 3)
    assert hpa.HF_PACKED_ATTENTION_STATS["mask_checks"] >= 2
    checks = hpa.HF_PACKED_ATTENTION_STATS["mask_checks"]
    assert hpa._mask_is_block_causal(full, cu, 15)
    assert hpa._mask_is_block_causal(sliding, cu, 15, 3)
    assert hpa.HF_PACKED_ATTENTION_STATS["mask_checks"] == checks
    # The same mask object asked about another window is checked, not answered from the cache.
    assert not hpa._mask_is_block_causal(full, cu, 15, 3)


def test_mask_verdict_is_cached_per_mask_object():
    lengths = [4, 4]
    mask, cu = _block_causal(lengths), _cu(lengths)
    assert hpa._mask_is_block_causal(mask, cu, 8)
    # Same object: the cached verdict stands. A new object is checked again.
    assert hpa._mask_is_block_causal(mask, cu, 8)
    other = torch.ones(8, 8, dtype = torch.bool).tril()[None, None]
    assert not hpa._mask_is_block_causal(other, cu, 8)


def _calls():
    seen = []

    def orig(
        module,
        q,
        k,
        v,
        mask,
        dropout = 0.0,
        scaling = None,
        is_causal = None,
        **kwargs,
    ):
        seen.append(kwargs)
        return "orig", None

    return seen, orig


def test_unpacked_calls_go_straight_through(monkeypatch):
    seen, orig = _calls()
    hpa._ORIG_SDPA[0] = orig
    q = torch.zeros(2, 4, 6, 8)
    out = hpa._sdpa_packed_varlen(types.SimpleNamespace(), q, q, q, None, scaling = 0.3, foo = 1)
    assert out == ("orig", None) and seen == [{"foo": 1}]


_CUDA = pytest.mark.skipif(not has_real_cuda(), reason = "needs a CUDA device")
# torch varlen is never used on ROCm (_torch_varlen returns None there).
_TORCH_VARLEN_HOST = pytest.mark.skipif(
    torch.version.hip is not None, reason = "torch varlen is off on ROCm"
)


@_CUDA
@pytest.mark.parametrize(
    "why",
    [
        "none",
        "window_inside_documents",
        "sliding_wrong_window",
        "zero_window",
        "float_window",
        "softcap",
        "sinks",
        "fp32",
        "dropout",
        "batched",
        "cpu",
        "head_dim_640",
        "mla_value_width",
        "autocast_dtype",
        "paged_cache",
        "no_mask",
        "module_softcap",
        "not_causal",
        "positional",
        "kv_batched",
    ],
)
def test_packed_calls_that_varlen_cannot_express_fall_back(monkeypatch, why):
    """Each reason alone sends the call to the wrapped sdpa; "none" is the control that takes varlen."""
    from unsloth.utils.attention_dispatch import HAS_FLASH_ATTENTION, HAS_XFORMERS

    if not (HAS_FLASH_ATTENTION or HAS_XFORMERS or hpa._torch_varlen(torch.device("cuda", 0))):
        pytest.skip("needs a varlen kernel")
    seen, orig = _calls()
    hpa._ORIG_SDPA[0] = orig
    monkeypatch.delenv("UNSLOTH_HF_PACKED_VARLEN", raising = False)
    lengths, D = [3, 3], 64
    module = types.SimpleNamespace(is_causal = True)
    device, dtype, B = "cuda", torch.bfloat16, 1
    kwargs = {"packed_seq_lengths": torch.tensor(lengths, dtype = torch.int32)}
    dropout, mask, v_width = 0.0, _block_causal(lengths, "cuda"), D
    if why == "window_inside_documents":
        # Both documents fit the window, so the band is the plain block-causal mask.
        module.sliding_window = 4
    elif why == "sliding_wrong_window":
        # The layer says 2, the mask has no band.
        module.sliding_window = 2
    elif why == "zero_window":
        kwargs["sliding_window"] = 0
    elif why == "float_window":
        module.sliding_window = 4.0
    elif why == "softcap":
        module.config = types.SimpleNamespace(attn_logit_softcapping = 50.0)
    elif why == "module_softcap":
        module.attn_logit_softcapping = 30.0
    elif why == "sinks":
        module.sinks = torch.zeros(4)
    elif why == "fp32":
        dtype = torch.float32
    elif why == "dropout":
        dropout = 0.1
    elif why == "batched":
        B = 2
    elif why == "cpu":
        device, mask = "cpu", _block_causal(lengths)
    elif why == "head_dim_640":
        # Past both flash (256) and the efficient route's tested width (512).
        D = v_width = 640
    elif why == "mla_value_width":
        D, v_width = 192, 128
    elif why == "paged_cache":
        kwargs["cache"] = object()
    elif why == "no_mask":
        mask = None
    q = torch.randn(B, 4, 6, D, device = device, dtype = dtype)
    v = torch.randn(B, 4, 6, v_width, device = device, dtype = dtype)
    k = q
    if why == "kv_batched":
        k, v = q.expand(2, -1, -1, -1), v.expand(2, -1, -1, -1)
    before = dict(hpa.HF_PACKED_ATTENTION_STATS)
    if why == "autocast_dtype":
        with torch.autocast("cuda", dtype = torch.float16):
            out = hpa._sdpa_packed_varlen(module, q, k, v, mask, dropout = dropout, **kwargs)
    elif why == "not_causal":
        out = hpa._sdpa_packed_varlen(module, q, k, v, mask, is_causal = False, **kwargs)
    elif why == "positional":
        out = hpa._sdpa_packed_varlen(module, q, k, v, mask, 0.0, **kwargs)
    else:
        out = hpa._sdpa_packed_varlen(module, q, k, v, mask, dropout = dropout, **kwargs)
    if why in ("none", "window_inside_documents"):
        assert out[0].shape == (1, 6, 4, D) and seen == []
        assert hpa.HF_PACKED_ATTENTION_STATS["fast"] == before["fast"] + 1
    else:
        assert out == ("orig", None)
        assert hpa.HF_PACKED_ATTENTION_STATS["fallback"] == before["fallback"] + 1


def test_an_edited_mask_is_checked_again():
    lengths = [3, 3]
    mask, cu = _block_causal(lengths).clone(), _cu(lengths)
    assert hpa._mask_is_block_causal(mask, cu, 6)
    mask[0, 0, 3, 0] = True
    assert not hpa._mask_is_block_causal(mask, cu, 6)
    with torch.inference_mode():
        frozen, frozen_cu = _block_causal(lengths), _cu(lengths)
    assert hpa._mask_is_block_causal(frozen, frozen_cu, 6)


def test_a_missing_mask_is_not_block_causal():
    assert not hpa._mask_is_block_causal(None, _cu([6]), 6)


def _plain_sdpa(
    module,
    q,
    k,
    v,
    mask,
    dropout = 0.0,
    scaling = None,
    is_causal = None,
    **kwargs,
):
    # Not transformers' sdpa_attention_forward: compiling that after importing unsloth trips a Dynamo
    # guard bug on torch <= 2.13, with or without this wrapper.
    out = torch.nn.functional.scaled_dot_product_attention(q, k, v, attn_mask = mask, scale = scaling)
    return out.transpose(1, 2), None


@_CUDA
def test_fullgraph_compile_keeps_the_wrapped_sdpa(monkeypatch):
    sdpa_attention_forward = _plain_sdpa
    hpa._ORIG_SDPA[0] = sdpa_attention_forward
    monkeypatch.delenv("UNSLOTH_HF_PACKED_VARLEN", raising = False)
    lengths = [3, 3]
    module = types.SimpleNamespace(is_causal = True, num_key_value_groups = 1, training = False)
    q = torch.randn(1, 4, 6, 64, device = "cuda", dtype = torch.bfloat16)
    mask, psl = _block_causal(lengths, "cuda"), torch.tensor(lengths, dtype = torch.int32)
    fn = torch.compile(
        lambda q: hpa._sdpa_packed_varlen(module, q, q, q, mask, packed_seq_lengths = psl)[0],
        backend = "eager",
        fullgraph = True,
    )
    torch.testing.assert_close(fn(q), sdpa_attention_forward(module, q, q, q, mask)[0])


@_CUDA
@pytest.mark.parametrize("route", ["torch", "dispatch"])
def test_reentrant_checkpoint_of_grouped_heads(monkeypatch, route):
    from unsloth.utils.attention_dispatch import HAS_FLASH_ATTENTION, HAS_XFORMERS

    if route == "torch" and hpa._torch_varlen(torch.device("cuda", 0)) is None:
        pytest.skip("this torch has no window_size / enable_gqa varlen_attn")
    if route == "dispatch":
        if not (HAS_FLASH_ATTENTION or HAS_XFORMERS):
            pytest.skip("needs xformers or flash-attn")
        monkeypatch.setattr(hpa, "_torch_varlen", lambda device: None)
    from torch.utils.checkpoint import checkpoint
    from transformers.integrations.sdpa_attention import sdpa_attention_forward

    hpa._ORIG_SDPA[0] = sdpa_attention_forward
    monkeypatch.delenv("UNSLOTH_HF_PACKED_VARLEN", raising = False)
    lengths = [40, 24]
    module = types.SimpleNamespace(is_causal = True, num_key_value_groups = 4, training = True)
    q = torch.randn(1, 8, 64, 64, device = "cuda", dtype = torch.bfloat16, requires_grad = True)
    k = torch.randn(1, 2, 64, 64, device = "cuda", dtype = torch.bfloat16, requires_grad = True)
    v = torch.randn(1, 2, 64, 64, device = "cuda", dtype = torch.bfloat16, requires_grad = True)
    mask, psl = _block_causal(lengths, "cuda"), torch.tensor(lengths, dtype = torch.int32)
    fn = lambda q, k, v: hpa._sdpa_packed_varlen(module, q, k, v, mask, packed_seq_lengths = psl)[0]
    out = checkpoint(fn, q, k, v, use_reentrant = True)
    out.float().sum().backward()
    ref = sdpa_attention_forward(module, q, k, v, mask)[0]
    torch.testing.assert_close(out.float(), ref.float(), atol = 2e-2, rtol = 2e-2)


def test_kill_switch(monkeypatch):
    seen, orig = _calls()
    hpa._ORIG_SDPA[0] = orig
    monkeypatch.setenv("UNSLOTH_HF_PACKED_VARLEN", "0")
    q = torch.zeros(1, 4, 6, 8, dtype = torch.bfloat16)
    lengths = torch.tensor([3, 3], dtype = torch.int32)
    out = hpa._sdpa_packed_varlen(
        types.SimpleNamespace(), q, q, q, _block_causal([3, 3]), packed_seq_lengths = lengths
    )
    assert out == ("orig", None)
    assert not hpa.enable_hf_packed_attention()


def test_install_forwards_router_sentinels_and_runs_once(monkeypatch):
    mapping_mod = pytest.importorskip("transformers.modeling_utils")
    mapping = mapping_mod.ALL_ATTENTION_FUNCTIONS

    def router(*args, **kwargs):
        return None

    router._unsloth_gemma4_flash = True
    monkeypatch.setitem(mapping, "sdpa", router)
    hpa._ORIG_SDPA[0] = None
    monkeypatch.delenv("UNSLOTH_HF_PACKED_VARLEN", raising = False)
    assert hpa.enable_hf_packed_attention()
    installed = mapping["sdpa"]
    assert installed is hpa._sdpa_packed_varlen and hpa._ORIG_SDPA[0] is router
    # A zoo router re-running on the next load sees its own sentinel and leaves the chain alone.
    assert getattr(installed, "_unsloth_gemma4_flash", False)
    # A router stacked above later never gets wrapped a second time.
    monkeypatch.setitem(mapping, "sdpa", router)
    assert hpa.enable_hf_packed_attention()
    assert mapping["sdpa"] is router and hpa._ORIG_SDPA[0] is router


@_CUDA
@pytest.mark.parametrize("window", [None, 4])
def test_rocm_keeps_windowed_rows_off_flash_attn(monkeypatch, window):
    """flash-attn's ROCm backend lists sliding windows as work in progress."""
    from unsloth.utils import attention_dispatch as ad

    seen, orig = _calls()
    hpa._ORIG_SDPA[0] = orig
    dispatched = []
    monkeypatch.delenv("UNSLOTH_HF_PACKED_VARLEN", raising = False)
    monkeypatch.setattr(torch.version, "hip", "6.4")
    monkeypatch.setattr(ad, "select_attention_backend", lambda use_varlen = False: ad.FLASH_VARLEN)
    monkeypatch.setattr(ad, "run_attention", lambda **kwargs: dispatched.append(1) or kwargs["Q"])
    lengths = [3, 5]
    q = torch.randn(1, 4, 8, 64, device = "cuda", dtype = torch.bfloat16)
    mask = (
        _block_causal(lengths, "cuda")
        if window is None
        else _hf_window_mask(lengths, window, "cuda")
    )
    module = types.SimpleNamespace(is_causal = True, num_key_value_groups = 1, sliding_window = window)
    hpa._sdpa_packed_varlen(
        module, q, q, q, mask, packed_seq_lengths = torch.tensor(lengths, dtype = torch.int32)
    )
    if window is None:
        assert dispatched and not seen
    else:
        assert seen and not dispatched


def _block_flag(
    monkeypatch,
    value,
    leaves = None,
):
    monkeypatch.setattr(hpa, "IN_COMPILED_BLOCK", [value])
    if leaves is not None:
        monkeypatch.setattr(hpa, "_leaves_block", lambda *args: leaves)


@pytest.mark.parametrize("in_block,leaves", [(True, True), (True, None), (False, True)])
def test_packed_rows_leave_a_compiled_block_for_the_eager_wrapper(monkeypatch, in_block, leaves):
    """Inside an Unsloth compiled block a packed call the predicate sends out runs the wrapper
    eagerly (one graph break); CPU rows and a user's own compile keep sdpa in the graph."""
    hpa._ORIG_SDPA[0] = _plain_sdpa
    monkeypatch.delenv("UNSLOTH_HF_PACKED_VARLEN", raising = False)
    _block_flag(monkeypatch, in_block, leaves)
    leaving = in_block and leaves is True
    lengths = [3, 3]
    module = types.SimpleNamespace(is_causal = True, num_key_value_groups = 1, training = False)
    q = torch.randn(1, 4, 6, 16)
    mask, psl = _block_causal(lengths), torch.tensor(lengths, dtype = torch.int32)
    torch._dynamo.reset()
    torch._dynamo.utils.counters.clear()
    fn = torch.compile(
        lambda q: hpa._sdpa_packed_varlen(module, q, q, q, mask, packed_seq_lengths = psl)[0] * 2,
        backend = "eager",
    )
    before = dict(hpa.HF_PACKED_ATTENTION_STATS)
    out = fn(q)
    torch.testing.assert_close(out, _plain_sdpa(module, q, q, q, mask)[0] * 2)
    breaks = sum(torch._dynamo.utils.counters["graph_break"].values())
    # CPU rows take the eager wrapper's own fallback, counted only outside a trace.
    if leaving:
        assert hpa.HF_PACKED_ATTENTION_STATS["fallback"] == before["fallback"] + 1
        assert breaks >= 1
    else:
        assert hpa.HF_PACKED_ATTENTION_STATS == before
        assert breaks == 0
    # Unpacked rows stay in the graph either way.
    torch._dynamo.reset()
    torch._dynamo.utils.counters.clear()
    plain = torch.compile(
        lambda q: hpa._sdpa_packed_varlen(module, q, q, q, mask)[0], backend = "eager", fullgraph = True
    )
    plain(q)
    assert hpa.HF_PACKED_ATTENTION_STATS["fallback"] == before["fallback"] + int(leaving)


@pytest.mark.parametrize(
    "args",
    [
        ("cpu", None, torch.bfloat16, 64, None),
        ("cuda", 0, torch.bfloat16, 64, -1),  # a window varlen cannot take
        ("cuda", 0, torch.bfloat16, 512, None),  # past flash
        ("cuda", 0, torch.bfloat16, 60, None),
    ],
)
def test_compiled_block_keeps_rows_without_a_varlen_win(monkeypatch, args):
    monkeypatch.setattr(hpa, "_LEAVES_BLOCK", {})
    assert hpa._leaves_block_eager(*args) is False


def test_compiled_block_keeps_rows_on_torch_with_cheap_cudnn_plans(monkeypatch):
    monkeypatch.setattr(hpa, "_SLOW_CUDNN_PLANS", False)
    monkeypatch.setattr(hpa, "_LEAVES_BLOCK", {})
    hpa._LEAVES_BLOCK.clear()
    assert hpa._leaves_block_eager("cuda", 0, torch.bfloat16, 64, None) is False


def test_compiled_block_verdict_is_computed_eagerly_while_tracing(monkeypatch):
    """The probe runs real tensors once per key; traced instead (an lru_cache wrapper is traced
    through __wrapped__) it saw fake tensors and kept every row in the graph."""
    calls = []

    def probe(*key):
        calls.append(type(torch.zeros(1)) is torch.Tensor)
        return True

    monkeypatch.setattr(hpa, "_leaves_block_probe", probe)
    monkeypatch.setattr(hpa, "_LEAVES_BLOCK", {})
    torch._dynamo.reset()
    fn = torch.compile(
        lambda x: x * 2 if hpa._leaves_block("cuda", 0, torch.bfloat16, 64, None) else x,
        backend = "eager",
        fullgraph = True,
    )
    torch.testing.assert_close(fn(torch.ones(3)), torch.full((3,), 2.0))
    fn(torch.ones(3))
    assert calls == [True]


@_CUDA
@_TORCH_VARLEN_HOST
def test_packed_rows_in_a_compiled_block_take_varlen(monkeypatch):
    """Real predicate, evaluated while tracing: it must agree with the eager verdict, and a row it
    sends out takes the varlen kernel."""
    if hpa._torch_varlen(torch.device("cuda", 0)) is None:
        pytest.skip(
            reason = "varlen kernel needs torch's varlen_attn with window_size (CUDA torch >= 2.11)"
        )
    from transformers.integrations.sdpa_attention import sdpa_attention_forward

    hpa._ORIG_SDPA[0] = sdpa_attention_forward
    monkeypatch.delenv("UNSLOTH_HF_PACKED_VARLEN", raising = False)
    _block_flag(monkeypatch, True)
    monkeypatch.setattr(hpa, "_LEAVES_BLOCK", {})
    torch.manual_seed(0)
    lengths = [37, 100, 2, 5, 230]
    T, H, n_kv, D = sum(lengths), 8, 2, 64
    module = types.SimpleNamespace(is_causal = True, num_key_value_groups = H // n_kv, training = True)
    q = torch.randn(1, H, T, D, device = "cuda", dtype = torch.bfloat16, requires_grad = True)
    k = torch.randn(1, n_kv, T, D, device = "cuda", dtype = torch.bfloat16, requires_grad = True)
    v = torch.randn(1, n_kv, T, D, device = "cuda", dtype = torch.bfloat16, requires_grad = True)
    mask, psl = _block_causal(lengths, "cuda"), torch.tensor(lengths, dtype = torch.int32)
    torch._dynamo.reset()
    fn = torch.compile(
        lambda q, k, v: hpa._sdpa_packed_varlen(
            module, q, k, v, mask, scaling = 0.11, packed_seq_lengths = psl
        )[0],
    )
    for t, dims in ((q, (2,)), (k, (2,)), (v, (2,)), (mask, (2, 3))):
        torch._dynamo.mark_dynamic(t, dims)  # as _block_compile marks the sequence
    before = hpa.HF_PACKED_ATTENTION_STATS["fast"]
    fast = fn(q, k, v)
    expected = hpa._leaves_block_probe("cuda", 0, torch.bfloat16, D, None)
    assert hpa._LEAVES_BLOCK[("cuda", 0, torch.bfloat16, D, None)] is expected
    assert hpa.HF_PACKED_ATTENTION_STATS["fast"] == before + int(expected)
    if hpa._SLOW_CUDNN_PLANS and torch.cuda.get_device_capability(0)[0] == 10:
        assert expected  # the case this exists for: SM100 picks cuDNN for the dense mask
    ref, _ = sdpa_attention_forward(module, q, k, v, mask, scaling = 0.11)
    torch.testing.assert_close(fast.float(), ref.float(), atol = 2e-2, rtol = 2e-2)


@_CUDA
@pytest.mark.parametrize("window", [None, 3, 4, 128])
@pytest.mark.parametrize("route", ["torch", "torch_2_11", "dispatch"])
@pytest.mark.parametrize("n_kv", [8, 2])
def test_varlen_matches_sdpa_over_the_dense_mask(monkeypatch, n_kv, route, window):
    from unsloth.utils.attention_dispatch import HAS_FLASH_ATTENTION, HAS_XFORMERS

    if route != "dispatch" and hpa._torch_varlen(torch.device("cuda", 0)) is None:
        pytest.skip("this torch has no window_size varlen_attn")
    if route == "torch_2_11":
        # torch 2.11's varlen_attn has scale / window_size but no enable_gqa.
        native = hpa._TORCH_VARLEN[0]

        def without_gqa(*args, **kwargs):
            assert "enable_gqa" not in kwargs
            return native(*args, **kwargs)

        monkeypatch.setattr(hpa, "_TORCH_VARLEN", [without_gqa])
        monkeypatch.setattr(hpa, "_TORCH_VARLEN_GQA", [False])
    if route == "dispatch":
        if not (HAS_FLASH_ATTENTION or HAS_XFORMERS):
            pytest.skip("needs xformers or flash-attn")
        monkeypatch.setattr(hpa, "_torch_varlen", lambda device: None)
    from transformers.integrations.sdpa_attention import sdpa_attention_forward

    hpa._ORIG_SDPA[0] = sdpa_attention_forward
    monkeypatch.delenv("UNSLOTH_HF_PACKED_VARLEN", raising = False)
    torch.manual_seed(0)
    lengths = [37, 100, 2, 5, 230]
    T, H, D = sum(lengths), 8, 64
    module = types.SimpleNamespace(
        is_causal = True, num_key_value_groups = H // n_kv, training = True, sliding_window = window
    )
    q = torch.randn(1, H, T, D, device = "cuda", dtype = torch.bfloat16, requires_grad = True)
    k = torch.randn(1, n_kv, T, D, device = "cuda", dtype = torch.bfloat16, requires_grad = True)
    v = torch.randn(1, n_kv, T, D, device = "cuda", dtype = torch.bfloat16, requires_grad = True)
    mask = (
        _block_causal(lengths, "cuda")
        if window is None
        else _hf_window_mask(lengths, window, "cuda")
    )
    psl = torch.tensor(lengths, dtype = torch.int32)
    before = hpa.HF_PACKED_ATTENTION_STATS["fast"]
    fast, _ = hpa._sdpa_packed_varlen(module, q, k, v, mask, scaling = 0.11, packed_seq_lengths = psl)
    assert hpa.HF_PACKED_ATTENTION_STATS["fast"] == before + 1
    ref, _ = sdpa_attention_forward(module, q, k, v, mask, scaling = 0.11)
    assert fast.shape == ref.shape
    torch.testing.assert_close(fast.float(), ref.float(), atol = 2e-2, rtol = 2e-2)
    g = torch.randn_like(ref)
    grads_fast = torch.autograd.grad(fast, (q, k, v), g)
    grads_ref = torch.autograd.grad(ref, (q, k, v), g)
    for a, b in zip(grads_fast, grads_ref):
        torch.testing.assert_close(a.float(), b.float(), atol = 5e-2, rtol = 5e-2)


def test_new_lengths_tensor_with_a_cached_mask_is_checked_again():
    lengths = [3, 3]
    mask = _block_causal(lengths)
    assert hpa._mask_is_block_causal(mask, _cu(lengths), 6)
    # A different cu_seqlens object must not be compared elementwise against the cached one.
    assert hpa._mask_is_block_causal(mask, _cu(lengths), 6)


def test_long_masks_are_checked_in_row_blocks(monkeypatch):
    monkeypatch.setattr(hpa, "_MASK_CHECK_ROWS", 4)
    lengths = [5, 7, 6]
    mask = _block_causal(lengths).clone()
    assert hpa._mask_is_block_causal(mask, _cu(lengths), 18)
    mask[0, 0, 17, 0] = True
    assert not hpa._mask_is_block_causal(mask, _cu(lengths), 18)


@_CUDA
def test_compiling_leaves_the_counters_alone(monkeypatch):
    hpa._ORIG_SDPA[0] = _plain_sdpa
    monkeypatch.delenv("UNSLOTH_HF_PACKED_VARLEN", raising = False)
    lengths = [3, 3]
    module = types.SimpleNamespace(is_causal = True, num_key_value_groups = 1, training = False)
    q = torch.randn(1, 4, 6, 64, device = "cuda", dtype = torch.bfloat16)
    mask, psl = _block_causal(lengths, "cuda"), torch.tensor(lengths, dtype = torch.int32)
    fn = torch.compile(
        lambda q: hpa._sdpa_packed_varlen(module, q, q, q, mask, packed_seq_lengths = psl)[0],
        backend = "eager",
        fullgraph = True,
    )
    positional = torch.compile(
        lambda q: hpa._sdpa_packed_varlen(module, q, q, q, mask, 0.0, packed_seq_lengths = psl)[0],
        backend = "eager",
        fullgraph = True,
    )
    before = dict(hpa.HF_PACKED_ATTENTION_STATS)
    fn(q)
    positional(q)
    assert hpa.HF_PACKED_ATTENTION_STATS == before


@_CUDA
@pytest.mark.parametrize("mode", ["training_no_grad", "eval_grad", "eval_no_grad"])
def test_int32_guard_follows_training_or_grad_mode(monkeypatch, mode):
    """A reentrant checkpoint's no_grad forward (training) and its recompute both fall back; a pure
    no_grad eval forward allocates no backward buffer and keeps varlen."""
    import unsloth.utils.attention_dispatch as ad

    seen, orig = _calls()
    hpa._ORIG_SDPA[0] = orig
    monkeypatch.delenv("UNSLOTH_HF_PACKED_VARLEN", raising = False)
    monkeypatch.setattr(ad, "_varlen_backward_overflows_int32", lambda *a: True)
    lengths = [3, 3]
    module = types.SimpleNamespace(is_causal = True, training = mode == "training_no_grad")
    q = torch.randn(1, 4, 6, 64, device = "cuda", dtype = torch.bfloat16)
    psl = torch.tensor(lengths, dtype = torch.int32)
    with torch.set_grad_enabled(mode == "eval_grad"):
        out = hpa._sdpa_packed_varlen(
            module, q, q, q, _block_causal(lengths, "cuda"), packed_seq_lengths = psl
        )
    if mode == "eval_no_grad":
        if hpa._torch_varlen(q.device) is None and not (ad.HAS_FLASH_ATTENTION or ad.HAS_XFORMERS):
            pytest.skip("needs a varlen kernel")
        assert out[0].shape == (1, 6, 4, 64)
    else:
        assert out == ("orig", None)


# (L + 128 * n) * H * round32(D) at H = 8, D = 64: 400 tokens' worth per group.
_SMALL_INT32_LIMIT = 400 * 8 * 64


def test_int32_safe_groups_split_at_document_boundaries(monkeypatch):
    import unsloth.utils.attention_dispatch as ad

    monkeypatch.setattr(ad, "_INT32_ELEMENTS", _SMALL_INT32_LIMIT)
    lengths = [37, 100, 2, 5, 230, 64, 9]
    groups = hpa._int32_safe_groups(lengths, 8, 64)
    assert groups[0][0] == 0 and groups[-1][1] == len(lengths) and groups[-1][3] == sum(lengths)
    for (first, end, start, stop, longest), following in zip(groups, groups[1:] + [None]):
        assert stop - start == sum(lengths[first:end]) and longest == max(lengths[first:end])
        assert not ad._varlen_backward_overflows_int32(end - first, stop - start, 8, 64)
        if following is not None:
            assert following[0] == end and following[2] == stop
    assert len(groups) > 1
    # One document alone past the limit cannot be split.
    assert hpa._int32_safe_groups([37, 300], 8, 64) is None


@_CUDA
@pytest.mark.parametrize("window", [None, 4])
@pytest.mark.parametrize("route", ["torch", "dispatch"])
def test_int32_guard_splits_documents_instead_of_the_dense_fallback(monkeypatch, route, window):
    """A row past the varlen backward's int32 limit runs the kernel per group of documents, and the
    output and gradients are the unsplit kernel's; SDPA over the dense mask is never reached."""
    import unsloth.utils.attention_dispatch as ad

    if route == "torch" and hpa._torch_varlen(torch.device("cuda", 0)) is None:
        pytest.skip("this torch has no window_size varlen_attn")
    if route == "dispatch":
        if not (ad.HAS_FLASH_ATTENTION or ad.HAS_XFORMERS):
            pytest.skip("needs xformers or flash-attn")
        monkeypatch.setattr(hpa, "_torch_varlen", lambda device: None)
        # Where torch's packed kernels beat xformers the dispatcher picks sdpa: pin the varlen one.
        backend = ad.FLASH_VARLEN if ad.HAS_FLASH_ATTENTION else ad.XFORMERS
        monkeypatch.setattr(ad, "select_attention_backend", lambda **kwargs: backend)

    def _dense(*args, **kwargs):
        raise AssertionError("packed row fell back to SDPA over the dense mask")

    hpa._ORIG_SDPA[0] = _dense
    monkeypatch.delenv("UNSLOTH_HF_PACKED_VARLEN", raising = False)
    torch.manual_seed(0)
    lengths = [37, 100, 2, 5, 230, 64, 9]
    T, H, n_kv, D = sum(lengths), 8, 2, 64
    module = types.SimpleNamespace(
        is_causal = True, num_key_value_groups = H // n_kv, training = True, sliding_window = window
    )
    q, k, v = (
        torch.randn(1, h, T, D, device = "cuda", dtype = torch.bfloat16, requires_grad = True)
        for h in (H, n_kv, n_kv)
    )
    mask = (
        _block_causal(lengths, "cuda")
        if window is None
        else _hf_window_mask(lengths, window, "cuda")
    )
    psl = torch.tensor(lengths, dtype = torch.int32)
    g = torch.randn(1, T, H, D, device = "cuda", dtype = torch.bfloat16)

    def run():
        out, _ = hpa._sdpa_packed_varlen(
            module, q, k, v, mask, scaling = 0.11, packed_seq_lengths = psl
        )
        return (out, *torch.autograd.grad(out, (q, k, v), g))

    whole = run()
    monkeypatch.setattr(ad, "_INT32_ELEMENTS", _SMALL_INT32_LIMIT)
    assert ad._varlen_backward_overflows_int32(len(lengths), T, H, D)
    before = dict(hpa.HF_PACKED_ATTENTION_STATS)
    split = run()
    assert hpa.HF_PACKED_ATTENTION_STATS["int32_split"] == before["int32_split"] + 1
    assert hpa.HF_PACKED_ATTENTION_STATS["fallback"] == before["fallback"]
    for a, b in zip(split, whole):
        torch.testing.assert_close(a, b, atol = 0, rtol = 0)


def test_block_varlen_kwargs_sends_rows_past_int32_to_the_eager_wrapper(monkeypatch):
    """The traced kernel cannot split a row, so the eager wrapper (which can) runs it; a document that
    alone overflows keeps the previous dense path."""
    import unsloth.utils.attention_dispatch as ad

    hpa._ORIG_SDPA[0] = _plain_sdpa
    monkeypatch.delenv("UNSLOTH_HF_PACKED_VARLEN", raising = False)
    monkeypatch.setattr(torch.Tensor, "is_cuda", property(lambda self: True))
    monkeypatch.setattr(hpa, "_torch_varlen", lambda device: object())
    monkeypatch.setattr(hpa, "_sdpa_policy_requested", lambda *args: False)
    monkeypatch.setattr(hpa, "_BLOCK_PREP", {})
    monkeypatch.setattr(ad, "_INT32_ELEMENTS", 200 * 4 * 32)
    layer = _block_layer(head_dim = 32, n_heads = 4)
    for lengths, expected in (([20, 30, 40], hpa._LEAVE_BLOCK), ([20, 80], None)):
        hidden = torch.randn(1, sum(lengths), 8)
        kwargs = {
            "packed_seq_lengths": torch.tensor(lengths),
            "attention_mask": _block_causal(lengths),
        }
        before = hpa.HF_PACKED_ATTENTION_STATS["compiled"]
        assert hpa.block_varlen_kwargs(layer, hidden, kwargs) == expected
        assert hpa.HF_PACKED_ATTENTION_STATS["compiled"] == before


def test_traced_leave_block_call_runs_the_eager_wrapper(monkeypatch):
    seen = []

    @torch.compiler.disable
    def eager(module, q, k, v, mask, **kwargs):
        seen.append(set(kwargs))
        return q.transpose(1, 2) * 3, None

    hpa._ORIG_SDPA[0] = lambda *args, **kwargs: pytest.fail("the dense path stayed in the graph")
    monkeypatch.delenv("UNSLOTH_HF_PACKED_VARLEN", raising = False)
    monkeypatch.setattr(hpa, "_sdpa_packed_varlen_eager", eager)
    lengths = [3, 3]
    module = types.SimpleNamespace(is_causal = True, num_key_value_groups = 1, training = True)
    q = torch.randn(1, 4, 6, 16, dtype = torch.bfloat16)
    mask, psl = _block_causal(lengths), torch.tensor(lengths, dtype = torch.int32)
    torch._dynamo.reset()
    fn = torch.compile(
        lambda q: hpa._sdpa_packed_varlen(
            module,
            q,
            q,
            q,
            mask,
            packed_seq_lengths = psl,
            **{hpa.BLOCK_VARLEN_KWARG: hpa._LEAVE_BLOCK},
        )[0],
        backend = "eager",
    )
    torch.testing.assert_close(fn(q), q.transpose(1, 2) * 3)
    assert seen and all(hpa.BLOCK_VARLEN_KWARG not in kwargs for kwargs in seen)


@_CUDA
@_TORCH_VARLEN_HOST
def test_a_refusing_torch_varlen_hands_the_call_back(monkeypatch):
    seen, orig = _calls()
    hpa._ORIG_SDPA[0] = orig
    monkeypatch.delenv("UNSLOTH_HF_PACKED_VARLEN", raising = False)

    def refuse(*args, **kwargs):
        raise RuntimeError("GQA is not supported with the cuDNN backend.")

    monkeypatch.setattr(hpa, "_TORCH_VARLEN", [refuse])
    monkeypatch.setattr(hpa, "_TORCH_VARLEN_DEVICES", {})
    lengths = [3, 3]
    q = torch.randn(1, 4, 6, 64, device = "cuda", dtype = torch.bfloat16)
    psl = torch.tensor(lengths, dtype = torch.int32)
    out = hpa._sdpa_packed_varlen(
        types.SimpleNamespace(is_causal = True),
        q,
        q,
        q,
        _block_causal(lengths, "cuda"),
        packed_seq_lengths = psl,
    )
    assert out == ("orig", None)
    assert hpa._TORCH_VARLEN == [None]


def test_torch_varlen_kill_switch(monkeypatch):
    monkeypatch.setenv("UNSLOTH_TORCH_VARLEN", "0")
    monkeypatch.setattr(hpa, "_TORCH_VARLEN", [])
    monkeypatch.setattr(hpa, "_TORCH_VARLEN_GQA", [])
    assert hpa._torch_varlen(torch.device("cuda", 0)) is None
    assert hpa._TORCH_VARLEN == [None]


@_CUDA
@_TORCH_VARLEN_HOST
@pytest.mark.parametrize("policy", ["math_only", "flash_disabled", "math_first", "deterministic"])
def test_an_sdpa_policy_keeps_every_varlen_kernel_off(monkeypatch, policy):
    """The wrapped sdpa runs, not torch varlen nor the xformers / flash-attn dispatch; outside, varlen runs."""
    import contextlib
    from torch.nn.attention import SDPBackend, sdpa_kernel
    from unsloth.utils import attention_dispatch as ad

    if policy == "math_first" and "set_priority" not in inspect.signature(sdpa_kernel).parameters:
        pytest.skip("this torch has no sdpa_kernel(set_priority = True)")
    calls = []

    def spy(q, *args, **kwargs):
        calls.append(1)
        return torch.zeros_like(q)

    monkeypatch.setattr(hpa, "_TORCH_VARLEN", [spy])
    monkeypatch.setattr(hpa, "_TORCH_VARLEN_DEVICES", {})
    monkeypatch.setattr(ad, "run_attention", lambda **kwargs: calls.append(2) or kwargs["Q"])
    seen, orig = _calls()
    hpa._ORIG_SDPA[0] = orig
    monkeypatch.delenv("UNSLOTH_HF_PACKED_VARLEN", raising = False)
    lengths = [3, 3]
    q = torch.randn(1, 4, 6, 64, device = "cuda", dtype = torch.bfloat16)
    k = torch.randn(1, 2, 6, 64, device = "cuda", dtype = torch.bfloat16)
    module = types.SimpleNamespace(is_causal = True)

    def run():
        return hpa._sdpa_packed_varlen(
            module,
            q,
            k,
            k,
            _block_causal(lengths, "cuda"),
            packed_seq_lengths = torch.tensor(lengths, dtype = torch.int32),
        )

    if policy == "math_only":
        context = sdpa_kernel([SDPBackend.MATH])
    elif policy == "flash_disabled":
        context = sdpa_kernel([SDPBackend.EFFICIENT_ATTENTION, SDPBackend.MATH])
    elif policy == "math_first":
        context = sdpa_kernel([SDPBackend.MATH, SDPBackend.FLASH_ATTENTION], set_priority = True)
    else:
        context = contextlib.nullcontext()
        deterministic = torch.are_deterministic_algorithms_enabled()
        torch.use_deterministic_algorithms(True)
    try:
        with context:
            assert run() == ("orig", None)
    finally:
        if policy == "deterministic":
            torch.use_deterministic_algorithms(deterministic)
    assert calls == [] and len(seen) == 1
    run()
    assert calls == [1]


@_CUDA
@pytest.mark.parametrize("window", [None, 4])
def test_tiny_mistral_packed_row_matches_the_masked_path(monkeypatch, window):
    """A real sliding-window decoder: packed logits and grads with the wrapper equal the wrapped sdpa."""
    transformers = pytest.importorskip("transformers")
    from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS
    from unsloth.utils.attention_dispatch import HAS_FLASH_ATTENTION, HAS_XFORMERS

    if not (HAS_FLASH_ATTENTION or HAS_XFORMERS or hpa._torch_varlen(torch.device("cuda", 0))):
        pytest.skip("no varlen kernel on this build")
    hpa._ORIG_SDPA[0] = None
    monkeypatch.delenv("UNSLOTH_HF_PACKED_VARLEN", raising = False)
    from transformers.integrations.sdpa_attention import sdpa_attention_forward

    # transformers' own entry: an earlier test can leave the installed wrapper in the mapping.
    monkeypatch.setitem(ALL_ATTENTION_FUNCTIONS, "sdpa", sdpa_attention_forward)
    assert hpa.enable_hf_packed_attention()
    torch.manual_seed(0)
    config = transformers.MistralConfig(
        vocab_size = 128,
        hidden_size = 256,
        intermediate_size = 256,
        num_hidden_layers = 2,
        num_attention_heads = 4,
        num_key_value_heads = 2,
        head_dim = 64,
        sliding_window = window,
        attn_implementation = "sdpa",
    )
    model = transformers.MistralForCausalLM(config).cuda().to(torch.bfloat16)
    lengths = [3, 11, 7]
    ids = torch.randint(0, 128, (1, sum(lengths)), device = "cuda")
    position_ids = torch.cat([torch.arange(n) for n in lengths])[None].cuda()
    psl = torch.tensor(lengths, dtype = torch.int32)

    def run():
        model.zero_grad()
        out = model(
            input_ids = ids, position_ids = position_ids, packed_seq_lengths = psl, use_cache = False
        )
        out.logits.float().square().mean().backward()
        return out.logits.detach().float(), [p.grad.float() for p in model.parameters()]

    before = hpa.HF_PACKED_ATTENTION_STATS["fast"]
    fast_logits, fast_grads = run()
    assert hpa.HF_PACKED_ATTENTION_STATS["fast"] == before + 2
    monkeypatch.setenv("UNSLOTH_HF_PACKED_VARLEN", "0")
    ref_logits, ref_grads = run()
    torch.testing.assert_close(fast_logits, ref_logits, atol = 3e-2, rtol = 3e-2)
    for a, b in zip(fast_grads, ref_grads):
        torch.testing.assert_close(a, b, atol = 3e-2, rtol = 5e-2)


# Gemma-4 global layers: head_dim 512, past every flash kernel, take SDPA's efficient kernel over cu_seqlens.
def _efficient_varlen_op() -> bool:
    try:
        forward = torch.ops.aten._efficient_attention_forward.default._schema
        backward = torch.ops.aten._efficient_attention_backward.default._schema
    except Exception:
        return False
    return {"cu_seqlens_q", "scale", "window_size"} <= {a.name for a in forward.arguments} and (
        "window_size" in {a.name for a in backward.arguments}
    )


_EFFICIENT_HOST = pytest.mark.skipif(
    torch.version.hip is not None or not _efficient_varlen_op(),
    reason = "the efficient varlen route is off on ROCm and on torch without its varlen schema",
)


@pytest.fixture
def _efficient_fresh(monkeypatch):
    monkeypatch.setattr(hpa, "_EFFICIENT_VARLEN", [])
    monkeypatch.setattr(hpa, "_EFFICIENT_DEVICES", {})
    monkeypatch.delenv("UNSLOTH_HF_PACKED_VARLEN", raising = False)
    monkeypatch.delenv(hpa._EFFICIENT_ENV, raising = False)


def _wide_inputs(
    lengths,
    H,
    n_kv,
    D,
    scale = 0.1,
):
    T = sum(lengths)

    def make(heads):
        x = torch.randn(1, heads, T, D, device = "cuda", dtype = torch.bfloat16) * scale
        return x.requires_grad_()

    return make(H), make(n_kv), make(n_kv)


@_CUDA
@_EFFICIENT_HOST
@pytest.mark.parametrize("window", [None, 5])
@pytest.mark.parametrize("n_kv", [8, 2, 1])
def test_wide_heads_match_sdpa_over_the_dense_mask(_efficient_fresh, n_kv, window):
    """Output and dQ / dK / dV equal transformers' sdpa over the real mask, Gemma-4 scaling 1.0."""
    from transformers.integrations.sdpa_attention import sdpa_attention_forward

    hpa._ORIG_SDPA[0] = sdpa_attention_forward
    torch.manual_seed(0)
    lengths = [37, 100, 2, 5, 230]
    H, D = 8, 512
    module = types.SimpleNamespace(
        is_causal = True, num_key_value_groups = H // n_kv, training = True, sliding_window = window
    )
    q, k, v = _wide_inputs(lengths, H, n_kv, D)
    mask = (
        _block_causal(lengths, "cuda")
        if window is None
        else _hf_window_mask(lengths, window, "cuda")
    )
    psl = torch.tensor(lengths, dtype = torch.int32)
    before = dict(hpa.HF_PACKED_ATTENTION_STATS)
    fast, _ = hpa._sdpa_packed_varlen(module, q, k, v, mask, scaling = 1.0, packed_seq_lengths = psl)
    assert hpa.HF_PACKED_ATTENTION_STATS["efficient"] == before["efficient"] + 1
    assert hpa.HF_PACKED_ATTENTION_STATS["fast"] == before["fast"] + 1
    ref, _ = sdpa_attention_forward(module, q, k, v, mask, scaling = 1.0)
    assert fast.shape == ref.shape == (1, sum(lengths), H, D)
    torch.testing.assert_close(fast.float(), ref.float(), atol = 1e-2, rtol = 1e-2)
    g = torch.randn_like(ref)
    for a, b in zip(
        torch.autograd.grad(fast, (q, k, v), g), torch.autograd.grad(ref, (q, k, v), g)
    ):
        torch.testing.assert_close(a.float(), b.float(), atol = 2e-2, rtol = 2e-2)


@_CUDA
@_EFFICIENT_HOST
def test_wide_heads_keep_documents_apart(_efficient_fresh):
    """Each document of the packed row equals that document attended alone."""
    from transformers.integrations.sdpa_attention import sdpa_attention_forward

    hpa._ORIG_SDPA[0] = sdpa_attention_forward
    torch.manual_seed(1)
    lengths = [60, 3, 130]
    module = types.SimpleNamespace(is_causal = True, num_key_value_groups = 4, training = True)
    q, k, v = _wide_inputs(lengths, 8, 2, 512)
    fast, _ = hpa._sdpa_packed_varlen(
        module,
        q,
        k,
        v,
        _block_causal(lengths, "cuda"),
        scaling = 1.0,
        packed_seq_lengths = torch.tensor(lengths, dtype = torch.int32),
    )
    start = 0
    for n in lengths:
        sl = slice(start, start + n)
        alone, _ = sdpa_attention_forward(
            module, q[:, :, sl], k[:, :, sl], v[:, :, sl], None, scaling = 1.0, is_causal = True
        )
        torch.testing.assert_close(fast[:, sl].float(), alone.float(), atol = 1e-2, rtol = 1e-2)
        start += n


@_CUDA
@_EFFICIENT_HOST
@pytest.mark.parametrize(
    "why",
    [
        "none",
        "cross_document",
        "wrong_window",
        "softcap",
        "sinks",
        "dropout",
        "fp32",
        "head_dim_640",
        "kill_switch",
        "deterministic",
        "math_only",
        "efficient_disabled",
        "no_op",
    ],
)
def test_wide_heads_fall_back_to_the_wrapped_sdpa(_efficient_fresh, monkeypatch, why):
    """Each reason alone sends a head_dim 512 call to the wrapped sdpa; "none" is the control."""
    import contextlib
    from torch.nn.attention import SDPBackend, sdpa_kernel

    seen, orig = _calls()
    hpa._ORIG_SDPA[0] = orig
    lengths, D, dtype = [5, 7], 512, torch.bfloat16
    module = types.SimpleNamespace(is_causal = True, num_key_value_groups = 2)
    mask, dropout, context = _block_causal(lengths, "cuda"), 0.0, contextlib.nullcontext()
    if why == "cross_document":
        mask = mask.clone()
        mask[0, 0, 6, 1] = True
    elif why == "wrong_window":
        module.sliding_window = 3  # the mask has no band
    elif why == "softcap":
        module.config = types.SimpleNamespace(attn_logit_softcapping = 50.0)
    elif why == "sinks":
        module.sinks = torch.zeros(4)
    elif why == "dropout":
        dropout = 0.1
    elif why == "fp32":
        dtype = torch.float32
    elif why == "head_dim_640":
        D = 640
    elif why == "kill_switch":
        monkeypatch.setenv(hpa._EFFICIENT_ENV, "0")
    elif why == "math_only":
        context = sdpa_kernel([SDPBackend.MATH])
    elif why == "efficient_disabled":
        context = sdpa_kernel([SDPBackend.FLASH_ATTENTION, SDPBackend.MATH])
    elif why == "no_op":
        monkeypatch.setattr(hpa, "_EFFICIENT_VARLEN", [False])
    q = torch.randn(1, 4, 12, D, device = "cuda", dtype = dtype)
    k = torch.randn(1, 2, 12, D, device = "cuda", dtype = dtype)
    before = dict(hpa.HF_PACKED_ATTENTION_STATS)
    deterministic = torch.are_deterministic_algorithms_enabled()
    if why == "deterministic":
        torch.use_deterministic_algorithms(True)
    try:
        with context:
            out = hpa._sdpa_packed_varlen(
                module,
                q,
                k,
                k,
                mask,
                dropout = dropout,
                scaling = 1.0,
                packed_seq_lengths = torch.tensor(lengths, dtype = torch.int32),
            )
    finally:
        torch.use_deterministic_algorithms(deterministic)
    if why == "none":
        assert out[0].shape == (1, 12, 4, D) and seen == []
        assert hpa.HF_PACKED_ATTENTION_STATS["efficient"] == before["efficient"] + 1
    else:
        assert out == ("orig", None)
        assert hpa.HF_PACKED_ATTENTION_STATS["efficient"] == before["efficient"]
        assert hpa.HF_PACKED_ATTENTION_STATS["fallback"] == before["fallback"] + 1


@_CUDA
@_EFFICIENT_HOST
def test_a_refusing_efficient_kernel_hands_the_call_back(_efficient_fresh, monkeypatch):
    seen, orig = _calls()
    hpa._ORIG_SDPA[0] = orig

    def refuse(*args, **kwargs):
        raise RuntimeError("No available kernel.")

    monkeypatch.setattr(hpa._EfficientVarlenAttention, "apply", refuse)
    lengths = [3, 3]
    q = torch.randn(1, 4, 6, 512, device = "cuda", dtype = torch.bfloat16)
    before = dict(hpa.HF_PACKED_ATTENTION_STATS)
    out = hpa._sdpa_packed_varlen(
        types.SimpleNamespace(is_causal = True),
        q,
        q,
        q,
        _block_causal(lengths, "cuda"),
        packed_seq_lengths = torch.tensor(lengths, dtype = torch.int32),
    )
    assert out == ("orig", None) and len(seen) == 1
    assert hpa._EFFICIENT_VARLEN == [False]
    assert hpa.HF_PACKED_ATTENTION_STATS["fast"] == before["fast"]
    assert hpa.HF_PACKED_ATTENTION_STATS["fallback"] == before["fallback"] + 1


@_CUDA
@_EFFICIENT_HOST
def test_the_efficient_kill_switch_is_read_on_every_call(_efficient_fresh, monkeypatch):
    q = torch.randn(1, 4, 12, 512, device = "cuda", dtype = torch.bfloat16)
    assert hpa._efficient_varlen_takes(q, None)
    monkeypatch.setenv(hpa._EFFICIENT_ENV, "0")
    assert not hpa._efficient_varlen_takes(q, None)
    monkeypatch.delenv(hpa._EFFICIENT_ENV)
    assert hpa._efficient_varlen_takes(q, None)


@_CUDA
@_EFFICIENT_HOST
def test_a_non_contiguous_head_dim_is_copied_not_refused(_efficient_fresh):
    cu = torch.tensor([0, 5, 12], device = "cuda", dtype = torch.int32)
    base = torch.randn(1, 4, 12, 1024, device = "cuda", dtype = torch.bfloat16)
    q = base[..., ::2]  # head dim 512 with stride 2
    assert q.stride(-1) == 2
    out = hpa._efficient_varlen(q, q, q, cu, 7, 512**-0.5, None)
    ref = hpa._efficient_varlen(
        q.contiguous(), q.contiguous(), q.contiguous(), cu, 7, 512**-0.5, None
    )
    torch.testing.assert_close(out, ref)


@_CUDA
@_EFFICIENT_HOST
def test_a_cuda_fault_in_the_efficient_kernel_is_not_a_refusal(_efficient_fresh, monkeypatch):
    if hpa._ACCELERATOR_ERROR is None:
        pytest.skip("this torch has no AcceleratorError")
    seen, orig = _calls()
    hpa._ORIG_SDPA[0] = orig

    def fault(*args, **kwargs):
        raise hpa._ACCELERATOR_ERROR("CUDA error: an illegal memory access was encountered")

    monkeypatch.setattr(hpa._EfficientVarlenAttention, "apply", fault)
    lengths = [3, 3]
    q = torch.randn(1, 4, 6, 512, device = "cuda", dtype = torch.bfloat16)
    with pytest.raises(hpa._ACCELERATOR_ERROR):
        hpa._sdpa_packed_varlen(
            types.SimpleNamespace(is_causal = True),
            q,
            q,
            q,
            _block_causal(lengths, "cuda"),
            packed_seq_lengths = torch.tensor(lengths, dtype = torch.int32),
        )
    assert seen == [] and hpa._EFFICIENT_VARLEN != [False]


@_CUDA
@_EFFICIENT_HOST
@pytest.mark.parametrize("backend", ["sdpa", "flash_varlen"])
def test_a_shape_the_flash_impl_refuses_takes_the_efficient_kernel(
    _efficient_fresh, monkeypatch, backend
):
    """FA4 on sm100 lacks hd256 + window: without flash-attn / xformers it takes the efficient kernel."""
    from transformers.integrations.sdpa_attention import sdpa_attention_forward
    from unsloth.utils import attention_dispatch as ad
    from unsloth.utils import flash_impl

    if hpa._torch_varlen(torch.device("cuda", 0)) is None:
        pytest.skip("no torch varlen_attn to refuse")
    monkeypatch.setattr(flash_impl, "flash_impl_allows", lambda head_dim, window, device: False)
    monkeypatch.setattr(ad, "select_attention_backend", lambda use_varlen = False: backend)
    dispatched = []
    monkeypatch.setattr(
        ad, "run_attention", lambda **kwargs: dispatched.append(1) or kwargs["Q"].transpose(1, 2)
    )
    hpa._ORIG_SDPA[0] = sdpa_attention_forward
    torch.manual_seed(2)
    lengths, window = [9, 20, 4], 6
    module = types.SimpleNamespace(
        is_causal = True, num_key_value_groups = 4, training = True, sliding_window = window
    )
    q, k, v = _wide_inputs(lengths, 8, 2, 256)
    mask = _hf_window_mask(lengths, window, "cuda")
    before = dict(hpa.HF_PACKED_ATTENTION_STATS)
    fast, _ = hpa._sdpa_packed_varlen(
        module,
        q,
        k,
        v,
        mask,
        scaling = 1.0,
        packed_seq_lengths = torch.tensor(lengths, dtype = torch.int32),
    )
    if backend == "flash_varlen":
        assert dispatched == [1]
        assert hpa.HF_PACKED_ATTENTION_STATS["efficient"] == before["efficient"]
        return
    assert dispatched == []
    assert hpa.HF_PACKED_ATTENTION_STATS["efficient"] == before["efficient"] + 1
    ref, _ = sdpa_attention_forward(module, q, k, v, mask, scaling = 1.0)
    torch.testing.assert_close(fast.float(), ref.float(), atol = 1e-2, rtol = 1e-2)
    g = torch.randn_like(ref)
    for a, b in zip(
        torch.autograd.grad(fast, (q, k, v), g), torch.autograd.grad(ref, (q, k, v), g)
    ):
        torch.testing.assert_close(a.float(), b.float(), atol = 2e-2, rtol = 2e-2)


@_CUDA
@_EFFICIENT_HOST
def test_wide_heads_under_reentrant_checkpoint(_efficient_fresh):
    from torch.utils.checkpoint import checkpoint
    from transformers.integrations.sdpa_attention import sdpa_attention_forward

    hpa._ORIG_SDPA[0] = sdpa_attention_forward
    lengths = [40, 24]
    module = types.SimpleNamespace(is_causal = True, num_key_value_groups = 4, training = True)
    q, k, v = _wide_inputs(lengths, 8, 2, 512)
    mask, psl = _block_causal(lengths, "cuda"), torch.tensor(lengths, dtype = torch.int32)
    fn = lambda q, k, v: hpa._sdpa_packed_varlen(
        module, q, k, v, mask, scaling = 1.0, packed_seq_lengths = psl
    )[0]
    out = checkpoint(fn, q, k, v, use_reentrant = True)
    g = torch.randn_like(out)
    out.backward(g)
    grads = [x.grad for x in (q, k, v)]
    ref = sdpa_attention_forward(module, q, k, v, mask, scaling = 1.0)[0]
    torch.testing.assert_close(out.float(), ref.float(), atol = 1e-2, rtol = 1e-2)
    for a, b in zip(grads, torch.autograd.grad(ref, (q, k, v), g)):
        torch.testing.assert_close(a.float(), b.float(), atol = 2e-2, rtol = 2e-2)


@_CUDA
@_EFFICIENT_HOST
def test_wide_heads_leave_a_compiled_caller_alone(_efficient_fresh):
    """Under fullgraph compile the wrapper returns the wrapped sdpa: no graph break, no counters."""
    hpa._ORIG_SDPA[0] = _plain_sdpa
    lengths = [3, 5]
    module = types.SimpleNamespace(is_causal = True, num_key_value_groups = 1, training = False)
    q = torch.randn(1, 4, 8, 512, device = "cuda", dtype = torch.bfloat16)
    mask, psl = _block_causal(lengths, "cuda"), torch.tensor(lengths, dtype = torch.int32)
    before = dict(hpa.HF_PACKED_ATTENTION_STATS)
    fn = torch.compile(
        lambda q: hpa._sdpa_packed_varlen(
            module, q, q, q, mask, scaling = 1.0, packed_seq_lengths = psl
        )[0],
        backend = "eager",
        fullgraph = True,
    )
    torch.testing.assert_close(fn(q), _plain_sdpa(module, q, q, q, mask, scaling = 1.0)[0])
    assert hpa.HF_PACKED_ATTENTION_STATS == before


@_CUDA
@_EFFICIENT_HOST
@pytest.mark.parametrize("k_eq_v", [False, True])
def test_tiny_gemma4_packed_row_matches_each_document(_efficient_fresh, monkeypatch, k_eq_v):
    """Gemma4 global_head_dim 512: efficient kernel; logits / grads match sdpa and each doc alone."""
    transformers = pytest.importorskip("transformers")
    if not hasattr(transformers, "Gemma4TextConfig"):
        pytest.skip("this transformers has no Gemma 4")
    from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS

    hpa._ORIG_SDPA[0] = None
    from transformers.integrations.sdpa_attention import sdpa_attention_forward

    # transformers' own entry: an earlier test can leave the installed wrapper in the mapping.
    monkeypatch.setitem(ALL_ATTENTION_FUNCTIONS, "sdpa", sdpa_attention_forward)
    assert hpa.enable_hf_packed_attention()
    torch.manual_seed(0)
    config = transformers.Gemma4TextConfig(
        vocab_size = 128,
        vocab_size_per_layer_input = 128,
        hidden_size = 64,
        hidden_size_per_layer_input = 8,
        intermediate_size = 128,
        num_hidden_layers = 2,
        layer_types = ["sliding_attention", "full_attention"],
        num_attention_heads = 4,
        num_key_value_heads = 2,
        num_global_key_value_heads = 1 if k_eq_v else None,
        attention_k_eq_v = k_eq_v,
        head_dim = 256,
        global_head_dim = 512,
        sliding_window = 4,
        attn_implementation = "sdpa",
    )
    model = transformers.Gemma4ForCausalLM(config).cuda().to(torch.bfloat16)
    global_attn = model.model.layers[1].self_attn
    assert global_attn.head_dim == 512 and model.model.layers[0].self_attn.head_dim == 256
    lengths = [3, 11, 7]
    ids = torch.randint(0, 128, (1, sum(lengths)), device = "cuda")
    position_ids = torch.cat([torch.arange(n) for n in lengths])[None].cuda()
    psl = torch.tensor(lengths, dtype = torch.int32)

    def run():
        model.zero_grad()
        out = model(
            input_ids = ids, position_ids = position_ids, packed_seq_lengths = psl, use_cache = False
        )
        out.logits.float().square().mean().backward()
        return out.logits.detach().float(), [
            p.grad.float() for p in model.parameters() if p.grad is not None
        ]

    before = dict(hpa.HF_PACKED_ATTENTION_STATS)
    fast_logits, fast_grads = run()
    stats = hpa.HF_PACKED_ATTENTION_STATS
    assert stats["efficient"] == before["efficient"] + 1
    # Without torch varlen (torch 2.10 and older) the windowed sliding layer may have no varlen route.
    assert stats["fast"] + stats["fallback"] == before["fast"] + before["fallback"] + 2
    if hpa._torch_varlen(ids.device) is not None:
        assert stats["fast"] == before["fast"] + 2
    monkeypatch.setenv("UNSLOTH_HF_PACKED_VARLEN", "0")
    ref_logits, ref_grads = run()
    torch.testing.assert_close(fast_logits, ref_logits, atol = 3e-2, rtol = 3e-2)
    for a, b in zip(fast_grads, ref_grads):
        torch.testing.assert_close(a, b, atol = 3e-2, rtol = 5e-2)
    with torch.no_grad():
        start = 0
        for n in lengths:
            alone = model(input_ids = ids[:, start : start + n], use_cache = False).logits.float()
            torch.testing.assert_close(
                fast_logits[:, start : start + n], alone, atol = 3e-2, rtol = 3e-2
            )
            start += n


def _block_layer(
    window = None,
    impl = "sdpa",
    head_dim = 16,
    n_heads = 4,
):
    config = types.SimpleNamespace(_attn_implementation = impl, num_attention_heads = n_heads)
    attn = types.SimpleNamespace(config = config, head_dim = head_dim, sliding_window = window)
    return types.SimpleNamespace(self_attn = attn)


@pytest.mark.parametrize(
    "case", ["cpu", "unpacked", "eager_impl", "bad_window", "no_attention", "disabled"]
)
def test_block_varlen_kwargs_refusals_keep_the_dense_path(monkeypatch, case):
    """Every refusal leaves the compiled block on its previous path (SDPA over the dense mask)."""
    hpa._ORIG_SDPA[0] = _plain_sdpa
    monkeypatch.delenv("UNSLOTH_HF_PACKED_VARLEN", raising = False)
    lengths = [3, 3]
    hidden = torch.randn(1, 6, 8)
    kwargs = {"packed_seq_lengths": torch.tensor(lengths), "attention_mask": _block_causal(lengths)}
    layer = _block_layer()
    if case == "cpu":
        pass  # hidden states on CPU
    else:
        monkeypatch.setattr(torch.Tensor, "is_cuda", property(lambda self: True))
    if case == "unpacked":
        kwargs.pop("packed_seq_lengths")
    elif case == "eager_impl":
        layer = _block_layer(impl = "eager")
    elif case == "bad_window":
        layer = _block_layer(window = 0)
    elif case == "no_attention":
        layer = types.SimpleNamespace()
    elif case == "disabled":
        monkeypatch.setenv("UNSLOTH_HF_PACKED_VARLEN", "0")
    if case != "cpu":
        # No CUDA here: refusals must come before any device work.
        monkeypatch.setattr(
            hpa, "_torch_varlen", lambda device: None if case == "cpu" else object()
        )
        monkeypatch.setattr(
            "unsloth.utils.packing.get_packed_info_from_kwargs",
            lambda *a, **k: pytest.fail("refused rows must not build packed metadata"),
        )
    assert hpa.block_varlen_kwargs(layer, hidden, kwargs) is None


def test_block_varlen_kwargs_prepares_metadata_and_checks_the_mask(monkeypatch):
    hpa._ORIG_SDPA[0] = _plain_sdpa
    monkeypatch.delenv("UNSLOTH_HF_PACKED_VARLEN", raising = False)
    monkeypatch.setattr(torch.Tensor, "is_cuda", property(lambda self: True))
    monkeypatch.setattr(hpa, "_torch_varlen", lambda device: object())
    monkeypatch.setattr(hpa, "_sdpa_policy_requested", lambda *args: False)
    monkeypatch.setattr(hpa, "_BLOCK_PREP", {})
    lengths = [3, 5]
    cu = torch.tensor([0, 3, 8], dtype = torch.int32)
    monkeypatch.setattr(
        "unsloth.utils.packing.get_packed_info_from_kwargs",
        lambda kwargs, device: (torch.tensor(lengths), cu, 5),
    )
    hidden = torch.randn(1, 8, 8)
    kwargs = {"packed_seq_lengths": torch.tensor(lengths), "attention_mask": _block_causal(lengths)}
    before = hpa.HF_PACKED_ATTENTION_STATS["compiled"]
    assert hpa.block_varlen_kwargs(_block_layer(), hidden, kwargs) == (cu, 5)
    assert hpa.HF_PACKED_ATTENTION_STATS["compiled"] == before + 1
    # A mask that is not this block-causal one (one causal span over both documents) is refused.
    kwargs["attention_mask"] = _block_causal([8])
    assert hpa.block_varlen_kwargs(_block_layer(), hidden, kwargs) is None


def test_block_varlen_kwargs_rechecks_a_mask_edited_in_place(monkeypatch):
    """The per-forward cache keys on tensor versions: a mask edited in place is checked again."""
    hpa._ORIG_SDPA[0] = _plain_sdpa
    monkeypatch.delenv("UNSLOTH_HF_PACKED_VARLEN", raising = False)
    monkeypatch.setattr(torch.Tensor, "is_cuda", property(lambda self: True))
    monkeypatch.setattr(hpa, "_torch_varlen", lambda device: object())
    monkeypatch.setattr(hpa, "_sdpa_policy_requested", lambda *args: False)
    monkeypatch.setattr(hpa, "_BLOCK_PREP", {})
    lengths = [3, 5]
    cu = torch.tensor([0, 3, 8], dtype = torch.int32)
    monkeypatch.setattr(
        "unsloth.utils.packing.get_packed_info_from_kwargs",
        lambda kwargs, device: (torch.tensor(lengths), cu, 5),
    )
    hidden = torch.randn(1, 8, 8)
    mask = _block_causal(lengths)
    kwargs = {"packed_seq_lengths": torch.tensor(lengths), "attention_mask": mask}
    assert hpa.block_varlen_kwargs(_block_layer(), hidden, kwargs) == (cu, 5)
    mask[..., 4, 0] = True  # the second document now sees the first
    assert hpa.block_varlen_kwargs(_block_layer(), hidden, kwargs) is None


def test_block_varlen_kwargs_does_not_serve_inference_metadata_to_training(monkeypatch):
    """inference_mode forward then a training forward on the same batch: fresh cu_seqlens."""
    hpa._ORIG_SDPA[0] = _plain_sdpa
    monkeypatch.delenv("UNSLOTH_HF_PACKED_VARLEN", raising = False)
    monkeypatch.setattr(torch.Tensor, "is_cuda", property(lambda self: True))
    monkeypatch.setattr(hpa, "_torch_varlen", lambda device: object())
    monkeypatch.setattr(hpa, "_sdpa_policy_requested", lambda *args: False)
    monkeypatch.setattr(hpa, "_BLOCK_PREP", {})
    lengths = [3, 5]
    monkeypatch.setattr(
        "unsloth.utils.packing.get_packed_info_from_kwargs",
        lambda kwargs, device: (torch.tensor(lengths), _cu(lengths), 5),
    )
    hidden = torch.randn(1, 8, 8)
    kwargs = {"packed_seq_lengths": torch.tensor(lengths), "attention_mask": _block_causal(lengths)}
    with torch.inference_mode():
        cu, _ = hpa.block_varlen_kwargs(_block_layer(), hidden, kwargs)
        assert hpa.block_varlen_kwargs(_block_layer(), hidden, kwargs)[0] is cu
    assert torch.is_inference(cu)
    train, _ = hpa.block_varlen_kwargs(_block_layer(), hidden, kwargs)
    assert not torch.is_inference(train) and train.tolist() == [0, 3, 8]
    assert hpa.block_varlen_kwargs(_block_layer(), hidden, kwargs)[0] is train
    with torch.inference_mode():
        assert hpa.block_varlen_kwargs(_block_layer(), hidden, kwargs)[0] is train


def test_block_varlen_kwargs_survives_a_collected_mask_then_none(monkeypatch):
    """Same lengths, the cached mask collected, then a call without a mask: refused, not a crash."""
    hpa._ORIG_SDPA[0] = _plain_sdpa
    monkeypatch.delenv("UNSLOTH_HF_PACKED_VARLEN", raising = False)
    monkeypatch.setattr(torch.Tensor, "is_cuda", property(lambda self: True))
    monkeypatch.setattr(hpa, "_torch_varlen", lambda device: object())
    monkeypatch.setattr(hpa, "_sdpa_policy_requested", lambda *args: False)
    monkeypatch.setattr(hpa, "_BLOCK_PREP", {})
    lengths = torch.tensor([3, 5])
    cu = torch.tensor([0, 3, 8], dtype = torch.int32)
    monkeypatch.setattr(
        "unsloth.utils.packing.get_packed_info_from_kwargs", lambda kwargs, device: (lengths, cu, 5)
    )
    hidden = torch.randn(1, 8, 8)
    kwargs = {"packed_seq_lengths": lengths, "attention_mask": _block_causal([3, 5])}
    assert hpa.block_varlen_kwargs(_block_layer(), hidden, kwargs) == (cu, 5)
    del kwargs["attention_mask"]  # the only reference: the cached weakref is now dead
    assert hpa.block_varlen_kwargs(_block_layer(), hidden, kwargs) is None


def test_block_varlen_kwargs_honours_the_sdpa_policy_each_step(monkeypatch):
    """Deterministic mode / a math-only sdpa_kernel is read per step, not cached with the hardware
    verdict: while it asks for SDPA the block keeps the dense path, and the fast path returns after."""
    hpa._ORIG_SDPA[0] = _plain_sdpa
    monkeypatch.delenv("UNSLOTH_HF_PACKED_VARLEN", raising = False)
    monkeypatch.setattr(torch.Tensor, "is_cuda", property(lambda self: True))
    monkeypatch.setattr(hpa, "_torch_varlen", lambda device: object())
    monkeypatch.setattr(hpa, "_BLOCK_PREP", {})
    lengths = [3, 5]
    cu = torch.tensor([0, 3, 8], dtype = torch.int32)
    monkeypatch.setattr(
        "unsloth.utils.packing.get_packed_info_from_kwargs",
        lambda kwargs, device: (torch.tensor(lengths), cu, 5),
    )
    policy = [True]
    monkeypatch.setattr(hpa, "_sdpa_policy_requested", lambda *args: policy[0])
    hidden = torch.randn(1, 8, 8)
    for wants_sdpa in (True, False):
        policy[0] = wants_sdpa
        # A new step: new lengths and mask objects.
        kwargs = {
            "packed_seq_lengths": torch.tensor(lengths),
            "attention_mask": _block_causal(lengths),
        }
        got = hpa.block_varlen_kwargs(_block_layer(), hidden, kwargs)
        assert got is None if wants_sdpa else got == (cu, 5)


def test_block_varlen_kwargs_rechecks_the_sdpa_policy_on_reused_tensors(monkeypatch):
    """A cache hit must still refuse cached metadata once the sdpa policy flips."""
    hpa._ORIG_SDPA[0] = _plain_sdpa
    monkeypatch.delenv("UNSLOTH_HF_PACKED_VARLEN", raising = False)
    monkeypatch.setattr(torch.Tensor, "is_cuda", property(lambda self: True))
    monkeypatch.setattr(hpa, "_torch_varlen", lambda device: object())
    monkeypatch.setattr(hpa, "_BLOCK_PREP", {})
    lengths = [3, 5]
    cu = torch.tensor([0, 3, 8], dtype = torch.int32)
    monkeypatch.setattr(
        "unsloth.utils.packing.get_packed_info_from_kwargs",
        lambda kwargs, device: (torch.tensor(lengths), cu, 5),
    )
    policy = [False]
    monkeypatch.setattr(hpa, "_sdpa_policy_requested", lambda *args: policy[0])
    hidden = torch.randn(1, 8, 8)
    kwargs = {"packed_seq_lengths": torch.tensor(lengths), "attention_mask": _block_causal(lengths)}
    layer = _block_layer()
    assert hpa.block_varlen_kwargs(layer, hidden, kwargs) == (cu, 5)
    assert hpa.block_varlen_kwargs(layer, hidden, kwargs) == (cu, 5)
    policy[0] = True
    assert hpa.block_varlen_kwargs(layer, hidden, kwargs) is None
    assert hpa.block_varlen_kwargs(layer, hidden, kwargs) is None
    policy[0] = False
    assert hpa.block_varlen_kwargs(layer, hidden, kwargs) == (cu, 5)


@pytest.mark.parametrize("in_graph", [True, False])
def test_traced_block_varlen_call_stays_in_the_graph(monkeypatch, in_graph):
    """With prepared metadata the traced wrapper calls the varlen op without a graph break; when the
    op cannot take the shape it keeps the wrapped sdpa, and the private kwarg never reaches it."""
    seen = {}

    def orig(module, q, k, v, mask, **kwargs):
        seen["orig_kwargs"] = set(kwargs)
        return _plain_sdpa(module, q, k, v, mask, **kwargs)

    hpa._ORIG_SDPA[0] = orig
    monkeypatch.delenv("UNSLOTH_HF_PACKED_VARLEN", raising = False)
    # Fake tensors keep their real (CPU) device here; the GPU test below covers the real checks.
    monkeypatch.setattr(hpa, "_static_window", lambda *args: None)
    monkeypatch.setattr(hpa, "_varlen_in_graph", lambda *args: in_graph)
    monkeypatch.setattr(
        hpa,
        "_flash_varlen",
        lambda q, k, v, cu, m, scale, window: q.transpose(1, 2) * scale,
    )
    lengths = [3, 3]
    module = types.SimpleNamespace(is_causal = True, num_key_value_groups = 1, training = True)
    q = torch.randn(1, 4, 6, 16, dtype = torch.bfloat16)
    mask, psl = _block_causal(lengths), torch.tensor(lengths, dtype = torch.int32)
    cu = torch.tensor([0, 3, 6], dtype = torch.int32)
    torch._dynamo.reset()
    torch._dynamo.utils.counters.clear()
    fn = torch.compile(
        lambda q: hpa._sdpa_packed_varlen(
            module, q, q, q, mask, packed_seq_lengths = psl, **{hpa.BLOCK_VARLEN_KWARG: (cu, 3)}
        )[0],
        backend = "eager",
        fullgraph = True,
    )
    out = fn(q)
    assert sum(torch._dynamo.utils.counters["graph_break"].values()) == 0
    if in_graph:
        torch.testing.assert_close(out, q.transpose(1, 2) * 16**-0.5)
        assert "orig_kwargs" not in seen
    else:
        assert hpa.BLOCK_VARLEN_KWARG not in seen["orig_kwargs"]


@pytest.mark.parametrize("in_graph", [True, False])
def test_traced_packed_probes_take_symbolic_head_dims(monkeypatch, in_graph):
    """dynamic = True head sizes reach the probes as plain ints; the sequence dim stays dynamic."""
    from torch._dynamo.testing import CompileCounter

    keys = []

    def probe(name):
        def run(*key):
            keys.append((name, key))
            return in_graph if name == "varlen" else False

        return run

    def gqa_sdpa(
        module,
        q,
        k,
        v,
        mask,
        scaling = None,
        **kwargs,
    ):
        out = torch.nn.functional.scaled_dot_product_attention(
            q, k, v, attn_mask = mask, scale = scaling, enable_gqa = True
        )
        return out.transpose(1, 2), None

    hpa._ORIG_SDPA[0] = gqa_sdpa
    monkeypatch.delenv("UNSLOTH_HF_PACKED_VARLEN", raising = False)
    _block_flag(monkeypatch, True)
    monkeypatch.setattr(hpa, "_static_window", lambda *args: None)
    monkeypatch.setattr(hpa, "_VARLEN_IN_GRAPH", {})
    monkeypatch.setattr(hpa, "_LEAVES_BLOCK", {})
    monkeypatch.setattr(hpa, "_varlen_in_graph_probe", probe("varlen"))
    monkeypatch.setattr(hpa, "_leaves_block_probe", probe("leaves"))
    monkeypatch.setattr(
        hpa,
        "_flash_varlen",
        lambda q, k, v, cu, m, scale, window: q.transpose(1, 2) * scale,
    )
    module = types.SimpleNamespace(is_causal = True, num_key_value_groups = 2, training = True)
    torch._dynamo.reset()
    torch._dynamo.utils.counters.clear()
    counter = CompileCounter()
    fn = torch.compile(
        lambda q, k, mask, psl, cu: hpa._sdpa_packed_varlen(
            module, q, k, k, mask, packed_seq_lengths = psl, **{hpa.BLOCK_VARLEN_KWARG: (cu, 3)}
        )[0],
        backend = counter,
        fullgraph = True,
        dynamic = True,
    )
    # Segment counts and lengths avoid the head sizes (4, 2, 16): duck sizing would tie them.
    for lengths in ([2, 2, 2], [2, 5, 4, 1, 1], [7, 1, 1, 6, 2, 3]):
        T = sum(lengths)
        q = torch.randn(1, 4, T, 16)
        k = torch.randn(1, 2, T, 16)
        mask, psl = _block_causal(lengths), torch.tensor(lengths, dtype = torch.int32)
        cu = torch.tensor([0, *torch.tensor(lengths).cumsum(0).tolist()], dtype = torch.int32)
        out = fn(q, k, mask, psl, cu)
        if in_graph:
            torch.testing.assert_close(out, q.transpose(1, 2) * 16**-0.5)
        else:
            ref = gqa_sdpa(module, q, k, k, mask)[0]
            torch.testing.assert_close(out, ref)
    assert sum(torch._dynamo.utils.counters["graph_break"].values()) == 0
    assert counter.frame_count == 1
    expected = [("varlen", (None, torch.float32, 16, 4, 2, None))]
    if not in_graph:
        expected.append(("leaves", ("cpu", None, torch.float32, 16, None)))
    assert keys == expected
    assert all(type(x) is int for _, key in keys for x in key if isinstance(x, int))


def test_shared_forward_with_mixed_head_dims_compiles(monkeypatch):
    """Gemma4-style mixed head_dim: allow_unspec_int_on_nn_module (set by unsloth_zoo) makes
    self.head_dim symbolic even with dynamic = None; the probes must still get plain ints."""
    from torch._dynamo.testing import CompileCounter

    if not hasattr(torch._dynamo.config, "allow_unspec_int_on_nn_module"):
        pytest.skip("torch without allow_unspec_int_on_nn_module keeps module ints static")
    keys = []

    def probe(name):
        def run(*key):
            keys.append((name, key))
            return False

        return run

    def gqa_sdpa(
        module,
        q,
        k,
        v,
        mask,
        scaling = None,
        **kwargs,
    ):
        out = torch.nn.functional.scaled_dot_product_attention(
            q, k, v, attn_mask = mask, scale = scaling, enable_gqa = True
        )
        return out.transpose(1, 2), None

    class Attn(torch.nn.Module):
        def __init__(self, heads, kv_heads, head_dim):
            super().__init__()
            self.head_dim, self.is_causal = head_dim, True
            self.num_key_value_groups = heads // kv_heads
            self.q_proj = torch.nn.Linear(64, heads * head_dim, bias = False)
            self.k_proj = torch.nn.Linear(64, kv_heads * head_dim, bias = False)

        def forward(self, h, mask, psl):
            shape = (*h.shape[:-1], -1, self.head_dim)
            q = self.q_proj(h).view(shape).transpose(1, 2)
            k = self.k_proj(h).view(shape).transpose(1, 2)
            return hpa._sdpa_packed_varlen(self, q, k, k, mask, packed_seq_lengths = psl)[0]

    hpa._ORIG_SDPA[0] = gqa_sdpa
    monkeypatch.delenv("UNSLOTH_HF_PACKED_VARLEN", raising = False)
    _block_flag(monkeypatch, True)
    monkeypatch.setattr(hpa, "_static_window", lambda *args: None)
    monkeypatch.setattr(hpa, "_VARLEN_IN_GRAPH", {})
    monkeypatch.setattr(hpa, "_LEAVES_BLOCK", {})
    monkeypatch.setattr(hpa, "_varlen_in_graph_probe", probe("varlen"))
    monkeypatch.setattr(hpa, "_leaves_block_probe", probe("leaves"))
    torch.manual_seed(0)
    layers = [Attn(4, 2, 16), Attn(2, 1, 32), Attn(4, 2, 16), Attn(2, 1, 32)]
    lengths = [2, 5, 4]
    h = torch.randn(1, sum(lengths), 64)
    mask, psl = _block_causal(lengths), torch.tensor(lengths, dtype = torch.int32)
    torch._dynamo.reset()
    torch._dynamo.utils.counters.clear()
    counter = CompileCounter()
    with torch._dynamo.config.patch(allow_unspec_int_on_nn_module = True):
        compiled = torch.compile(Attn.forward, backend = counter, fullgraph = True)
        for layer in layers:
            out = compiled(layer, h, mask, psl)
            shape = (*h.shape[:-1], -1, layer.head_dim)
            q = layer.q_proj(h).view(shape).transpose(1, 2)
            k = layer.k_proj(h).view(shape).transpose(1, 2)
            torch.testing.assert_close(out, gqa_sdpa(layer, q, k, k, mask)[0])
    assert sum(torch._dynamo.utils.counters["graph_break"].values()) == 0
    assert {key[3] for name, key in keys if name == "leaves"} == {16, 32}
    assert all(type(x) is int for _, key in keys for x in key if isinstance(x, int))


@_CUDA
@_TORCH_VARLEN_HOST
@pytest.mark.parametrize("window", [None, 4])
def test_block_varlen_runs_inside_the_graph_and_matches_dense_sdpa(monkeypatch, window):
    """Real op, compiled fullgraph with the sequence dim dynamic like _block_compile: no graph break,
    one graph across segment layouts after the first change, forward and backward match SDPA."""
    if hpa._torch_varlen(torch.device("cuda", 0)) is None:
        pytest.skip(
            reason = "varlen kernel needs torch's varlen_attn with window_size (CUDA torch >= 2.11)"
        )
    from transformers.integrations.sdpa_attention import sdpa_attention_forward

    # Engagement without a Python side effect (Dynamo would guard on a counter): the dense fallback
    # fails the test, so every layout below ran the in-graph kernel.
    def _dense(*args, **kwargs):
        raise AssertionError("packed row fell back to SDPA over the dense mask")

    hpa._ORIG_SDPA[0] = _dense
    monkeypatch.delenv("UNSLOTH_HF_PACKED_VARLEN", raising = False)
    monkeypatch.setattr(hpa, "_VARLEN_IN_GRAPH", {})
    H, n_kv, D = 8, 2, 64
    module = types.SimpleNamespace(
        is_causal = True, num_key_value_groups = H // n_kv, training = True, sliding_window = window
    )
    fn = torch.compile(
        lambda q, k, v, mask, psl, varlen: hpa._sdpa_packed_varlen(
            module,
            q,
            k,
            v,
            mask,
            scaling = 0.11,
            packed_seq_lengths = psl,
            **{hpa.BLOCK_VARLEN_KWARG: varlen},
        )[0],
        fullgraph = True,
    )
    torch._dynamo.reset()
    torch._dynamo.utils.counters.clear()
    torch.manual_seed(0)
    for lengths in ([37, 100, 2, 5, 230], [64, 64, 9], [300, 1, 41, 7], [128] * 3):
        T = sum(lengths)
        q, k, v = (
            torch.randn(1, h, T, D, device = "cuda", dtype = torch.bfloat16, requires_grad = True)
            for h in (H, n_kv, n_kv)
        )
        mask = _block_causal(lengths, "cuda")
        if window is not None:
            idx = torch.arange(T, device = "cuda")
            mask = mask & (idx[None, :] > idx[:, None] - window)[None, None]
        cu = torch.tensor([0] + lengths, device = "cuda").cumsum(0).to(torch.int32)
        for t, dims in ((q, (2,)), (k, (2,)), (v, (2,)), (mask, (2, 3)), (cu, (0,))):
            torch._dynamo.maybe_mark_dynamic(t, dims)
        out = fn(q, k, v, mask, torch.tensor(lengths, dtype = torch.int32), (cu, max(lengths)))
        grad = torch.randn_like(out)
        got = torch.autograd.grad(out, (q, k, v), grad)
        # Math reference: cuDNN on torch 2.6 - 2.13 can reuse a stale backward plan (wrong grads).
        with torch.nn.attention.sdpa_kernel(torch.nn.attention.SDPBackend.MATH):
            ref_out, _ = sdpa_attention_forward(module, q, k, v, mask, scaling = 0.11)
        ref = torch.autograd.grad(ref_out, (q, k, v), grad)
        torch.testing.assert_close(out.float(), ref_out.float(), atol = 2e-2, rtol = 2e-2)
        for g, r in zip(got, ref):
            torch.testing.assert_close(g.float(), r.float(), atol = 3e-2, rtol = 3e-2)
    assert sum(torch._dynamo.utils.counters["graph_break"].values()) == 0
    # max_seqlen specializes once, then is dynamic: at most two graphs for four layouts.
    assert torch._dynamo.utils.counters["stats"]["unique_graphs"] <= 2


@pytest.mark.parametrize("mode", ["no_grad", "inference_mode"])
def test_varlen_in_graph_probe_runs_its_backward_with_grad_enabled(monkeypatch, mode):
    """Probe under no_grad must still run and check a backward (verdict is cached for all modes)."""
    seen = []

    def torch_varlen(device):
        seen.append((torch.is_grad_enabled(), torch.is_inference_mode_enabled()))
        return None

    monkeypatch.setattr(hpa, "_torch_varlen", torch_varlen)
    context = torch.no_grad() if mode == "no_grad" else torch.inference_mode()
    with context:
        assert hpa._varlen_in_graph_probe(0, torch.bfloat16, 64, 8, 2, None) is False
        assert torch.is_grad_enabled() is False  # the caller's mode is restored
    assert seen == [(True, False)]


@pytest.mark.parametrize("compiled", [False, True])
def test_varlen_in_graph_probe_inside_a_nonreentrant_checkpoint(monkeypatch, compiled):
    """First trace inside a non-reentrant checkpoint (VLMs under DDP): the probe's own backward must
    not go through the checkpoint's saved-tensor hooks, and the outer backward must still recompute."""
    from torch.utils.checkpoint import checkpoint

    seen = []

    def torch_varlen(device):
        # Stands in for the probe's kernel forward and backward (CUDA only).
        try:
            q = torch.zeros(4, 4, requires_grad = True)
            (q @ q).sin().sum().backward()
            seen.append(q.grad is not None)
        except Exception as error:
            seen.append(repr(error))
        return None

    monkeypatch.setattr(hpa, "_torch_varlen", torch_varlen)
    monkeypatch.setattr(hpa, "_VARLEN_IN_GRAPH", {})

    def block(x):
        if hpa._varlen_in_graph(0, torch.bfloat16, 64, 8, 2, None):
            return (x * 3).cos()
        return (x * 2).sin()

    if compiled:
        # Dynamo calls the assume_constant_result probe eagerly while tracing, hooks still live.
        torch._dynamo.reset()
        block = torch.compile(block, backend = "eager", fullgraph = True)
    x = torch.randn(4, 4, requires_grad = True)
    checkpoint(block, x, use_reentrant = False).sum().backward()
    assert seen == [True]
    torch.testing.assert_close(x.grad, 2 * torch.cos(2 * x.detach()))


@_CUDA
@_TORCH_VARLEN_HOST
@pytest.mark.parametrize("mode", ["no_grad", "inference_mode"])
def test_varlen_in_graph_verdict_does_not_depend_on_the_first_grad_mode(monkeypatch, mode):
    """Real probe under no_grad / inference_mode caches the grad-enabled answer."""
    if hpa._torch_varlen(torch.device("cuda", 0)) is None:
        pytest.skip(
            reason = "varlen kernel needs torch's varlen_attn with window_size (CUDA torch >= 2.11)"
        )
    key = (0, torch.bfloat16, 64, 8, 2, None)
    expected = hpa._varlen_in_graph_probe(*key)
    if not expected:
        pytest.skip(reason = "this build's flash varlen does not take a 64-wide GQA row")
    monkeypatch.setattr(hpa, "_VARLEN_IN_GRAPH", {})
    context = torch.no_grad() if mode == "no_grad" else torch.inference_mode()
    with context:
        assert hpa._varlen_in_graph_eager(*key) is expected
    assert hpa._varlen_in_graph_eager(*key) is expected


@_CUDA
@_TORCH_VARLEN_HOST
def test_reentrant_checkpoint_first_trace_keeps_the_block_varlen_path(monkeypatch):
    """First compiled inside a reentrant checkpoint: forward and recompute both run varlen."""
    if hpa._torch_varlen(torch.device("cuda", 0)) is None:
        pytest.skip(
            reason = "varlen kernel needs torch's varlen_attn with window_size (CUDA torch >= 2.11)"
        )
    from torch.utils.checkpoint import checkpoint
    from transformers.integrations.sdpa_attention import sdpa_attention_forward

    def _dense(*args, **kwargs):
        raise AssertionError("packed row fell back to SDPA over the dense mask")

    hpa._ORIG_SDPA[0] = _dense
    monkeypatch.delenv("UNSLOTH_HF_PACKED_VARLEN", raising = False)
    monkeypatch.setattr(hpa, "_VARLEN_IN_GRAPH", {})
    H, n_kv, D = 8, 2, 64
    module = types.SimpleNamespace(
        is_causal = True, num_key_value_groups = H // n_kv, training = True, sliding_window = None
    )
    lengths = [37, 100, 2, 5, 230]
    T = sum(lengths)
    mask = _block_causal(lengths, "cuda")
    psl = torch.tensor(lengths, dtype = torch.int32)
    cu = _cu(lengths, "cuda")
    fn = torch.compile(
        lambda q, k, v: hpa._sdpa_packed_varlen(
            module,
            q,
            k,
            v,
            mask,
            scaling = 0.11,
            packed_seq_lengths = psl,
            **{hpa.BLOCK_VARLEN_KWARG: (cu, max(lengths))},
        )[0],
        fullgraph = True,
    )
    torch._dynamo.reset()
    torch.manual_seed(0)
    q, k, v = (
        torch.randn(1, h, T, D, device = "cuda", dtype = torch.bfloat16, requires_grad = True)
        for h in (H, n_kv, n_kv)
    )
    out = checkpoint(fn, q, k, v, use_reentrant = True)
    grad = torch.randn_like(out)
    out.backward(grad)  # reentrant checkpoint refuses autograd.grad
    got = (q.grad, k.grad, v.grad)
    # Math reference: the dense-mask backend sdpa picks can change with earlier tests in this file.
    with torch.nn.attention.sdpa_kernel(torch.nn.attention.SDPBackend.MATH):
        ref_out, _ = sdpa_attention_forward(module, q, k, v, mask, scaling = 0.11)
    ref = torch.autograd.grad(ref_out, (q, k, v), grad)
    torch.testing.assert_close(out.float(), ref_out.float(), atol = 2e-2, rtol = 2e-2)
    for g, r in zip(got, ref):
        torch.testing.assert_close(g.float(), r.float(), atol = 3e-2, rtol = 3e-2)


def _dtype_seen():
    seen = []

    def orig(
        module,
        q,
        k,
        v,
        mask,
        dropout = 0.0,
        scaling = None,
        is_causal = None,
        **kwargs,
    ):
        seen.append((q.dtype, k.dtype, v.dtype))
        return torch.nn.functional.scaled_dot_product_attention(
            q, k, v, attn_mask = mask, scale = scaling
        ), None

    return seen, orig


@pytest.mark.parametrize(
    "case, autocast, dtype, packed, expected",
    [
        # Full finetuning: fp32 q_norm / k_norm leave Q and K in fp32 under bf16 autocast.
        ("fp32_under_bf16_autocast", torch.bfloat16, torch.float32, True, torch.bfloat16),
        ("fp32_under_fp16_autocast", torch.float16, torch.float32, True, torch.float16),
        ("fp32_without_autocast", None, torch.float32, True, torch.float32),
        ("bf16_under_fp16_autocast", torch.float16, torch.bfloat16, True, torch.bfloat16),
        ("unpacked_fp32_under_autocast", torch.bfloat16, torch.float32, False, torch.float32),
    ],
)
def test_packed_fp32_rows_under_autocast_run_in_the_autocast_dtype(
    monkeypatch, case, autocast, dtype, packed, expected
):
    """Packed fp32 Q / K / V under autocast reach the varlen checks in the dtype SDPA runs in."""
    seen, orig = _dtype_seen()
    hpa._ORIG_SDPA[0] = orig
    monkeypatch.delenv("UNSLOTH_HF_PACKED_VARLEN", raising = False)
    lengths = [3, 5]
    q, k, v = (torch.randn(1, 2, 8, 16, dtype = dtype) for _ in range(3))
    kwargs = {"packed_seq_lengths": torch.tensor(lengths, dtype = torch.int32)} if packed else {}
    module = types.SimpleNamespace(is_causal = True)
    mask = _block_causal(lengths)
    context = (
        torch.autocast("cpu", dtype = autocast) if autocast is not None else contextlib.nullcontext()
    )
    with context:
        out, _ = hpa._sdpa_packed_varlen(module, q, k, v, mask, scaling = 0.25, **kwargs)
        ref = torch.nn.functional.scaled_dot_product_attention(q, k, v, attn_mask = mask, scale = 0.25)
    assert seen == [(dtype, dtype, dtype)]
    assert out.dtype == ref.dtype
    torch.testing.assert_close(out, ref, atol = 0, rtol = 0)
    if packed:
        with context:
            assert {t.dtype for t in hpa._autocast_float32(q, k, v)} == {expected}


@pytest.mark.parametrize("autocast", [torch.bfloat16, torch.float16])
@pytest.mark.parametrize("n_kv", [2, 8])
def test_packed_fp32_fallback_rows_keep_their_gradients(monkeypatch, autocast, n_kv):
    """Fallback rows must get uncast Q / K / V: GQA's dK / dV reduce in fp32 after repeat_kv."""
    from transformers.integrations.sdpa_attention import sdpa_attention_forward

    hpa._ORIG_SDPA[0] = sdpa_attention_forward
    monkeypatch.delenv("UNSLOTH_HF_PACKED_VARLEN", raising = False)
    torch.manual_seed(0)
    lengths = [37, 100, 2, 5, 56]
    T, H, D = sum(lengths), 8, 64
    module = types.SimpleNamespace(is_causal = True, num_key_value_groups = H // n_kv, training = True)
    q = torch.randn(1, H, T, D, requires_grad = True)
    k = torch.randn(1, n_kv, T, D, requires_grad = True)
    v = torch.randn(1, n_kv, T, D, requires_grad = True)
    mask, psl = _block_causal(lengths), torch.tensor(lengths, dtype = torch.int32)
    with torch.autocast("cpu", dtype = autocast):
        out, _ = hpa._sdpa_packed_varlen(
            module, q, k, v, mask, scaling = 0.125, packed_seq_lengths = psl
        )
        ref, _ = sdpa_attention_forward(module, q, k, v, mask, scaling = 0.125)
    g = torch.randn_like(ref)
    got = torch.autograd.grad(out, (q, k, v), g)
    want = torch.autograd.grad(ref, (q, k, v), g)
    torch.testing.assert_close(out, ref, atol = 0, rtol = 0)
    for a, b in zip(got, want):
        assert a.dtype == b.dtype == torch.float32
        torch.testing.assert_close(a, b, atol = 0, rtol = 0)


@_CUDA
@_TORCH_VARLEN_HOST
@pytest.mark.parametrize("autocast", [torch.bfloat16, torch.float16])
def test_packed_fp32_rows_under_autocast_take_varlen(monkeypatch, autocast):
    if hpa._torch_varlen(torch.device("cuda", 0)) is None:
        pytest.skip("this torch has no window_size varlen_attn")
    from transformers.integrations.sdpa_attention import sdpa_attention_forward

    hpa._ORIG_SDPA[0] = sdpa_attention_forward
    monkeypatch.delenv("UNSLOTH_HF_PACKED_VARLEN", raising = False)
    torch.manual_seed(0)
    lengths = [37, 100, 2, 5, 230]
    T, H, n_kv, D = sum(lengths), 8, 2, 64
    module = types.SimpleNamespace(is_causal = True, num_key_value_groups = H // n_kv, training = True)
    q = torch.randn(1, H, T, D, device = "cuda", requires_grad = True)
    k = torch.randn(1, n_kv, T, D, device = "cuda", requires_grad = True)
    v = torch.randn(1, n_kv, T, D, device = "cuda", requires_grad = True)
    mask, psl = _block_causal(lengths, "cuda"), torch.tensor(lengths, dtype = torch.int32)
    before = dict(hpa.HF_PACKED_ATTENTION_STATS)
    with torch.autocast("cuda", dtype = autocast):
        fast, _ = hpa._sdpa_packed_varlen(
            module, q, k, v, mask, scaling = 0.11, packed_seq_lengths = psl
        )
        ref, _ = sdpa_attention_forward(module, q, k, v, mask, scaling = 0.11)
    assert hpa.HF_PACKED_ATTENTION_STATS["fast"] == before["fast"] + 1
    assert hpa.HF_PACKED_ATTENTION_STATS["fallback"] == before["fallback"]
    assert fast.dtype == ref.dtype == autocast and fast.shape == ref.shape
    torch.testing.assert_close(fast.float(), ref.float(), atol = 2e-2, rtol = 2e-2)
    g = torch.randn_like(ref)
    grads_fast = torch.autograd.grad(fast, (q, k, v), g)
    grads_ref = torch.autograd.grad(ref, (q, k, v), g)
    for a, b in zip(grads_fast, grads_ref):
        assert a.dtype == torch.float32
        torch.testing.assert_close(a, b, atol = 5e-2, rtol = 5e-2)