Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
"""head_dim 512 attention (unsloth/kernels/large_head_attention.py).
CPU: mask -> per-row key interval extraction, and the router leaving every unsupported call
to the original SDPA. CUDA: fwd + bwd against fp32 SDPA math over causal / packed / padded /
GQA shapes, and the router engaging on a Gemma-4 style call.
"""
import importlib.util
import os
import pytest
import torch
from real_accelerator import has_real_cuda # tests/_shared, on sys.path via tests/conftest.py
pytest.importorskip("triton")
_PATH = os.path.join(
os.path.dirname(__file__), "..", "unsloth", "kernels", "large_head_attention.py"
)
def _load():
spec = importlib.util.spec_from_file_location("_unsloth_large_head_attention", _PATH)
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
LH = _load()
_SM80 = has_real_cuda() and torch.cuda.get_device_capability()[0] >= 8
def _causal(S):
i = torch.arange(S)
return i[None, :] <= i[:, None]
def _packed(S, starts):
doc = torch.zeros(S, dtype = torch.long)
for s in starts:
doc[s:] += 1
return _causal(S) & (doc[None, :] == doc[:, None])
def test_intervals_causal_packed_padded_float():
S = 37
lo, hi = LH.intervals_from_mask(_causal(S)[None, None], 1, S)
assert lo.tolist() == [[0] * S] and hi.tolist() == [list(range(S))]
lo, hi = LH.intervals_from_mask(_packed(S, [5, 6, 20])[None, None], 1, S)
assert lo[0, :5].tolist() == [0] * 5 and lo[0, 5].item() == 5 and lo[0, 6].item() == 6
assert lo[0, 20:].tolist() == [20] * (S - 20) and hi.tolist() == [list(range(S))]
i = torch.arange(S)
right_pad = (_causal(S) & (i[None, :] < 30))[None, None]
lo, hi = LH.intervals_from_mask(right_pad.expand(2, 1, S, S), 2, S)
assert hi[1, 29:].tolist() == [29] * (S - 29)
additive = torch.zeros(1, 1, S, S)
additive[~_packed(S, [9])[None, None]] = torch.finfo(torch.float32).min
lo, _ = LH.intervals_from_mask(additive, 3, S)
assert lo.shape == (3, S) and lo[2, 9].item() == 9
def test_intervals_reject_unrepresentable_masks():
S = 16
i = torch.arange(S)
left_pad = _causal(S) & (i[None, :] >= 3) # rows 0..2 attend nothing
assert LH.intervals_from_mask(left_pad[None, None], 1, S) is None
holes = _causal(S).clone()
holes[10, 4] = False
assert LH.intervals_from_mask(holes[None, None], 1, S) is None
per_head = _causal(S)[None, None].expand(1, 2, S, S)
assert LH.intervals_from_mask(per_head, 1, S) is None
soft = torch.zeros(1, 1, S, S)
soft[0, 0, 5, 2] = -1.0 # a bias, not a mask
assert LH.intervals_from_mask(soft, 1, S) is None
assert LH.intervals_from_mask(_causal(S)[None, None, :, :8], 1, S) is None
def test_mask_verdict_is_cached_and_invalidated():
S = 12
mask = _causal(S)[None, None].clone()
assert LH.intervals_from_mask(mask, 1, S) is not None
mask[0, 0, 7, 2] = False
assert LH.intervals_from_mask(mask, 1, S) is None
def test_inference_mask_is_rescanned_after_in_place_update():
S = 12
with torch.inference_mode():
mask = _causal(S)[None, None].clone()
assert LH.intervals_from_mask(mask, 1, S) is not None
mask[0, 0, 7, 2] = False
assert LH.intervals_from_mask(mask, 1, S) is None
mask.copy_(_packed(S, [5, 7])[None, None])
lo, hi = LH.intervals_from_mask(mask, 1, S)
assert lo[0, 6].item() == 5 and hi[0, 4].item() == 4
def _sentinel():
calls = []
def original(
module,
query,
key,
value,
attention_mask,
dropout = 0.0,
scaling = None,
is_causal = None,
**kwargs,
):
calls.append(dict(dropout = dropout, scaling = scaling, is_causal = is_causal, **kwargs))
return "original", None
return calls, original
class _Module(torch.nn.Module):
is_causal = True
def test_router_defers_everything_it_cannot_run():
calls, original = _sentinel()
router = LH._make_router(original)
before = LH.LARGE_HEAD_ENGAGED["calls"]
q = torch.zeros(1, 4, 8, 512)
kv = torch.zeros(1, 2, 8, 512)
# CPU tensors, a 256 head, dropout and a softcap never reach the kernel.
assert router(_Module(), q, kv, kv, None, scaling = 1.0)[0] == "original"
small = torch.zeros(1, 4, 8, 256)
assert router(_Module(), small, small[:, :2], small[:, :2], None)[0] == "original"
assert router(_Module(), q, kv, kv, None, dropout = 0.1)[0] == "original"
assert router(_Module(), q, kv, kv, None, softcap = 30.0, sliding_window = None)[0] == "original"
assert len(calls) == 4
assert calls[0]["scaling"] == 1.0 and calls[3]["softcap"] == 30.0
assert LH.LARGE_HEAD_ENGAGED["calls"] == before
def test_kill_switch_skips_install(monkeypatch):
monkeypatch.setenv("UNSLOTH_HD512_ATTENTION", "0")
LH._enabled.cache_clear()
try:
assert LH.patch_large_head_attention() is False
finally:
monkeypatch.delenv("UNSLOTH_HD512_ATTENTION")
LH._enabled.cache_clear()
def _reference(q, k, v, allowed):
groups = q.shape[1] // k.shape[1]
k = k.repeat_interleave(groups, 1)
v = v.repeat_interleave(groups, 1)
scores = (q @ k.transpose(-1, -2)) * q.shape[-1] ** -0.5
scores = scores.masked_fill(~allowed, float("-inf"))
return (scores.softmax(-1) @ v).transpose(1, 2)
@pytest.mark.skipif(not has_real_cuda(), reason = "needs CUDA")
@pytest.mark.parametrize("S", [17, 64, 131])
@pytest.mark.parametrize("heads", [(4, 4), (8, 2), (8, 1)])
@pytest.mark.parametrize("kind", ["causal", "packed", "right_pad"])
def test_kernel_matches_reference(S, heads, kind):
H, HKV = heads
dtype = torch.bfloat16 if torch.cuda.get_device_capability()[0] >= 8 else torch.float16
torch.manual_seed(S + H)
B, D = 2, 512
allowed = (
{
"causal": _causal(S),
"packed": _packed(S, [1, S // 3, S // 3 + 2]),
"right_pad": _causal(S) & (torch.arange(S)[None, :] < S - 5),
}[kind]
.cuda()[None, None]
.expand(B, 1, S, S)
)
q = torch.randn(B, S, H, D, device = "cuda", dtype = dtype).transpose(1, 2)
k = torch.randn(B, S, HKV, D, device = "cuda", dtype = dtype).transpose(1, 2)
v = torch.randn(B, S, HKV, D, device = "cuda", dtype = dtype).transpose(1, 2)
grad = torch.randn(B, S, H, D, device = "cuda", dtype = dtype)
lo, hi = LH.intervals_from_mask(allowed, B, S)
ours_in = [x.detach().clone().requires_grad_() for x in (q, k, v)]
out = LH.large_head_attention(*ours_in, lo, hi)
out.backward(grad)
ref_in = [x.detach().float().requires_grad_() for x in (q, k, v)]
ref = _reference(*ref_in, allowed)
ref.backward(grad.float())
for a, b in zip([out] + [x.grad for x in ours_in], [ref] + [x.grad for x in ref_in]):
assert torch.isfinite(a).all()
err = (a.float() - b).abs()
assert err.max() <= 0.03 * b.abs().max() + 0.02
assert err.mean() <= 0.01 * b.abs().mean() + 2e-3
@pytest.mark.skipif(not _SM80, reason = "the router engages on sm80+ only")
def test_router_engages_on_gemma4_global_call():
calls, original = _sentinel()
router = LH._make_router(original)
dtype = torch.bfloat16 if torch.cuda.get_device_capability()[0] >= 8 else torch.float16
q = torch.randn(1, 8, 40, 512, device = "cuda", dtype = dtype)
kv = torch.randn(1, 1, 40, 512, device = "cuda", dtype = dtype)
before = LH.LARGE_HEAD_ENGAGED["calls"]
out, weights = router(_Module(), q, kv, kv, None, dropout = 0.0, scaling = 1.0, sliding_window = None)
assert weights is None and out.shape == (1, 40, 8, 512) and out.is_contiguous()
assert LH.LARGE_HEAD_ENGAGED["calls"] == before + 1 and not calls
left_pad = _causal(40).cuda() & (torch.arange(40, device = "cuda")[None, :] >= 3)
assert router(_Module(), q, kv, kv, left_pad[None, None])[0] == "original"
@pytest.mark.skipif(not _SM80, reason = "the router engages on sm80+ only")
def test_router_defers_trainable_additive_mask():
calls, original = _sentinel()
router = LH._make_router(original)
dtype = _cuda_dtype()
q = torch.randn(1, 4, 24, 512, device = "cuda", dtype = dtype)
kv = torch.randn(1, 2, 24, 512, device = "cuda", dtype = dtype)
additive = torch.zeros(1, 1, 24, 24, device = "cuda", dtype = dtype)
additive[~_causal(24).cuda()[None, None]] = torch.finfo(dtype).min
before = LH.LARGE_HEAD_ENGAGED["calls"]
assert router(_Module(), q, kv, kv, additive)[0] != "original"
assert LH.LARGE_HEAD_ENGAGED["calls"] == before + 1
assert router(_Module(), q, kv, kv, additive.requires_grad_())[0] == "original"
assert LH.LARGE_HEAD_ENGAGED["calls"] == before + 1 and len(calls) == 1
def _cuda_dtype():
return torch.bfloat16 if torch.cuda.get_device_capability()[0] >= 8 else torch.float16
@pytest.mark.skipif(not _SM80, reason = "the router engages on sm80+ only")
def test_self_check_leaves_global_rng_alone(monkeypatch):
device, dtype = torch.device("cuda", torch.cuda.current_device()), _cuda_dtype()
LH._SELF_CHECK.pop((str(device), dtype), None)
seeded = []
monkeypatch.setattr(torch.cuda, "manual_seed_all", lambda seed: seeded.append(seed))
state = torch.cuda.get_rng_state(device)
assert LH._self_check(device, dtype) is True
assert not seeded # would reseed the model's other GPUs
assert torch.equal(torch.cuda.get_rng_state(device), state)
@pytest.mark.skipif(not _SM80, reason = "the router engages on sm80+ only")
def test_self_check_under_inference_mode():
device, dtype = torch.device("cuda", torch.cuda.current_device()), _cuda_dtype()
LH._SELF_CHECK.pop((str(device), dtype), None)
with torch.inference_mode():
assert LH._self_check(device, dtype) is True
def _self_check_in_checkpoint(device, dtype):
# Cold first call from inside a non-reentrant checkpointed layer (DDP / FSDP2, Gemma-4
# E-series): returns the cached verdict, the layer's run count and whether w got a grad.
from torch.utils.checkpoint import checkpoint
LH._SELF_CHECK.pop((str(device), dtype), None)
calls = []
w = (torch.randn(8, 8, device = device) * 0.1).requires_grad_()
def layer(x):
calls.append(1)
h = (x @ w).sigmoid()
LH._self_check(device, dtype)
return (h @ w).sigmoid()
checkpoint(layer, torch.randn(4, 8, device = device), use_reentrant = False).sum().backward()
return LH._SELF_CHECK[(str(device), dtype)], len(calls), w.grad is not None
def test_self_check_stays_out_of_a_non_reentrant_checkpoint(monkeypatch):
# The check's tensors were packed into the layer's checkpoint and its autograd.grad recomputed
# the layer mid-forward: CheckpointError, cached as False, the kernel off for the process.
def kernel(q, k, v, lo, hi, scale):
groups = q.shape[1] // k.shape[1]
out = torch.nn.functional.scaled_dot_product_attention(
q, k.repeat_interleave(groups, 1), v.repeat_interleave(groups, 1), is_causal = True
)
return out.transpose(1, 2)
monkeypatch.setattr(LH, "large_head_attention", kernel)
monkeypatch.setattr(LH, "_measured_faster_on", lambda device: True)
monkeypatch.setattr(LH, "_dense_intervals", lambda *args: (None, None))
key = (str(torch.device("cpu")), torch.float32)
monkeypatch.setitem(LH._SELF_CHECK, key, None)
assert _self_check_in_checkpoint(torch.device("cpu"), torch.float32) == (True, 2, True)
@pytest.mark.skipif(not _SM80, reason = "the router engages on sm80+ only")
def test_real_self_check_stays_out_of_a_non_reentrant_checkpoint():
device, dtype = torch.device("cuda", torch.cuda.current_device()), _cuda_dtype()
assert _self_check_in_checkpoint(device, dtype) == (True, 2, True)
@pytest.mark.skipif(not _SM80, reason = "the router engages on sm80+ only")
def test_patch_self_checks_so_compiled_graphs_use_the_kernel(monkeypatch):
from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS
device, dtype = torch.device("cuda", torch.cuda.current_device()), _cuda_dtype()
LH._SELF_CHECK.pop((str(device), dtype), None)
model = torch.nn.Linear(4, 4).to(device = device, dtype = dtype)
model.head_dim = 512
saved = ALL_ATTENTION_FUNCTIONS["sdpa"]
monkeypatch.setattr(LH, "_PATCHED", [False])
try:
assert LH.patch_large_head_attention(model) is True
finally:
ALL_ATTENTION_FUNCTIONS["sdpa"] = saved
graphs = []
def backend(gm, example_inputs):
graphs.append(gm.code)
return gm.forward
calls, original = _sentinel()
router = LH._make_router(original)
q = torch.randn(1, 4, 24, 512, device = device, dtype = dtype)
kv = torch.randn(1, 2, 24, 512, device = device, dtype = dtype)
torch._dynamo.reset()
out, _ = torch.compile(router, backend = backend)(_Module(), q, kv, kv, None, scaling = 512**-0.5)
assert not calls and out.shape == (1, 24, 4, 512)
assert any("large_head_attention_fwd" in code for code in graphs)
assert LH._SELF_CHECK.get((str(device), dtype)) is True
@pytest.mark.skipif(not _SM80, reason = "the router engages on sm80+ only")
def test_router_defers_autocast_dtype_change():
calls, original = _sentinel()
router = LH._make_router(original)
q = torch.randn(1, 4, 24, 512, device = "cuda", dtype = torch.float16)
kv = torch.randn(1, 2, 24, 512, device = "cuda", dtype = torch.float16)
before = LH.LARGE_HEAD_ENGAGED["calls"]
with torch.autocast("cuda", dtype = torch.float16):
assert router(_Module(), q, kv, kv, None)[0] != "original"
assert LH.LARGE_HEAD_ENGAGED["calls"] == before + 1
with torch.autocast("cuda", dtype = torch.bfloat16):
assert router(_Module(), q, kv, kv, None)[0] == "original"
assert LH.LARGE_HEAD_ENGAGED["calls"] == before + 1 and len(calls) == 1
@pytest.mark.skipif(not _SM80, reason = "the router engages on sm80+ only")
def test_router_honours_math_sdpa_selection():
from torch.nn.attention import SDPBackend, sdpa_kernel
calls, original = _sentinel()
router = LH._make_router(original)
dtype = _cuda_dtype()
q = torch.randn(1, 4, 24, 512, device = "cuda", dtype = dtype)
kv = torch.randn(1, 2, 24, 512, device = "cuda", dtype = dtype)
before = LH.LARGE_HEAD_ENGAGED["calls"]
with sdpa_kernel([SDPBackend.MATH]):
assert router(_Module(), q, kv, kv, None)[0] == "original"
if hasattr(torch._C, "_get_sdp_priority_order"):
with sdpa_kernel([SDPBackend.MATH, SDPBackend.EFFICIENT_ATTENTION], set_priority = True):
assert router(_Module(), q, kv, kv, None)[0] == "original"
with sdpa_kernel([SDPBackend.EFFICIENT_ATTENTION, SDPBackend.MATH]):
assert router(_Module(), q, kv, kv, None)[0] != "original"
assert LH.LARGE_HEAD_ENGAGED["calls"] == before + 1
def test_router_keeps_wrapped_router_sentinels():
# unsloth_zoo's gemma-4 sliding router re-patches on every model load unless the current sdpa
# carries its sentinel; without it, it would wrap this router and the two would recurse.
mapping, orig = {}, [None]
def sdpa(module, query, key, value, attention_mask, **kwargs):
return "sdpa", None
def sliding(module, query, key, value, attention_mask, **kwargs):
return orig[0](module, query, key, value, attention_mask, **kwargs)
def patch_sliding():
if getattr(mapping["sdpa"], "_unsloth_gemma4_flash", False):
return
orig[0] = mapping["sdpa"]
sliding._unsloth_gemma4_flash = True
mapping["sdpa"] = sliding
mapping["sdpa"] = sdpa
patch_sliding()
router = LH._make_router(mapping["sdpa"])
mapping["sdpa"] = router
patch_sliding() # second model load
assert (
mapping["sdpa"] is router
and router._unsloth_large_head
and router._unsloth_original is sliding
)
q = torch.randn(1, 2, 4, 64)
assert mapping["sdpa"](None, q, q, q, None)[0] == "sdpa"
def test_install_sits_beneath_the_packed_router(monkeypatch):
# A packed trainer built before the model load already installed the packed router; the route of a
# packed call must not depend on which of the two was installed first.
import sys
import types
from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS
def inner(module, query, key, value, attention_mask, **kwargs):
return "inner", None
packed = types.ModuleType("unsloth.utils.hf_packed_attention")
packed._ORIG_SDPA = [inner]
def packed_router(module, query, key, value, attention_mask, **kwargs):
return packed._ORIG_SDPA[0](module, query, key, value, attention_mask, **kwargs)
model = torch.nn.Linear(4, 4)
model.head_dim = 512
saved = ALL_ATTENTION_FUNCTIONS["sdpa"]
monkeypatch.setitem(sys.modules, "unsloth.utils.hf_packed_attention", packed)
monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
monkeypatch.setattr(torch.version, "hip", None)
monkeypatch.setattr(LH, "_PATCHED", [False])
try:
ALL_ATTENTION_FUNCTIONS["sdpa"] = packed_router
assert LH.patch_large_head_attention(model) is True
assert ALL_ATTENTION_FUNCTIONS["sdpa"] is packed_router
assert packed._ORIG_SDPA[0]._unsloth_large_head
assert packed._ORIG_SDPA[0]._unsloth_original is inner
finally:
ALL_ATTENTION_FUNCTIONS["sdpa"] = saved
def test_autotune_skips_the_flush_buffer_benchmarker():
# Triton's default benchmarker allocates a 256 MB L2 flush buffer, enough to OOM a model that fits.
for kernel in (LH._lh_fwd, LH._lh_bwd_dq, LH._lh_bwd_dkdv):
assert kernel.do_bench is LH._bench_in_place
def test_self_check_oom_falls_back_to_sdpa(monkeypatch):
def oom(*args, **kwargs):
raise torch.cuda.OutOfMemoryError("CUDA out of memory.")
monkeypatch.setattr(LH, "large_head_attention", oom)
monkeypatch.setattr(LH, "_measured_faster_on", lambda device: True)
key = (str(torch.device("cpu")), torch.float32)
monkeypatch.setitem(LH._SELF_CHECK, key, None)
LH._SELF_CHECK.pop(key)
assert LH._self_check(torch.device("cpu"), torch.float32) is False
assert LH._SELF_CHECK[key] is False
@pytest.mark.skipif(not has_real_cuda(), reason = "needs CUDA")
def test_autotune_of_a_new_shape_adds_no_flush_buffer():
dtype = _cuda_dtype()
# G = 3 is not used elsewhere in this file, so this call autotunes fwd and bwd from scratch.
q = torch.randn(1, 6, 40, 512, device = "cuda", dtype = dtype, requires_grad = True)
k = torch.randn(1, 2, 40, 512, device = "cuda", dtype = dtype, requires_grad = True)
v = torch.randn(1, 2, 40, 512, device = "cuda", dtype = dtype, requires_grad = True)
lo, hi = LH._dense_intervals(1, 40, True, q.device)
torch.cuda.synchronize()
torch.cuda.reset_peak_memory_stats()
base = torch.cuda.memory_allocated()
LH.large_head_attention(q, k, v, lo, hi).float().sum().backward()
torch.cuda.synchronize()
assert torch.cuda.max_memory_allocated() - base < 32 * 1024**2
@pytest.mark.skipif(not _SM80, reason = "the router engages on sm80+ only")
def test_router_falls_back_to_sdpa_on_kernel_oom(monkeypatch):
calls, original = _sentinel()
router = LH._make_router(original)
dtype = _cuda_dtype()
assert LH._self_check(torch.device("cuda", torch.cuda.current_device()), dtype) is True
def oom(*args, **kwargs):
raise torch.cuda.OutOfMemoryError("CUDA out of memory.")
monkeypatch.setattr(LH, "_attention", oom)
q = torch.randn(1, 4, 24, 512, device = "cuda", dtype = dtype)
kv = torch.randn(1, 2, 24, 512, device = "cuda", dtype = dtype)
before = LH.LARGE_HEAD_ENGAGED["fallback"]
assert router(_Module(), q, kv, kv, None)[0] == "original"
assert LH.LARGE_HEAD_ENGAGED["fallback"] == before + 1 and len(calls) == 1
@pytest.mark.skipif(not _SM80, reason = "the router engages on sm80+ only")
def test_router_skips_the_mask_scan_when_it_cannot_use_the_result(monkeypatch):
# The scan needs a host sync, and a failed self check makes the result unusable.
def scan(*args, **kwargs):
raise AssertionError("mask scanned")
calls, original = _sentinel()
router = LH._make_router(original)
device, dtype = torch.device("cuda", torch.cuda.current_device()), _cuda_dtype()
assert LH._self_check(device, dtype) is True
monkeypatch.setattr(LH, "intervals_from_mask", scan)
q = torch.randn(1, 4, 24, 512, device = "cuda", dtype = dtype)
kv = torch.randn(1, 2, 24, 512, device = "cuda", dtype = dtype)
monkeypatch.setitem(LH._SELF_CHECK, (str(device), dtype), False)
assert router(_Module(), q, kv, kv, _causal(24).cuda()[None, None])[0] == "original"
assert len(calls) == 1
@pytest.mark.skipif(not _SM80, reason = "the router engages on sm80+ only")
def test_router_leaves_an_explicit_is_causal_beside_a_mask_to_sdpa():
calls, original = _sentinel()
router = LH._make_router(original)
dtype = _cuda_dtype()
q = torch.randn(1, 4, 24, 512, device = "cuda", dtype = dtype)
kv = torch.randn(1, 2, 24, 512, device = "cuda", dtype = dtype)
allowed = torch.ones(1, 1, 24, 24, device = "cuda", dtype = torch.bool)
before = LH.LARGE_HEAD_ENGAGED["calls"]
assert router(_Module(), q, kv, kv, allowed, is_causal = True)[0] == "original"
assert calls[-1]["is_causal"] is True and LH.LARGE_HEAD_ENGAGED["calls"] == before
causal = _causal(24).cuda()[None, None]
assert router(_Module(), q, kv, kv, causal)[0] != "original"
assert router(_Module(), q, kv, kv, causal, is_causal = False)[0] != "original"
assert LH.LARGE_HEAD_ENGAGED["calls"] == before + 2 and len(calls) == 1
@pytest.mark.skipif(not _SM80, reason = "the router engages on sm80+ only")
def test_router_falls_back_to_sdpa_on_mask_scan_oom(monkeypatch):
def oom(*args, **kwargs):
raise torch.cuda.OutOfMemoryError("CUDA out of memory.")
calls, original = _sentinel()
router = LH._make_router(original)
device, dtype = torch.device("cuda", torch.cuda.current_device()), _cuda_dtype()
assert LH._self_check(device, dtype) is True
monkeypatch.setattr(LH, "intervals_from_mask", oom)
q = torch.randn(1, 4, 24, 512, device = "cuda", dtype = dtype)
kv = torch.randn(1, 2, 24, 512, device = "cuda", dtype = dtype)
before = LH.LARGE_HEAD_ENGAGED["fallback"]
assert router(_Module(), q, kv, kv, _causal(24).cuda()[None, None])[0] == "original"
assert LH.LARGE_HEAD_ENGAGED["fallback"] == before + 1 and len(calls) == 1
def test_pre_ampere_keeps_sdpa_without_compiling(monkeypatch):
def kernel(*args, **kwargs):
raise AssertionError("kernel compiled on sm75")
monkeypatch.setattr(LH, "large_head_attention", kernel)
monkeypatch.setattr(torch.cuda, "get_device_capability", lambda device = None: (7, 5))
key = (str(torch.device("cpu")), torch.float16)
monkeypatch.setitem(LH._SELF_CHECK, key, None)
LH._SELF_CHECK.pop(key)
assert LH._self_check(torch.device("cpu"), torch.float16) is False
def test_install_skipped_when_every_self_check_fails(monkeypatch):
from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS
class _Param:
is_cuda = True
device = torch.device("cuda", 0)
dtype = torch.float16
model = torch.nn.Linear(4, 4)
model.head_dim = 512
monkeypatch.setattr(model, "parameters", lambda: iter([_Param()]))
monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
monkeypatch.setattr(torch.version, "hip", None)
monkeypatch.setattr(LH, "_PATCHED", [False])
monkeypatch.setattr(LH, "_self_check", lambda device, dtype: False)
saved = ALL_ATTENTION_FUNCTIONS["sdpa"]
try:
assert LH.patch_large_head_attention(model) is False
assert ALL_ATTENTION_FUNCTIONS["sdpa"] is saved
finally:
ALL_ATTENTION_FUNCTIONS["sdpa"] = saved
def test_enabled_reads_the_same_value_compiled_and_eager(monkeypatch):
# An lru_cache wrapper is traced through by dynamo (with a warning), so a compiled caller
# re-read the environment while eager callers kept the cached value.
import sys
# dynamo resolves globals through sys.modules; this test loads the file standalone.
monkeypatch.setitem(sys.modules, LH.__name__, LH)
monkeypatch.setenv("UNSLOTH_HD512_ATTENTION", "1")
LH._enabled.cache_clear()
try:
assert LH._enabled() is True
monkeypatch.setenv("UNSLOTH_HD512_ATTENTION", "0")
def f(x):
return x + 1 if LH._enabled() else x - 1
x = torch.zeros(3)
torch._dynamo.reset()
compiled = torch.compile(f, backend = "eager", fullgraph = True)(x)
assert torch.equal(compiled, f(x))
assert torch.equal(compiled, x + 1)
finally:
monkeypatch.delenv("UNSLOTH_HD512_ATTENTION")
LH._enabled.cache_clear()
torch._dynamo.reset()
def _install_with_params(monkeypatch, params, **attrs):
from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS
model = torch.nn.Linear(4, 4)
model.head_dim = 512
for name, value in attrs.items():
setattr(model, name, value)
monkeypatch.setattr(model, "parameters", lambda: iter(params))
monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
monkeypatch.setattr(torch.version, "hip", None)
monkeypatch.setattr(LH, "_PATCHED", [False])
checked = []
monkeypatch.setattr(
LH, "_self_check", lambda device, dtype: checked.append((device, dtype)) or True
)
saved = ALL_ATTENTION_FUNCTIONS["sdpa"]
try:
installed = LH.patch_large_head_attention(model)
finally:
ALL_ATTENTION_FUNCTIONS["sdpa"] = saved
return installed, checked
class _FakeParam:
is_cuda = True
device = torch.device("cuda", 0)
def __init__(
self,
dtype,
quant_dtype = None,
):
self.dtype = dtype
if quant_dtype is not None:
self.quant_state = type("QuantState", (), {"dtype": quant_dtype})()
def test_patch_self_checks_the_compute_dtype_of_4bit_only_shards(monkeypatch):
# A GPU holding only bnb 4-bit weights (uint8 storage), e.g. embeddings offloaded to CPU, must
# still get its load-time check, else compiled blocks never use the kernel there.
cpu = type(
"CpuParam", (), {"is_cuda": False, "device": torch.device("cpu"), "dtype": torch.bfloat16}
)()
installed, checked = _install_with_params(
monkeypatch, [cpu, _FakeParam(torch.uint8, torch.bfloat16)]
)
assert installed is True
assert checked == [(torch.device("cuda", 0), torch.bfloat16)]
def test_patch_self_checks_the_compute_dtype_of_float_packed_4bit(monkeypatch):
# bnb_4bit_quant_storage = float16 with bfloat16 compute: the storage dtype is a half type, but
# attention runs in bfloat16, so that is the verdict compiled blocks look up.
installed, checked = _install_with_params(
monkeypatch, [_FakeParam(torch.float16, torch.bfloat16)]
)
assert installed is True
assert checked == [(torch.device("cuda", 0), torch.bfloat16)]
def test_patch_skips_the_fp16_self_check_under_forced_float32(monkeypatch):
# zoo's forced float32 Gemma-4 attention runs float32 SDPA directly, so the router never sees it.
installed, checked = _install_with_params(
monkeypatch, [_FakeParam(torch.float16)], _unsloth_forced_float32 = True
)
assert installed is False and checked == []
installed, checked = _install_with_params(monkeypatch, [_FakeParam(torch.float16)])
assert installed is True and checked == [(torch.device("cuda", 0), torch.float16)]
@pytest.mark.skipif(not _SM80, reason = "the router engages on sm80+ only")
def test_router_uses_the_kernel_for_masks_made_under_inference_mode():
# generate() / eval build their masks under inference_mode. Sending those to SDPA repeated the
# KV heads and kept the dense mask (OOM at 8192 tokens where the kernel fits); they are rescanned
# per call instead, since an inference tensor has no version counter to cache the verdict on.
calls, original = _sentinel()
router = LH._make_router(original)
device, dtype = torch.device("cuda", torch.cuda.current_device()), _cuda_dtype()
assert LH._self_check(device, dtype) is True
q = torch.randn(1, 4, 24, 512, device = "cuda", dtype = dtype)
kv = torch.randn(1, 2, 24, 512, device = "cuda", dtype = dtype)
with torch.inference_mode():
mask = _causal(24).cuda()[None, None]
before = LH.LARGE_HEAD_ENGAGED["calls"]
out, _ = router(_Module(), q, kv, kv, mask)
reference, _ = router(_Module(), q, kv, kv, None)
assert not isinstance(out, str) and len(calls) == 0
assert LH.LARGE_HEAD_ENGAGED["calls"] == before + 2
torch.testing.assert_close(out, reference)
class _FakeCudaOperand:
"""A meta tensor that reports CUDA, so `_shapes_qualify` can be checked without a GPU."""
def __init__(
self,
shape,
stride,
dtype = torch.float16,
):
self._t = torch.empty_strided(shape, stride, device = "meta", dtype = dtype)
self.shape, self.dtype, self.is_cuda, self.device = self._t.shape, dtype, True, "cuda:0"
def dim(self):
return self._t.dim()
def numel(self):
return self._t.numel()
def stride(self, dim = None):
return self._t.stride() if dim is None else self._t.stride(dim)
def test_shapes_qualify_bounds_the_reachable_row_offset():
# 2,560 elements, but row 4 starts 2**31 elements in: int32 row offsets wrap in the kernel.
dense = _FakeCudaOperand((1, 1, 5, 512), (2560, 2560, 512, 1))
wide = _FakeCudaOperand((1, 1, 5, 512), (0, 0, 1 << 29, 1))
assert LH._shapes_qualify(dense, dense, dense)
assert not LH._shapes_qualify(wide, dense, dense)
assert not LH._shapes_qualify(dense, wide, dense)
assert not LH._shapes_qualify(dense, dense, wide)
# Batch and head offsets are int64 in the kernels, so a wide batch stride still qualifies.
batch = _FakeCudaOperand((2, 1, 5, 512), (1 << 31, 2560, 512, 1))
assert LH._shapes_qualify(batch, batch, batch)
# The last row's last element at exactly 2**31 - 1 is the largest offset that still fits.
edge = _FakeCudaOperand((1, 1, 2, 512), (0, 0, LH._INT32_MAX - 511, 1))
dense_k = _FakeCudaOperand((1, 1, 2, 512), (1024, 1024, 512, 1))
assert LH._shapes_qualify(edge, dense_k, dense_k)
over = _FakeCudaOperand((1, 1, 2, 512), (0, 0, LH._INT32_MAX - 510, 1))
assert not LH._shapes_qualify(over, dense_k, dense_k)
def _wide_row_stride(shape, dtype, seed):
# The storage runs to the last row's offset: 2**29 elements per row, over 4 GiB for five rows.
need = ((shape[-2] - 1) * (1 << 29) + shape[-1]) * torch.empty((), dtype = dtype).element_size()
if torch.cuda.mem_get_info()[0] < need + (1 << 30):
pytest.skip(reason = f"needs {need / 2**30:.1f} GiB free for a 2**29 element row stride")
gen = torch.Generator(device = "cuda").manual_seed(seed)
x = torch.empty_strided(shape, (0, 0, 1 << 29, 1), device = "cuda", dtype = dtype)
x.copy_(torch.randn(shape, device = "cuda", dtype = dtype, generator = gen))
return x
@pytest.mark.skipif(not _SM80, reason = "the router engages on sm80+ only")
def test_router_sends_a_wide_row_stride_query_to_sdpa():
dtype = _cuda_dtype()
S = 5
q = _wide_row_stride((1, 1, S, 512), dtype, 0)
k = torch.randn(1, 1, S, 512, device = "cuda", dtype = dtype)
v = torch.randn_like(k)
def original(module, query, key, value, attention_mask, **kwargs):
out = torch.nn.functional.scaled_dot_product_attention(query, key, value, is_causal = True)
return out.transpose(1, 2).contiguous(), None
router = LH._make_router(original)
before = LH.LARGE_HEAD_ENGAGED["calls"]
out, _ = router(_Module(), q, k, v, None)
torch.cuda.synchronize()
assert LH.LARGE_HEAD_ENGAGED["calls"] == before
ref = _reference(q.float(), k.float(), v.float(), _causal(S).cuda())
assert torch.isfinite(out).all()
assert (out.float() - ref).abs().max() <= 0.03 * ref.abs().max() + 0.02
del q
torch.cuda.empty_cache()
@pytest.mark.skipif(not has_real_cuda(), reason = "needs CUDA")
def test_backward_copies_a_wide_row_stride_grad():
dtype = torch.bfloat16 if torch.cuda.get_device_capability()[0] >= 8 else torch.float16
S = 5
torch.manual_seed(1)
q, k, v = [torch.randn(1, 1, S, 512, device = "cuda", dtype = dtype) for _ in range(3)]
lo, hi = LH._dense_intervals(1, S, True, q.device)
ours_in = [x.clone().requires_grad_() for x in (q, k, v)]
out = LH.large_head_attention(*ours_in, lo, hi)
grad = _wide_row_stride((1, 1, S, 512), dtype, 2) # [B, H, S, D], as the kernel reads dO
out.backward(grad.transpose(1, 2))
torch.cuda.synchronize()
ref_in = [x.detach().float().requires_grad_() for x in (q, k, v)]
_reference(*ref_in, _causal(S).cuda()).backward(grad.float().transpose(1, 2))
for a, b in zip([x.grad for x in ours_in], [x.grad for x in ref_in]):
assert torch.isfinite(a).all()
assert (a.float() - b).abs().max() <= 0.03 * b.abs().max() + 0.02
del grad
torch.cuda.empty_cache()
@pytest.mark.skipif(not _SM80, reason = "the router engages on sm80+ only")
def test_router_captures_into_a_cuda_graph_with_an_inference_mask():
# The mask scan syncs the host ("operation not permitted when stream is capturing"), and a graph
# would replay one scan's verdict for whatever the mask buffer holds later, so capture uses SDPA.
def original(module, query, key, value, attention_mask, **kwargs):
key, value = key.repeat_interleave(2, 1), value.repeat_interleave(2, 1)
out = torch.nn.functional.scaled_dot_product_attention(
query, key, value, attn_mask = attention_mask
)
return out.transpose(1, 2).contiguous(), None
router = LH._make_router(original)
device, dtype = torch.device("cuda", torch.cuda.current_device()), _cuda_dtype()
assert LH._self_check(device, dtype) is True
with torch.inference_mode():
q = torch.randn(1, 4, 24, 512, device = "cuda", dtype = dtype)
kv = torch.randn(1, 2, 24, 512, device = "cuda", dtype = dtype)
mask = _causal(24).cuda()[None, None]
eager_masked = router(_Module(), q, kv, kv, mask)[0]
eager_dense = router(_Module(), q, kv, kv, None)[0]
stream = torch.cuda.Stream()
stream.wait_stream(torch.cuda.current_stream())
with torch.cuda.stream(stream):
for _ in range(3):
router(_Module(), q, kv, kv, mask)
router(_Module(), q, kv, kv, None)
stream.synchronize()
before = LH.LARGE_HEAD_ENGAGED["calls"]
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph, stream = stream):
masked = router(_Module(), q, kv, kv, mask)[0]
dense = router(_Module(), q, kv, kv, None)[0]
# The unmasked call still runs the kernel inside the graph; the masked one runs SDPA.
assert LH.LARGE_HEAD_ENGAGED["calls"] == before + 1
graph.replay()
torch.cuda.synchronize()
assert LH._SELF_CHECK[(str(device), dtype)] is True
torch.testing.assert_close(masked, eager_masked, atol = 2e-2, rtol = 2e-2)
torch.testing.assert_close(dense, eager_dense)