Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

head_dim 512 attention: use the kernel for masks made under inference mode (#14202)

Daniel Han9faa9952172d
diff --git a/tests/test_large_head_attention.py b/tests/test_large_head_attention.py
index eae63d174cc..1c5c439d06d 100644
--- a/tests/test_large_head_attention.py
+++ b/tests/test_large_head_attention.py
@@ -443,8 +443,7 @@ def test_router_falls_back_to_sdpa_on_kernel_oom(monkeypatch):
@pytest.mark.skipif(not _SM80, reason = "the router engages on sm80+ only")
def test_router_skips_the_mask_scan_when_it_cannot_use_the_result(monkeypatch):
- # The scan needs a host sync: inference masks would pay it on every layer, and a failed self
- # check makes the result unusable.
+ # The scan needs a host sync, and a failed self check makes the result unusable.
def scan(*args, **kwargs):
raise AssertionError("mask scanned")
@@ -455,12 +454,9 @@ def test_router_skips_the_mask_scan_when_it_cannot_use_the_result(monkeypatch):
monkeypatch.setattr(LH, "intervals_from_mask", scan)
q = torch.randn(1, 4, 24, 512, device = "cuda", dtype = dtype)
kv = torch.randn(1, 2, 24, 512, device = "cuda", dtype = dtype)
- with torch.inference_mode():
- mask = _causal(24).cuda()[None, None]
- assert router(_Module(), q, kv, kv, mask)[0] == "original"
monkeypatch.setitem(LH._SELF_CHECK, (str(device), dtype), False)
assert router(_Module(), q, kv, kv, _causal(24).cuda()[None, None])[0] == "original"
- assert len(calls) == 2
+ assert len(calls) == 1
def test_pre_ampere_keeps_sdpa_without_compiling(monkeypatch):
@@ -496,3 +492,24 @@ def test_install_skipped_when_every_self_check_fails(monkeypatch):
assert ALL_ATTENTION_FUNCTIONS["sdpa"] is saved
finally:
ALL_ATTENTION_FUNCTIONS["sdpa"] = saved
+
+
[email protected](not _SM80, reason = "the router engages on sm80+ only")
+def test_router_uses_the_kernel_for_masks_made_under_inference_mode():
+ # generate() / eval build their masks under inference_mode. Sending those to SDPA repeated the
+ # KV heads and kept the dense mask (OOM at 8192 tokens where the kernel fits); they are rescanned
+ # per call instead, since an inference tensor has no version counter to cache the verdict on.
+ calls, original = _sentinel()
+ router = LH._make_router(original)
+ device, dtype = torch.device("cuda", torch.cuda.current_device()), _cuda_dtype()
+ assert LH._self_check(device, dtype) is True
+ q = torch.randn(1, 4, 24, 512, device = "cuda", dtype = dtype)
+ kv = torch.randn(1, 2, 24, 512, device = "cuda", dtype = dtype)
+ with torch.inference_mode():
+ mask = _causal(24).cuda()[None, None]
+ before = LH.LARGE_HEAD_ENGAGED["calls"]
+ out, _ = router(_Module(), q, kv, kv, mask)
+ reference, _ = router(_Module(), q, kv, kv, None)
+ assert not isinstance(out, str) and len(calls) == 0
+ assert LH.LARGE_HEAD_ENGAGED["calls"] == before + 2
+ torch.testing.assert_close(out, reference)
diff --git a/unsloth/kernels/large_head_attention.py b/unsloth/kernels/large_head_attention.py
index f031bcfbd3c..48f6508f32e 100644
--- a/unsloth/kernels/large_head_attention.py
+++ b/unsloth/kernels/large_head_attention.py
@@ -865,13 +865,10 @@ def _make_router(original):
if attention_mask is None:
causal = is_causal if is_causal is not None else getattr(module, "is_causal", True)
intervals = _dense_intervals(B, S, bool(causal), query.device)
- elif (
- not compiling
- and not (torch.is_tensor(attention_mask) and attention_mask.is_inference())
- and _SELF_CHECK.get((str(query.device), query.dtype), True)
- ):
+ elif not compiling and _SELF_CHECK.get((str(query.device), query.dtype), True):
# Reading a mask's structure needs a host sync; compiled graphs keep SDPA. Inference
- # masks cannot cache the verdict across layers, so they keep SDPA too.
+ # masks cannot cache the verdict, so they are rescanned per layer: one sync each, only
+ # at prefill (decode never qualifies), cheaper than SDPA's repeated KV heads and mask.
intervals = intervals_from_mask(attention_mask, B, S)
if compiling:
checked = _SELF_CHECK.get((str(query.device), query.dtype), False)