Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
head_dim 512 attention: use the kernel for masks made under inference mode (#14202)
Daniel Han9faa9952172d
diff --git a/tests/test_large_head_attention.py b/tests/test_large_head_attention.pyindex eae63d174cc..1c5c439d06d 100644--- a/tests/test_large_head_attention.py+++ b/tests/test_large_head_attention.py@@ -443,8 +443,7 @@ def test_router_falls_back_to_sdpa_on_kernel_oom(monkeypatch):@pytest.mark.skipif(not _SM80, reason = "the router engages on sm80+ only")def test_router_skips_the_mask_scan_when_it_cannot_use_the_result(monkeypatch):- # The scan needs a host sync: inference masks would pay it on every layer, and a failed self- # check makes the result unusable.+ # The scan needs a host sync, and a failed self check makes the result unusable.def scan(*args, **kwargs):raise AssertionError("mask scanned")@@ -455,12 +454,9 @@ def test_router_skips_the_mask_scan_when_it_cannot_use_the_result(monkeypatch):monkeypatch.setattr(LH, "intervals_from_mask", scan)q = torch.randn(1, 4, 24, 512, device = "cuda", dtype = dtype)kv = torch.randn(1, 2, 24, 512, device = "cuda", dtype = dtype)- with torch.inference_mode():- mask = _causal(24).cuda()[None, None]- assert router(_Module(), q, kv, kv, mask)[0] == "original"monkeypatch.setitem(LH._SELF_CHECK, (str(device), dtype), False)assert router(_Module(), q, kv, kv, _causal(24).cuda()[None, None])[0] == "original"- assert len(calls) == 2+ assert len(calls) == 1def test_pre_ampere_keeps_sdpa_without_compiling(monkeypatch):@@ -496,3 +492,24 @@ def test_install_skipped_when_every_self_check_fails(monkeypatch):assert ALL_ATTENTION_FUNCTIONS["sdpa"] is savedfinally:ALL_ATTENTION_FUNCTIONS["sdpa"] = saved++[email protected](not _SM80, reason = "the router engages on sm80+ only")+def test_router_uses_the_kernel_for_masks_made_under_inference_mode():+ # generate() / eval build their masks under inference_mode. Sending those to SDPA repeated the+ # KV heads and kept the dense mask (OOM at 8192 tokens where the kernel fits); they are rescanned+ # per call instead, since an inference tensor has no version counter to cache the verdict on.+ calls, original = _sentinel()+ router = LH._make_router(original)+ device, dtype = torch.device("cuda", torch.cuda.current_device()), _cuda_dtype()+ assert LH._self_check(device, dtype) is True+ q = torch.randn(1, 4, 24, 512, device = "cuda", dtype = dtype)+ kv = torch.randn(1, 2, 24, 512, device = "cuda", dtype = dtype)+ with torch.inference_mode():+ mask = _causal(24).cuda()[None, None]+ before = LH.LARGE_HEAD_ENGAGED["calls"]+ out, _ = router(_Module(), q, kv, kv, mask)+ reference, _ = router(_Module(), q, kv, kv, None)+ assert not isinstance(out, str) and len(calls) == 0+ assert LH.LARGE_HEAD_ENGAGED["calls"] == before + 2+ torch.testing.assert_close(out, reference)diff --git a/unsloth/kernels/large_head_attention.py b/unsloth/kernels/large_head_attention.pyindex f031bcfbd3c..48f6508f32e 100644--- a/unsloth/kernels/large_head_attention.py+++ b/unsloth/kernels/large_head_attention.py@@ -865,13 +865,10 @@ def _make_router(original):if attention_mask is None:causal = is_causal if is_causal is not None else getattr(module, "is_causal", True)intervals = _dense_intervals(B, S, bool(causal), query.device)- elif (- not compiling- and not (torch.is_tensor(attention_mask) and attention_mask.is_inference())- and _SELF_CHECK.get((str(query.device), query.dtype), True)- ):+ elif not compiling and _SELF_CHECK.get((str(query.device), query.dtype), True):# Reading a mask's structure needs a host sync; compiled graphs keep SDPA. Inference- # masks cannot cache the verdict across layers, so they keep SDPA too.+ # masks cannot cache the verdict, so they are rescanned per layer: one sync each, only+ # at prefill (decode never qualifies), cheaper than SDPA's repeated KV heads and mask.intervals = intervals_from_mask(attention_mask, B, S)if compiling:checked = _SELF_CHECK.get((str(query.device), query.dtype), False)