Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Attention routing tests: follow torch's own GQA kernel choice (fixes two failures on torch 2.14) (#14672)
Daniel Han0c001fcebe14
diff --git a/tests/utils/test_attention_native_routing.py b/tests/utils/test_attention_native_routing.pyindex 028c1d0a6a6..711850290da 100644--- a/tests/utils/test_attention_native_routing.py+++ b/tests/utils/test_attention_native_routing.py@@ -99,11 +99,28 @@ def _dense(monkeypatch, dtype, mask):return out, seen, (Q, K, V, attention_mask)+def _gqa_falls_to_math(Q, K, masked):+ # Expand K/V only where GQA leaves math (or nothing) while the expanded call has a faster kernel:+ # before torch 2.14 the memory-efficient kernel rejects GQA, so masked fp32 GQA had only math.+ mask = torch.ones(Q.shape[0], 1, Q.shape[2], K.shape[2], dtype = torch.bool, device = Q.device) if masked else None+ math = int(SDPBackend.MATH)++ def choice(k, gqa):+ try:+ return int(torch._fused_sdp_choice(Q, k, k, attn_mask = mask, is_causal = not masked, enable_gqa = gqa))+ except RuntimeError: # no enabled backend takes this call+ return None++ gqa = choice(K, True)+ mha = choice(K.repeat_interleave(Q.shape[1] // K.shape[1], dim = 1), False)+ return gqa is None or (gqa == math and mha not in (math, None))++@needs_cudadef test_fp32_masked_gqa_expands_instead_of_math(monkeypatch):- # fp32 (DoRA) with a padding mask: flash cannot run, the memory-efficient kernel rejects GQA.+ # fp32 (DoRA) with a padding mask: flash cannot run; expand K/V only where GQA would leave math.out, seen, (Q, K, V, mask) = _dense(monkeypatch, torch.float32, True)- assert seen == [(8, False)]+ assert seen == ([(8, False)] if _gqa_falls_to_math(Q, K, True) else [(2, True)])with torch.nn.attention.sdpa_kernel([SDPBackend.MATH]):ref = torch.nn.functional.scaled_dot_product_attention(Q,@@ -200,7 +217,10 @@ def test_windowed_packed_runs_expand_where_gqa_would_force_math(monkeypatch):Q, K, K, lengths, n_groups = 2, is_causal = True, sliding_window = 4, sdpa_kwargs = {})assert out is not None- assert all(not gqa and heads == 4 for heads, gqa, _ in seen if _), seen+ if _gqa_falls_to_math(Q, K, True):+ assert all(not gqa and heads == 4 for heads, gqa, _ in seen if _), seen+ else:+ assert all(gqa and heads == 2 for heads, gqa, _ in seen if _), seendef test_gqa_probe_failure_still_learns_the_expanded_choice(monkeypatch):