Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Attention routing tests: follow torch's own GQA kernel choice (fixes two failures on torch 2.14) (#14672)

Daniel Han0c001fcebe14
diff --git a/tests/utils/test_attention_native_routing.py b/tests/utils/test_attention_native_routing.py
index 028c1d0a6a6..711850290da 100644
--- a/tests/utils/test_attention_native_routing.py
+++ b/tests/utils/test_attention_native_routing.py
@@ -99,11 +99,28 @@ def _dense(monkeypatch, dtype, mask):
return out, seen, (Q, K, V, attention_mask)
+def _gqa_falls_to_math(Q, K, masked):
+ # Expand K/V only where GQA leaves math (or nothing) while the expanded call has a faster kernel:
+ # before torch 2.14 the memory-efficient kernel rejects GQA, so masked fp32 GQA had only math.
+ mask = torch.ones(Q.shape[0], 1, Q.shape[2], K.shape[2], dtype = torch.bool, device = Q.device) if masked else None
+ math = int(SDPBackend.MATH)
+
+ def choice(k, gqa):
+ try:
+ return int(torch._fused_sdp_choice(Q, k, k, attn_mask = mask, is_causal = not masked, enable_gqa = gqa))
+ except RuntimeError: # no enabled backend takes this call
+ return None
+
+ gqa = choice(K, True)
+ mha = choice(K.repeat_interleave(Q.shape[1] // K.shape[1], dim = 1), False)
+ return gqa is None or (gqa == math and mha not in (math, None))
+
+
@needs_cuda
def test_fp32_masked_gqa_expands_instead_of_math(monkeypatch):
- # fp32 (DoRA) with a padding mask: flash cannot run, the memory-efficient kernel rejects GQA.
+ # fp32 (DoRA) with a padding mask: flash cannot run; expand K/V only where GQA would leave math.
out, seen, (Q, K, V, mask) = _dense(monkeypatch, torch.float32, True)
- assert seen == [(8, False)]
+ assert seen == ([(8, False)] if _gqa_falls_to_math(Q, K, True) else [(2, True)])
with torch.nn.attention.sdpa_kernel([SDPBackend.MATH]):
ref = torch.nn.functional.scaled_dot_product_attention(
Q,
@@ -200,7 +217,10 @@ def test_windowed_packed_runs_expand_where_gqa_would_force_math(monkeypatch):
Q, K, K, lengths, n_groups = 2, is_causal = True, sliding_window = 4, sdpa_kwargs = {}
)
assert out is not None
- assert all(not gqa and heads == 4 for heads, gqa, _ in seen if _), seen
+ if _gqa_falls_to_math(Q, K, True):
+ assert all(not gqa and heads == 4 for heads, gqa, _ in seen if _), seen
+ else:
+ assert all(gqa and heads == 2 for heads, gqa, _ in seen if _), seen
def test_gqa_probe_failure_still_learns_the_expanded_choice(monkeypatch):