Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Attention: GQA never forces SDPA math, cuDNN-over-varlen on sm100 only, native before xformers (#14158)

Daniel Hane6c95c292b3a
diff --git a/tests/utils/test_attention_native_routing.py b/tests/utils/test_attention_native_routing.py
new file mode 100644
index 00000000000..028c1d0a6a6
--- /dev/null
+++ b/tests/utils/test_attention_native_routing.py
@@ -0,0 +1,471 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+"""PyTorch-native attention routing: GQA never forces SDPA's math kernel, cuDNN-over-varlen only
+where it was measured, and native is preferred over xformers where it is at least as fast."""
+
+import pytest
+import torch
+from torch.nn.attention import SDPBackend
+from real_accelerator import has_real_cuda # tests/_shared, on sys.path via tests/conftest.py
+
+import unsloth # noqa: F401
+from unsloth.utils import attention_dispatch as ad
+from unsloth.utils import packing
+
+needs_cuda = pytest.mark.skipif(not has_real_cuda(), reason = "needs a real CUDA device")
+
+MATH = int(SDPBackend.MATH)
+FLASH = int(SDPBackend.FLASH_ATTENTION)
+EFFICIENT = int(SDPBackend.EFFICIENT_ATTENTION)
+CUDNN = int(SDPBackend.CUDNN_ATTENTION)
+
+
+def fake_choice(monkeypatch):
+ """Make torch._fused_sdp_choice answer from a table keyed on (gqa, mask)."""
+ table = {}
+
+ def choice(
+ q,
+ k,
+ v,
+ attn_mask = None,
+ is_causal = False,
+ enable_gqa = False,
+ **_,
+ ):
+ return table[(bool(enable_gqa), attn_mask is not None)]
+
+ monkeypatch.setattr(torch, "_fused_sdp_choice", choice)
+ ad._sdpa_gqa_choices.cache_clear()
+ yield table
+ ad._sdpa_gqa_choices.cache_clear()
+
+
+ "gqa_choice, mha_choice, expect",
+ [
+ (FLASH, FLASH, True), # flash takes GQA: no K/V copy
+ (CUDNN, CUDNN, True), # cuDNN takes GQA
+ (MATH, EFFICIENT, False), # GQA would leave only math: expand K/V instead
+ (MATH, MATH, True), # math either way: let math broadcast
+ (MATH, -1, True), # unknown: unchanged behaviour
+ ],
+)
+def test_gqa_only_where_it_does_not_force_math(fake_choice, gqa_choice, mha_choice, expect):
+ fake_choice[(True, True)] = gqa_choice
+ fake_choice[(False, True)] = mha_choice
+ assert ad._sdpa_gqa_state("cpu", torch.float32, 64, 8, 2, False, True) is expect
+
+
+def test_cpu_tensors_keep_enable_gqa():
+ q = torch.zeros(1, 8, 4, 64)
+ k = torch.zeros(1, 2, 4, 64)
+ assert ad._sdpa_takes_gqa(q, k, True, False) is ad.SDPA_HAS_GQA
+
+
+def _dense(monkeypatch, dtype, mask):
+ seen = []
+ real = ad.scaled_dot_product_attention
+
+ def spy(Q, K, V, **kwargs):
+ seen.append((K.shape[1], kwargs.get("enable_gqa", False)))
+ return real(Q, K, V, **kwargs)
+
+ monkeypatch.setattr(ad, "scaled_dot_product_attention", spy)
+ bsz, n_heads, n_kv, T, D = 2, 8, 2, 32, 64
+ g = torch.Generator(device = "cuda").manual_seed(0)
+ Q = torch.randn(bsz, n_heads, T, D, device = "cuda", generator = g).to(dtype)
+ K = torch.randn(bsz, n_kv, T, D, device = "cuda", generator = g).to(dtype)
+ V = torch.randn(bsz, n_kv, T, D, device = "cuda", generator = g).to(dtype)
+ attention_mask = None
+ if mask:
+ attention_mask = torch.ones(bsz, T, dtype = torch.long, device = "cuda")
+ attention_mask[1, :5] = 0
+ config = ad.AttentionConfig(backend = ad.SDPA, n_kv_heads = n_kv, n_groups = n_heads // n_kv)
+ context = ad.AttentionContext(
+ bsz = bsz,
+ q_len = T,
+ kv_seq_len = T,
+ n_heads = n_heads,
+ head_dim = D,
+ requires_grad = True,
+ seq_info = None,
+ attention_mask = attention_mask,
+ causal_mask = None,
+ is_causal = True,
+ sliding_window = None,
+ )
+ out = ad.run_attention(config = config, context = context, Q = Q, K = K, V = V)
+ return out, seen, (Q, K, V, attention_mask)
+
+
+@needs_cuda
+def test_fp32_masked_gqa_expands_instead_of_math(monkeypatch):
+ # fp32 (DoRA) with a padding mask: flash cannot run, the memory-efficient kernel rejects GQA.
+ out, seen, (Q, K, V, mask) = _dense(monkeypatch, torch.float32, True)
+ assert seen == [(8, False)]
+ with torch.nn.attention.sdpa_kernel([SDPBackend.MATH]):
+ ref = torch.nn.functional.scaled_dot_product_attention(
+ Q,
+ K.repeat_interleave(4, dim = 1),
+ V.repeat_interleave(4, dim = 1),
+ attn_mask = None,
+ is_causal = True,
+ )
+ # Row 0 has no padding: identical to plain causal attention.
+ torch.testing.assert_close(out[0], ref.transpose(1, 2)[0], atol = 1e-4, rtol = 1e-4)
+
+
+@needs_cuda
+ has_real_cuda() and torch.cuda.get_device_capability()[0] < 8, reason = "flash needs sm80+"
+)
+def test_bf16_causal_gqa_keeps_flash_gqa(monkeypatch):
+ _, seen, _ = _dense(monkeypatch, torch.bfloat16, False)
+ assert seen == [(2, True)]
+
+
+ "capability, expect_varlen",
+ [((10, 0), False), ((10, 3), False), ((9, 0), True), ((12, 0), True), ((8, 0), True)],
+)
+def test_cudnn_over_varlen_only_on_the_measured_arch(monkeypatch, capability, expect_varlen):
+ # Mean segment length 1767: cuDNN per segment won on B200 but is 2.2x+ slower on H100.
+ monkeypatch.setattr(ad, "_sdpa_backend_state", lambda *a: (True, CUDNN))
+ monkeypatch.setattr(torch.cuda, "get_device_capability", lambda *a: capability)
+ lengths = (1500, 2000, 1800)
+ Q = torch.zeros(1, 4, sum(lengths), 64, dtype = torch.bfloat16)
+ assert ad._torch_varlen_beats_segments(Q, Q, lengths, 1, True) is expect_varlen
+ # Short documents still go to varlen everywhere.
+ short = (300, 700, 24)
+ Q = torch.zeros(1, 4, sum(short), 64, dtype = torch.bfloat16)
+ assert ad._torch_varlen_beats_segments(Q, Q, short, 1, True) is True
+
+
+ "capabilities, varlen, hip, expect",
+ [
+ (((8, 0),), True, None, True),
+ (((9, 0), (9, 0)), True, None, True),
+ (((12, 0),), True, None, True),
+ (((7, 5),), True, None, False), # T4: xformers' block-diagonal kernel is faster
+ (((8, 6), (7, 5)), True, None, False), # mixed host keeps xformers for the old card
+ (((8, 0),), False, None, False), # torch 2.10 / 2.11: no varlen kernel
+ (((9, 0),), True, "7.0", False), # ROCm unchanged
+ ((), True, None, False),
+ ],
+)
+def test_native_beats_xformers(monkeypatch, capabilities, varlen, hip, expect):
+ monkeypatch.setattr(torch.version, "hip", hip)
+ assert ad._native_beats_xformers(capabilities, varlen) is expect
+
+
[email protected]("use_varlen", [False, True])
+def test_select_prefers_native_over_xformers(monkeypatch, use_varlen):
+ monkeypatch.setattr(ad, "_ring_attention_active", lambda: False)
+ monkeypatch.setattr(ad, "HAS_FLASH_ATTENTION", False)
+ monkeypatch.setattr(ad, "HAS_XFORMERS", True)
+ monkeypatch.setattr(ad, "_NATIVE_BEATS_XFORMERS", True)
+ monkeypatch.setattr(ad, "_SDPA_PACKED_SEGMENTS", True)
+ # Packed route prefers torch's kernels; dense calls keep xformers (block-diagonal causal_mask,
+ # no KV-cache expansion on fp32 decode).
+ assert ad.select_attention_backend(use_varlen) == (ad.SDPA if use_varlen else ad.XFORMERS)
+ monkeypatch.setattr(ad, "_NATIVE_BEATS_XFORMERS", False)
+ assert ad.select_attention_backend(use_varlen) == ad.XFORMERS
+ # FlashAttention installed: unchanged.
+ monkeypatch.setattr(ad, "HAS_FLASH_ATTENTION", True)
+ monkeypatch.setattr(ad, "_NATIVE_BEATS_XFORMERS", True)
+ assert ad.select_attention_backend(use_varlen) == (
+ ad.FLASH_VARLEN if use_varlen else ad.FLASH_DENSE
+ )
+
+
+@needs_cuda
+def test_windowed_packed_runs_expand_where_gqa_would_force_math(monkeypatch):
+ # fp32 packed segments with a sliding window: the windowed run carries a mask.
+ seen = []
+ real = ad.scaled_dot_product_attention
+
+ def spy(Q, K, V, **kwargs):
+ seen.append((K.shape[1], kwargs.get("enable_gqa", False), "attn_mask" in kwargs))
+ return real(Q, K, V, **kwargs)
+
+ monkeypatch.setattr(ad, "scaled_dot_product_attention", spy)
+ packing.clear_packed_caches()
+ lengths = (12, 3)
+ T = sum(lengths)
+ Q = torch.randn(1, 4, T, 64, device = "cuda")
+ K = torch.randn(1, 2, T, 64, device = "cuda")
+ out = ad._sdpa_packed_segments(
+ Q, K, K, lengths, n_groups = 2, is_causal = True, sliding_window = 4, sdpa_kwargs = {}
+ )
+ assert out is not None
+ assert all(not gqa and heads == 4 for heads, gqa, _ in seen if _), seen
+
+
+def test_gqa_probe_failure_still_learns_the_expanded_choice(monkeypatch):
+ """No enabled backend takes GQA (torch raises): expand K/V when the expanded call has one."""
+
+ def choice(
+ q,
+ k,
+ v,
+ attn_mask = None,
+ is_causal = False,
+ enable_gqa = False,
+ **_,
+ ):
+ if enable_gqa:
+ raise RuntimeError("No available kernel")
+ return EFFICIENT
+
+ monkeypatch.setattr(torch, "_fused_sdp_choice", choice)
+ ad._sdpa_gqa_choices.cache_clear()
+ try:
+ assert ad._sdpa_gqa_state("cpu", torch.float32, 64, 8, 2, False, True) is False
+ finally:
+ ad._sdpa_gqa_choices.cache_clear()
+
+
+def test_both_probes_failing_keep_enable_gqa(monkeypatch):
+ def choice(*args, **kwargs):
+ raise RuntimeError("No available kernel")
+
+ monkeypatch.setattr(torch, "_fused_sdp_choice", choice)
+ ad._sdpa_gqa_choices.cache_clear()
+ try:
+ assert ad._sdpa_gqa_state("cpu", torch.float32, 64, 8, 2, False, True) is True
+ finally:
+ ad._sdpa_gqa_choices.cache_clear()
+
+
+def test_packed_segments_opt_out_keeps_xformers_for_packed_calls(monkeypatch):
+ monkeypatch.setattr(ad, "_ring_attention_active", lambda: False)
+ monkeypatch.setattr(ad, "HAS_FLASH_ATTENTION", False)
+ monkeypatch.setattr(ad, "HAS_XFORMERS", True)
+ monkeypatch.setattr(ad, "_NATIVE_BEATS_XFORMERS", True)
+ monkeypatch.setattr(ad, "_SDPA_PACKED_SEGMENTS", False)
+ # Packed SDPA would build the dense (T, T) mask under the opt-out.
+ assert ad.select_attention_backend(use_varlen = True) == ad.XFORMERS
+ assert ad.select_attention_backend(use_varlen = False) == ad.XFORMERS
+
+
+def test_sentence_unpadding_keeps_xformers_when_training_prefers_native(monkeypatch):
+ from unsloth.models import _sentence_transformer_unpadding as unpad
+
+ # Read at call time from unsloth.utils.packing; xformers itself is not needed for the routing.
+ monkeypatch.setattr(packing, "_XFormersBidirectionalMask", object())
+ monkeypatch.setattr(ad, "_ring_attention_active", lambda: False)
+ monkeypatch.setattr(ad, "HAS_FLASH_ATTENTION", False)
+ monkeypatch.setattr(ad, "HAS_XFORMERS", True)
+ monkeypatch.setattr(ad, "_NATIVE_BEATS_XFORMERS", True)
+ monkeypatch.setattr(ad, "_SDPA_PACKED_SEGMENTS", True)
+ assert ad.select_attention_backend(use_varlen = True) == ad.SDPA
+ assert unpad._unpadding_backend() == ad.XFORMERS
+ # No xformers either: unpadding stays off rather than padding silently on an unsupported path.
+ monkeypatch.setattr(ad, "HAS_XFORMERS", False)
+ assert unpad._unpadding_backend() is None
+
+
+ "needs_grad, dropout_p, expect",
+ [
+ (False, 0.0, True), # inference: flash takes GQA
+ (True, 0.0, False), # training: only math takes GQA, mem-efficient takes expanded K/V
+ (False, 0.1, False), # dropout: same
+ ],
+)
+def test_gqa_probe_carries_grad_and_dropout(monkeypatch, needs_grad, dropout_p, expect):
+ seen = []
+
+ def choice(
+ q,
+ k,
+ v,
+ attn_mask = None,
+ dropout_p = 0.0,
+ is_causal = False,
+ enable_gqa = False,
+ **_,
+ ):
+ training = q.requires_grad and torch.is_grad_enabled()
+ seen.append((training, dropout_p))
+ if training or dropout_p:
+ return MATH if enable_gqa else EFFICIENT
+ return FLASH
+
+ monkeypatch.setattr(torch, "_fused_sdp_choice", choice)
+ ad._sdpa_gqa_choices.cache_clear()
+ try:
+ assert (
+ ad._sdpa_gqa_state("cpu", torch.float32, 64, 8, 2, True, False, needs_grad, dropout_p)
+ is expect
+ )
+ assert seen and all(entry == (needs_grad, dropout_p) for entry in seen)
+ finally:
+ ad._sdpa_gqa_choices.cache_clear()
+
+
+def test_takes_gqa_reads_requires_grad_and_dropout(monkeypatch):
+ calls = []
+ monkeypatch.setattr(ad, "_sdpa_gqa_state", lambda *args: calls.append(args[-4:-2]) or True)
+ Q = torch.empty(1, 8, 4, 64)
+ K = torch.empty(1, 2, 4, 64)
+ monkeypatch.setattr(torch.Tensor, "is_cuda", property(lambda self: True))
+ ad._sdpa_takes_gqa(Q, K, True, False)
+ ad._sdpa_takes_gqa(Q.requires_grad_(), K, True, False, None, 0.25)
+ with torch.no_grad():
+ ad._sdpa_takes_gqa(Q, K, True, False)
+ assert calls == [(False, 0.0), (True, 0.25), (False, 0.0)]
+
+
+def test_probe_matches_single_query_decode(monkeypatch):
+ """A decode call (q_len 1) is probed with q_len 1: cuDNN rejects single-query dropout."""
+ from torch.nn.attention import SDPBackend
+
+ MATH, EFFICIENT, CUDNN = (
+ int(SDPBackend.MATH),
+ int(SDPBackend.EFFICIENT_ATTENTION),
+ int(SDPBackend.CUDNN_ATTENTION),
+ )
+ lengths = []
+
+ def choice(
+ q,
+ k,
+ v,
+ attn_mask = None,
+ dropout_p = 0.0,
+ is_causal = False,
+ enable_gqa = False,
+ **kwargs,
+ ):
+ lengths.append(q.shape[-2])
+ if q.shape[-2] == 1 and dropout_p:
+ return MATH if enable_gqa else EFFICIENT
+ return CUDNN if enable_gqa else EFFICIENT
+
+ monkeypatch.setattr(torch, "_fused_sdp_choice", choice)
+ ad._sdpa_gqa_choices.cache_clear()
+ try:
+ state = ad._sdpa_gqa_state
+ assert state("cpu", torch.float32, 64, 8, 2, False, False, False, 0.1, False) is True
+ assert state("cpu", torch.float32, 64, 8, 2, False, False, False, 0.1, True) is False
+ assert set(lengths) == {128, 1}
+ finally:
+ ad._sdpa_gqa_choices.cache_clear()
+
+
+def test_probe_cache_follows_deterministic_mode(monkeypatch):
+ """cuDNN SDPA is ineligible under strict deterministic mode, so a cached answer must not carry over."""
+ from torch.nn.attention import SDPBackend
+
+ MATH, EFFICIENT, CUDNN = (
+ int(SDPBackend.MATH),
+ int(SDPBackend.EFFICIENT_ATTENTION),
+ int(SDPBackend.CUDNN_ATTENTION),
+ )
+
+ def choice(
+ q,
+ k,
+ v,
+ attn_mask = None,
+ dropout_p = 0.0,
+ is_causal = False,
+ enable_gqa = False,
+ **kwargs,
+ ):
+ if torch.are_deterministic_algorithms_enabled():
+ return MATH if enable_gqa else EFFICIENT
+ return CUDNN if enable_gqa else EFFICIENT
+
+ monkeypatch.setattr(torch, "_fused_sdp_choice", choice)
+ ad._sdpa_gqa_choices.cache_clear()
+ was = torch.are_deterministic_algorithms_enabled()
+ try:
+ assert ad._sdpa_gqa_state("cpu", torch.float32, 64, 8, 2, False, False) is True
+ torch.use_deterministic_algorithms(True)
+ assert ad._sdpa_gqa_state("cpu", torch.float32, 64, 8, 2, False, False) is False
+ finally:
+ torch.use_deterministic_algorithms(was)
+ ad._sdpa_gqa_choices.cache_clear()
+
+
+def test_probe_matches_singleton_key_length(monkeypatch):
+ """A one-token prefill (q_len == kv_len == 1) is probed with K/V length 1: cuDNN rejects it."""
+ from torch.nn.attention import SDPBackend
+
+ MATH, EFFICIENT, CUDNN = (
+ int(SDPBackend.MATH),
+ int(SDPBackend.EFFICIENT_ATTENTION),
+ int(SDPBackend.CUDNN_ATTENTION),
+ )
+ kv_lengths = []
+
+ def choice(
+ q,
+ k,
+ v,
+ attn_mask = None,
+ dropout_p = 0.0,
+ is_causal = False,
+ enable_gqa = False,
+ **kwargs,
+ ):
+ kv_lengths.append(k.shape[-2])
+ if k.shape[-2] == 1:
+ return MATH if enable_gqa else EFFICIENT
+ return CUDNN if enable_gqa else EFFICIENT
+
+ monkeypatch.setattr(torch, "_fused_sdp_choice", choice)
+ ad._sdpa_gqa_choices.cache_clear()
+ try:
+ state = ad._sdpa_gqa_state
+ assert state("cpu", torch.float32, 64, 8, 2, False, False, False, 0.0, True, False) is True
+ assert state("cpu", torch.float32, 64, 8, 2, False, False, False, 0.0, True, True) is False
+ assert set(kv_lengths) == {128, 1}
+ finally:
+ ad._sdpa_gqa_choices.cache_clear()
+
+
+def test_packed_dropout_keeps_xformers(monkeypatch):
+ """Torch varlen takes no dropout: a packed call with dropout keeps xformers' single launch."""
+ monkeypatch.setattr(ad, "_ring_attention_active", lambda: False)
+ monkeypatch.setattr(ad, "HAS_FLASH_ATTENTION", False)
+ monkeypatch.setattr(ad, "HAS_XFORMERS", True)
+ monkeypatch.setattr(ad, "_NATIVE_BEATS_XFORMERS", True)
+ monkeypatch.setattr(ad, "_SDPA_PACKED_SEGMENTS", True)
+ assert ad.select_attention_backend(True) == ad.SDPA
+ assert ad.select_attention_backend(True, 0.1) == ad.XFORMERS
+ assert ad.select_attention_backend(True, dropout_p = 0.0) == ad.SDPA
+
+
+def test_packed_runs_are_probed_with_their_own_lengths(monkeypatch):
+ """A one-token run is probed as one token, not with the whole row's length."""
+ seen = []
+
+ def takes_gqa(
+ Q,
+ K,
+ is_causal,
+ has_mask,
+ V = None,
+ dropout_p = 0.0,
+ ):
+ seen.append((Q.shape[-2], K.shape[-2]))
+ return True
+
+ monkeypatch.setattr(ad, "_sdpa_takes_gqa", takes_gqa)
+ monkeypatch.setattr(
+ ad, "scaled_dot_product_attention", lambda Q, K, V, **kw: torch.zeros_like(Q)
+ )
+ packing.clear_packed_caches()
+ lengths = (5, 1)
+ T = sum(lengths)
+ Q = torch.randn(1, 4, T, 16)
+ K = torch.randn(1, 2, T, 16)
+ ad._sdpa_packed_segments(
+ Q, K, K, lengths, n_groups = 2, is_causal = True, sliding_window = None, sdpa_kwargs = {}
+ )
+ assert (1, 1) in seen and all(q == k for q, k in seen)
diff --git a/tests/utils/test_sdpa_packed_segments.py b/tests/utils/test_sdpa_packed_segments.py
index a2ed75443b3..3f00c6a84a5 100644
--- a/tests/utils/test_sdpa_packed_segments.py
+++ b/tests/utils/test_sdpa_packed_segments.py
@@ -110,6 +110,8 @@ def test_peak_memory_is_linear_in_tokens(monkeypatch):
_run(monkeypatch, segments, (2048, 2048), total, n_heads, n_kv, qkv)
torch.cuda.synchronize()
peaks[segments] = (torch.cuda.max_memory_allocated() - base) / 2**20
- # The dense path holds (16, 4096, 4096) fp32 scores (1 GiB).
+ # Neither path holds (16, 4096, 4096) fp32 scores (1 GiB): the segments never build the dense
+ # mask, and the dense-mask path expands K/V rather than passing enable_gqa, which would leave
+ # SDPA only its math kernel next to a mask flash cannot take.
assert peaks[True] < 256, peaks
- assert peaks[False] > 4 * peaks[True], peaks
+ assert peaks[False] < 512, peaks
diff --git a/unsloth/models/_sentence_transformer_unpadding.py b/unsloth/models/_sentence_transformer_unpadding.py
index d41f3340c06..3d8f91cc0d3 100644
--- a/unsloth/models/_sentence_transformer_unpadding.py
+++ b/unsloth/models/_sentence_transformer_unpadding.py
@@ -24,8 +24,11 @@ def _unpadding_backend():
backend = ad.select_attention_backend(use_varlen = True)
if backend == ad.FLASH_VARLEN:
return backend
- if backend == ad.XFORMERS and _XFormersBidirectionalMask is not None:
- return backend
+ # Training may prefer torch's kernels over xformers; this route only runs on xformers or FA2.
+ if (
+ backend == ad.XFORMERS or (backend == ad.SDPA and ad.HAS_XFORMERS)
+ ) and _XFormersBidirectionalMask is not None:
+ return ad.XFORMERS
return None
diff --git a/unsloth/models/granite.py b/unsloth/models/granite.py
index 811e2ee3cf4..13b2891f594 100644
--- a/unsloth/models/granite.py
+++ b/unsloth/models/granite.py
@@ -140,7 +140,9 @@ def GraniteAttention_fast_forward(
use_varlen = attention_mask is None and seq_info is not None and not had_cache
- backend = SDPA if attention_mask is not None else select_attention_backend(use_varlen)
+ backend = (
+ SDPA if attention_mask is not None else select_attention_backend(use_varlen, dropout_p)
+ )
window = (kv_seq_len, kv_seq_len)
softmax_scale = getattr(self, "scaling", None)
diff --git a/unsloth/utils/attention_dispatch.py b/unsloth/utils/attention_dispatch.py
index 25670f919f7..52c808d56a7 100644
--- a/unsloth/utils/attention_dispatch.py
+++ b/unsloth/utils/attention_dispatch.py
@@ -233,7 +233,10 @@ def _torch_varlen_takes(
# Above this mean segment length cuDNN's per-segment SDPA beats torch's varlen flash kernel (B200 sweep).
+# Only sm_100-family: on H100 the same cuDNN choice is 2.2x to 2.6x slower than varlen at mean lengths
+# 1024 to 2048, so there varlen always wins.
_VARLEN_CUDNN_MAX_MEAN_LEN = 1024
+_VARLEN_CUDNN_ARCH_MAJORS = frozenset({10})
@functools.lru_cache(maxsize = 64)
@@ -266,6 +269,150 @@ if hasattr(torch.compiler, "assume_constant_result"):
_sdpa_backend_state = torch.compiler.assume_constant_result(_sdpa_backend_state)
[email protected]_cache(maxsize = 128)
+def _sdpa_gqa_choices(
+ device,
+ dtype,
+ head_dim,
+ n_heads,
+ n_kv_heads,
+ is_causal,
+ has_mask,
+ needs_grad,
+ dropout_p,
+ backends,
+ single_query = False,
+ single_key = False,
+):
+ """(SDPBackend with enable_gqa, SDPBackend with K/V expanded) for these shapes, -1 if unknown.
+ The probe tensors carry the real call's requires_grad, dropout and single-query (decode) shape:
+ backends differ on all three (cuDNN rejects single-query dropout, for one)."""
+ # Lengths of 1 change eligibility (cuDNN rejects a K/V length of 1); longer ones do not.
+ q_len = 1 if single_query else 128
+ kv_len = 1 if single_key else 128
+ try:
+ q = torch.empty(1, n_heads, q_len, head_dim, device = device, dtype = dtype)
+ k = torch.empty(1, n_kv_heads, kv_len, head_dim, device = device, dtype = dtype)
+ k_full = torch.empty(1, n_heads, kv_len, head_dim, device = device, dtype = dtype)
+ if needs_grad:
+ q, k, k_full = (t.requires_grad_() for t in (q, k, k_full))
+ mask = (
+ torch.ones(1, 1, q_len, kv_len, device = device, dtype = torch.bool) if has_mask else None
+ )
+ except Exception:
+ return -1, -1
+ causal = is_causal and not has_mask
+
+ def probe(key, **kwargs):
+ # Each on its own: torch raises when no enabled backend fits, and a GQA call nothing
+ # takes must still learn whether the expanded call would run.
+ try:
+ with torch.set_grad_enabled(needs_grad):
+ return int(
+ torch._fused_sdp_choice(
+ q, key, key, attn_mask = mask, dropout_p = dropout_p, is_causal = causal, **kwargs
+ )
+ )
+ except Exception:
+ return None
+
+ gqa = probe(k, enable_gqa = True)
+ mha = probe(k_full)
+ if gqa is None:
+ # Nothing takes GQA here: expand when the expanded call has a backend, else unknown.
+ return (_SDPA_NO_BACKEND, mha) if mha is not None else (-1, -1)
+ return gqa, -1 if mha is None else mha
+
+
+# _sdpa_gqa_choices' answer when no enabled SDPA backend accepts the GQA call at all.
+_SDPA_NO_BACKEND = -2
+
+
+def _sdpa_gqa_state(
+ device,
+ dtype,
+ head_dim,
+ n_heads,
+ n_kv_heads,
+ is_causal,
+ has_mask,
+ needs_grad = False,
+ dropout_p = 0.0,
+ single_query = False,
+ single_key = False,
+) -> bool:
+ """False when enable_gqa would leave SDPA only its math kernel while expanded K/V would not.
+ Flash and cuDNN take GQA, the memory-efficient kernel does not (before torch 2.14), so fp32,
+ pre-sm80 GPUs and masked calls flash cannot run would otherwise fall to math: several times
+ slower and a (T, T) score matrix per head. Asked of torch, so ROCm's backend set is covered too."""
+ cuda = torch.backends.cuda
+ backends = (
+ cuda.flash_sdp_enabled(),
+ cuda.cudnn_sdp_enabled(),
+ cuda.mem_efficient_sdp_enabled(),
+ cuda.math_sdp_enabled(),
+ tuple(getattr(torch._C, "_get_sdp_priority_order", tuple)()),
+ # Strict deterministic mode makes cuDNN SDPA ineligible.
+ torch.are_deterministic_algorithms_enabled(),
+ torch.is_deterministic_algorithms_warn_only_enabled(),
+ )
+ gqa, mha = _sdpa_gqa_choices(
+ device,
+ dtype,
+ head_dim,
+ n_heads,
+ n_kv_heads,
+ is_causal,
+ has_mask,
+ needs_grad,
+ dropout_p,
+ backends,
+ single_query,
+ single_key,
+ )
+ from torch.nn.attention import SDPBackend
+
+ math = int(SDPBackend.MATH)
+ if gqa == _SDPA_NO_BACKEND:
+ return False
+ return not (gqa == math and mha not in (math, -1))
+
+
+if hasattr(torch.compiler, "assume_constant_result"):
+ _sdpa_gqa_state = torch.compiler.assume_constant_result(_sdpa_gqa_state)
+
+
+def _sdpa_takes_gqa(
+ Q: Tensor,
+ K: Tensor,
+ is_causal: bool,
+ has_mask: bool,
+ V: Optional[Tensor] = None,
+ dropout_p = 0.0,
+) -> bool:
+ """Whether to pass enable_gqa (True) or expand K/V (False) for this SDPA call."""
+ if not SDPA_HAS_GQA:
+ return False
+ if not Q.is_cuda:
+ return True
+ needs_grad = torch.is_grad_enabled() and any(
+ t is not None and t.requires_grad for t in (Q, K, V)
+ )
+ return _sdpa_gqa_state(
+ str(Q.device),
+ Q.dtype,
+ Q.shape[-1],
+ Q.shape[1],
+ K.shape[1],
+ bool(is_causal),
+ bool(has_mask),
+ bool(needs_grad),
+ float(dropout_p or 0.0),
+ Q.shape[-2] == 1,
+ K.shape[-2] == 1,
+ )
+
+
def _torch_varlen_beats_segments(
Q: Tensor, K: Tensor, lengths: Tuple[int, ...], n_groups: int, is_causal: bool
) -> bool:
@@ -295,6 +442,8 @@ def _torch_varlen_beats_segments(
if choice == int(SDPBackend.FLASH_ATTENTION):
return True
if choice == int(SDPBackend.CUDNN_ATTENTION):
+ if torch.cuda.get_device_capability(Q.device)[0] not in _VARLEN_CUDNN_ARCH_MAJORS:
+ return True
return sum(lengths) / len(lengths) <= _VARLEN_CUDNN_MAX_MEAN_LEN
return False
@@ -365,10 +514,19 @@ def _sdpa_packed_segments(
None when the lengths do not tile the row."""
if sum(length for length in lengths if length > 0) != Q.shape[-2]:
return None
- gqa = n_groups != 1 and _sdpa_flash_takes_gqa(Q)
- if n_groups != 1 and not gqa:
- K = K.repeat_interleave(n_groups, dim = 1)
- V = V.repeat_interleave(n_groups, dim = 1)
+ expanded = None
+
+ def kv(gqa):
+ nonlocal expanded
+ if n_groups == 1 or gqa:
+ return K, V
+ if expanded is None:
+ expanded = (
+ K.repeat_interleave(n_groups, dim = 1),
+ V.repeat_interleave(n_groups, dim = 1),
+ )
+ return expanded
+
outs, start, i = [], 0, 0
lengths = [length for length in lengths if length > 0]
while i < len(lengths):
@@ -393,9 +551,15 @@ def _sdpa_packed_segments(
is_causal = is_causal,
)
kwargs["is_causal"] = is_causal and not windowed
+ Q_run = split(Q)
+ # Probe the run's own tensors: a one-token run has different eligibility than the row.
+ gqa = n_groups != 1 and _sdpa_takes_gqa(
+ Q_run, split(K), kwargs["is_causal"], windowed, split(V), kwargs.get("dropout_p", 0.0)
+ )
if gqa:
kwargs["enable_gqa"] = True
- out = scaled_dot_product_attention(split(Q), split(K), split(V), **kwargs)
+ K_run, V_run = kv(gqa)
+ out = scaled_dot_product_attention(Q_run, split(K_run), split(V_run), **kwargs)
outs.append(out.transpose(1, 2).flatten(0, 1))
start, i = end, j
return torch.cat(outs, dim = 0).unsqueeze(0)
@@ -534,7 +698,23 @@ def _ring_attention_active() -> bool:
return manager is not None and manager.backend == "ring"
-def select_attention_backend(use_varlen: bool = False) -> str:
+def _native_beats_xformers(capabilities, varlen_loaded: bool) -> bool:
+ """Without FlashAttention, torch's own kernels match or beat xformers 0.0.35 (which itself calls
+ torch's flash / memory-efficient ops) once torch has a varlen kernel (2.12+) on sm_80+ CUDA:
+ packed / padding-free 1.00x to 1.06x on A100, L4, RTX PRO 6000 and H100. Below sm_80 there is no
+ torch varlen, and xformers' block-diagonal kernel stays 8% to 21% faster (T4); a mixed host keeps
+ xformers for its older card."""
+ if not varlen_loaded or not capabilities or torch.version.hip is not None:
+ return False
+ return all(capability[0] >= 8 for capability in capabilities)
+
+
+_NATIVE_BEATS_XFORMERS = _native_beats_xformers(
+ _visible_capabilities(), _TORCH_VARLEN_ATTN is not None
+)
+
+
+def select_attention_backend(use_varlen: bool = False, dropout_p: float = 0.0) -> str:
"""Return attention backend based on availability / priority order."""
if _ring_attention_active():
@@ -549,7 +729,14 @@ def select_attention_backend(use_varlen: bool = False) -> str:
return FLASH_VARLEN
else:
return FLASH_DENSE
- if HAS_XFORMERS:
+ # Only the packed route prefers torch's kernels (varlen / per-segment SDPA). Dense calls keep
+ # xformers: it alone consumes a caller's block-diagonal causal_mask, and expanding a GQA KV cache
+ # for SDPA on fp32 decode costs memory. With UNSLOTH_SDPA_PACKED_SEGMENTS off, packed SDPA would
+ # build a dense (T, T) mask, so xformers stays there too. Torch varlen takes no dropout, so a
+ # packed call with dropout would become one SDPA launch per run: xformers keeps those as well.
+ if HAS_XFORMERS and not (
+ use_varlen and not dropout_p and _NATIVE_BEATS_XFORMERS and _SDPA_PACKED_SEGMENTS
+ ):
return XFORMERS
return SDPA
@@ -1005,6 +1192,16 @@ def _run_attention(
# expansion for deterministic row-wise behavior.
use_sdpa_gqa = False
+ if use_sdpa_gqa and not _sdpa_takes_gqa(
+ Q,
+ K,
+ kwargs.get("is_causal", False),
+ kwargs.get("attn_mask") is not None,
+ V,
+ kwargs.get("dropout_p", 0.0),
+ ):
+ use_sdpa_gqa = False
+
if use_sdpa_gqa:
kwargs.setdefault("enable_gqa", True)
out = scaled_dot_product_attention(Q, K, V, **kwargs)