Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Attention: GQA never forces SDPA math, cuDNN-over-varlen on sm100 only, native before xformers (#14158)
Daniel Hane6c95c292b3a
diff --git a/tests/utils/test_attention_native_routing.py b/tests/utils/test_attention_native_routing.pynew file mode 100644index 00000000000..028c1d0a6a6--- /dev/null+++ b/tests/utils/test_attention_native_routing.py@@ -0,0 +1,471 @@+# SPDX-License-Identifier: AGPL-3.0-only+"""PyTorch-native attention routing: GQA never forces SDPA's math kernel, cuDNN-over-varlen only+where it was measured, and native is preferred over xformers where it is at least as fast."""++import pytest+import torch+from torch.nn.attention import SDPBackend+from real_accelerator import has_real_cuda # tests/_shared, on sys.path via tests/conftest.py++import unsloth # noqa: F401+from unsloth.utils import attention_dispatch as ad+from unsloth.utils import packing++needs_cuda = pytest.mark.skipif(not has_real_cuda(), reason = "needs a real CUDA device")++MATH = int(SDPBackend.MATH)+FLASH = int(SDPBackend.FLASH_ATTENTION)+EFFICIENT = int(SDPBackend.EFFICIENT_ATTENTION)+CUDNN = int(SDPBackend.CUDNN_ATTENTION)+++def fake_choice(monkeypatch):+ """Make torch._fused_sdp_choice answer from a table keyed on (gqa, mask)."""+ table = {}++ def choice(+ q,+ k,+ v,+ attn_mask = None,+ is_causal = False,+ enable_gqa = False,+ **_,+ ):+ return table[(bool(enable_gqa), attn_mask is not None)]++ monkeypatch.setattr(torch, "_fused_sdp_choice", choice)+ ad._sdpa_gqa_choices.cache_clear()+ yield table+ ad._sdpa_gqa_choices.cache_clear()+++ "gqa_choice, mha_choice, expect",+ [+ (FLASH, FLASH, True), # flash takes GQA: no K/V copy+ (CUDNN, CUDNN, True), # cuDNN takes GQA+ (MATH, EFFICIENT, False), # GQA would leave only math: expand K/V instead+ (MATH, MATH, True), # math either way: let math broadcast+ (MATH, -1, True), # unknown: unchanged behaviour+ ],+)+def test_gqa_only_where_it_does_not_force_math(fake_choice, gqa_choice, mha_choice, expect):+ fake_choice[(True, True)] = gqa_choice+ fake_choice[(False, True)] = mha_choice+ assert ad._sdpa_gqa_state("cpu", torch.float32, 64, 8, 2, False, True) is expect+++def test_cpu_tensors_keep_enable_gqa():+ q = torch.zeros(1, 8, 4, 64)+ k = torch.zeros(1, 2, 4, 64)+ assert ad._sdpa_takes_gqa(q, k, True, False) is ad.SDPA_HAS_GQA+++def _dense(monkeypatch, dtype, mask):+ seen = []+ real = ad.scaled_dot_product_attention++ def spy(Q, K, V, **kwargs):+ seen.append((K.shape[1], kwargs.get("enable_gqa", False)))+ return real(Q, K, V, **kwargs)++ monkeypatch.setattr(ad, "scaled_dot_product_attention", spy)+ bsz, n_heads, n_kv, T, D = 2, 8, 2, 32, 64+ g = torch.Generator(device = "cuda").manual_seed(0)+ Q = torch.randn(bsz, n_heads, T, D, device = "cuda", generator = g).to(dtype)+ K = torch.randn(bsz, n_kv, T, D, device = "cuda", generator = g).to(dtype)+ V = torch.randn(bsz, n_kv, T, D, device = "cuda", generator = g).to(dtype)+ attention_mask = None+ if mask:+ attention_mask = torch.ones(bsz, T, dtype = torch.long, device = "cuda")+ attention_mask[1, :5] = 0+ config = ad.AttentionConfig(backend = ad.SDPA, n_kv_heads = n_kv, n_groups = n_heads // n_kv)+ context = ad.AttentionContext(+ bsz = bsz,+ q_len = T,+ kv_seq_len = T,+ n_heads = n_heads,+ head_dim = D,+ requires_grad = True,+ seq_info = None,+ attention_mask = attention_mask,+ causal_mask = None,+ is_causal = True,+ sliding_window = None,+ )+ out = ad.run_attention(config = config, context = context, Q = Q, K = K, V = V)+ return out, seen, (Q, K, V, attention_mask)+++@needs_cuda+def test_fp32_masked_gqa_expands_instead_of_math(monkeypatch):+ # fp32 (DoRA) with a padding mask: flash cannot run, the memory-efficient kernel rejects GQA.+ out, seen, (Q, K, V, mask) = _dense(monkeypatch, torch.float32, True)+ assert seen == [(8, False)]+ with torch.nn.attention.sdpa_kernel([SDPBackend.MATH]):+ ref = torch.nn.functional.scaled_dot_product_attention(+ Q,+ K.repeat_interleave(4, dim = 1),+ V.repeat_interleave(4, dim = 1),+ attn_mask = None,+ is_causal = True,+ )+ # Row 0 has no padding: identical to plain causal attention.+ torch.testing.assert_close(out[0], ref.transpose(1, 2)[0], atol = 1e-4, rtol = 1e-4)+++@needs_cuda+ has_real_cuda() and torch.cuda.get_device_capability()[0] < 8, reason = "flash needs sm80+"+)+def test_bf16_causal_gqa_keeps_flash_gqa(monkeypatch):+ _, seen, _ = _dense(monkeypatch, torch.bfloat16, False)+ assert seen == [(2, True)]+++ "capability, expect_varlen",+ [((10, 0), False), ((10, 3), False), ((9, 0), True), ((12, 0), True), ((8, 0), True)],+)+def test_cudnn_over_varlen_only_on_the_measured_arch(monkeypatch, capability, expect_varlen):+ # Mean segment length 1767: cuDNN per segment won on B200 but is 2.2x+ slower on H100.+ monkeypatch.setattr(ad, "_sdpa_backend_state", lambda *a: (True, CUDNN))+ monkeypatch.setattr(torch.cuda, "get_device_capability", lambda *a: capability)+ lengths = (1500, 2000, 1800)+ Q = torch.zeros(1, 4, sum(lengths), 64, dtype = torch.bfloat16)+ assert ad._torch_varlen_beats_segments(Q, Q, lengths, 1, True) is expect_varlen+ # Short documents still go to varlen everywhere.+ short = (300, 700, 24)+ Q = torch.zeros(1, 4, sum(short), 64, dtype = torch.bfloat16)+ assert ad._torch_varlen_beats_segments(Q, Q, short, 1, True) is True+++ "capabilities, varlen, hip, expect",+ [+ (((8, 0),), True, None, True),+ (((9, 0), (9, 0)), True, None, True),+ (((12, 0),), True, None, True),+ (((7, 5),), True, None, False), # T4: xformers' block-diagonal kernel is faster+ (((8, 6), (7, 5)), True, None, False), # mixed host keeps xformers for the old card+ (((8, 0),), False, None, False), # torch 2.10 / 2.11: no varlen kernel+ (((9, 0),), True, "7.0", False), # ROCm unchanged+ ((), True, None, False),+ ],+)+def test_native_beats_xformers(monkeypatch, capabilities, varlen, hip, expect):+ monkeypatch.setattr(torch.version, "hip", hip)+ assert ad._native_beats_xformers(capabilities, varlen) is expect++[email protected]("use_varlen", [False, True])+def test_select_prefers_native_over_xformers(monkeypatch, use_varlen):+ monkeypatch.setattr(ad, "_ring_attention_active", lambda: False)+ monkeypatch.setattr(ad, "HAS_FLASH_ATTENTION", False)+ monkeypatch.setattr(ad, "HAS_XFORMERS", True)+ monkeypatch.setattr(ad, "_NATIVE_BEATS_XFORMERS", True)+ monkeypatch.setattr(ad, "_SDPA_PACKED_SEGMENTS", True)+ # Packed route prefers torch's kernels; dense calls keep xformers (block-diagonal causal_mask,+ # no KV-cache expansion on fp32 decode).+ assert ad.select_attention_backend(use_varlen) == (ad.SDPA if use_varlen else ad.XFORMERS)+ monkeypatch.setattr(ad, "_NATIVE_BEATS_XFORMERS", False)+ assert ad.select_attention_backend(use_varlen) == ad.XFORMERS+ # FlashAttention installed: unchanged.+ monkeypatch.setattr(ad, "HAS_FLASH_ATTENTION", True)+ monkeypatch.setattr(ad, "_NATIVE_BEATS_XFORMERS", True)+ assert ad.select_attention_backend(use_varlen) == (+ ad.FLASH_VARLEN if use_varlen else ad.FLASH_DENSE+ )+++@needs_cuda+def test_windowed_packed_runs_expand_where_gqa_would_force_math(monkeypatch):+ # fp32 packed segments with a sliding window: the windowed run carries a mask.+ seen = []+ real = ad.scaled_dot_product_attention++ def spy(Q, K, V, **kwargs):+ seen.append((K.shape[1], kwargs.get("enable_gqa", False), "attn_mask" in kwargs))+ return real(Q, K, V, **kwargs)++ monkeypatch.setattr(ad, "scaled_dot_product_attention", spy)+ packing.clear_packed_caches()+ lengths = (12, 3)+ T = sum(lengths)+ Q = torch.randn(1, 4, T, 64, device = "cuda")+ K = torch.randn(1, 2, T, 64, device = "cuda")+ out = ad._sdpa_packed_segments(+ Q, K, K, lengths, n_groups = 2, is_causal = True, sliding_window = 4, sdpa_kwargs = {}+ )+ assert out is not None+ assert all(not gqa and heads == 4 for heads, gqa, _ in seen if _), seen+++def test_gqa_probe_failure_still_learns_the_expanded_choice(monkeypatch):+ """No enabled backend takes GQA (torch raises): expand K/V when the expanded call has one."""++ def choice(+ q,+ k,+ v,+ attn_mask = None,+ is_causal = False,+ enable_gqa = False,+ **_,+ ):+ if enable_gqa:+ raise RuntimeError("No available kernel")+ return EFFICIENT++ monkeypatch.setattr(torch, "_fused_sdp_choice", choice)+ ad._sdpa_gqa_choices.cache_clear()+ try:+ assert ad._sdpa_gqa_state("cpu", torch.float32, 64, 8, 2, False, True) is False+ finally:+ ad._sdpa_gqa_choices.cache_clear()+++def test_both_probes_failing_keep_enable_gqa(monkeypatch):+ def choice(*args, **kwargs):+ raise RuntimeError("No available kernel")++ monkeypatch.setattr(torch, "_fused_sdp_choice", choice)+ ad._sdpa_gqa_choices.cache_clear()+ try:+ assert ad._sdpa_gqa_state("cpu", torch.float32, 64, 8, 2, False, True) is True+ finally:+ ad._sdpa_gqa_choices.cache_clear()+++def test_packed_segments_opt_out_keeps_xformers_for_packed_calls(monkeypatch):+ monkeypatch.setattr(ad, "_ring_attention_active", lambda: False)+ monkeypatch.setattr(ad, "HAS_FLASH_ATTENTION", False)+ monkeypatch.setattr(ad, "HAS_XFORMERS", True)+ monkeypatch.setattr(ad, "_NATIVE_BEATS_XFORMERS", True)+ monkeypatch.setattr(ad, "_SDPA_PACKED_SEGMENTS", False)+ # Packed SDPA would build the dense (T, T) mask under the opt-out.+ assert ad.select_attention_backend(use_varlen = True) == ad.XFORMERS+ assert ad.select_attention_backend(use_varlen = False) == ad.XFORMERS+++def test_sentence_unpadding_keeps_xformers_when_training_prefers_native(monkeypatch):+ from unsloth.models import _sentence_transformer_unpadding as unpad++ # Read at call time from unsloth.utils.packing; xformers itself is not needed for the routing.+ monkeypatch.setattr(packing, "_XFormersBidirectionalMask", object())+ monkeypatch.setattr(ad, "_ring_attention_active", lambda: False)+ monkeypatch.setattr(ad, "HAS_FLASH_ATTENTION", False)+ monkeypatch.setattr(ad, "HAS_XFORMERS", True)+ monkeypatch.setattr(ad, "_NATIVE_BEATS_XFORMERS", True)+ monkeypatch.setattr(ad, "_SDPA_PACKED_SEGMENTS", True)+ assert ad.select_attention_backend(use_varlen = True) == ad.SDPA+ assert unpad._unpadding_backend() == ad.XFORMERS+ # No xformers either: unpadding stays off rather than padding silently on an unsupported path.+ monkeypatch.setattr(ad, "HAS_XFORMERS", False)+ assert unpad._unpadding_backend() is None+++ "needs_grad, dropout_p, expect",+ [+ (False, 0.0, True), # inference: flash takes GQA+ (True, 0.0, False), # training: only math takes GQA, mem-efficient takes expanded K/V+ (False, 0.1, False), # dropout: same+ ],+)+def test_gqa_probe_carries_grad_and_dropout(monkeypatch, needs_grad, dropout_p, expect):+ seen = []++ def choice(+ q,+ k,+ v,+ attn_mask = None,+ dropout_p = 0.0,+ is_causal = False,+ enable_gqa = False,+ **_,+ ):+ training = q.requires_grad and torch.is_grad_enabled()+ seen.append((training, dropout_p))+ if training or dropout_p:+ return MATH if enable_gqa else EFFICIENT+ return FLASH++ monkeypatch.setattr(torch, "_fused_sdp_choice", choice)+ ad._sdpa_gqa_choices.cache_clear()+ try:+ assert (+ ad._sdpa_gqa_state("cpu", torch.float32, 64, 8, 2, True, False, needs_grad, dropout_p)+ is expect+ )+ assert seen and all(entry == (needs_grad, dropout_p) for entry in seen)+ finally:+ ad._sdpa_gqa_choices.cache_clear()+++def test_takes_gqa_reads_requires_grad_and_dropout(monkeypatch):+ calls = []+ monkeypatch.setattr(ad, "_sdpa_gqa_state", lambda *args: calls.append(args[-4:-2]) or True)+ Q = torch.empty(1, 8, 4, 64)+ K = torch.empty(1, 2, 4, 64)+ monkeypatch.setattr(torch.Tensor, "is_cuda", property(lambda self: True))+ ad._sdpa_takes_gqa(Q, K, True, False)+ ad._sdpa_takes_gqa(Q.requires_grad_(), K, True, False, None, 0.25)+ with torch.no_grad():+ ad._sdpa_takes_gqa(Q, K, True, False)+ assert calls == [(False, 0.0), (True, 0.25), (False, 0.0)]+++def test_probe_matches_single_query_decode(monkeypatch):+ """A decode call (q_len 1) is probed with q_len 1: cuDNN rejects single-query dropout."""+ from torch.nn.attention import SDPBackend++ MATH, EFFICIENT, CUDNN = (+ int(SDPBackend.MATH),+ int(SDPBackend.EFFICIENT_ATTENTION),+ int(SDPBackend.CUDNN_ATTENTION),+ )+ lengths = []++ def choice(+ q,+ k,+ v,+ attn_mask = None,+ dropout_p = 0.0,+ is_causal = False,+ enable_gqa = False,+ **kwargs,+ ):+ lengths.append(q.shape[-2])+ if q.shape[-2] == 1 and dropout_p:+ return MATH if enable_gqa else EFFICIENT+ return CUDNN if enable_gqa else EFFICIENT++ monkeypatch.setattr(torch, "_fused_sdp_choice", choice)+ ad._sdpa_gqa_choices.cache_clear()+ try:+ state = ad._sdpa_gqa_state+ assert state("cpu", torch.float32, 64, 8, 2, False, False, False, 0.1, False) is True+ assert state("cpu", torch.float32, 64, 8, 2, False, False, False, 0.1, True) is False+ assert set(lengths) == {128, 1}+ finally:+ ad._sdpa_gqa_choices.cache_clear()+++def test_probe_cache_follows_deterministic_mode(monkeypatch):+ """cuDNN SDPA is ineligible under strict deterministic mode, so a cached answer must not carry over."""+ from torch.nn.attention import SDPBackend++ MATH, EFFICIENT, CUDNN = (+ int(SDPBackend.MATH),+ int(SDPBackend.EFFICIENT_ATTENTION),+ int(SDPBackend.CUDNN_ATTENTION),+ )++ def choice(+ q,+ k,+ v,+ attn_mask = None,+ dropout_p = 0.0,+ is_causal = False,+ enable_gqa = False,+ **kwargs,+ ):+ if torch.are_deterministic_algorithms_enabled():+ return MATH if enable_gqa else EFFICIENT+ return CUDNN if enable_gqa else EFFICIENT++ monkeypatch.setattr(torch, "_fused_sdp_choice", choice)+ ad._sdpa_gqa_choices.cache_clear()+ was = torch.are_deterministic_algorithms_enabled()+ try:+ assert ad._sdpa_gqa_state("cpu", torch.float32, 64, 8, 2, False, False) is True+ torch.use_deterministic_algorithms(True)+ assert ad._sdpa_gqa_state("cpu", torch.float32, 64, 8, 2, False, False) is False+ finally:+ torch.use_deterministic_algorithms(was)+ ad._sdpa_gqa_choices.cache_clear()+++def test_probe_matches_singleton_key_length(monkeypatch):+ """A one-token prefill (q_len == kv_len == 1) is probed with K/V length 1: cuDNN rejects it."""+ from torch.nn.attention import SDPBackend++ MATH, EFFICIENT, CUDNN = (+ int(SDPBackend.MATH),+ int(SDPBackend.EFFICIENT_ATTENTION),+ int(SDPBackend.CUDNN_ATTENTION),+ )+ kv_lengths = []++ def choice(+ q,+ k,+ v,+ attn_mask = None,+ dropout_p = 0.0,+ is_causal = False,+ enable_gqa = False,+ **kwargs,+ ):+ kv_lengths.append(k.shape[-2])+ if k.shape[-2] == 1:+ return MATH if enable_gqa else EFFICIENT+ return CUDNN if enable_gqa else EFFICIENT++ monkeypatch.setattr(torch, "_fused_sdp_choice", choice)+ ad._sdpa_gqa_choices.cache_clear()+ try:+ state = ad._sdpa_gqa_state+ assert state("cpu", torch.float32, 64, 8, 2, False, False, False, 0.0, True, False) is True+ assert state("cpu", torch.float32, 64, 8, 2, False, False, False, 0.0, True, True) is False+ assert set(kv_lengths) == {128, 1}+ finally:+ ad._sdpa_gqa_choices.cache_clear()+++def test_packed_dropout_keeps_xformers(monkeypatch):+ """Torch varlen takes no dropout: a packed call with dropout keeps xformers' single launch."""+ monkeypatch.setattr(ad, "_ring_attention_active", lambda: False)+ monkeypatch.setattr(ad, "HAS_FLASH_ATTENTION", False)+ monkeypatch.setattr(ad, "HAS_XFORMERS", True)+ monkeypatch.setattr(ad, "_NATIVE_BEATS_XFORMERS", True)+ monkeypatch.setattr(ad, "_SDPA_PACKED_SEGMENTS", True)+ assert ad.select_attention_backend(True) == ad.SDPA+ assert ad.select_attention_backend(True, 0.1) == ad.XFORMERS+ assert ad.select_attention_backend(True, dropout_p = 0.0) == ad.SDPA+++def test_packed_runs_are_probed_with_their_own_lengths(monkeypatch):+ """A one-token run is probed as one token, not with the whole row's length."""+ seen = []++ def takes_gqa(+ Q,+ K,+ is_causal,+ has_mask,+ V = None,+ dropout_p = 0.0,+ ):+ seen.append((Q.shape[-2], K.shape[-2]))+ return True++ monkeypatch.setattr(ad, "_sdpa_takes_gqa", takes_gqa)+ monkeypatch.setattr(+ ad, "scaled_dot_product_attention", lambda Q, K, V, **kw: torch.zeros_like(Q)+ )+ packing.clear_packed_caches()+ lengths = (5, 1)+ T = sum(lengths)+ Q = torch.randn(1, 4, T, 16)+ K = torch.randn(1, 2, T, 16)+ ad._sdpa_packed_segments(+ Q, K, K, lengths, n_groups = 2, is_causal = True, sliding_window = None, sdpa_kwargs = {}+ )+ assert (1, 1) in seen and all(q == k for q, k in seen)diff --git a/tests/utils/test_sdpa_packed_segments.py b/tests/utils/test_sdpa_packed_segments.pyindex a2ed75443b3..3f00c6a84a5 100644--- a/tests/utils/test_sdpa_packed_segments.py+++ b/tests/utils/test_sdpa_packed_segments.py@@ -110,6 +110,8 @@ def test_peak_memory_is_linear_in_tokens(monkeypatch):_run(monkeypatch, segments, (2048, 2048), total, n_heads, n_kv, qkv)torch.cuda.synchronize()peaks[segments] = (torch.cuda.max_memory_allocated() - base) / 2**20- # The dense path holds (16, 4096, 4096) fp32 scores (1 GiB).+ # Neither path holds (16, 4096, 4096) fp32 scores (1 GiB): the segments never build the dense+ # mask, and the dense-mask path expands K/V rather than passing enable_gqa, which would leave+ # SDPA only its math kernel next to a mask flash cannot take.assert peaks[True] < 256, peaks- assert peaks[False] > 4 * peaks[True], peaks+ assert peaks[False] < 512, peaksdiff --git a/unsloth/models/_sentence_transformer_unpadding.py b/unsloth/models/_sentence_transformer_unpadding.pyindex d41f3340c06..3d8f91cc0d3 100644--- a/unsloth/models/_sentence_transformer_unpadding.py+++ b/unsloth/models/_sentence_transformer_unpadding.py@@ -24,8 +24,11 @@ def _unpadding_backend():backend = ad.select_attention_backend(use_varlen = True)if backend == ad.FLASH_VARLEN:return backend- if backend == ad.XFORMERS and _XFormersBidirectionalMask is not None:- return backend+ # Training may prefer torch's kernels over xformers; this route only runs on xformers or FA2.+ if (+ backend == ad.XFORMERS or (backend == ad.SDPA and ad.HAS_XFORMERS)+ ) and _XFormersBidirectionalMask is not None:+ return ad.XFORMERSreturn Nonediff --git a/unsloth/models/granite.py b/unsloth/models/granite.pyindex 811e2ee3cf4..13b2891f594 100644--- a/unsloth/models/granite.py+++ b/unsloth/models/granite.py@@ -140,7 +140,9 @@ def GraniteAttention_fast_forward(use_varlen = attention_mask is None and seq_info is not None and not had_cache- backend = SDPA if attention_mask is not None else select_attention_backend(use_varlen)+ backend = (+ SDPA if attention_mask is not None else select_attention_backend(use_varlen, dropout_p)+ )window = (kv_seq_len, kv_seq_len)softmax_scale = getattr(self, "scaling", None)diff --git a/unsloth/utils/attention_dispatch.py b/unsloth/utils/attention_dispatch.pyindex 25670f919f7..52c808d56a7 100644--- a/unsloth/utils/attention_dispatch.py+++ b/unsloth/utils/attention_dispatch.py@@ -233,7 +233,10 @@ def _torch_varlen_takes(# Above this mean segment length cuDNN's per-segment SDPA beats torch's varlen flash kernel (B200 sweep).+# Only sm_100-family: on H100 the same cuDNN choice is 2.2x to 2.6x slower than varlen at mean lengths+# 1024 to 2048, so there varlen always wins._VARLEN_CUDNN_MAX_MEAN_LEN = 1024+_VARLEN_CUDNN_ARCH_MAJORS = frozenset({10})@functools.lru_cache(maxsize = 64)@@ -266,6 +269,150 @@ if hasattr(torch.compiler, "assume_constant_result"):_sdpa_backend_state = torch.compiler.assume_constant_result(_sdpa_backend_state)[email protected]_cache(maxsize = 128)+def _sdpa_gqa_choices(+ device,+ dtype,+ head_dim,+ n_heads,+ n_kv_heads,+ is_causal,+ has_mask,+ needs_grad,+ dropout_p,+ backends,+ single_query = False,+ single_key = False,+):+ """(SDPBackend with enable_gqa, SDPBackend with K/V expanded) for these shapes, -1 if unknown.+ The probe tensors carry the real call's requires_grad, dropout and single-query (decode) shape:+ backends differ on all three (cuDNN rejects single-query dropout, for one)."""+ # Lengths of 1 change eligibility (cuDNN rejects a K/V length of 1); longer ones do not.+ q_len = 1 if single_query else 128+ kv_len = 1 if single_key else 128+ try:+ q = torch.empty(1, n_heads, q_len, head_dim, device = device, dtype = dtype)+ k = torch.empty(1, n_kv_heads, kv_len, head_dim, device = device, dtype = dtype)+ k_full = torch.empty(1, n_heads, kv_len, head_dim, device = device, dtype = dtype)+ if needs_grad:+ q, k, k_full = (t.requires_grad_() for t in (q, k, k_full))+ mask = (+ torch.ones(1, 1, q_len, kv_len, device = device, dtype = torch.bool) if has_mask else None+ )+ except Exception:+ return -1, -1+ causal = is_causal and not has_mask++ def probe(key, **kwargs):+ # Each on its own: torch raises when no enabled backend fits, and a GQA call nothing+ # takes must still learn whether the expanded call would run.+ try:+ with torch.set_grad_enabled(needs_grad):+ return int(+ torch._fused_sdp_choice(+ q, key, key, attn_mask = mask, dropout_p = dropout_p, is_causal = causal, **kwargs+ )+ )+ except Exception:+ return None++ gqa = probe(k, enable_gqa = True)+ mha = probe(k_full)+ if gqa is None:+ # Nothing takes GQA here: expand when the expanded call has a backend, else unknown.+ return (_SDPA_NO_BACKEND, mha) if mha is not None else (-1, -1)+ return gqa, -1 if mha is None else mha+++# _sdpa_gqa_choices' answer when no enabled SDPA backend accepts the GQA call at all.+_SDPA_NO_BACKEND = -2+++def _sdpa_gqa_state(+ device,+ dtype,+ head_dim,+ n_heads,+ n_kv_heads,+ is_causal,+ has_mask,+ needs_grad = False,+ dropout_p = 0.0,+ single_query = False,+ single_key = False,+) -> bool:+ """False when enable_gqa would leave SDPA only its math kernel while expanded K/V would not.+ Flash and cuDNN take GQA, the memory-efficient kernel does not (before torch 2.14), so fp32,+ pre-sm80 GPUs and masked calls flash cannot run would otherwise fall to math: several times+ slower and a (T, T) score matrix per head. Asked of torch, so ROCm's backend set is covered too."""+ cuda = torch.backends.cuda+ backends = (+ cuda.flash_sdp_enabled(),+ cuda.cudnn_sdp_enabled(),+ cuda.mem_efficient_sdp_enabled(),+ cuda.math_sdp_enabled(),+ tuple(getattr(torch._C, "_get_sdp_priority_order", tuple)()),+ # Strict deterministic mode makes cuDNN SDPA ineligible.+ torch.are_deterministic_algorithms_enabled(),+ torch.is_deterministic_algorithms_warn_only_enabled(),+ )+ gqa, mha = _sdpa_gqa_choices(+ device,+ dtype,+ head_dim,+ n_heads,+ n_kv_heads,+ is_causal,+ has_mask,+ needs_grad,+ dropout_p,+ backends,+ single_query,+ single_key,+ )+ from torch.nn.attention import SDPBackend++ math = int(SDPBackend.MATH)+ if gqa == _SDPA_NO_BACKEND:+ return False+ return not (gqa == math and mha not in (math, -1))+++if hasattr(torch.compiler, "assume_constant_result"):+ _sdpa_gqa_state = torch.compiler.assume_constant_result(_sdpa_gqa_state)+++def _sdpa_takes_gqa(+ Q: Tensor,+ K: Tensor,+ is_causal: bool,+ has_mask: bool,+ V: Optional[Tensor] = None,+ dropout_p = 0.0,+) -> bool:+ """Whether to pass enable_gqa (True) or expand K/V (False) for this SDPA call."""+ if not SDPA_HAS_GQA:+ return False+ if not Q.is_cuda:+ return True+ needs_grad = torch.is_grad_enabled() and any(+ t is not None and t.requires_grad for t in (Q, K, V)+ )+ return _sdpa_gqa_state(+ str(Q.device),+ Q.dtype,+ Q.shape[-1],+ Q.shape[1],+ K.shape[1],+ bool(is_causal),+ bool(has_mask),+ bool(needs_grad),+ float(dropout_p or 0.0),+ Q.shape[-2] == 1,+ K.shape[-2] == 1,+ )++def _torch_varlen_beats_segments(Q: Tensor, K: Tensor, lengths: Tuple[int, ...], n_groups: int, is_causal: bool) -> bool:@@ -295,6 +442,8 @@ def _torch_varlen_beats_segments(if choice == int(SDPBackend.FLASH_ATTENTION):return Trueif choice == int(SDPBackend.CUDNN_ATTENTION):+ if torch.cuda.get_device_capability(Q.device)[0] not in _VARLEN_CUDNN_ARCH_MAJORS:+ return Truereturn sum(lengths) / len(lengths) <= _VARLEN_CUDNN_MAX_MEAN_LENreturn False@@ -365,10 +514,19 @@ def _sdpa_packed_segments(None when the lengths do not tile the row."""if sum(length for length in lengths if length > 0) != Q.shape[-2]:return None- gqa = n_groups != 1 and _sdpa_flash_takes_gqa(Q)- if n_groups != 1 and not gqa:- K = K.repeat_interleave(n_groups, dim = 1)- V = V.repeat_interleave(n_groups, dim = 1)+ expanded = None++ def kv(gqa):+ nonlocal expanded+ if n_groups == 1 or gqa:+ return K, V+ if expanded is None:+ expanded = (+ K.repeat_interleave(n_groups, dim = 1),+ V.repeat_interleave(n_groups, dim = 1),+ )+ return expanded+outs, start, i = [], 0, 0lengths = [length for length in lengths if length > 0]while i < len(lengths):@@ -393,9 +551,15 @@ def _sdpa_packed_segments(is_causal = is_causal,)kwargs["is_causal"] = is_causal and not windowed+ Q_run = split(Q)+ # Probe the run's own tensors: a one-token run has different eligibility than the row.+ gqa = n_groups != 1 and _sdpa_takes_gqa(+ Q_run, split(K), kwargs["is_causal"], windowed, split(V), kwargs.get("dropout_p", 0.0)+ )if gqa:kwargs["enable_gqa"] = True- out = scaled_dot_product_attention(split(Q), split(K), split(V), **kwargs)+ K_run, V_run = kv(gqa)+ out = scaled_dot_product_attention(Q_run, split(K_run), split(V_run), **kwargs)outs.append(out.transpose(1, 2).flatten(0, 1))start, i = end, jreturn torch.cat(outs, dim = 0).unsqueeze(0)@@ -534,7 +698,23 @@ def _ring_attention_active() -> bool:return manager is not None and manager.backend == "ring"-def select_attention_backend(use_varlen: bool = False) -> str:+def _native_beats_xformers(capabilities, varlen_loaded: bool) -> bool:+ """Without FlashAttention, torch's own kernels match or beat xformers 0.0.35 (which itself calls+ torch's flash / memory-efficient ops) once torch has a varlen kernel (2.12+) on sm_80+ CUDA:+ packed / padding-free 1.00x to 1.06x on A100, L4, RTX PRO 6000 and H100. Below sm_80 there is no+ torch varlen, and xformers' block-diagonal kernel stays 8% to 21% faster (T4); a mixed host keeps+ xformers for its older card."""+ if not varlen_loaded or not capabilities or torch.version.hip is not None:+ return False+ return all(capability[0] >= 8 for capability in capabilities)+++_NATIVE_BEATS_XFORMERS = _native_beats_xformers(+ _visible_capabilities(), _TORCH_VARLEN_ATTN is not None+)+++def select_attention_backend(use_varlen: bool = False, dropout_p: float = 0.0) -> str:"""Return attention backend based on availability / priority order."""if _ring_attention_active():@@ -549,7 +729,14 @@ def select_attention_backend(use_varlen: bool = False) -> str:return FLASH_VARLENelse:return FLASH_DENSE- if HAS_XFORMERS:+ # Only the packed route prefers torch's kernels (varlen / per-segment SDPA). Dense calls keep+ # xformers: it alone consumes a caller's block-diagonal causal_mask, and expanding a GQA KV cache+ # for SDPA on fp32 decode costs memory. With UNSLOTH_SDPA_PACKED_SEGMENTS off, packed SDPA would+ # build a dense (T, T) mask, so xformers stays there too. Torch varlen takes no dropout, so a+ # packed call with dropout would become one SDPA launch per run: xformers keeps those as well.+ if HAS_XFORMERS and not (+ use_varlen and not dropout_p and _NATIVE_BEATS_XFORMERS and _SDPA_PACKED_SEGMENTS+ ):return XFORMERSreturn SDPA@@ -1005,6 +1192,16 @@ def _run_attention(# expansion for deterministic row-wise behavior.use_sdpa_gqa = False+ if use_sdpa_gqa and not _sdpa_takes_gqa(+ Q,+ K,+ kwargs.get("is_causal", False),+ kwargs.get("attn_mask") is not None,+ V,+ kwargs.get("dropout_p", 0.0),+ ):+ use_sdpa_gqa = False+if use_sdpa_gqa:kwargs.setdefault("enable_gqa", True)out = scaled_dot_product_attention(Q, K, V, **kwargs)