Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

SDPA on cuDNN: bucket masked and prefill calls, keep decode off cuDNN where torch would pick it (#15268)

Daniel Hana2aca8d475bb
diff --git a/tests/test_sdpa_cudnn_length_bucket_masked.py b/tests/test_sdpa_cudnn_length_bucket_masked.py
new file mode 100644
index 00000000000..4524eac2bb0
--- /dev/null
+++ b/tests/test_sdpa_cudnn_length_bucket_masked.py
@@ -0,0 +1,324 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+"""Masked training SDPA on cuDNN attention (GRPO / DPO padding) pads Q / K / V and the mask to a
+length bucket: outputs and gradients match the unpadded call against an fp32 oracle."""
+
+import types
+
+import pytest
+import torch
+import torch.nn.functional as F
+
+from unsloth.utils import attention_dispatch as ad
+
+
+def _cudnn_sdpa_available():
+ if not torch.cuda.is_available() or ad._CUDNN_CHOICE is None:
+ return False
+ try:
+ from torch.nn.attention import SDPBackend, sdpa_kernel
+
+ q = torch.randn(2, 2, 64, 64, device = "cuda", dtype = torch.bfloat16)
+ m = torch.ones(2, 1, 64, 64, device = "cuda", dtype = torch.bool)
+ with sdpa_kernel([SDPBackend.CUDNN_ATTENTION]):
+ F.scaled_dot_product_attention(q, q, q, attn_mask = m)
+ return torch.cuda.get_device_capability()[0] >= 8
+ except Exception:
+ return False
+
+
+needs_cudnn = pytest.mark.skipif(not _cudnn_sdpa_available(), reason = "no masked cuDNN SDPA here")
+
+
+def _qkv(
+ bsz,
+ n_heads,
+ n_kv,
+ q_len,
+ head_dim,
+ gqa,
+ seed = 0,
+):
+ g = torch.Generator(device = "cuda").manual_seed(seed)
+ mk = lambda h: torch.randn(
+ bsz, h, q_len, head_dim, device = "cuda", dtype = torch.bfloat16, generator = g
+ )
+ q, k, v = mk(n_heads), mk(n_kv), mk(n_kv)
+ if not gqa:
+ k = k.repeat_interleave(n_heads // n_kv, 1)
+ v = v.repeat_interleave(n_heads // n_kv, 1)
+ return [t.requires_grad_() for t in (q, k, v)]
+
+
+def _mask(
+ kind,
+ bsz,
+ q_len,
+ seed = 0,
+):
+ """The masks run_attention hands SDPA: (bsz, 1, q, k) bool keep masks with fully masked rows
+ opened up, a (bsz, 1, 1, k) key-padding mask, or an HF-style additive bf16 mask."""
+ g = torch.Generator().manual_seed(seed)
+ real = [max(1, int(q_len * f)) for f in torch.rand(bsz, generator = g).clamp(0.3, 1.0)]
+ real[0] = q_len # one unpadded row
+ pos = torch.arange(q_len)
+ keep = torch.stack(
+ [(pos >= q_len - n) if kind.startswith("left") else (pos < n) for n in real]
+ ).cuda()
+ if kind == "keys_only":
+ return keep[:, None, None, :]
+ causal = torch.ones(q_len, q_len, dtype = torch.bool, device = "cuda").tril()
+ mask = causal[None, None] & keep[:, None, None, :]
+ mask = mask | ~mask.any(dim = -1, keepdim = True)
+ if kind.endswith("additive"):
+ return torch.zeros(mask.shape, dtype = torch.bfloat16, device = "cuda").masked_fill(
+ ~mask, torch.finfo(torch.bfloat16).min
+ )
+ return mask
+
+
+def _run(fn, q, k, v, grad, **kw):
+ for t in (q, k, v):
+ t.grad = None
+ out = fn(q, k, v, **kw)
+ out.backward(grad)
+ return out.detach().float(), [t.grad.float() for t in (q, k, v)]
+
+
+def _rel(a, b):
+ return ((a - b).norm() / b.norm().clamp_min(1e-12)).item()
+
+
+def test_gate_masked(monkeypatch):
+ if ad._CUDNN_CHOICE is None or not torch.cuda.is_available():
+ pytest.skip("needs CUDA and torch's cuDNN SDPA backend")
+ monkeypatch.setattr(ad, "_sdpa_masked_choice", lambda *a, **k: ad._CUDNN_CHOICE)
+ q = torch.zeros(2, 4, 8, 64, device = "cuda", dtype = torch.bfloat16)
+ m4 = torch.ones(2, 1, 8, 8, dtype = torch.bool, device = "cuda")
+ applies = ad._cudnn_length_bucket_applies
+ assert applies(q, q, q, attn_mask = m4)
+ assert applies(q, q, q, attn_mask = m4[:, :, :1]) # key-padding broadcast over queries
+ assert applies(q, q, q, attn_mask = torch.zeros(2, 4, 8, 8, dtype = q.dtype, device = "cuda"))
+ # Batch-shared (cached) masks and single-row batches keep the unpadded call.
+ assert not applies(q, q, q, attn_mask = m4[:1])
+ assert not applies(q[:1], q[:1], q[:1], attn_mask = m4[:1])
+ assert not applies(q, q, q, attn_mask = m4, is_causal = True) # SDPA rejects both
+ assert not applies(q, q, q, attn_mask = m4, dropout_p = 0.1)
+ assert not applies(q, q, q, attn_mask = m4[0, 0]) # 2D: not a shape run_attention builds
+ assert not applies(q, q, q, attn_mask = m4.float()) # neither bool nor Q's dtype
+ assert not applies(q, q, q, attn_mask = m4[..., :4])
+ grad_mask = torch.zeros(2, 1, 8, 8, dtype = q.dtype, device = "cuda", requires_grad = True)
+ assert not applies(q, q, q, attn_mask = grad_mask)
+ assert not applies(q, q, q) # not causal, no mask
+ monkeypatch.setattr(ad, "_sdpa_masked_choice", lambda *a, **k: 1) # flash
+ assert not applies(q, q, q, attn_mask = m4)
+ monkeypatch.setattr(ad, "_sdpa_masked_choice", lambda *a, **k: ad._CUDNN_CHOICE)
+ monkeypatch.setattr(ad, "_CUDNN_LENGTH_BUCKET", 0)
+ assert not applies(q, q, q, attn_mask = m4)
+
+
[email protected]("backend", ["ring", "ulysses"])
+def test_masked_not_applied_under_context_parallelism(monkeypatch, backend):
+ if not torch.cuda.is_available():
+ pytest.skip("needs CUDA")
+ monkeypatch.setattr(ad, "_sdpa_masked_choice", lambda *a, **k: ad._CUDNN_CHOICE)
+ monkeypatch.setattr(ad, "get_cp_manager", lambda: types.SimpleNamespace(backend = backend))
+ q = torch.randn(2, 2, 200, 64, device = "cuda", dtype = torch.bfloat16)
+ m = torch.ones(2, 1, 200, 200, dtype = torch.bool, device = "cuda")
+ assert not ad._cudnn_length_bucket_applies(q, q, q, attn_mask = m)
+
+
+@needs_cudnn
[email protected]("q_len", [2, 3, 127, 128, 129, 300, 513, 1000])
[email protected]("kind", ["left", "right", "keys_only", "left_additive"])
[email protected]("gqa", [True, False])
+def test_masked_bucketed_matches_unpadded(q_len, kind, gqa):
+ from torch.nn.attention import SDPBackend, sdpa_kernel
+
+ kw = dict(enable_gqa = True) if gqa else {}
+ bsz = 3
+ q, k, v = _qkv(bsz, 8, 2, q_len, 128, gqa, seed = q_len)
+ mask = _mask(kind, bsz, q_len, seed = q_len)
+ grad = torch.randn(bsz, 8, q_len, 128, device = "cuda", dtype = torch.bfloat16)
+ with sdpa_kernel([SDPBackend.CUDNN_ATTENTION]):
+ ref, ref_g = _run(F.scaled_dot_product_attention, q, k, v, grad, attn_mask = mask, **kw)
+ got, got_g = _run(ad._cudnn_length_bucketed_sdpa, q, k, v, grad, attn_mask = mask, **kw)
+ # fp32 oracle on the same bf16 inputs.
+ q32, k32, v32 = (t.detach().float().requires_grad_() for t in (q, k, v))
+ m32 = mask if mask.dtype == torch.bool else mask.float()
+ with sdpa_kernel([SDPBackend.MATH]):
+ orc, orc_g = _run(
+ F.scaled_dot_product_attention, q32, k32, v32, grad.float(), attn_mask = m32, **kw
+ )
+ assert got.shape == ref.shape
+ for a, b, o in [(got, ref, orc)] + list(zip(got_g, ref_g, orc_g)):
+ assert torch.isfinite(a).all()
+ # The bucketed call is as close to fp32 as the unpadded cuDNN call (bf16 rounding).
+ assert _rel(a, o) <= 1.25 * _rel(b, o) + 2e-3, (_rel(a, o), _rel(b, o))
+
+
+@needs_cudnn
+def test_masked_training_sdpa_buckets_lengths(monkeypatch):
+ from torch.nn.attention import SDPBackend, sdpa_kernel
+
+ seen = []
+ real = ad.scaled_dot_product_attention
+
+ def record(Q, K, V, **kwargs):
+ seen.append((Q.shape[-2], kwargs["attn_mask"].shape[-1]))
+ return real(Q, K, V, **kwargs)
+
+ monkeypatch.setattr(ad, "scaled_dot_product_attention", record)
+ monkeypatch.setattr(ad, "_sdpa_masked_choice", lambda *a, **k: ad._CUDNN_CHOICE)
+ with sdpa_kernel([SDPBackend.CUDNN_ATTENTION]):
+ for q_len in (5, 130, 256, 700):
+ q, k, v = _qkv(2, 4, 4, q_len, 64, False)
+ out = ad._training_sdpa(q, k, v, attn_mask = _mask("left", 2, q_len))
+ assert out.shape[-2] == q_len
+ if ad._avoids_cudnn(q, True):
+ assert seen == [(5, 5), (130, 130), (256, 256), (700, 700)] # SM100 pin: unchanged
+ else:
+ assert seen == [(128, 128), (256, 256), (384, 384), (768, 768)]
+
+
+@needs_cudnn
+def test_masked_compiled_dynamic_lengths_one_graph():
+ import torch._dynamo
+ from torch._dynamo.utils import counters
+ from torch.nn.attention import SDPBackend, sdpa_kernel
+
+ torch._dynamo.reset()
+ counters.clear()
+
+ def f(q, k, v, m):
+ return ad._cudnn_length_bucketed_sdpa(q, k, v, attn_mask = m, enable_gqa = True)
+
+ compiled = torch.compile(f, dynamic = True, fullgraph = True)
+ with sdpa_kernel([SDPBackend.CUDNN_ATTENTION]):
+ for q_len in (300, 517, 1000, 1024):
+ q, k, v = _qkv(2, 8, 2, q_len, 128, True)
+ m = _mask("left", 2, q_len)
+ torch.testing.assert_close(compiled(q, k, v, m), f(q, k, v, m), atol = 2e-2, rtol = 2e-2)
+ assert counters["stats"]["unique_graphs"] == 1, dict(counters["stats"])
+
+
+@needs_cudnn
+def test_masked_gate_compiles_without_graph_breaks(monkeypatch):
+ """The real gate (constant probe on the mask shape) traces fullgraph across lengths."""
+ import torch._dynamo
+ from torch._dynamo.utils import counters
+
+ if ad._avoids_cudnn(torch.empty(1, device = "cuda"), True):
+ pytest.skip("SM100 training stays off cuDNN")
+ torch._dynamo.reset()
+ counters.clear()
+
+ def f(q, k, v, m):
+ return ad._training_sdpa(q, k, v, attn_mask = m, enable_gqa = True)
+
+ compiled = torch.compile(f, dynamic = True, fullgraph = True)
+ for q_len in (300, 517, 1000):
+ q, k, v = _qkv(2, 8, 2, q_len, 128, True)
+ m = _mask("left", 2, q_len)
+ torch.testing.assert_close(compiled(q, k, v, m), f(q, k, v, m), atol = 2e-2, rtol = 2e-2)
+ assert counters["stats"]["unique_graphs"] == 1, dict(counters["stats"])
+
+
+@needs_cudnn
[email protected]("masked", [True, False])
+def test_inference_prefill_bucketed_decode_untouched(monkeypatch, masked):
+ """No-grad prefill (q_len == k_len) is bucketed like training; a decode step (q_len 1) is not."""
+ from torch.nn.attention import SDPBackend, sdpa_kernel
+
+ seen = []
+ real = ad.scaled_dot_product_attention
+
+ def record(Q, K, V, **kwargs):
+ seen.append((Q.shape[-2], K.shape[-2]))
+ return real(Q, K, V, **kwargs)
+
+ monkeypatch.setattr(ad, "scaled_dot_product_attention", record)
+ monkeypatch.setattr(ad, "_sdpa_masked_choice", lambda *a, **k: ad._CUDNN_CHOICE)
+ monkeypatch.setattr(ad, "_sdpa_backend_state", lambda *a, **k: (True, ad._CUDNN_CHOICE))
+ L = 300
+ q, k, v = (t.detach() for t in _qkv(2, 8, 8, L, 128, False))
+ kw = dict(attn_mask = _mask("left", 2, L)) if masked else dict(is_causal = True)
+ with torch.no_grad(), sdpa_kernel([SDPBackend.CUDNN_ATTENTION]):
+ got = ad._inference_sdpa(q, k, v, **kw)
+ want = real(q, k, v, **kw)
+ # cuDNN takes no single-query call: the decode step runs on torch's default kernels.
+ step_mask = dict(attn_mask = torch.ones(2, 1, 1, L, dtype = torch.bool, device = "cuda"))
+ with torch.no_grad():
+ ad._inference_sdpa(q[:, :, -1:], k, v, **(step_mask if masked else {}))
+ assert seen[0] == (384, 384) and seen[-1] == (1, L), seen
+ assert got.shape == want.shape
+ # fp32 oracle: the bucketed prefill is as accurate as the unpadded one.
+ m32 = kw.get("attn_mask")
+ with sdpa_kernel([SDPBackend.MATH]):
+ orc = F.scaled_dot_product_attention(
+ q.float(),
+ k.float(),
+ v.float(),
+ **({"attn_mask": m32} if masked else {"is_causal": True}),
+ )
+ assert _rel(got.float(), orc) <= 1.25 * _rel(want.float(), orc) + 2e-3
+
+
[email protected]("masked", [True, False])
+def test_decode_sdpa_avoids_cudnn_where_flagged(monkeypatch, masked):
+ """Decode (one query against a growing cache) runs off cuDNN on flagged devices: cuDNN would
+ build a plan for every new cache length, i.e. every generated token."""
+ if not torch.cuda.is_available() or ad._CUDNN_CHOICE is None:
+ pytest.skip("needs CUDA and torch's cuDNN SDPA backend")
+ seen = []
+ real = F.scaled_dot_product_attention
+
+ def record(*a, **k):
+ seen.append(torch.backends.cuda.cudnn_sdp_enabled())
+ return real(*a, **k)
+
+ monkeypatch.setattr(F, "scaled_dot_product_attention", record)
+ monkeypatch.setattr(ad, "_decode_picks_cudnn", lambda *a: True)
+ q = torch.randn(4, 8, 1, 64, device = "cuda", dtype = torch.bfloat16)
+ kv = torch.randn(4, 8, 300, 64, device = "cuda", dtype = torch.bfloat16)
+ kw = dict(attn_mask = torch.ones(4, 1, 1, 300, dtype = torch.bool, device = "cuda")) if masked else {}
+ monkeypatch.setattr(ad, "_NO_CUDNN_DECODE", frozenset({q.device.index}))
+ out = ad.decode_scaled_dot_product_attention(q, kv, kv, **kw)
+ monkeypatch.setattr(ad, "_NO_CUDNN_DECODE", frozenset())
+ ref = ad.decode_scaled_dot_product_attention(q, kv, kv, **kw)
+ assert seen[0] is False and seen[1] == torch.backends.cuda.cudnn_sdp_enabled()
+ torch.testing.assert_close(out, ref, atol = 2e-2, rtol = 2e-2)
+ # Prefill-shaped calls are never pinned.
+ monkeypatch.setattr(ad, "_NO_CUDNN_DECODE", frozenset({q.device.index}))
+ ad.decode_scaled_dot_product_attention(kv, kv, kv, is_causal = True)
+ assert seen[-1] == torch.backends.cuda.cudnn_sdp_enabled()
+
+
[email protected]("masked", [True, False])
+def test_decode_sdpa_pins_only_where_torch_picks_cudnn(monkeypatch, masked):
+ """The pin is entered only when torch's own pick for the call is cuDNN: elsewhere (torch 2.10 on
+ H100) sdpa_kernel is pure overhead per decode call."""
+ if not torch.cuda.is_available() or ad._CUDNN_CHOICE is None:
+ pytest.skip("needs CUDA and torch's cuDNN SDPA backend")
+ q = torch.randn(4, 8, 1, 64, device = "cuda", dtype = torch.bfloat16)
+ kv = torch.randn(4, 8, 300, 64, device = "cuda", dtype = torch.bfloat16)
+ mask = torch.ones(4, 1, 1, 300, dtype = torch.bool, device = "cuda") if masked else None
+ torch_pick = int(torch._fused_sdp_choice(q, kv, kv, attn_mask = mask))
+ ad._DECODE_PICKS_CUDNN.clear()
+ assert ad._decode_picks_cudnn(q, kv, mask, False) == (torch_pick == ad._CUDNN_CHOICE)
+ entered = []
+ real = ad._sdpa_kernel
+ monkeypatch.setattr(ad, "_sdpa_kernel", lambda b: entered.append(1) or real(b))
+ monkeypatch.setattr(ad, "_NO_CUDNN_DECODE", frozenset({q.device.index}))
+ for picks in (False, True):
+ monkeypatch.setattr(ad, "_decode_picks_cudnn", lambda *a, p = picks: p)
+ ad.decode_scaled_dot_product_attention(q, kv, kv, attn_mask = mask)
+ assert entered == [1]
+
+
+def test_decode_no_cudnn_devices(monkeypatch):
+ monkeypatch.setattr(ad, "_visible_capabilities", lambda: ((8, 0), (9, 0), (10, 0), (12, 0)))
+ monkeypatch.delenv("UNSLOTH_SDPA_DECODE_CUDNN", raising = False)
+ if torch.version.hip is None:
+ assert ad._decode_no_cudnn_devices() == frozenset({1, 2})
+ monkeypatch.setenv("UNSLOTH_SDPA_DECODE_CUDNN", "1")
+ assert ad._decode_no_cudnn_devices() == frozenset()
diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py
index 08caf5ea346..7a6136c89dc 100644
--- a/unsloth/models/llama.py
+++ b/unsloth/models/llama.py
@@ -103,7 +103,7 @@ from ..utils.attention_dispatch import (
select_attention_backend,
resolve_prefix_seg_info,
)
-from torch.nn.functional import scaled_dot_product_attention
+from torch.nn.functional import scaled_dot_product_attention as _torch_sdpa
from transformers import __version__ as transformers_version
from unsloth_zoo.utils import Version, _get_dtype
from unsloth_zoo.hf_utils import (
@@ -266,7 +266,11 @@ from math import sqrt as math_sqrt
KV_CACHE_INCREMENT = 512
torch_nn_functional_softmax = torch.nn.functional.softmax
# SDPA has GQA internally.
-SDPA_HAS_GQA = "enable_gqa" in scaled_dot_product_attention.__doc__
+SDPA_HAS_GQA = "enable_gqa" in (_torch_sdpa.__doc__ or "")
+# Decode paths here and in models importing this name avoid cuDNN on sm90 / sm100.
+from ..utils.attention_dispatch import (
+ decode_scaled_dot_product_attention as scaled_dot_product_attention,
+)
from peft.utils.other import ModulesToSaveWrapper
diff --git a/unsloth/utils/attention_dispatch.py b/unsloth/utils/attention_dispatch.py
index 304a832aade..82d6529d249 100644
--- a/unsloth/utils/attention_dispatch.py
+++ b/unsloth/utils/attention_dispatch.py
@@ -116,12 +116,19 @@ def _training_sdpa(*args, **kwargs):
return scaled_dot_product_attention(*args, **kwargs)
+def _inference_sdpa(*args, **kwargs):
+ """Non-training SDPA: a prefill also pays cuDNN's per-length plan, so it is bucketed too."""
+ if len(args) >= 3 and _cudnn_length_bucket_applies(*args[:3], **kwargs):
+ return _cudnn_length_bucketed_sdpa(*args, **kwargs)
+ return scaled_dot_product_attention(*args, **kwargs)
+
+
# Wherever SDPA picks cuDNN attention for a training call (H100 on torch 2.14: the default
# priority), every new sequence length builds a new cuDNN plan, ~170 ms forward plus backward on
# H100. Padded batches change length every step, so that cost landed on nearly every step. Padding
# Q / K / V along the sequence to a multiple of this bucket bounds the plans to max_len / bucket.
-# Only causal calls with no mask and q_len == k_len: padded keys come after every real query, so
-# causal masking hides them and the real rows are what the unpadded call returns. 0 disables.
+# Causal calls: causal masking hides the padded keys. Masked calls (GRPO / DPO padding): the mask is
+# padded too (new keys masked out, new query rows attend everything and get zero gradient). 0 disables.
try:
_CUDNN_LENGTH_BUCKET = max(int(os.environ.get("UNSLOTH_SDPA_CUDNN_BUCKET", "128")), 0)
except ValueError:
@@ -143,12 +150,29 @@ def _cudnn_length_bucket_applies(
# zero keys inside the global sequence.
if get_cp_manager() is not None:
return False
- if attn_mask is not None or not is_causal or dropout_p:
+ if dropout_p or (attn_mask is None) != bool(is_causal):
return False
if not (isinstance(Q, Tensor) and Q.is_cuda and Q.dim() == 4 and K.dim() == 4 and V.dim() == 4):
return False
if Q.dtype not in (torch.float16, torch.bfloat16) or Q.shape[-2] != K.shape[-2]:
return False
+ enable_gqa = True if kwargs.get("enable_gqa", False) else False
+ if attn_mask is not None:
+ if not _cudnn_bucket_mask_ok(attn_mask, Q, K):
+ return False
+ needs_grad = torch.is_grad_enabled() and (
+ Q.requires_grad or K.requires_grad or V.requires_grad
+ )
+ choice = _sdpa_masked_choice(
+ str(Q.device),
+ Q.dtype,
+ Q.shape[-1],
+ Q.shape[1],
+ K.shape[1],
+ enable_gqa,
+ True if needs_grad else False,
+ )
+ return choice == _CUDNN_CHOICE
_, choice = _sdpa_backend_state(
str(Q.device),
Q.dtype,
@@ -156,11 +180,45 @@ def _cudnn_length_bucket_applies(
Q.shape[1],
K.shape[1],
True,
- bool(kwargs.get("enable_gqa", False)),
+ enable_gqa,
)
return choice == _CUDNN_CHOICE
+def _cudnn_bucket_mask_ok(attn_mask, Q, K):
+ """Per-batch 4D padding masks only (bool or Q-dtype additive, no grad). Batch-shared cached masks
+ are skipped: a padded copy per layer would multiply their memory."""
+ if not isinstance(attn_mask, Tensor) or attn_mask.dim() != 4 or attn_mask.requires_grad:
+ return False
+ if attn_mask.dtype != torch.bool and attn_mask.dtype != Q.dtype:
+ return False
+ if attn_mask.device != Q.device or attn_mask.shape[-1] != K.shape[-2]:
+ return False
+ if attn_mask.shape[-2] != 1 and attn_mask.shape[-2] != Q.shape[-2]:
+ return False
+ if Q.shape[0] == 1 or attn_mask.shape[0] != Q.shape[0]:
+ return False
+ return attn_mask.shape[1] == 1 or attn_mask.shape[1] == Q.shape[1]
+
+
+def _sdpa_masked_choice(device, dtype, head_dim, n_heads, n_kv_heads, enable_gqa, needs_grad):
+ """SDPBackend SDPA picks for a masked, non-causal call with these shapes (-1 if unknown)."""
+ cuda = torch.backends.cuda
+ backends = (
+ cuda.flash_sdp_enabled(),
+ cuda.cudnn_sdp_enabled(),
+ cuda.mem_efficient_sdp_enabled(),
+ cuda.math_sdp_enabled(),
+ tuple(getattr(torch._C, "_get_sdp_priority_order", tuple)()),
+ torch.are_deterministic_algorithms_enabled(),
+ torch.is_deterministic_algorithms_warn_only_enabled(),
+ )
+ gqa, mha = _sdpa_gqa_choices(
+ device, dtype, head_dim, n_heads, n_kv_heads, False, True, needs_grad, 0.0, backends
+ )
+ return gqa if enable_gqa else mha
+
+
def _cudnn_length_bucketed_sdpa(
Q,
K,
@@ -175,7 +233,20 @@ def _cudnn_length_bucketed_sdpa(
q_len = Q.shape[-2]
pad = _CUDNN_LENGTH_BUCKET - q_len % _CUDNN_LENGTH_BUCKET
Q, K, V = (F.pad(x, (0, 0, 0, pad)) for x in (Q, K, V))
- out = scaled_dot_product_attention(Q, K, V, is_causal = True, **kwargs)
+ if attn_mask is None:
+ out = scaled_dot_product_attention(Q, K, V, is_causal = True, **kwargs)
+ return out[..., :q_len, :]
+ # Padded query rows attend everything (no fully masked row) and are sliced off; padded keys are masked.
+ rows = attn_mask.shape[-2]
+ shape = (*attn_mask.shape[:-2], rows if rows == 1 else rows + pad, q_len + pad)
+ if attn_mask.dtype == torch.bool:
+ padded = attn_mask.new_ones(shape)
+ padded[..., :rows, q_len:] = False
+ else:
+ padded = attn_mask.new_zeros(shape)
+ padded[..., :rows, q_len:] = torch.finfo(attn_mask.dtype).min
+ padded[..., :rows, :q_len] = attn_mask
+ out = scaled_dot_product_attention(Q, K, V, attn_mask = padded, **kwargs)
return out[..., :q_len, :]
@@ -241,6 +312,77 @@ def _no_cudnn_training_devices():
_NO_CUDNN_TRAINING = _no_cudnn_training_devices()
+def _decode_no_cudnn_devices():
+ """sm90 / sm100 devices whose decode SDPA avoids cuDNN: it builds a plan per key length, i.e. per
+ generated token. UNSLOTH_SDPA_DECODE_CUDNN=1 keeps it."""
+ if os.environ.get("UNSLOTH_SDPA_DECODE_CUDNN", "0") == "1" or torch.version.hip is not None:
+ return frozenset()
+ return frozenset(i for i, cap in enumerate(_visible_capabilities()) if cap[0] in (9, 10))
+
+
+_NO_CUDNN_DECODE = _decode_no_cudnn_devices()
+
+
+_DECODE_PICKS_CUDNN = {}
+
+
+def _decode_picks_cudnn(query, key, attn_mask, enable_gqa):
+ """Whether torch would pick cuDNN for this decode call (else sdpa_kernel is ~25 us of overhead)."""
+ cache_key = (
+ query.device.index,
+ query.dtype,
+ query.shape[-1],
+ query.shape[1],
+ key.shape[1],
+ attn_mask is not None,
+ enable_gqa,
+ torch.backends.cuda.cudnn_sdp_enabled(),
+ )
+ picks = _DECODE_PICKS_CUDNN.get(cache_key)
+ if picks is None:
+ cuda = torch.backends.cuda
+ backends = (
+ cuda.flash_sdp_enabled(),
+ cuda.cudnn_sdp_enabled(),
+ cuda.mem_efficient_sdp_enabled(),
+ cuda.math_sdp_enabled(),
+ tuple(getattr(torch._C, "_get_sdp_priority_order", tuple)()),
+ )
+ gqa, mha = _sdpa_gqa_choices(
+ str(query.device), query.dtype, query.shape[-1], query.shape[1], key.shape[1],
+ False, attn_mask is not None, False, 0.0, backends, single_query = True,
+ ) # fmt: skip
+ picks = _DECODE_PICKS_CUDNN[cache_key] = (gqa if enable_gqa else mha) == _CUDNN_CHOICE
+ return picks
+
+
+def decode_scaled_dot_product_attention(query, key, value, *args, **kwargs):
+ """Decode SDPA, kept off cuDNN where torch would pick it on _NO_CUDNN_DECODE devices."""
+ if (
+ _CUDNN_CHOICE is not None
+ and query.is_cuda
+ and query.shape[-2] == 1
+ and query.device.index in _NO_CUDNN_DECODE
+ and (
+ torch.compiler.is_compiling()
+ or _decode_picks_cudnn(
+ query,
+ key,
+ kwargs.get("attn_mask", args[0] if args else None),
+ bool(kwargs.get("enable_gqa", False)),
+ )
+ ) # fmt: skip
+ ):
+ with _sdpa_kernel(_XFORMERS_LIKE_BACKENDS):
+ return F.scaled_dot_product_attention(query, key, value, *args, **kwargs)
+ return F.scaled_dot_product_attention(query, key, value, *args, **kwargs)
+
+
+decode_scaled_dot_product_attention.__doc__ = (
+ F.scaled_dot_product_attention.__doc__ or ""
+) + decode_scaled_dot_product_attention.__doc__
+
+
# Mixed-generation hosts only (e.g. sm_86 + sm_75): the backend is re-checked against Q's device, since a
# device-0 verdict is wrong for layers a device_map places on another card.
_DEVICE_BACKENDS = _per_device_backends(_visible_capabilities())
@@ -527,6 +669,7 @@ def _constant_probe(fn):
_sdpa_backend_state = _constant_probe(_sdpa_backend_state)
+_sdpa_masked_choice = _constant_probe(_sdpa_masked_choice)
@functools.lru_cache(maxsize = 128)
@@ -844,7 +987,7 @@ def _sdpa_packed_segments(
if gqa:
kwargs["enable_gqa"] = True
K_run, V_run = kv(gqa)
- out = (_training_sdpa if training else scaled_dot_product_attention)(
+ out = (_training_sdpa if training else _inference_sdpa)(
Q_run, split(K_run), split(V_run), **kwargs
)
outs.append(out.transpose(1, 2).flatten(0, 1))
@@ -1497,7 +1640,7 @@ def _run_attention(
)
kwargs = dict(sdpa_kwargs)
- sdpa = _training_sdpa if context.training else scaled_dot_product_attention
+ sdpa = _training_sdpa if context.training else _inference_sdpa
kwargs.setdefault("attn_mask", local_mask)
kwargs.setdefault("is_causal", is_causal_local)