Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

cuDNN SDPA: keep known-bad decode and NaN-dQ shapes off cuDNN before torch 2.14 / 2.15 (#15934)

Daniel Han8161e8188d9b
diff --git a/tests/test_cudnn_sdpa_known_bad_shapes.py b/tests/test_cudnn_sdpa_known_bad_shapes.py
new file mode 100644
index 00000000000..e9e80dd1223
--- /dev/null
+++ b/tests/test_cudnn_sdpa_known_bad_shapes.py
@@ -0,0 +1,640 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""SDPA stays off cuDNN for the calls torch later refused (pytorch#193893, #196678).
+CPU only: versions and capability are mocked; the wrapper is checked via the backend flags it runs under."""
+
+import shutil
+import sys
+import types
+
+import pytest
+
+torch = pytest.importorskip("torch")
+from torch.nn.attention import SDPBackend, sdpa_kernel # noqa: E402
+
+import unsloth.import_fixes as fixes # noqa: E402
+
+
+ "torch_version, cudnn, cap, expected",
+ [
+ ("2.13.0+cu130", 92000, (10, 0), True), # the issue's B200 setup on 2.13
+ ("2.11.0+cu128", 91900, (10, 0), True), # the original report
+ ("2.12.1+cu130", 92000, (10, 3), True), # GB300 / B300
+ ("2.13.0+cu130", 92000, (11, 0), True), # Thor
+ ("2.13.0+cu130", 92500, (10, 0), True), # 9.25.0 still broken
+ ("2.13.0+cu130", 92400, (10, 0), True),
+ ("2.13.0+cu130", 92401, (10, 0), False), # 9.24.1 fixed
+ ("2.13.0+cu130", 92501, (10, 0), False), # 9.25.1 fixed
+ ("2.13.0+cu130", 92600, (10, 0), False),
+ ("2.13.0+cu126", 91002, (10, 0), False), # cu126 wheels ship cuDNN 9.10.2
+ ("2.13.0+cu130", 91800, (10, 0), False),
+ ("2.13.0+cu130", 92000, (12, 0), False), # sm_120 is clean
+ ("2.13.0+cu130", 92000, (9, 0), False),
+ ("2.14.0+cu130", 92400, (10, 0), False), # torch 2.14 refuses it itself
+ ("2.14.1+cu130", 92400, (10, 0), False),
+ ("2.15.0+cu132", 92600, (10, 0), False),
+ ("2.13.0+cu130", None, (10, 0), False),
+ ("not a version", 92000, (10, 0), False),
+ ],
+)
+def test_decode_gate(torch_version, cudnn, cap, expected):
+ assert fixes._cudnn_sdpa_decode_corrupts(torch_version, cudnn, cap) is expected
+
+
+ "torch_version, cudnn, cap, expected",
+ [
+ ("2.14.1+cu130", 92400, (9, 0), True), # what pip install unsloth gets on H100
+ ("2.14.0+cu130", 92400, (9, 0), True),
+ ("2.13.0+cu130", 92000, (9, 0), True), # the issue's version
+ ("2.11.0+cu128", 91900, (9, 0), True),
+ ("2.14.0+cu130", 92501, (9, 0), True), # 9.25.1 still broken (issue comment)
+ ("2.14.0+cu126", 91002, (9, 0), True), # torch's own guard is any cuDNN < 9.26
+ ("2.14.0+cu130", 92600, (9, 0), False), # cuDNN 9.26 fixed
+ ("2.15.0+cu132", 92600, (9, 0), False),
+ ("2.15.0+cu132", 92400, (9, 0), False), # torch 2.15 refuses it itself
+ ("2.14.0+cu130", 92400, (10, 0), False),
+ ("2.14.0+cu130", 92400, (8, 0), False),
+ ("2.14.0+cu130", 92400, (8, 9), False),
+ ("2.14.0+cu130", 92400, (12, 0), False),
+ ("2.14.0+cu130", None, (9, 0), False),
+ ],
+)
+def test_dq_nan_gate(torch_version, cudnn, cap, expected):
+ assert fixes._cudnn_sdpa_dq_nan(torch_version, cudnn, cap) is expected
+
+
+class _FakeCuda:
+ def __init__(
+ self,
+ *shape,
+ dtype = None,
+ requires_grad = False,
+ index = 0,
+ ):
+ self.shape = torch.Size(shape)
+ self.dtype = dtype or torch.bfloat16
+ self.requires_grad = requires_grad
+ self.is_cuda = True
+ self.is_nested = False
+ self.device = types.SimpleNamespace(type = "cuda", index = index)
+
+ def dim(self):
+ return len(self.shape)
+
+
+D0 = frozenset({0})
+ALL_BACKENDS = [
+ SDPBackend.FLASH_ATTENTION,
+ SDPBackend.EFFICIENT_ATTENTION,
+ SDPBackend.MATH,
+ SDPBackend.CUDNN_ATTENTION,
+]
+NONE = frozenset()
+
+
+def _bad(
+ decode,
+ dq,
+ q,
+ k,
+ is_causal = False,
+):
+ return fixes._sdpa_cudnn_known_bad(decode, dq, q, k, is_causal)
+
+
+class TestPerCallDecision:
+ def test_decode_one_query_row(self):
+ q, k = _FakeCuda(2, 28, 1, 128), _FakeCuda(2, 4, 257, 128)
+ assert _bad(D0, NONE, q, k) == "decode"
+ assert _bad(D0, NONE, q, k, is_causal = True) == "decode"
+
+ def test_decode_only_one_row(self):
+ assert _bad(D0, NONE, _FakeCuda(2, 28, 2, 128), _FakeCuda(2, 4, 257, 128)) is None
+ assert _bad(D0, NONE, _FakeCuda(1, 28, 512, 128), _FakeCuda(1, 4, 512, 128)) is None
+
+ def test_decode_other_device_or_cpu(self):
+ q, k = _FakeCuda(1, 8, 1, 64, index = 1), _FakeCuda(1, 8, 9, 64, index = 1)
+ assert _bad(D0, NONE, q, k) is None
+ cpu = torch.zeros(1, 8, 1, 64, dtype = torch.bfloat16)
+ assert _bad(frozenset({0, None}), frozenset({0, None}), cpu, cpu) is None
+
+ def test_decode_fp32_only_under_autocast(self):
+ q, k = _FakeCuda(1, 8, 1, 64, dtype = torch.float32), _FakeCuda(1, 8, 9, 64)
+ assert _bad(D0, NONE, q, k) is None
+
+ @pytest.mark.parametrize("s_kv", [64, 192, 704, 2112])
+ def test_dq_nan_lengths(self, s_kv):
+ q = _FakeCuda(1, 8, s_kv, 64, requires_grad = True)
+ k = _FakeCuda(1, 8, s_kv, 64)
+ assert _bad(NONE, D0, q, k) == "dq_nan"
+
+ @pytest.mark.parametrize("s_kv", [63, 65, 128, 640, 768, 1024])
+ def test_dq_nan_other_lengths_keep_cudnn(self, s_kv):
+ q = _FakeCuda(1, 8, s_kv, 64, requires_grad = True)
+ assert _bad(NONE, D0, q, _FakeCuda(1, 8, s_kv, 64)) is None
+
+ def test_dq_nan_needs_noncausal_grad(self):
+ k = _FakeCuda(1, 8, 704, 64)
+ assert (
+ _bad(NONE, D0, _FakeCuda(1, 8, 704, 64, requires_grad = True), k, is_causal = True) is None
+ )
+ assert _bad(NONE, D0, _FakeCuda(1, 8, 704, 64), k) is None
+ with torch.no_grad():
+ assert _bad(NONE, D0, _FakeCuda(1, 8, 704, 64, requires_grad = True), k) is None
+
+ def test_dq_nan_reads_key_length(self):
+ # Upstream gates on s_k: a 1-row training query against 704 keys is caught, 640 is not.
+ q = _FakeCuda(1, 8, 100, 64, requires_grad = True)
+ assert _bad(NONE, D0, q, _FakeCuda(1, 8, 704, 64)) == "dq_nan"
+ assert _bad(NONE, D0, q, _FakeCuda(1, 8, 640, 64)) is None
+
+ def test_dq_nan_dtype(self):
+ k = _FakeCuda(1, 8, 704, 64)
+ for dtype in (torch.float16, torch.bfloat16):
+ assert _bad(NONE, D0, _FakeCuda(1, 8, 704, 64, dtype = dtype, requires_grad = True), k)
+ q32 = _FakeCuda(1, 8, 704, 64, dtype = torch.float32, requires_grad = True)
+ assert _bad(NONE, D0, q32, k) is None
+
+
+class TestDevices:
+ def _mock(
+ self,
+ monkeypatch,
+ version,
+ cudnn,
+ caps,
+ hip = None,
+ ):
+ monkeypatch.setattr(torch, "__version__", version)
+ monkeypatch.setattr(torch.version, "hip", hip, raising = False)
+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
+ monkeypatch.setattr(torch.cuda, "device_count", lambda: len(caps))
+ monkeypatch.setattr(torch.cuda, "get_device_capability", lambda i = 0: caps[i])
+ monkeypatch.setattr(torch.backends.cudnn, "version", lambda: cudnn)
+
+ def test_mixed_host(self, monkeypatch):
+ self._mock(monkeypatch, "2.13.0+cu130", 92000, [(9, 0), (10, 0), (12, 0), (11, 0)])
+ assert fixes._sdpa_cudnn_known_bad_devices() == (frozenset({1, 3}), frozenset({0}))
+
+ def test_torch_214_h100(self, monkeypatch):
+ self._mock(monkeypatch, "2.14.0+cu130", 92400, [(9, 0), (10, 0)])
+ assert fixes._sdpa_cudnn_known_bad_devices() == (frozenset(), frozenset({0}))
+
+ def test_torch_215_is_clean(self, monkeypatch):
+ self._mock(monkeypatch, "2.15.0+cu132", 92600, [(9, 0), (10, 0)])
+ assert fixes._sdpa_cudnn_known_bad_devices() == (frozenset(), frozenset())
+
+ def test_rocm_is_clean(self, monkeypatch):
+ self._mock(monkeypatch, "2.13.0+rocm7.2", 92000, [(9, 0)], hip = "7.2")
+ assert fixes._sdpa_cudnn_known_bad_devices() == (frozenset(), frozenset())
+
+
+def recorder(monkeypatch):
+ import torch.nn.functional as F
+
+ calls = []
+
+ def fake_sdpa(
+ query,
+ key,
+ value,
+ attn_mask = None,
+ dropout_p = 0.0,
+ is_causal = False,
+ **kw,
+ ):
+ cuda = torch.backends.cuda
+ calls.append(
+ dict(
+ cudnn = cuda.cudnn_sdp_enabled(),
+ flash = cuda.flash_sdp_enabled(),
+ efficient = cuda.mem_efficient_sdp_enabled(),
+ math = cuda.math_sdp_enabled(),
+ mask = attn_mask,
+ is_causal = is_causal,
+ kw = kw,
+ )
+ )
+ return "out"
+
+ monkeypatch.setattr(F, "scaled_dot_product_attention", fake_sdpa)
+ monkeypatch.setattr(fixes, "_sdpa_cudnn_known_bad_devices", lambda: (D0, D0))
+ monkeypatch.delenv("UNSLOTH_ALLOW_BUGGY_CUDNN_SDPA", raising = False)
+ fixes.fix_cudnn_sdpa_known_bad_shapes()
+ assert F.scaled_dot_product_attention is not fake_sdpa
+ with sdpa_kernel(ALL_BACKENDS):
+ yield F, calls
+
+
+ALL_ON = dict(cudnn = True, flash = True, efficient = True, math = True)
+NO_CUDNN = dict(cudnn = False, flash = True, efficient = True, math = True)
+
+
+def _flags(call):
+ return {k: call[k] for k in ("cudnn", "flash", "efficient", "math")}
+
+
+class TestWrapper:
+ def test_trigger_calls_run_without_cudnn(self, recorder):
+ F, calls = recorder
+ mask = object()
+ assert F.scaled_dot_product_attention(
+ _FakeCuda(1, 28, 1, 128), _FakeCuda(1, 4, 300, 128), _FakeCuda(1, 4, 300, 128),
+ attn_mask = mask, enable_gqa = True,
+ ) == "out" # fmt: skip
+ q = _FakeCuda(2, 8, 704, 64, requires_grad = True)
+ F.scaled_dot_product_attention(q, _FakeCuda(2, 8, 704, 64), _FakeCuda(2, 8, 704, 64), mask)
+ assert [_flags(c) for c in calls] == [NO_CUDNN, NO_CUDNN]
+ assert calls[0]["mask"] is mask and calls[0]["kw"] == {"enable_gqa": True}
+ assert torch.backends.cuda.cudnn_sdp_enabled()
+
+ def test_other_calls_keep_cudnn(self, recorder):
+ F, calls = recorder
+ q = _FakeCuda(2, 8, 768, 64, requires_grad = True)
+ F.scaled_dot_product_attention(q, _FakeCuda(2, 8, 768, 64), _FakeCuda(2, 8, 768, 64))
+ q = _FakeCuda(2, 8, 704, 64, requires_grad = True)
+ F.scaled_dot_product_attention(q, q, q, None, 0.0, True) # causal
+ F.scaled_dot_product_attention(_FakeCuda(1, 8, 4, 64), _FakeCuda(1, 8, 9, 64), q) # prefill
+ assert [_flags(c) for c in calls] == [ALL_ON] * 3
+ assert calls[1]["is_causal"] is True
+
+ def test_user_policy_is_honoured(self, recorder):
+ F, calls = recorder
+ q = _FakeCuda(1, 8, 1, 64)
+ with sdpa_kernel([SDPBackend.MATH, SDPBackend.CUDNN_ATTENTION]):
+ F.scaled_dot_product_attention(q, q, q)
+ with sdpa_kernel([SDPBackend.CUDNN_ATTENTION]): # cuDNN-only: left alone
+ F.scaled_dot_product_attention(q, q, q)
+ with sdpa_kernel([SDPBackend.FLASH_ATTENTION]): # cuDNN already off
+ F.scaled_dot_product_attention(q, q, q)
+ assert [_flags(c) for c in calls] == [
+ dict(cudnn = False, flash = False, efficient = False, math = True),
+ dict(cudnn = True, flash = False, efficient = False, math = False),
+ dict(cudnn = False, flash = True, efficient = False, math = False),
+ ]
+
+ def test_idempotent(self, recorder):
+ F, _ = recorder
+ wrapped = F.scaled_dot_product_attention
+ fixes.fix_cudnn_sdpa_known_bad_shapes()
+ assert F.scaled_dot_product_attention is wrapped
+
+ def test_compiled_fullgraph(self, recorder, monkeypatch):
+ # Must trace without a graph break, with the backend switch around the real SDPA.
+ F, _ = recorder
+ monkeypatch.setattr(fixes, "_sdpa_cudnn_known_bad", lambda *a: "dq_nan")
+ real = torch._C._nn.scaled_dot_product_attention
+ graphs = []
+
+ def backend(gm, example_inputs):
+ graphs.append(gm.code)
+ return gm.forward
+
+ torch._dynamo.reset()
+ monkeypatch.setattr(F, "scaled_dot_product_attention", real)
+ fixes.fix_cudnn_sdpa_known_bad_shapes()
+ wrapped = F.scaled_dot_product_attention
+ assert wrapped is not real
+
+ def fn(q):
+ return wrapped(q, q, q)
+
+ q = torch.randn(1, 2, 4, 8)
+ out = torch.compile(fn, backend = backend, fullgraph = True)(q)
+ torch.testing.assert_close(out, real(q, q, q))
+ code = "\n".join(graphs)
+ assert "_sdpa_without_cudnn(" in code, code
+ torch._dynamo.reset()
+
+
+def test_cudnn_off_tests_the_key_length_value():
+ with sdpa_kernel(ALL_BACKENDS):
+ for length, off in ((192, True), (256, False), (None, True)):
+ switch = fixes._sdpa_cudnn_off(length)
+ assert switch is off
+ assert torch.backends.cuda.cudnn_sdp_enabled() is not off
+ fixes._sdpa_cudnn_restore(switch)
+ assert torch.backends.cuda.cudnn_sdp_enabled()
+ with sdpa_kernel([SDPBackend.CUDNN_ATTENTION]): # cuDNN-only: left alone
+ assert fixes._sdpa_cudnn_off(192) is False
+ assert torch.backends.cuda.cudnn_sdp_enabled()
+ fixes._sdpa_cudnn_restore(False)
+
+
+def cpu_dq_nan(monkeypatch):
+ """CPU SDPA as an affected H100: only the device check is mocked."""
+ import torch.nn.functional as F
+
+ length = getattr(fixes, "_sdpa_cudnn_dq_nan_length", lambda k: k.shape[-2] % 128 == 64)
+
+ def known_bad(decode, dq, query, key, is_causal):
+ hit = not is_causal and query.requires_grad and torch.is_grad_enabled() and length(key)
+ return fixes._SDPA_CUDNN_DQ_NAN if hit else None
+
+ real = torch._C._nn.scaled_dot_product_attention
+ monkeypatch.setattr(F, "scaled_dot_product_attention", real)
+ monkeypatch.setattr(fixes, "_sdpa_cudnn_known_bad_devices", lambda: (D0, D0))
+ monkeypatch.setattr(fixes, "_sdpa_cudnn_known_bad", known_bad)
+ monkeypatch.delenv("UNSLOTH_ALLOW_BUGGY_CUDNN_SDPA", raising = False)
+ fixes.fix_cudnn_sdpa_known_bad_shapes()
+ wrapped = F.scaled_dot_product_attention
+ assert wrapped is not real
+ torch._dynamo.reset()
+ yield wrapped
+ torch._dynamo.reset()
+
+
+def _qkv(s_kv):
+ q = torch.randn(1, 2, 8, 16, dtype = torch.float16, requires_grad = True)
+ k = torch.randn(1, 2, s_kv, 16, dtype = torch.float16, requires_grad = True)
+ return q, k, torch.randn_like(k, requires_grad = True)
+
+
[email protected]("strict", [False, True])
+def test_compiled_dynamic_key_length_is_not_specialized(cpu_dq_nan, strict):
+ # One graph for lengths on and off 64 mod 128; strict mark_dynamic must compile.
+ wrapped = cpu_dq_nan
+ graphs = []
+
+ def backend(gm, example_inputs):
+ graphs.append(gm)
+ return gm.forward
+
+ fn = torch.compile(lambda q, k, v: wrapped(q, k, v).float().sum(), backend = backend,
+ fullgraph = True, dynamic = True) # fmt: skip
+ with sdpa_kernel([SDPBackend.MATH, SDPBackend.CUDNN_ATTENTION]):
+ for s_kv in (192, 256, 320, 384, 448):
+ q, k, v = _qkv(s_kv)
+ if strict:
+ torch._dynamo.mark_dynamic(k, 2, min = 128, max = 512)
+ torch._dynamo.mark_dynamic(v, 2, min = 128, max = 512)
+ fn(q, k, v).backward()
+ assert len(graphs) == 1
+ names = ("_sdpa_without_cudnn(", "_sdpa_cudnn_off(", "_sdpa_kernel(")
+ assert any(name in graphs[0].code for name in names)
+
+
+def test_compiled_call_reads_policy_at_call_time(cpu_dq_nan):
+ # Compiled under one policy, called under another: the later policy decides and is kept.
+ wrapped = cpu_dq_nan
+ fn = torch.compile(lambda q, k, v: wrapped(q, k, v), backend = lambda gm, ex: gm.forward,
+ fullgraph = True) # fmt: skip
+ q, k, v = _qkv(192)
+ cuda = torch.backends.cuda
+ with sdpa_kernel(ALL_BACKENDS):
+ fn(q, k, v).sum().backward()
+ with sdpa_kernel([SDPBackend.MATH, SDPBackend.CUDNN_ATTENTION]):
+ fn(q, k, v)
+ assert (cuda.flash_sdp_enabled(), cuda.math_sdp_enabled(), cuda.cudnn_sdp_enabled()) == (
+ False, True, True,
+ ) # fmt: skip
+ with sdpa_kernel([SDPBackend.CUDNN_ATTENTION]): # cuDNN-only: left alone, as unwrapped SDPA
+ with pytest.raises(RuntimeError, match = "No viable backend"):
+ fn(q, k, v)
+ assert not cuda.math_sdp_enabled() and cuda.cudnn_sdp_enabled()
+
+
+_RAISING = []
+
+
+def _raising_op():
+ """An opaque op that raises at run time (as a CUDA OOM would), not while Dynamo traces it."""
+ if not _RAISING:
+
+ @torch.library.custom_op("unsloth_cudnn_sdpa_probe::boom", mutates_args = ())
+ def boom(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor) -> torch.Tensor:
+ raise RuntimeError("CUDA out of memory (simulated)")
+
+ @boom.register_fake
+ def _(q, k, v):
+ return torch.empty_like(q)
+
+ _RAISING.append(boom)
+ return _RAISING[0]
+
+
[email protected]("mode", ["plain", "eager_graph"])
+def test_raising_sdpa_restores_the_policy(cpu_dq_nan, monkeypatch, mode):
+ # A raising SDPA (e.g. OOM) still restores the flags, also from an FX graph.
+ import torch.nn.functional as F
+
+ boom = _raising_op()
+ monkeypatch.setattr(F, "scaled_dot_product_attention", lambda q, k, v, *a, **kw: boom(q, k, v))
+ fixes.fix_cudnn_sdpa_known_bad_shapes()
+ wrapped = F.scaled_dot_product_attention
+ torch._dynamo.reset()
+ fn = lambda q, k, v: wrapped(q, k, v) # noqa: E731
+ if mode == "eager_graph":
+ fn = torch.compile(fn, backend = "eager", fullgraph = True, dynamic = False)
+ depth = fixes._SDPA_CUDNN_HELD[0]
+ try:
+ with sdpa_kernel(CUDNN_MATH):
+ before = _sdp_flags()
+ for _ in range(2):
+ with pytest.raises(RuntimeError, match = "simulated"):
+ fn(*_qkv(192))
+ assert _sdp_flags() == before
+ assert fixes._SDPA_CUDNN_HELD[0] == depth
+ finally:
+ fixes._SDPA_CUDNN_HELD[0] = depth
+ torch._dynamo.reset()
+
+
+_PROBE = []
+
+
+def _probe_op():
+ """Like aten SDPA, picks its kernel from the live cuDNN flag when decomposed."""
+ if not _PROBE:
+ lib = torch.library.Library("unsloth_cudnn_sdpa_probe", "FRAGMENT")
+ lib.define("probe(Tensor q, Tensor k, Tensor v) -> Tensor")
+ lib.impl(
+ "probe",
+ lambda q, k, v: q * (2.0 if torch._C._get_cudnn_sdp_enabled() else 3.0),
+ "CompositeImplicitAutograd",
+ )
+ _PROBE.append((lib, torch.ops.unsloth_cudnn_sdpa_probe.probe))
+ return _PROBE[0][1]
+
+
+def probe_dq_nan(cpu_dq_nan, monkeypatch):
+ """cpu_dq_nan over the probe op: an output of 2 * q ran the cuDNN kernel, 3 * q a fallback."""
+ import torch.nn.functional as F
+
+ probe = _probe_op()
+ monkeypatch.setattr(F, "scaled_dot_product_attention", lambda q, k, v, *a, **kw: probe(q, k, v))
+ fixes.fix_cudnn_sdpa_known_bad_shapes()
+ torch._dynamo.reset()
+ yield F.scaled_dot_product_attention
+ torch._dynamo.reset()
+
+
+CUDNN_ONLY = [SDPBackend.CUDNN_ATTENTION]
+CUDNN_MATH = [SDPBackend.MATH, SDPBackend.CUDNN_ATTENTION]
+
+
+def _sdp_flags():
+ cuda = torch.backends.cuda
+ return (cuda.cudnn_sdp_enabled(), cuda.flash_sdp_enabled(), cuda.mem_efficient_sdp_enabled(),
+ cuda.math_sdp_enabled()) # fmt: skip
+
+
+def cudnn_sdp_on():
+ """Start from torch's default, cuDNN SDPA on: importing vLLM turns it off process-wide."""
+ was = torch.backends.cuda.cudnn_sdp_enabled()
+ torch.backends.cuda.enable_cudnn_sdp(True)
+ yield
+ torch.backends.cuda.enable_cudnn_sdp(was)
+
+
+_INDUCTOR = pytest.param(
+ "inductor",
+ marks = pytest.mark.skipif(
+ sys.platform == "win32" and shutil.which("cl") is None,
+ reason = "CPU inductor needs MSVC on Windows",
+ ),
+)
+
+
[email protected]("backend", ["aot_eager", _INDUCTOR])
[email protected]("first", ["cudnn_only", "fallback"])
+def test_aot_compiled_call_follows_a_later_policy(cudnn_sdp_on, probe_dq_nan, backend, first):
+ # AOTAutograd fixes the backend at trace time: a later policy must not keep the old kernel.
+ wrapped = probe_dq_nan
+ fn = torch.compile(lambda q, k, v: wrapped(q, k, v), backend = backend, fullgraph = True,
+ dynamic = False) # fmt: skip
+ policies = [CUDNN_ONLY, CUDNN_MATH] if first == "cudnn_only" else [CUDNN_MATH, CUDNN_ONLY]
+ for s_kv, hit in ((192, True), (256, False)):
+ q, k, v = _qkv(s_kv)
+ for policy in policies * 2:
+ with sdpa_kernel(policy):
+ before = _sdp_flags()
+ ratio = (fn(q, k, v) / q).detach().flatten()[0].item()
+ expected = 3.0 if hit and policy is CUDNN_MATH else 2.0
+ assert ratio == pytest.approx(expected, rel = 0.01), (s_kv, policy) # fp16 2q / 3q
+ eager = (wrapped(q, k, v) / q).detach().flatten()[0].item()
+ assert eager == pytest.approx(expected, rel = 0.01)
+ assert _sdp_flags() == before
+ assert torch.backends.cuda.cudnn_sdp_enabled()
+
+
+def test_opt_out_and_unaffected_host(monkeypatch):
+ import torch.nn.functional as F
+
+ sentinel = lambda *a, **k: None # noqa: E731
+ monkeypatch.setattr(F, "scaled_dot_product_attention", sentinel)
+ monkeypatch.setattr(fixes, "_sdpa_cudnn_known_bad_devices", lambda: (D0, D0))
+ monkeypatch.setenv("UNSLOTH_ALLOW_BUGGY_CUDNN_SDPA", "1")
+ fixes.fix_cudnn_sdpa_known_bad_shapes()
+ assert F.scaled_dot_product_attention is sentinel
+ monkeypatch.delenv("UNSLOTH_ALLOW_BUGGY_CUDNN_SDPA")
+ monkeypatch.setattr(fixes, "_sdpa_cudnn_known_bad_devices", lambda: (NONE, NONE))
+ fixes.fix_cudnn_sdpa_known_bad_shapes()
+ assert F.scaled_dot_product_attention is sentinel
+
+
+def _held_count():
+ return fixes._SDPA_CUDNN_HELD[0]
+
+
+def _cell(wrapper, name):
+ return wrapper.__closure__[wrapper.__code__.co_freevars.index(name)]
+
+
+def test_overlapping_calls_keep_cudnn_off(recorder):
+ # A call starting while another holds cuDNN off still runs without cuDNN after the first restores.
+ import threading
+
+ F, calls = recorder
+ a_inside, b_inside, a_done = threading.Event(), threading.Event(), threading.Event()
+ seen = {}
+ token = _cell(F.scaled_dot_product_attention, "token").cell_contents
+ inner = fixes._SDPA_CUDNN_ORIGINALS[token]
+
+ def gated(query, key, value, *args, **kwargs):
+ if threading.current_thread().name == "A":
+ seen["A"] = torch.backends.cuda.cudnn_sdp_enabled()
+ a_inside.set()
+ b_inside.wait(5)
+ else:
+ b_inside.set()
+ a_done.wait(5) # backend selection after A restored its flag
+ seen["B"] = torch.backends.cuda.cudnn_sdp_enabled()
+ return inner(query, key, value, *args, **kwargs)
+
+ fixes._SDPA_CUDNN_ORIGINALS[token] = gated
+ wrapper = F.scaled_dot_product_attention
+ original_cell = _cell(wrapper, "original")
+ saved = original_cell.cell_contents
+ original_cell.cell_contents = gated
+ try:
+ q, k = _FakeCuda(1, 8, 1, 64), _FakeCuda(1, 8, 9, 64)
+
+ def run_a():
+ F.scaled_dot_product_attention(q, k, k)
+ a_done.set()
+
+ def run_b():
+ a_inside.wait(5)
+ F.scaled_dot_product_attention(q, k, k)
+
+ threads = [threading.Thread(target = run_a, name = "A"),
+ threading.Thread(target = run_b, name = "B")] # fmt: skip
+ for t in threads:
+ t.start()
+ for t in threads:
+ t.join(10)
+ assert seen == {"A": False, "B": False}
+ assert torch.backends.cuda.cudnn_sdp_enabled()
+ assert _held_count() == 0
+ finally:
+ original_cell.cell_contents = saved
+ fixes._SDPA_CUDNN_ORIGINALS[token] = inner
+
+
+def test_many_threads_never_see_cudnn(recorder):
+ # 8 threads: every matching call runs with cuDNN off; flag back on, nothing held, at the end.
+ import threading
+ import time
+
+ F, calls = recorder
+ token = _cell(F.scaled_dot_product_attention, "token").cell_contents
+ inner = fixes._SDPA_CUDNN_ORIGINALS[token]
+ wrapper = F.scaled_dot_product_attention
+ original_cell = _cell(wrapper, "original")
+ saved = original_cell.cell_contents
+ bad = []
+
+ def slow(query, key, value, *args, **kwargs):
+ time.sleep(0.0002) # yields the GIL, as a real kernel launch does
+ if torch.backends.cuda.cudnn_sdp_enabled():
+ bad.append(threading.current_thread().name)
+ return "out"
+
+ fixes._SDPA_CUDNN_ORIGINALS[token] = slow
+ original_cell.cell_contents = slow
+ try:
+ q, k = _FakeCuda(1, 8, 1, 64), _FakeCuda(1, 8, 9, 64)
+
+ def worker():
+ for _ in range(200):
+ F.scaled_dot_product_attention(q, k, k)
+
+ threads = [threading.Thread(target = worker, name = f"t{i}") for i in range(8)]
+ for t in threads:
+ t.start()
+ for t in threads:
+ t.join(60)
+ assert not bad, f"{len(bad)} matching calls ran with cuDNN on"
+ assert torch.backends.cuda.cudnn_sdp_enabled()
+ assert _held_count() == 0
+ finally:
+ original_cell.cell_contents = saved
+ fixes._SDPA_CUDNN_ORIGINALS[token] = inner
diff --git a/unsloth/_gpu_init.py b/unsloth/_gpu_init.py
index 4d22588c5de..20de3236175 100644
--- a/unsloth/_gpu_init.py
+++ b/unsloth/_gpu_init.py
@@ -371,6 +371,7 @@ from .import_fixes import (
fix_vllm_pdl_blackwell,
fix_cudnn_sdpa_d256_masked_backward,
fix_cudnn_sdpa_backward_grad_layout,
+ fix_cudnn_sdpa_known_bad_shapes,
fix_rocm_windows_fused_sdpa,
fix_triton_compiled_kernel_missing_attrs,
fix_dynamo_config_thread_visibility,
@@ -462,6 +463,8 @@ fix_trl_vllm_ascend()
fix_vllm_pdl_blackwell()
fix_cudnn_sdpa_d256_masked_backward()
fix_cudnn_sdpa_backward_grad_layout()
+# torch 2.11 - 2.14 cuDNN SDPA bugs fixed by cuDNN 9.26 / torch 2.15 (pytorch#193893, #196678).
+fix_cudnn_sdpa_known_bad_shapes()
# Windows ROCm only, probe-gated: fused attention fails on every call there (gfx1151, torch 2.11).
fix_rocm_windows_fused_sdpa()
fix_triton_compiled_kernel_missing_attrs()
@@ -530,6 +533,7 @@ del fix_trl_vllm_ascend
del fix_vllm_pdl_blackwell
del fix_cudnn_sdpa_d256_masked_backward
del fix_cudnn_sdpa_backward_grad_layout
+del fix_cudnn_sdpa_known_bad_shapes
del fix_rocm_windows_fused_sdpa
del fix_triton_compiled_kernel_missing_attrs
del fix_dynamo_config_thread_visibility
diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.py
index c391ce2d34c..367e972cea5 100644
--- a/unsloth/import_fixes.py
+++ b/unsloth/import_fixes.py
@@ -7900,6 +7900,228 @@ def fix_cudnn_sdpa_backward_grad_layout():
)
+_SDPA_CUDNN_KNOWN_BAD_FLAG = "_unsloth_avoids_cudnn_sdpa_known_bad_shapes"
+_SDPA_CUDNN_KNOWN_BAD_WARNED = set()
+# Backend flags are process-wide: refcount wrapped calls in flight so overlapping calls never see cuDNN back on early.
+_SDPA_CUDNN_HELD = [0]
+_SDPA_CUDNN_LOCK = threading.Lock()
+_SDPA_CUDNN_DECODE = "decode"
+_SDPA_CUDNN_DQ_NAN = "dq_nan"
+
+
+def _torch_release(torch_version):
+ try:
+ return TrueVersion(TrueVersion(str(torch_version)).base_version)
+ except Exception:
+ return None
+
+
+def _cudnn_sdpa_decode_corrupts(torch_version, cudnn_version, capability):
+ """pytorch#193893: cuDNN 9.19 - 9.25.0 (not 9.24.1) one-row decode on SM 10.x / 11.x is wrong.
+ torch 2.14 refuses it (pytorch#194963); <= 2.13 cu128+ wheels ship 9.19 / 9.20."""
+ release = _torch_release(torch_version)
+ if release is None or release >= TrueVersion("2.14") or cudnn_version is None:
+ return False
+ cudnn_version = int(cudnn_version)
+ if cudnn_version < 91900 or cudnn_version > 92500 or 92400 < cudnn_version < 92500:
+ return False
+ return int(capability[0]) in (10, 11)
+
+
+def _cudnn_sdpa_dq_nan(torch_version, cudnn_version, capability):
+ """pytorch#196678: cuDNN < 9.26 on SM90 gives NaN dQ for non-causal s_kv % 128 == 64.
+ torch 2.15 refuses it (pytorch#196861)."""
+ release = _torch_release(torch_version)
+ if release is None or release >= TrueVersion("2.15") or cudnn_version is None:
+ return False
+ return int(cudnn_version) < 92600 and tuple(int(x) for x in capability[:2]) == (9, 0)
+
+
+def _sdpa_cudnn_known_bad_devices():
+ try:
+ import torch
+
+ if not torch.cuda.is_available() or getattr(torch.version, "hip", None):
+ return frozenset(), frozenset()
+ cudnn_version = torch.backends.cudnn.version()
+ capabilities = [
+ torch.cuda.get_device_capability(i) for i in range(torch.cuda.device_count())
+ ]
+ except Exception:
+ return frozenset(), frozenset()
+ decode = frozenset(
+ i
+ for i, cap in enumerate(capabilities)
+ if _cudnn_sdpa_decode_corrupts(torch.__version__, cudnn_version, cap)
+ )
+ dq_nan = frozenset(
+ i
+ for i, cap in enumerate(capabilities)
+ if _cudnn_sdpa_dq_nan(torch.__version__, cudnn_version, cap)
+ )
+ return decode, dq_nan
+
+
+def _sdpa_cudnn_dq_nan_length(key):
+ # Compiling: _sdpa_cudnn_off tests the value, so a dynamic length adds no guard.
+ import torch
+ return torch.compiler.is_compiling() or key.shape[-2] % 128 == 64
+
+
+def _sdpa_cudnn_known_bad(decode_devices, dq_devices, query, key, is_causal):
+ """The known cuDNN bug this call would hit, mirroring torch's later guards; None if none."""
+ import torch
+
+ if not getattr(query, "is_cuda", False) or getattr(query, "is_nested", False):
+ return None
+ index = query.device.index
+ if index not in decode_devices and index not in dq_devices:
+ return None
+ dtype = query.dtype
+ # Autocast casts fp32 SDPA inputs to half before dispatch.
+ if dtype == torch.float32 and torch.is_autocast_enabled("cuda"):
+ dtype = torch.get_autocast_dtype("cuda")
+ if dtype not in (torch.float16, torch.bfloat16) or query.dim() < 2:
+ return None
+ if index in decode_devices and query.shape[-2] == 1:
+ return _SDPA_CUDNN_DECODE
+ if (
+ index in dq_devices
+ and not is_causal
+ and query.requires_grad
+ and torch.is_grad_enabled()
+ and getattr(key, "dim", lambda: 0)() >= 2
+ and _sdpa_cudnn_dq_nan_length(key)
+ ):
+ return _SDPA_CUDNN_DQ_NAN
+ return None
+
+
+class _SdpaCudnnPolicy(types.ModuleType):
+ """Dynamo folds SDPA backend flags without a guard, but guards a module attribute by value:
+ reading the policy here makes a compiled graph recompile when the policy flips its backend."""
+
+ @property
+ def cudnn_off(self):
+ import torch
+ cuda = torch.backends.cuda
+ # A held switch counts as on, so another thread's matching call also takes it.
+ return (cuda.cudnn_sdp_enabled() or _SDPA_CUDNN_HELD[0] > 0) and (
+ cuda.flash_sdp_enabled() or cuda.mem_efficient_sdp_enabled() or cuda.math_sdp_enabled()
+ )
+
+
+_SDPA_CUDNN_POLICY = _SdpaCudnnPolicy("unsloth_sdpa_cudnn_policy")
+
+
+def _sdpa_cudnn_off(key_length = None):
+ """Turn cuDNN SDPA off unless it is the only backend; a symbolic key_length counts as a hit.
+ The count rises before the flag drops, so an unlocked policy read never sees cuDNN off."""
+ import torch
+
+ if not (key_length is None or isinstance(key_length, torch.SymInt) or key_length % 128 == 64):
+ return False
+ with _SDPA_CUDNN_LOCK:
+ if _SDPA_CUDNN_HELD[0] == 0 and not _SDPA_CUDNN_POLICY.cudnn_off:
+ return False
+ _SDPA_CUDNN_HELD[0] += 1
+ if _SDPA_CUDNN_HELD[0] == 1:
+ torch._C._set_sdp_use_cudnn(False)
+ return True
+
+
+def _sdpa_cudnn_restore(switch):
+ import torch
+ if not switch:
+ return
+ with _SDPA_CUDNN_LOCK:
+ if _SDPA_CUDNN_HELD[0] == 1:
+ torch._C._set_sdp_use_cudnn(True)
+ _SDPA_CUDNN_HELD[0] -= 1
+
+
+_SDPA_CUDNN_ORIGINALS = {} # str token -> wrapped SDPA (a graph constant, unlike a function)
+_SDPA_CUDNN_MESSAGES = {
+ _SDPA_CUDNN_DECODE: "single-token decode on this Blackwell GPU (cuDNN 9.19 - 9.25.0 "
+ "returns wrong attention there, pytorch#193893)",
+ _SDPA_CUDNN_DQ_NAN: "training with a key length of 64 mod 128 on this H100 / H200 (cuDNN "
+ "< 9.26 returns NaN query gradients there, pytorch#196678)",
+}
+
+
+def _sdpa_without_cudnn(token, bug, key_length, query, key, value, *args, **kwargs):
+ """allow_in_graph: one eager-graph node whose finally runs even if SDPA raises; AOTAutograd
+ traces through it, so the traced SDPA avoids cuDNN."""
+ import torch
+
+ switch = _sdpa_cudnn_off(key_length)
+ try:
+ if switch and not torch.compiler.is_compiling() and bug not in _SDPA_CUDNN_KNOWN_BAD_WARNED:
+ _SDPA_CUDNN_KNOWN_BAD_WARNED.add(bug)
+ logger.warning(
+ f"Unsloth: running SDPA without cuDNN attention for {_SDPA_CUDNN_MESSAGES[bug]}. "
+ "Upgrade to torch 2.15 / cuDNN 9.26, or set UNSLOTH_ALLOW_BUGGY_CUDNN_SDPA=1."
+ )
+ return _SDPA_CUDNN_ORIGINALS[token](query, key, value, *args, **kwargs)
+ finally:
+ _sdpa_cudnn_restore(switch)
+
+
+def fix_cudnn_sdpa_known_bad_shapes():
+ """Keep SDPA off cuDNN for the calls torch 2.14 / 2.15 later refuse (pytorch#193893, #196678),
+ only when a visible device is affected and another backend is enabled.
+ UNSLOTH_ALLOW_BUGGY_CUDNN_SDPA=1 opts out."""
+ if os.environ.get("UNSLOTH_ALLOW_BUGGY_CUDNN_SDPA", "0") == "1":
+ return
+ decode_devices, dq_devices = _sdpa_cudnn_known_bad_devices()
+ if not decode_devices and not dq_devices:
+ return
+ try:
+ import torch
+ import torch._dynamo
+ import torch.nn.functional as F
+ except Exception as e:
+ logger.info(f"Unsloth: Skipping the cuDNN SDPA known-bad-shape fix ({e})")
+ return
+ if getattr(F.scaled_dot_product_attention, _SDPA_CUDNN_KNOWN_BAD_FLAG, False):
+ return
+
+ # Switch, SDPA and restore stay one graph call: a trailing restore node would be skipped on a raise.
+ torch._dynamo.allow_in_graph(_sdpa_without_cudnn)
+ original = F.scaled_dot_product_attention
+ token = f"sdpa_{len(_SDPA_CUDNN_ORIGINALS)}" # a str: Dynamo may make an int symbolic
+ _SDPA_CUDNN_ORIGINALS[token] = original
+
+ @functools.wraps(original)
+ def scaled_dot_product_attention(
+ query,
+ key,
+ value,
+ attn_mask = None,
+ dropout_p = 0.0,
+ is_causal = False,
+ *args,
+ **kwargs,
+ ):
+ bug = _sdpa_cudnn_known_bad(decode_devices, dq_devices, query, key, is_causal)
+ # Compiled, this policy read is a guard.
+ if bug is not None and _SDPA_CUDNN_POLICY.cudnn_off:
+ key_length = key.shape[-2] if bug == _SDPA_CUDNN_DQ_NAN else None
+ return _sdpa_without_cudnn(
+ token, bug, key_length, query, key, value, attn_mask, dropout_p, is_causal,
+ *args, **kwargs,
+ ) # fmt: skip
+ return original(query, key, value, attn_mask, dropout_p, is_causal, *args, **kwargs)
+
+ scaled_dot_product_attention.__wrapped__ = original
+ setattr(scaled_dot_product_attention, _SDPA_CUDNN_KNOWN_BAD_FLAG, True)
+ F.scaled_dot_product_attention = scaled_dot_product_attention
+ logger.info(
+ "Unsloth: cuDNN SDPA known-bad shapes avoided "
+ f"(decode devices {sorted(decode_devices)}, NaN dQ devices {sorted(dq_devices)})"
+ )
+
+
def _rocm_windows_broken_sdpa_backends():
"""Fused SDPA backends ("flash", "mem_efficient") failing on this process's Windows ROCm GPU.