Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
cuDNN SDPA: keep known-bad decode and NaN-dQ shapes off cuDNN before torch 2.14 / 2.15 (#15934)
Daniel Han8161e8188d9b
diff --git a/tests/test_cudnn_sdpa_known_bad_shapes.py b/tests/test_cudnn_sdpa_known_bad_shapes.pynew file mode 100644index 00000000000..e9e80dd1223--- /dev/null+++ b/tests/test_cudnn_sdpa_known_bad_shapes.py@@ -0,0 +1,640 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""SDPA stays off cuDNN for the calls torch later refused (pytorch#193893, #196678).+CPU only: versions and capability are mocked; the wrapper is checked via the backend flags it runs under."""++import shutil+import sys+import types++import pytest++torch = pytest.importorskip("torch")+from torch.nn.attention import SDPBackend, sdpa_kernel # noqa: E402++import unsloth.import_fixes as fixes # noqa: E402+++ "torch_version, cudnn, cap, expected",+ [+ ("2.13.0+cu130", 92000, (10, 0), True), # the issue's B200 setup on 2.13+ ("2.11.0+cu128", 91900, (10, 0), True), # the original report+ ("2.12.1+cu130", 92000, (10, 3), True), # GB300 / B300+ ("2.13.0+cu130", 92000, (11, 0), True), # Thor+ ("2.13.0+cu130", 92500, (10, 0), True), # 9.25.0 still broken+ ("2.13.0+cu130", 92400, (10, 0), True),+ ("2.13.0+cu130", 92401, (10, 0), False), # 9.24.1 fixed+ ("2.13.0+cu130", 92501, (10, 0), False), # 9.25.1 fixed+ ("2.13.0+cu130", 92600, (10, 0), False),+ ("2.13.0+cu126", 91002, (10, 0), False), # cu126 wheels ship cuDNN 9.10.2+ ("2.13.0+cu130", 91800, (10, 0), False),+ ("2.13.0+cu130", 92000, (12, 0), False), # sm_120 is clean+ ("2.13.0+cu130", 92000, (9, 0), False),+ ("2.14.0+cu130", 92400, (10, 0), False), # torch 2.14 refuses it itself+ ("2.14.1+cu130", 92400, (10, 0), False),+ ("2.15.0+cu132", 92600, (10, 0), False),+ ("2.13.0+cu130", None, (10, 0), False),+ ("not a version", 92000, (10, 0), False),+ ],+)+def test_decode_gate(torch_version, cudnn, cap, expected):+ assert fixes._cudnn_sdpa_decode_corrupts(torch_version, cudnn, cap) is expected+++ "torch_version, cudnn, cap, expected",+ [+ ("2.14.1+cu130", 92400, (9, 0), True), # what pip install unsloth gets on H100+ ("2.14.0+cu130", 92400, (9, 0), True),+ ("2.13.0+cu130", 92000, (9, 0), True), # the issue's version+ ("2.11.0+cu128", 91900, (9, 0), True),+ ("2.14.0+cu130", 92501, (9, 0), True), # 9.25.1 still broken (issue comment)+ ("2.14.0+cu126", 91002, (9, 0), True), # torch's own guard is any cuDNN < 9.26+ ("2.14.0+cu130", 92600, (9, 0), False), # cuDNN 9.26 fixed+ ("2.15.0+cu132", 92600, (9, 0), False),+ ("2.15.0+cu132", 92400, (9, 0), False), # torch 2.15 refuses it itself+ ("2.14.0+cu130", 92400, (10, 0), False),+ ("2.14.0+cu130", 92400, (8, 0), False),+ ("2.14.0+cu130", 92400, (8, 9), False),+ ("2.14.0+cu130", 92400, (12, 0), False),+ ("2.14.0+cu130", None, (9, 0), False),+ ],+)+def test_dq_nan_gate(torch_version, cudnn, cap, expected):+ assert fixes._cudnn_sdpa_dq_nan(torch_version, cudnn, cap) is expected+++class _FakeCuda:+ def __init__(+ self,+ *shape,+ dtype = None,+ requires_grad = False,+ index = 0,+ ):+ self.shape = torch.Size(shape)+ self.dtype = dtype or torch.bfloat16+ self.requires_grad = requires_grad+ self.is_cuda = True+ self.is_nested = False+ self.device = types.SimpleNamespace(type = "cuda", index = index)++ def dim(self):+ return len(self.shape)+++D0 = frozenset({0})+ALL_BACKENDS = [+ SDPBackend.FLASH_ATTENTION,+ SDPBackend.EFFICIENT_ATTENTION,+ SDPBackend.MATH,+ SDPBackend.CUDNN_ATTENTION,+]+NONE = frozenset()+++def _bad(+ decode,+ dq,+ q,+ k,+ is_causal = False,+):+ return fixes._sdpa_cudnn_known_bad(decode, dq, q, k, is_causal)+++class TestPerCallDecision:+ def test_decode_one_query_row(self):+ q, k = _FakeCuda(2, 28, 1, 128), _FakeCuda(2, 4, 257, 128)+ assert _bad(D0, NONE, q, k) == "decode"+ assert _bad(D0, NONE, q, k, is_causal = True) == "decode"++ def test_decode_only_one_row(self):+ assert _bad(D0, NONE, _FakeCuda(2, 28, 2, 128), _FakeCuda(2, 4, 257, 128)) is None+ assert _bad(D0, NONE, _FakeCuda(1, 28, 512, 128), _FakeCuda(1, 4, 512, 128)) is None++ def test_decode_other_device_or_cpu(self):+ q, k = _FakeCuda(1, 8, 1, 64, index = 1), _FakeCuda(1, 8, 9, 64, index = 1)+ assert _bad(D0, NONE, q, k) is None+ cpu = torch.zeros(1, 8, 1, 64, dtype = torch.bfloat16)+ assert _bad(frozenset({0, None}), frozenset({0, None}), cpu, cpu) is None++ def test_decode_fp32_only_under_autocast(self):+ q, k = _FakeCuda(1, 8, 1, 64, dtype = torch.float32), _FakeCuda(1, 8, 9, 64)+ assert _bad(D0, NONE, q, k) is None++ @pytest.mark.parametrize("s_kv", [64, 192, 704, 2112])+ def test_dq_nan_lengths(self, s_kv):+ q = _FakeCuda(1, 8, s_kv, 64, requires_grad = True)+ k = _FakeCuda(1, 8, s_kv, 64)+ assert _bad(NONE, D0, q, k) == "dq_nan"++ @pytest.mark.parametrize("s_kv", [63, 65, 128, 640, 768, 1024])+ def test_dq_nan_other_lengths_keep_cudnn(self, s_kv):+ q = _FakeCuda(1, 8, s_kv, 64, requires_grad = True)+ assert _bad(NONE, D0, q, _FakeCuda(1, 8, s_kv, 64)) is None++ def test_dq_nan_needs_noncausal_grad(self):+ k = _FakeCuda(1, 8, 704, 64)+ assert (+ _bad(NONE, D0, _FakeCuda(1, 8, 704, 64, requires_grad = True), k, is_causal = True) is None+ )+ assert _bad(NONE, D0, _FakeCuda(1, 8, 704, 64), k) is None+ with torch.no_grad():+ assert _bad(NONE, D0, _FakeCuda(1, 8, 704, 64, requires_grad = True), k) is None++ def test_dq_nan_reads_key_length(self):+ # Upstream gates on s_k: a 1-row training query against 704 keys is caught, 640 is not.+ q = _FakeCuda(1, 8, 100, 64, requires_grad = True)+ assert _bad(NONE, D0, q, _FakeCuda(1, 8, 704, 64)) == "dq_nan"+ assert _bad(NONE, D0, q, _FakeCuda(1, 8, 640, 64)) is None++ def test_dq_nan_dtype(self):+ k = _FakeCuda(1, 8, 704, 64)+ for dtype in (torch.float16, torch.bfloat16):+ assert _bad(NONE, D0, _FakeCuda(1, 8, 704, 64, dtype = dtype, requires_grad = True), k)+ q32 = _FakeCuda(1, 8, 704, 64, dtype = torch.float32, requires_grad = True)+ assert _bad(NONE, D0, q32, k) is None+++class TestDevices:+ def _mock(+ self,+ monkeypatch,+ version,+ cudnn,+ caps,+ hip = None,+ ):+ monkeypatch.setattr(torch, "__version__", version)+ monkeypatch.setattr(torch.version, "hip", hip, raising = False)+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)+ monkeypatch.setattr(torch.cuda, "device_count", lambda: len(caps))+ monkeypatch.setattr(torch.cuda, "get_device_capability", lambda i = 0: caps[i])+ monkeypatch.setattr(torch.backends.cudnn, "version", lambda: cudnn)++ def test_mixed_host(self, monkeypatch):+ self._mock(monkeypatch, "2.13.0+cu130", 92000, [(9, 0), (10, 0), (12, 0), (11, 0)])+ assert fixes._sdpa_cudnn_known_bad_devices() == (frozenset({1, 3}), frozenset({0}))++ def test_torch_214_h100(self, monkeypatch):+ self._mock(monkeypatch, "2.14.0+cu130", 92400, [(9, 0), (10, 0)])+ assert fixes._sdpa_cudnn_known_bad_devices() == (frozenset(), frozenset({0}))++ def test_torch_215_is_clean(self, monkeypatch):+ self._mock(monkeypatch, "2.15.0+cu132", 92600, [(9, 0), (10, 0)])+ assert fixes._sdpa_cudnn_known_bad_devices() == (frozenset(), frozenset())++ def test_rocm_is_clean(self, monkeypatch):+ self._mock(monkeypatch, "2.13.0+rocm7.2", 92000, [(9, 0)], hip = "7.2")+ assert fixes._sdpa_cudnn_known_bad_devices() == (frozenset(), frozenset())+++def recorder(monkeypatch):+ import torch.nn.functional as F++ calls = []++ def fake_sdpa(+ query,+ key,+ value,+ attn_mask = None,+ dropout_p = 0.0,+ is_causal = False,+ **kw,+ ):+ cuda = torch.backends.cuda+ calls.append(+ dict(+ cudnn = cuda.cudnn_sdp_enabled(),+ flash = cuda.flash_sdp_enabled(),+ efficient = cuda.mem_efficient_sdp_enabled(),+ math = cuda.math_sdp_enabled(),+ mask = attn_mask,+ is_causal = is_causal,+ kw = kw,+ )+ )+ return "out"++ monkeypatch.setattr(F, "scaled_dot_product_attention", fake_sdpa)+ monkeypatch.setattr(fixes, "_sdpa_cudnn_known_bad_devices", lambda: (D0, D0))+ monkeypatch.delenv("UNSLOTH_ALLOW_BUGGY_CUDNN_SDPA", raising = False)+ fixes.fix_cudnn_sdpa_known_bad_shapes()+ assert F.scaled_dot_product_attention is not fake_sdpa+ with sdpa_kernel(ALL_BACKENDS):+ yield F, calls+++ALL_ON = dict(cudnn = True, flash = True, efficient = True, math = True)+NO_CUDNN = dict(cudnn = False, flash = True, efficient = True, math = True)+++def _flags(call):+ return {k: call[k] for k in ("cudnn", "flash", "efficient", "math")}+++class TestWrapper:+ def test_trigger_calls_run_without_cudnn(self, recorder):+ F, calls = recorder+ mask = object()+ assert F.scaled_dot_product_attention(+ _FakeCuda(1, 28, 1, 128), _FakeCuda(1, 4, 300, 128), _FakeCuda(1, 4, 300, 128),+ attn_mask = mask, enable_gqa = True,+ ) == "out" # fmt: skip+ q = _FakeCuda(2, 8, 704, 64, requires_grad = True)+ F.scaled_dot_product_attention(q, _FakeCuda(2, 8, 704, 64), _FakeCuda(2, 8, 704, 64), mask)+ assert [_flags(c) for c in calls] == [NO_CUDNN, NO_CUDNN]+ assert calls[0]["mask"] is mask and calls[0]["kw"] == {"enable_gqa": True}+ assert torch.backends.cuda.cudnn_sdp_enabled()++ def test_other_calls_keep_cudnn(self, recorder):+ F, calls = recorder+ q = _FakeCuda(2, 8, 768, 64, requires_grad = True)+ F.scaled_dot_product_attention(q, _FakeCuda(2, 8, 768, 64), _FakeCuda(2, 8, 768, 64))+ q = _FakeCuda(2, 8, 704, 64, requires_grad = True)+ F.scaled_dot_product_attention(q, q, q, None, 0.0, True) # causal+ F.scaled_dot_product_attention(_FakeCuda(1, 8, 4, 64), _FakeCuda(1, 8, 9, 64), q) # prefill+ assert [_flags(c) for c in calls] == [ALL_ON] * 3+ assert calls[1]["is_causal"] is True++ def test_user_policy_is_honoured(self, recorder):+ F, calls = recorder+ q = _FakeCuda(1, 8, 1, 64)+ with sdpa_kernel([SDPBackend.MATH, SDPBackend.CUDNN_ATTENTION]):+ F.scaled_dot_product_attention(q, q, q)+ with sdpa_kernel([SDPBackend.CUDNN_ATTENTION]): # cuDNN-only: left alone+ F.scaled_dot_product_attention(q, q, q)+ with sdpa_kernel([SDPBackend.FLASH_ATTENTION]): # cuDNN already off+ F.scaled_dot_product_attention(q, q, q)+ assert [_flags(c) for c in calls] == [+ dict(cudnn = False, flash = False, efficient = False, math = True),+ dict(cudnn = True, flash = False, efficient = False, math = False),+ dict(cudnn = False, flash = True, efficient = False, math = False),+ ]++ def test_idempotent(self, recorder):+ F, _ = recorder+ wrapped = F.scaled_dot_product_attention+ fixes.fix_cudnn_sdpa_known_bad_shapes()+ assert F.scaled_dot_product_attention is wrapped++ def test_compiled_fullgraph(self, recorder, monkeypatch):+ # Must trace without a graph break, with the backend switch around the real SDPA.+ F, _ = recorder+ monkeypatch.setattr(fixes, "_sdpa_cudnn_known_bad", lambda *a: "dq_nan")+ real = torch._C._nn.scaled_dot_product_attention+ graphs = []++ def backend(gm, example_inputs):+ graphs.append(gm.code)+ return gm.forward++ torch._dynamo.reset()+ monkeypatch.setattr(F, "scaled_dot_product_attention", real)+ fixes.fix_cudnn_sdpa_known_bad_shapes()+ wrapped = F.scaled_dot_product_attention+ assert wrapped is not real++ def fn(q):+ return wrapped(q, q, q)++ q = torch.randn(1, 2, 4, 8)+ out = torch.compile(fn, backend = backend, fullgraph = True)(q)+ torch.testing.assert_close(out, real(q, q, q))+ code = "\n".join(graphs)+ assert "_sdpa_without_cudnn(" in code, code+ torch._dynamo.reset()+++def test_cudnn_off_tests_the_key_length_value():+ with sdpa_kernel(ALL_BACKENDS):+ for length, off in ((192, True), (256, False), (None, True)):+ switch = fixes._sdpa_cudnn_off(length)+ assert switch is off+ assert torch.backends.cuda.cudnn_sdp_enabled() is not off+ fixes._sdpa_cudnn_restore(switch)+ assert torch.backends.cuda.cudnn_sdp_enabled()+ with sdpa_kernel([SDPBackend.CUDNN_ATTENTION]): # cuDNN-only: left alone+ assert fixes._sdpa_cudnn_off(192) is False+ assert torch.backends.cuda.cudnn_sdp_enabled()+ fixes._sdpa_cudnn_restore(False)+++def cpu_dq_nan(monkeypatch):+ """CPU SDPA as an affected H100: only the device check is mocked."""+ import torch.nn.functional as F++ length = getattr(fixes, "_sdpa_cudnn_dq_nan_length", lambda k: k.shape[-2] % 128 == 64)++ def known_bad(decode, dq, query, key, is_causal):+ hit = not is_causal and query.requires_grad and torch.is_grad_enabled() and length(key)+ return fixes._SDPA_CUDNN_DQ_NAN if hit else None++ real = torch._C._nn.scaled_dot_product_attention+ monkeypatch.setattr(F, "scaled_dot_product_attention", real)+ monkeypatch.setattr(fixes, "_sdpa_cudnn_known_bad_devices", lambda: (D0, D0))+ monkeypatch.setattr(fixes, "_sdpa_cudnn_known_bad", known_bad)+ monkeypatch.delenv("UNSLOTH_ALLOW_BUGGY_CUDNN_SDPA", raising = False)+ fixes.fix_cudnn_sdpa_known_bad_shapes()+ wrapped = F.scaled_dot_product_attention+ assert wrapped is not real+ torch._dynamo.reset()+ yield wrapped+ torch._dynamo.reset()+++def _qkv(s_kv):+ q = torch.randn(1, 2, 8, 16, dtype = torch.float16, requires_grad = True)+ k = torch.randn(1, 2, s_kv, 16, dtype = torch.float16, requires_grad = True)+ return q, k, torch.randn_like(k, requires_grad = True)++[email protected]("strict", [False, True])+def test_compiled_dynamic_key_length_is_not_specialized(cpu_dq_nan, strict):+ # One graph for lengths on and off 64 mod 128; strict mark_dynamic must compile.+ wrapped = cpu_dq_nan+ graphs = []++ def backend(gm, example_inputs):+ graphs.append(gm)+ return gm.forward++ fn = torch.compile(lambda q, k, v: wrapped(q, k, v).float().sum(), backend = backend,+ fullgraph = True, dynamic = True) # fmt: skip+ with sdpa_kernel([SDPBackend.MATH, SDPBackend.CUDNN_ATTENTION]):+ for s_kv in (192, 256, 320, 384, 448):+ q, k, v = _qkv(s_kv)+ if strict:+ torch._dynamo.mark_dynamic(k, 2, min = 128, max = 512)+ torch._dynamo.mark_dynamic(v, 2, min = 128, max = 512)+ fn(q, k, v).backward()+ assert len(graphs) == 1+ names = ("_sdpa_without_cudnn(", "_sdpa_cudnn_off(", "_sdpa_kernel(")+ assert any(name in graphs[0].code for name in names)+++def test_compiled_call_reads_policy_at_call_time(cpu_dq_nan):+ # Compiled under one policy, called under another: the later policy decides and is kept.+ wrapped = cpu_dq_nan+ fn = torch.compile(lambda q, k, v: wrapped(q, k, v), backend = lambda gm, ex: gm.forward,+ fullgraph = True) # fmt: skip+ q, k, v = _qkv(192)+ cuda = torch.backends.cuda+ with sdpa_kernel(ALL_BACKENDS):+ fn(q, k, v).sum().backward()+ with sdpa_kernel([SDPBackend.MATH, SDPBackend.CUDNN_ATTENTION]):+ fn(q, k, v)+ assert (cuda.flash_sdp_enabled(), cuda.math_sdp_enabled(), cuda.cudnn_sdp_enabled()) == (+ False, True, True,+ ) # fmt: skip+ with sdpa_kernel([SDPBackend.CUDNN_ATTENTION]): # cuDNN-only: left alone, as unwrapped SDPA+ with pytest.raises(RuntimeError, match = "No viable backend"):+ fn(q, k, v)+ assert not cuda.math_sdp_enabled() and cuda.cudnn_sdp_enabled()+++_RAISING = []+++def _raising_op():+ """An opaque op that raises at run time (as a CUDA OOM would), not while Dynamo traces it."""+ if not _RAISING:++ @torch.library.custom_op("unsloth_cudnn_sdpa_probe::boom", mutates_args = ())+ def boom(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor) -> torch.Tensor:+ raise RuntimeError("CUDA out of memory (simulated)")++ @boom.register_fake+ def _(q, k, v):+ return torch.empty_like(q)++ _RAISING.append(boom)+ return _RAISING[0]++[email protected]("mode", ["plain", "eager_graph"])+def test_raising_sdpa_restores_the_policy(cpu_dq_nan, monkeypatch, mode):+ # A raising SDPA (e.g. OOM) still restores the flags, also from an FX graph.+ import torch.nn.functional as F++ boom = _raising_op()+ monkeypatch.setattr(F, "scaled_dot_product_attention", lambda q, k, v, *a, **kw: boom(q, k, v))+ fixes.fix_cudnn_sdpa_known_bad_shapes()+ wrapped = F.scaled_dot_product_attention+ torch._dynamo.reset()+ fn = lambda q, k, v: wrapped(q, k, v) # noqa: E731+ if mode == "eager_graph":+ fn = torch.compile(fn, backend = "eager", fullgraph = True, dynamic = False)+ depth = fixes._SDPA_CUDNN_HELD[0]+ try:+ with sdpa_kernel(CUDNN_MATH):+ before = _sdp_flags()+ for _ in range(2):+ with pytest.raises(RuntimeError, match = "simulated"):+ fn(*_qkv(192))+ assert _sdp_flags() == before+ assert fixes._SDPA_CUDNN_HELD[0] == depth+ finally:+ fixes._SDPA_CUDNN_HELD[0] = depth+ torch._dynamo.reset()+++_PROBE = []+++def _probe_op():+ """Like aten SDPA, picks its kernel from the live cuDNN flag when decomposed."""+ if not _PROBE:+ lib = torch.library.Library("unsloth_cudnn_sdpa_probe", "FRAGMENT")+ lib.define("probe(Tensor q, Tensor k, Tensor v) -> Tensor")+ lib.impl(+ "probe",+ lambda q, k, v: q * (2.0 if torch._C._get_cudnn_sdp_enabled() else 3.0),+ "CompositeImplicitAutograd",+ )+ _PROBE.append((lib, torch.ops.unsloth_cudnn_sdpa_probe.probe))+ return _PROBE[0][1]+++def probe_dq_nan(cpu_dq_nan, monkeypatch):+ """cpu_dq_nan over the probe op: an output of 2 * q ran the cuDNN kernel, 3 * q a fallback."""+ import torch.nn.functional as F++ probe = _probe_op()+ monkeypatch.setattr(F, "scaled_dot_product_attention", lambda q, k, v, *a, **kw: probe(q, k, v))+ fixes.fix_cudnn_sdpa_known_bad_shapes()+ torch._dynamo.reset()+ yield F.scaled_dot_product_attention+ torch._dynamo.reset()+++CUDNN_ONLY = [SDPBackend.CUDNN_ATTENTION]+CUDNN_MATH = [SDPBackend.MATH, SDPBackend.CUDNN_ATTENTION]+++def _sdp_flags():+ cuda = torch.backends.cuda+ return (cuda.cudnn_sdp_enabled(), cuda.flash_sdp_enabled(), cuda.mem_efficient_sdp_enabled(),+ cuda.math_sdp_enabled()) # fmt: skip+++def cudnn_sdp_on():+ """Start from torch's default, cuDNN SDPA on: importing vLLM turns it off process-wide."""+ was = torch.backends.cuda.cudnn_sdp_enabled()+ torch.backends.cuda.enable_cudnn_sdp(True)+ yield+ torch.backends.cuda.enable_cudnn_sdp(was)+++_INDUCTOR = pytest.param(+ "inductor",+ marks = pytest.mark.skipif(+ sys.platform == "win32" and shutil.which("cl") is None,+ reason = "CPU inductor needs MSVC on Windows",+ ),+)++[email protected]("backend", ["aot_eager", _INDUCTOR])[email protected]("first", ["cudnn_only", "fallback"])+def test_aot_compiled_call_follows_a_later_policy(cudnn_sdp_on, probe_dq_nan, backend, first):+ # AOTAutograd fixes the backend at trace time: a later policy must not keep the old kernel.+ wrapped = probe_dq_nan+ fn = torch.compile(lambda q, k, v: wrapped(q, k, v), backend = backend, fullgraph = True,+ dynamic = False) # fmt: skip+ policies = [CUDNN_ONLY, CUDNN_MATH] if first == "cudnn_only" else [CUDNN_MATH, CUDNN_ONLY]+ for s_kv, hit in ((192, True), (256, False)):+ q, k, v = _qkv(s_kv)+ for policy in policies * 2:+ with sdpa_kernel(policy):+ before = _sdp_flags()+ ratio = (fn(q, k, v) / q).detach().flatten()[0].item()+ expected = 3.0 if hit and policy is CUDNN_MATH else 2.0+ assert ratio == pytest.approx(expected, rel = 0.01), (s_kv, policy) # fp16 2q / 3q+ eager = (wrapped(q, k, v) / q).detach().flatten()[0].item()+ assert eager == pytest.approx(expected, rel = 0.01)+ assert _sdp_flags() == before+ assert torch.backends.cuda.cudnn_sdp_enabled()+++def test_opt_out_and_unaffected_host(monkeypatch):+ import torch.nn.functional as F++ sentinel = lambda *a, **k: None # noqa: E731+ monkeypatch.setattr(F, "scaled_dot_product_attention", sentinel)+ monkeypatch.setattr(fixes, "_sdpa_cudnn_known_bad_devices", lambda: (D0, D0))+ monkeypatch.setenv("UNSLOTH_ALLOW_BUGGY_CUDNN_SDPA", "1")+ fixes.fix_cudnn_sdpa_known_bad_shapes()+ assert F.scaled_dot_product_attention is sentinel+ monkeypatch.delenv("UNSLOTH_ALLOW_BUGGY_CUDNN_SDPA")+ monkeypatch.setattr(fixes, "_sdpa_cudnn_known_bad_devices", lambda: (NONE, NONE))+ fixes.fix_cudnn_sdpa_known_bad_shapes()+ assert F.scaled_dot_product_attention is sentinel+++def _held_count():+ return fixes._SDPA_CUDNN_HELD[0]+++def _cell(wrapper, name):+ return wrapper.__closure__[wrapper.__code__.co_freevars.index(name)]+++def test_overlapping_calls_keep_cudnn_off(recorder):+ # A call starting while another holds cuDNN off still runs without cuDNN after the first restores.+ import threading++ F, calls = recorder+ a_inside, b_inside, a_done = threading.Event(), threading.Event(), threading.Event()+ seen = {}+ token = _cell(F.scaled_dot_product_attention, "token").cell_contents+ inner = fixes._SDPA_CUDNN_ORIGINALS[token]++ def gated(query, key, value, *args, **kwargs):+ if threading.current_thread().name == "A":+ seen["A"] = torch.backends.cuda.cudnn_sdp_enabled()+ a_inside.set()+ b_inside.wait(5)+ else:+ b_inside.set()+ a_done.wait(5) # backend selection after A restored its flag+ seen["B"] = torch.backends.cuda.cudnn_sdp_enabled()+ return inner(query, key, value, *args, **kwargs)++ fixes._SDPA_CUDNN_ORIGINALS[token] = gated+ wrapper = F.scaled_dot_product_attention+ original_cell = _cell(wrapper, "original")+ saved = original_cell.cell_contents+ original_cell.cell_contents = gated+ try:+ q, k = _FakeCuda(1, 8, 1, 64), _FakeCuda(1, 8, 9, 64)++ def run_a():+ F.scaled_dot_product_attention(q, k, k)+ a_done.set()++ def run_b():+ a_inside.wait(5)+ F.scaled_dot_product_attention(q, k, k)++ threads = [threading.Thread(target = run_a, name = "A"),+ threading.Thread(target = run_b, name = "B")] # fmt: skip+ for t in threads:+ t.start()+ for t in threads:+ t.join(10)+ assert seen == {"A": False, "B": False}+ assert torch.backends.cuda.cudnn_sdp_enabled()+ assert _held_count() == 0+ finally:+ original_cell.cell_contents = saved+ fixes._SDPA_CUDNN_ORIGINALS[token] = inner+++def test_many_threads_never_see_cudnn(recorder):+ # 8 threads: every matching call runs with cuDNN off; flag back on, nothing held, at the end.+ import threading+ import time++ F, calls = recorder+ token = _cell(F.scaled_dot_product_attention, "token").cell_contents+ inner = fixes._SDPA_CUDNN_ORIGINALS[token]+ wrapper = F.scaled_dot_product_attention+ original_cell = _cell(wrapper, "original")+ saved = original_cell.cell_contents+ bad = []++ def slow(query, key, value, *args, **kwargs):+ time.sleep(0.0002) # yields the GIL, as a real kernel launch does+ if torch.backends.cuda.cudnn_sdp_enabled():+ bad.append(threading.current_thread().name)+ return "out"++ fixes._SDPA_CUDNN_ORIGINALS[token] = slow+ original_cell.cell_contents = slow+ try:+ q, k = _FakeCuda(1, 8, 1, 64), _FakeCuda(1, 8, 9, 64)++ def worker():+ for _ in range(200):+ F.scaled_dot_product_attention(q, k, k)++ threads = [threading.Thread(target = worker, name = f"t{i}") for i in range(8)]+ for t in threads:+ t.start()+ for t in threads:+ t.join(60)+ assert not bad, f"{len(bad)} matching calls ran with cuDNN on"+ assert torch.backends.cuda.cudnn_sdp_enabled()+ assert _held_count() == 0+ finally:+ original_cell.cell_contents = saved+ fixes._SDPA_CUDNN_ORIGINALS[token] = innerdiff --git a/unsloth/_gpu_init.py b/unsloth/_gpu_init.pyindex 4d22588c5de..20de3236175 100644--- a/unsloth/_gpu_init.py+++ b/unsloth/_gpu_init.py@@ -371,6 +371,7 @@ from .import_fixes import (fix_vllm_pdl_blackwell,fix_cudnn_sdpa_d256_masked_backward,fix_cudnn_sdpa_backward_grad_layout,+ fix_cudnn_sdpa_known_bad_shapes,fix_rocm_windows_fused_sdpa,fix_triton_compiled_kernel_missing_attrs,fix_dynamo_config_thread_visibility,@@ -462,6 +463,8 @@ fix_trl_vllm_ascend()fix_vllm_pdl_blackwell()fix_cudnn_sdpa_d256_masked_backward()fix_cudnn_sdpa_backward_grad_layout()+# torch 2.11 - 2.14 cuDNN SDPA bugs fixed by cuDNN 9.26 / torch 2.15 (pytorch#193893, #196678).+fix_cudnn_sdpa_known_bad_shapes()# Windows ROCm only, probe-gated: fused attention fails on every call there (gfx1151, torch 2.11).fix_rocm_windows_fused_sdpa()fix_triton_compiled_kernel_missing_attrs()@@ -530,6 +533,7 @@ del fix_trl_vllm_ascenddel fix_vllm_pdl_blackwelldel fix_cudnn_sdpa_d256_masked_backwarddel fix_cudnn_sdpa_backward_grad_layout+del fix_cudnn_sdpa_known_bad_shapesdel fix_rocm_windows_fused_sdpadel fix_triton_compiled_kernel_missing_attrsdel fix_dynamo_config_thread_visibilitydiff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.pyindex c391ce2d34c..367e972cea5 100644--- a/unsloth/import_fixes.py+++ b/unsloth/import_fixes.py@@ -7900,6 +7900,228 @@ def fix_cudnn_sdpa_backward_grad_layout():)+_SDPA_CUDNN_KNOWN_BAD_FLAG = "_unsloth_avoids_cudnn_sdpa_known_bad_shapes"+_SDPA_CUDNN_KNOWN_BAD_WARNED = set()+# Backend flags are process-wide: refcount wrapped calls in flight so overlapping calls never see cuDNN back on early.+_SDPA_CUDNN_HELD = [0]+_SDPA_CUDNN_LOCK = threading.Lock()+_SDPA_CUDNN_DECODE = "decode"+_SDPA_CUDNN_DQ_NAN = "dq_nan"+++def _torch_release(torch_version):+ try:+ return TrueVersion(TrueVersion(str(torch_version)).base_version)+ except Exception:+ return None+++def _cudnn_sdpa_decode_corrupts(torch_version, cudnn_version, capability):+ """pytorch#193893: cuDNN 9.19 - 9.25.0 (not 9.24.1) one-row decode on SM 10.x / 11.x is wrong.+ torch 2.14 refuses it (pytorch#194963); <= 2.13 cu128+ wheels ship 9.19 / 9.20."""+ release = _torch_release(torch_version)+ if release is None or release >= TrueVersion("2.14") or cudnn_version is None:+ return False+ cudnn_version = int(cudnn_version)+ if cudnn_version < 91900 or cudnn_version > 92500 or 92400 < cudnn_version < 92500:+ return False+ return int(capability[0]) in (10, 11)+++def _cudnn_sdpa_dq_nan(torch_version, cudnn_version, capability):+ """pytorch#196678: cuDNN < 9.26 on SM90 gives NaN dQ for non-causal s_kv % 128 == 64.+ torch 2.15 refuses it (pytorch#196861)."""+ release = _torch_release(torch_version)+ if release is None or release >= TrueVersion("2.15") or cudnn_version is None:+ return False+ return int(cudnn_version) < 92600 and tuple(int(x) for x in capability[:2]) == (9, 0)+++def _sdpa_cudnn_known_bad_devices():+ try:+ import torch++ if not torch.cuda.is_available() or getattr(torch.version, "hip", None):+ return frozenset(), frozenset()+ cudnn_version = torch.backends.cudnn.version()+ capabilities = [+ torch.cuda.get_device_capability(i) for i in range(torch.cuda.device_count())+ ]+ except Exception:+ return frozenset(), frozenset()+ decode = frozenset(+ i+ for i, cap in enumerate(capabilities)+ if _cudnn_sdpa_decode_corrupts(torch.__version__, cudnn_version, cap)+ )+ dq_nan = frozenset(+ i+ for i, cap in enumerate(capabilities)+ if _cudnn_sdpa_dq_nan(torch.__version__, cudnn_version, cap)+ )+ return decode, dq_nan+++def _sdpa_cudnn_dq_nan_length(key):+ # Compiling: _sdpa_cudnn_off tests the value, so a dynamic length adds no guard.+ import torch+ return torch.compiler.is_compiling() or key.shape[-2] % 128 == 64+++def _sdpa_cudnn_known_bad(decode_devices, dq_devices, query, key, is_causal):+ """The known cuDNN bug this call would hit, mirroring torch's later guards; None if none."""+ import torch++ if not getattr(query, "is_cuda", False) or getattr(query, "is_nested", False):+ return None+ index = query.device.index+ if index not in decode_devices and index not in dq_devices:+ return None+ dtype = query.dtype+ # Autocast casts fp32 SDPA inputs to half before dispatch.+ if dtype == torch.float32 and torch.is_autocast_enabled("cuda"):+ dtype = torch.get_autocast_dtype("cuda")+ if dtype not in (torch.float16, torch.bfloat16) or query.dim() < 2:+ return None+ if index in decode_devices and query.shape[-2] == 1:+ return _SDPA_CUDNN_DECODE+ if (+ index in dq_devices+ and not is_causal+ and query.requires_grad+ and torch.is_grad_enabled()+ and getattr(key, "dim", lambda: 0)() >= 2+ and _sdpa_cudnn_dq_nan_length(key)+ ):+ return _SDPA_CUDNN_DQ_NAN+ return None+++class _SdpaCudnnPolicy(types.ModuleType):+ """Dynamo folds SDPA backend flags without a guard, but guards a module attribute by value:+ reading the policy here makes a compiled graph recompile when the policy flips its backend."""++ @property+ def cudnn_off(self):+ import torch+ cuda = torch.backends.cuda+ # A held switch counts as on, so another thread's matching call also takes it.+ return (cuda.cudnn_sdp_enabled() or _SDPA_CUDNN_HELD[0] > 0) and (+ cuda.flash_sdp_enabled() or cuda.mem_efficient_sdp_enabled() or cuda.math_sdp_enabled()+ )+++_SDPA_CUDNN_POLICY = _SdpaCudnnPolicy("unsloth_sdpa_cudnn_policy")+++def _sdpa_cudnn_off(key_length = None):+ """Turn cuDNN SDPA off unless it is the only backend; a symbolic key_length counts as a hit.+ The count rises before the flag drops, so an unlocked policy read never sees cuDNN off."""+ import torch++ if not (key_length is None or isinstance(key_length, torch.SymInt) or key_length % 128 == 64):+ return False+ with _SDPA_CUDNN_LOCK:+ if _SDPA_CUDNN_HELD[0] == 0 and not _SDPA_CUDNN_POLICY.cudnn_off:+ return False+ _SDPA_CUDNN_HELD[0] += 1+ if _SDPA_CUDNN_HELD[0] == 1:+ torch._C._set_sdp_use_cudnn(False)+ return True+++def _sdpa_cudnn_restore(switch):+ import torch+ if not switch:+ return+ with _SDPA_CUDNN_LOCK:+ if _SDPA_CUDNN_HELD[0] == 1:+ torch._C._set_sdp_use_cudnn(True)+ _SDPA_CUDNN_HELD[0] -= 1+++_SDPA_CUDNN_ORIGINALS = {} # str token -> wrapped SDPA (a graph constant, unlike a function)+_SDPA_CUDNN_MESSAGES = {+ _SDPA_CUDNN_DECODE: "single-token decode on this Blackwell GPU (cuDNN 9.19 - 9.25.0 "+ "returns wrong attention there, pytorch#193893)",+ _SDPA_CUDNN_DQ_NAN: "training with a key length of 64 mod 128 on this H100 / H200 (cuDNN "+ "< 9.26 returns NaN query gradients there, pytorch#196678)",+}+++def _sdpa_without_cudnn(token, bug, key_length, query, key, value, *args, **kwargs):+ """allow_in_graph: one eager-graph node whose finally runs even if SDPA raises; AOTAutograd+ traces through it, so the traced SDPA avoids cuDNN."""+ import torch++ switch = _sdpa_cudnn_off(key_length)+ try:+ if switch and not torch.compiler.is_compiling() and bug not in _SDPA_CUDNN_KNOWN_BAD_WARNED:+ _SDPA_CUDNN_KNOWN_BAD_WARNED.add(bug)+ logger.warning(+ f"Unsloth: running SDPA without cuDNN attention for {_SDPA_CUDNN_MESSAGES[bug]}. "+ "Upgrade to torch 2.15 / cuDNN 9.26, or set UNSLOTH_ALLOW_BUGGY_CUDNN_SDPA=1."+ )+ return _SDPA_CUDNN_ORIGINALS[token](query, key, value, *args, **kwargs)+ finally:+ _sdpa_cudnn_restore(switch)+++def fix_cudnn_sdpa_known_bad_shapes():+ """Keep SDPA off cuDNN for the calls torch 2.14 / 2.15 later refuse (pytorch#193893, #196678),+ only when a visible device is affected and another backend is enabled.+ UNSLOTH_ALLOW_BUGGY_CUDNN_SDPA=1 opts out."""+ if os.environ.get("UNSLOTH_ALLOW_BUGGY_CUDNN_SDPA", "0") == "1":+ return+ decode_devices, dq_devices = _sdpa_cudnn_known_bad_devices()+ if not decode_devices and not dq_devices:+ return+ try:+ import torch+ import torch._dynamo+ import torch.nn.functional as F+ except Exception as e:+ logger.info(f"Unsloth: Skipping the cuDNN SDPA known-bad-shape fix ({e})")+ return+ if getattr(F.scaled_dot_product_attention, _SDPA_CUDNN_KNOWN_BAD_FLAG, False):+ return++ # Switch, SDPA and restore stay one graph call: a trailing restore node would be skipped on a raise.+ torch._dynamo.allow_in_graph(_sdpa_without_cudnn)+ original = F.scaled_dot_product_attention+ token = f"sdpa_{len(_SDPA_CUDNN_ORIGINALS)}" # a str: Dynamo may make an int symbolic+ _SDPA_CUDNN_ORIGINALS[token] = original++ @functools.wraps(original)+ def scaled_dot_product_attention(+ query,+ key,+ value,+ attn_mask = None,+ dropout_p = 0.0,+ is_causal = False,+ *args,+ **kwargs,+ ):+ bug = _sdpa_cudnn_known_bad(decode_devices, dq_devices, query, key, is_causal)+ # Compiled, this policy read is a guard.+ if bug is not None and _SDPA_CUDNN_POLICY.cudnn_off:+ key_length = key.shape[-2] if bug == _SDPA_CUDNN_DQ_NAN else None+ return _sdpa_without_cudnn(+ token, bug, key_length, query, key, value, attn_mask, dropout_p, is_causal,+ *args, **kwargs,+ ) # fmt: skip+ return original(query, key, value, attn_mask, dropout_p, is_causal, *args, **kwargs)++ scaled_dot_product_attention.__wrapped__ = original+ setattr(scaled_dot_product_attention, _SDPA_CUDNN_KNOWN_BAD_FLAG, True)+ F.scaled_dot_product_attention = scaled_dot_product_attention+ logger.info(+ "Unsloth: cuDNN SDPA known-bad shapes avoided "+ f"(decode devices {sorted(decode_devices)}, NaN dQ devices {sorted(dq_devices)})"+ )++def _rocm_windows_broken_sdpa_backends():"""Fused SDPA backends ("flash", "mem_efficient") failing on this process's Windows ROCm GPU.