Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Keep masked head_dim 256 SDPA training off cuDNN attention on SM100 (torch 2.14 NaN grads) (#12344)

Daniel Han3e070b48ae06
diff --git a/tests/test_cudnn_sdpa_d256_masked_backward.py b/tests/test_cudnn_sdpa_d256_masked_backward.py
new file mode 100644
index 00000000000..270c9e37dfc
--- /dev/null
+++ b/tests/test_cudnn_sdpa_d256_masked_backward.py
@@ -0,0 +1,154 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""Masked head_dim-256 SDPA training must not get NaN gradients from cuDNN on SM100 (torch 2.14,
+cuDNN 9.24). The GPU tests only `import unsloth` in a fresh interpreter, so they fail on a tree
+without the fix wherever the bug exists."""
+
+import json
+import os
+import subprocess
+import sys
+import textwrap
+
+import pytest
+
+torch = pytest.importorskip("torch")
+
+
+def _detour(*args):
+ # Per test, so the GPU tests still run (and fail) on a tree without the fix.
+ from unsloth.import_fixes import _sdpa_needs_cudnn_d256_detour
+ return _sdpa_needs_cudnn_d256_detour(*args)
+
+
+def _t(
+ *shape,
+ dtype = None,
+ requires_grad = False,
+ device = "cpu",
+):
+ return torch.zeros(
+ *shape, dtype = dtype or torch.bfloat16, device = device, requires_grad = requires_grad
+ )
+
+
+class TestDetourDecision:
+ """The detour is for masked, grad-building, half-precision head_dim-256 calls only."""
+
+ def test_no_mask_is_never_detoured(self):
+ q = _t(1, 2, 8, 256, requires_grad = True)
+ assert not _detour(frozenset({0}), q, q, None)
+
+ def test_cpu_query_is_never_detoured(self):
+ q = _t(1, 2, 8, 256, requires_grad = True)
+ m = torch.ones(8, 8, dtype = torch.bool)
+ assert not _detour(frozenset({0, None}), q, q, m)
+
+ def test_no_grad_is_never_detoured(self):
+ q = _t(1, 2, 8, 256, requires_grad = True)
+ m = torch.ones(8, 8, dtype = torch.bool)
+ with torch.no_grad():
+ assert not _detour(frozenset({0, None}), q, q, m)
+
+
+def _gpu_sm100():
+ if not torch.cuda.is_available() or getattr(torch.version, "hip", None):
+ return False
+ try:
+ return any(
+ torch.cuda.get_device_capability(i)[0] == 10 for i in range(torch.cuda.device_count())
+ )
+ except Exception:
+ return False
+
+
+_PROBE = textwrap.dedent(
+ """
+ import json, sys
+ import unsloth # noqa: F401 (applies the import-time fixes)
+ import torch
+ import torch.nn.functional as F
+
+ dev = next(i for i in range(torch.cuda.device_count()) if torch.cuda.get_device_capability(i)[0] == 10)
+ torch.cuda.set_device(dev)
+ out = {}
+
+ def grads(fn, D, mask, dtype):
+ g = torch.Generator(device = "cuda").manual_seed(0)
+ B, H, S = 2, 4, 24
+ q, k, v = (torch.randn(B, H, S, D, device = "cuda", dtype = dtype, generator = g).requires_grad_() for _ in range(3))
+ m = torch.ones(S, S, device = "cuda", dtype = torch.bool).tril().expand(B, 1, S, S) if mask else None
+ o = fn(q, k, v, m)
+ o.float().square().sum().backward()
+ return [x.grad.float() for x in (q, k, v)]
+
+ def sdpa(q, k, v, m):
+ return F.scaled_dot_product_attention(q, k, v, attn_mask = m, is_causal = m is None)
+
+ compiled = torch.compile(sdpa, fullgraph = True, dynamic = True)
+ for name, fn in (("eager", sdpa), ("compiled", compiled)):
+ for D in (256, 128):
+ for mask in (True, False):
+ got = grads(fn, D, mask, torch.bfloat16)
+ with torch.nn.attention.sdpa_kernel([torch.nn.attention.SDPBackend.MATH]):
+ ref = grads(sdpa, D, mask, torch.float32)
+ out[f"{name}-D{D}-mask{int(mask)}"] = [
+ None if not torch.isfinite(a).all() else float((a - r).norm() / r.norm())
+ for a, r in zip(got, ref)
+ ]
+ # fp32 Q/K/V (DoRA promotes them) under bf16 autocast: SDPA casts to bf16 before dispatch.
+ for name, fn in (("eager", sdpa), ("compiled", compiled)):
+ with torch.autocast("cuda", dtype = torch.bfloat16):
+ got = grads(fn, 256, True, torch.float32)
+ with torch.nn.attention.sdpa_kernel([torch.nn.attention.SDPBackend.MATH]):
+ ref = grads(sdpa, 256, True, torch.float32)
+ out[f"{name}-autocast-fp32-D256-mask1"] = [
+ None if not torch.isfinite(a).all() else float((a - r).norm() / r.norm())
+ for a, r in zip(got, ref)
+ ]
+ with torch.no_grad():
+ q = torch.randn(1, 2, 8, 256, device = "cuda", dtype = torch.bfloat16)
+ m = torch.ones(8, 8, device = "cuda", dtype = torch.bool)
+ out["inference_finite"] = bool(torch.isfinite(F.scaled_dot_product_attention(q, q, q, attn_mask = m)).all())
+ print("PROBE_RESULT " + json.dumps(out))
+ """
+)
+
+
+def _run_probe(extra_env = None):
+ env = dict(os.environ)
+ env.update(extra_env or {})
+ r = subprocess.run(
+ [sys.executable, "-c", _PROBE], capture_output = True, text = True, env = env, timeout = 900
+ )
+ line = [x for x in r.stdout.splitlines() if x.startswith("PROBE_RESULT ")]
+ assert line, f"probe crashed (rc {r.returncode}):\n{r.stdout[-3000:]}\n{r.stderr[-3000:]}"
+ return json.loads(line[-1][len("PROBE_RESULT ") :])
+
+
[email protected](not _gpu_sm100(), reason = "needs an SM100 (B200 / B300) GPU")
+def test_masked_head_dim_256_backward_is_finite_after_import():
+ res = _run_probe()
+ assert res.pop("inference_finite") is True
+ bad = {k: v for k, v in res.items() if any(e is None or e > 2e-2 for e in v)}
+ assert not bad, f"non-finite or wrong SDPA gradients (rel err dq, dk, dv vs fp32 math): {bad}"
+
+
[email protected](not _gpu_sm100(), reason = "needs an SM100 (B200 / B300) GPU")
+def test_opt_out_env_leaves_torch_dispatch_alone():
+ """With the opt-out set the wrapper is not installed, so torch's own dispatch runs."""
+ code = (
+ "import unsloth, torch.nn.functional as F\n"
+ "print('WRAPPED', getattr(F.scaled_dot_product_attention, '_unsloth_avoids_cudnn_d256_masked_backward', False))\n"
+ )
+ for env, want in (({"UNSLOTH_ALLOW_CUDNN_SDPA_D256": "1"}, "False"), ({}, "True")):
+ e = dict(os.environ)
+ e.pop("UNSLOTH_ALLOW_CUDNN_SDPA_D256", None)
+ e.update(env)
+ r = subprocess.run(
+ [sys.executable, "-c", code], capture_output = True, text = True, env = e, timeout = 600
+ )
+ assert f"WRAPPED {want}" in r.stdout, (env, r.stdout[-2000:], r.stderr[-2000:])
diff --git a/tests/test_flex_head_dim_version_gate.py b/tests/test_flex_head_dim_version_gate.py
index 31d32ba0ead..fc823584763 100644
--- a/tests/test_flex_head_dim_version_gate.py
+++ b/tests/test_flex_head_dim_version_gate.py
@@ -101,15 +101,43 @@ def test_gate_is_off_below_blackwell(monkeypatch):
assert u._sdpa_reaches_cudnn_at_head_dim_256() is False
-def test_gate_is_on_for_blackwell_on_torch_2_14(monkeypatch):
+def _plain_sdpa(monkeypatch):
+ # The SM100 cuDNN head_dim 256 detour wraps SDPA at import on a B200; these gate tests are about plain SDPA.
+ sdpa = u.torch.nn.functional.scaled_dot_product_attention
+ monkeypatch.setattr(
+ u.torch.nn.functional, "scaled_dot_product_attention", getattr(sdpa, "__wrapped__", sdpa)
+ )
+
+
+def _blackwell_torch_2_14(monkeypatch):
monkeypatch.setattr(u.torch, "__version__", "2.14.0+cu130")
monkeypatch.setattr(u.torch.version, "hip", None, raising = False)
monkeypatch.setattr(u.torch.cuda, "is_available", lambda: True)
monkeypatch.setattr(u.torch.cuda, "device_count", lambda: 1)
monkeypatch.setattr(u.torch.cuda, "get_device_capability", lambda index: (10, 0))
+
+
+def test_gate_is_on_for_blackwell_on_torch_2_14(monkeypatch):
+ _plain_sdpa(monkeypatch)
+ _blackwell_torch_2_14(monkeypatch)
assert u._sdpa_reaches_cudnn_at_head_dim_256() is True
+def test_gate_is_off_while_the_cudnn_d256_detour_is_installed(monkeypatch):
+ # Masked head_dim 256 training then runs the efficient kernel, not cuDNN: flex is faster, as on torch <= 2.13.
+ _blackwell_torch_2_14(monkeypatch)
+ sdpa = u.torch.nn.functional.scaled_dot_product_attention
+ original = getattr(sdpa, "__wrapped__", sdpa)
+
+ def detour(*args, **kwargs):
+ return original(*args, **kwargs)
+
+ detour._unsloth_avoids_cudnn_d256_masked_backward = True
+ monkeypatch.setattr(u.torch.nn.functional, "scaled_dot_product_attention", detour)
+ assert u._sdpa_reaches_cudnn_at_head_dim_256() is False
+ assert u._prefers_flex_for_head_dim(_cfg(256)) is True
+
+
def test_a_mixed_box_falls_back_to_the_weakest_card(monkeypatch):
monkeypatch.setattr(u.torch, "__version__", "2.14.0+cu130")
monkeypatch.setattr(u.torch.version, "hip", None, raising = False)
diff --git a/unsloth/_gpu_init.py b/unsloth/_gpu_init.py
index 6fd5bb0b001..3f8790078df 100644
--- a/unsloth/_gpu_init.py
+++ b/unsloth/_gpu_init.py
@@ -281,6 +281,7 @@ from .import_fixes import (
check_vllm_torch_sm100_compatibility,
fix_vllm_guided_decoding_params,
fix_vllm_pdl_blackwell,
+ fix_cudnn_sdpa_d256_masked_backward,
fix_triton_compiled_kernel_missing_attrs,
fix_dynamo_config_thread_visibility,
patch_trunc_normal_precision_issue,
@@ -364,6 +365,7 @@ check_vllm_torch_sm100_compatibility()
fix_vllm_guided_decoding_params()
fix_trl_vllm_ascend()
fix_vllm_pdl_blackwell()
+fix_cudnn_sdpa_d256_masked_backward()
fix_triton_compiled_kernel_missing_attrs()
# Must run before unsloth_zoo's patch_torch_compile and the gpt-oss patches raise the dynamo
# recompile limits, so those settings reach the autograd worker threads on torch >= 2.12.
@@ -422,6 +424,7 @@ del check_vllm_torch_sm100_compatibility
del fix_vllm_guided_decoding_params
del fix_trl_vllm_ascend
del fix_vllm_pdl_blackwell
+del fix_cudnn_sdpa_d256_masked_backward
del fix_triton_compiled_kernel_missing_attrs
del fix_dynamo_config_thread_visibility
del patch_trunc_normal_precision_issue
diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.py
index d1c14fd95a9..0edc427ba6a 100644
--- a/unsloth/import_fixes.py
+++ b/unsloth/import_fixes.py
@@ -6515,6 +6515,102 @@ def fix_vllm_pdl_blackwell():
logger.info(f"Unsloth: Set TRITON_DISABLE_PDL=1 for SM100 ({sm100_gpu_name})")
+_SDPA_CUDNN_D256_FLAG = "_unsloth_avoids_cudnn_d256_masked_backward"
+_SDPA_CUDNN_D256_WARNED = False
+
+
+def _sdpa_cudnn_d256_sm100_devices():
+ try:
+ import torch
+ if not torch.cuda.is_available() or getattr(torch.version, "hip", None):
+ return frozenset()
+ return frozenset(
+ i
+ for i in range(torch.cuda.device_count())
+ if torch.cuda.get_device_capability(i)[0] == 10
+ )
+ except Exception:
+ return frozenset()
+
+
+def _sdpa_needs_cudnn_d256_detour(sm100_devices, query, key, attn_mask):
+ """True for the calls whose cuDNN backward returns NaN dQ on SM100 (no-grad never reaches it)."""
+ import torch
+
+ if attn_mask is None or not torch.is_grad_enabled():
+ return False
+ if not isinstance(query, torch.Tensor) or not query.is_cuda:
+ return False
+ dtype = query.dtype
+ # CUDA autocast casts fp32 SDPA inputs (e.g. DoRA-promoted Q/K/V) to half before dispatch.
+ if dtype == torch.float32 and torch.is_autocast_enabled("cuda"):
+ dtype = torch.get_autocast_dtype("cuda")
+ if dtype not in (torch.float16, torch.bfloat16):
+ return False
+ if query.shape[-1] != 256:
+ return False
+ if not (query.requires_grad or (isinstance(key, torch.Tensor) and key.requires_grad)):
+ return False
+ return query.device.index in sm100_devices
+
+
+def fix_cudnn_sdpa_d256_masked_backward():
+ """Run masked head_dim-256 SDPA training without cuDNN attention on SM100.
+
+ torch 2.14 (cuDNN 9.24) first dispatches such calls to cuDNN, whose backward returns NaN dQ
+ (bf16 / fp16); torch <= 2.13 never picks cuDNN here, so the detour is a no-op
+ there. Gated on the device, not the torch version. UNSLOTH_ALLOW_CUDNN_SDPA_D256=1 opts out.
+ """
+ if os.environ.get("UNSLOTH_ALLOW_CUDNN_SDPA_D256", "0") == "1":
+ return
+ sm100_devices = _sdpa_cudnn_d256_sm100_devices()
+ if not sm100_devices:
+ return
+ try:
+ import torch
+ import torch.nn.functional as F
+ from torch.nn.attention import SDPBackend, sdpa_kernel
+ except Exception as e:
+ logger.info(f"Unsloth: Skipping the cuDNN SDPA head_dim 256 fix ({e})")
+ return
+ if getattr(F.scaled_dot_product_attention, _SDPA_CUDNN_D256_FLAG, False):
+ return
+
+ original = F.scaled_dot_product_attention
+ backends = [SDPBackend.FLASH_ATTENTION, SDPBackend.EFFICIENT_ATTENTION, SDPBackend.MATH]
+
+ @functools.wraps(original)
+ def scaled_dot_product_attention(
+ query,
+ key,
+ value,
+ attn_mask = None,
+ *args,
+ **kwargs,
+ ):
+ global _SDPA_CUDNN_D256_WARNED
+ if _sdpa_needs_cudnn_d256_detour(sm100_devices, query, key, attn_mask):
+ if not torch.compiler.is_compiling():
+ if not _SDPA_CUDNN_D256_WARNED:
+ _SDPA_CUDNN_D256_WARNED = True
+ logger.warning(
+ "Unsloth: head_dim 256 attention with a mask is training on an SM100 GPU; "
+ "running it without cuDNN attention, whose backward returns NaN gradients "
+ "there (set UNSLOTH_ALLOW_CUDNN_SDPA_D256=1 to keep cuDNN)."
+ )
+ with sdpa_kernel(backends):
+ return original(query, key, value, attn_mask, *args, **kwargs)
+ return original(query, key, value, attn_mask, *args, **kwargs)
+
+ scaled_dot_product_attention.__wrapped__ = original
+ setattr(scaled_dot_product_attention, _SDPA_CUDNN_D256_FLAG, True)
+ F.scaled_dot_product_attention = scaled_dot_product_attention
+ logger.info(
+ "Unsloth: SM100 GPU found; masked head_dim 256 SDPA training will avoid cuDNN attention "
+ f"(devices {sorted(sm100_devices)})"
+ )
+
+
def patch_openspiel_env_async():
"""Apply nest_asyncio for OpenEnv EnvClient async compatibility.
diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py
index 6c055053365..88c43c894aa 100644
--- a/unsloth/models/_utils.py
+++ b/unsloth/models/_utils.py
@@ -491,7 +491,11 @@ _CUDNN_LARGE_HEAD_DIM_MIN_CAPABILITY = (10, 0)
def _sdpa_reaches_cudnn_at_head_dim_256():
- """True when plain SDPA already dispatches cuDNN for a MASKED head_dim 256 on this box."""
+ """True when plain SDPA already dispatches cuDNN for a MASKED head_dim 256 on this box.
+
+ False under fix_cudnn_sdpa_d256_masked_backward: it moves that training onto the efficient
+ kernel, where flex is faster again.
+ """
try:
if Version(torch.__version__.split("+")[0]) < Version(_CUDNN_LARGE_HEAD_DIM_TORCH_VERSION):
return False
@@ -499,6 +503,12 @@ def _sdpa_reaches_cudnn_at_head_dim_256():
return False
if not torch.cuda.is_available():
return False
+ if getattr(
+ torch.nn.functional.scaled_dot_product_attention,
+ "_unsloth_avoids_cudnn_d256_masked_backward",
+ False,
+ ):
+ return False
return all(
torch.cuda.get_device_capability(index) >= _CUDNN_LARGE_HEAD_DIM_MIN_CAPABILITY
for index in range(torch.cuda.device_count())