Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Keep masked head_dim 256 SDPA training off cuDNN attention on SM100 (torch 2.14 NaN grads) (#12344)
Daniel Han3e070b48ae06
diff --git a/tests/test_cudnn_sdpa_d256_masked_backward.py b/tests/test_cudnn_sdpa_d256_masked_backward.pynew file mode 100644index 00000000000..270c9e37dfc--- /dev/null+++ b/tests/test_cudnn_sdpa_d256_masked_backward.py@@ -0,0 +1,154 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""Masked head_dim-256 SDPA training must not get NaN gradients from cuDNN on SM100 (torch 2.14,+cuDNN 9.24). The GPU tests only `import unsloth` in a fresh interpreter, so they fail on a tree+without the fix wherever the bug exists."""++import json+import os+import subprocess+import sys+import textwrap++import pytest++torch = pytest.importorskip("torch")+++def _detour(*args):+ # Per test, so the GPU tests still run (and fail) on a tree without the fix.+ from unsloth.import_fixes import _sdpa_needs_cudnn_d256_detour+ return _sdpa_needs_cudnn_d256_detour(*args)+++def _t(+ *shape,+ dtype = None,+ requires_grad = False,+ device = "cpu",+):+ return torch.zeros(+ *shape, dtype = dtype or torch.bfloat16, device = device, requires_grad = requires_grad+ )+++class TestDetourDecision:+ """The detour is for masked, grad-building, half-precision head_dim-256 calls only."""++ def test_no_mask_is_never_detoured(self):+ q = _t(1, 2, 8, 256, requires_grad = True)+ assert not _detour(frozenset({0}), q, q, None)++ def test_cpu_query_is_never_detoured(self):+ q = _t(1, 2, 8, 256, requires_grad = True)+ m = torch.ones(8, 8, dtype = torch.bool)+ assert not _detour(frozenset({0, None}), q, q, m)++ def test_no_grad_is_never_detoured(self):+ q = _t(1, 2, 8, 256, requires_grad = True)+ m = torch.ones(8, 8, dtype = torch.bool)+ with torch.no_grad():+ assert not _detour(frozenset({0, None}), q, q, m)+++def _gpu_sm100():+ if not torch.cuda.is_available() or getattr(torch.version, "hip", None):+ return False+ try:+ return any(+ torch.cuda.get_device_capability(i)[0] == 10 for i in range(torch.cuda.device_count())+ )+ except Exception:+ return False+++_PROBE = textwrap.dedent(+ """+ import json, sys+ import unsloth # noqa: F401 (applies the import-time fixes)+ import torch+ import torch.nn.functional as F++ dev = next(i for i in range(torch.cuda.device_count()) if torch.cuda.get_device_capability(i)[0] == 10)+ torch.cuda.set_device(dev)+ out = {}++ def grads(fn, D, mask, dtype):+ g = torch.Generator(device = "cuda").manual_seed(0)+ B, H, S = 2, 4, 24+ q, k, v = (torch.randn(B, H, S, D, device = "cuda", dtype = dtype, generator = g).requires_grad_() for _ in range(3))+ m = torch.ones(S, S, device = "cuda", dtype = torch.bool).tril().expand(B, 1, S, S) if mask else None+ o = fn(q, k, v, m)+ o.float().square().sum().backward()+ return [x.grad.float() for x in (q, k, v)]++ def sdpa(q, k, v, m):+ return F.scaled_dot_product_attention(q, k, v, attn_mask = m, is_causal = m is None)++ compiled = torch.compile(sdpa, fullgraph = True, dynamic = True)+ for name, fn in (("eager", sdpa), ("compiled", compiled)):+ for D in (256, 128):+ for mask in (True, False):+ got = grads(fn, D, mask, torch.bfloat16)+ with torch.nn.attention.sdpa_kernel([torch.nn.attention.SDPBackend.MATH]):+ ref = grads(sdpa, D, mask, torch.float32)+ out[f"{name}-D{D}-mask{int(mask)}"] = [+ None if not torch.isfinite(a).all() else float((a - r).norm() / r.norm())+ for a, r in zip(got, ref)+ ]+ # fp32 Q/K/V (DoRA promotes them) under bf16 autocast: SDPA casts to bf16 before dispatch.+ for name, fn in (("eager", sdpa), ("compiled", compiled)):+ with torch.autocast("cuda", dtype = torch.bfloat16):+ got = grads(fn, 256, True, torch.float32)+ with torch.nn.attention.sdpa_kernel([torch.nn.attention.SDPBackend.MATH]):+ ref = grads(sdpa, 256, True, torch.float32)+ out[f"{name}-autocast-fp32-D256-mask1"] = [+ None if not torch.isfinite(a).all() else float((a - r).norm() / r.norm())+ for a, r in zip(got, ref)+ ]+ with torch.no_grad():+ q = torch.randn(1, 2, 8, 256, device = "cuda", dtype = torch.bfloat16)+ m = torch.ones(8, 8, device = "cuda", dtype = torch.bool)+ out["inference_finite"] = bool(torch.isfinite(F.scaled_dot_product_attention(q, q, q, attn_mask = m)).all())+ print("PROBE_RESULT " + json.dumps(out))+ """+)+++def _run_probe(extra_env = None):+ env = dict(os.environ)+ env.update(extra_env or {})+ r = subprocess.run(+ [sys.executable, "-c", _PROBE], capture_output = True, text = True, env = env, timeout = 900+ )+ line = [x for x in r.stdout.splitlines() if x.startswith("PROBE_RESULT ")]+ assert line, f"probe crashed (rc {r.returncode}):\n{r.stdout[-3000:]}\n{r.stderr[-3000:]}"+ return json.loads(line[-1][len("PROBE_RESULT ") :])++[email protected](not _gpu_sm100(), reason = "needs an SM100 (B200 / B300) GPU")+def test_masked_head_dim_256_backward_is_finite_after_import():+ res = _run_probe()+ assert res.pop("inference_finite") is True+ bad = {k: v for k, v in res.items() if any(e is None or e > 2e-2 for e in v)}+ assert not bad, f"non-finite or wrong SDPA gradients (rel err dq, dk, dv vs fp32 math): {bad}"++[email protected](not _gpu_sm100(), reason = "needs an SM100 (B200 / B300) GPU")+def test_opt_out_env_leaves_torch_dispatch_alone():+ """With the opt-out set the wrapper is not installed, so torch's own dispatch runs."""+ code = (+ "import unsloth, torch.nn.functional as F\n"+ "print('WRAPPED', getattr(F.scaled_dot_product_attention, '_unsloth_avoids_cudnn_d256_masked_backward', False))\n"+ )+ for env, want in (({"UNSLOTH_ALLOW_CUDNN_SDPA_D256": "1"}, "False"), ({}, "True")):+ e = dict(os.environ)+ e.pop("UNSLOTH_ALLOW_CUDNN_SDPA_D256", None)+ e.update(env)+ r = subprocess.run(+ [sys.executable, "-c", code], capture_output = True, text = True, env = e, timeout = 600+ )+ assert f"WRAPPED {want}" in r.stdout, (env, r.stdout[-2000:], r.stderr[-2000:])diff --git a/tests/test_flex_head_dim_version_gate.py b/tests/test_flex_head_dim_version_gate.pyindex 31d32ba0ead..fc823584763 100644--- a/tests/test_flex_head_dim_version_gate.py+++ b/tests/test_flex_head_dim_version_gate.py@@ -101,15 +101,43 @@ def test_gate_is_off_below_blackwell(monkeypatch):assert u._sdpa_reaches_cudnn_at_head_dim_256() is False-def test_gate_is_on_for_blackwell_on_torch_2_14(monkeypatch):+def _plain_sdpa(monkeypatch):+ # The SM100 cuDNN head_dim 256 detour wraps SDPA at import on a B200; these gate tests are about plain SDPA.+ sdpa = u.torch.nn.functional.scaled_dot_product_attention+ monkeypatch.setattr(+ u.torch.nn.functional, "scaled_dot_product_attention", getattr(sdpa, "__wrapped__", sdpa)+ )+++def _blackwell_torch_2_14(monkeypatch):monkeypatch.setattr(u.torch, "__version__", "2.14.0+cu130")monkeypatch.setattr(u.torch.version, "hip", None, raising = False)monkeypatch.setattr(u.torch.cuda, "is_available", lambda: True)monkeypatch.setattr(u.torch.cuda, "device_count", lambda: 1)monkeypatch.setattr(u.torch.cuda, "get_device_capability", lambda index: (10, 0))+++def test_gate_is_on_for_blackwell_on_torch_2_14(monkeypatch):+ _plain_sdpa(monkeypatch)+ _blackwell_torch_2_14(monkeypatch)assert u._sdpa_reaches_cudnn_at_head_dim_256() is True+def test_gate_is_off_while_the_cudnn_d256_detour_is_installed(monkeypatch):+ # Masked head_dim 256 training then runs the efficient kernel, not cuDNN: flex is faster, as on torch <= 2.13.+ _blackwell_torch_2_14(monkeypatch)+ sdpa = u.torch.nn.functional.scaled_dot_product_attention+ original = getattr(sdpa, "__wrapped__", sdpa)++ def detour(*args, **kwargs):+ return original(*args, **kwargs)++ detour._unsloth_avoids_cudnn_d256_masked_backward = True+ monkeypatch.setattr(u.torch.nn.functional, "scaled_dot_product_attention", detour)+ assert u._sdpa_reaches_cudnn_at_head_dim_256() is False+ assert u._prefers_flex_for_head_dim(_cfg(256)) is True++def test_a_mixed_box_falls_back_to_the_weakest_card(monkeypatch):monkeypatch.setattr(u.torch, "__version__", "2.14.0+cu130")monkeypatch.setattr(u.torch.version, "hip", None, raising = False)diff --git a/unsloth/_gpu_init.py b/unsloth/_gpu_init.pyindex 6fd5bb0b001..3f8790078df 100644--- a/unsloth/_gpu_init.py+++ b/unsloth/_gpu_init.py@@ -281,6 +281,7 @@ from .import_fixes import (check_vllm_torch_sm100_compatibility,fix_vllm_guided_decoding_params,fix_vllm_pdl_blackwell,+ fix_cudnn_sdpa_d256_masked_backward,fix_triton_compiled_kernel_missing_attrs,fix_dynamo_config_thread_visibility,patch_trunc_normal_precision_issue,@@ -364,6 +365,7 @@ check_vllm_torch_sm100_compatibility()fix_vllm_guided_decoding_params()fix_trl_vllm_ascend()fix_vllm_pdl_blackwell()+fix_cudnn_sdpa_d256_masked_backward()fix_triton_compiled_kernel_missing_attrs()# Must run before unsloth_zoo's patch_torch_compile and the gpt-oss patches raise the dynamo# recompile limits, so those settings reach the autograd worker threads on torch >= 2.12.@@ -422,6 +424,7 @@ del check_vllm_torch_sm100_compatibilitydel fix_vllm_guided_decoding_paramsdel fix_trl_vllm_ascenddel fix_vllm_pdl_blackwell+del fix_cudnn_sdpa_d256_masked_backwarddel fix_triton_compiled_kernel_missing_attrsdel fix_dynamo_config_thread_visibilitydel patch_trunc_normal_precision_issuediff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.pyindex d1c14fd95a9..0edc427ba6a 100644--- a/unsloth/import_fixes.py+++ b/unsloth/import_fixes.py@@ -6515,6 +6515,102 @@ def fix_vllm_pdl_blackwell():logger.info(f"Unsloth: Set TRITON_DISABLE_PDL=1 for SM100 ({sm100_gpu_name})")+_SDPA_CUDNN_D256_FLAG = "_unsloth_avoids_cudnn_d256_masked_backward"+_SDPA_CUDNN_D256_WARNED = False+++def _sdpa_cudnn_d256_sm100_devices():+ try:+ import torch+ if not torch.cuda.is_available() or getattr(torch.version, "hip", None):+ return frozenset()+ return frozenset(+ i+ for i in range(torch.cuda.device_count())+ if torch.cuda.get_device_capability(i)[0] == 10+ )+ except Exception:+ return frozenset()+++def _sdpa_needs_cudnn_d256_detour(sm100_devices, query, key, attn_mask):+ """True for the calls whose cuDNN backward returns NaN dQ on SM100 (no-grad never reaches it)."""+ import torch++ if attn_mask is None or not torch.is_grad_enabled():+ return False+ if not isinstance(query, torch.Tensor) or not query.is_cuda:+ return False+ dtype = query.dtype+ # CUDA autocast casts fp32 SDPA inputs (e.g. DoRA-promoted Q/K/V) to half before dispatch.+ if dtype == torch.float32 and torch.is_autocast_enabled("cuda"):+ dtype = torch.get_autocast_dtype("cuda")+ if dtype not in (torch.float16, torch.bfloat16):+ return False+ if query.shape[-1] != 256:+ return False+ if not (query.requires_grad or (isinstance(key, torch.Tensor) and key.requires_grad)):+ return False+ return query.device.index in sm100_devices+++def fix_cudnn_sdpa_d256_masked_backward():+ """Run masked head_dim-256 SDPA training without cuDNN attention on SM100.++ torch 2.14 (cuDNN 9.24) first dispatches such calls to cuDNN, whose backward returns NaN dQ+ (bf16 / fp16); torch <= 2.13 never picks cuDNN here, so the detour is a no-op+ there. Gated on the device, not the torch version. UNSLOTH_ALLOW_CUDNN_SDPA_D256=1 opts out.+ """+ if os.environ.get("UNSLOTH_ALLOW_CUDNN_SDPA_D256", "0") == "1":+ return+ sm100_devices = _sdpa_cudnn_d256_sm100_devices()+ if not sm100_devices:+ return+ try:+ import torch+ import torch.nn.functional as F+ from torch.nn.attention import SDPBackend, sdpa_kernel+ except Exception as e:+ logger.info(f"Unsloth: Skipping the cuDNN SDPA head_dim 256 fix ({e})")+ return+ if getattr(F.scaled_dot_product_attention, _SDPA_CUDNN_D256_FLAG, False):+ return++ original = F.scaled_dot_product_attention+ backends = [SDPBackend.FLASH_ATTENTION, SDPBackend.EFFICIENT_ATTENTION, SDPBackend.MATH]++ @functools.wraps(original)+ def scaled_dot_product_attention(+ query,+ key,+ value,+ attn_mask = None,+ *args,+ **kwargs,+ ):+ global _SDPA_CUDNN_D256_WARNED+ if _sdpa_needs_cudnn_d256_detour(sm100_devices, query, key, attn_mask):+ if not torch.compiler.is_compiling():+ if not _SDPA_CUDNN_D256_WARNED:+ _SDPA_CUDNN_D256_WARNED = True+ logger.warning(+ "Unsloth: head_dim 256 attention with a mask is training on an SM100 GPU; "+ "running it without cuDNN attention, whose backward returns NaN gradients "+ "there (set UNSLOTH_ALLOW_CUDNN_SDPA_D256=1 to keep cuDNN)."+ )+ with sdpa_kernel(backends):+ return original(query, key, value, attn_mask, *args, **kwargs)+ return original(query, key, value, attn_mask, *args, **kwargs)++ scaled_dot_product_attention.__wrapped__ = original+ setattr(scaled_dot_product_attention, _SDPA_CUDNN_D256_FLAG, True)+ F.scaled_dot_product_attention = scaled_dot_product_attention+ logger.info(+ "Unsloth: SM100 GPU found; masked head_dim 256 SDPA training will avoid cuDNN attention "+ f"(devices {sorted(sm100_devices)})"+ )++def patch_openspiel_env_async():"""Apply nest_asyncio for OpenEnv EnvClient async compatibility.diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.pyindex 6c055053365..88c43c894aa 100644--- a/unsloth/models/_utils.py+++ b/unsloth/models/_utils.py@@ -491,7 +491,11 @@ _CUDNN_LARGE_HEAD_DIM_MIN_CAPABILITY = (10, 0)def _sdpa_reaches_cudnn_at_head_dim_256():- """True when plain SDPA already dispatches cuDNN for a MASKED head_dim 256 on this box."""+ """True when plain SDPA already dispatches cuDNN for a MASKED head_dim 256 on this box.++ False under fix_cudnn_sdpa_d256_masked_backward: it moves that training onto the efficient+ kernel, where flex is faster again.+ """try:if Version(torch.__version__.split("+")[0]) < Version(_CUDNN_LARGE_HEAD_DIM_TORCH_VERSION):return False@@ -499,6 +503,12 @@ def _sdpa_reaches_cudnn_at_head_dim_256():return Falseif not torch.cuda.is_available():return False+ if getattr(+ torch.nn.functional.scaled_dot_product_attention,+ "_unsloth_avoids_cudnn_d256_masked_backward",+ False,+ ):+ return Falsereturn all(torch.cuda.get_device_capability(index) >= _CUDNN_LARGE_HEAD_DIM_MIN_CAPABILITYfor index in range(torch.cuda.device_count())