Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

SDPA training on SM100: keep the caller's backend policy when avoiding cuDNN (#16230)

Daniel Han65e4bc66fc4e
diff --git a/tests/utils/test_attention_training_no_cudnn.py b/tests/utils/test_attention_training_no_cudnn.py
index 5ef8736e33d..17643d33470 100644
--- a/tests/utils/test_attention_training_no_cudnn.py
+++ b/tests/utils/test_attention_training_no_cudnn.py
@@ -1,13 +1,6 @@
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
-"""Training SDPA on SM100 avoids cuDNN attention; eval / generation calls are untouched.
-
-torch 2.13+ on B200 picks cuDNN attention for SDPA, which builds a plan for every new sequence length
-(~150 ms each). Padded training batches change length every step: Qwen3-0.6B LoRA went from 2777-3060
-to 5164-5333 useful tokens/s with flash / efficient / math (exclusive B200, no xformers, interleaved).
-"""
-
import pytest
import torch
import torch.nn.functional as F
@@ -111,12 +104,10 @@ def test_training_sdpa_traces_without_a_break(monkeypatch):
def test_gqa_probe_runs_under_the_training_restriction(monkeypatch):
- """The enable_gqa decision is probed with the kernels the call runs with (cuDNN off)."""
probed = []
def fake_now(*args):
probed.append((args, torch.backends.cuda.cudnn_sdp_enabled()))
- # cuDNN takes this masked GQA call; without it only math would.
return torch.backends.cuda.cudnn_sdp_enabled()
monkeypatch.setattr(ad, "_sdpa_gqa_state_now", fake_now)
@@ -131,7 +122,7 @@ def test_gqa_probe_runs_under_the_training_restriction(monkeypatch):
ad._sdpa_takes_gqa(Q, K, False, True, K, 0.0, no_cudnn = ad._avoids_cudnn(Q, False)) is before
)
assert [cudnn for _, cudnn in probed] == [False, before]
- assert probed[0][0] == probed[1][0] # same shapes, only the backend set differs
+ assert probed[0][0] == probed[1][0]
assert torch.backends.cuda.cudnn_sdp_enabled() == before
assert ad._avoids_cudnn(Q, True) and not ad._avoids_cudnn(Q, False)
monkeypatch.setattr(ad, "_NO_CUDNN_TRAINING", frozenset())
@@ -162,3 +153,103 @@ def test_dense_and_segment_paths_probe_without_cudnn_when_training(monkeypatch):
training = training,
)
assert flags == [True, True, False, False]
+
+
+_POLICIES = {
+ "all": ("FLASH_ATTENTION", "EFFICIENT_ATTENTION", "MATH", "CUDNN_ATTENTION"),
+ "math": ("MATH",),
+ "math_cudnn": ("MATH", "CUDNN_ATTENTION"),
+ "efficient_math_cudnn": ("EFFICIENT_ATTENTION", "MATH", "CUDNN_ATTENTION"),
+ "cudnn": ("CUDNN_ATTENTION",),
+}
+
+
+def _policy(name):
+ from torch.nn.attention import SDPBackend
+ if not hasattr(SDPBackend, "CUDNN_ATTENTION"):
+ pytest.skip("torch has no cuDNN SDPA backend")
+ return [getattr(SDPBackend, b) for b in _POLICIES[name]]
+
+
+def _flags():
+ cuda = torch.backends.cuda
+ return (
+ cuda.flash_sdp_enabled(),
+ cuda.mem_efficient_sdp_enabled(),
+ cuda.math_sdp_enabled(),
+ cuda.cudnn_sdp_enabled(),
+ )
+
+
+def _expected(name):
+ names = _POLICIES[name]
+ keep_cudnn = names == ("CUDNN_ATTENTION",)
+ return (
+ "FLASH_ATTENTION" in names,
+ "EFFICIENT_ATTENTION" in names,
+ "MATH" in names,
+ keep_cudnn,
+ )
+
+
[email protected]("name", list(_POLICIES))
+def test_training_pin_keeps_caller_backend_policy(monkeypatch, name):
+ from torch.nn.attention import sdpa_kernel
+
+ seen = []
+ monkeypatch.setattr(
+ F, "scaled_dot_product_attention", lambda q, k, v, **kw: seen.append(_flags()) or q
+ )
+ monkeypatch.setattr(ad, "_NO_CUDNN_TRAINING", frozenset({None}))
+ Q = torch.randn(1, 2, 6, 8).as_subclass(_CudaLooking)
+ with sdpa_kernel(_policy(name)):
+ before = _flags()
+ ad._training_sdpa(Q, Q, Q, is_causal = True)
+ ad._rerouted_sdpa(Q, Q, Q, is_causal = True)
+ assert _flags() == before
+ assert seen == [_expected(name)] * 2
+
+
[email protected]("name", ["all", "math_cudnn", "cudnn"])
+def test_gqa_probe_uses_the_callers_cudnn_free_set(monkeypatch, name):
+ from torch.nn.attention import sdpa_kernel
+
+ probed = []
+ monkeypatch.setattr(ad, "_sdpa_gqa_state_now", lambda *a: probed.append(_flags()) or True)
+ Q = torch.randn(1, 4, 6, 8).as_subclass(_CudaLooking)
+ K = torch.randn(1, 2, 6, 8).as_subclass(_CudaLooking)
+ with sdpa_kernel(_policy(name)):
+ ad._sdpa_takes_gqa(Q, K, False, True, K, 0.0, no_cudnn = True)
+ assert probed == [_expected(name)]
+
+
[email protected]("name", ["all", "math", "math_cudnn"])
+def test_compiled_training_pin_keeps_caller_backend_policy(monkeypatch, name):
+ from torch._dynamo.utils import counters
+ from torch.nn.attention import sdpa_kernel
+
+ pinned = []
+ real = ad._sdpa_kernel
+
+ def recording(backends):
+ pinned.append(tuple(backends))
+ return real(backends)
+
+ monkeypatch.setattr(ad, "_sdpa_kernel", recording)
+ monkeypatch.setattr(ad, "_avoids_cudnn", lambda q, training: bool(training))
+ torch._dynamo.reset()
+ counters.clear()
+
+ def f(q):
+ return ad._training_sdpa(q, q, q, is_causal = True)
+
+ policy = _policy(name)
+ c = torch.compile(f, backend = "eager", fullgraph = True, dynamic = True)
+ with sdpa_kernel(policy):
+ for L in (5, 9, 16):
+ q = torch.randn(1, 2, L, 8, requires_grad = True)
+ c(q).sum().backward()
+ assert dict(counters["graph_break"]) == {}
+ expected = tuple(b for b in ad._XFORMERS_LIKE_BACKENDS if b in policy)
+ assert set(pinned) == ({expected} if any("CUDNN" in str(b) for b in policy) else set())
+ torch._dynamo.reset()
diff --git a/unsloth/utils/attention_dispatch.py b/unsloth/utils/attention_dispatch.py
index 11d431b03a5..b8fe615ddc2 100644
--- a/unsloth/utils/attention_dispatch.py
+++ b/unsloth/utils/attention_dispatch.py
@@ -107,12 +107,13 @@ def _avoids_cudnn(Q, training):
def _training_sdpa(*args, **kwargs):
- """SDPA for a training forward (AttentionContext.training, passed in, no global state: these
- calls run traced). On SM100 SDPA picks cuDNN attention, which builds a plan for every new
- sequence length (~150 ms each), and training batches change length every step."""
+ """Avoid per-length cuDNN planning on SM100 training calls, including traced forwards."""
if args and _avoids_cudnn(args[0], True):
- with _sdpa_kernel(_XFORMERS_LIKE_BACKENDS):
- return scaled_dot_product_attention(*args, **kwargs)
+ # Narrows the caller's sdpa_kernel policy (MATH stays math); cuDNN-only runs as asked.
+ backends = _CUDNN_FREE_BACKENDS[_cudnn_free_index()]
+ if backends is not None:
+ with _sdpa_kernel(backends):
+ return scaled_dot_product_attention(*args, **kwargs)
if len(args) >= 3 and _cudnn_length_bucket_applies(*args[:3], **kwargs):
return _cudnn_length_bucketed_sdpa(*args, **kwargs)
return scaled_dot_product_attention(*args, **kwargs)
@@ -258,10 +259,11 @@ def _padded_bucket_mask(attn_mask, pad):
def _rerouted_sdpa(*args, **kwargs):
- """SDPA standing in for xformers inside a compiled block, on the kernels xformers would run.
- Without the pin, torch 2.13+ on SM100 takes cuDNN attention, which builds a new plan for every
- new sequence length (~150 ms per length): padded batches of varying length stalled every step."""
- with _sdpa_kernel(_XFORMERS_LIKE_BACKENDS):
+ """Compiled xformers stand-in: avoid cuDNN's per-length planning."""
+ backends = _CUDNN_FREE_BACKENDS[_cudnn_free_index()]
+ if backends is None:
+ return F.scaled_dot_product_attention(*args, **kwargs)
+ with _sdpa_kernel(backends):
return F.scaled_dot_product_attention(*args, **kwargs)
@@ -274,8 +276,7 @@ try:
_SDPBackend.EFFICIENT_ATTENTION,
_SDPBackend.MATH,
]
- # Decode pins, indexed by _decode_cudnn_free_index: the caller's enabled flash / efficient / math.
- _DECODE_CUDNN_FREE_BACKENDS = (None,) + tuple(
+ _CUDNN_FREE_BACKENDS = (None,) + tuple(
[b for j, b in enumerate(_XFORMERS_LIKE_BACKENDS) if i >> j & 1] for i in range(1, 8)
)
_CUDNN_CHOICE = (
@@ -283,7 +284,7 @@ try:
)
except Exception:
_rerouted_sdpa = scaled_dot_product_attention
- _DECODE_CUDNN_FREE_BACKENDS = (None,) * 8
+ _CUDNN_FREE_BACKENDS = (None,) * 8
_CUDNN_CHOICE = None
@@ -391,14 +392,14 @@ def decode_scaled_dot_product_attention(query, key, value, *args, **kwargs):
):
# Only the caller's own non-cuDNN backends (sdpa_kernel([MATH]) stays math); a cuDNN-only
# policy is left as is.
- backends = _DECODE_CUDNN_FREE_BACKENDS[_decode_cudnn_free_index()]
+ backends = _CUDNN_FREE_BACKENDS[_cudnn_free_index()]
if backends is not None:
with _sdpa_kernel(backends):
return F.scaled_dot_product_attention(query, key, value, *args, **kwargs)
return F.scaled_dot_product_attention(query, key, value, *args, **kwargs)
-def _decode_cudnn_free_index():
+def _cudnn_free_index():
"""Bit 0 / 1 / 2: flash / efficient / math SDPA enabled; 0 when cuDNN is disabled (nothing to avoid)."""
cuda = torch.backends.cuda
if not cuda.cudnn_sdp_enabled():
@@ -755,8 +756,7 @@ def _constant_probe(fn):
_sdpa_backend_state = _constant_probe(_sdpa_backend_state)
_sdpa_masked_choice = _constant_probe(_sdpa_masked_choice)
-# Traced: read under the caller's sdpa_kernel at trace time, like SDPA's own backend pick.
-_decode_cudnn_free_index = _constant_probe(_decode_cudnn_free_index)
+_cudnn_free_index = _constant_probe(_cudnn_free_index)
@functools.lru_cache(maxsize = 128)
@@ -871,8 +871,9 @@ def _sdpa_gqa_state_now(
def _sdpa_gqa_state(*args, no_cudnn = False):
"""_sdpa_gqa_state_now; no_cudnn probes under the kernels the call runs with (_training_sdpa,
_rerouted_sdpa), whose enabled flags are part of the probe's cache key."""
- if no_cudnn:
- with _sdpa_kernel(_XFORMERS_LIKE_BACKENDS):
+ backends = _CUDNN_FREE_BACKENDS[_cudnn_free_index()] if no_cudnn else None
+ if backends is not None:
+ with _sdpa_kernel(backends):
return _sdpa_gqa_state_now(*args)
return _sdpa_gqa_state_now(*args)