Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Attention: keep training SDPA off cuDNN attention on SM100 (#15019)

Daniel Han5b258ae93f29
diff --git a/tests/utils/test_attention_training_no_cudnn.py b/tests/utils/test_attention_training_no_cudnn.py
new file mode 100644
index 00000000000..5ef8736e33d
--- /dev/null
+++ b/tests/utils/test_attention_training_no_cudnn.py
@@ -0,0 +1,164 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
+
+"""Training SDPA on SM100 avoids cuDNN attention; eval / generation calls are untouched.
+
+torch 2.13+ on B200 picks cuDNN attention for SDPA, which builds a plan for every new sequence length
+(~150 ms each). Padded training batches change length every step: Qwen3-0.6B LoRA went from 2777-3060
+to 5164-5333 useful tokens/s with flash / efficient / math (exclusive B200, no xformers, interleaved).
+"""
+
+import pytest
+import torch
+import torch.nn.functional as F
+
+ad = pytest.importorskip("unsloth.utils.attention_dispatch")
+
+
+class _CudaLooking(torch.Tensor):
+ @property
+ def is_cuda(self):
+ return True
+
+
+def _run(
+ training,
+ monkeypatch,
+ devices = frozenset({None}),
+):
+ seen = []
+
+ def fake_sdpa(q, k, v, **kwargs):
+ seen.append(torch.backends.cuda.cudnn_sdp_enabled())
+ return torch.zeros_like(q)
+
+ monkeypatch.setattr(F, "scaled_dot_product_attention", fake_sdpa)
+ monkeypatch.setattr(ad, "_NO_CUDNN_TRAINING", devices)
+ Q = torch.randn(1, 2, 6, 8).as_subclass(_CudaLooking)
+ config = ad.AttentionConfig(backend = ad.SDPA, n_kv_heads = 2, n_groups = 1)
+ context = ad.AttentionContext(
+ bsz = 1,
+ q_len = 6,
+ kv_seq_len = 6,
+ n_heads = 2,
+ head_dim = 8,
+ requires_grad = False,
+ seq_info = None,
+ attention_mask = None,
+ causal_mask = None,
+ training = training,
+ )
+ ad.run_attention(config = config, context = context, Q = Q, K = Q, V = Q)
+ return seen
+
+
+def test_training_call_disables_cudnn(monkeypatch):
+ before = torch.backends.cuda.cudnn_sdp_enabled()
+ assert _run(True, monkeypatch) == [False]
+ assert torch.backends.cuda.cudnn_sdp_enabled() == before
+
+
+def test_eval_call_and_other_devices_untouched(monkeypatch):
+ before = torch.backends.cuda.cudnn_sdp_enabled()
+ assert _run(False, monkeypatch) == [before]
+ assert _run(True, monkeypatch, devices = frozenset()) == [before]
+
+
+def test_only_sm100_devices_and_opt_out(monkeypatch):
+ monkeypatch.setattr(ad, "_visible_capabilities", lambda: ((9, 0), (10, 0), (12, 0), (10, 3)))
+ monkeypatch.delenv("UNSLOTH_SDPA_TRAIN_CUDNN", raising = False)
+ assert ad._no_cudnn_training_devices() == frozenset({1, 3})
+ monkeypatch.setenv("UNSLOTH_SDPA_TRAIN_CUDNN", "1")
+ assert ad._no_cudnn_training_devices() == frozenset()
+
+
+def test_checkpointed_no_grad_forward_still_pins(monkeypatch):
+ """A checkpointed layer's first forward runs inside autograd.Function.forward (grad off)."""
+
+ class Ckpt(torch.autograd.Function):
+ @staticmethod
+ def forward(ctx, x):
+ assert not torch.is_grad_enabled()
+ seen.extend(_run(True, monkeypatch))
+ return x.clone()
+
+ @staticmethod
+ def backward(ctx, g):
+ return g
+
+ seen = []
+ x = torch.randn(2, requires_grad = True)
+ Ckpt.apply(x)
+ assert seen == [False]
+
+
+def test_training_sdpa_traces_without_a_break(monkeypatch):
+ from torch._dynamo.utils import counters
+
+ monkeypatch.setattr(ad, "_NO_CUDNN_TRAINING", frozenset({None}))
+ torch._dynamo.reset()
+ counters.clear()
+
+ def f(q):
+ return ad._training_sdpa(q, q, q, is_causal = True)
+
+ c = torch.compile(f, backend = "eager", dynamic = True)
+ for L in (5, 9, 16):
+ q = torch.randn(1, 2, L, 8)
+ torch.testing.assert_close(c(q), F.scaled_dot_product_attention(q, q, q, is_causal = True))
+ assert dict(counters["graph_break"]) == {}
+ torch._dynamo.reset()
+
+
+def test_gqa_probe_runs_under_the_training_restriction(monkeypatch):
+ """The enable_gqa decision is probed with the kernels the call runs with (cuDNN off)."""
+ probed = []
+
+ def fake_now(*args):
+ probed.append((args, torch.backends.cuda.cudnn_sdp_enabled()))
+ # cuDNN takes this masked GQA call; without it only math would.
+ return torch.backends.cuda.cudnn_sdp_enabled()
+
+ monkeypatch.setattr(ad, "_sdpa_gqa_state_now", fake_now)
+ monkeypatch.setattr(ad, "_NO_CUDNN_TRAINING", frozenset({None}))
+ Q = torch.randn(1, 4, 6, 8).as_subclass(_CudaLooking)
+ K = torch.randn(1, 2, 6, 8).as_subclass(_CudaLooking)
+ before = torch.backends.cuda.cudnn_sdp_enabled()
+ assert (
+ ad._sdpa_takes_gqa(Q, K, False, True, K, 0.0, no_cudnn = ad._avoids_cudnn(Q, True)) is False
+ )
+ assert (
+ ad._sdpa_takes_gqa(Q, K, False, True, K, 0.0, no_cudnn = ad._avoids_cudnn(Q, False)) is before
+ )
+ assert [cudnn for _, cudnn in probed] == [False, before]
+ assert probed[0][0] == probed[1][0] # same shapes, only the backend set differs
+ assert torch.backends.cuda.cudnn_sdp_enabled() == before
+ assert ad._avoids_cudnn(Q, True) and not ad._avoids_cudnn(Q, False)
+ monkeypatch.setattr(ad, "_NO_CUDNN_TRAINING", frozenset())
+ assert not ad._avoids_cudnn(Q, True)
+
+
+def test_dense_and_segment_paths_probe_without_cudnn_when_training(monkeypatch):
+ flags = []
+ monkeypatch.setattr(ad, "_NO_CUDNN_TRAINING", frozenset({None}))
+ monkeypatch.setattr(
+ ad, "_sdpa_takes_gqa", lambda *a, no_cudnn = False, **k: flags.append(no_cudnn) or False
+ )
+ monkeypatch.setattr(
+ F, "scaled_dot_product_attention", lambda q, k, v, **kw: torch.zeros_like(q)
+ )
+ Q = torch.randn(1, 4, 6, 8).as_subclass(_CudaLooking)
+ K = torch.randn(1, 2, 6, 8).as_subclass(_CudaLooking)
+ for training in (True, False):
+ ad._sdpa_packed_segments(
+ Q,
+ K,
+ K,
+ (2, 4),
+ n_groups = 2,
+ is_causal = True,
+ sliding_window = None,
+ sdpa_kwargs = {},
+ training = training,
+ )
+ assert flags == [True, True, False, False]
diff --git a/unsloth/models/cohere.py b/unsloth/models/cohere.py
index 00461d6d327..d96a6f01224 100644
--- a/unsloth/models/cohere.py
+++ b/unsloth/models/cohere.py
@@ -168,6 +168,7 @@ def CohereAttention_fast_forward(
head_dim = head_dim,
requires_grad = hidden_states.requires_grad,
seq_info = seq_info,
+ training = self.training,
attention_mask = attention_mask,
causal_mask = causal_mask,
prefix_seg_info = _pg_seg,
diff --git a/unsloth/models/falcon_h1.py b/unsloth/models/falcon_h1.py
index 338140aac8b..8858c40fed5 100644
--- a/unsloth/models/falcon_h1.py
+++ b/unsloth/models/falcon_h1.py
@@ -167,6 +167,7 @@ def FalconH1Attention_fast_forward(
head_dim = head_dim,
requires_grad = hidden_states.requires_grad,
seq_info = seq_info,
+ training = self.training,
attention_mask = attention_mask,
causal_mask = causal_mask,
)
diff --git a/unsloth/models/gemma2.py b/unsloth/models/gemma2.py
index fa1ef3a8605..b5240b2470f 100644
--- a/unsloth/models/gemma2.py
+++ b/unsloth/models/gemma2.py
@@ -244,6 +244,7 @@ def Gemma2Attention_fast_forward(
head_dim = head_dim,
requires_grad = hidden_states.requires_grad,
seq_info = seq_info,
+ training = self.training,
attention_mask = attention_mask,
causal_mask = causal_mask,
sliding_window = sliding_window,
diff --git a/unsloth/models/granite.py b/unsloth/models/granite.py
index e980bd84298..2fea7d7c85d 100644
--- a/unsloth/models/granite.py
+++ b/unsloth/models/granite.py
@@ -188,6 +188,7 @@ def GraniteAttention_fast_forward(
head_dim = head_dim,
requires_grad = hidden_states.requires_grad,
seq_info = seq_info,
+ training = self.training,
attention_mask = attention_mask,
causal_mask = causal_mask,
prefix_seg_info = _pg_seg,
diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py
index 3ced5f471fa..db388bf568e 100644
--- a/unsloth/models/llama.py
+++ b/unsloth/models/llama.py
@@ -1060,6 +1060,7 @@ def LlamaAttention_fast_forward(
head_dim = head_dim,
requires_grad = hidden_states.requires_grad,
seq_info = seq_info,
+ training = self.training,
attention_mask = attention_mask,
causal_mask = causal_mask,
prefix_seg_info = _pg_seg,
diff --git a/unsloth/models/mistral.py b/unsloth/models/mistral.py
index 89334513380..812818f2a00 100644
--- a/unsloth/models/mistral.py
+++ b/unsloth/models/mistral.py
@@ -155,6 +155,7 @@ def MistralAttention_fast_forward(
head_dim = head_dim,
requires_grad = hidden_states.requires_grad,
seq_info = seq_info,
+ training = self.training,
attention_mask = attention_mask,
causal_mask = causal_mask,
# The window the flash path already gets through window_size. SDPA needs it too, and not only in
diff --git a/unsloth/models/qwen3.py b/unsloth/models/qwen3.py
index 4ffb91e710b..cef4a20b2f8 100644
--- a/unsloth/models/qwen3.py
+++ b/unsloth/models/qwen3.py
@@ -158,6 +158,7 @@ def Qwen3Attention_fast_forward(
head_dim = head_dim,
requires_grad = hidden_states.requires_grad,
seq_info = seq_info,
+ training = self.training,
attention_mask = attention_mask,
causal_mask = causal_mask,
prefix_seg_info = _pg_seg,
diff --git a/unsloth/utils/attention_dispatch.py b/unsloth/utils/attention_dispatch.py
index b3beac205aa..5a8d4629bfd 100644
--- a/unsloth/utils/attention_dispatch.py
+++ b/unsloth/utils/attention_dispatch.py
@@ -99,6 +99,21 @@ def scaled_dot_product_attention(*args, **kwargs):
return F.scaled_dot_product_attention(*args, **kwargs)
+def _avoids_cudnn(Q, training):
+ """Training calls on SM100 run without cuDNN attention (_training_sdpa); probes must match."""
+ return bool(training) and Q.is_cuda and Q.device.index in _NO_CUDNN_TRAINING
+
+
+def _training_sdpa(*args, **kwargs):
+ """SDPA for a training forward (AttentionContext.training, passed in, no global state: these
+ calls run traced). On SM100 SDPA picks cuDNN attention, which builds a plan for every new
+ sequence length (~150 ms each), and training batches change length every step."""
+ if args and _avoids_cudnn(args[0], True):
+ with _sdpa_kernel(_XFORMERS_LIKE_BACKENDS):
+ return scaled_dot_product_attention(*args, **kwargs)
+ return scaled_dot_product_attention(*args, **kwargs)
+
+
def _rerouted_sdpa(*args, **kwargs):
"""SDPA standing in for xformers inside a compiled block, on the kernels xformers would run.
Without the pin, torch 2.13+ on SM100 takes cuDNN attention, which builds a new plan for every
@@ -147,6 +162,16 @@ def _visible_capabilities():
return ()
+def _no_cudnn_training_devices():
+ """Device indices whose training SDPA calls avoid cuDNN attention (SM100). UNSLOTH_SDPA_TRAIN_CUDNN=1 keeps it."""
+ if os.environ.get("UNSLOTH_SDPA_TRAIN_CUDNN", "0") == "1" or torch.version.hip is not None:
+ return frozenset()
+ return frozenset(i for i, cap in enumerate(_visible_capabilities()) if cap[0] == 10)
+
+
+_NO_CUDNN_TRAINING = _no_cudnn_training_devices()
+
+
# Mixed-generation hosts only (e.g. sm_86 + sm_75): the backend is re-checked against Q's device, since a
# device-0 verdict is wrong for layers a device_map places on another card.
_DEVICE_BACKENDS = _per_device_backends(_visible_capabilities())
@@ -384,7 +409,7 @@ def _sdpa_gqa_choices(
_SDPA_NO_BACKEND = -2
-def _sdpa_gqa_state(
+def _sdpa_gqa_state_now(
device,
dtype,
head_dim,
@@ -434,6 +459,15 @@ def _sdpa_gqa_state(
return not (gqa == math and mha not in (math, -1))
+def _sdpa_gqa_state(*args, no_cudnn = False):
+ """_sdpa_gqa_state_now; no_cudnn probes under the kernels the call runs with (_training_sdpa,
+ _rerouted_sdpa), whose enabled flags are part of the probe's cache key."""
+ if no_cudnn:
+ with _sdpa_kernel(_XFORMERS_LIKE_BACKENDS):
+ return _sdpa_gqa_state_now(*args)
+ return _sdpa_gqa_state_now(*args)
+
+
_sdpa_gqa_state = _constant_probe(_sdpa_gqa_state)
@@ -444,8 +478,10 @@ def _sdpa_takes_gqa(
has_mask: bool,
V: Optional[Tensor] = None,
dropout_p = 0.0,
+ no_cudnn = False,
) -> bool:
- """Whether to pass enable_gqa (True) or expand K/V (False) for this SDPA call."""
+ """Whether to pass enable_gqa (True) or expand K/V (False) for this SDPA call. no_cudnn: the
+ call runs without cuDNN attention (SM100 training, xformers stand-in), so the probe does too."""
if not SDPA_HAS_GQA:
return False
if not Q.is_cuda:
@@ -458,7 +494,7 @@ def _sdpa_takes_gqa(
single_key = True if K.shape[-2] == 1 else False
is_causal = True if is_causal else False
has_mask = True if has_mask else False
- return _sdpa_gqa_state(
+ args = (
str(Q.device),
Q.dtype,
Q.shape[-1],
@@ -471,6 +507,9 @@ def _sdpa_takes_gqa(
single_query,
single_key,
)
+ if no_cudnn:
+ return _sdpa_gqa_state(*args, no_cudnn = True)
+ return _sdpa_gqa_state(*args)
def _torch_varlen_beats_segments(
@@ -569,6 +608,7 @@ def _sdpa_packed_segments(
is_causal: bool,
sliding_window: Optional[int],
sdpa_kwargs: dict,
+ training: bool = False,
) -> Optional[Tensor]:
"""Q (1, H, T, D), K / V (1, H_kv, T, D) -> (1, T, H, D), the block-diagonal mask's result;
None when the lengths do not tile the row."""
@@ -614,12 +654,21 @@ def _sdpa_packed_segments(
Q_run = split(Q)
# Probe the run's own tensors: a one-token run has different eligibility than the row.
gqa = n_groups != 1 and _sdpa_takes_gqa(
- Q_run, split(K), kwargs["is_causal"], windowed, split(V), kwargs.get("dropout_p", 0.0)
+ Q_run,
+ split(K),
+ kwargs["is_causal"],
+ windowed,
+ split(V),
+ kwargs.get("dropout_p", 0.0),
+ # Passed only when set: callers and tests wrap the probe with the old signature.
+ **({"no_cudnn": True} if _avoids_cudnn(Q_run, training) else {}),
)
if gqa:
kwargs["enable_gqa"] = True
K_run, V_run = kv(gqa)
- out = scaled_dot_product_attention(Q_run, split(K_run), split(V_run), **kwargs)
+ out = (_training_sdpa if training else scaled_dot_product_attention)(
+ Q_run, split(K_run), split(V_run), **kwargs
+ )
outs.append(out.transpose(1, 2).flatten(0, 1))
start, i = end, j
return torch.cat(outs, dim = 0).unsqueeze(0)
@@ -751,6 +800,8 @@ class AttentionContext:
# every existing construction and behavior unchanged.
prefix_seg_info: Optional[Any] = None
is_causal: bool = True
+ # The attention module's .training: also true for a checkpointed forward run without grad.
+ training: bool = False
def _ring_attention_active() -> bool:
@@ -1193,6 +1244,7 @@ def _run_attention(
is_causal = context.is_causal,
sliding_window = sliding_window,
sdpa_kwargs = sdpa_kwargs,
+ training = context.training,
)
if out is not None:
return out
@@ -1267,6 +1319,7 @@ def _run_attention(
)
kwargs = dict(sdpa_kwargs)
+ sdpa = _training_sdpa if context.training else scaled_dot_product_attention
kwargs.setdefault("attn_mask", local_mask)
kwargs.setdefault("is_causal", is_causal_local)
@@ -1296,14 +1349,17 @@ def _run_attention(
kwargs.get("attn_mask") is not None,
V,
kwargs.get("dropout_p", 0.0),
+ **(
+ {"no_cudnn": True}
+ if xformers_rerouted or _avoids_cudnn(Q, context.training)
+ else {}
+ ),
):
use_sdpa_gqa = False
if use_sdpa_gqa:
kwargs.setdefault("enable_gqa", True)
- out = (_rerouted_sdpa if xformers_rerouted else scaled_dot_product_attention)(
- Q, K, V, **kwargs
- )
+ out = (_rerouted_sdpa if xformers_rerouted else sdpa)(Q, K, V, **kwargs)
out = out.transpose(1, 2)
return out if cp_out_dtype is None else out.to(cp_out_dtype)
@@ -1319,7 +1375,7 @@ def _run_attention(
K_mod = K_mod.reshape(bsz, n_heads, kv_seq_len, head_dim)
V_mod = V_mod.reshape(bsz, n_heads, kv_seq_len, head_dim)
- out = (_rerouted_sdpa if xformers_rerouted else scaled_dot_product_attention)(
+ out = (_rerouted_sdpa if xformers_rerouted else sdpa)(
Q.contiguous(),
K_mod.contiguous(),
V_mod.contiguous(),