Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Attention: keep training SDPA off cuDNN attention on SM100 (#15019)
Daniel Han5b258ae93f29
diff --git a/tests/utils/test_attention_training_no_cudnn.py b/tests/utils/test_attention_training_no_cudnn.pynew file mode 100644index 00000000000..5ef8736e33d--- /dev/null+++ b/tests/utils/test_attention_training_no_cudnn.py@@ -0,0 +1,164 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.++"""Training SDPA on SM100 avoids cuDNN attention; eval / generation calls are untouched.++torch 2.13+ on B200 picks cuDNN attention for SDPA, which builds a plan for every new sequence length+(~150 ms each). Padded training batches change length every step: Qwen3-0.6B LoRA went from 2777-3060+to 5164-5333 useful tokens/s with flash / efficient / math (exclusive B200, no xformers, interleaved).+"""++import pytest+import torch+import torch.nn.functional as F++ad = pytest.importorskip("unsloth.utils.attention_dispatch")+++class _CudaLooking(torch.Tensor):+ @property+ def is_cuda(self):+ return True+++def _run(+ training,+ monkeypatch,+ devices = frozenset({None}),+):+ seen = []++ def fake_sdpa(q, k, v, **kwargs):+ seen.append(torch.backends.cuda.cudnn_sdp_enabled())+ return torch.zeros_like(q)++ monkeypatch.setattr(F, "scaled_dot_product_attention", fake_sdpa)+ monkeypatch.setattr(ad, "_NO_CUDNN_TRAINING", devices)+ Q = torch.randn(1, 2, 6, 8).as_subclass(_CudaLooking)+ config = ad.AttentionConfig(backend = ad.SDPA, n_kv_heads = 2, n_groups = 1)+ context = ad.AttentionContext(+ bsz = 1,+ q_len = 6,+ kv_seq_len = 6,+ n_heads = 2,+ head_dim = 8,+ requires_grad = False,+ seq_info = None,+ attention_mask = None,+ causal_mask = None,+ training = training,+ )+ ad.run_attention(config = config, context = context, Q = Q, K = Q, V = Q)+ return seen+++def test_training_call_disables_cudnn(monkeypatch):+ before = torch.backends.cuda.cudnn_sdp_enabled()+ assert _run(True, monkeypatch) == [False]+ assert torch.backends.cuda.cudnn_sdp_enabled() == before+++def test_eval_call_and_other_devices_untouched(monkeypatch):+ before = torch.backends.cuda.cudnn_sdp_enabled()+ assert _run(False, monkeypatch) == [before]+ assert _run(True, monkeypatch, devices = frozenset()) == [before]+++def test_only_sm100_devices_and_opt_out(monkeypatch):+ monkeypatch.setattr(ad, "_visible_capabilities", lambda: ((9, 0), (10, 0), (12, 0), (10, 3)))+ monkeypatch.delenv("UNSLOTH_SDPA_TRAIN_CUDNN", raising = False)+ assert ad._no_cudnn_training_devices() == frozenset({1, 3})+ monkeypatch.setenv("UNSLOTH_SDPA_TRAIN_CUDNN", "1")+ assert ad._no_cudnn_training_devices() == frozenset()+++def test_checkpointed_no_grad_forward_still_pins(monkeypatch):+ """A checkpointed layer's first forward runs inside autograd.Function.forward (grad off)."""++ class Ckpt(torch.autograd.Function):+ @staticmethod+ def forward(ctx, x):+ assert not torch.is_grad_enabled()+ seen.extend(_run(True, monkeypatch))+ return x.clone()++ @staticmethod+ def backward(ctx, g):+ return g++ seen = []+ x = torch.randn(2, requires_grad = True)+ Ckpt.apply(x)+ assert seen == [False]+++def test_training_sdpa_traces_without_a_break(monkeypatch):+ from torch._dynamo.utils import counters++ monkeypatch.setattr(ad, "_NO_CUDNN_TRAINING", frozenset({None}))+ torch._dynamo.reset()+ counters.clear()++ def f(q):+ return ad._training_sdpa(q, q, q, is_causal = True)++ c = torch.compile(f, backend = "eager", dynamic = True)+ for L in (5, 9, 16):+ q = torch.randn(1, 2, L, 8)+ torch.testing.assert_close(c(q), F.scaled_dot_product_attention(q, q, q, is_causal = True))+ assert dict(counters["graph_break"]) == {}+ torch._dynamo.reset()+++def test_gqa_probe_runs_under_the_training_restriction(monkeypatch):+ """The enable_gqa decision is probed with the kernels the call runs with (cuDNN off)."""+ probed = []++ def fake_now(*args):+ probed.append((args, torch.backends.cuda.cudnn_sdp_enabled()))+ # cuDNN takes this masked GQA call; without it only math would.+ return torch.backends.cuda.cudnn_sdp_enabled()++ monkeypatch.setattr(ad, "_sdpa_gqa_state_now", fake_now)+ monkeypatch.setattr(ad, "_NO_CUDNN_TRAINING", frozenset({None}))+ Q = torch.randn(1, 4, 6, 8).as_subclass(_CudaLooking)+ K = torch.randn(1, 2, 6, 8).as_subclass(_CudaLooking)+ before = torch.backends.cuda.cudnn_sdp_enabled()+ assert (+ ad._sdpa_takes_gqa(Q, K, False, True, K, 0.0, no_cudnn = ad._avoids_cudnn(Q, True)) is False+ )+ assert (+ ad._sdpa_takes_gqa(Q, K, False, True, K, 0.0, no_cudnn = ad._avoids_cudnn(Q, False)) is before+ )+ assert [cudnn for _, cudnn in probed] == [False, before]+ assert probed[0][0] == probed[1][0] # same shapes, only the backend set differs+ assert torch.backends.cuda.cudnn_sdp_enabled() == before+ assert ad._avoids_cudnn(Q, True) and not ad._avoids_cudnn(Q, False)+ monkeypatch.setattr(ad, "_NO_CUDNN_TRAINING", frozenset())+ assert not ad._avoids_cudnn(Q, True)+++def test_dense_and_segment_paths_probe_without_cudnn_when_training(monkeypatch):+ flags = []+ monkeypatch.setattr(ad, "_NO_CUDNN_TRAINING", frozenset({None}))+ monkeypatch.setattr(+ ad, "_sdpa_takes_gqa", lambda *a, no_cudnn = False, **k: flags.append(no_cudnn) or False+ )+ monkeypatch.setattr(+ F, "scaled_dot_product_attention", lambda q, k, v, **kw: torch.zeros_like(q)+ )+ Q = torch.randn(1, 4, 6, 8).as_subclass(_CudaLooking)+ K = torch.randn(1, 2, 6, 8).as_subclass(_CudaLooking)+ for training in (True, False):+ ad._sdpa_packed_segments(+ Q,+ K,+ K,+ (2, 4),+ n_groups = 2,+ is_causal = True,+ sliding_window = None,+ sdpa_kwargs = {},+ training = training,+ )+ assert flags == [True, True, False, False]diff --git a/unsloth/models/cohere.py b/unsloth/models/cohere.pyindex 00461d6d327..d96a6f01224 100644--- a/unsloth/models/cohere.py+++ b/unsloth/models/cohere.py@@ -168,6 +168,7 @@ def CohereAttention_fast_forward(head_dim = head_dim,requires_grad = hidden_states.requires_grad,seq_info = seq_info,+ training = self.training,attention_mask = attention_mask,causal_mask = causal_mask,prefix_seg_info = _pg_seg,diff --git a/unsloth/models/falcon_h1.py b/unsloth/models/falcon_h1.pyindex 338140aac8b..8858c40fed5 100644--- a/unsloth/models/falcon_h1.py+++ b/unsloth/models/falcon_h1.py@@ -167,6 +167,7 @@ def FalconH1Attention_fast_forward(head_dim = head_dim,requires_grad = hidden_states.requires_grad,seq_info = seq_info,+ training = self.training,attention_mask = attention_mask,causal_mask = causal_mask,)diff --git a/unsloth/models/gemma2.py b/unsloth/models/gemma2.pyindex fa1ef3a8605..b5240b2470f 100644--- a/unsloth/models/gemma2.py+++ b/unsloth/models/gemma2.py@@ -244,6 +244,7 @@ def Gemma2Attention_fast_forward(head_dim = head_dim,requires_grad = hidden_states.requires_grad,seq_info = seq_info,+ training = self.training,attention_mask = attention_mask,causal_mask = causal_mask,sliding_window = sliding_window,diff --git a/unsloth/models/granite.py b/unsloth/models/granite.pyindex e980bd84298..2fea7d7c85d 100644--- a/unsloth/models/granite.py+++ b/unsloth/models/granite.py@@ -188,6 +188,7 @@ def GraniteAttention_fast_forward(head_dim = head_dim,requires_grad = hidden_states.requires_grad,seq_info = seq_info,+ training = self.training,attention_mask = attention_mask,causal_mask = causal_mask,prefix_seg_info = _pg_seg,diff --git a/unsloth/models/llama.py b/unsloth/models/llama.pyindex 3ced5f471fa..db388bf568e 100644--- a/unsloth/models/llama.py+++ b/unsloth/models/llama.py@@ -1060,6 +1060,7 @@ def LlamaAttention_fast_forward(head_dim = head_dim,requires_grad = hidden_states.requires_grad,seq_info = seq_info,+ training = self.training,attention_mask = attention_mask,causal_mask = causal_mask,prefix_seg_info = _pg_seg,diff --git a/unsloth/models/mistral.py b/unsloth/models/mistral.pyindex 89334513380..812818f2a00 100644--- a/unsloth/models/mistral.py+++ b/unsloth/models/mistral.py@@ -155,6 +155,7 @@ def MistralAttention_fast_forward(head_dim = head_dim,requires_grad = hidden_states.requires_grad,seq_info = seq_info,+ training = self.training,attention_mask = attention_mask,causal_mask = causal_mask,# The window the flash path already gets through window_size. SDPA needs it too, and not only indiff --git a/unsloth/models/qwen3.py b/unsloth/models/qwen3.pyindex 4ffb91e710b..cef4a20b2f8 100644--- a/unsloth/models/qwen3.py+++ b/unsloth/models/qwen3.py@@ -158,6 +158,7 @@ def Qwen3Attention_fast_forward(head_dim = head_dim,requires_grad = hidden_states.requires_grad,seq_info = seq_info,+ training = self.training,attention_mask = attention_mask,causal_mask = causal_mask,prefix_seg_info = _pg_seg,diff --git a/unsloth/utils/attention_dispatch.py b/unsloth/utils/attention_dispatch.pyindex b3beac205aa..5a8d4629bfd 100644--- a/unsloth/utils/attention_dispatch.py+++ b/unsloth/utils/attention_dispatch.py@@ -99,6 +99,21 @@ def scaled_dot_product_attention(*args, **kwargs):return F.scaled_dot_product_attention(*args, **kwargs)+def _avoids_cudnn(Q, training):+ """Training calls on SM100 run without cuDNN attention (_training_sdpa); probes must match."""+ return bool(training) and Q.is_cuda and Q.device.index in _NO_CUDNN_TRAINING+++def _training_sdpa(*args, **kwargs):+ """SDPA for a training forward (AttentionContext.training, passed in, no global state: these+ calls run traced). On SM100 SDPA picks cuDNN attention, which builds a plan for every new+ sequence length (~150 ms each), and training batches change length every step."""+ if args and _avoids_cudnn(args[0], True):+ with _sdpa_kernel(_XFORMERS_LIKE_BACKENDS):+ return scaled_dot_product_attention(*args, **kwargs)+ return scaled_dot_product_attention(*args, **kwargs)++def _rerouted_sdpa(*args, **kwargs):"""SDPA standing in for xformers inside a compiled block, on the kernels xformers would run.Without the pin, torch 2.13+ on SM100 takes cuDNN attention, which builds a new plan for every@@ -147,6 +162,16 @@ def _visible_capabilities():return ()+def _no_cudnn_training_devices():+ """Device indices whose training SDPA calls avoid cuDNN attention (SM100). UNSLOTH_SDPA_TRAIN_CUDNN=1 keeps it."""+ if os.environ.get("UNSLOTH_SDPA_TRAIN_CUDNN", "0") == "1" or torch.version.hip is not None:+ return frozenset()+ return frozenset(i for i, cap in enumerate(_visible_capabilities()) if cap[0] == 10)+++_NO_CUDNN_TRAINING = _no_cudnn_training_devices()++# Mixed-generation hosts only (e.g. sm_86 + sm_75): the backend is re-checked against Q's device, since a# device-0 verdict is wrong for layers a device_map places on another card._DEVICE_BACKENDS = _per_device_backends(_visible_capabilities())@@ -384,7 +409,7 @@ def _sdpa_gqa_choices(_SDPA_NO_BACKEND = -2-def _sdpa_gqa_state(+def _sdpa_gqa_state_now(device,dtype,head_dim,@@ -434,6 +459,15 @@ def _sdpa_gqa_state(return not (gqa == math and mha not in (math, -1))+def _sdpa_gqa_state(*args, no_cudnn = False):+ """_sdpa_gqa_state_now; no_cudnn probes under the kernels the call runs with (_training_sdpa,+ _rerouted_sdpa), whose enabled flags are part of the probe's cache key."""+ if no_cudnn:+ with _sdpa_kernel(_XFORMERS_LIKE_BACKENDS):+ return _sdpa_gqa_state_now(*args)+ return _sdpa_gqa_state_now(*args)++_sdpa_gqa_state = _constant_probe(_sdpa_gqa_state)@@ -444,8 +478,10 @@ def _sdpa_takes_gqa(has_mask: bool,V: Optional[Tensor] = None,dropout_p = 0.0,+ no_cudnn = False,) -> bool:- """Whether to pass enable_gqa (True) or expand K/V (False) for this SDPA call."""+ """Whether to pass enable_gqa (True) or expand K/V (False) for this SDPA call. no_cudnn: the+ call runs without cuDNN attention (SM100 training, xformers stand-in), so the probe does too."""if not SDPA_HAS_GQA:return Falseif not Q.is_cuda:@@ -458,7 +494,7 @@ def _sdpa_takes_gqa(single_key = True if K.shape[-2] == 1 else Falseis_causal = True if is_causal else Falsehas_mask = True if has_mask else False- return _sdpa_gqa_state(+ args = (str(Q.device),Q.dtype,Q.shape[-1],@@ -471,6 +507,9 @@ def _sdpa_takes_gqa(single_query,single_key,)+ if no_cudnn:+ return _sdpa_gqa_state(*args, no_cudnn = True)+ return _sdpa_gqa_state(*args)def _torch_varlen_beats_segments(@@ -569,6 +608,7 @@ def _sdpa_packed_segments(is_causal: bool,sliding_window: Optional[int],sdpa_kwargs: dict,+ training: bool = False,) -> Optional[Tensor]:"""Q (1, H, T, D), K / V (1, H_kv, T, D) -> (1, T, H, D), the block-diagonal mask's result;None when the lengths do not tile the row."""@@ -614,12 +654,21 @@ def _sdpa_packed_segments(Q_run = split(Q)# Probe the run's own tensors: a one-token run has different eligibility than the row.gqa = n_groups != 1 and _sdpa_takes_gqa(- Q_run, split(K), kwargs["is_causal"], windowed, split(V), kwargs.get("dropout_p", 0.0)+ Q_run,+ split(K),+ kwargs["is_causal"],+ windowed,+ split(V),+ kwargs.get("dropout_p", 0.0),+ # Passed only when set: callers and tests wrap the probe with the old signature.+ **({"no_cudnn": True} if _avoids_cudnn(Q_run, training) else {}),)if gqa:kwargs["enable_gqa"] = TrueK_run, V_run = kv(gqa)- out = scaled_dot_product_attention(Q_run, split(K_run), split(V_run), **kwargs)+ out = (_training_sdpa if training else scaled_dot_product_attention)(+ Q_run, split(K_run), split(V_run), **kwargs+ )outs.append(out.transpose(1, 2).flatten(0, 1))start, i = end, jreturn torch.cat(outs, dim = 0).unsqueeze(0)@@ -751,6 +800,8 @@ class AttentionContext:# every existing construction and behavior unchanged.prefix_seg_info: Optional[Any] = Noneis_causal: bool = True+ # The attention module's .training: also true for a checkpointed forward run without grad.+ training: bool = Falsedef _ring_attention_active() -> bool:@@ -1193,6 +1244,7 @@ def _run_attention(is_causal = context.is_causal,sliding_window = sliding_window,sdpa_kwargs = sdpa_kwargs,+ training = context.training,)if out is not None:return out@@ -1267,6 +1319,7 @@ def _run_attention()kwargs = dict(sdpa_kwargs)+ sdpa = _training_sdpa if context.training else scaled_dot_product_attentionkwargs.setdefault("attn_mask", local_mask)kwargs.setdefault("is_causal", is_causal_local)@@ -1296,14 +1349,17 @@ def _run_attention(kwargs.get("attn_mask") is not None,V,kwargs.get("dropout_p", 0.0),+ **(+ {"no_cudnn": True}+ if xformers_rerouted or _avoids_cudnn(Q, context.training)+ else {}+ ),):use_sdpa_gqa = Falseif use_sdpa_gqa:kwargs.setdefault("enable_gqa", True)- out = (_rerouted_sdpa if xformers_rerouted else scaled_dot_product_attention)(- Q, K, V, **kwargs- )+ out = (_rerouted_sdpa if xformers_rerouted else sdpa)(Q, K, V, **kwargs)out = out.transpose(1, 2)return out if cp_out_dtype is None else out.to(cp_out_dtype)@@ -1319,7 +1375,7 @@ def _run_attention(K_mod = K_mod.reshape(bsz, n_heads, kv_seq_len, head_dim)V_mod = V_mod.reshape(bsz, n_heads, kv_seq_len, head_dim)- out = (_rerouted_sdpa if xformers_rerouted else scaled_dot_product_attention)(+ out = (_rerouted_sdpa if xformers_rerouted else sdpa)(Q.contiguous(),K_mod.contiguous(),V_mod.contiguous(),