Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
SDPA training on SM100: keep the caller's backend policy when avoiding cuDNN (#16230)
Daniel Han65e4bc66fc4e
diff --git a/tests/utils/test_attention_training_no_cudnn.py b/tests/utils/test_attention_training_no_cudnn.pyindex 5ef8736e33d..17643d33470 100644--- a/tests/utils/test_attention_training_no_cudnn.py+++ b/tests/utils/test_attention_training_no_cudnn.py@@ -1,13 +1,6 @@# SPDX-License-Identifier: AGPL-3.0-only# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.-"""Training SDPA on SM100 avoids cuDNN attention; eval / generation calls are untouched.--torch 2.13+ on B200 picks cuDNN attention for SDPA, which builds a plan for every new sequence length-(~150 ms each). Padded training batches change length every step: Qwen3-0.6B LoRA went from 2777-3060-to 5164-5333 useful tokens/s with flash / efficient / math (exclusive B200, no xformers, interleaved).-"""-import pytestimport torchimport torch.nn.functional as F@@ -111,12 +104,10 @@ def test_training_sdpa_traces_without_a_break(monkeypatch):def test_gqa_probe_runs_under_the_training_restriction(monkeypatch):- """The enable_gqa decision is probed with the kernels the call runs with (cuDNN off)."""probed = []def fake_now(*args):probed.append((args, torch.backends.cuda.cudnn_sdp_enabled()))- # cuDNN takes this masked GQA call; without it only math would.return torch.backends.cuda.cudnn_sdp_enabled()monkeypatch.setattr(ad, "_sdpa_gqa_state_now", fake_now)@@ -131,7 +122,7 @@ def test_gqa_probe_runs_under_the_training_restriction(monkeypatch):ad._sdpa_takes_gqa(Q, K, False, True, K, 0.0, no_cudnn = ad._avoids_cudnn(Q, False)) is before)assert [cudnn for _, cudnn in probed] == [False, before]- assert probed[0][0] == probed[1][0] # same shapes, only the backend set differs+ assert probed[0][0] == probed[1][0]assert torch.backends.cuda.cudnn_sdp_enabled() == beforeassert ad._avoids_cudnn(Q, True) and not ad._avoids_cudnn(Q, False)monkeypatch.setattr(ad, "_NO_CUDNN_TRAINING", frozenset())@@ -162,3 +153,103 @@ def test_dense_and_segment_paths_probe_without_cudnn_when_training(monkeypatch):training = training,)assert flags == [True, True, False, False]+++_POLICIES = {+ "all": ("FLASH_ATTENTION", "EFFICIENT_ATTENTION", "MATH", "CUDNN_ATTENTION"),+ "math": ("MATH",),+ "math_cudnn": ("MATH", "CUDNN_ATTENTION"),+ "efficient_math_cudnn": ("EFFICIENT_ATTENTION", "MATH", "CUDNN_ATTENTION"),+ "cudnn": ("CUDNN_ATTENTION",),+}+++def _policy(name):+ from torch.nn.attention import SDPBackend+ if not hasattr(SDPBackend, "CUDNN_ATTENTION"):+ pytest.skip("torch has no cuDNN SDPA backend")+ return [getattr(SDPBackend, b) for b in _POLICIES[name]]+++def _flags():+ cuda = torch.backends.cuda+ return (+ cuda.flash_sdp_enabled(),+ cuda.mem_efficient_sdp_enabled(),+ cuda.math_sdp_enabled(),+ cuda.cudnn_sdp_enabled(),+ )+++def _expected(name):+ names = _POLICIES[name]+ keep_cudnn = names == ("CUDNN_ATTENTION",)+ return (+ "FLASH_ATTENTION" in names,+ "EFFICIENT_ATTENTION" in names,+ "MATH" in names,+ keep_cudnn,+ )++[email protected]("name", list(_POLICIES))+def test_training_pin_keeps_caller_backend_policy(monkeypatch, name):+ from torch.nn.attention import sdpa_kernel++ seen = []+ monkeypatch.setattr(+ F, "scaled_dot_product_attention", lambda q, k, v, **kw: seen.append(_flags()) or q+ )+ monkeypatch.setattr(ad, "_NO_CUDNN_TRAINING", frozenset({None}))+ Q = torch.randn(1, 2, 6, 8).as_subclass(_CudaLooking)+ with sdpa_kernel(_policy(name)):+ before = _flags()+ ad._training_sdpa(Q, Q, Q, is_causal = True)+ ad._rerouted_sdpa(Q, Q, Q, is_causal = True)+ assert _flags() == before+ assert seen == [_expected(name)] * 2++[email protected]("name", ["all", "math_cudnn", "cudnn"])+def test_gqa_probe_uses_the_callers_cudnn_free_set(monkeypatch, name):+ from torch.nn.attention import sdpa_kernel++ probed = []+ monkeypatch.setattr(ad, "_sdpa_gqa_state_now", lambda *a: probed.append(_flags()) or True)+ Q = torch.randn(1, 4, 6, 8).as_subclass(_CudaLooking)+ K = torch.randn(1, 2, 6, 8).as_subclass(_CudaLooking)+ with sdpa_kernel(_policy(name)):+ ad._sdpa_takes_gqa(Q, K, False, True, K, 0.0, no_cudnn = True)+ assert probed == [_expected(name)]++[email protected]("name", ["all", "math", "math_cudnn"])+def test_compiled_training_pin_keeps_caller_backend_policy(monkeypatch, name):+ from torch._dynamo.utils import counters+ from torch.nn.attention import sdpa_kernel++ pinned = []+ real = ad._sdpa_kernel++ def recording(backends):+ pinned.append(tuple(backends))+ return real(backends)++ monkeypatch.setattr(ad, "_sdpa_kernel", recording)+ monkeypatch.setattr(ad, "_avoids_cudnn", lambda q, training: bool(training))+ torch._dynamo.reset()+ counters.clear()++ def f(q):+ return ad._training_sdpa(q, q, q, is_causal = True)++ policy = _policy(name)+ c = torch.compile(f, backend = "eager", fullgraph = True, dynamic = True)+ with sdpa_kernel(policy):+ for L in (5, 9, 16):+ q = torch.randn(1, 2, L, 8, requires_grad = True)+ c(q).sum().backward()+ assert dict(counters["graph_break"]) == {}+ expected = tuple(b for b in ad._XFORMERS_LIKE_BACKENDS if b in policy)+ assert set(pinned) == ({expected} if any("CUDNN" in str(b) for b in policy) else set())+ torch._dynamo.reset()diff --git a/unsloth/utils/attention_dispatch.py b/unsloth/utils/attention_dispatch.pyindex 11d431b03a5..b8fe615ddc2 100644--- a/unsloth/utils/attention_dispatch.py+++ b/unsloth/utils/attention_dispatch.py@@ -107,12 +107,13 @@ def _avoids_cudnn(Q, training):def _training_sdpa(*args, **kwargs):- """SDPA for a training forward (AttentionContext.training, passed in, no global state: these- calls run traced). On SM100 SDPA picks cuDNN attention, which builds a plan for every new- sequence length (~150 ms each), and training batches change length every step."""+ """Avoid per-length cuDNN planning on SM100 training calls, including traced forwards."""if args and _avoids_cudnn(args[0], True):- with _sdpa_kernel(_XFORMERS_LIKE_BACKENDS):- return scaled_dot_product_attention(*args, **kwargs)+ # Narrows the caller's sdpa_kernel policy (MATH stays math); cuDNN-only runs as asked.+ backends = _CUDNN_FREE_BACKENDS[_cudnn_free_index()]+ if backends is not None:+ with _sdpa_kernel(backends):+ return scaled_dot_product_attention(*args, **kwargs)if len(args) >= 3 and _cudnn_length_bucket_applies(*args[:3], **kwargs):return _cudnn_length_bucketed_sdpa(*args, **kwargs)return scaled_dot_product_attention(*args, **kwargs)@@ -258,10 +259,11 @@ def _padded_bucket_mask(attn_mask, pad):def _rerouted_sdpa(*args, **kwargs):- """SDPA standing in for xformers inside a compiled block, on the kernels xformers would run.- Without the pin, torch 2.13+ on SM100 takes cuDNN attention, which builds a new plan for every- new sequence length (~150 ms per length): padded batches of varying length stalled every step."""- with _sdpa_kernel(_XFORMERS_LIKE_BACKENDS):+ """Compiled xformers stand-in: avoid cuDNN's per-length planning."""+ backends = _CUDNN_FREE_BACKENDS[_cudnn_free_index()]+ if backends is None:+ return F.scaled_dot_product_attention(*args, **kwargs)+ with _sdpa_kernel(backends):return F.scaled_dot_product_attention(*args, **kwargs)@@ -274,8 +276,7 @@ try:_SDPBackend.EFFICIENT_ATTENTION,_SDPBackend.MATH,]- # Decode pins, indexed by _decode_cudnn_free_index: the caller's enabled flash / efficient / math.- _DECODE_CUDNN_FREE_BACKENDS = (None,) + tuple(+ _CUDNN_FREE_BACKENDS = (None,) + tuple([b for j, b in enumerate(_XFORMERS_LIKE_BACKENDS) if i >> j & 1] for i in range(1, 8))_CUDNN_CHOICE = (@@ -283,7 +284,7 @@ try:)except Exception:_rerouted_sdpa = scaled_dot_product_attention- _DECODE_CUDNN_FREE_BACKENDS = (None,) * 8+ _CUDNN_FREE_BACKENDS = (None,) * 8_CUDNN_CHOICE = None@@ -391,14 +392,14 @@ def decode_scaled_dot_product_attention(query, key, value, *args, **kwargs):):# Only the caller's own non-cuDNN backends (sdpa_kernel([MATH]) stays math); a cuDNN-only# policy is left as is.- backends = _DECODE_CUDNN_FREE_BACKENDS[_decode_cudnn_free_index()]+ backends = _CUDNN_FREE_BACKENDS[_cudnn_free_index()]if backends is not None:with _sdpa_kernel(backends):return F.scaled_dot_product_attention(query, key, value, *args, **kwargs)return F.scaled_dot_product_attention(query, key, value, *args, **kwargs)-def _decode_cudnn_free_index():+def _cudnn_free_index():"""Bit 0 / 1 / 2: flash / efficient / math SDPA enabled; 0 when cuDNN is disabled (nothing to avoid)."""cuda = torch.backends.cudaif not cuda.cudnn_sdp_enabled():@@ -755,8 +756,7 @@ def _constant_probe(fn):_sdpa_backend_state = _constant_probe(_sdpa_backend_state)_sdpa_masked_choice = _constant_probe(_sdpa_masked_choice)-# Traced: read under the caller's sdpa_kernel at trace time, like SDPA's own backend pick.-_decode_cudnn_free_index = _constant_probe(_decode_cudnn_free_index)+_cudnn_free_index = _constant_probe(_cudnn_free_index)@functools.lru_cache(maxsize = 128)@@ -871,8 +871,9 @@ def _sdpa_gqa_state_now(def _sdpa_gqa_state(*args, no_cudnn = False):"""_sdpa_gqa_state_now; no_cudnn probes under the kernels the call runs with (_training_sdpa,_rerouted_sdpa), whose enabled flags are part of the probe's cache key."""- if no_cudnn:- with _sdpa_kernel(_XFORMERS_LIKE_BACKENDS):+ backends = _CUDNN_FREE_BACKENDS[_cudnn_free_index()] if no_cudnn else None+ if backends is not None:+ with _sdpa_kernel(backends):return _sdpa_gqa_state_now(*args)return _sdpa_gqa_state_now(*args)