Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Training SDPA: bucket sequence lengths where torch picks cuDNN, so padded batches stop rebuilding plans (#15134)
Daniel Hana224ff36d622
diff --git a/tests/test_sdpa_cudnn_length_bucket.py b/tests/test_sdpa_cudnn_length_bucket.pynew file mode 100644index 00000000000..e35823bdeb2--- /dev/null+++ b/tests/test_sdpa_cudnn_length_bucket.py@@ -0,0 +1,159 @@+# SPDX-License-Identifier: AGPL-3.0-only+"""Training SDPA on cuDNN attention pads the sequence to a length bucket (bounded cuDNN plans)."""++import types++import pytest+import torch++from unsloth.utils import attention_dispatch as ad+++def _cudnn_sdpa_available():+ if not torch.cuda.is_available() or ad._CUDNN_CHOICE is None:+ return False+ try:+ from torch.nn.attention import SDPBackend, sdpa_kernel++ q = torch.randn(1, 2, 64, 64, device = "cuda", dtype = torch.bfloat16)+ with sdpa_kernel([SDPBackend.CUDNN_ATTENTION]):+ torch.nn.functional.scaled_dot_product_attention(q, q, q, is_causal = True)+ return torch.cuda.get_device_capability()[0] >= 8+ except Exception:+ return False+++needs_cudnn = pytest.mark.skipif(not _cudnn_sdpa_available(), reason = "no cuDNN SDPA on this device")+++def _qkv(+ bsz,+ n_heads,+ n_kv,+ q_len,+ head_dim,+ gqa,+ seed = 0,+):+ g = torch.Generator(device = "cuda").manual_seed(seed)+ mk = lambda h: torch.randn(+ bsz, h, q_len, head_dim, device = "cuda", dtype = torch.bfloat16, generator = g+ )+ q = mk(n_heads)+ k, v = mk(n_kv), mk(n_kv)+ if not gqa:+ k = k.repeat_interleave(n_heads // n_kv, 1)+ v = v.repeat_interleave(n_heads // n_kv, 1)+ return [t.requires_grad_() for t in (q, k, v)]+++def test_applies_only_to_causal_unmasked_square_cuda_calls(monkeypatch):+ monkeypatch.setattr(ad, "_sdpa_backend_state", lambda *a, **k: (True, ad._CUDNN_CHOICE))+ if ad._CUDNN_CHOICE is None:+ pytest.skip("torch has no cuDNN SDPA backend")+ cpu = torch.zeros(1, 2, 8, 16, dtype = torch.bfloat16)+ assert not ad._cudnn_length_bucket_applies(cpu, cpu, cpu, is_causal = True)+ if not torch.cuda.is_available():+ return+ q = cpu.cuda()+ k_short = torch.zeros(1, 2, 4, 16, device = "cuda", dtype = torch.bfloat16)+ assert ad._cudnn_length_bucket_applies(q, q, q, is_causal = True)+ assert not ad._cudnn_length_bucket_applies(q, q, q, is_causal = False)+ assert not ad._cudnn_length_bucket_applies(+ q, q, q, attn_mask = torch.ones(8, 8, dtype = torch.bool, device = "cuda"), is_causal = True+ )+ assert not ad._cudnn_length_bucket_applies(q, q, q, dropout_p = 0.1, is_causal = True)+ assert not ad._cudnn_length_bucket_applies(q, k_short, k_short, is_causal = True)+ assert not ad._cudnn_length_bucket_applies(q.float(), q.float(), q.float(), is_causal = True)+ monkeypatch.setattr(ad, "_CUDNN_LENGTH_BUCKET", 0)+ assert not ad._cudnn_length_bucket_applies(q, q, q, is_causal = True)++[email protected]("backend", ["ring", "ulysses"])+def test_not_applied_under_context_parallelism(monkeypatch, backend):+ # Sequence shards: padding a shard would add unmasked zero keys inside the global sequence.+ if not torch.cuda.is_available():+ pytest.skip("needs CUDA")+ monkeypatch.setattr(ad, "_sdpa_backend_state", lambda *a, **k: (True, ad._CUDNN_CHOICE))+ monkeypatch.setattr(ad, "get_cp_manager", lambda: types.SimpleNamespace(backend = backend))+ q = torch.randn(1, 2, 200, 64, device = "cuda", dtype = torch.bfloat16)+ assert not ad._cudnn_length_bucket_applies(q, q, q, is_causal = True)+++def test_not_applied_when_sdpa_would_not_pick_cudnn(monkeypatch):+ if not torch.cuda.is_available():+ pytest.skip("needs CUDA")+ monkeypatch.setattr(ad, "_sdpa_backend_state", lambda *a, **k: (True, 1)) # flash+ q = torch.zeros(1, 2, 8, 64, device = "cuda", dtype = torch.bfloat16)+ assert not ad._cudnn_length_bucket_applies(q, q, q, is_causal = True)+++@needs_cudnn[email protected]("q_len", [2, 127, 128, 129, 300, 1000])[email protected]("gqa", [True, False])+def test_bucketed_matches_unpadded_cudnn(q_len, gqa):+ from torch.nn.attention import SDPBackend, sdpa_kernel++ kw = dict(enable_gqa = True) if gqa else {}+ q, k, v = _qkv(2, 8, 2, q_len, 128, gqa)+ grad = torch.randn(2, 8, q_len, 128, device = "cuda", dtype = torch.bfloat16)+ with sdpa_kernel([SDPBackend.CUDNN_ATTENTION]):+ ref = torch.nn.functional.scaled_dot_product_attention(q, k, v, is_causal = True, **kw)+ ref.backward(grad)+ ref_grads = [t.grad.clone() for t in (q, k, v)]+ for t in (q, k, v):+ t.grad = None+ out = ad._cudnn_length_bucketed_sdpa(q, k, v, is_causal = True, **kw)+ out.backward(grad)+ assert out.shape == ref.shape+ assert torch.isfinite(out).all()+ # Same kernel family on a longer (padded) sequence: tiles differ, so compare within bf16 rounding.+ torch.testing.assert_close(out, ref, atol = 2e-2, rtol = 2e-2)+ for got, want in zip((q.grad, k.grad, v.grad), ref_grads):+ assert torch.isfinite(got).all()+ torch.testing.assert_close(got, want, atol = 3e-2, rtol = 3e-2)+++@needs_cudnn+def test_padded_lengths_are_bucket_multiples(monkeypatch):+ from torch.nn.attention import SDPBackend, sdpa_kernel++ seen = []+ real = ad.scaled_dot_product_attention++ def record(Q, K, V, **kwargs):+ seen.append(Q.shape[-2])+ return real(Q, K, V, **kwargs)++ monkeypatch.setattr(ad, "scaled_dot_product_attention", record)+ with sdpa_kernel([SDPBackend.CUDNN_ATTENTION]):+ for q_len in (5, 130, 255, 256, 700):+ q, k, v = _qkv(1, 4, 4, q_len, 64, False)+ out = ad._training_sdpa(q, k, v, is_causal = True)+ assert out.shape[-2] == q_len+ if ad._avoids_cudnn(q, True):+ assert seen == [5, 130, 255, 256, 700] # the SM100 pin keeps these calls off cuDNN+ else:+ assert seen == [128, 256, 256, 384, 768]+++@needs_cudnn+def test_compiled_dynamic_lengths_one_graph():+ import torch._dynamo+ from torch._dynamo.utils import counters+ from torch.nn.attention import SDPBackend, sdpa_kernel++ torch._dynamo.reset()+ counters.clear()++ def f(q, k, v):+ return ad._cudnn_length_bucketed_sdpa(q, k, v, is_causal = True, enable_gqa = True)++ compiled = torch.compile(f, dynamic = True, fullgraph = True)+ with sdpa_kernel([SDPBackend.CUDNN_ATTENTION]):+ for q_len in (300, 517, 1000, 1024):+ q, k, v = _qkv(2, 8, 2, q_len, 128, True)+ got = compiled(q, k, v)+ want = f(q, k, v)+ torch.testing.assert_close(got, want, atol = 2e-2, rtol = 2e-2)+ assert counters["stats"]["unique_graphs"] == 1, dict(counters["stats"])diff --git a/unsloth/utils/attention_dispatch.py b/unsloth/utils/attention_dispatch.pyindex 5a8d4629bfd..d39c8eb99c6 100644--- a/unsloth/utils/attention_dispatch.py+++ b/unsloth/utils/attention_dispatch.py@@ -111,9 +111,74 @@ def _training_sdpa(*args, **kwargs):if args and _avoids_cudnn(args[0], True):with _sdpa_kernel(_XFORMERS_LIKE_BACKENDS):return scaled_dot_product_attention(*args, **kwargs)+ if len(args) >= 3 and _cudnn_length_bucket_applies(*args[:3], **kwargs):+ return _cudnn_length_bucketed_sdpa(*args, **kwargs)return scaled_dot_product_attention(*args, **kwargs)+# Wherever SDPA picks cuDNN attention for a training call (H100 on torch 2.14: the default+# priority), every new sequence length builds a new cuDNN plan, ~170 ms forward plus backward on+# H100. Padded batches change length every step, so that cost landed on nearly every step. Padding+# Q / K / V along the sequence to a multiple of this bucket bounds the plans to max_len / bucket.+# Only causal calls with no mask and q_len == k_len: padded keys come after every real query, so+# causal masking hides them and the real rows are what the unpadded call returns. 0 disables.+try:+ _CUDNN_LENGTH_BUCKET = max(int(os.environ.get("UNSLOTH_SDPA_CUDNN_BUCKET", "128")), 0)+except ValueError:+ _CUDNN_LENGTH_BUCKET = 128+++def _cudnn_length_bucket_applies(+ Q,+ K,+ V,+ attn_mask = None,+ dropout_p = 0.0,+ is_causal = False,+ **kwargs,+):+ if not _CUDNN_LENGTH_BUCKET or _CUDNN_CHOICE is None:+ return False+ # Under context parallelism the tensors can be sequence shards: padding a shard would put unmasked+ # zero keys inside the global sequence.+ if get_cp_manager() is not None:+ return False+ if attn_mask is not None or not is_causal or dropout_p:+ return False+ if not (isinstance(Q, Tensor) and Q.is_cuda and Q.dim() == 4 and K.dim() == 4 and V.dim() == 4):+ return False+ if Q.dtype not in (torch.float16, torch.bfloat16) or Q.shape[-2] != K.shape[-2]:+ return False+ _, choice = _sdpa_backend_state(+ str(Q.device),+ Q.dtype,+ Q.shape[-1],+ Q.shape[1],+ K.shape[1],+ True,+ bool(kwargs.get("enable_gqa", False)),+ )+ return choice == _CUDNN_CHOICE+++def _cudnn_length_bucketed_sdpa(+ Q,+ K,+ V,+ attn_mask = None,+ dropout_p = 0.0,+ is_causal = False,+ **kwargs,+):+ # Always pad by 1..bucket rows and slice: a zero pad specialises under dynamo (a recompile when the+ # length becomes a bucket multiple), so an exact multiple carries one spare bucket instead.+ q_len = Q.shape[-2]+ pad = _CUDNN_LENGTH_BUCKET - q_len % _CUDNN_LENGTH_BUCKET+ Q, K, V = (F.pad(x, (0, 0, 0, pad)) for x in (Q, K, V))+ out = scaled_dot_product_attention(Q, K, V, is_causal = True, **kwargs)+ return out[..., :q_len, :]++def _rerouted_sdpa(*args, **kwargs):"""SDPA standing in for xformers inside a compiled block, on the kernels xformers would run.Without the pin, torch 2.13+ on SM100 takes cuDNN attention, which builds a new plan for every@@ -130,8 +195,12 @@ try:_SDPBackend.EFFICIENT_ATTENTION,_SDPBackend.MATH,]+ _CUDNN_CHOICE = (+ int(_SDPBackend.CUDNN_ATTENTION) if hasattr(_SDPBackend, "CUDNN_ATTENTION") else None+ )except Exception:_rerouted_sdpa = scaled_dot_product_attention+ _CUDNN_CHOICE = Nonedef _per_device_backends(capabilities):