Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Training SDPA: bucket sequence lengths where torch picks cuDNN, so padded batches stop rebuilding plans (#15134)

Daniel Hana224ff36d622
diff --git a/tests/test_sdpa_cudnn_length_bucket.py b/tests/test_sdpa_cudnn_length_bucket.py
new file mode 100644
index 00000000000..e35823bdeb2
--- /dev/null
+++ b/tests/test_sdpa_cudnn_length_bucket.py
@@ -0,0 +1,159 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+"""Training SDPA on cuDNN attention pads the sequence to a length bucket (bounded cuDNN plans)."""
+
+import types
+
+import pytest
+import torch
+
+from unsloth.utils import attention_dispatch as ad
+
+
+def _cudnn_sdpa_available():
+ if not torch.cuda.is_available() or ad._CUDNN_CHOICE is None:
+ return False
+ try:
+ from torch.nn.attention import SDPBackend, sdpa_kernel
+
+ q = torch.randn(1, 2, 64, 64, device = "cuda", dtype = torch.bfloat16)
+ with sdpa_kernel([SDPBackend.CUDNN_ATTENTION]):
+ torch.nn.functional.scaled_dot_product_attention(q, q, q, is_causal = True)
+ return torch.cuda.get_device_capability()[0] >= 8
+ except Exception:
+ return False
+
+
+needs_cudnn = pytest.mark.skipif(not _cudnn_sdpa_available(), reason = "no cuDNN SDPA on this device")
+
+
+def _qkv(
+ bsz,
+ n_heads,
+ n_kv,
+ q_len,
+ head_dim,
+ gqa,
+ seed = 0,
+):
+ g = torch.Generator(device = "cuda").manual_seed(seed)
+ mk = lambda h: torch.randn(
+ bsz, h, q_len, head_dim, device = "cuda", dtype = torch.bfloat16, generator = g
+ )
+ q = mk(n_heads)
+ k, v = mk(n_kv), mk(n_kv)
+ if not gqa:
+ k = k.repeat_interleave(n_heads // n_kv, 1)
+ v = v.repeat_interleave(n_heads // n_kv, 1)
+ return [t.requires_grad_() for t in (q, k, v)]
+
+
+def test_applies_only_to_causal_unmasked_square_cuda_calls(monkeypatch):
+ monkeypatch.setattr(ad, "_sdpa_backend_state", lambda *a, **k: (True, ad._CUDNN_CHOICE))
+ if ad._CUDNN_CHOICE is None:
+ pytest.skip("torch has no cuDNN SDPA backend")
+ cpu = torch.zeros(1, 2, 8, 16, dtype = torch.bfloat16)
+ assert not ad._cudnn_length_bucket_applies(cpu, cpu, cpu, is_causal = True)
+ if not torch.cuda.is_available():
+ return
+ q = cpu.cuda()
+ k_short = torch.zeros(1, 2, 4, 16, device = "cuda", dtype = torch.bfloat16)
+ assert ad._cudnn_length_bucket_applies(q, q, q, is_causal = True)
+ assert not ad._cudnn_length_bucket_applies(q, q, q, is_causal = False)
+ assert not ad._cudnn_length_bucket_applies(
+ q, q, q, attn_mask = torch.ones(8, 8, dtype = torch.bool, device = "cuda"), is_causal = True
+ )
+ assert not ad._cudnn_length_bucket_applies(q, q, q, dropout_p = 0.1, is_causal = True)
+ assert not ad._cudnn_length_bucket_applies(q, k_short, k_short, is_causal = True)
+ assert not ad._cudnn_length_bucket_applies(q.float(), q.float(), q.float(), is_causal = True)
+ monkeypatch.setattr(ad, "_CUDNN_LENGTH_BUCKET", 0)
+ assert not ad._cudnn_length_bucket_applies(q, q, q, is_causal = True)
+
+
[email protected]("backend", ["ring", "ulysses"])
+def test_not_applied_under_context_parallelism(monkeypatch, backend):
+ # Sequence shards: padding a shard would add unmasked zero keys inside the global sequence.
+ if not torch.cuda.is_available():
+ pytest.skip("needs CUDA")
+ monkeypatch.setattr(ad, "_sdpa_backend_state", lambda *a, **k: (True, ad._CUDNN_CHOICE))
+ monkeypatch.setattr(ad, "get_cp_manager", lambda: types.SimpleNamespace(backend = backend))
+ q = torch.randn(1, 2, 200, 64, device = "cuda", dtype = torch.bfloat16)
+ assert not ad._cudnn_length_bucket_applies(q, q, q, is_causal = True)
+
+
+def test_not_applied_when_sdpa_would_not_pick_cudnn(monkeypatch):
+ if not torch.cuda.is_available():
+ pytest.skip("needs CUDA")
+ monkeypatch.setattr(ad, "_sdpa_backend_state", lambda *a, **k: (True, 1)) # flash
+ q = torch.zeros(1, 2, 8, 64, device = "cuda", dtype = torch.bfloat16)
+ assert not ad._cudnn_length_bucket_applies(q, q, q, is_causal = True)
+
+
+@needs_cudnn
[email protected]("q_len", [2, 127, 128, 129, 300, 1000])
[email protected]("gqa", [True, False])
+def test_bucketed_matches_unpadded_cudnn(q_len, gqa):
+ from torch.nn.attention import SDPBackend, sdpa_kernel
+
+ kw = dict(enable_gqa = True) if gqa else {}
+ q, k, v = _qkv(2, 8, 2, q_len, 128, gqa)
+ grad = torch.randn(2, 8, q_len, 128, device = "cuda", dtype = torch.bfloat16)
+ with sdpa_kernel([SDPBackend.CUDNN_ATTENTION]):
+ ref = torch.nn.functional.scaled_dot_product_attention(q, k, v, is_causal = True, **kw)
+ ref.backward(grad)
+ ref_grads = [t.grad.clone() for t in (q, k, v)]
+ for t in (q, k, v):
+ t.grad = None
+ out = ad._cudnn_length_bucketed_sdpa(q, k, v, is_causal = True, **kw)
+ out.backward(grad)
+ assert out.shape == ref.shape
+ assert torch.isfinite(out).all()
+ # Same kernel family on a longer (padded) sequence: tiles differ, so compare within bf16 rounding.
+ torch.testing.assert_close(out, ref, atol = 2e-2, rtol = 2e-2)
+ for got, want in zip((q.grad, k.grad, v.grad), ref_grads):
+ assert torch.isfinite(got).all()
+ torch.testing.assert_close(got, want, atol = 3e-2, rtol = 3e-2)
+
+
+@needs_cudnn
+def test_padded_lengths_are_bucket_multiples(monkeypatch):
+ from torch.nn.attention import SDPBackend, sdpa_kernel
+
+ seen = []
+ real = ad.scaled_dot_product_attention
+
+ def record(Q, K, V, **kwargs):
+ seen.append(Q.shape[-2])
+ return real(Q, K, V, **kwargs)
+
+ monkeypatch.setattr(ad, "scaled_dot_product_attention", record)
+ with sdpa_kernel([SDPBackend.CUDNN_ATTENTION]):
+ for q_len in (5, 130, 255, 256, 700):
+ q, k, v = _qkv(1, 4, 4, q_len, 64, False)
+ out = ad._training_sdpa(q, k, v, is_causal = True)
+ assert out.shape[-2] == q_len
+ if ad._avoids_cudnn(q, True):
+ assert seen == [5, 130, 255, 256, 700] # the SM100 pin keeps these calls off cuDNN
+ else:
+ assert seen == [128, 256, 256, 384, 768]
+
+
+@needs_cudnn
+def test_compiled_dynamic_lengths_one_graph():
+ import torch._dynamo
+ from torch._dynamo.utils import counters
+ from torch.nn.attention import SDPBackend, sdpa_kernel
+
+ torch._dynamo.reset()
+ counters.clear()
+
+ def f(q, k, v):
+ return ad._cudnn_length_bucketed_sdpa(q, k, v, is_causal = True, enable_gqa = True)
+
+ compiled = torch.compile(f, dynamic = True, fullgraph = True)
+ with sdpa_kernel([SDPBackend.CUDNN_ATTENTION]):
+ for q_len in (300, 517, 1000, 1024):
+ q, k, v = _qkv(2, 8, 2, q_len, 128, True)
+ got = compiled(q, k, v)
+ want = f(q, k, v)
+ torch.testing.assert_close(got, want, atol = 2e-2, rtol = 2e-2)
+ assert counters["stats"]["unique_graphs"] == 1, dict(counters["stats"])
diff --git a/unsloth/utils/attention_dispatch.py b/unsloth/utils/attention_dispatch.py
index 5a8d4629bfd..d39c8eb99c6 100644
--- a/unsloth/utils/attention_dispatch.py
+++ b/unsloth/utils/attention_dispatch.py
@@ -111,9 +111,74 @@ def _training_sdpa(*args, **kwargs):
if args and _avoids_cudnn(args[0], True):
with _sdpa_kernel(_XFORMERS_LIKE_BACKENDS):
return scaled_dot_product_attention(*args, **kwargs)
+ if len(args) >= 3 and _cudnn_length_bucket_applies(*args[:3], **kwargs):
+ return _cudnn_length_bucketed_sdpa(*args, **kwargs)
return scaled_dot_product_attention(*args, **kwargs)
+# Wherever SDPA picks cuDNN attention for a training call (H100 on torch 2.14: the default
+# priority), every new sequence length builds a new cuDNN plan, ~170 ms forward plus backward on
+# H100. Padded batches change length every step, so that cost landed on nearly every step. Padding
+# Q / K / V along the sequence to a multiple of this bucket bounds the plans to max_len / bucket.
+# Only causal calls with no mask and q_len == k_len: padded keys come after every real query, so
+# causal masking hides them and the real rows are what the unpadded call returns. 0 disables.
+try:
+ _CUDNN_LENGTH_BUCKET = max(int(os.environ.get("UNSLOTH_SDPA_CUDNN_BUCKET", "128")), 0)
+except ValueError:
+ _CUDNN_LENGTH_BUCKET = 128
+
+
+def _cudnn_length_bucket_applies(
+ Q,
+ K,
+ V,
+ attn_mask = None,
+ dropout_p = 0.0,
+ is_causal = False,
+ **kwargs,
+):
+ if not _CUDNN_LENGTH_BUCKET or _CUDNN_CHOICE is None:
+ return False
+ # Under context parallelism the tensors can be sequence shards: padding a shard would put unmasked
+ # zero keys inside the global sequence.
+ if get_cp_manager() is not None:
+ return False
+ if attn_mask is not None or not is_causal or dropout_p:
+ return False
+ if not (isinstance(Q, Tensor) and Q.is_cuda and Q.dim() == 4 and K.dim() == 4 and V.dim() == 4):
+ return False
+ if Q.dtype not in (torch.float16, torch.bfloat16) or Q.shape[-2] != K.shape[-2]:
+ return False
+ _, choice = _sdpa_backend_state(
+ str(Q.device),
+ Q.dtype,
+ Q.shape[-1],
+ Q.shape[1],
+ K.shape[1],
+ True,
+ bool(kwargs.get("enable_gqa", False)),
+ )
+ return choice == _CUDNN_CHOICE
+
+
+def _cudnn_length_bucketed_sdpa(
+ Q,
+ K,
+ V,
+ attn_mask = None,
+ dropout_p = 0.0,
+ is_causal = False,
+ **kwargs,
+):
+ # Always pad by 1..bucket rows and slice: a zero pad specialises under dynamo (a recompile when the
+ # length becomes a bucket multiple), so an exact multiple carries one spare bucket instead.
+ q_len = Q.shape[-2]
+ pad = _CUDNN_LENGTH_BUCKET - q_len % _CUDNN_LENGTH_BUCKET
+ Q, K, V = (F.pad(x, (0, 0, 0, pad)) for x in (Q, K, V))
+ out = scaled_dot_product_attention(Q, K, V, is_causal = True, **kwargs)
+ return out[..., :q_len, :]
+
+
def _rerouted_sdpa(*args, **kwargs):
"""SDPA standing in for xformers inside a compiled block, on the kernels xformers would run.
Without the pin, torch 2.13+ on SM100 takes cuDNN attention, which builds a new plan for every
@@ -130,8 +195,12 @@ try:
_SDPBackend.EFFICIENT_ATTENTION,
_SDPBackend.MATH,
]
+ _CUDNN_CHOICE = (
+ int(_SDPBackend.CUDNN_ATTENTION) if hasattr(_SDPBackend, "CUDNN_ATTENTION") else None
+ )
except Exception:
_rerouted_sdpa = scaled_dot_product_attention
+ _CUDNN_CHOICE = None
def _per_device_backends(capabilities):