Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Compiled blocks: keep the xformers stand-in SDPA off cuDNN attention (#15009)

Daniel Han217fa4c71d06
diff --git a/tests/utils/test_attention_rerouted_sdpa.py b/tests/utils/test_attention_rerouted_sdpa.py
new file mode 100644
index 00000000000..3fd4dc988ec
--- /dev/null
+++ b/tests/utils/test_attention_rerouted_sdpa.py
@@ -0,0 +1,57 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
+
+"""SDPA standing in for xformers inside a compiled block keeps cuDNN attention off.
+
+torch 2.13+ on SM100 picks cuDNN attention for plain causal SDPA, and cuDNN builds a new plan for
+every new sequence length: padded training batches of varying length stalled every step (Llama-3.2-1B
+LoRA on B200: 439 ms median step compiled vs 227 ms stock). xformers itself runs flash there.
+"""
+
+import pytest
+import torch
+import torch.nn.functional as F
+
+ad = pytest.importorskip("unsloth.utils.attention_dispatch")
+
+
+def test_rerouted_sdpa_disables_cudnn_only(monkeypatch):
+ seen = []
+
+ def fake_sdpa(q, k, v, **kwargs):
+ b = torch.backends.cuda
+ seen.append(
+ (
+ b.cudnn_sdp_enabled(),
+ b.flash_sdp_enabled(),
+ b.mem_efficient_sdp_enabled(),
+ b.math_sdp_enabled(),
+ )
+ )
+ return q
+
+ monkeypatch.setattr(F, "scaled_dot_product_attention", fake_sdpa)
+ before = torch.backends.cuda.cudnn_sdp_enabled()
+ q = torch.zeros(1, 2, 4, 8)
+ ad._rerouted_sdpa(q, q, q, is_causal = True)
+ ad.scaled_dot_product_attention(q, q, q, is_causal = True)
+ assert seen[0] == (False, True, True, True)
+ assert seen[1][0] == before # the plain path is untouched
+ assert torch.backends.cuda.cudnn_sdp_enabled() == before
+
+
+def test_rerouted_sdpa_traces_without_a_break():
+ from torch._dynamo.utils import counters
+
+ torch._dynamo.reset()
+ counters.clear()
+
+ def f(q):
+ return ad._rerouted_sdpa(q, q, q, is_causal = True)
+
+ c = torch.compile(f, backend = "eager", dynamic = True)
+ for L in (5, 9, 16, 23):
+ q = torch.randn(1, 2, L, 8)
+ torch.testing.assert_close(c(q), F.scaled_dot_product_attention(q, q, q, is_causal = True))
+ assert dict(counters["graph_break"]) == {}
+ torch._dynamo.reset()
diff --git a/unsloth/utils/attention_dispatch.py b/unsloth/utils/attention_dispatch.py
index 173eae42601..b3beac205aa 100644
--- a/unsloth/utils/attention_dispatch.py
+++ b/unsloth/utils/attention_dispatch.py
@@ -99,6 +99,26 @@ def scaled_dot_product_attention(*args, **kwargs):
return F.scaled_dot_product_attention(*args, **kwargs)
+def _rerouted_sdpa(*args, **kwargs):
+ """SDPA standing in for xformers inside a compiled block, on the kernels xformers would run.
+ Without the pin, torch 2.13+ on SM100 takes cuDNN attention, which builds a new plan for every
+ new sequence length (~150 ms per length): padded batches of varying length stalled every step."""
+ with _sdpa_kernel(_XFORMERS_LIKE_BACKENDS):
+ return F.scaled_dot_product_attention(*args, **kwargs)
+
+
+# Module level: reading SDPBackend's attributes while traced breaks the graph.
+try:
+ from torch.nn.attention import SDPBackend as _SDPBackend, sdpa_kernel as _sdpa_kernel
+ _XFORMERS_LIKE_BACKENDS = [
+ _SDPBackend.FLASH_ATTENTION,
+ _SDPBackend.EFFICIENT_ATTENTION,
+ _SDPBackend.MATH,
+ ]
+except Exception:
+ _rerouted_sdpa = scaled_dot_product_attention
+
+
def _per_device_backends(capabilities):
"""(flash runs, xformers runs, xformers rejects fp32, bf16 kernels) per CUDA device index, or None
when every device has device 0's capability class, so homogeneous hosts never consult the table.
@@ -1281,7 +1301,9 @@ def _run_attention(
if use_sdpa_gqa:
kwargs.setdefault("enable_gqa", True)
- out = scaled_dot_product_attention(Q, K, V, **kwargs)
+ out = (_rerouted_sdpa if xformers_rerouted else scaled_dot_product_attention)(
+ Q, K, V, **kwargs
+ )
out = out.transpose(1, 2)
return out if cp_out_dtype is None else out.to(cp_out_dtype)
@@ -1297,7 +1319,7 @@ def _run_attention(
K_mod = K_mod.reshape(bsz, n_heads, kv_seq_len, head_dim)
V_mod = V_mod.reshape(bsz, n_heads, kv_seq_len, head_dim)
- out = scaled_dot_product_attention(
+ out = (_rerouted_sdpa if xformers_rerouted else scaled_dot_product_attention)(
Q.contiguous(),
K_mod.contiguous(),
V_mod.contiguous(),