Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Compiled blocks: keep the xformers stand-in SDPA off cuDNN attention (#15009)
Daniel Han217fa4c71d06
diff --git a/tests/utils/test_attention_rerouted_sdpa.py b/tests/utils/test_attention_rerouted_sdpa.pynew file mode 100644index 00000000000..3fd4dc988ec--- /dev/null+++ b/tests/utils/test_attention_rerouted_sdpa.py@@ -0,0 +1,57 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.++"""SDPA standing in for xformers inside a compiled block keeps cuDNN attention off.++torch 2.13+ on SM100 picks cuDNN attention for plain causal SDPA, and cuDNN builds a new plan for+every new sequence length: padded training batches of varying length stalled every step (Llama-3.2-1B+LoRA on B200: 439 ms median step compiled vs 227 ms stock). xformers itself runs flash there.+"""++import pytest+import torch+import torch.nn.functional as F++ad = pytest.importorskip("unsloth.utils.attention_dispatch")+++def test_rerouted_sdpa_disables_cudnn_only(monkeypatch):+ seen = []++ def fake_sdpa(q, k, v, **kwargs):+ b = torch.backends.cuda+ seen.append(+ (+ b.cudnn_sdp_enabled(),+ b.flash_sdp_enabled(),+ b.mem_efficient_sdp_enabled(),+ b.math_sdp_enabled(),+ )+ )+ return q++ monkeypatch.setattr(F, "scaled_dot_product_attention", fake_sdpa)+ before = torch.backends.cuda.cudnn_sdp_enabled()+ q = torch.zeros(1, 2, 4, 8)+ ad._rerouted_sdpa(q, q, q, is_causal = True)+ ad.scaled_dot_product_attention(q, q, q, is_causal = True)+ assert seen[0] == (False, True, True, True)+ assert seen[1][0] == before # the plain path is untouched+ assert torch.backends.cuda.cudnn_sdp_enabled() == before+++def test_rerouted_sdpa_traces_without_a_break():+ from torch._dynamo.utils import counters++ torch._dynamo.reset()+ counters.clear()++ def f(q):+ return ad._rerouted_sdpa(q, q, q, is_causal = True)++ c = torch.compile(f, backend = "eager", dynamic = True)+ for L in (5, 9, 16, 23):+ q = torch.randn(1, 2, L, 8)+ torch.testing.assert_close(c(q), F.scaled_dot_product_attention(q, q, q, is_causal = True))+ assert dict(counters["graph_break"]) == {}+ torch._dynamo.reset()diff --git a/unsloth/utils/attention_dispatch.py b/unsloth/utils/attention_dispatch.pyindex 173eae42601..b3beac205aa 100644--- a/unsloth/utils/attention_dispatch.py+++ b/unsloth/utils/attention_dispatch.py@@ -99,6 +99,26 @@ def scaled_dot_product_attention(*args, **kwargs):return F.scaled_dot_product_attention(*args, **kwargs)+def _rerouted_sdpa(*args, **kwargs):+ """SDPA standing in for xformers inside a compiled block, on the kernels xformers would run.+ Without the pin, torch 2.13+ on SM100 takes cuDNN attention, which builds a new plan for every+ new sequence length (~150 ms per length): padded batches of varying length stalled every step."""+ with _sdpa_kernel(_XFORMERS_LIKE_BACKENDS):+ return F.scaled_dot_product_attention(*args, **kwargs)+++# Module level: reading SDPBackend's attributes while traced breaks the graph.+try:+ from torch.nn.attention import SDPBackend as _SDPBackend, sdpa_kernel as _sdpa_kernel+ _XFORMERS_LIKE_BACKENDS = [+ _SDPBackend.FLASH_ATTENTION,+ _SDPBackend.EFFICIENT_ATTENTION,+ _SDPBackend.MATH,+ ]+except Exception:+ _rerouted_sdpa = scaled_dot_product_attention++def _per_device_backends(capabilities):"""(flash runs, xformers runs, xformers rejects fp32, bf16 kernels) per CUDA device index, or Nonewhen every device has device 0's capability class, so homogeneous hosts never consult the table.@@ -1281,7 +1301,9 @@ def _run_attention(if use_sdpa_gqa:kwargs.setdefault("enable_gqa", True)- out = scaled_dot_product_attention(Q, K, V, **kwargs)+ out = (_rerouted_sdpa if xformers_rerouted else scaled_dot_product_attention)(+ Q, K, V, **kwargs+ )out = out.transpose(1, 2)return out if cp_out_dtype is None else out.to(cp_out_dtype)@@ -1297,7 +1319,7 @@ def _run_attention(K_mod = K_mod.reshape(bsz, n_heads, kv_seq_len, head_dim)V_mod = V_mod.reshape(bsz, n_heads, kv_seq_len, head_dim)- out = scaled_dot_product_attention(+ out = (_rerouted_sdpa if xformers_rerouted else scaled_dot_product_attention)(Q.contiguous(),K_mod.contiguous(),V_mod.contiguous(),