Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: fall back to native when SageAttention does not run on this GPU (#11997)

Daniel Hanf5d3cd3908ae
diff --git a/studio/backend/core/inference/diffusion_attention.py b/studio/backend/core/inference/diffusion_attention.py
index bfd24252c99..f9dfed8237f 100644
--- a/studio/backend/core/inference/diffusion_attention.py
+++ b/studio/backend/core/inference/diffusion_attention.py
@@ -274,6 +274,63 @@ def _cudnn_attention_supported() -> bool:
return have is None or have >= (8, 0)
+# sageattn dispatches on an exact arch match whose set varies by build (2.2.0: sm80/86/89/90/120; community builds add
+# sm75/87/100) and diffusers only checks the version, so ask the kernel. (device, dtype) -> "" or its error.
+_SAGE_PROBE_CACHE: dict[tuple[str, str], str] = {}
+
+
+def _run_sage_probe(device: str, dtype: Any) -> str:
+ """Empty when a tiny ``sageattn`` ran on ``device``, else its error; raises when unaskable (import, device, OOM)."""
+ import torch
+ from sageattention import sageattn
+
+ if dtype not in (torch.float16, torch.bfloat16):
+ dtype = torch.float16
+ q = torch.zeros((1, 128, 2, 128), device = device, dtype = dtype)
+ try:
+ sageattn(q, q, q, tensor_layout = "NHD")
+ torch.cuda.synchronize(device)
+ except torch.cuda.OutOfMemoryError:
+ raise
+ except Exception as exc: # noqa: BLE001
+ return f"{type(exc).__name__}: {exc}"
+ return ""
+
+
+def _indexed_cuda_device(device: str) -> str:
+ """Bare "cuda" -> the card pinned on this thread, so one card's verdict is never reused for another."""
+ if device != "cuda":
+ return device
+ try:
+ import torch
+ return f"cuda:{torch.cuda.current_device()}"
+ except Exception: # noqa: BLE001
+ return device
+
+
+def _sage_kernel_runs(target: Any, logger: Any = None) -> Optional[bool]:
+ """False only when the kernel raised; None (unaskable, not cached) keeps the requested backend."""
+ device = str(getattr(target, "torch_device", None) or getattr(target, "device", None) or "")
+ if not device.startswith("cuda"):
+ return None
+ device = _indexed_cuda_device(device)
+ dtype = getattr(target, "dtype", None)
+ key = (device, str(dtype))
+ error = _SAGE_PROBE_CACHE.get(key)
+ if error is None:
+ try:
+ error = _run_sage_probe(device, dtype)
+ except Exception: # noqa: BLE001
+ return None
+ error = _SAGE_PROBE_CACHE.setdefault(key, error)
+ if error and logger is not None:
+ logger.warning(
+ "diffusion.attention: SageAttention does not run on this GPU (%s); using the default backend",
+ error,
+ )
+ return not error
+
+
# Optional kernels installable on demand: dispatcher name -> (probe module, pip package). Wheels only
# (--only-binary=:all:), since a source build needs a CUDA toolchain the host may lack.
_INSTALLABLE_BACKENDS: dict[str, tuple[str, str]] = {
@@ -616,6 +673,9 @@ def apply_attention_backend(
return None
if backend is not None:
_ensure_attention_backend_installed(backend, logger)
+ if backend == "sage" and target is not None and _sage_kernel_runs(target, logger) is False:
+ backend = None
+ if backend is not None:
engaged = False
for fn in setters:
try:
diff --git a/studio/backend/tests/test_diffusion_attention_sage_probe.py b/studio/backend/tests/test_diffusion_attention_sage_probe.py
new file mode 100644
index 00000000000..0212113adb1
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_attention_sage_probe.py
@@ -0,0 +1,185 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Explicit ``sage`` on a GPU the installed kernel does not serve falls back to native. Probe stubbed."""
+
+from __future__ import annotations
+
+import types
+
+import pytest
+
+import core.inference.diffusion_attention as att
+from core.inference.diffusion_attention import apply_attention_backend
+
+_UNSUPPORTED = "ValueError: Unsupported CUDA architecture: sm100"
+
+
+class _Transformer:
+ def __init__(self):
+ self.calls: list = []
+
+ def set_attention_backend(self, name):
+ self.calls.append(name)
+
+
+class _Logger:
+ def __init__(self):
+ self.warnings: list = []
+
+ def warning(self, msg, *args):
+ self.warnings.append(msg % args if args else msg)
+
+ def info(self, *_a, **_k):
+ pass
+
+
+def _target(
+ device = "cuda",
+ dtype = "bf16",
+ torch_device = None,
+):
+ return types.SimpleNamespace(device = device, dtype = dtype, torch_device = torch_device or device)
+
+
[email protected](autouse = True)
+def _isolated(monkeypatch):
+ monkeypatch.setattr(att, "_SAGE_PROBE_CACHE", {})
+ monkeypatch.setattr(att, "_ensure_attention_backend_installed", lambda *a, **k: None)
+ monkeypatch.setattr(att, "_active_attention_backend", lambda: "native")
+ monkeypatch.setattr(att, "warn_if_sdpa_math_only", lambda *a, **k: False)
+ monkeypatch.setattr(att, "_indexed_cuda_device", lambda device: device)
+
+
+def _stub_probe(
+ monkeypatch,
+ result,
+ seen = None,
+):
+ def _probe(device, dtype):
+ if seen is not None:
+ seen.append((device, dtype))
+ if isinstance(result, BaseException):
+ raise result
+ return result
+
+ monkeypatch.setattr(att, "_run_sage_probe", _probe)
+
+
+def test_sage_on_a_card_its_kernel_rejects_falls_back_to_native(monkeypatch):
+ _stub_probe(monkeypatch, _UNSUPPORTED)
+ t, log = _Transformer(), _Logger()
+ assert (
+ apply_attention_backend(
+ types.SimpleNamespace(transformer = t), "sage", logger = log, target = _target()
+ )
+ is None
+ )
+ assert "sage" not in t.calls
+ assert any("Unsupported CUDA architecture: sm100" in w for w in log.warnings)
+
+
+def test_sage_stays_engaged_where_its_kernel_runs(monkeypatch):
+ _stub_probe(monkeypatch, "")
+ t = _Transformer()
+ assert (
+ apply_attention_backend(types.SimpleNamespace(transformer = t), "sage", target = _target())
+ == "sage"
+ )
+ assert t.calls == ["sage"]
+
+
+ "exc", [ImportError("no sageattention"), RuntimeError("CUDA out of memory")]
+)
+def test_an_unanswerable_probe_keeps_the_request(monkeypatch, exc):
+ seen: list = []
+ _stub_probe(monkeypatch, exc, seen)
+ t = _Transformer()
+ assert (
+ apply_attention_backend(types.SimpleNamespace(transformer = t), "sage", target = _target())
+ == "sage"
+ )
+ assert t.calls == ["sage"]
+ assert att._sage_kernel_runs(_target()) is None and len(seen) == 2
+
+
+def test_probe_is_memoised_per_device_and_dtype_and_warns_each_time(monkeypatch):
+ seen: list = []
+ _stub_probe(monkeypatch, _UNSUPPORTED, seen)
+ log = _Logger()
+ assert att._sage_kernel_runs(_target(), log) is False
+ assert att._sage_kernel_runs(_target(), log) is False
+ assert len(seen) == 1 and len(log.warnings) == 2
+ att._sage_kernel_runs(_target(dtype = "fp16"))
+ att._sage_kernel_runs(_target(torch_device = "cuda:1"))
+ assert seen[1:] == [("cuda", "fp16"), ("cuda:1", "bf16")]
+
+
+def test_other_backends_and_targetless_calls_never_probe(monkeypatch):
+ seen: list = []
+ _stub_probe(monkeypatch, _UNSUPPORTED, seen)
+ for backend in ("_native_cudnn", "flash", "_flash_3_hub", "flash_4_hub", "xformers", "aiter"):
+ t = _Transformer()
+ assert (
+ apply_attention_backend(types.SimpleNamespace(transformer = t), backend, target = _target())
+ == backend
+ )
+ t = _Transformer()
+ assert apply_attention_backend(types.SimpleNamespace(transformer = t), "sage") == "sage"
+ assert seen == []
+
+
[email protected]("device", ["cpu", "mps", "xpu", "", None])
+def test_non_cuda_targets_are_not_probed(monkeypatch, device):
+ seen: list = []
+ _stub_probe(monkeypatch, _UNSUPPORTED, seen)
+ assert att._sage_kernel_runs(types.SimpleNamespace(device = device, dtype = None)) is None
+ assert seen == []
+
+
+def test_run_probe_reports_the_kernels_own_error(monkeypatch):
+ torch = pytest.importorskip("torch")
+
+ def _sageattn(
+ q,
+ k,
+ v,
+ tensor_layout = "HND",
+ ):
+ assert tensor_layout == "NHD" and q.dtype == torch.float16 and q.shape == (1, 128, 2, 128)
+ raise ValueError("Unsupported CUDA architecture: sm75")
+
+ monkeypatch.setitem(
+ __import__("sys").modules, "sageattention", types.SimpleNamespace(sageattn = _sageattn)
+ )
+ assert att._run_sage_probe("cpu", None) == "ValueError: Unsupported CUDA architecture: sm75"
+
+
+def test_run_probe_raises_when_the_package_is_missing(monkeypatch):
+ pytest.importorskip("torch")
+ monkeypatch.setitem(__import__("sys").modules, "sageattention", None)
+ with pytest.raises(ImportError):
+ att._run_sage_probe("cpu", None)
+
+
+def test_bare_cuda_is_keyed_by_the_card_pinned_on_this_thread(monkeypatch):
+ seen: list = []
+ current = {"card": 0}
+ monkeypatch.setattr(
+ att, "_indexed_cuda_device", lambda d: f"cuda:{current['card']}" if d == "cuda" else d
+ )
+ monkeypatch.setattr(
+ att,
+ "_run_sage_probe",
+ lambda d, dt: seen.append(d) or ("" if d == "cuda:1" else _UNSUPPORTED),
+ )
+ bare = types.SimpleNamespace(device = "cuda", dtype = "bf16")
+ assert att._sage_kernel_runs(bare) is False
+ current["card"] = 1
+ assert att._sage_kernel_runs(bare) is True
+ assert seen == ["cuda:0", "cuda:1"]
+
+
+def test_indexed_cuda_device_leaves_indexed_names_alone():
+ assert att._indexed_cuda_device("cuda:3") == "cuda:3"