Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: honour an explicit flash attention request on ROCm (gfx1151 8 to 12% faster per step) (#12667)

Daniel Han52f0d166a271
diff --git a/studio/backend/core/inference/diffusion_attention.py b/studio/backend/core/inference/diffusion_attention.py
index b0ca92cd697..d7f6ae43f63 100644
--- a/studio/backend/core/inference/diffusion_attention.py
+++ b/studio/backend/core/inference/diffusion_attention.py
@@ -24,6 +24,7 @@ Attention is bandwidth-bound, so a better kernel is a real win orthogonal to wei
sage - SageAttention 2 (INT8 QK); quantized, small quality cost. A pip SageAttention 2 when installed, else the
kernels-community build from the Hugging Face kernels hub (``sage_hub``, Ampere / Ada / Hopper).
xformers / aiter - memory-efficient (NVIDIA) / AITER (AMD ROCm).
+ flash on ROCm - only when the DAO-AILab ROCm build imports and passes an on-card check; ``auto`` stays native.
Best-effort: an unavailable backend falls back to the diffusers default. torch/diffusers lazy.
"""
@@ -264,8 +265,10 @@ def select_attention_backend(
if getattr(target, "device", None) == "cuda" and not _is_cuda_nvidia(target):
return backend
return None
- # cuDNN / flash* / sage are CUDA+NVIDIA-only; elsewhere the first generation crashes.
+ # cuDNN / flash* / sage are CUDA+NVIDIA-only (elsewhere the first generation crashes), except verified ROCm FA2.
if not _is_cuda_nvidia(target):
+ if backend == "flash" and _is_cuda_rocm(target) and _rocm_flash_attn_runs(target):
+ return backend
return None
# An arch-gated kernel (flash3/flash4) on a card that can't run it sets fine then crashes.
if not _backend_arch_supported(backend):
@@ -276,9 +279,100 @@ def select_attention_backend(
return backend
if speed_active and _is_cuda_nvidia(target) and _cudnn_attention_supported():
return "_native_cudnn"
+ if speed_active and ROCM_AUTO_FLASH and _is_cuda_rocm(target) and _rocm_flash_attn_runs(target):
+ return "flash"
return None
+def _is_cuda_rocm(target: Any) -> bool:
+ return getattr(target, "device", None) == "cuda" and not _is_cuda_nvidia(target)
+
+
+# ``auto`` -> verified ROCm flash under a speed profile. Off: gfx1151 gains over AOTriton SDPA were too small/narrow.
+ROCM_AUTO_FLASH = False
+
+# Max abs error of the probe's flash_attn output against an fp32 reference (bf16 eps is ~4e-3).
+_ROCM_FLASH_PROBE_TOL = 2e-2
+# (device, dtype) -> whether flash_attn ran and matched. Only answers are cached; an unaskable probe is retried.
+_ROCM_FLASH_PROBE_CACHE: dict[tuple[str, str], bool] = {}
+_ROCM_FLASH_MISSING_LOGGED: list[bool] = []
+
+_ROCM_FLASH_HINT = (
+ "the PyPI flash-attn wheel is the NVIDIA CUDA build; on ROCm install the DAO-AILab flash-attention ROCm build "
+ "(github.com/Dao-AILab/flash-attention, CK backend, or FLASH_ATTENTION_TRITON_AMD_ENABLE=TRUE for Triton) "
+ "or use attention_backend=aiter"
+)
+
+
+def _module_logger() -> Any:
+ try:
+ from loggers import get_logger
+ return get_logger(__name__)
+ except Exception: # noqa: BLE001
+ import logging
+ return logging.getLogger(__name__)
+
+
+def _run_rocm_flash_probe(device: str, dtype: Any) -> bool:
+ """Tiny ``flash_attn_func`` on ``device`` vs an fp32 reference; False when it raises or disagrees."""
+ import torch
+ from flash_attn import flash_attn_func
+
+ if dtype not in (torch.float16, torch.bfloat16):
+ # diffusers' flash backend is half precision only, and the CK build raises on fp32 at the first call
+ if dtype is not None:
+ return False
+ dtype = torch.bfloat16
+ gen = torch.Generator().manual_seed(0)
+ q, k, v = (torch.randn((1, 128, 2, 64), generator = gen).to(device, dtype) for _ in range(3))
+ try:
+ out = flash_attn_func(q, k, v)
+ torch.cuda.synchronize(device)
+ except torch.cuda.OutOfMemoryError:
+ raise
+ except Exception: # noqa: BLE001 - no kernel for this arch is the answer
+ return False
+ qf, kf, vf = (t.float().transpose(1, 2) for t in (q, k, v))
+ ref = torch.softmax(qf @ kf.transpose(-1, -2) * qf.shape[-1] ** -0.5, dim = -1) @ vf
+ err = (out.float() - ref.transpose(1, 2)).abs().max()
+ return bool(torch.isfinite(err).item() and err.item() <= _ROCM_FLASH_PROBE_TOL)
+
+
+def _rocm_flash_attn_runs(target: Any) -> bool:
+ """True only when flash_attn imports and its kernel ran and matched on this ROCm card at this dtype."""
+ device = _indexed_cuda_device(
+ str(getattr(target, "torch_device", None) or getattr(target, "device", None) or "")
+ )
+ dtype = getattr(target, "dtype", None)
+ key = (device, str(dtype))
+ cached = _ROCM_FLASH_PROBE_CACHE.get(key)
+ if cached is not None:
+ return cached
+ try:
+ import flash_attn # noqa: F401
+ except Exception as exc: # noqa: BLE001
+ if not _ROCM_FLASH_MISSING_LOGGED:
+ _ROCM_FLASH_MISSING_LOGGED.append(True)
+ _module_logger().warning(
+ "diffusion.attention: flash requested on ROCm but flash_attn is not importable (%s); %s. "
+ "Using the default backend",
+ exc,
+ _ROCM_FLASH_HINT,
+ )
+ return False
+ try:
+ ok = _run_rocm_flash_probe(device, dtype)
+ except Exception: # noqa: BLE001 - OOM / device trouble: no answer, keep native this time
+ return False
+ if not ok:
+ _module_logger().warning(
+ "diffusion.attention: flash_attn did not run or did not match SDPA on %s (%s); using the default backend",
+ device,
+ dtype,
+ )
+ return _ROCM_FLASH_PROBE_CACHE.setdefault(key, ok)
+
+
def _cudnn_attention_supported() -> bool:
"""cuDNN fused SDPA needs Ampere+ (SM80); on pre-SM80 cards (T4/V100) diffusers accepts it
then fails at generation, so gate the upgrade on capability. Unknown capability allows it."""
@@ -1368,6 +1462,18 @@ def _ensure_backend_package(backend: str, logger: Any = None) -> Optional[str]:
return None
except Exception: # noqa: BLE001 - a broken install probes as missing; try the install
pass
+ # PyPI flash-attn is the CUDA build: never pip it onto a ROCm torch. Policy, so no _INSTALL_ATTEMPTED record.
+ if backend == "flash" and _torch_is_rocm():
+ reason = "flash-attn on PyPI is the NVIDIA CUDA build"
+ if logger is not None:
+ logger.warning(
+ "diffusion.attention: not installing %s for backend=%s on ROCm: %s; %s. Using the default backend",
+ package,
+ backend,
+ reason,
+ _ROCM_FLASH_HINT,
+ )
+ return reason
# XFormers ships a compiled extension tied to one exact (torch, CUDA) pair, so the name `xformers` is not a safe
# thing to hand pip: PyPI serves only the CUDA-12.8 build and --no-deps below stops pip from ever reading its
# `Requires-Dist: torch==X`. Resolve the matching wheel URL instead, and REFUSE when there is none -- like the
@@ -1450,6 +1556,16 @@ def _ensure_backend_package(backend: str, logger: Any = None) -> Optional[str]:
return None
+def _torch_is_rocm() -> bool:
+ try:
+ import torch
+
+ from core._torchao_stub import _module_is_rocm
+ return _module_is_rocm(torch)
+ except Exception: # noqa: BLE001
+ return False
+
+
def _attention_dits(pipe: Any) -> list:
"""Every DiT the denoise loop runs: the primary ``transformer`` plus a second expert some
families carry (Ideogram's ``unconditional_transformer``, an MoE ``transformer_2``). The
@@ -1611,6 +1727,14 @@ def apply_attention_backend(
if too_old and logger is not None:
logger.info("diffusion.attention: ignoring %s", too_old)
backend = _sage_hub_backend(pipe, target, logger)
+ # Re-verify at the dtype the pipeline runs in: selection may have seen the pre-promotion (fp16) dtype.
+ if (
+ backend == "flash"
+ and target is not None
+ and _is_cuda_rocm(target)
+ and not _rocm_flash_attn_runs(target)
+ ):
+ backend = None
if (
backend == "_native_cudnn"
and target is not None
diff --git a/studio/backend/tests/test_diffusion_attention_rocm_flash.py b/studio/backend/tests/test_diffusion_attention_rocm_flash.py
new file mode 100644
index 00000000000..c9e04529045
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_attention_rocm_flash.py
@@ -0,0 +1,242 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Explicit ``flash`` on ROCm: honored only when flash_attn imports and the per-device check passes; ``auto`` and every
+NVIDIA path unchanged; the CUDA flash-attn wheel is never pip installed onto ROCm. Hermetic, CPU only."""
+
+from __future__ import annotations
+
+import subprocess
+import sys
+import types
+
+import pytest
+
+import core.inference.diffusion_attention as att
+from core.inference.diffusion_attention import select_attention_backend
+
+
[email protected](autouse = True)
+def _clean(monkeypatch):
+ monkeypatch.setattr(att, "_ROCM_FLASH_PROBE_CACHE", {}, raising = False)
+ monkeypatch.setattr(
+ att, "_indexed_cuda_device", lambda d: "cuda:0" if d == "cuda" else d, raising = False
+ )
+ monkeypatch.setattr(att, "_cuda_capability", lambda: (9, 0))
+ monkeypatch.setattr(att, "_INSTALL_ATTEMPTED", set())
+
+
+def _target(device = "cuda", dtype = "torch.bfloat16"):
+ return types.SimpleNamespace(device = device, dtype = dtype)
+
+
+def _rocm(
+ monkeypatch,
+ *,
+ flash_attn = True,
+ probe = True,
+):
+ monkeypatch.setattr(att, "_is_cuda_nvidia", lambda target: False)
+ monkeypatch.setitem(
+ sys.modules, "flash_attn", types.ModuleType("flash_attn") if flash_attn else None
+ )
+ calls = []
+
+ def fake_probe(device, dtype):
+ calls.append((device, dtype))
+ return probe
+
+ monkeypatch.setattr(att, "_run_rocm_flash_probe", fake_probe, raising = False)
+ return calls
+
+
[email protected]("alias", ["flash", "flash2", "FLASH"])
+def test_rocm_flash_honored_when_imported_and_check_passes(monkeypatch, alias):
+ calls = _rocm(monkeypatch)
+ assert select_attention_backend(_target(), alias, speed_active = False) == "flash"
+ assert select_attention_backend(_target(), alias, speed_active = True) == "flash"
+ assert calls == [("cuda:0", "torch.bfloat16")] # cached per (device, dtype)
+
+
+def test_rocm_flash_check_cached_per_device_and_dtype(monkeypatch):
+ calls = _rocm(monkeypatch)
+ select_attention_backend(_target(), "flash", speed_active = True)
+ select_attention_backend(_target("cuda", "torch.float16"), "flash", speed_active = True)
+ select_attention_backend(
+ types.SimpleNamespace(device = "cuda", torch_device = "cuda:1", dtype = "torch.float16"),
+ "flash",
+ speed_active = True,
+ )
+ assert calls == [
+ ("cuda:0", "torch.bfloat16"),
+ ("cuda:0", "torch.float16"),
+ ("cuda:1", "torch.float16"),
+ ]
+
+
+def test_rocm_flash_without_flash_attn_is_native(monkeypatch):
+ calls = _rocm(monkeypatch, flash_attn = False)
+ assert select_attention_backend(_target(), "flash", speed_active = True) is None
+ assert calls == []
+
+
+def test_rocm_flash_check_failure_is_native(monkeypatch):
+ _rocm(monkeypatch, probe = False)
+ assert select_attention_backend(_target(), "flash", speed_active = True) is None
+
+
+def test_rocm_unaskable_check_is_native_and_not_cached(monkeypatch):
+ _rocm(monkeypatch)
+
+ def boom(device, dtype):
+ raise RuntimeError("out of memory")
+
+ monkeypatch.setattr(att, "_run_rocm_flash_probe", boom)
+ assert select_attention_backend(_target(), "flash", speed_active = True) is None
+ assert att._ROCM_FLASH_PROBE_CACHE == {}
+
+
+def test_rocm_auto_unchanged(monkeypatch):
+ calls = _rocm(monkeypatch)
+ assert select_attention_backend(_target(), "auto", speed_active = True) is None
+ assert select_attention_backend(_target(), "auto", speed_active = False) is None
+ assert calls == []
+
+
[email protected]("alias", ["flash3", "flash4", "sage", "cudnn", "xformers"])
+def test_rocm_other_nvidia_kernels_still_dropped(monkeypatch, alias):
+ _rocm(monkeypatch)
+ assert select_attention_backend(_target(), alias, speed_active = True) is None
+
+
[email protected]("device", ["mps", "cpu", "xpu"])
+def test_flash_off_cuda_still_native(monkeypatch, device):
+ _rocm(monkeypatch)
+ assert select_attention_backend(_target(device), "flash", speed_active = True) is None
+
+
+# origin/main behaviour on an NVIDIA target at SM90 (flash3 in range, flash4 out). Frozen: must never move.
+_NVIDIA_EXPECTED = {
+ "auto": ("_native_cudnn", None),
+ "native": (None, None),
+ "sdpa": (None, None),
+ "cudnn": ("_native_cudnn", "_native_cudnn"),
+ "flash": ("flash", "flash"),
+ "flash2": ("flash", "flash"),
+ "flash3": ("_flash_3_hub", "_flash_3_hub"),
+ "flash4": (None, None),
+ "sage": ("sage", "sage"),
+ "xformers": ("xformers", "xformers"),
+ "aiter": (None, None),
+}
+
+
[email protected]("alias", sorted(_NVIDIA_EXPECTED))
+def test_nvidia_selection_identical_to_main(monkeypatch, alias):
+ monkeypatch.setattr(att, "_is_cuda_nvidia", lambda target: True)
+
+ def never(*a, **k):
+ raise AssertionError("ROCm flash check must not run on NVIDIA")
+
+ monkeypatch.setattr(att, "_run_rocm_flash_probe", never, raising = False)
+ on, off = _NVIDIA_EXPECTED[alias]
+ assert select_attention_backend(_target(), alias, speed_active = True) == on
+ assert select_attention_backend(_target(), alias, speed_active = False) == off
+
+
+def test_alias_table_covered():
+ assert set(_NVIDIA_EXPECTED) == set(att.ATTN_ALIASES)
+
+
+def _fake_torch(monkeypatch, *, hip):
+ torch = types.ModuleType("torch")
+ torch.version = types.SimpleNamespace(hip = hip)
+ torch.__version__ = "2.9.0+rocm6.4" if hip else "2.9.0+cu128"
+ monkeypatch.setitem(sys.modules, "torch", torch)
+
+
+def test_installer_never_pip_installs_flash_attn_on_rocm(monkeypatch):
+ _fake_torch(monkeypatch, hip = "6.4.0")
+ monkeypatch.delitem(sys.modules, "flash_attn", raising = False)
+ monkeypatch.setattr("importlib.util.find_spec", lambda name, *a, **k: None)
+ monkeypatch.delenv(att._ATTENTION_INSTALL_ENV, raising = False)
+ ran = []
+ monkeypatch.setattr(subprocess, "run", lambda *a, **k: ran.append(a))
+ reason = att._ensure_attention_backend_installed("flash")
+ assert ran == []
+ assert reason and "CUDA" in reason
+ assert att._INSTALL_ATTEMPTED == set()
+
+
+def test_installer_still_installs_flash_attn_on_nvidia(monkeypatch):
+ _fake_torch(monkeypatch, hip = None)
+ monkeypatch.delitem(sys.modules, "flash_attn", raising = False)
+ monkeypatch.setattr("importlib.util.find_spec", lambda name, *a, **k: None)
+ monkeypatch.delenv(att._ATTENTION_INSTALL_ENV, raising = False)
+ ran = []
+ monkeypatch.setattr(subprocess, "run", lambda *a, **k: ran.append(a[0]))
+ assert att._ensure_attention_backend_installed("flash") is None
+ assert len(ran) == 1 and ran[0][-1] == "flash-attn"
+
+
[email protected]("dtype_name", ["bfloat16", "float16", "float32"])
[email protected]("good", [True, False])
+def test_probe_compares_against_reference(monkeypatch, dtype_name, good):
+ torch = pytest.importorskip("torch")
+ fa = types.ModuleType("flash_attn")
+
+ def flash_attn_func(q, k, v):
+ assert q.dtype in (torch.float16, torch.bfloat16)
+ out = (
+ torch.nn.functional.scaled_dot_product_attention(
+ *(t.float().transpose(1, 2) for t in (q, k, v))
+ )
+ .transpose(1, 2)
+ .to(q.dtype)
+ )
+ return out if good else out + 0.1
+
+ fa.flash_attn_func = flash_attn_func
+ monkeypatch.setitem(sys.modules, "flash_attn", fa)
+ monkeypatch.setattr(torch.cuda, "synchronize", lambda *a, **k: None)
+ expected = good and dtype_name != "float32"
+ assert att._run_rocm_flash_probe("cpu", getattr(torch, dtype_name)) is expected
+
+
+def test_apply_reverifies_rocm_flash_at_run_dtype(monkeypatch):
+ calls = _rocm(monkeypatch, probe = False)
+ set_to = []
+ dit = types.SimpleNamespace(set_attention_backend = set_to.append)
+ pipe = types.SimpleNamespace(transformer = dit)
+ monkeypatch.setattr(att, "_active_attention_backend", lambda: att.ATTN_NATIVE)
+ monkeypatch.setattr(att, "warn_if_sdpa_math_only", lambda *a, **k: False)
+ run_target = _target(dtype = "torch.float32")
+ assert att.apply_attention_backend(pipe, "flash", target = run_target) is None
+ assert calls == [("cuda:0", "torch.float32")]
+ assert set_to == []
+
+
+def test_probe_kernel_error_is_false(monkeypatch):
+ torch = pytest.importorskip("torch")
+ fa = types.ModuleType("flash_attn")
+
+ def flash_attn_func(q, k, v):
+ raise RuntimeError("HIP error: invalid device function")
+
+ fa.flash_attn_func = flash_attn_func
+ monkeypatch.setitem(sys.modules, "flash_attn", fa)
+ assert att._run_rocm_flash_probe("cpu", torch.bfloat16) is False
+
+
+def test_cudnn_head_dim_probe_still_answers_fp32_at_bf16(monkeypatch):
+ torch = pytest.importorskip("torch")
+ import torch.backends.cuda as tbc
+
+ if not hasattr(tbc, "can_use_cudnn_attention"):
+ pytest.skip("torch without can_use_cudnn_attention")
+ seen = []
+ monkeypatch.setattr(
+ tbc, "can_use_cudnn_attention", lambda p, debug: seen.append(p.query.dtype) or True
+ )
+ assert att._run_cudnn_head_dim_probe("cpu", torch.float32, 128) is True
+ assert seen == [torch.bfloat16]