Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: honour an explicit flash attention request on ROCm (gfx1151 8 to 12% faster per step) (#12667)
Daniel Han52f0d166a271
diff --git a/studio/backend/core/inference/diffusion_attention.py b/studio/backend/core/inference/diffusion_attention.pyindex b0ca92cd697..d7f6ae43f63 100644--- a/studio/backend/core/inference/diffusion_attention.py+++ b/studio/backend/core/inference/diffusion_attention.py@@ -24,6 +24,7 @@ Attention is bandwidth-bound, so a better kernel is a real win orthogonal to weisage - SageAttention 2 (INT8 QK); quantized, small quality cost. A pip SageAttention 2 when installed, else thekernels-community build from the Hugging Face kernels hub (``sage_hub``, Ampere / Ada / Hopper).xformers / aiter - memory-efficient (NVIDIA) / AITER (AMD ROCm).+ flash on ROCm - only when the DAO-AILab ROCm build imports and passes an on-card check; ``auto`` stays native.Best-effort: an unavailable backend falls back to the diffusers default. torch/diffusers lazy."""@@ -264,8 +265,10 @@ def select_attention_backend(if getattr(target, "device", None) == "cuda" and not _is_cuda_nvidia(target):return backendreturn None- # cuDNN / flash* / sage are CUDA+NVIDIA-only; elsewhere the first generation crashes.+ # cuDNN / flash* / sage are CUDA+NVIDIA-only (elsewhere the first generation crashes), except verified ROCm FA2.if not _is_cuda_nvidia(target):+ if backend == "flash" and _is_cuda_rocm(target) and _rocm_flash_attn_runs(target):+ return backendreturn None# An arch-gated kernel (flash3/flash4) on a card that can't run it sets fine then crashes.if not _backend_arch_supported(backend):@@ -276,9 +279,100 @@ def select_attention_backend(return backendif speed_active and _is_cuda_nvidia(target) and _cudnn_attention_supported():return "_native_cudnn"+ if speed_active and ROCM_AUTO_FLASH and _is_cuda_rocm(target) and _rocm_flash_attn_runs(target):+ return "flash"return None+def _is_cuda_rocm(target: Any) -> bool:+ return getattr(target, "device", None) == "cuda" and not _is_cuda_nvidia(target)+++# ``auto`` -> verified ROCm flash under a speed profile. Off: gfx1151 gains over AOTriton SDPA were too small/narrow.+ROCM_AUTO_FLASH = False++# Max abs error of the probe's flash_attn output against an fp32 reference (bf16 eps is ~4e-3).+_ROCM_FLASH_PROBE_TOL = 2e-2+# (device, dtype) -> whether flash_attn ran and matched. Only answers are cached; an unaskable probe is retried.+_ROCM_FLASH_PROBE_CACHE: dict[tuple[str, str], bool] = {}+_ROCM_FLASH_MISSING_LOGGED: list[bool] = []++_ROCM_FLASH_HINT = (+ "the PyPI flash-attn wheel is the NVIDIA CUDA build; on ROCm install the DAO-AILab flash-attention ROCm build "+ "(github.com/Dao-AILab/flash-attention, CK backend, or FLASH_ATTENTION_TRITON_AMD_ENABLE=TRUE for Triton) "+ "or use attention_backend=aiter"+)+++def _module_logger() -> Any:+ try:+ from loggers import get_logger+ return get_logger(__name__)+ except Exception: # noqa: BLE001+ import logging+ return logging.getLogger(__name__)+++def _run_rocm_flash_probe(device: str, dtype: Any) -> bool:+ """Tiny ``flash_attn_func`` on ``device`` vs an fp32 reference; False when it raises or disagrees."""+ import torch+ from flash_attn import flash_attn_func++ if dtype not in (torch.float16, torch.bfloat16):+ # diffusers' flash backend is half precision only, and the CK build raises on fp32 at the first call+ if dtype is not None:+ return False+ dtype = torch.bfloat16+ gen = torch.Generator().manual_seed(0)+ q, k, v = (torch.randn((1, 128, 2, 64), generator = gen).to(device, dtype) for _ in range(3))+ try:+ out = flash_attn_func(q, k, v)+ torch.cuda.synchronize(device)+ except torch.cuda.OutOfMemoryError:+ raise+ except Exception: # noqa: BLE001 - no kernel for this arch is the answer+ return False+ qf, kf, vf = (t.float().transpose(1, 2) for t in (q, k, v))+ ref = torch.softmax(qf @ kf.transpose(-1, -2) * qf.shape[-1] ** -0.5, dim = -1) @ vf+ err = (out.float() - ref.transpose(1, 2)).abs().max()+ return bool(torch.isfinite(err).item() and err.item() <= _ROCM_FLASH_PROBE_TOL)+++def _rocm_flash_attn_runs(target: Any) -> bool:+ """True only when flash_attn imports and its kernel ran and matched on this ROCm card at this dtype."""+ device = _indexed_cuda_device(+ str(getattr(target, "torch_device", None) or getattr(target, "device", None) or "")+ )+ dtype = getattr(target, "dtype", None)+ key = (device, str(dtype))+ cached = _ROCM_FLASH_PROBE_CACHE.get(key)+ if cached is not None:+ return cached+ try:+ import flash_attn # noqa: F401+ except Exception as exc: # noqa: BLE001+ if not _ROCM_FLASH_MISSING_LOGGED:+ _ROCM_FLASH_MISSING_LOGGED.append(True)+ _module_logger().warning(+ "diffusion.attention: flash requested on ROCm but flash_attn is not importable (%s); %s. "+ "Using the default backend",+ exc,+ _ROCM_FLASH_HINT,+ )+ return False+ try:+ ok = _run_rocm_flash_probe(device, dtype)+ except Exception: # noqa: BLE001 - OOM / device trouble: no answer, keep native this time+ return False+ if not ok:+ _module_logger().warning(+ "diffusion.attention: flash_attn did not run or did not match SDPA on %s (%s); using the default backend",+ device,+ dtype,+ )+ return _ROCM_FLASH_PROBE_CACHE.setdefault(key, ok)++def _cudnn_attention_supported() -> bool:"""cuDNN fused SDPA needs Ampere+ (SM80); on pre-SM80 cards (T4/V100) diffusers accepts itthen fails at generation, so gate the upgrade on capability. Unknown capability allows it."""@@ -1368,6 +1462,18 @@ def _ensure_backend_package(backend: str, logger: Any = None) -> Optional[str]:return Noneexcept Exception: # noqa: BLE001 - a broken install probes as missing; try the installpass+ # PyPI flash-attn is the CUDA build: never pip it onto a ROCm torch. Policy, so no _INSTALL_ATTEMPTED record.+ if backend == "flash" and _torch_is_rocm():+ reason = "flash-attn on PyPI is the NVIDIA CUDA build"+ if logger is not None:+ logger.warning(+ "diffusion.attention: not installing %s for backend=%s on ROCm: %s; %s. Using the default backend",+ package,+ backend,+ reason,+ _ROCM_FLASH_HINT,+ )+ return reason# XFormers ships a compiled extension tied to one exact (torch, CUDA) pair, so the name `xformers` is not a safe# thing to hand pip: PyPI serves only the CUDA-12.8 build and --no-deps below stops pip from ever reading its# `Requires-Dist: torch==X`. Resolve the matching wheel URL instead, and REFUSE when there is none -- like the@@ -1450,6 +1556,16 @@ def _ensure_backend_package(backend: str, logger: Any = None) -> Optional[str]:return None+def _torch_is_rocm() -> bool:+ try:+ import torch++ from core._torchao_stub import _module_is_rocm+ return _module_is_rocm(torch)+ except Exception: # noqa: BLE001+ return False++def _attention_dits(pipe: Any) -> list:"""Every DiT the denoise loop runs: the primary ``transformer`` plus a second expert somefamilies carry (Ideogram's ``unconditional_transformer``, an MoE ``transformer_2``). The@@ -1611,6 +1727,14 @@ def apply_attention_backend(if too_old and logger is not None:logger.info("diffusion.attention: ignoring %s", too_old)backend = _sage_hub_backend(pipe, target, logger)+ # Re-verify at the dtype the pipeline runs in: selection may have seen the pre-promotion (fp16) dtype.+ if (+ backend == "flash"+ and target is not None+ and _is_cuda_rocm(target)+ and not _rocm_flash_attn_runs(target)+ ):+ backend = Noneif (backend == "_native_cudnn"and target is not Nonediff --git a/studio/backend/tests/test_diffusion_attention_rocm_flash.py b/studio/backend/tests/test_diffusion_attention_rocm_flash.pynew file mode 100644index 00000000000..c9e04529045--- /dev/null+++ b/studio/backend/tests/test_diffusion_attention_rocm_flash.py@@ -0,0 +1,242 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Explicit ``flash`` on ROCm: honored only when flash_attn imports and the per-device check passes; ``auto`` and every+NVIDIA path unchanged; the CUDA flash-attn wheel is never pip installed onto ROCm. Hermetic, CPU only."""++from __future__ import annotations++import subprocess+import sys+import types++import pytest++import core.inference.diffusion_attention as att+from core.inference.diffusion_attention import select_attention_backend++[email protected](autouse = True)+def _clean(monkeypatch):+ monkeypatch.setattr(att, "_ROCM_FLASH_PROBE_CACHE", {}, raising = False)+ monkeypatch.setattr(+ att, "_indexed_cuda_device", lambda d: "cuda:0" if d == "cuda" else d, raising = False+ )+ monkeypatch.setattr(att, "_cuda_capability", lambda: (9, 0))+ monkeypatch.setattr(att, "_INSTALL_ATTEMPTED", set())+++def _target(device = "cuda", dtype = "torch.bfloat16"):+ return types.SimpleNamespace(device = device, dtype = dtype)+++def _rocm(+ monkeypatch,+ *,+ flash_attn = True,+ probe = True,+):+ monkeypatch.setattr(att, "_is_cuda_nvidia", lambda target: False)+ monkeypatch.setitem(+ sys.modules, "flash_attn", types.ModuleType("flash_attn") if flash_attn else None+ )+ calls = []++ def fake_probe(device, dtype):+ calls.append((device, dtype))+ return probe++ monkeypatch.setattr(att, "_run_rocm_flash_probe", fake_probe, raising = False)+ return calls++[email protected]("alias", ["flash", "flash2", "FLASH"])+def test_rocm_flash_honored_when_imported_and_check_passes(monkeypatch, alias):+ calls = _rocm(monkeypatch)+ assert select_attention_backend(_target(), alias, speed_active = False) == "flash"+ assert select_attention_backend(_target(), alias, speed_active = True) == "flash"+ assert calls == [("cuda:0", "torch.bfloat16")] # cached per (device, dtype)+++def test_rocm_flash_check_cached_per_device_and_dtype(monkeypatch):+ calls = _rocm(monkeypatch)+ select_attention_backend(_target(), "flash", speed_active = True)+ select_attention_backend(_target("cuda", "torch.float16"), "flash", speed_active = True)+ select_attention_backend(+ types.SimpleNamespace(device = "cuda", torch_device = "cuda:1", dtype = "torch.float16"),+ "flash",+ speed_active = True,+ )+ assert calls == [+ ("cuda:0", "torch.bfloat16"),+ ("cuda:0", "torch.float16"),+ ("cuda:1", "torch.float16"),+ ]+++def test_rocm_flash_without_flash_attn_is_native(monkeypatch):+ calls = _rocm(monkeypatch, flash_attn = False)+ assert select_attention_backend(_target(), "flash", speed_active = True) is None+ assert calls == []+++def test_rocm_flash_check_failure_is_native(monkeypatch):+ _rocm(monkeypatch, probe = False)+ assert select_attention_backend(_target(), "flash", speed_active = True) is None+++def test_rocm_unaskable_check_is_native_and_not_cached(monkeypatch):+ _rocm(monkeypatch)++ def boom(device, dtype):+ raise RuntimeError("out of memory")++ monkeypatch.setattr(att, "_run_rocm_flash_probe", boom)+ assert select_attention_backend(_target(), "flash", speed_active = True) is None+ assert att._ROCM_FLASH_PROBE_CACHE == {}+++def test_rocm_auto_unchanged(monkeypatch):+ calls = _rocm(monkeypatch)+ assert select_attention_backend(_target(), "auto", speed_active = True) is None+ assert select_attention_backend(_target(), "auto", speed_active = False) is None+ assert calls == []++[email protected]("alias", ["flash3", "flash4", "sage", "cudnn", "xformers"])+def test_rocm_other_nvidia_kernels_still_dropped(monkeypatch, alias):+ _rocm(monkeypatch)+ assert select_attention_backend(_target(), alias, speed_active = True) is None++[email protected]("device", ["mps", "cpu", "xpu"])+def test_flash_off_cuda_still_native(monkeypatch, device):+ _rocm(monkeypatch)+ assert select_attention_backend(_target(device), "flash", speed_active = True) is None+++# origin/main behaviour on an NVIDIA target at SM90 (flash3 in range, flash4 out). Frozen: must never move.+_NVIDIA_EXPECTED = {+ "auto": ("_native_cudnn", None),+ "native": (None, None),+ "sdpa": (None, None),+ "cudnn": ("_native_cudnn", "_native_cudnn"),+ "flash": ("flash", "flash"),+ "flash2": ("flash", "flash"),+ "flash3": ("_flash_3_hub", "_flash_3_hub"),+ "flash4": (None, None),+ "sage": ("sage", "sage"),+ "xformers": ("xformers", "xformers"),+ "aiter": (None, None),+}++[email protected]("alias", sorted(_NVIDIA_EXPECTED))+def test_nvidia_selection_identical_to_main(monkeypatch, alias):+ monkeypatch.setattr(att, "_is_cuda_nvidia", lambda target: True)++ def never(*a, **k):+ raise AssertionError("ROCm flash check must not run on NVIDIA")++ monkeypatch.setattr(att, "_run_rocm_flash_probe", never, raising = False)+ on, off = _NVIDIA_EXPECTED[alias]+ assert select_attention_backend(_target(), alias, speed_active = True) == on+ assert select_attention_backend(_target(), alias, speed_active = False) == off+++def test_alias_table_covered():+ assert set(_NVIDIA_EXPECTED) == set(att.ATTN_ALIASES)+++def _fake_torch(monkeypatch, *, hip):+ torch = types.ModuleType("torch")+ torch.version = types.SimpleNamespace(hip = hip)+ torch.__version__ = "2.9.0+rocm6.4" if hip else "2.9.0+cu128"+ monkeypatch.setitem(sys.modules, "torch", torch)+++def test_installer_never_pip_installs_flash_attn_on_rocm(monkeypatch):+ _fake_torch(monkeypatch, hip = "6.4.0")+ monkeypatch.delitem(sys.modules, "flash_attn", raising = False)+ monkeypatch.setattr("importlib.util.find_spec", lambda name, *a, **k: None)+ monkeypatch.delenv(att._ATTENTION_INSTALL_ENV, raising = False)+ ran = []+ monkeypatch.setattr(subprocess, "run", lambda *a, **k: ran.append(a))+ reason = att._ensure_attention_backend_installed("flash")+ assert ran == []+ assert reason and "CUDA" in reason+ assert att._INSTALL_ATTEMPTED == set()+++def test_installer_still_installs_flash_attn_on_nvidia(monkeypatch):+ _fake_torch(monkeypatch, hip = None)+ monkeypatch.delitem(sys.modules, "flash_attn", raising = False)+ monkeypatch.setattr("importlib.util.find_spec", lambda name, *a, **k: None)+ monkeypatch.delenv(att._ATTENTION_INSTALL_ENV, raising = False)+ ran = []+ monkeypatch.setattr(subprocess, "run", lambda *a, **k: ran.append(a[0]))+ assert att._ensure_attention_backend_installed("flash") is None+ assert len(ran) == 1 and ran[0][-1] == "flash-attn"++[email protected]("dtype_name", ["bfloat16", "float16", "float32"])[email protected]("good", [True, False])+def test_probe_compares_against_reference(monkeypatch, dtype_name, good):+ torch = pytest.importorskip("torch")+ fa = types.ModuleType("flash_attn")++ def flash_attn_func(q, k, v):+ assert q.dtype in (torch.float16, torch.bfloat16)+ out = (+ torch.nn.functional.scaled_dot_product_attention(+ *(t.float().transpose(1, 2) for t in (q, k, v))+ )+ .transpose(1, 2)+ .to(q.dtype)+ )+ return out if good else out + 0.1++ fa.flash_attn_func = flash_attn_func+ monkeypatch.setitem(sys.modules, "flash_attn", fa)+ monkeypatch.setattr(torch.cuda, "synchronize", lambda *a, **k: None)+ expected = good and dtype_name != "float32"+ assert att._run_rocm_flash_probe("cpu", getattr(torch, dtype_name)) is expected+++def test_apply_reverifies_rocm_flash_at_run_dtype(monkeypatch):+ calls = _rocm(monkeypatch, probe = False)+ set_to = []+ dit = types.SimpleNamespace(set_attention_backend = set_to.append)+ pipe = types.SimpleNamespace(transformer = dit)+ monkeypatch.setattr(att, "_active_attention_backend", lambda: att.ATTN_NATIVE)+ monkeypatch.setattr(att, "warn_if_sdpa_math_only", lambda *a, **k: False)+ run_target = _target(dtype = "torch.float32")+ assert att.apply_attention_backend(pipe, "flash", target = run_target) is None+ assert calls == [("cuda:0", "torch.float32")]+ assert set_to == []+++def test_probe_kernel_error_is_false(monkeypatch):+ torch = pytest.importorskip("torch")+ fa = types.ModuleType("flash_attn")++ def flash_attn_func(q, k, v):+ raise RuntimeError("HIP error: invalid device function")++ fa.flash_attn_func = flash_attn_func+ monkeypatch.setitem(sys.modules, "flash_attn", fa)+ assert att._run_rocm_flash_probe("cpu", torch.bfloat16) is False+++def test_cudnn_head_dim_probe_still_answers_fp32_at_bf16(monkeypatch):+ torch = pytest.importorskip("torch")+ import torch.backends.cuda as tbc++ if not hasattr(tbc, "can_use_cudnn_attention"):+ pytest.skip("torch without can_use_cudnn_attention")+ seen = []+ monkeypatch.setattr(+ tbc, "can_use_cudnn_attention", lambda p, debug: seen.append(p.query.dtype) or True+ )+ assert att._run_cudnn_head_dim_probe("cpu", torch.float32, 128) is True+ assert seen == [torch.bfloat16]