Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: install flash-attn's Triton AMD backend on ROCm RDNA3 / RDNA4 (10-15% faster diffusion steps on gfx1151) (#16767)

Daniel Hanf0b5800fba78
diff --git a/studio/backend/core/inference/__init__.py b/studio/backend/core/inference/__init__.py
index c746a53c880..732eee41b7e 100644
--- a/studio/backend/core/inference/__init__.py
+++ b/studio/backend/core/inference/__init__.py
@@ -22,6 +22,13 @@ from typing import TYPE_CHECKING
# Same ROCm AOTriton opt-in as main.py, for entries that skip main.py; read at first SDPA dispatch, setdefault keeps explicit 0; else gfx1151 runs MATH
os.environ.setdefault("TORCH_ROCM_AOTRITON_ENABLE_EXPERIMENTAL", "1")
+# Same flash-attn switch as main.py, for entries that skip it.
+try:
+ from utils.rocm_flash_attn import default_triton_amd_env as _default_triton_amd_env
+ _default_triton_amd_env()
+except Exception: # noqa: BLE001
+ pass
+
__all__ = [
"InferenceBackend",
"InferenceOrchestrator",
diff --git a/studio/backend/core/inference/diffusion_attention.py b/studio/backend/core/inference/diffusion_attention.py
index fd798dd4c24..8d72405295e 100644
--- a/studio/backend/core/inference/diffusion_attention.py
+++ b/studio/backend/core/inference/diffusion_attention.py
@@ -973,6 +973,8 @@ def _sync_rocm_flash_baseline() -> None:
ROCM_FLASH_ADMISSIONS.clear()
_ROCM_FLASH_BUCKETS.clear()
_ROCM_FLASH_STORED_MISSES.clear()
+ _ROCM_FLASH_SEGMENTS_MEMO.clear()
+ _ROCM_FLASH_TRACED_KEYS.clear()
_ROCM_FLASH_BASELINE[0] = current
@@ -1113,11 +1115,30 @@ def _rocm_flash_traced_reason(
dtype = getattr(torch, dtype_name.rpartition(".")[2], None)
if not isinstance(dtype, torch.dtype):
return "compiled or captured"
- return _rocm_flash_bucket_reason(device, dtype, geometries, q_bucket, kv_bucket, b_bucket)
+ before = set(_ROCM_FLASH_BUCKETS)
+ try:
+ return _rocm_flash_bucket_reason(
+ device, dtype, geometries, q_bucket, kv_bucket, b_bucket
+ )
+ finally:
+ _ROCM_FLASH_TRACED_KEYS.update(set(_ROCM_FLASH_BUCKETS) - before)
except Exception: # noqa: BLE001
return "compiled or captured"
+# Dynamo replays a frame after a graph break, so segments must answer as on the first pass (else
+# SpeculationLogDivergence, Wan2.2 on gfx1151). Key excludes verdicts added by traced calls.
+_ROCM_FLASH_TRACED_KEYS: set = set()
+_ROCM_FLASH_SEGMENTS_MEMO: dict = {}
+
+
+def _rocm_flash_segments_key(args: tuple) -> tuple:
+ known = frozenset(
+ (k, v) for k, v in _ROCM_FLASH_BUCKETS.items() if k not in _ROCM_FLASH_TRACED_KEYS
+ )
+ return (*args, id(_active_rocm_flash_gate()), known)
+
+
# Traced query lengths are routed by segments up to this many tokens; longer ones guard on their own bucket.
_ROCM_FLASH_SEGMENT_LIMIT = 1 << 22
# Admission keys with no stored verdict at first stored-only lookup (later verdicts reach the per-bucket route).
@@ -1168,6 +1189,30 @@ def _rocm_flash_traced_segments(
own (``known`` False). ``kv_bucket`` None means keys are as long as queries. Run in plain Python while Dynamo
traces (constant-marked like ``_rocm_flash_traced_reason``); ``()`` when anything is odd (the caller falls back
to the per-bucket route)."""
+ args = (raw_device, dtype_name, heads, head_dim, geometries, kv_bucket, b_bucket)
+ try:
+ key = _rocm_flash_segments_key(args)
+ if key in _ROCM_FLASH_SEGMENTS_MEMO:
+ return _ROCM_FLASH_SEGMENTS_MEMO[key]
+ segments = _rocm_flash_traced_segments_uncached(*args)
+ # Under both keys: reading stored verdicts while building the segments moves the key itself.
+ _ROCM_FLASH_SEGMENTS_MEMO[key] = _ROCM_FLASH_SEGMENTS_MEMO[
+ _rocm_flash_segments_key(args)
+ ] = segments
+ return segments
+ except Exception: # noqa: BLE001
+ return ()
+
+
+def _rocm_flash_traced_segments_uncached(
+ raw_device: str,
+ dtype_name: str,
+ heads: int,
+ head_dim: int,
+ geometries: tuple,
+ kv_bucket: Optional[int],
+ b_bucket: int,
+) -> tuple:
try:
import torch
diff --git a/studio/backend/main.py b/studio/backend/main.py
index 5c20404ebf9..07bfb65a96d 100644
--- a/studio/backend/main.py
+++ b/studio/backend/main.py
@@ -30,6 +30,13 @@ try:
except Exception:
pass
+# Triton-only flash-attn (RDNA3/4) imports only with its backend switch; set before any flash_attn import.
+try:
+ from utils.rocm_flash_attn import default_triton_amd_env as _default_triton_amd_env
+ _default_triton_amd_env()
+except Exception:
+ pass
+
# normalize allocator booleans before torch parses them; spawned workers inherit the environment.
from utils.allocator_conf import normalize_allocator_conf as _normalize_allocator_conf
diff --git a/studio/backend/tests/test_diffusion_attention_rocm_flash.py b/studio/backend/tests/test_diffusion_attention_rocm_flash.py
index 1484bcbb546..a0a0a50de31 100644
--- a/studio/backend/tests/test_diffusion_attention_rocm_flash.py
+++ b/studio/backend/tests/test_diffusion_attention_rocm_flash.py
@@ -40,6 +40,8 @@ def _clean(monkeypatch, tmp_path):
monkeypatch.setattr(att, "_ROCM_FLASH_LOCAL", __import__("threading").local(), raising = False)
monkeypatch.setattr(att, "_ROCM_FLASH_BUCKETS", {}, raising = False)
monkeypatch.setattr(att, "_ROCM_FLASH_STORED_MISSES", set(), raising = False)
+ monkeypatch.setattr(att, "_ROCM_FLASH_SEGMENTS_MEMO", {}, raising = False)
+ monkeypatch.setattr(att, "_ROCM_FLASH_TRACED_KEYS", set(), raising = False)
monkeypatch.setattr(att, "_ROCM_FLASH_BASELINE", [None], raising = False)
monkeypatch.setattr(att, "_rocm_flash_probe_fits", lambda device, need: True, raising = False)
monkeypatch.setattr(att, "_ROCM_FLASH_ROUTED", {})
@@ -2461,3 +2463,59 @@ def test_reapply_to_native_removes_rocm_flash_tracking(monkeypatch):
assert state.resolved["attention_backend"] == entry
assert att.apply_attention_backend(pipe, att.ROCM_FLASH_CANDIDATE, target = _target()) == "flash"
assert att.attention_may_run_native(pipe)
+
+
+class _CompiledDitWithBreak(_CompiledDit):
+ """A traced attention call followed by a graph break in the same frame: Dynamo compiles the prefix and restarts
+ the frame's analysis, replaying it against its speculation log (diffusers' Wan blocks under regional compile)."""
+
+ def forward(self, q):
+ import torch
+
+ out = self._guarded(q, q, q, None)
+ torch._dynamo.graph_break()
+ return out * 1
+
+
+def test_traced_probe_survives_a_dynamo_restart_of_the_frame(monkeypatch):
+ # The traced call times an untimed bucket (8192) during the first analysis; the restart after the graph break
+ # must see the same segments, else SpeculationLogDivergence (Wan2.2 on gfx1151 with flash-attn, auto backend).
+ torch, dit, timed = _compiled_env(
+ monkeypatch, True, {4096: 70.0, 8192: 70.0}, _CompiledDitWithBreak
+ )
+ torch._dynamo.reset()
+ compiled = torch.compile(dit, backend = "eager")
+ q8 = torch.zeros(1, 9000, 2, 16, dtype = torch.bfloat16)
+ assert float(compiled(q8).flatten()[0]) == 1.0
+ assert timed == [4096, 8192]
+ # A later trace after an eager verdict sees it: the key moves with verdicts the traced calls did not add.
+ q16 = torch.zeros(1, 17000, 2, 16, dtype = torch.bfloat16)
+ assert float(compiled(q16).flatten()[0]) == -1.0
+ torch._dynamo.reset()
+
+
+def test_a_new_sdpa_baseline_drops_the_memoized_segments(monkeypatch):
+ # A load that changes the enabled native SDPA kernels clears the verdicts; the traced segments built from the old
+ # verdicts must go with them, or a later trace would bake the old baseline's route into its graph.
+ att._ROCM_FLASH_BUCKETS[("cuda:0", "torch.bfloat16", 32, 2, 16, 4096, 4096, 1)] = True
+ monkeypatch.setattr(att, "_native_sdpa_baseline", lambda: "sdpa-A")
+ att._sync_rocm_flash_baseline()
+ att._ROCM_FLASH_SEGMENTS_MEMO[("stale",)] = ((8192, 4096, True, None),)
+ att._ROCM_FLASH_TRACED_KEYS.add(("cuda:0", "torch.bfloat16", 32, 2, 16, 8192, 8192, 1))
+ monkeypatch.setattr(att, "_native_sdpa_baseline", lambda: "sdpa-B")
+ att._sync_rocm_flash_baseline()
+ assert att._ROCM_FLASH_SEGMENTS_MEMO == {} and att._ROCM_FLASH_TRACED_KEYS == set()
+ assert att._ROCM_FLASH_BUCKETS == {}
+
+
+def test_segments_memo_key_follows_eager_verdicts_not_traced_ones(monkeypatch):
+ args = ("cuda:0", "torch.bfloat16", 2, 16, ((32, 2, 16),), None, 1)
+ k0 = att._rocm_flash_segments_key(args)
+ traced = ("cuda:0", "torch.bfloat16", 32, 2, 16, 8192, 8192, 1)
+ att._ROCM_FLASH_BUCKETS[traced] = True
+ att._ROCM_FLASH_TRACED_KEYS.add(traced)
+ assert (
+ att._rocm_flash_segments_key(args) == k0
+ ) # a traced call's verdict keeps the key (Dynamo restart)
+ att._ROCM_FLASH_BUCKETS[("cuda:0", "torch.bfloat16", 32, 2, 16, 16384, 16384, 1)] = False
+ assert att._rocm_flash_segments_key(args) != k0 # an eager / stored verdict moves it
diff --git a/studio/backend/tests/test_rocm_flash_attn_install.py b/studio/backend/tests/test_rocm_flash_attn_install.py
new file mode 100644
index 00000000000..ad91dd1092c
--- /dev/null
+++ b/studio/backend/tests/test_rocm_flash_attn_install.py
@@ -0,0 +1,348 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""flash-attn's Triton AMD backend: installed by setup on Linux ROCm RDNA3/4 only, best effort, and
+its import switch defaulted only for a Triton-only build on ROCm torch."""
+
+from __future__ import annotations
+
+import importlib.machinery
+import os
+import subprocess
+import sys
+import types
+from pathlib import Path
+
+import pytest
+
+from utils import rocm_flash_attn as rfa
+
+_INSTALL_SCRIPT = Path(__file__).resolve().parents[2] / "install_python_stack.py"
+
+_ROCM_ENV = {
+ "python_tag": "cp312",
+ "torch_mm": "2.11",
+ "torch_version": "2.11.0+rocm7.13.0",
+ "cuda_version": "",
+ "cuda_major": "",
+ "hip_version": "7.13.0",
+ "cxx11abi": "TRUE",
+ "platform_tag": "linux_x86_64",
+}
+_CUDA_ENV = {**_ROCM_ENV, "hip_version": "", "cuda_major": "13", "cuda_version": "13.0"}
+
+
+def ips(monkeypatch):
+ sys.modules.pop("install_python_stack", None)
+ monkeypatch.syspath_prepend(str(_INSTALL_SCRIPT.parent))
+ import install_python_stack
+
+ monkeypatch.delenv("UNSLOTH_STUDIO_SKIP_FLASHATTN_INSTALL", raising = False)
+ monkeypatch.delenv(rfa.SKIP_INSTALL_ENV, raising = False)
+ return install_python_stack
+
+
+class _Host:
+ """install_python_stack with every host fact and side effect replaced."""
+
+ def __init__(
+ self,
+ ips,
+ monkeypatch,
+ *,
+ env,
+ gfx = ("gfx1151",),
+ windows = False,
+ install_ok = True,
+ importable_after = True,
+ installed_version = None,
+ ):
+ self.calls: list[tuple] = []
+ self.notes: list[str] = []
+ self.steps: list[str] = []
+ self.removed = 0
+ self.wheel_urls: list[str] = []
+ self.installed = False
+ self.env_during_install: dict = {}
+ monkeypatch.setattr(ips, "NO_TORCH", False)
+ monkeypatch.setattr(ips, "IS_MACOS", False)
+ monkeypatch.setattr(ips, "IS_WINDOWS", windows)
+ monkeypatch.setattr(ips.sys, "platform", "win32" if windows else "linux")
+ monkeypatch.setattr(ips, "probe_torch_wheel_env", lambda **_: dict(env))
+ monkeypatch.setattr(ips, "_detect_amd_gfx_codes", lambda *a, **k: list(gfx))
+ monkeypatch.setattr(ips, "_flash_attn_gpu_below_floor", lambda: False)
+ monkeypatch.setattr(
+ ips, "_flash_attn_importable", lambda: self.installed and importable_after
+ )
+ monkeypatch.setattr(
+ ips,
+ "_installed_distribution_version",
+ lambda name: "2.8.4" if self.installed else installed_version,
+ )
+ monkeypatch.setattr(ips, "_remove_stale_flash_attn", self._remove)
+ monkeypatch.setattr(ips, "_note", lambda message, *a, **k: self.notes.append(message))
+ monkeypatch.setattr(ips, "_step", lambda label, value, *a, **k: self.steps.append(value))
+ monkeypatch.setattr(ips, "url_exists", lambda url: self.wheel_urls.append(url) or False)
+ self.install_ok = install_ok
+ monkeypatch.setattr(ips, "pip_install_try", self._pip_install_try)
+
+ def _remove(self):
+ self.removed += 1
+
+ def _pip_install_try(self, label, *args, **kwargs):
+ self.calls.append(args)
+ if rfa.FLASH_ATTN_TRITON_AMD_SPEC in args:
+ self.env_during_install = {
+ name: os.environ.get(name) for name in rfa.FLASH_ATTN_TRITON_AMD_BUILD_ENV
+ }
+ self.installed = self.install_ok
+ return self.install_ok
+ return True
+
+ @property
+ def fa_install_args(self):
+ return [args for args in self.calls if rfa.FLASH_ATTN_TRITON_AMD_SPEC in args]
+
+
+# --- installer ---------------------------------------------------------------------------------
+
+
[email protected]("gfx", ["gfx1151", "gfx1100", "gfx1150", "gfx1201"])
+def test_installs_on_linux_rocm_rdna(ips, monkeypatch, gfx):
+ host = _Host(ips, monkeypatch, env = _ROCM_ENV, gfx = (gfx,))
+ ips._ensure_flash_attn()
+ assert len(host.fa_install_args) == 1
+ args = host.fa_install_args[0]
+ assert "--no-build-isolation" in args and "--no-deps" in args
+ # Build deps first, in a separate call.
+ assert set(rfa.FLASH_ATTN_TRITON_AMD_BUILD_DEPS) <= set(host.calls[0])
+ # The Triton switch and FORCE_BUILD reach the build, then leave the installer's environment.
+ assert host.env_during_install == {
+ "FLASH_ATTENTION_TRITON_AMD_ENABLE": "TRUE",
+ "FLASH_ATTENTION_FORCE_BUILD": "TRUE",
+ }
+ assert host.removed == 0
+ assert not host.wheel_urls # no CUDA wheel lookup on ROCm
+
+
+def test_spec_is_a_pinned_commit_archive():
+ assert rfa.FLASH_ATTN_TRITON_AMD_SPEC == (
+ "flash-attn @ https://github.com/Dao-AILab/flash-attention/archive/"
+ "bbe25bacff9f337568b66e69950e549df9839068.tar.gz"
+ )
+
+
+def test_stale_copy_is_reinstalled(ips, monkeypatch):
+ host = _Host(ips, monkeypatch, env = _ROCM_ENV, installed_version = "2.8.4")
+ ips._ensure_flash_attn()
+ assert "--force-reinstall" in host.fa_install_args[0]
+
+
+def test_build_env_restored_after_install(ips, monkeypatch):
+ monkeypatch.setenv("FLASH_ATTENTION_FORCE_BUILD", "FALSE")
+ monkeypatch.delenv("FLASH_ATTENTION_TRITON_AMD_ENABLE", raising = False)
+ _Host(ips, monkeypatch, env = _ROCM_ENV)
+ ips._ensure_flash_attn()
+ assert os.environ["FLASH_ATTENTION_FORCE_BUILD"] == "FALSE"
+ assert "FLASH_ATTENTION_TRITON_AMD_ENABLE" not in os.environ
+
+
+ "case",
+ ["unsupported_gfx", "cdna", "no_gfx", "opt_out", "global_opt_out"],
+)
+def test_skipped(ips, monkeypatch, case):
+ gfx = {"unsupported_gfx": ("gfx1030",), "cdna": ("gfx942",), "no_gfx": ()}.get(
+ case, ("gfx1151",)
+ )
+ if case == "opt_out":
+ monkeypatch.setenv(rfa.SKIP_INSTALL_ENV, "1")
+ if case == "global_opt_out":
+ monkeypatch.setenv("UNSLOTH_STUDIO_SKIP_FLASHATTN_INSTALL", "1")
+ host = _Host(ips, monkeypatch, env = _ROCM_ENV, gfx = gfx)
+ ips._ensure_flash_attn()
+ assert host.calls == []
+ assert not host.wheel_urls
+
+
+def test_skipped_rocm_removes_an_unimportable_copy(ips, monkeypatch):
+ # As before: an installed flash-attn that does not import breaks diffusers / xformers imports.
+ host = _Host(ips, monkeypatch, env = _ROCM_ENV, gfx = ("gfx1030",), installed_version = "2.8.3")
+ ips._ensure_flash_attn()
+ assert host.calls == [] and host.removed == 1
+
+
+def test_nvidia_keeps_the_wheel_route(ips, monkeypatch):
+ host = _Host(ips, monkeypatch, env = _CUDA_ENV, gfx = ())
+ ips._ensure_flash_attn()
+ assert host.fa_install_args == []
+ assert host.wheel_urls # the CUDA prebuilt-wheel lookup ran as before
+
+
+def test_cpu_torch_installs_nothing(ips, monkeypatch):
+ cpu = {**_ROCM_ENV, "hip_version": "", "cuda_major": "", "cuda_version": ""}
+ host = _Host(ips, monkeypatch, env = cpu)
+ ips._ensure_flash_attn()
+ assert host.fa_install_args == []
+
+
+def test_windows_rocm_never_builds(ips, monkeypatch):
+ host = _Host(ips, monkeypatch, env = {**_ROCM_ENV, "platform_tag": "win_amd64"}, windows = True)
+ ips._ensure_flash_attn()
+ assert host.fa_install_args == []
+
+
+def test_skip_reason_is_linux_only():
+ assert rfa.install_skip_reason(platform = "win32", hip_version = "7.1", gfx_codes = ["gfx1151"])
+ assert rfa.install_skip_reason(platform = "darwin", hip_version = "7.1", gfx_codes = ["gfx1151"])
+ assert (
+ rfa.install_skip_reason(platform = "linux", hip_version = "7.1", gfx_codes = ["gfx1151"]) is None
+ )
+ # A mixed host installs when any visible card is RDNA3/4.
+ assert (
+ rfa.install_skip_reason(
+ platform = "linux", hip_version = "7.1", gfx_codes = ["gfx942", "gfx1100"]
+ )
+ is None
+ )
+
+
[email protected]("failure", ["install_fails", "not_importable", "deps_fail"])
+def test_failure_is_not_fatal(ips, monkeypatch, failure):
+ host = _Host(
+ ips,
+ monkeypatch,
+ env = _ROCM_ENV,
+ install_ok = failure != "install_fails",
+ importable_after = failure != "not_importable",
+ )
+ if failure == "deps_fail":
+ monkeypatch.setattr(ips, "pip_install_try", lambda *a, **k: False)
+ ips._ensure_flash_attn() # never raises or exits
+ assert any("Continuing without flash-attn" in step for step in host.steps)
+ if failure == "not_importable":
+ assert host.removed == 1 # an unimportable copy is never left behind
+
+
+def test_import_probe_enables_triton_only_on_rocm():
+ probe = rfa.IMPORT_PROBE
+ assert "flash_attn_2_cuda" in probe and "torch.version, 'hip'" in probe
+ assert "setdefault('FLASH_ATTENTION_TRITON_AMD_ENABLE', 'TRUE')" in probe
+ assert probe.rstrip().endswith("import flash_attn")
+ compile(probe, "<probe>", "exec")
+
+
+def test_installer_import_probe_uses_it(ips, monkeypatch):
+ seen = []
+ monkeypatch.setattr(
+ ips.subprocess,
+ "run",
+ lambda cmd, **kwargs: seen.append(cmd) or subprocess.CompletedProcess(cmd, 0),
+ )
+ assert ips._flash_attn_importable() is True
+ assert seen[0][1:] == ["-c", rfa.IMPORT_PROBE]
+
+
+# --- runtime env default -------------------------------------------------------------------------
+
+
+def _fake_specs(
+ monkeypatch,
+ *,
+ flash_attn = True,
+ extension = False,
+ hip = True,
+):
+ specs = {
+ "flash_attn": flash_attn,
+ "flash_attn_2_cuda": extension,
+ }
+ real = rfa._find_spec
+
+ def find_spec(name):
+ if name in specs:
+ return importlib.machinery.ModuleSpec(name, None) if specs[name] else None
+ return real(name)
+
+ monkeypatch.setattr(rfa, "_find_spec", find_spec)
+ monkeypatch.setattr(rfa, "_torch_is_rocm", lambda: hip)
+ monkeypatch.setattr(rfa.sys, "platform", "linux")
+ monkeypatch.delitem(sys.modules, "flash_attn", raising = False)
+
+
+def test_env_default_set_on_rocm_triton_only_build(monkeypatch):
+ _fake_specs(monkeypatch)
+ env: dict = {}
+ assert rfa.default_triton_amd_env(env) is True
+ assert env == {"FLASH_ATTENTION_TRITON_AMD_ENABLE": "TRUE", rfa.DIFFUSION_ONLY_ENV: "1"}
+
+
+ "kwargs",
+ [
+ {"hip": False}, # NVIDIA / CPU torch: would misroute a CUDA flash_attn
+ {"extension": True}, # CK or CUDA build: has its compiled extension
+ {"flash_attn": False}, # not installed
+ ],
+)
+def test_env_default_not_set_elsewhere(monkeypatch, kwargs):
+ _fake_specs(monkeypatch, **kwargs)
+ env: dict = {}
+ assert rfa.default_triton_amd_env(env) is False
+ assert env == {}
+
+
+def test_env_default_not_set_on_windows(monkeypatch):
+ _fake_specs(monkeypatch)
+ monkeypatch.setattr(rfa.sys, "platform", "win32")
+ env: dict = {}
+ assert rfa.default_triton_amd_env(env) is False and env == {}
+
+
[email protected]("value", ["TRUE", "FALSE"])
+def test_explicit_value_wins_and_opts_training_in(monkeypatch, value):
+ _fake_specs(monkeypatch)
+ env = {"FLASH_ATTENTION_TRITON_AMD_ENABLE": value}
+ assert rfa.default_triton_amd_env(env) is False
+ assert env == {"FLASH_ATTENTION_TRITON_AMD_ENABLE": value} # no diffusion-only marker
+
+
+def test_explicit_diffusion_only_zero_is_kept(monkeypatch):
+ _fake_specs(monkeypatch)
+ env = {rfa.DIFFUSION_ONLY_ENV: "0"}
+ assert rfa.default_triton_amd_env(env) is True
+ assert env[rfa.DIFFUSION_ONLY_ENV] == "0"
+
+
+def test_torch_is_rocm_reads_version_file_without_importing(monkeypatch, tmp_path):
+ for hip_line, expected in (
+ ("hip: Optional[str] = '7.13.0-abc'", True),
+ ("hip: Optional[str] = None", False),
+ ):
+ pkg = tmp_path / ("rocm" if expected else "cuda") / "torch"
+ pkg.mkdir(parents = True)
+ (pkg / "version.py").write_text(f"__version__ = '2.11.0'\n{hip_line}\n")
+ spec = importlib.machinery.ModuleSpec("torch", None, is_package = True)
+ spec.submodule_search_locations = [str(pkg)]
+ monkeypatch.setattr(rfa, "_find_spec", lambda name, spec = spec: spec)
+ monkeypatch.delitem(sys.modules, "torch", raising = False)
+ assert rfa._torch_is_rocm() is expected
+
+
+def test_torch_is_rocm_prefers_imported_torch(monkeypatch):
+ fake = types.SimpleNamespace(version = types.SimpleNamespace(hip = "7.1"))
+ monkeypatch.setitem(sys.modules, "torch", fake)
+ assert rfa._torch_is_rocm() is True
+ fake.version.hip = None
+ assert rfa._torch_is_rocm() is False
+
+
+def test_unsloth_gates_llm_flash_attention_on_the_marker():
+ # The training side reads the marker in its hip branch, before probing flash_attn.
+ source = (Path(__file__).resolve().parents[3] / "unsloth" / "models" / "_utils.py").read_text(
+ encoding = "utf-8"
+ )
+ hip_branch = source.split('elif DEVICE_TYPE == "hip":', 1)[1].split("elif DEVICE_TYPE ==", 1)[0]
+ gate = hip_branch.index(f'os.environ.get("{rfa.DIFFUSION_ONLY_ENV}") == "1"')
+ assert gate < hip_branch.index('_package_available("flash_attn")')
diff --git a/studio/backend/utils/rocm_flash_attn.py b/studio/backend/utils/rocm_flash_attn.py
new file mode 100644
index 00000000000..9f646ec7b06
--- /dev/null
+++ b/studio/backend/utils/rocm_flash_attn.py
@@ -0,0 +1,109 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""flash-attn on ROCm via its Triton AMD backend. Stdlib only: main.py imports it before torch.
+
+A Triton-only build imports only with FLASH_ATTENTION_TRITON_AMD_ENABLE=TRUE.
+"""
+
+from __future__ import annotations
+
+import importlib.util
+import os
+import re
+import sys
+from typing import Iterable, MutableMapping, Optional
+
+TRITON_AMD_ENV = "FLASH_ATTENTION_TRITON_AMD_ENABLE"
+# Set only when Studio supplied the TRITON_AMD_ENV default: unsloth keeps LLM attention on SDPA.
+DIFFUSION_ONLY_ENV = "UNSLOTH_ROCM_FLASH_ATTN_DIFFUSION_ONLY"
+SKIP_INSTALL_ENV = "UNSLOTH_STUDIO_SKIP_ROCM_FLASHATTN_INSTALL"
+
+# flash-attention main (2.8.4 dev), as built by the gfx1151 CI runner.
+FLASH_ATTN_TRITON_AMD_SHA = "bbe25bacff9f337568b66e69950e549df9839068"
+FLASH_ATTN_TRITON_AMD_URL = (
+ f"https://github.com/Dao-AILab/flash-attention/archive/{FLASH_ATTN_TRITON_AMD_SHA}.tar.gz"
+)
+FLASH_ATTN_TRITON_AMD_SPEC = f"flash-attn @ {FLASH_ATTN_TRITON_AMD_URL}"
+# setup.py imports / setup_requires under --no-build-isolation, plus runtime einops.
+FLASH_ATTN_TRITON_AMD_BUILD_DEPS = ("setuptools", "wheel", "packaging", "psutil", "ninja", "einops")
+# FORCE_BUILD skips setup.py's guessed prebuilt-wheel download, which could fetch a CK build.
+FLASH_ATTN_TRITON_AMD_BUILD_ENV = {TRITON_AMD_ENV: "TRUE", "FLASH_ATTENTION_FORCE_BUILD": "TRUE"}
+
+# RDNA3/3.5/4 only; CDNA has CK and aiter routes.
+_SUPPORTED_GFX = re.compile(r"^gfx1[12][0-9a-f]{2}$")
+
+# Import probe mirroring Studio: a Triton-only build on ROCm gets the backend switch first.
+IMPORT_PROBE = (
+ "import importlib.util, os\n"
+ "if importlib.util.find_spec('flash_attn') is not None and "
+ "importlib.util.find_spec('flash_attn_2_cuda') is None:\n"
+ " import torch\n"
+ " if getattr(torch.version, 'hip', None):\n"
+ f" os.environ.setdefault({TRITON_AMD_ENV!r}, 'TRUE')\n"
+ "import flash_attn\n"
+)
+
+
+def supported_gfx(gfx_codes: Iterable[str]) -> list[str]:
+ return [code for code in gfx_codes if _SUPPORTED_GFX.match(str(code).strip().lower())]
+
+
+def install_skip_reason(
+ *, platform: str, hip_version: Optional[str], gfx_codes: Iterable[str]
+) -> Optional[str]:
+ if os.environ.get(SKIP_INSTALL_ENV) == "1":
+ return f"{SKIP_INSTALL_ENV}=1"
+ if not platform.startswith("linux"):
+ return "Linux only"
+ if not hip_version:
+ return "needs a ROCm build of torch"
+ codes = [str(code) for code in gfx_codes]
+ if not codes:
+ return "no AMD GPU architecture detected"
+ if not supported_gfx(codes):
+ return f"no RDNA3/RDNA4 GPU ({', '.join(sorted(set(codes)))})"
+ return None
+
+
+def _find_spec(name: str):
+ try:
+ return importlib.util.find_spec(name)
+ except Exception: # noqa: BLE001
+ return None
+
+
+def _torch_is_rocm() -> bool:
+ """Whether the resident torch is a ROCm build, without importing torch (main.py defers it)."""
+ torch = sys.modules.get("torch")
+ if torch is not None:
+ return bool(getattr(getattr(torch, "version", None), "hip", None))
+ spec = _find_spec("torch")
+ locations = list(getattr(spec, "submodule_search_locations", None) or [])
+ for location in locations:
+ try:
+ with open(os.path.join(location, "version.py"), encoding = "utf-8") as fh:
+ text = fh.read()
+ except OSError:
+ continue
+ match = re.search(r"^hip\s*(?::[^=\n]*)?=\s*(.+)$", text, re.MULTILINE)
+ return bool(match) and match.group(1).strip() not in ("None", "''", '""')
+ return False
+
+
+def triton_only_flash_attn_installed() -> bool:
+ return _find_spec("flash_attn") is not None and _find_spec("flash_attn_2_cuda") is None
+
+
+def default_triton_amd_env(environ: Optional[MutableMapping[str, str]] = None) -> bool:
+ """Default the Triton AMD switch on; an explicit value (even FALSE) wins. True if applied."""
+ env = os.environ if environ is None else environ
+ if not sys.platform.startswith("linux"):
+ return False
+ if TRITON_AMD_ENV in env or "flash_attn" in sys.modules:
+ return False
+ if not triton_only_flash_attn_installed() or not _torch_is_rocm():
+ return False
+ env[TRITON_AMD_ENV] = "TRUE"
+ env.setdefault(DIFFUSION_ONLY_ENV, "1")
+ return True
diff --git a/studio/install_python_stack.py b/studio/install_python_stack.py
index bfdfcb41ff8..e90a8a751e6 100644
--- a/studio/install_python_stack.py
+++ b/studio/install_python_stack.py
@@ -69,6 +69,7 @@ from backend.utils.wheel_utils import (
xformers_torch_requirement_unmet,
)
from backend.utils.uv_path_safety import uv_safe_path as _uv_safe_path
+from backend.utils import rocm_flash_attn as _rocm_flash_attn
IS_WINDOWS = sys.platform == "win32"
IS_MACOS = sys.platform == "darwin"
@@ -8056,7 +8057,7 @@ def _flash_attn_importable() -> bool:
"""
try:
result = subprocess.run(
- [sys.executable, "-c", "import flash_attn"],
+ [sys.executable, "-c", _rocm_flash_attn.IMPORT_PROBE],
stdout = subprocess.DEVNULL,
stderr = subprocess.DEVNULL,
timeout = _FLASH_ATTN_IMPORT_PROBE_TIMEOUT,
@@ -8111,6 +8112,9 @@ def _ensure_flash_attn() -> None:
)
# a failed flash-attn probe proves staleness only when torch still imports.
stale = env is not None and _installed_distribution_version("flash-attn") is not None
+ if env is not None and env.get("hip_version") and not IS_WINDOWS:
+ _ensure_rocm_triton_flash_attn(env, stale)
+ return
wheel_url = _build_flash_attn_wheel_url(env) if env else None
if wheel_url and not (env or {}).get("hip_version") and _flash_attn_gpu_below_floor():
# Unsloth only turns FlashAttention 2 on from sm80; a T4 would download it for nothing.
@@ -8175,6 +8179,66 @@ def _ensure_flash_attn() -> None:
_remove_stale_flash_attn()
+def _rocm_gfx_codes() -> list[str]:
+ try:
+ return _detect_amd_gfx_codes()
+ except Exception:
+ return []
+
+
+def _temporary_environ(values: dict[str, str]):
+ saved = {name: os.environ.get(name) for name in values}
+ os.environ.update(values)
+ try:
+ yield
+ finally:
+ for name, value in saved.items():
+ if value is None:
+ os.environ.pop(name, None)
+ else:
+ os.environ[name] = value
+
+
+def _ensure_rocm_triton_flash_attn(env: dict[str, str], stale: bool) -> None:
+ """Best-effort Triton AMD flash-attn on ROCm RDNA3/4; failure leaves SDPA."""
+ reason = _rocm_flash_attn.install_skip_reason(
+ platform = sys.platform,
+ hip_version = env.get("hip_version"),
+ gfx_codes = _rocm_gfx_codes(),
+ )
+ if reason is not None:
+ _note(f"flash-attn (ROCm Triton backend) skipped: {reason}")
+ if stale:
+ _remove_stale_flash_attn()
+ return
+ with _install_heartbeat("flash-attn"):
+ deps_ok = pip_install_try(
+ "Installing flash-attn build dependencies",
+ "--no-cache-dir",
+ *_rocm_flash_attn.FLASH_ATTN_TRITON_AMD_BUILD_DEPS,
+ )
+ installed = False
+ if deps_ok:
+ with _temporary_environ(_rocm_flash_attn.FLASH_ATTN_TRITON_AMD_BUILD_ENV):
+ installed = pip_install_try(
+ "Installing flash-attn (ROCm Triton backend)",
+ "--no-build-isolation",
+ "--no-deps",
+ "--no-cache-dir",
+ *(["--force-reinstall"] if stale else []),
+ _rocm_flash_attn.FLASH_ATTN_TRITON_AMD_SPEC,
+ constrain = False,
+ )
+ if installed and _flash_attn_importable():
+ _note("flash-attn installed (ROCm Triton backend)")
+ return
+ if _installed_distribution_version("flash-attn") is not None:
+ # An unimportable copy breaks diffusers and xformers imports: never leave one behind.
+ _remove_stale_flash_attn()
+ _step("warning", "Continuing without flash-attn (ROCm Triton backend did not install)", _warn)
+
+
# The compiled extension each SSM kernel loads (kernel_install.KERNELS checks the same import).
_SSM_KERNEL_EXTENSIONS = {"causal_conv1d": "causal_conv1d_cuda", "mamba_ssm": "selective_scan_cuda"}
# Exit code of the probe below when torch imported and the extension raised ImportError / OSError.
diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py
index c2f383b0e5a..55b381e33fc 100644
--- a/unsloth/models/_utils.py
+++ b/unsloth/models/_utils.py
@@ -4763,7 +4763,15 @@ elif DEVICE_TYPE == "cuda":
HAS_FLASH_ATTENTION = False
elif DEVICE_TYPE == "hip":
SUPPORTS_BFLOAT16 = torch.cuda.is_bf16_supported()
- if _package_available("flash_attn"):
+ if os.environ.get("UNSLOTH_ROCM_FLASH_ATTN_DIFFUSION_ONLY") == "1":
+ # Studio's diffusion-only Triton flash-attn: keep LLM attention on SDPA.
+ import transformers.utils.import_utils
+
+ transformers.utils.import_utils.is_flash_attn_2_available = lambda *args, **kwargs: False
+ import transformers.utils
+
+ transformers.utils.is_flash_attn_2_available = lambda *args, **kwargs: False
+ elif _package_available("flash_attn"):
# Check for CUDA linking errors "undefined symbol: _ZNK3c106SymIntltEl" and for softcapping; see #1437.
try:
try: