Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: install flash-attn's Triton AMD backend on ROCm RDNA3 / RDNA4 (10-15% faster diffusion steps on gfx1151) (#16767)
Daniel Hanf0b5800fba78
diff --git a/studio/backend/core/inference/__init__.py b/studio/backend/core/inference/__init__.pyindex c746a53c880..732eee41b7e 100644--- a/studio/backend/core/inference/__init__.py+++ b/studio/backend/core/inference/__init__.py@@ -22,6 +22,13 @@ from typing import TYPE_CHECKING# Same ROCm AOTriton opt-in as main.py, for entries that skip main.py; read at first SDPA dispatch, setdefault keeps explicit 0; else gfx1151 runs MATHos.environ.setdefault("TORCH_ROCM_AOTRITON_ENABLE_EXPERIMENTAL", "1")+# Same flash-attn switch as main.py, for entries that skip it.+try:+ from utils.rocm_flash_attn import default_triton_amd_env as _default_triton_amd_env+ _default_triton_amd_env()+except Exception: # noqa: BLE001+ pass+__all__ = ["InferenceBackend","InferenceOrchestrator",diff --git a/studio/backend/core/inference/diffusion_attention.py b/studio/backend/core/inference/diffusion_attention.pyindex fd798dd4c24..8d72405295e 100644--- a/studio/backend/core/inference/diffusion_attention.py+++ b/studio/backend/core/inference/diffusion_attention.py@@ -973,6 +973,8 @@ def _sync_rocm_flash_baseline() -> None:ROCM_FLASH_ADMISSIONS.clear()_ROCM_FLASH_BUCKETS.clear()_ROCM_FLASH_STORED_MISSES.clear()+ _ROCM_FLASH_SEGMENTS_MEMO.clear()+ _ROCM_FLASH_TRACED_KEYS.clear()_ROCM_FLASH_BASELINE[0] = current@@ -1113,11 +1115,30 @@ def _rocm_flash_traced_reason(dtype = getattr(torch, dtype_name.rpartition(".")[2], None)if not isinstance(dtype, torch.dtype):return "compiled or captured"- return _rocm_flash_bucket_reason(device, dtype, geometries, q_bucket, kv_bucket, b_bucket)+ before = set(_ROCM_FLASH_BUCKETS)+ try:+ return _rocm_flash_bucket_reason(+ device, dtype, geometries, q_bucket, kv_bucket, b_bucket+ )+ finally:+ _ROCM_FLASH_TRACED_KEYS.update(set(_ROCM_FLASH_BUCKETS) - before)except Exception: # noqa: BLE001return "compiled or captured"+# Dynamo replays a frame after a graph break, so segments must answer as on the first pass (else+# SpeculationLogDivergence, Wan2.2 on gfx1151). Key excludes verdicts added by traced calls.+_ROCM_FLASH_TRACED_KEYS: set = set()+_ROCM_FLASH_SEGMENTS_MEMO: dict = {}+++def _rocm_flash_segments_key(args: tuple) -> tuple:+ known = frozenset(+ (k, v) for k, v in _ROCM_FLASH_BUCKETS.items() if k not in _ROCM_FLASH_TRACED_KEYS+ )+ return (*args, id(_active_rocm_flash_gate()), known)++# Traced query lengths are routed by segments up to this many tokens; longer ones guard on their own bucket._ROCM_FLASH_SEGMENT_LIMIT = 1 << 22# Admission keys with no stored verdict at first stored-only lookup (later verdicts reach the per-bucket route).@@ -1168,6 +1189,30 @@ def _rocm_flash_traced_segments(own (``known`` False). ``kv_bucket`` None means keys are as long as queries. Run in plain Python while Dynamotraces (constant-marked like ``_rocm_flash_traced_reason``); ``()`` when anything is odd (the caller falls backto the per-bucket route)."""+ args = (raw_device, dtype_name, heads, head_dim, geometries, kv_bucket, b_bucket)+ try:+ key = _rocm_flash_segments_key(args)+ if key in _ROCM_FLASH_SEGMENTS_MEMO:+ return _ROCM_FLASH_SEGMENTS_MEMO[key]+ segments = _rocm_flash_traced_segments_uncached(*args)+ # Under both keys: reading stored verdicts while building the segments moves the key itself.+ _ROCM_FLASH_SEGMENTS_MEMO[key] = _ROCM_FLASH_SEGMENTS_MEMO[+ _rocm_flash_segments_key(args)+ ] = segments+ return segments+ except Exception: # noqa: BLE001+ return ()+++def _rocm_flash_traced_segments_uncached(+ raw_device: str,+ dtype_name: str,+ heads: int,+ head_dim: int,+ geometries: tuple,+ kv_bucket: Optional[int],+ b_bucket: int,+) -> tuple:try:import torchdiff --git a/studio/backend/main.py b/studio/backend/main.pyindex 5c20404ebf9..07bfb65a96d 100644--- a/studio/backend/main.py+++ b/studio/backend/main.py@@ -30,6 +30,13 @@ try:except Exception:pass+# Triton-only flash-attn (RDNA3/4) imports only with its backend switch; set before any flash_attn import.+try:+ from utils.rocm_flash_attn import default_triton_amd_env as _default_triton_amd_env+ _default_triton_amd_env()+except Exception:+ pass+# normalize allocator booleans before torch parses them; spawned workers inherit the environment.from utils.allocator_conf import normalize_allocator_conf as _normalize_allocator_confdiff --git a/studio/backend/tests/test_diffusion_attention_rocm_flash.py b/studio/backend/tests/test_diffusion_attention_rocm_flash.pyindex 1484bcbb546..a0a0a50de31 100644--- a/studio/backend/tests/test_diffusion_attention_rocm_flash.py+++ b/studio/backend/tests/test_diffusion_attention_rocm_flash.py@@ -40,6 +40,8 @@ def _clean(monkeypatch, tmp_path):monkeypatch.setattr(att, "_ROCM_FLASH_LOCAL", __import__("threading").local(), raising = False)monkeypatch.setattr(att, "_ROCM_FLASH_BUCKETS", {}, raising = False)monkeypatch.setattr(att, "_ROCM_FLASH_STORED_MISSES", set(), raising = False)+ monkeypatch.setattr(att, "_ROCM_FLASH_SEGMENTS_MEMO", {}, raising = False)+ monkeypatch.setattr(att, "_ROCM_FLASH_TRACED_KEYS", set(), raising = False)monkeypatch.setattr(att, "_ROCM_FLASH_BASELINE", [None], raising = False)monkeypatch.setattr(att, "_rocm_flash_probe_fits", lambda device, need: True, raising = False)monkeypatch.setattr(att, "_ROCM_FLASH_ROUTED", {})@@ -2461,3 +2463,59 @@ def test_reapply_to_native_removes_rocm_flash_tracking(monkeypatch):assert state.resolved["attention_backend"] == entryassert att.apply_attention_backend(pipe, att.ROCM_FLASH_CANDIDATE, target = _target()) == "flash"assert att.attention_may_run_native(pipe)+++class _CompiledDitWithBreak(_CompiledDit):+ """A traced attention call followed by a graph break in the same frame: Dynamo compiles the prefix and restarts+ the frame's analysis, replaying it against its speculation log (diffusers' Wan blocks under regional compile)."""++ def forward(self, q):+ import torch++ out = self._guarded(q, q, q, None)+ torch._dynamo.graph_break()+ return out * 1+++def test_traced_probe_survives_a_dynamo_restart_of_the_frame(monkeypatch):+ # The traced call times an untimed bucket (8192) during the first analysis; the restart after the graph break+ # must see the same segments, else SpeculationLogDivergence (Wan2.2 on gfx1151 with flash-attn, auto backend).+ torch, dit, timed = _compiled_env(+ monkeypatch, True, {4096: 70.0, 8192: 70.0}, _CompiledDitWithBreak+ )+ torch._dynamo.reset()+ compiled = torch.compile(dit, backend = "eager")+ q8 = torch.zeros(1, 9000, 2, 16, dtype = torch.bfloat16)+ assert float(compiled(q8).flatten()[0]) == 1.0+ assert timed == [4096, 8192]+ # A later trace after an eager verdict sees it: the key moves with verdicts the traced calls did not add.+ q16 = torch.zeros(1, 17000, 2, 16, dtype = torch.bfloat16)+ assert float(compiled(q16).flatten()[0]) == -1.0+ torch._dynamo.reset()+++def test_a_new_sdpa_baseline_drops_the_memoized_segments(monkeypatch):+ # A load that changes the enabled native SDPA kernels clears the verdicts; the traced segments built from the old+ # verdicts must go with them, or a later trace would bake the old baseline's route into its graph.+ att._ROCM_FLASH_BUCKETS[("cuda:0", "torch.bfloat16", 32, 2, 16, 4096, 4096, 1)] = True+ monkeypatch.setattr(att, "_native_sdpa_baseline", lambda: "sdpa-A")+ att._sync_rocm_flash_baseline()+ att._ROCM_FLASH_SEGMENTS_MEMO[("stale",)] = ((8192, 4096, True, None),)+ att._ROCM_FLASH_TRACED_KEYS.add(("cuda:0", "torch.bfloat16", 32, 2, 16, 8192, 8192, 1))+ monkeypatch.setattr(att, "_native_sdpa_baseline", lambda: "sdpa-B")+ att._sync_rocm_flash_baseline()+ assert att._ROCM_FLASH_SEGMENTS_MEMO == {} and att._ROCM_FLASH_TRACED_KEYS == set()+ assert att._ROCM_FLASH_BUCKETS == {}+++def test_segments_memo_key_follows_eager_verdicts_not_traced_ones(monkeypatch):+ args = ("cuda:0", "torch.bfloat16", 2, 16, ((32, 2, 16),), None, 1)+ k0 = att._rocm_flash_segments_key(args)+ traced = ("cuda:0", "torch.bfloat16", 32, 2, 16, 8192, 8192, 1)+ att._ROCM_FLASH_BUCKETS[traced] = True+ att._ROCM_FLASH_TRACED_KEYS.add(traced)+ assert (+ att._rocm_flash_segments_key(args) == k0+ ) # a traced call's verdict keeps the key (Dynamo restart)+ att._ROCM_FLASH_BUCKETS[("cuda:0", "torch.bfloat16", 32, 2, 16, 16384, 16384, 1)] = False+ assert att._rocm_flash_segments_key(args) != k0 # an eager / stored verdict moves itdiff --git a/studio/backend/tests/test_rocm_flash_attn_install.py b/studio/backend/tests/test_rocm_flash_attn_install.pynew file mode 100644index 00000000000..ad91dd1092c--- /dev/null+++ b/studio/backend/tests/test_rocm_flash_attn_install.py@@ -0,0 +1,348 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""flash-attn's Triton AMD backend: installed by setup on Linux ROCm RDNA3/4 only, best effort, and+its import switch defaulted only for a Triton-only build on ROCm torch."""++from __future__ import annotations++import importlib.machinery+import os+import subprocess+import sys+import types+from pathlib import Path++import pytest++from utils import rocm_flash_attn as rfa++_INSTALL_SCRIPT = Path(__file__).resolve().parents[2] / "install_python_stack.py"++_ROCM_ENV = {+ "python_tag": "cp312",+ "torch_mm": "2.11",+ "torch_version": "2.11.0+rocm7.13.0",+ "cuda_version": "",+ "cuda_major": "",+ "hip_version": "7.13.0",+ "cxx11abi": "TRUE",+ "platform_tag": "linux_x86_64",+}+_CUDA_ENV = {**_ROCM_ENV, "hip_version": "", "cuda_major": "13", "cuda_version": "13.0"}+++def ips(monkeypatch):+ sys.modules.pop("install_python_stack", None)+ monkeypatch.syspath_prepend(str(_INSTALL_SCRIPT.parent))+ import install_python_stack++ monkeypatch.delenv("UNSLOTH_STUDIO_SKIP_FLASHATTN_INSTALL", raising = False)+ monkeypatch.delenv(rfa.SKIP_INSTALL_ENV, raising = False)+ return install_python_stack+++class _Host:+ """install_python_stack with every host fact and side effect replaced."""++ def __init__(+ self,+ ips,+ monkeypatch,+ *,+ env,+ gfx = ("gfx1151",),+ windows = False,+ install_ok = True,+ importable_after = True,+ installed_version = None,+ ):+ self.calls: list[tuple] = []+ self.notes: list[str] = []+ self.steps: list[str] = []+ self.removed = 0+ self.wheel_urls: list[str] = []+ self.installed = False+ self.env_during_install: dict = {}+ monkeypatch.setattr(ips, "NO_TORCH", False)+ monkeypatch.setattr(ips, "IS_MACOS", False)+ monkeypatch.setattr(ips, "IS_WINDOWS", windows)+ monkeypatch.setattr(ips.sys, "platform", "win32" if windows else "linux")+ monkeypatch.setattr(ips, "probe_torch_wheel_env", lambda **_: dict(env))+ monkeypatch.setattr(ips, "_detect_amd_gfx_codes", lambda *a, **k: list(gfx))+ monkeypatch.setattr(ips, "_flash_attn_gpu_below_floor", lambda: False)+ monkeypatch.setattr(+ ips, "_flash_attn_importable", lambda: self.installed and importable_after+ )+ monkeypatch.setattr(+ ips,+ "_installed_distribution_version",+ lambda name: "2.8.4" if self.installed else installed_version,+ )+ monkeypatch.setattr(ips, "_remove_stale_flash_attn", self._remove)+ monkeypatch.setattr(ips, "_note", lambda message, *a, **k: self.notes.append(message))+ monkeypatch.setattr(ips, "_step", lambda label, value, *a, **k: self.steps.append(value))+ monkeypatch.setattr(ips, "url_exists", lambda url: self.wheel_urls.append(url) or False)+ self.install_ok = install_ok+ monkeypatch.setattr(ips, "pip_install_try", self._pip_install_try)++ def _remove(self):+ self.removed += 1++ def _pip_install_try(self, label, *args, **kwargs):+ self.calls.append(args)+ if rfa.FLASH_ATTN_TRITON_AMD_SPEC in args:+ self.env_during_install = {+ name: os.environ.get(name) for name in rfa.FLASH_ATTN_TRITON_AMD_BUILD_ENV+ }+ self.installed = self.install_ok+ return self.install_ok+ return True++ @property+ def fa_install_args(self):+ return [args for args in self.calls if rfa.FLASH_ATTN_TRITON_AMD_SPEC in args]+++# --- installer ---------------------------------------------------------------------------------++[email protected]("gfx", ["gfx1151", "gfx1100", "gfx1150", "gfx1201"])+def test_installs_on_linux_rocm_rdna(ips, monkeypatch, gfx):+ host = _Host(ips, monkeypatch, env = _ROCM_ENV, gfx = (gfx,))+ ips._ensure_flash_attn()+ assert len(host.fa_install_args) == 1+ args = host.fa_install_args[0]+ assert "--no-build-isolation" in args and "--no-deps" in args+ # Build deps first, in a separate call.+ assert set(rfa.FLASH_ATTN_TRITON_AMD_BUILD_DEPS) <= set(host.calls[0])+ # The Triton switch and FORCE_BUILD reach the build, then leave the installer's environment.+ assert host.env_during_install == {+ "FLASH_ATTENTION_TRITON_AMD_ENABLE": "TRUE",+ "FLASH_ATTENTION_FORCE_BUILD": "TRUE",+ }+ assert host.removed == 0+ assert not host.wheel_urls # no CUDA wheel lookup on ROCm+++def test_spec_is_a_pinned_commit_archive():+ assert rfa.FLASH_ATTN_TRITON_AMD_SPEC == (+ "flash-attn @ https://github.com/Dao-AILab/flash-attention/archive/"+ "bbe25bacff9f337568b66e69950e549df9839068.tar.gz"+ )+++def test_stale_copy_is_reinstalled(ips, monkeypatch):+ host = _Host(ips, monkeypatch, env = _ROCM_ENV, installed_version = "2.8.4")+ ips._ensure_flash_attn()+ assert "--force-reinstall" in host.fa_install_args[0]+++def test_build_env_restored_after_install(ips, monkeypatch):+ monkeypatch.setenv("FLASH_ATTENTION_FORCE_BUILD", "FALSE")+ monkeypatch.delenv("FLASH_ATTENTION_TRITON_AMD_ENABLE", raising = False)+ _Host(ips, monkeypatch, env = _ROCM_ENV)+ ips._ensure_flash_attn()+ assert os.environ["FLASH_ATTENTION_FORCE_BUILD"] == "FALSE"+ assert "FLASH_ATTENTION_TRITON_AMD_ENABLE" not in os.environ+++ "case",+ ["unsupported_gfx", "cdna", "no_gfx", "opt_out", "global_opt_out"],+)+def test_skipped(ips, monkeypatch, case):+ gfx = {"unsupported_gfx": ("gfx1030",), "cdna": ("gfx942",), "no_gfx": ()}.get(+ case, ("gfx1151",)+ )+ if case == "opt_out":+ monkeypatch.setenv(rfa.SKIP_INSTALL_ENV, "1")+ if case == "global_opt_out":+ monkeypatch.setenv("UNSLOTH_STUDIO_SKIP_FLASHATTN_INSTALL", "1")+ host = _Host(ips, monkeypatch, env = _ROCM_ENV, gfx = gfx)+ ips._ensure_flash_attn()+ assert host.calls == []+ assert not host.wheel_urls+++def test_skipped_rocm_removes_an_unimportable_copy(ips, monkeypatch):+ # As before: an installed flash-attn that does not import breaks diffusers / xformers imports.+ host = _Host(ips, monkeypatch, env = _ROCM_ENV, gfx = ("gfx1030",), installed_version = "2.8.3")+ ips._ensure_flash_attn()+ assert host.calls == [] and host.removed == 1+++def test_nvidia_keeps_the_wheel_route(ips, monkeypatch):+ host = _Host(ips, monkeypatch, env = _CUDA_ENV, gfx = ())+ ips._ensure_flash_attn()+ assert host.fa_install_args == []+ assert host.wheel_urls # the CUDA prebuilt-wheel lookup ran as before+++def test_cpu_torch_installs_nothing(ips, monkeypatch):+ cpu = {**_ROCM_ENV, "hip_version": "", "cuda_major": "", "cuda_version": ""}+ host = _Host(ips, monkeypatch, env = cpu)+ ips._ensure_flash_attn()+ assert host.fa_install_args == []+++def test_windows_rocm_never_builds(ips, monkeypatch):+ host = _Host(ips, monkeypatch, env = {**_ROCM_ENV, "platform_tag": "win_amd64"}, windows = True)+ ips._ensure_flash_attn()+ assert host.fa_install_args == []+++def test_skip_reason_is_linux_only():+ assert rfa.install_skip_reason(platform = "win32", hip_version = "7.1", gfx_codes = ["gfx1151"])+ assert rfa.install_skip_reason(platform = "darwin", hip_version = "7.1", gfx_codes = ["gfx1151"])+ assert (+ rfa.install_skip_reason(platform = "linux", hip_version = "7.1", gfx_codes = ["gfx1151"]) is None+ )+ # A mixed host installs when any visible card is RDNA3/4.+ assert (+ rfa.install_skip_reason(+ platform = "linux", hip_version = "7.1", gfx_codes = ["gfx942", "gfx1100"]+ )+ is None+ )++[email protected]("failure", ["install_fails", "not_importable", "deps_fail"])+def test_failure_is_not_fatal(ips, monkeypatch, failure):+ host = _Host(+ ips,+ monkeypatch,+ env = _ROCM_ENV,+ install_ok = failure != "install_fails",+ importable_after = failure != "not_importable",+ )+ if failure == "deps_fail":+ monkeypatch.setattr(ips, "pip_install_try", lambda *a, **k: False)+ ips._ensure_flash_attn() # never raises or exits+ assert any("Continuing without flash-attn" in step for step in host.steps)+ if failure == "not_importable":+ assert host.removed == 1 # an unimportable copy is never left behind+++def test_import_probe_enables_triton_only_on_rocm():+ probe = rfa.IMPORT_PROBE+ assert "flash_attn_2_cuda" in probe and "torch.version, 'hip'" in probe+ assert "setdefault('FLASH_ATTENTION_TRITON_AMD_ENABLE', 'TRUE')" in probe+ assert probe.rstrip().endswith("import flash_attn")+ compile(probe, "<probe>", "exec")+++def test_installer_import_probe_uses_it(ips, monkeypatch):+ seen = []+ monkeypatch.setattr(+ ips.subprocess,+ "run",+ lambda cmd, **kwargs: seen.append(cmd) or subprocess.CompletedProcess(cmd, 0),+ )+ assert ips._flash_attn_importable() is True+ assert seen[0][1:] == ["-c", rfa.IMPORT_PROBE]+++# --- runtime env default -------------------------------------------------------------------------+++def _fake_specs(+ monkeypatch,+ *,+ flash_attn = True,+ extension = False,+ hip = True,+):+ specs = {+ "flash_attn": flash_attn,+ "flash_attn_2_cuda": extension,+ }+ real = rfa._find_spec++ def find_spec(name):+ if name in specs:+ return importlib.machinery.ModuleSpec(name, None) if specs[name] else None+ return real(name)++ monkeypatch.setattr(rfa, "_find_spec", find_spec)+ monkeypatch.setattr(rfa, "_torch_is_rocm", lambda: hip)+ monkeypatch.setattr(rfa.sys, "platform", "linux")+ monkeypatch.delitem(sys.modules, "flash_attn", raising = False)+++def test_env_default_set_on_rocm_triton_only_build(monkeypatch):+ _fake_specs(monkeypatch)+ env: dict = {}+ assert rfa.default_triton_amd_env(env) is True+ assert env == {"FLASH_ATTENTION_TRITON_AMD_ENABLE": "TRUE", rfa.DIFFUSION_ONLY_ENV: "1"}+++ "kwargs",+ [+ {"hip": False}, # NVIDIA / CPU torch: would misroute a CUDA flash_attn+ {"extension": True}, # CK or CUDA build: has its compiled extension+ {"flash_attn": False}, # not installed+ ],+)+def test_env_default_not_set_elsewhere(monkeypatch, kwargs):+ _fake_specs(monkeypatch, **kwargs)+ env: dict = {}+ assert rfa.default_triton_amd_env(env) is False+ assert env == {}+++def test_env_default_not_set_on_windows(monkeypatch):+ _fake_specs(monkeypatch)+ monkeypatch.setattr(rfa.sys, "platform", "win32")+ env: dict = {}+ assert rfa.default_triton_amd_env(env) is False and env == {}++[email protected]("value", ["TRUE", "FALSE"])+def test_explicit_value_wins_and_opts_training_in(monkeypatch, value):+ _fake_specs(monkeypatch)+ env = {"FLASH_ATTENTION_TRITON_AMD_ENABLE": value}+ assert rfa.default_triton_amd_env(env) is False+ assert env == {"FLASH_ATTENTION_TRITON_AMD_ENABLE": value} # no diffusion-only marker+++def test_explicit_diffusion_only_zero_is_kept(monkeypatch):+ _fake_specs(monkeypatch)+ env = {rfa.DIFFUSION_ONLY_ENV: "0"}+ assert rfa.default_triton_amd_env(env) is True+ assert env[rfa.DIFFUSION_ONLY_ENV] == "0"+++def test_torch_is_rocm_reads_version_file_without_importing(monkeypatch, tmp_path):+ for hip_line, expected in (+ ("hip: Optional[str] = '7.13.0-abc'", True),+ ("hip: Optional[str] = None", False),+ ):+ pkg = tmp_path / ("rocm" if expected else "cuda") / "torch"+ pkg.mkdir(parents = True)+ (pkg / "version.py").write_text(f"__version__ = '2.11.0'\n{hip_line}\n")+ spec = importlib.machinery.ModuleSpec("torch", None, is_package = True)+ spec.submodule_search_locations = [str(pkg)]+ monkeypatch.setattr(rfa, "_find_spec", lambda name, spec = spec: spec)+ monkeypatch.delitem(sys.modules, "torch", raising = False)+ assert rfa._torch_is_rocm() is expected+++def test_torch_is_rocm_prefers_imported_torch(monkeypatch):+ fake = types.SimpleNamespace(version = types.SimpleNamespace(hip = "7.1"))+ monkeypatch.setitem(sys.modules, "torch", fake)+ assert rfa._torch_is_rocm() is True+ fake.version.hip = None+ assert rfa._torch_is_rocm() is False+++def test_unsloth_gates_llm_flash_attention_on_the_marker():+ # The training side reads the marker in its hip branch, before probing flash_attn.+ source = (Path(__file__).resolve().parents[3] / "unsloth" / "models" / "_utils.py").read_text(+ encoding = "utf-8"+ )+ hip_branch = source.split('elif DEVICE_TYPE == "hip":', 1)[1].split("elif DEVICE_TYPE ==", 1)[0]+ gate = hip_branch.index(f'os.environ.get("{rfa.DIFFUSION_ONLY_ENV}") == "1"')+ assert gate < hip_branch.index('_package_available("flash_attn")')diff --git a/studio/backend/utils/rocm_flash_attn.py b/studio/backend/utils/rocm_flash_attn.pynew file mode 100644index 00000000000..9f646ec7b06--- /dev/null+++ b/studio/backend/utils/rocm_flash_attn.py@@ -0,0 +1,109 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""flash-attn on ROCm via its Triton AMD backend. Stdlib only: main.py imports it before torch.++A Triton-only build imports only with FLASH_ATTENTION_TRITON_AMD_ENABLE=TRUE.+"""++from __future__ import annotations++import importlib.util+import os+import re+import sys+from typing import Iterable, MutableMapping, Optional++TRITON_AMD_ENV = "FLASH_ATTENTION_TRITON_AMD_ENABLE"+# Set only when Studio supplied the TRITON_AMD_ENV default: unsloth keeps LLM attention on SDPA.+DIFFUSION_ONLY_ENV = "UNSLOTH_ROCM_FLASH_ATTN_DIFFUSION_ONLY"+SKIP_INSTALL_ENV = "UNSLOTH_STUDIO_SKIP_ROCM_FLASHATTN_INSTALL"++# flash-attention main (2.8.4 dev), as built by the gfx1151 CI runner.+FLASH_ATTN_TRITON_AMD_SHA = "bbe25bacff9f337568b66e69950e549df9839068"+FLASH_ATTN_TRITON_AMD_URL = (+ f"https://github.com/Dao-AILab/flash-attention/archive/{FLASH_ATTN_TRITON_AMD_SHA}.tar.gz"+)+FLASH_ATTN_TRITON_AMD_SPEC = f"flash-attn @ {FLASH_ATTN_TRITON_AMD_URL}"+# setup.py imports / setup_requires under --no-build-isolation, plus runtime einops.+FLASH_ATTN_TRITON_AMD_BUILD_DEPS = ("setuptools", "wheel", "packaging", "psutil", "ninja", "einops")+# FORCE_BUILD skips setup.py's guessed prebuilt-wheel download, which could fetch a CK build.+FLASH_ATTN_TRITON_AMD_BUILD_ENV = {TRITON_AMD_ENV: "TRUE", "FLASH_ATTENTION_FORCE_BUILD": "TRUE"}++# RDNA3/3.5/4 only; CDNA has CK and aiter routes.+_SUPPORTED_GFX = re.compile(r"^gfx1[12][0-9a-f]{2}$")++# Import probe mirroring Studio: a Triton-only build on ROCm gets the backend switch first.+IMPORT_PROBE = (+ "import importlib.util, os\n"+ "if importlib.util.find_spec('flash_attn') is not None and "+ "importlib.util.find_spec('flash_attn_2_cuda') is None:\n"+ " import torch\n"+ " if getattr(torch.version, 'hip', None):\n"+ f" os.environ.setdefault({TRITON_AMD_ENV!r}, 'TRUE')\n"+ "import flash_attn\n"+)+++def supported_gfx(gfx_codes: Iterable[str]) -> list[str]:+ return [code for code in gfx_codes if _SUPPORTED_GFX.match(str(code).strip().lower())]+++def install_skip_reason(+ *, platform: str, hip_version: Optional[str], gfx_codes: Iterable[str]+) -> Optional[str]:+ if os.environ.get(SKIP_INSTALL_ENV) == "1":+ return f"{SKIP_INSTALL_ENV}=1"+ if not platform.startswith("linux"):+ return "Linux only"+ if not hip_version:+ return "needs a ROCm build of torch"+ codes = [str(code) for code in gfx_codes]+ if not codes:+ return "no AMD GPU architecture detected"+ if not supported_gfx(codes):+ return f"no RDNA3/RDNA4 GPU ({', '.join(sorted(set(codes)))})"+ return None+++def _find_spec(name: str):+ try:+ return importlib.util.find_spec(name)+ except Exception: # noqa: BLE001+ return None+++def _torch_is_rocm() -> bool:+ """Whether the resident torch is a ROCm build, without importing torch (main.py defers it)."""+ torch = sys.modules.get("torch")+ if torch is not None:+ return bool(getattr(getattr(torch, "version", None), "hip", None))+ spec = _find_spec("torch")+ locations = list(getattr(spec, "submodule_search_locations", None) or [])+ for location in locations:+ try:+ with open(os.path.join(location, "version.py"), encoding = "utf-8") as fh:+ text = fh.read()+ except OSError:+ continue+ match = re.search(r"^hip\s*(?::[^=\n]*)?=\s*(.+)$", text, re.MULTILINE)+ return bool(match) and match.group(1).strip() not in ("None", "''", '""')+ return False+++def triton_only_flash_attn_installed() -> bool:+ return _find_spec("flash_attn") is not None and _find_spec("flash_attn_2_cuda") is None+++def default_triton_amd_env(environ: Optional[MutableMapping[str, str]] = None) -> bool:+ """Default the Triton AMD switch on; an explicit value (even FALSE) wins. True if applied."""+ env = os.environ if environ is None else environ+ if not sys.platform.startswith("linux"):+ return False+ if TRITON_AMD_ENV in env or "flash_attn" in sys.modules:+ return False+ if not triton_only_flash_attn_installed() or not _torch_is_rocm():+ return False+ env[TRITON_AMD_ENV] = "TRUE"+ env.setdefault(DIFFUSION_ONLY_ENV, "1")+ return Truediff --git a/studio/install_python_stack.py b/studio/install_python_stack.pyindex bfdfcb41ff8..e90a8a751e6 100644--- a/studio/install_python_stack.py+++ b/studio/install_python_stack.py@@ -69,6 +69,7 @@ from backend.utils.wheel_utils import (xformers_torch_requirement_unmet,)from backend.utils.uv_path_safety import uv_safe_path as _uv_safe_path+from backend.utils import rocm_flash_attn as _rocm_flash_attnIS_WINDOWS = sys.platform == "win32"IS_MACOS = sys.platform == "darwin"@@ -8056,7 +8057,7 @@ def _flash_attn_importable() -> bool:"""try:result = subprocess.run(- [sys.executable, "-c", "import flash_attn"],+ [sys.executable, "-c", _rocm_flash_attn.IMPORT_PROBE],stdout = subprocess.DEVNULL,stderr = subprocess.DEVNULL,timeout = _FLASH_ATTN_IMPORT_PROBE_TIMEOUT,@@ -8111,6 +8112,9 @@ def _ensure_flash_attn() -> None:)# a failed flash-attn probe proves staleness only when torch still imports.stale = env is not None and _installed_distribution_version("flash-attn") is not None+ if env is not None and env.get("hip_version") and not IS_WINDOWS:+ _ensure_rocm_triton_flash_attn(env, stale)+ returnwheel_url = _build_flash_attn_wheel_url(env) if env else Noneif wheel_url and not (env or {}).get("hip_version") and _flash_attn_gpu_below_floor():# Unsloth only turns FlashAttention 2 on from sm80; a T4 would download it for nothing.@@ -8175,6 +8179,66 @@ def _ensure_flash_attn() -> None:_remove_stale_flash_attn()+def _rocm_gfx_codes() -> list[str]:+ try:+ return _detect_amd_gfx_codes()+ except Exception:+ return []+++def _temporary_environ(values: dict[str, str]):+ saved = {name: os.environ.get(name) for name in values}+ os.environ.update(values)+ try:+ yield+ finally:+ for name, value in saved.items():+ if value is None:+ os.environ.pop(name, None)+ else:+ os.environ[name] = value+++def _ensure_rocm_triton_flash_attn(env: dict[str, str], stale: bool) -> None:+ """Best-effort Triton AMD flash-attn on ROCm RDNA3/4; failure leaves SDPA."""+ reason = _rocm_flash_attn.install_skip_reason(+ platform = sys.platform,+ hip_version = env.get("hip_version"),+ gfx_codes = _rocm_gfx_codes(),+ )+ if reason is not None:+ _note(f"flash-attn (ROCm Triton backend) skipped: {reason}")+ if stale:+ _remove_stale_flash_attn()+ return+ with _install_heartbeat("flash-attn"):+ deps_ok = pip_install_try(+ "Installing flash-attn build dependencies",+ "--no-cache-dir",+ *_rocm_flash_attn.FLASH_ATTN_TRITON_AMD_BUILD_DEPS,+ )+ installed = False+ if deps_ok:+ with _temporary_environ(_rocm_flash_attn.FLASH_ATTN_TRITON_AMD_BUILD_ENV):+ installed = pip_install_try(+ "Installing flash-attn (ROCm Triton backend)",+ "--no-build-isolation",+ "--no-deps",+ "--no-cache-dir",+ *(["--force-reinstall"] if stale else []),+ _rocm_flash_attn.FLASH_ATTN_TRITON_AMD_SPEC,+ constrain = False,+ )+ if installed and _flash_attn_importable():+ _note("flash-attn installed (ROCm Triton backend)")+ return+ if _installed_distribution_version("flash-attn") is not None:+ # An unimportable copy breaks diffusers and xformers imports: never leave one behind.+ _remove_stale_flash_attn()+ _step("warning", "Continuing without flash-attn (ROCm Triton backend did not install)", _warn)++# The compiled extension each SSM kernel loads (kernel_install.KERNELS checks the same import)._SSM_KERNEL_EXTENSIONS = {"causal_conv1d": "causal_conv1d_cuda", "mamba_ssm": "selective_scan_cuda"}# Exit code of the probe below when torch imported and the extension raised ImportError / OSError.diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.pyindex c2f383b0e5a..55b381e33fc 100644--- a/unsloth/models/_utils.py+++ b/unsloth/models/_utils.py@@ -4763,7 +4763,15 @@ elif DEVICE_TYPE == "cuda":HAS_FLASH_ATTENTION = Falseelif DEVICE_TYPE == "hip":SUPPORTS_BFLOAT16 = torch.cuda.is_bf16_supported()- if _package_available("flash_attn"):+ if os.environ.get("UNSLOTH_ROCM_FLASH_ATTN_DIFFUSION_ONLY") == "1":+ # Studio's diffusion-only Triton flash-attn: keep LLM attention on SDPA.+ import transformers.utils.import_utils++ transformers.utils.import_utils.is_flash_attn_2_available = lambda *args, **kwargs: False+ import transformers.utils++ transformers.utils.is_flash_attn_2_available = lambda *args, **kwargs: False+ elif _package_available("flash_attn"):# Check for CUDA linking errors "undefined symbol: _ZNK3c106SymIntltEl" and for softcapping; see #1437.try:try: