Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: run fp16 on ROCm GPUs without native bf16 (RDNA2 and older, Vega) (#12676)
Daniel Han8216fa94b41a
diff --git a/studio/backend/core/inference/diffusion_device.py b/studio/backend/core/inference/diffusion_device.pyindex 685a5a836a3..2c7519b3ef6 100644--- a/studio/backend/core/inference/diffusion_device.py+++ b/studio/backend/core/inference/diffusion_device.py@@ -400,11 +400,11 @@ def _cuda_or_rocm_target(ordinal: Optional[int] = None,) -> DiffusionDeviceTarget:if is_rocm:- # ROCm lacks NVIDIA's pre-Ampere bf16-emulation quirk, so is_bf16_supported() is trustworthy. It takes no device- # argument, so the selected card is asked by scoping the current device.+ # is_bf16_supported() takes no device argument: scope the selected card current.+ from .rocm_bf16 import rocm_bf16_supportedtry:with diffusion_device_scope(ordinal):- bf16_ok = bool(torch.cuda.is_bf16_supported())+ bf16_ok = rocm_bf16_supported(torch, ordinal)except Exception:bf16_ok = Falsedtype = torch.bfloat16 if bf16_ok else torch.float16diff --git a/studio/backend/core/inference/native_audio.py b/studio/backend/core/inference/native_audio.pyindex f9c60ea4903..92540eecbb2 100644--- a/studio/backend/core/inference/native_audio.py+++ b/studio/backend/core/inference/native_audio.py@@ -864,8 +864,12 @@ class NativeAudioBackend:import torchif self.device == "cuda":- if getattr(torch.version, "hip", None):- supports_bf16 = torch.cuda.is_bf16_supported()+ from .rocm_bf16 import is_rocm_torch, rocm_bf16_supported+ if is_rocm_torch(torch):+ try:+ supports_bf16 = rocm_bf16_supported(torch)+ except Exception:+ supports_bf16 = Falseelse:try:major, _minor = torch.cuda.get_device_capability()diff --git a/studio/backend/core/inference/rocm_bf16.py b/studio/backend/core/inference/rocm_bf16.pynew file mode 100644index 00000000000..be7548e83fd--- /dev/null+++ b/studio/backend/core/inference/rocm_bf16.py@@ -0,0 +1,72 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Native bf16 on a ROCm card, by gfx target.++``torch.cuda.is_bf16_supported()`` returns True on every HIP build without looking at the device.+bf16 math needs MFMA (gfx908+ CDNA), WMMA or bf16 dot (gfx11+) per llvm/lib/Target/AMDGPU/AMDGPU.td;+GCN, Vega and RDNA1/2 emulate it through fp32, so they get float16, as pre-Ampere NVIDIA does.+``UNSLOTH_STUDIO_ROCM_BF16=1`` restores torch's answer everywhere; ``=0`` forces float16.+"""++from __future__ import annotations++import os+import re+from typing import Any, Optional++ROCM_BF16_ENV = "UNSLOTH_STUDIO_ROCM_BF16"++# GCN gfx6xx-8xx, Vega gfx90x except gfx908 / gfx90a (MFMA), RDNA1/2 gfx101x-103x. Unknown targets keep torch's answer.+_NO_NATIVE_BF16_ARCH = re.compile(r"^gfx(?:[6-8][0-9a-f]{2}|90[0-79c]|10[0-3][0-9a-f])$")+++def is_rocm_torch(torch: Any) -> bool:+ """ROCm build, incl. AMD SDK / Radeon wheels that leave ``torch.version.hip`` unset (tag in ``__version__``)."""+ return bool(+ getattr(getattr(torch, "version", None), "hip", None)+ or "rocm" in str(getattr(torch, "__version__", "") or "").lower()+ )+++def rocm_bf16_forced_off() -> bool:+ return os.environ.get(ROCM_BF16_ENV, "").strip().lower() in ("0", "false", "no", "off")+++def normalize_gfx_arch(arch: Any) -> str:+ """``gfx906:sramecc+:xnack-`` -> ``gfx906``; anything unreadable -> ""."""+ try:+ return str(arch or "").split(":")[0].strip().lower()+ except Exception: # noqa: BLE001+ return ""+++def gfx_arch_lacks_native_bf16(arch: Any) -> bool:+ """True for a gfx target known to have no bf16 MFMA / WMMA / dot instructions."""+ return bool(_NO_NATIVE_BF16_ARCH.match(normalize_gfx_arch(arch)))+++def _device_gfx_arch(torch: Any, ordinal: Optional[int]) -> str:+ try:+ index = torch.cuda.current_device() if ordinal is None else ordinal+ props = torch.cuda.get_device_properties(index)+ except Exception: # noqa: BLE001 -- unreadable properties: no arch, torch's answer stands+ return ""+ # gcnArchName alone is empty on some AMD SDK / Radeon wheels (same spellings utils.hardware reads).+ for attr in ("gcnArchName", "gcn_arch_name", "arch_name", "gfx_arch_name"):+ arch = normalize_gfx_arch(getattr(props, attr, ""))+ if arch:+ return arch+ return ""+++def rocm_bf16_supported(torch: Any, ordinal: Optional[int] = None) -> bool:+ """Native bf16 on the ROCm card ``ordinal`` (current device when None); HIP builds only. Raises what torch raises."""+ if rocm_bf16_forced_off():+ return False+ override = os.environ.get(ROCM_BF16_ENV, "").strip().lower()+ if override not in ("1", "true", "yes", "on") and gfx_arch_lacks_native_bf16(+ _device_gfx_arch(torch, ordinal)+ ):+ return False+ return bool(torch.cuda.is_bf16_supported())diff --git a/studio/backend/core/systemone/laya_runtime.py b/studio/backend/core/systemone/laya_runtime.pyindex 7af744ccdb8..c951f227cf1 100644--- a/studio/backend/core/systemone/laya_runtime.py+++ b/studio/backend/core/systemone/laya_runtime.py@@ -440,8 +440,10 @@ def _precision(device, fp16_checkpoint: bool):if device.type == "cuda":if fp16_checkpoint:return torch.float16, torch.float16- if torch.version.hip:- bf16 = torch.cuda.is_bf16_supported()+ from core.inference.rocm_bf16 import is_rocm_torch, rocm_bf16_supported++ if is_rocm_torch(torch):+ bf16 = rocm_bf16_supported(torch, device.index)else:# By capability: pre-Ampere NVIDIA reports is_bf16_supported() through slow emulation.bf16 = torch.cuda.get_device_capability(device)[0] >= 8diff --git a/studio/backend/core/training/diffusion_dit_trainer.py b/studio/backend/core/training/diffusion_dit_trainer.pyindex 2f8ffb14e9e..d0cf1376020 100644--- a/studio/backend/core/training/diffusion_dit_trainer.py+++ b/studio/backend/core/training/diffusion_dit_trainer.py@@ -57,7 +57,7 @@ from core.training.diffusion_train_common import (_restore_perf_flags,discover_image_caption_pairs,has_functional_torchao,- native_bf16_supported,+ flow_bf16_trainable,native_bf16_supported_xpu,PermutationBatchSampler,repo_is_prequantized,@@ -1870,7 +1870,7 @@ def run_dit_lora_training(device = resolve_train_device()# bf16 throughout (fp32 on a CPU-only box, to keep import/unit tests architecture-agnostic). Both accelerator# guards gate on NATIVE bf16, since is_bf16_supported() counts emulation on CUDA and on XPU alike.- if device == "cuda" and not native_bf16_supported():+ if device == "cuda" and not flow_bf16_trainable():raise ValueError("This trainer requires a bfloat16-capable GPU (Ampere or newer); ""this CUDA device does not support bf16."diff --git a/studio/backend/core/training/diffusion_h3_trainer.py b/studio/backend/core/training/diffusion_h3_trainer.pyindex ac266ae0dde..10cd7e82a7a 100644--- a/studio/backend/core/training/diffusion_h3_trainer.py+++ b/studio/backend/core/training/diffusion_h3_trainer.py@@ -82,7 +82,7 @@ from core.training.diffusion_train_common import (_latent_cache_over_budget,_restore_perf_flags,h3_train_unsupported_reason,- native_bf16_supported,+ flow_bf16_trainable,native_bf16_supported_xpu,resolve_train_device,resolve_train_steps,@@ -483,7 +483,7 @@ def run_h3_lora_training(return Truedevice = resolve_train_device()- if device == "cuda" and not native_bf16_supported():+ if device == "cuda" and not flow_bf16_trainable():raise ValueError("This trainer requires a bfloat16-capable GPU (Ampere or newer); ""this CUDA device does not support bf16."diff --git a/studio/backend/core/training/diffusion_train_common.py b/studio/backend/core/training/diffusion_train_common.pyindex f7707677680..142734ff949 100644--- a/studio/backend/core/training/diffusion_train_common.py+++ b/studio/backend/core/training/diffusion_train_common.py@@ -421,7 +421,7 @@ def train_precision_modes() -> tuple[list[str], str]:recommended = "nf4"try:import torch- if native_bf16_supported():+ if flow_bf16_trainable():modes.append("bf16")# ROCm capability values are gfx versions, not the NVIDIA SM levels checked below.torchao_ok = has_functional_torchao() and not torch_is_rocm()@@ -631,21 +631,41 @@ def native_bf16_supported() -> bool:``torch.cuda.is_bf16_supported()`` defaults to counting EMULATED bf16, which every pre-AmpereCUDA card (T4 / V100 / RTX 20xx) reports as supported even though the DiT trainer needs realAmpere-or-newer bf16. Gate NVIDIA on compute capability major >= 8 instead, the same #6658- fix the inference device resolver already uses; ROCm has no such quirk. Never raises. Shared- by the /info modes, the start preflight, and the trainer guard so all three stay in sync."""+ fix the inference device resolver already uses; ROCm by gfx arch (``rocm_bf16_supported``).+ Never raises. The flow trainers' admission is ``flow_bf16_trainable``."""try:import torchif not torch.cuda.is_available():return False- is_rocm = bool(getattr(getattr(torch, "version", None), "hip", None))- if is_rocm:- return bool(torch.cuda.is_bf16_supported())+ from core.inference.rocm_bf16 import is_rocm_torch, rocm_bf16_supported++ if is_rocm_torch(torch):+ return rocm_bf16_supported(torch)return torch.cuda.get_device_capability()[0] >= 8except Exception: # noqa: BLE001 -- no torch / probe failure -> treat as unsupportedreturn False+def flow_bf16_trainable() -> bool:+ """Whether the bf16-only flow trainers (DiT, MiniMax-H3; no fp16 path) may run on this GPU. ROCm targets+ without native bf16 (RDNA2-and-older / Vega) still run bf16 through fp32 emulation, slowly, so they stay+ admitted as before the arch gate; only pre-Ampere NVIDIA and ``UNSLOTH_STUDIO_ROCM_BF16=0`` are refused. Never raises."""+ if native_bf16_supported():+ return True+ try:+ import torch+ from core.inference.rocm_bf16 import rocm_bf16_forced_off++ if rocm_bf16_forced_off():+ return False+ return bool(+ torch.cuda.is_available() and torch_is_rocm() and torch.cuda.is_bf16_supported()+ )+ except Exception: # noqa: BLE001 -- no torch / probe failure -> treat as unsupported+ return False++def resolve_train_device() -> str:"""The device a flow-matching trainer runs on: CUDA first (so a box with both is unaffected),then Intel XPU, else CPU.@@ -714,7 +734,7 @@ def bf16_unsupported_reason(resolved_family: str) -> Optional[str]:return Nonetry:import torch- if torch.cuda.is_available() and not native_bf16_supported():+ if torch.cuda.is_available() and not flow_bf16_trainable():return ("This trainer requires a bfloat16-capable GPU (Ampere or newer); this CUDA ""device does not support bf16. Train the DiT families on a newer GPU."diff --git a/studio/backend/tests/test_rocm_bf16_arch_gate.py b/studio/backend/tests/test_rocm_bf16_arch_gate.pynew file mode 100644index 00000000000..7695d501c7c--- /dev/null+++ b/studio/backend/tests/test_rocm_bf16_arch_gate.py@@ -0,0 +1,205 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""ROCm bf16 is gated by gfx arch: torch.cuda.is_bf16_supported() is True on every HIP build, so+RDNA2-and-older / Vega cards (no bf16 MFMA / WMMA / dot) must resolve float16, while CDNA (gfx908+),+RDNA3+, gfx1151 and every NVIDIA card keep their previous pick."""++from __future__ import annotations++import sys+import types++import pytest++import core.inference.diffusion_device as dd+import core.inference.native_audio as na+import core.training.diffusion_train_common as tc++BF16, FP16, FP32 = "bf16", "fp16", "fp32"+++def _fake_torch(+ *,+ hip,+ arch = "",+ capability = (8, 0),+):+ torch = types.ModuleType("torch")+ torch.bfloat16, torch.float16, torch.float32 = BF16, FP16, FP32+ torch.version = types.SimpleNamespace(hip = hip)+ props = types.SimpleNamespace(gcnArchName = arch, major = capability[0], minor = capability[1])++ def _is_bf16_supported(including_emulation = True):+ # torch's real body: any HIP build answers True before looking at the device.+ if hip:+ return True+ return capability[0] >= 8 or including_emulation++ torch.cuda = types.SimpleNamespace(+ is_available = lambda: True,+ is_bf16_supported = _is_bf16_supported,+ get_device_capability = lambda device = None: capability,+ get_device_properties = lambda device = None: props,+ current_device = lambda: 0,+ device_count = lambda: 1,+ mem_get_info = lambda index = None: (0, 0),+ set_device = lambda index: None,+ )+ torch.backends = types.SimpleNamespace(mps = types.SimpleNamespace(is_available = lambda: False))+ return torch+++def _install(monkeypatch, torch, *, is_rocm):+ monkeypatch.setitem(sys.modules, "torch", torch)++ class _DT:+ CUDA, XPU, MLX, CPU = "cuda", "xpu", "mlx", "cpu"++ fake_uh = types.ModuleType("utils.hardware")+ fake_uh.DeviceType = _DT+ fake_uh.get_device = lambda: "cuda"+ fake_uh.hardware = types.SimpleNamespace(IS_ROCM = is_rocm)+ monkeypatch.setitem(sys.modules, "utils.hardware", fake_uh)+ monkeypatch.delenv("UNSLOTH_STUDIO_ROCM_BF16", raising = False)+++AMD_CASES = [+ ("gfx1030", FP16), # RX 6800/6900, RDNA2+ ("gfx1031", FP16), # RX 6700, RDNA2+ ("gfx1010", FP16), # RX 5700, RDNA1+ ("gfx906:sramecc+:xnack-", FP16), # MI50 / Radeon VII, suffix stripped+ ("gfx900:xnack-", FP16), # Vega 10+ ("gfx803", FP16), # Polaris+ ("gfx908:sramecc+:xnack-", BF16), # MI100, bf16 MFMA+ ("gfx90a:sramecc+:xnack-", BF16), # MI200+ ("gfx942:sramecc+:xnack-", BF16), # MI300+ ("gfx950", BF16), # MI350+ ("gfx1100", BF16), # RDNA3+ ("gfx1151", BF16), # Strix Halo+ ("gfx1201", BF16), # RDNA4+ ("", BF16), # no arch reported: torch's answer stands (previous behaviour)+]++[email protected]("arch,expected", AMD_CASES)+def test_diffusion_rocm_dtype_by_arch(monkeypatch, arch, expected):+ torch = _fake_torch(hip = "6.4", arch = arch, capability = (9, 0))+ _install(monkeypatch, torch, is_rocm = True)+ t = dd.resolve_diffusion_device_target()+ assert (t.backend, t.dtype) == ("rocm", expected)++[email protected]("arch,expected", AMD_CASES)+def test_native_audio_rocm_dtype_by_arch(monkeypatch, arch, expected):+ torch = _fake_torch(hip = "6.4", arch = arch, capability = (9, 0))+ _install(monkeypatch, torch, is_rocm = True)+ backend = types.SimpleNamespace(device = "cuda")+ assert na.NativeAudioBackend._dtype(backend) == expected++[email protected]("arch,expected", AMD_CASES)+def test_training_native_bf16_by_arch(monkeypatch, arch, expected):+ torch = _fake_torch(hip = "6.4", arch = arch, capability = (9, 0))+ _install(monkeypatch, torch, is_rocm = True)+ assert tc.native_bf16_supported() is (expected == BF16)++[email protected]("capability,expected", [((7, 5), FP16), ((8, 0), BF16), ((9, 0), BF16)])+def test_nvidia_unchanged(monkeypatch, capability, expected):+ # A stray gcnArchName must not matter: the arch gate is ROCm-only.+ torch = _fake_torch(hip = None, arch = "gfx1030", capability = capability)+ _install(monkeypatch, torch, is_rocm = False)+ assert dd.resolve_diffusion_device_target().dtype == expected+ assert na.NativeAudioBackend._dtype(types.SimpleNamespace(device = "cuda")) == expected+ assert tc.native_bf16_supported() is (expected == BF16)+++def test_env_override(monkeypatch):+ torch = _fake_torch(hip = "6.4", arch = "gfx1030")+ _install(monkeypatch, torch, is_rocm = True)+ monkeypatch.setenv("UNSLOTH_STUDIO_ROCM_BF16", "1")+ assert dd.resolve_diffusion_device_target().dtype == BF16+ torch_cdna = _fake_torch(hip = "6.4", arch = "gfx942")+ _install(monkeypatch, torch_cdna, is_rocm = True)+ monkeypatch.setenv("UNSLOTH_STUDIO_ROCM_BF16", "0")+ assert dd.resolve_diffusion_device_target().dtype == FP16+++def test_selected_ordinal_arch_is_read(monkeypatch):+ # Mixed box: card 0 is gfx1100, the selected card 1 is gfx1030 -> fp16 for card 1.+ from core.inference.rocm_bf16 import rocm_bf16_supported++ torch = _fake_torch(hip = "6.4")+ archs = {0: "gfx1100", 1: "gfx1030"}+ torch.cuda.get_device_properties = lambda i = None: types.SimpleNamespace(gcnArchName = archs[i])+ monkeypatch.setitem(sys.modules, "torch", torch)+ monkeypatch.delenv("UNSLOTH_STUDIO_ROCM_BF16", raising = False)+ assert rocm_bf16_supported(torch, 1) is False+ assert rocm_bf16_supported(torch, 0) is True+ assert rocm_bf16_supported(torch) is True+++ "arch,expected", [("gfx1030", FP16), ("gfx906", FP16), ("gfx90a", BF16), ("gfx1151", BF16)]+)+def test_laya_precision_by_arch(monkeypatch, arch, expected):+ import core.systemone.laya_runtime as lr++ torch = _fake_torch(hip = "6.4", arch = arch)+ _install(monkeypatch, torch, is_rocm = True)+ monkeypatch.delenv("UNSLOTH_SYSTEMONE_FP32", raising = False)+ device = types.SimpleNamespace(type = "cuda", index = 0)+ assert lr._precision(device, fp16_checkpoint = False) == (expected, expected)++[email protected]("arch", ["gfx1030", "gfx906", "gfx90a", "gfx1151"])+def test_flow_trainers_stay_admitted_on_rocm(monkeypatch, arch):+ # DiT / MiniMax-H3 train in bf16 only: an emulated-bf16 ROCm card keeps training (slowly) as before the gate.+ torch = _fake_torch(hip = "6.4", arch = arch)+ _install(monkeypatch, torch, is_rocm = True)+ assert tc.flow_bf16_trainable() is True+ assert tc.bf16_unsupported_reason("minimax-h3") is None+ assert "bf16" in tc.train_precision_modes()[0]++[email protected]("capability,expected", [((7, 5), False), ((8, 0), True)])+def test_flow_trainers_nvidia_unchanged(monkeypatch, capability, expected):+ torch = _fake_torch(hip = None, capability = capability)+ _install(monkeypatch, torch, is_rocm = False)+ assert tc.flow_bf16_trainable() is expected+ assert (tc.bf16_unsupported_reason("minimax-h3") is None) is expected+++def test_real_rocm_device_arch_is_read():+ torch = pytest.importorskip("torch")+ if not (getattr(torch.version, "hip", None) and torch.cuda.is_available()):+ pytest.skip("needs a ROCm GPU")+ from core.inference.rocm_bf16 import (+ _device_gfx_arch,+ gfx_arch_lacks_native_bf16,+ rocm_bf16_supported,+ )++ arch = _device_gfx_arch(torch, 0)+ assert arch.startswith("gfx"), arch+ assert rocm_bf16_supported(torch, 0) is (not gfx_arch_lacks_native_bf16(arch))+ print(f"ROCM_ARCH {arch} bf16={rocm_bf16_supported(torch, 0)}")++[email protected]("arch,expected", [("gfx1030", FP16), ("gfx906", FP16), ("gfx1151", BF16)])+def test_rocm_wheel_without_version_hip(monkeypatch, arch, expected):+ # AMD SDK / Radeon wheels leave torch.version.hip unset; the rocm tag is in __version__ and the capability is gfx.+ torch = _fake_torch(hip = None, arch = arch, capability = (10, 3))+ torch.__version__ = "2.9.0+rocmsdk20251116"+ _install(monkeypatch, torch, is_rocm = True)+ assert na.NativeAudioBackend._dtype(types.SimpleNamespace(device = "cuda")) == expected+ assert tc.native_bf16_supported() is (expected == BF16)+++def test_flow_trainers_honor_forced_fp16(monkeypatch):+ torch = _fake_torch(hip = "6.4", arch = "gfx1030")+ _install(monkeypatch, torch, is_rocm = True)+ monkeypatch.setenv("UNSLOTH_STUDIO_ROCM_BF16", "0")+ assert tc.flow_bf16_trainable() is False+ assert tc.bf16_unsupported_reason("minimax-h3") is not None