Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
FP8 on ROCm: use the dequant path on archs without e4m3fn Triton support (#13331)
Daniel Haneaad1bf289d5
diff --git a/tests/test_fp8_pre_sm89_dequant_fallback.py b/tests/test_fp8_pre_sm89_dequant_fallback.pyindex 3109527817c..e11b72091f1 100644--- a/tests/test_fp8_pre_sm89_dequant_fallback.py+++ b/tests/test_fp8_pre_sm89_dequant_fallback.py@@ -12,6 +12,8 @@ on any CUDA GPU including the pre-sm89 ones under discussion, which cannot compithey are asserted to select. Only the value comparison needs real fp8 hardware, and it skips."""+import types+import pytestimport torch@@ -47,6 +49,8 @@ def _route(weight,scale,hip = None,+ gcn_arch = None,+ triton_version = None,):"""Return "triton" or "torch" for a simulated device.@@ -72,6 +76,11 @@ def _route(monkeypatch.setattr(fp8, "weight_dequant_block", _stub)monkeypatch.setattr(torch.cuda, "get_device_capability", lambda *a, **k: capability)monkeypatch.setattr(torch.version, "hip", hip)+ if gcn_arch is not None:+ props = types.SimpleNamespace(gcnArchName = gcn_arch)+ monkeypatch.setattr(torch.cuda, "get_device_properties", lambda *a, **k: props)+ if triton_version is not None:+ monkeypatch.setattr(fp8.triton, "__version__", triton_version)fp8._blockwise_weight_dequant_any_shape(weight, scale, BLOCK, torch.bfloat16)return "triton" if calls else "torch"@@ -101,12 +110,61 @@ def test_float8_e5m2_is_never_diverted(monkeypatch, capability):assert _route(monkeypatch, capability, weight, scale) == "triton"[email protected]("capability", [(8, 0), (9, 0), (11, 5)])-def test_rocm_is_never_diverted(monkeypatch, capability):- # get_device_capability is gfx-derived on ROCm, not an SM number, and AMD's triton backend- # lists fp8e4nv unconditionally, so the NVIDIA-only guard must not fire there.+ "gcn_arch, capability, triton_version, expected",+ [+ # triton 3.5+ lists fp8e4nv on every AMD arch (emulated without OCP fp8 hardware).+ ("gfx950", (9, 5), "3.5.0", "triton"), # MI350X / MI355X+ ("gfx942:sramecc+:xnack-", (9, 4), "3.5.0", "triton"), # MI300X+ ("gfx90a:sramecc+:xnack-", (9, 0), "3.5.0", "triton"), # MI210 / MI250X (CDNA2)+ ("gfx1151", (11, 5), "3.6.0", "triton"), # Strix Halo: verified on real hardware+ ("gfx1030", (10, 3), "3.5.0", "triton"), # RX 6900 XT (RDNA2)+ # triton 3.4: gfx942 / gfx950 / gfx12 only.+ ("gfx1201", (12, 0), "3.4.0", "triton"), # RX 9070 XT (RDNA4)+ ("gfx942:sramecc+:xnack-", (9, 4), "3.4.0", "triton"),+ ("gfx1151", (11, 5), "3.4.0", "torch"),+ ("gfx90a:sramecc+:xnack-", (9, 0), "3.4.0", "torch"),+ # triton 3.3: CDNA3 / CDNA4 only.+ ("gfx950", (9, 5), "3.3.1", "triton"),+ ("gfx942:sramecc+:xnack-", (9, 4), "3.3.1", "triton"),+ ("gfx1201", (12, 0), "3.3.1", "torch"),+ ("gfx1100", (11, 0), "3.3.1", "torch"),+ # triton 3.2 (torch 2.6 ROCm): fp8e5 only.+ ("gfx942:sramecc+:xnack-", (9, 4), "3.2.0", "torch"),+ ("gfx950", (9, 5), "3.2.0", "torch"),+ ],+)+def test_rocm_routes_on_gfx_arch_not_capability(+ monkeypatch, gcn_arch, capability, triton_version, expected+):+ # get_device_capability is gfx-derived on ROCm (gfx1151 reports 11.5, above any sm89 cut), so+ # the guard keys on the arch: archs AMD triton cannot compile fp8e4nv for take the torch path.+ weight, scale = _make()+ route = _route(+ monkeypatch,+ capability,+ weight,+ scale,+ hip = "6.4.0",+ gcn_arch = gcn_arch,+ triton_version = triton_version,+ )+ assert route == expected+++def test_rocm_unknown_arch_takes_the_torch_path(monkeypatch):+ # A missing or unreadable gcnArchName must fall back, not compile a kernel that may not exist.+ from unsloth.kernels import fp8++ def _boom(*a, **k):+ raise RuntimeError("no device properties")++ monkeypatch.setattr(torch.cuda, "get_device_properties", _boom)weight, scale = _make()- assert _route(monkeypatch, capability, weight, scale, hip = "6.2.0") == "triton"+ # Only triton 3.3 / 3.4 need the arch; 3.5+ supports fp8e4nv everywhere without asking.+ assert (+ _route(monkeypatch, (9, 4), weight, scale, hip = "6.4.0", triton_version = "3.4.0") == "torch"+ )@pytest.mark.parametrize(@@ -268,3 +326,48 @@ def test_grouped_fp8_eval_uses_the_fallback_on_pre_sm89(monkeypatch):assert calls, "grouped eval bypassed the guarded dequant on a pre-sm89 device"assert torch.isfinite(out.float()).all()++[email protected]("gcn_arch", ["gfx1151", "gfx90a:sramecc+:xnack-"])+def test_rocm_forward_without_fp8e4nv_skips_act_quant(monkeypatch, gcn_arch):+ # The block forward quantizes the activation to e4m3fn in triton (act_quant) before the fp8 gemm;+ # on an arch without fp8e4nv both must be skipped for dequant plus a 16-bit matmul.+ from unsloth.kernels import fp8++ def _no_kernel(*a, **k):+ raise AssertionError("fp8e4nv triton kernel launched on an arch without fp8e4nv")++ monkeypatch.setattr(fp8, "act_quant", _no_kernel)+ monkeypatch.setattr(fp8, "fp8_block_matmul", _no_kernel)+ monkeypatch.setattr(fp8, "weight_dequant_block", _no_kernel)+ monkeypatch.setattr(torch.version, "hip", "6.4.0")+ props = types.SimpleNamespace(gcnArchName = gcn_arch)+ monkeypatch.setattr(torch.cuda, "get_device_properties", lambda *a, **k: props)+ monkeypatch.setattr(fp8.triton, "__version__", "3.4.0")++ weight, scale = _make()+ X = torch.randn(4, 256, device = "cuda", dtype = torch.bfloat16)+ out = fp8.FP8BlockQuantLinear.apply(X, weight, scale)+ W = (weight.float() * scale.repeat_interleave(BLOCK[0], 0).repeat_interleave(BLOCK[1], 1)).to(+ torch.bfloat16+ )+ torch.testing.assert_close(out, X @ W.T)+++def test_real_device_forward_matches_the_dequant_reference():+ # Unmocked: on any GPU the public forward must run and match a float32 dequant reference.+ from unsloth.kernels import fp8++ torch.manual_seed(0)+ bs, m, k = 128, 256, 512+ weight = (torch.randn(m, k, device = "cuda") * 0.3).to(torch.float8_e4m3fn)+ scale = torch.rand(m // bs, k // bs, device = "cuda", dtype = torch.float32) + 0.5+ scale.block_size = [bs, bs]+ X = torch.randn(16, k, device = "cuda", dtype = torch.bfloat16) * 0.5+ out = fp8.FP8BlockQuantLinear.apply(X, weight, scale)+ torch.cuda.synchronize()+ full_scale = scale.repeat_interleave(bs, 0).repeat_interleave(bs, 1)+ ref = X.float() @ (weight.float() * full_scale).t()+ rel = (out.float() - ref).norm() / ref.norm()+ # The fp8 kernels also quantize the activation (a few % error); the dequant path is ~bf16 exact.+ assert torch.isfinite(out.float()).all() and rel < 0.08, f"relative error {rel:.4f}"diff --git a/unsloth/kernels/fp8.py b/unsloth/kernels/fp8.pyindex fb216b54b81..24b73445261 100644--- a/unsloth/kernels/fp8.py+++ b/unsloth/kernels/fp8.py@@ -464,18 +464,39 @@ fp8_block_matmul = ()+def _rocm_fp8e4nv_kernel_supported(device):+ # AMD triton's supported_fp8_dtypes (third_party/amd/backend/compiler.py): no fp8e4nv before 3.3;+ # 3.3 adds gfx940-942 / gfx950, 3.4 gfx942 / gfx950 / gfx12*, and 3.5+ lists it on every arch+ # (software-emulated without OCP fp8 hardware; ran correctly on gfx1151 with triton 3.6).+ try:+ version = Version(triton.__version__).release[:2]+ if version >= (3, 5):+ return True+ if version < (3, 3):+ return False+ arch = torch.cuda.get_device_properties(device).gcnArchName.split(":")[0].lower()+ except Exception:+ return False+ if version >= (3, 4):+ return arch in ("gfx942", "gfx950") or arch.startswith("gfx12")+ return arch in ("gfx940", "gfx941", "gfx942", "gfx950")++@functools.lru_cache(maxsize = None)def _fp8_device_lacks_kernel(device):# A device's capability cannot change, so look it up once. Tests that simulate another# card must call _fp8_device_lacks_kernel.cache_clear().- return torch.version.hip is None and torch.cuda.get_device_capability(device) < (8, 9)+ if torch.version.hip is not None:+ return not _rocm_fp8e4nv_kernel_supported(device)+ return torch.cuda.get_device_capability(device) < (8, 9)def _fp8_kernel_unsupported(tensor, kernel_dtype = None):"""True when triton here cannot compile this fp8 dtype, so kernels taking it must be avoided.triton adds fp8e4nv only from sm89 on, so compare the full (major, minor): sm89 (4090,- L40S, L4) does support it. ROCm lists fp8e4nv always and its capability is gfx-derived.+ L40S, L4) does support it. ROCm's capability is gfx-derived, so it is gated on the gfx arch+ and triton version instead (_rocm_fp8e4nv_kernel_supported).kernel_dtype is the fp8 dtype the kernel really handles. It defaults to the tensor's own,which is right for a dequant, but a forward must pass e4m3fn: act_quant emits e4m3fn