Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

FP8 on ROCm: use the dequant path on archs without e4m3fn Triton support (#13331)

Daniel Haneaad1bf289d5
diff --git a/tests/test_fp8_pre_sm89_dequant_fallback.py b/tests/test_fp8_pre_sm89_dequant_fallback.py
index 3109527817c..e11b72091f1 100644
--- a/tests/test_fp8_pre_sm89_dequant_fallback.py
+++ b/tests/test_fp8_pre_sm89_dequant_fallback.py
@@ -12,6 +12,8 @@ on any CUDA GPU including the pre-sm89 ones under discussion, which cannot compi
they are asserted to select. Only the value comparison needs real fp8 hardware, and it skips.
"""
+import types
+
import pytest
import torch
@@ -47,6 +49,8 @@ def _route(
weight,
scale,
hip = None,
+ gcn_arch = None,
+ triton_version = None,
):
"""Return "triton" or "torch" for a simulated device.
@@ -72,6 +76,11 @@ def _route(
monkeypatch.setattr(fp8, "weight_dequant_block", _stub)
monkeypatch.setattr(torch.cuda, "get_device_capability", lambda *a, **k: capability)
monkeypatch.setattr(torch.version, "hip", hip)
+ if gcn_arch is not None:
+ props = types.SimpleNamespace(gcnArchName = gcn_arch)
+ monkeypatch.setattr(torch.cuda, "get_device_properties", lambda *a, **k: props)
+ if triton_version is not None:
+ monkeypatch.setattr(fp8.triton, "__version__", triton_version)
fp8._blockwise_weight_dequant_any_shape(weight, scale, BLOCK, torch.bfloat16)
return "triton" if calls else "torch"
@@ -101,12 +110,61 @@ def test_float8_e5m2_is_never_diverted(monkeypatch, capability):
assert _route(monkeypatch, capability, weight, scale) == "triton"
[email protected]("capability", [(8, 0), (9, 0), (11, 5)])
-def test_rocm_is_never_diverted(monkeypatch, capability):
- # get_device_capability is gfx-derived on ROCm, not an SM number, and AMD's triton backend
- # lists fp8e4nv unconditionally, so the NVIDIA-only guard must not fire there.
+ "gcn_arch, capability, triton_version, expected",
+ [
+ # triton 3.5+ lists fp8e4nv on every AMD arch (emulated without OCP fp8 hardware).
+ ("gfx950", (9, 5), "3.5.0", "triton"), # MI350X / MI355X
+ ("gfx942:sramecc+:xnack-", (9, 4), "3.5.0", "triton"), # MI300X
+ ("gfx90a:sramecc+:xnack-", (9, 0), "3.5.0", "triton"), # MI210 / MI250X (CDNA2)
+ ("gfx1151", (11, 5), "3.6.0", "triton"), # Strix Halo: verified on real hardware
+ ("gfx1030", (10, 3), "3.5.0", "triton"), # RX 6900 XT (RDNA2)
+ # triton 3.4: gfx942 / gfx950 / gfx12 only.
+ ("gfx1201", (12, 0), "3.4.0", "triton"), # RX 9070 XT (RDNA4)
+ ("gfx942:sramecc+:xnack-", (9, 4), "3.4.0", "triton"),
+ ("gfx1151", (11, 5), "3.4.0", "torch"),
+ ("gfx90a:sramecc+:xnack-", (9, 0), "3.4.0", "torch"),
+ # triton 3.3: CDNA3 / CDNA4 only.
+ ("gfx950", (9, 5), "3.3.1", "triton"),
+ ("gfx942:sramecc+:xnack-", (9, 4), "3.3.1", "triton"),
+ ("gfx1201", (12, 0), "3.3.1", "torch"),
+ ("gfx1100", (11, 0), "3.3.1", "torch"),
+ # triton 3.2 (torch 2.6 ROCm): fp8e5 only.
+ ("gfx942:sramecc+:xnack-", (9, 4), "3.2.0", "torch"),
+ ("gfx950", (9, 5), "3.2.0", "torch"),
+ ],
+)
+def test_rocm_routes_on_gfx_arch_not_capability(
+ monkeypatch, gcn_arch, capability, triton_version, expected
+):
+ # get_device_capability is gfx-derived on ROCm (gfx1151 reports 11.5, above any sm89 cut), so
+ # the guard keys on the arch: archs AMD triton cannot compile fp8e4nv for take the torch path.
+ weight, scale = _make()
+ route = _route(
+ monkeypatch,
+ capability,
+ weight,
+ scale,
+ hip = "6.4.0",
+ gcn_arch = gcn_arch,
+ triton_version = triton_version,
+ )
+ assert route == expected
+
+
+def test_rocm_unknown_arch_takes_the_torch_path(monkeypatch):
+ # A missing or unreadable gcnArchName must fall back, not compile a kernel that may not exist.
+ from unsloth.kernels import fp8
+
+ def _boom(*a, **k):
+ raise RuntimeError("no device properties")
+
+ monkeypatch.setattr(torch.cuda, "get_device_properties", _boom)
weight, scale = _make()
- assert _route(monkeypatch, capability, weight, scale, hip = "6.2.0") == "triton"
+ # Only triton 3.3 / 3.4 need the arch; 3.5+ supports fp8e4nv everywhere without asking.
+ assert (
+ _route(monkeypatch, (9, 4), weight, scale, hip = "6.4.0", triton_version = "3.4.0") == "torch"
+ )
@pytest.mark.parametrize(
@@ -268,3 +326,48 @@ def test_grouped_fp8_eval_uses_the_fallback_on_pre_sm89(monkeypatch):
assert calls, "grouped eval bypassed the guarded dequant on a pre-sm89 device"
assert torch.isfinite(out.float()).all()
+
+
[email protected]("gcn_arch", ["gfx1151", "gfx90a:sramecc+:xnack-"])
+def test_rocm_forward_without_fp8e4nv_skips_act_quant(monkeypatch, gcn_arch):
+ # The block forward quantizes the activation to e4m3fn in triton (act_quant) before the fp8 gemm;
+ # on an arch without fp8e4nv both must be skipped for dequant plus a 16-bit matmul.
+ from unsloth.kernels import fp8
+
+ def _no_kernel(*a, **k):
+ raise AssertionError("fp8e4nv triton kernel launched on an arch without fp8e4nv")
+
+ monkeypatch.setattr(fp8, "act_quant", _no_kernel)
+ monkeypatch.setattr(fp8, "fp8_block_matmul", _no_kernel)
+ monkeypatch.setattr(fp8, "weight_dequant_block", _no_kernel)
+ monkeypatch.setattr(torch.version, "hip", "6.4.0")
+ props = types.SimpleNamespace(gcnArchName = gcn_arch)
+ monkeypatch.setattr(torch.cuda, "get_device_properties", lambda *a, **k: props)
+ monkeypatch.setattr(fp8.triton, "__version__", "3.4.0")
+
+ weight, scale = _make()
+ X = torch.randn(4, 256, device = "cuda", dtype = torch.bfloat16)
+ out = fp8.FP8BlockQuantLinear.apply(X, weight, scale)
+ W = (weight.float() * scale.repeat_interleave(BLOCK[0], 0).repeat_interleave(BLOCK[1], 1)).to(
+ torch.bfloat16
+ )
+ torch.testing.assert_close(out, X @ W.T)
+
+
+def test_real_device_forward_matches_the_dequant_reference():
+ # Unmocked: on any GPU the public forward must run and match a float32 dequant reference.
+ from unsloth.kernels import fp8
+
+ torch.manual_seed(0)
+ bs, m, k = 128, 256, 512
+ weight = (torch.randn(m, k, device = "cuda") * 0.3).to(torch.float8_e4m3fn)
+ scale = torch.rand(m // bs, k // bs, device = "cuda", dtype = torch.float32) + 0.5
+ scale.block_size = [bs, bs]
+ X = torch.randn(16, k, device = "cuda", dtype = torch.bfloat16) * 0.5
+ out = fp8.FP8BlockQuantLinear.apply(X, weight, scale)
+ torch.cuda.synchronize()
+ full_scale = scale.repeat_interleave(bs, 0).repeat_interleave(bs, 1)
+ ref = X.float() @ (weight.float() * full_scale).t()
+ rel = (out.float() - ref).norm() / ref.norm()
+ # The fp8 kernels also quantize the activation (a few % error); the dequant path is ~bf16 exact.
+ assert torch.isfinite(out.float()).all() and rel < 0.08, f"relative error {rel:.4f}"
diff --git a/unsloth/kernels/fp8.py b/unsloth/kernels/fp8.py
index fb216b54b81..24b73445261 100644
--- a/unsloth/kernels/fp8.py
+++ b/unsloth/kernels/fp8.py
@@ -464,18 +464,39 @@ fp8_block_matmul = (
)
+def _rocm_fp8e4nv_kernel_supported(device):
+ # AMD triton's supported_fp8_dtypes (third_party/amd/backend/compiler.py): no fp8e4nv before 3.3;
+ # 3.3 adds gfx940-942 / gfx950, 3.4 gfx942 / gfx950 / gfx12*, and 3.5+ lists it on every arch
+ # (software-emulated without OCP fp8 hardware; ran correctly on gfx1151 with triton 3.6).
+ try:
+ version = Version(triton.__version__).release[:2]
+ if version >= (3, 5):
+ return True
+ if version < (3, 3):
+ return False
+ arch = torch.cuda.get_device_properties(device).gcnArchName.split(":")[0].lower()
+ except Exception:
+ return False
+ if version >= (3, 4):
+ return arch in ("gfx942", "gfx950") or arch.startswith("gfx12")
+ return arch in ("gfx940", "gfx941", "gfx942", "gfx950")
+
+
@functools.lru_cache(maxsize = None)
def _fp8_device_lacks_kernel(device):
# A device's capability cannot change, so look it up once. Tests that simulate another
# card must call _fp8_device_lacks_kernel.cache_clear().
- return torch.version.hip is None and torch.cuda.get_device_capability(device) < (8, 9)
+ if torch.version.hip is not None:
+ return not _rocm_fp8e4nv_kernel_supported(device)
+ return torch.cuda.get_device_capability(device) < (8, 9)
def _fp8_kernel_unsupported(tensor, kernel_dtype = None):
"""True when triton here cannot compile this fp8 dtype, so kernels taking it must be avoided.
triton adds fp8e4nv only from sm89 on, so compare the full (major, minor): sm89 (4090,
- L40S, L4) does support it. ROCm lists fp8e4nv always and its capability is gfx-derived.
+ L40S, L4) does support it. ROCm's capability is gfx-derived, so it is gated on the gfx arch
+ and triton version instead (_rocm_fp8e4nv_kernel_supported).
kernel_dtype is the fp8 dtype the kernel really handles. It defaults to the tensor's own,
which is right for a dequant, but a forward must pass e4m3fn: act_quant emits e4m3fn