Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

RMSNorm: enable the narrow-row kernels on ROCm with the DPP reduction order (#14637)

Daniel Han9dfba3be31cd
diff --git a/tests/test_rms_layernorm_multirow.py b/tests/test_rms_layernorm_multirow.py
index 515f056d967..3ea786ddb7d 100644
--- a/tests/test_rms_layernorm_multirow.py
+++ b/tests/test_rms_layernorm_multirow.py
@@ -19,17 +19,38 @@ import torch._dynamo.utils as dynamo_utils
from unsloth.kernels import rms_layernorm
from unsloth.kernels.rms_layernorm import fast_rms_layernorm
-if torch.version.hip:
- pytest.skip("the multi-row kernels are off on ROCm", allow_module_level = True)
-
BF16_OK = torch.cuda.get_device_capability()[0] >= 8
DTYPES = [torch.float16, torch.float32] + ([torch.bfloat16] if BF16_OK else [])
NARROW = [33, 64, 96, 100, 127, 128]
_eager = rms_layernorm._eager_kernel
+def _replay_matches_device():
+ # ROCm arches whose tl.sum is the generic butterfly (RDNA1/2, CDNA1) fail the DPP replay; production stays
+ # one-row there through the self-check, but most tests below force the replay.
+ if not torch.version.hip:
+ return True
+ device = torch.device("cuda", torch.cuda.current_device())
+ saved, rms_layernorm._MULTIROW = rms_layernorm._MULTIROW, True
+ try:
+ settings = rms_layernorm._multirow_settings(128, device)
+ finally:
+ rms_layernorm._MULTIROW = saved
+ return settings is not None and rms_layernorm._multirow_self_check(
+ device, torch.float32, torch.float32, 128, 1e-6, False, settings
+ )
+
+
+REPLAY_OK = _replay_matches_device()
+
+
@pytest.fixture
-def multirow(monkeypatch):
+def multirow(monkeypatch, request):
+ if (
+ not REPLAY_OK
+ and request.node.originalname != "test_rocm_without_the_dpp_order_stays_one_row"
+ ):
+ pytest.skip("this ROCm arch does not reduce in the replayed order")
monkeypatch.setattr(rms_layernorm, "_MULTIROW", True)
monkeypatch.setattr(rms_layernorm, "_MULTIROW_CHECKED", {})
return monkeypatch
@@ -323,3 +344,12 @@ def test_multirow_launches_on_the_tensors_device(multirow, path):
torch._dynamo.reset()
assert torch.equal(_bits(ref), _bits(got))
assert any(rms_layernorm._MULTIROW_CHECKED.values()) and rms_layernorm._MULTIROW
+
+
[email protected](REPLAY_OK, reason = "this device reduces in the replayed order")
+def test_rocm_without_the_dpp_order_stays_one_row(multirow):
+ """An arch the replay does not match never runs the multi-row kernels."""
+ X = torch.randn(64, 128, device = "cuda")
+ W = torch.randn(128, device = "cuda")
+ with pytest.warns(UserWarning, match = "self-check mismatch"):
+ assert rms_layernorm._multirow_checked(X, W, 1e-6, False) is None
diff --git a/tests/test_rms_layernorm_multirow_settings.py b/tests/test_rms_layernorm_multirow_settings.py
new file mode 100644
index 00000000000..49d8e386fbb
--- /dev/null
+++ b/tests/test_rms_layernorm_multirow_settings.py
@@ -0,0 +1,136 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
+
+"""CPU-only: narrow-row RMSNorm selection per backend, read from the kernel source (no GPU, no import).
+
+CUDA keeps the 32-lane replay and settings it shipped with; ROCm divides by the wavefront width.
+"""
+
+import ast
+import functools
+import types
+from pathlib import Path
+
+import pytest
+
+SOURCE = Path(__file__).resolve().parents[1] / "unsloth" / "kernels" / "rms_layernorm.py"
+TREE = ast.parse(SOURCE.read_text(encoding = "utf-8"))
+FUNCS = {n.name: n for n in TREE.body if isinstance(n, ast.FunctionDef)}
+
+# The CUDA replay as shipped before ROCm support; any edit changes CUDA bytes.
+CUDA_REPLAY = """
+def _row_dot_in_cuda_row_order(a, b, ROWS: tl.constexpr, WARPS: tl.constexpr):
+ a = tl.permute(tl.reshape(a, (ROWS, WARPS, 2, 16)), (0, 1, 3, 2))
+ b = tl.permute(tl.reshape(b, (ROWS, WARPS, 2, 16)), (0, 1, 3, 2))
+ a_lo, a_hi = tl.split(a)
+ b_lo, b_hi = tl.split(b)
+ acc = tl.fma(a_lo, b_lo, a_hi * b_hi)
+ acc = _fold_lanes(acc, ROWS, WARPS, 8)
+ acc = _fold_lanes(acc, ROWS, WARPS, 4)
+ acc = _fold_lanes(acc, ROWS, WARPS, 2)
+ acc = _fold_lanes(acc, ROWS, WARPS, 1)
+ acc = tl.reshape(acc, (ROWS, WARPS))
+ if WARPS == 4:
+ acc = tl.sum(tl.reshape(acc, (ROWS, 2, 2)), axis = 1)
+ return tl.sum(acc, axis = 1)
+"""
+
+
+def _calculate_settings(n):
+ block = 1 << (n - 1).bit_length()
+ warps = 32 if block >= 32768 else 16 if block >= 8192 else 8 if block >= 2048 else 4
+ return block, warps
+
+
+def _load(hip, lanes):
+ """Exec _multirow_settings (+ _hip_wavefront) against a fake torch."""
+ calls = []
+
+ def wavefront(index):
+ calls.append(index)
+ return lanes
+
+ torch = types.SimpleNamespace(
+ version = types.SimpleNamespace(hip = hip),
+ cuda = types.SimpleNamespace(current_device = lambda: 0),
+ )
+ ns = {
+ "torch": torch,
+ "functools": functools,
+ "_settings": _calculate_settings,
+ "_MULTIROW": True,
+ "_MULTIROW_ELEMENTS": 4096,
+ "_MULTIROW_NUM_WARPS": 4,
+ }
+ exec(compile(ast.Module([FUNCS["_multirow_settings"]], []), str(SOURCE), "exec"), ns)
+ ns["_hip_wavefront"] = wavefront
+ return ns["_multirow_settings"], calls
+
+
+def _shipped_cuda(n):
+ block, warps = _calculate_settings(n)
+ if warps != 4 or not (64 <= block <= 128):
+ return None
+ return block, 4096 // block, block // 32, 4
+
+
+def test_cuda_settings_unchanged_and_never_ask_for_a_wavefront():
+ settings, calls = _load(hip = None, lanes = 64)
+ for n in range(1, 4097):
+ assert settings(n) == _shipped_cuda(n), n
+ assert calls == []
+
+
[email protected]("lanes", [32, 64])
+def test_hip_settings_divide_by_the_wavefront(lanes):
+ settings, calls = _load(hip = "7.0", lanes = lanes)
+ for n in range(1, 4097):
+ cuda = _shipped_cuda(n)
+ expect = None if cuda is None else (cuda[0], cuda[1], cuda[0] // lanes, 4)
+ assert settings(n) == expect, n
+ assert calls and set(calls) == {0}
+ assert settings(128, types.SimpleNamespace(index = 3)) is not None and calls[-1] == 3
+
+
+def test_hip_unknown_wavefront_stays_one_row():
+ settings, _ = _load(hip = "7.0", lanes = None)
+ assert all(settings(n) is None for n in (33, 64, 100, 128))
+
+
+def test_cuda_replay_body_is_the_shipped_one():
+ shipped = ast.parse(CUDA_REPLAY).body[0]
+ current = FUNCS["_row_dot_in_cuda_row_order"]
+ assert [ast.dump(s) for s in current.body if not isinstance(s, ast.Expr)] == [
+ ast.dump(s) for s in shipped.body
+ ]
+ assert ast.dump(current.args) == ast.dump(shipped.args)
+
+
+def test_row_order_dispatch_is_a_compile_time_backend_switch():
+ flag = next(
+ n
+ for n in TREE.body
+ if isinstance(n, ast.Assign) and getattr(n.targets[0], "id", None) == "_HIP_ROW_ORDER"
+ )
+ assert ast.unparse(flag.value) == "tl.constexpr(bool(torch.version.hip))"
+ dispatch = FUNCS["_row_dot_in_row_kernel_order"]
+ branch = next(s for s in dispatch.body if isinstance(s, ast.If))
+ assert ast.unparse(branch.test) == "_HIP_ROW_ORDER"
+ assert "_row_dot_in_hip_row_order" in ast.unparse(branch.body[0])
+ assert "_row_dot_in_cuda_row_order" in ast.unparse(branch.orelse[0])
+ for name in ("_rms_layernorm_forward_rows", "_rms_backward_rows_block"):
+ text = ast.unparse(FUNCS[name])
+ assert "_row_dot_in_row_kernel_order(" in text
+ assert (
+ "_row_dot_in_cuda_row_order(" not in text and "_row_dot_in_hip_row_order(" not in text
+ )
+
+
+def test_multirow_is_no_longer_gated_off_on_hip():
+ gate = next(
+ n
+ for n in TREE.body
+ if isinstance(n, ast.Assign) and getattr(n.targets[0], "id", None) == "_MULTIROW"
+ )
+ assert "version.hip" not in ast.unparse(gate.value)
+ assert "_triton_at_least(3, 6)" in ast.unparse(gate.value)
diff --git a/unsloth/kernels/rms_layernorm.py b/unsloth/kernels/rms_layernorm.py
index 102547cdd8c..ac4b64bfcef 100644
--- a/unsloth/kernels/rms_layernorm.py
+++ b/unsloth/kernels/rms_layernorm.py
@@ -258,8 +258,35 @@ def _fold_lanes(x, ROWS: tl.constexpr, WARPS: tl.constexpr, HALF: tl.constexpr):
return tl.sum(tl.reshape(x, (ROWS, WARPS, 2, HALF)), axis = 2)
+# ROCm: Triton's tl.sum reduces each 16-lane row of a wavefront with DPP row_shr 8, 4, 2, 1,
+# then joins the rows (permlanex16 on RDNA, row_bcast 15 / 31 on CDNA), not CUDA's xor 16 .. 1.
+_HIP_ROW_ORDER = tl.constexpr(bool(torch.version.hip))
+
+
@triton.jit
-def _row_dot_in_row_kernel_order(a, b, ROWS: tl.constexpr, WARPS: tl.constexpr):
+def _row_dot_in_hip_row_order(a, b, ROWS: tl.constexpr, WARPS: tl.constexpr):
+ # Bit-exact replay of the one-row kernels' tl.sum on AMD (WARPS = BLOCK_SIZE // wavefront):
+ # fma(a_l, b_l, a_(l-8) * b_(l-8)), row_shr 4, 2, 1, 16-lane rows by lane bit 4 then 5, warps.
+ LANES: tl.constexpr = a.shape[1] // WARPS
+ ROWS16: tl.constexpr = LANES // 16
+ a = tl.permute(tl.reshape(a, (ROWS, WARPS, ROWS16, 2, 8)), (0, 1, 2, 4, 3))
+ b = tl.permute(tl.reshape(b, (ROWS, WARPS, ROWS16, 2, 8)), (0, 1, 2, 4, 3))
+ a_lo, a_hi = tl.split(a)
+ b_lo, b_hi = tl.split(b)
+ acc = tl.fma(a_hi, b_hi, a_lo * b_lo)
+ acc = tl.sum(tl.reshape(acc, (ROWS, WARPS, ROWS16, 2, 4)), axis = 3)
+ acc = tl.sum(tl.reshape(acc, (ROWS, WARPS, ROWS16, 2, 2)), axis = 3)
+ acc = tl.sum(acc, axis = 3)
+ if ROWS16 == 4:
+ acc = tl.sum(tl.reshape(acc, (ROWS, WARPS, 2, 2)), axis = 3)
+ acc = tl.sum(acc, axis = 2)
+ if WARPS == 4:
+ acc = tl.sum(tl.reshape(acc, (ROWS, 2, 2)), axis = 1)
+ return tl.sum(acc, axis = 1)
+
+
+def _row_dot_in_cuda_row_order(a, b, ROWS: tl.constexpr, WARPS: tl.constexpr):
# Bit-exact replay of the one-row kernels' tl.sum (4 warps, 64 <= BLOCK_SIZE <= 128):
# fma(a_l, b_l, a_(l+16) * b_(l+16)), lane xor 8, 4, 2, 1, warp xor 2, 1; explicit steps only.
a = tl.permute(tl.reshape(a, (ROWS, WARPS, 2, 16)), (0, 1, 3, 2))
@@ -277,6 +304,15 @@ def _row_dot_in_row_kernel_order(a, b, ROWS: tl.constexpr, WARPS: tl.constexpr):
return tl.sum(acc, axis = 1)
+def _row_dot_in_row_kernel_order(a, b, ROWS: tl.constexpr, WARPS: tl.constexpr):
+ if _HIP_ROW_ORDER:
+ total = _row_dot_in_hip_row_order(a, b, ROWS, WARPS)
+ else:
+ total = _row_dot_in_cuda_row_order(a, b, ROWS, WARPS)
+ return total
+
+
@triton.jit
def _rms_layernorm_forward_rows(
Y,
@@ -502,7 +538,7 @@ def _launch(wrap, kernel, grid, args, n_tensors, constexprs, device, **options):
wrap(kernel)[grid](*args, **constexprs, **options)
-# Off on ROCm (64-lane wavefronts reduce in another order) and before Triton 3.6 (slower backward).
+# Off before Triton 3.6 (slower backward). ROCm replays the DPP order; the self-check gates it per arch.
def _triton_at_least(major, minor):
try:
return tuple(int(v) for v in triton.__version__.split(".")[:2]) >= (major, minor)
@@ -510,22 +546,43 @@ def _triton_at_least(major, minor):
return False
-_MULTIROW = (
- os.environ.get("UNSLOTH_RMSNORM_MULTIROW", "1") != "0"
- and not torch.version.hip
- and _triton_at_least(3, 6)
-)
+_MULTIROW = os.environ.get("UNSLOTH_RMSNORM_MULTIROW", "1") != "0" and _triton_at_least(3, 6)
_MULTIROW_ELEMENTS = 4096
_MULTIROW_NUM_WARPS = 4
-def _multirow_settings(n_cols):
[email protected]_cache(maxsize = None)
+def _hip_wavefront(index):
+ # Lanes per wavefront the one-row kernels reduce over: 32 on RDNA, 64 on CDNA; None if unknown.
+ try:
+ with torch_gpu_device(torch.device("cuda", index)):
+ lanes = int(triton.runtime.driver.active.get_current_target().warp_size)
+ except Exception:
+ try:
+ lanes = int(torch.cuda.get_device_properties(index).warp_size)
+ except Exception:
+ return None
+ return lanes if lanes in (32, 64) else None
+
+
+def _multirow_settings(n_cols, device = None):
# Only widths 33..128 (one element per lane) can replay the one-row order; others stay one-row.
if not _MULTIROW:
return None
BLOCK_SIZE, num_warps = _settings(n_cols)
if num_warps != 4 or not (64 <= BLOCK_SIZE <= 128):
return None
+ if torch.version.hip:
+ index = device.index if device is not None and device.index is not None else None
+ lanes = _hip_wavefront(torch.cuda.current_device() if index is None else index)
+ if lanes is None:
+ return None
+ return (
+ BLOCK_SIZE,
+ _MULTIROW_ELEMENTS // BLOCK_SIZE,
+ BLOCK_SIZE // lanes,
+ _MULTIROW_NUM_WARPS,
+ )
return BLOCK_SIZE, _MULTIROW_ELEMENTS // BLOCK_SIZE, BLOCK_SIZE // 32, _MULTIROW_NUM_WARPS
@@ -577,7 +634,7 @@ def _multirow_self_check(device, dtype, W_dtype, n_cols, eps, gemma, multirow):
def _multirow_checked(X, W, eps, gemma):
n_cols = X.shape[1]
- multirow = _multirow_settings(n_cols)
+ multirow = _multirow_settings(n_cols, X.device)
if multirow is None:
return None
key = (X.device, X.dtype, W.dtype, int(n_cols), float(eps), gemma)