Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
RMSNorm: enable the narrow-row kernels on ROCm with the DPP reduction order (#14637)
Daniel Han9dfba3be31cd
diff --git a/tests/test_rms_layernorm_multirow.py b/tests/test_rms_layernorm_multirow.pyindex 515f056d967..3ea786ddb7d 100644--- a/tests/test_rms_layernorm_multirow.py+++ b/tests/test_rms_layernorm_multirow.py@@ -19,17 +19,38 @@ import torch._dynamo.utils as dynamo_utilsfrom unsloth.kernels import rms_layernormfrom unsloth.kernels.rms_layernorm import fast_rms_layernorm-if torch.version.hip:- pytest.skip("the multi-row kernels are off on ROCm", allow_module_level = True)-BF16_OK = torch.cuda.get_device_capability()[0] >= 8DTYPES = [torch.float16, torch.float32] + ([torch.bfloat16] if BF16_OK else [])NARROW = [33, 64, 96, 100, 127, 128]_eager = rms_layernorm._eager_kernel+def _replay_matches_device():+ # ROCm arches whose tl.sum is the generic butterfly (RDNA1/2, CDNA1) fail the DPP replay; production stays+ # one-row there through the self-check, but most tests below force the replay.+ if not torch.version.hip:+ return True+ device = torch.device("cuda", torch.cuda.current_device())+ saved, rms_layernorm._MULTIROW = rms_layernorm._MULTIROW, True+ try:+ settings = rms_layernorm._multirow_settings(128, device)+ finally:+ rms_layernorm._MULTIROW = saved+ return settings is not None and rms_layernorm._multirow_self_check(+ device, torch.float32, torch.float32, 128, 1e-6, False, settings+ )+++REPLAY_OK = _replay_matches_device()++@pytest.fixture-def multirow(monkeypatch):+def multirow(monkeypatch, request):+ if (+ not REPLAY_OK+ and request.node.originalname != "test_rocm_without_the_dpp_order_stays_one_row"+ ):+ pytest.skip("this ROCm arch does not reduce in the replayed order")monkeypatch.setattr(rms_layernorm, "_MULTIROW", True)monkeypatch.setattr(rms_layernorm, "_MULTIROW_CHECKED", {})return monkeypatch@@ -323,3 +344,12 @@ def test_multirow_launches_on_the_tensors_device(multirow, path):torch._dynamo.reset()assert torch.equal(_bits(ref), _bits(got))assert any(rms_layernorm._MULTIROW_CHECKED.values()) and rms_layernorm._MULTIROW++[email protected](REPLAY_OK, reason = "this device reduces in the replayed order")+def test_rocm_without_the_dpp_order_stays_one_row(multirow):+ """An arch the replay does not match never runs the multi-row kernels."""+ X = torch.randn(64, 128, device = "cuda")+ W = torch.randn(128, device = "cuda")+ with pytest.warns(UserWarning, match = "self-check mismatch"):+ assert rms_layernorm._multirow_checked(X, W, 1e-6, False) is Nonediff --git a/tests/test_rms_layernorm_multirow_settings.py b/tests/test_rms_layernorm_multirow_settings.pynew file mode 100644index 00000000000..49d8e386fbb--- /dev/null+++ b/tests/test_rms_layernorm_multirow_settings.py@@ -0,0 +1,136 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.++"""CPU-only: narrow-row RMSNorm selection per backend, read from the kernel source (no GPU, no import).++CUDA keeps the 32-lane replay and settings it shipped with; ROCm divides by the wavefront width.+"""++import ast+import functools+import types+from pathlib import Path++import pytest++SOURCE = Path(__file__).resolve().parents[1] / "unsloth" / "kernels" / "rms_layernorm.py"+TREE = ast.parse(SOURCE.read_text(encoding = "utf-8"))+FUNCS = {n.name: n for n in TREE.body if isinstance(n, ast.FunctionDef)}++# The CUDA replay as shipped before ROCm support; any edit changes CUDA bytes.+CUDA_REPLAY = """+def _row_dot_in_cuda_row_order(a, b, ROWS: tl.constexpr, WARPS: tl.constexpr):+ a = tl.permute(tl.reshape(a, (ROWS, WARPS, 2, 16)), (0, 1, 3, 2))+ b = tl.permute(tl.reshape(b, (ROWS, WARPS, 2, 16)), (0, 1, 3, 2))+ a_lo, a_hi = tl.split(a)+ b_lo, b_hi = tl.split(b)+ acc = tl.fma(a_lo, b_lo, a_hi * b_hi)+ acc = _fold_lanes(acc, ROWS, WARPS, 8)+ acc = _fold_lanes(acc, ROWS, WARPS, 4)+ acc = _fold_lanes(acc, ROWS, WARPS, 2)+ acc = _fold_lanes(acc, ROWS, WARPS, 1)+ acc = tl.reshape(acc, (ROWS, WARPS))+ if WARPS == 4:+ acc = tl.sum(tl.reshape(acc, (ROWS, 2, 2)), axis = 1)+ return tl.sum(acc, axis = 1)+"""+++def _calculate_settings(n):+ block = 1 << (n - 1).bit_length()+ warps = 32 if block >= 32768 else 16 if block >= 8192 else 8 if block >= 2048 else 4+ return block, warps+++def _load(hip, lanes):+ """Exec _multirow_settings (+ _hip_wavefront) against a fake torch."""+ calls = []++ def wavefront(index):+ calls.append(index)+ return lanes++ torch = types.SimpleNamespace(+ version = types.SimpleNamespace(hip = hip),+ cuda = types.SimpleNamespace(current_device = lambda: 0),+ )+ ns = {+ "torch": torch,+ "functools": functools,+ "_settings": _calculate_settings,+ "_MULTIROW": True,+ "_MULTIROW_ELEMENTS": 4096,+ "_MULTIROW_NUM_WARPS": 4,+ }+ exec(compile(ast.Module([FUNCS["_multirow_settings"]], []), str(SOURCE), "exec"), ns)+ ns["_hip_wavefront"] = wavefront+ return ns["_multirow_settings"], calls+++def _shipped_cuda(n):+ block, warps = _calculate_settings(n)+ if warps != 4 or not (64 <= block <= 128):+ return None+ return block, 4096 // block, block // 32, 4+++def test_cuda_settings_unchanged_and_never_ask_for_a_wavefront():+ settings, calls = _load(hip = None, lanes = 64)+ for n in range(1, 4097):+ assert settings(n) == _shipped_cuda(n), n+ assert calls == []++[email protected]("lanes", [32, 64])+def test_hip_settings_divide_by_the_wavefront(lanes):+ settings, calls = _load(hip = "7.0", lanes = lanes)+ for n in range(1, 4097):+ cuda = _shipped_cuda(n)+ expect = None if cuda is None else (cuda[0], cuda[1], cuda[0] // lanes, 4)+ assert settings(n) == expect, n+ assert calls and set(calls) == {0}+ assert settings(128, types.SimpleNamespace(index = 3)) is not None and calls[-1] == 3+++def test_hip_unknown_wavefront_stays_one_row():+ settings, _ = _load(hip = "7.0", lanes = None)+ assert all(settings(n) is None for n in (33, 64, 100, 128))+++def test_cuda_replay_body_is_the_shipped_one():+ shipped = ast.parse(CUDA_REPLAY).body[0]+ current = FUNCS["_row_dot_in_cuda_row_order"]+ assert [ast.dump(s) for s in current.body if not isinstance(s, ast.Expr)] == [+ ast.dump(s) for s in shipped.body+ ]+ assert ast.dump(current.args) == ast.dump(shipped.args)+++def test_row_order_dispatch_is_a_compile_time_backend_switch():+ flag = next(+ n+ for n in TREE.body+ if isinstance(n, ast.Assign) and getattr(n.targets[0], "id", None) == "_HIP_ROW_ORDER"+ )+ assert ast.unparse(flag.value) == "tl.constexpr(bool(torch.version.hip))"+ dispatch = FUNCS["_row_dot_in_row_kernel_order"]+ branch = next(s for s in dispatch.body if isinstance(s, ast.If))+ assert ast.unparse(branch.test) == "_HIP_ROW_ORDER"+ assert "_row_dot_in_hip_row_order" in ast.unparse(branch.body[0])+ assert "_row_dot_in_cuda_row_order" in ast.unparse(branch.orelse[0])+ for name in ("_rms_layernorm_forward_rows", "_rms_backward_rows_block"):+ text = ast.unparse(FUNCS[name])+ assert "_row_dot_in_row_kernel_order(" in text+ assert (+ "_row_dot_in_cuda_row_order(" not in text and "_row_dot_in_hip_row_order(" not in text+ )+++def test_multirow_is_no_longer_gated_off_on_hip():+ gate = next(+ n+ for n in TREE.body+ if isinstance(n, ast.Assign) and getattr(n.targets[0], "id", None) == "_MULTIROW"+ )+ assert "version.hip" not in ast.unparse(gate.value)+ assert "_triton_at_least(3, 6)" in ast.unparse(gate.value)diff --git a/unsloth/kernels/rms_layernorm.py b/unsloth/kernels/rms_layernorm.pyindex 102547cdd8c..ac4b64bfcef 100644--- a/unsloth/kernels/rms_layernorm.py+++ b/unsloth/kernels/rms_layernorm.py@@ -258,8 +258,35 @@ def _fold_lanes(x, ROWS: tl.constexpr, WARPS: tl.constexpr, HALF: tl.constexpr):return tl.sum(tl.reshape(x, (ROWS, WARPS, 2, HALF)), axis = 2)+# ROCm: Triton's tl.sum reduces each 16-lane row of a wavefront with DPP row_shr 8, 4, 2, 1,+# then joins the rows (permlanex16 on RDNA, row_bcast 15 / 31 on CDNA), not CUDA's xor 16 .. 1.+_HIP_ROW_ORDER = tl.constexpr(bool(torch.version.hip))++@triton.jit-def _row_dot_in_row_kernel_order(a, b, ROWS: tl.constexpr, WARPS: tl.constexpr):+def _row_dot_in_hip_row_order(a, b, ROWS: tl.constexpr, WARPS: tl.constexpr):+ # Bit-exact replay of the one-row kernels' tl.sum on AMD (WARPS = BLOCK_SIZE // wavefront):+ # fma(a_l, b_l, a_(l-8) * b_(l-8)), row_shr 4, 2, 1, 16-lane rows by lane bit 4 then 5, warps.+ LANES: tl.constexpr = a.shape[1] // WARPS+ ROWS16: tl.constexpr = LANES // 16+ a = tl.permute(tl.reshape(a, (ROWS, WARPS, ROWS16, 2, 8)), (0, 1, 2, 4, 3))+ b = tl.permute(tl.reshape(b, (ROWS, WARPS, ROWS16, 2, 8)), (0, 1, 2, 4, 3))+ a_lo, a_hi = tl.split(a)+ b_lo, b_hi = tl.split(b)+ acc = tl.fma(a_hi, b_hi, a_lo * b_lo)+ acc = tl.sum(tl.reshape(acc, (ROWS, WARPS, ROWS16, 2, 4)), axis = 3)+ acc = tl.sum(tl.reshape(acc, (ROWS, WARPS, ROWS16, 2, 2)), axis = 3)+ acc = tl.sum(acc, axis = 3)+ if ROWS16 == 4:+ acc = tl.sum(tl.reshape(acc, (ROWS, WARPS, 2, 2)), axis = 3)+ acc = tl.sum(acc, axis = 2)+ if WARPS == 4:+ acc = tl.sum(tl.reshape(acc, (ROWS, 2, 2)), axis = 1)+ return tl.sum(acc, axis = 1)+++def _row_dot_in_cuda_row_order(a, b, ROWS: tl.constexpr, WARPS: tl.constexpr):# Bit-exact replay of the one-row kernels' tl.sum (4 warps, 64 <= BLOCK_SIZE <= 128):# fma(a_l, b_l, a_(l+16) * b_(l+16)), lane xor 8, 4, 2, 1, warp xor 2, 1; explicit steps only.a = tl.permute(tl.reshape(a, (ROWS, WARPS, 2, 16)), (0, 1, 3, 2))@@ -277,6 +304,15 @@ def _row_dot_in_row_kernel_order(a, b, ROWS: tl.constexpr, WARPS: tl.constexpr):return tl.sum(acc, axis = 1)+def _row_dot_in_row_kernel_order(a, b, ROWS: tl.constexpr, WARPS: tl.constexpr):+ if _HIP_ROW_ORDER:+ total = _row_dot_in_hip_row_order(a, b, ROWS, WARPS)+ else:+ total = _row_dot_in_cuda_row_order(a, b, ROWS, WARPS)+ return total++@triton.jitdef _rms_layernorm_forward_rows(Y,@@ -502,7 +538,7 @@ def _launch(wrap, kernel, grid, args, n_tensors, constexprs, device, **options):wrap(kernel)[grid](*args, **constexprs, **options)-# Off on ROCm (64-lane wavefronts reduce in another order) and before Triton 3.6 (slower backward).+# Off before Triton 3.6 (slower backward). ROCm replays the DPP order; the self-check gates it per arch.def _triton_at_least(major, minor):try:return tuple(int(v) for v in triton.__version__.split(".")[:2]) >= (major, minor)@@ -510,22 +546,43 @@ def _triton_at_least(major, minor):return False-_MULTIROW = (- os.environ.get("UNSLOTH_RMSNORM_MULTIROW", "1") != "0"- and not torch.version.hip- and _triton_at_least(3, 6)-)+_MULTIROW = os.environ.get("UNSLOTH_RMSNORM_MULTIROW", "1") != "0" and _triton_at_least(3, 6)_MULTIROW_ELEMENTS = 4096_MULTIROW_NUM_WARPS = 4-def _multirow_settings(n_cols):[email protected]_cache(maxsize = None)+def _hip_wavefront(index):+ # Lanes per wavefront the one-row kernels reduce over: 32 on RDNA, 64 on CDNA; None if unknown.+ try:+ with torch_gpu_device(torch.device("cuda", index)):+ lanes = int(triton.runtime.driver.active.get_current_target().warp_size)+ except Exception:+ try:+ lanes = int(torch.cuda.get_device_properties(index).warp_size)+ except Exception:+ return None+ return lanes if lanes in (32, 64) else None+++def _multirow_settings(n_cols, device = None):# Only widths 33..128 (one element per lane) can replay the one-row order; others stay one-row.if not _MULTIROW:return NoneBLOCK_SIZE, num_warps = _settings(n_cols)if num_warps != 4 or not (64 <= BLOCK_SIZE <= 128):return None+ if torch.version.hip:+ index = device.index if device is not None and device.index is not None else None+ lanes = _hip_wavefront(torch.cuda.current_device() if index is None else index)+ if lanes is None:+ return None+ return (+ BLOCK_SIZE,+ _MULTIROW_ELEMENTS // BLOCK_SIZE,+ BLOCK_SIZE // lanes,+ _MULTIROW_NUM_WARPS,+ )return BLOCK_SIZE, _MULTIROW_ELEMENTS // BLOCK_SIZE, BLOCK_SIZE // 32, _MULTIROW_NUM_WARPS@@ -577,7 +634,7 @@ def _multirow_self_check(device, dtype, W_dtype, n_cols, eps, gemma, multirow):def _multirow_checked(X, W, eps, gemma):n_cols = X.shape[1]- multirow = _multirow_settings(n_cols)+ multirow = _multirow_settings(n_cols, X.device)if multirow is None:return Nonekey = (X.device, X.dtype, W.dtype, int(n_cols), float(eps), gemma)