Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Run narrow RMSNorm rows several per program, bit-identical (Qwen3 q/k norms) (#12754)

Daniel Hanb945fd7e8c2d
diff --git a/tests/test_rms_layernorm_multirow.py b/tests/test_rms_layernorm_multirow.py
new file mode 100644
index 00000000000..515f056d967
--- /dev/null
+++ b/tests/test_rms_layernorm_multirow.py
@@ -0,0 +1,325 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
+
+"""Narrow-row RMSNorm (several rows per program) is byte-identical to the one-row kernels."""
+
+import pytest
+import torch
+
+from real_accelerator import has_real_cuda
+
+# The CUDA spoof patches torch.cuda probes to True process-wide; ask the pre-spoof answer.
+if not has_real_cuda():
+ pytest.skip("needs a CUDA device", allow_module_level = True)
+
+pytest.importorskip("triton")
+
+import unsloth # noqa: F401 (patches first)
+import torch._dynamo.utils as dynamo_utils
+from unsloth.kernels import rms_layernorm
+from unsloth.kernels.rms_layernorm import fast_rms_layernorm
+
+if torch.version.hip:
+ pytest.skip("the multi-row kernels are off on ROCm", allow_module_level = True)
+
+BF16_OK = torch.cuda.get_device_capability()[0] >= 8
+DTYPES = [torch.float16, torch.float32] + ([torch.bfloat16] if BF16_OK else [])
+NARROW = [33, 64, 96, 100, 127, 128]
+_eager = rms_layernorm._eager_kernel
+
+
+def multirow(monkeypatch):
+ monkeypatch.setattr(rms_layernorm, "_MULTIROW", True)
+ monkeypatch.setattr(rms_layernorm, "_MULTIROW_CHECKED", {})
+ return monkeypatch
+
+
+def _bits(t):
+ return t.contiguous().view({2: torch.int16, 4: torch.int32}[t.element_size()])
+
+
+def _inputs(
+ n_rows,
+ n_cols,
+ dtype,
+ kind,
+ seed = 0,
+):
+ g = torch.Generator(device = "cuda").manual_seed(seed)
+ X = torch.randn(n_rows, n_cols, device = "cuda", generator = g)
+ dY = torch.randn(n_rows, n_cols, device = "cuda", generator = g)
+ if kind == "wide":
+ # Magnitudes spread over 2^+-12 make every change in the order of adds visible.
+ span = lambda: torch.randint(-12, 13, (n_rows, n_cols), device = "cuda", generator = g)
+ X = X * torch.exp2(span().float())
+ dY = dY * torch.exp2(span().float())
+ elif kind == "offset":
+ X = X + 30.0
+ X, dY = X.to(dtype), dY.to(dtype)
+ if kind == "zero_rows":
+ # All-zero rows with zero upstream grads: dX must keep the one-row kernel's signed zeros.
+ X[::3] = 0
+ dY[::3, ::2] = 0
+ W = torch.randn(n_cols, device = "cuda", generator = g).to(dtype)
+ return X, W, dY
+
+
+def _run(
+ X,
+ W,
+ dY,
+ gemma,
+ rows = None,
+):
+ Y, r = rms_layernorm._rms_forward(X, W, 1e-6, gemma, _eager, rows)
+ dY = dY.clone()
+ dX = torch.empty_like(dY) if gemma else dY
+ rms_layernorm._rms_backward(dY, dX, X, W, r, 1e-6, gemma, _eager, rows)
+ return Y, r, dX
+
+
+def _both(monkeypatch, X, W, dY, gemma):
+ # Forced past the self-check: a mismatch there would fall back and compare one row to itself.
+ settings = rms_layernorm._multirow_settings(X.shape[1])
+ assert settings is not None
+ return _run(X, W, dY, gemma, False), _run(X, W, dY, gemma, settings)
+
+
[email protected]("kind", ["normal", "wide", "offset", "zero_rows"])
[email protected]("n_rows", [1, 3, 129, 4099])
[email protected]("n_cols", NARROW)
[email protected]("gemma", [False, True], ids = ["llama", "gemma"])
[email protected]("dtype", DTYPES, ids = str)
+def test_multirow_matches_one_row_kernel(multirow, dtype, gemma, n_cols, n_rows, kind):
+ """Y, the saved rstd and dX are bit-identical to one row per program."""
+ assert rms_layernorm._multirow_settings(n_cols) is not None
+ X, W, dY = _inputs(n_rows, n_cols, dtype, kind)
+ one_row, many_rows = _both(multirow, X, W, dY, gemma)
+ for a, b, name in zip(one_row, many_rows, ("Y", "r", "dX")):
+ assert torch.equal(_bits(a), _bits(b)), name
+
+
[email protected]("elements,num_warps", [(512, 1), (1024, 1), (4096, 4), (8192, 8)])
+def test_multirow_bits_do_not_depend_on_tile_shape(multirow, elements, num_warps):
+ """The row sum spells out every add, so other tile shapes give the same bytes."""
+ multirow.setattr(rms_layernorm, "_MULTIROW_ELEMENTS", elements)
+ multirow.setattr(rms_layernorm, "_MULTIROW_NUM_WARPS", num_warps)
+ dtype = torch.bfloat16 if BF16_OK else torch.float16
+ for gemma in (False, True):
+ X, W, dY = _inputs(2051, 128, dtype, "wide", seed = 1)
+ one_row, many_rows = _both(multirow, X, W, dY, gemma)
+ for a, b, name in zip(one_row, many_rows, ("Y", "r", "dX")):
+ assert torch.equal(_bits(a), _bits(b)), name
+
+
+def test_multirow_selection(multirow):
+ """Only rows of 33 to 128 columns (one element per lane in the one-row kernels) qualify."""
+ for n_cols in (1, 16, 32, 129, 256, 1024, 4096):
+ assert rms_layernorm._multirow_settings(n_cols) is None, n_cols
+ for n_cols in (33, 64, 128):
+ assert rms_layernorm._multirow_settings(n_cols) is not None, n_cols
+ multirow.setattr(rms_layernorm, "_MULTIROW", False)
+ assert rms_layernorm._multirow_settings(128) is None
+
+
+class _Norm(torch.nn.Module):
+ def __init__(self, dim, dtype):
+ super().__init__()
+ g = torch.Generator(device = "cuda").manual_seed(1)
+ self.weight = torch.nn.Parameter((torch.randn(dim, device = "cuda", generator = g)).to(dtype))
+ self.variance_epsilon = 1e-6
+
+
+def _qk_norm_case(fn, dtype, gemma):
+ g = torch.Generator(device = "cuda").manual_seed(0)
+ norm = _Norm(128, dtype)
+ X = torch.randn(2, 37, 16, 128, device = "cuda", generator = g).to(dtype).requires_grad_(True)
+ dY = torch.randn(2, 37, 16, 128, device = "cuda", generator = g).to(dtype)
+ Y = fn(norm, X, gemma)
+ Y.backward(dY)
+ return Y.detach(), X.grad
+
+
[email protected]("gemma", [False, True], ids = ["llama", "gemma"])
+def test_multirow_compiled_matches_one_row_eager(multirow, gemma):
+ """Under torch.compile (fullgraph, no graph break) the bytes match the one-row eager kernel."""
+ if not getattr(torch.library, "triton_op", None):
+ pytest.skip("needs torch.library.triton_op")
+ dtype = torch.bfloat16 if rms_layernorm._BF16_TRACEABLE else torch.float16
+ multirow.setattr(rms_layernorm, "_MULTIROW", False)
+ ref = _qk_norm_case(fast_rms_layernorm, dtype, gemma)
+ multirow.setattr(rms_layernorm, "_MULTIROW", True)
+ torch._dynamo.reset()
+ dynamo_utils.counters.clear()
+ compiled = torch.compile(lambda n, x, gm: fast_rms_layernorm(n, x, gm), fullgraph = True)
+ got = _qk_norm_case(compiled, dtype, gemma)
+ torch._dynamo.reset()
+ assert sum(dynamo_utils.counters["graph_break"].values()) == 0
+ assert list(rms_layernorm._MULTIROW_CHECKED.values()) == [True]
+ assert rms_layernorm._MULTIROW
+ for a, b, name in zip(ref, got, ("Y", "dX")):
+ assert torch.equal(_bits(a), _bits(b)), name
+
+
+class _Raising:
+ def __init__(self, error):
+ self.error = error
+
+ def __getitem__(self, grid):
+ def launch(*args, **kwargs):
+ raise self.error
+
+ return launch
+
+
+class _OffByOneUlp:
+ def __init__(self, kernel):
+ self.kernel = kernel
+
+ def __getitem__(self, grid):
+ def launch(Y, *args, **kwargs):
+ self.kernel[grid](Y, *args, **kwargs)
+ _bits(Y).view(-1)[0] ^= 1
+
+ return launch
+
+
+def _narrow_case(monkeypatch, gemma = False):
+ dtype = torch.bfloat16 if BF16_OK else torch.float16
+ X, W, dY = _inputs(129, 128, dtype, "wide", seed = 2)
+ monkeypatch.setattr(rms_layernorm, "_MULTIROW", False)
+ one_row = _run(X, W, dY, gemma)
+ monkeypatch.setattr(rms_layernorm, "_MULTIROW", True)
+ return X, W, dY, one_row
+
+
+def _assert_bits(one_row, got):
+ for a, b, name in zip(one_row, got, ("Y", "r", "dX")):
+ assert torch.equal(_bits(a), _bits(b)), name
+
+
[email protected]("kernel", ["_rms_layernorm_forward_rows", "_rms_layernorm_backward_rows"])
+def test_multirow_launch_failure_falls_back(multirow, kernel):
+ """A failed multi-row launch runs the one-row kernel and turns the lever off process-wide."""
+ X, W, dY, one_row = _narrow_case(multirow)
+ multirow.setitem(
+ rms_layernorm._MULTIROW_CHECKED, (X.device, X.dtype, W.dtype, 128, 1e-6, False), True
+ )
+ multirow.setattr(rms_layernorm, kernel, _Raising(RuntimeError("PTX JIT compilation failed")))
+ with pytest.warns(UserWarning, match = "launch failed"):
+ got = _run(X, W, dY, False)
+ assert not rms_layernorm._MULTIROW
+ _assert_bits(one_row, got)
+
+
+def test_multirow_self_check_runs_per_eps(multirow):
+ """eps is a constexpr, so every distinct eps gets its own self-check."""
+ X, W, dY, _ = _narrow_case(multirow)
+ for eps in (1e-6, 1e-5, 1e-6):
+ rms_layernorm._rms_forward(X, W, eps, False, _eager)
+ assert sorted(key[4] for key in rms_layernorm._MULTIROW_CHECKED) == [1e-6, 1e-5]
+ assert all(rms_layernorm._MULTIROW_CHECKED.values())
+
+
+def test_multirow_self_check_failure_under_compile_falls_back(multirow):
+ """The first call's self-check launches re-raise while tracing; it must not escape."""
+ X, W, dY, one_row = _narrow_case(multirow)
+ multirow.setattr(
+ rms_layernorm,
+ "_rms_layernorm_forward_rows",
+ _Raising(RuntimeError("PTX JIT compilation failed")),
+ )
+ multirow.setattr(torch.compiler, "is_compiling", lambda: True)
+ with pytest.warns(UserWarning, match = "self-check failed"):
+ Y, r = rms_layernorm._rms_forward(X, W, 1e-6, False, _eager)
+ assert not rms_layernorm._MULTIROW
+ assert torch.equal(_bits(one_row[0]), _bits(Y))
+
+
+ "error",
+ [
+ torch.cuda.OutOfMemoryError("CUDA out of memory"),
+ torch._dynamo.exc.TorchRuntimeError("traced"),
+ ],
+ ids = ["oom", "dynamo"],
+)
+def test_multirow_launch_failure_reraises(multirow, error):
+ """OOM and dynamo errors are not launch failures: they propagate and keep the lever on."""
+ X, W, dY, _ = _narrow_case(multirow)
+ multirow.setitem(
+ rms_layernorm._MULTIROW_CHECKED, (X.device, X.dtype, W.dtype, 128, 1e-6, False), True
+ )
+ multirow.setattr(rms_layernorm, "_rms_layernorm_forward_rows", _Raising(error))
+ with pytest.raises(type(error)):
+ _run(X, W, dY, False)
+ with pytest.raises(RuntimeError):
+ rms_layernorm._rms_forward(X, W, 1e-6, False, lambda kernel: _Raising(RuntimeError("x")))
+ assert rms_layernorm._MULTIROW
+
+
[email protected]("gemma", [False, True], ids = ["llama", "gemma"])
+def test_multirow_self_check_disables_on_mismatch(multirow, gemma):
+ """A multi-row kernel that differs by one ulp is caught before use: one-row bytes, lever off."""
+ X, W, dY, one_row = _narrow_case(multirow, gemma)
+ multirow.setattr(
+ rms_layernorm,
+ "_rms_layernorm_forward_rows",
+ _OffByOneUlp(rms_layernorm._rms_layernorm_forward_rows),
+ )
+ with pytest.warns(UserWarning, match = "self-check mismatch"):
+ got = _run(X, W, dY, gemma)
+ assert not rms_layernorm._MULTIROW
+ assert list(rms_layernorm._MULTIROW_CHECKED.values()) == [False]
+ _assert_bits(one_row, got)
+
+
+def test_multirow_self_check_runs_once(multirow):
+ """One check per (device, dtypes, width, variant); later calls only read the verdict."""
+ X, W, dY, one_row = _narrow_case(multirow)
+ calls = []
+ check = rms_layernorm._multirow_self_check
+ multirow.setattr(rms_layernorm, "_multirow_self_check", lambda *a: calls.append(a) or check(*a))
+ launches = []
+ rows_kernel = rms_layernorm._rms_layernorm_forward_rows
+
+ class _Counting:
+ def __getitem__(self, grid):
+ launches.append(grid)
+ return rows_kernel[grid]
+
+ multirow.setattr(rms_layernorm, "_rms_layernorm_forward_rows", _Counting())
+ for _ in range(3):
+ _assert_bits(one_row, _run(X, W, dY, False))
+ assert len(calls) == 1 and rms_layernorm._MULTIROW
+ assert len(launches) == 1 + 3
+
+
[email protected](not has_real_cuda() or torch.cuda.device_count() < 2, reason = "needs two GPUs")
[email protected]("path", ["eager", "compiled", "op"])
+def test_multirow_launches_on_the_tensors_device(multirow, path):
+ """With cuda:0 current, cuda:1 inputs give the one-row bytes (launches use the tensor's device)."""
+ if path != "eager" and not getattr(torch.library, "triton_op", None):
+ pytest.skip("needs torch.library.triton_op")
+ dtype = torch.bfloat16 if BF16_OK else torch.float16
+ g = torch.Generator(device = "cuda:1").manual_seed(0)
+ X = torch.randn(1184, 128, device = "cuda:1", generator = g).to(dtype)
+ W = torch.randn(128, device = "cuda:1", generator = g).to(dtype)
+ multirow.setattr(rms_layernorm, "_MULTIROW", False)
+ with torch.cuda.device(1):
+ ref, _ = rms_layernorm._rms_forward(X, W, 1e-6, False, _eager)
+ multirow.setattr(rms_layernorm, "_MULTIROW", True)
+ norm = torch.nn.Module()
+ norm.weight, norm.variance_epsilon = torch.nn.Parameter(W), 1e-6
+ with torch.cuda.device(0):
+ if path == "op":
+ got, _ = torch.ops.unsloth.rms_layernorm(X, W, 1e-6, False)
+ else:
+ fn = lambda n, x: fast_rms_layernorm(n, x)
+ got = (torch.compile(fn, fullgraph = True) if path == "compiled" else fn)(
+ norm, X
+ ).detach()
+ torch._dynamo.reset()
+ assert torch.equal(_bits(ref), _bits(got))
+ assert any(rms_layernorm._MULTIROW_CHECKED.values()) and rms_layernorm._MULTIROW
diff --git a/unsloth/kernels/rms_layernorm.py b/unsloth/kernels/rms_layernorm.py
index fa4940c5f49..d9ea06a3cce 100644
--- a/unsloth/kernels/rms_layernorm.py
+++ b/unsloth/kernels/rms_layernorm.py
@@ -11,6 +11,7 @@
import hashlib
import os
+import warnings
import triton
import triton.language as tl
import torch
@@ -157,11 +158,258 @@ def _gemma_rms_layernorm_forward(
tl.store(Y + col_offsets, output, mask = mask)
-def _rms_forward(X, W, eps, gemma, wrap):
- n_rows, n_cols = X.shape
+def _fold_lanes(x, ROWS: tl.constexpr, WARPS: tl.constexpr, HALF: tl.constexpr):
+ return tl.sum(tl.reshape(x, (ROWS, WARPS, 2, HALF)), axis = 2)
+
+
+def _row_dot_in_row_kernel_order(a, b, ROWS: tl.constexpr, WARPS: tl.constexpr):
+ # Bit-exact replay of the one-row kernels' tl.sum (4 warps, 64 <= BLOCK_SIZE <= 128):
+ # fma(a_l, b_l, a_(l+16) * b_(l+16)), lane xor 8, 4, 2, 1, warp xor 2, 1; explicit steps only.
+ a = tl.permute(tl.reshape(a, (ROWS, WARPS, 2, 16)), (0, 1, 3, 2))
+ b = tl.permute(tl.reshape(b, (ROWS, WARPS, 2, 16)), (0, 1, 3, 2))
+ a_lo, a_hi = tl.split(a)
+ b_lo, b_hi = tl.split(b)
+ acc = tl.fma(a_lo, b_lo, a_hi * b_hi)
+ acc = _fold_lanes(acc, ROWS, WARPS, 8)
+ acc = _fold_lanes(acc, ROWS, WARPS, 4)
+ acc = _fold_lanes(acc, ROWS, WARPS, 2)
+ acc = _fold_lanes(acc, ROWS, WARPS, 1)
+ acc = tl.reshape(acc, (ROWS, WARPS))
+ if WARPS == 4:
+ acc = tl.sum(tl.reshape(acc, (ROWS, 2, 2)), axis = 1)
+ return tl.sum(acc, axis = 1)
+
+
+def _rms_layernorm_forward_rows(
+ Y,
+ Y_row_stride: tl.constexpr,
+ X,
+ X_row_stride: tl.constexpr,
+ W,
+ r,
+ n_rows,
+ n_cols: tl.constexpr,
+ eps: tl.constexpr,
+ GEMMA: tl.constexpr,
+ BLOCK_SIZE: tl.constexpr,
+ ROWS: tl.constexpr,
+ WARPS: tl.constexpr,
+):
+ rows = tl.program_id(0).to(tl.int64) * ROWS + tl.arange(0, ROWS)
+ col_offsets = tl.arange(0, BLOCK_SIZE)
+ row_mask = rows < n_rows
+ col_mask = col_offsets < n_cols
+ mask = row_mask[:, None] & col_mask[None, :]
+
+ X_rows = tl.load(
+ X + rows[:, None] * X_row_stride + col_offsets[None, :], mask = mask, other = 0
+ ).to(tl.float32)
+ if GEMMA:
+ W_row = tl.load(W + col_offsets, mask = col_mask, other = 0).to(tl.float32)
+ else:
+ W_row = tl.load(W + col_offsets, mask = col_mask, other = 0)
+
+ row_var = _row_dot_in_row_kernel_order(X_rows, X_rows, ROWS, WARPS) / n_cols
+ eps_f32 = tl.full((), eps, tl.float32)
+ inv_var = tl.math.rsqrt(row_var + eps_f32)
+ tl.store(r + rows, inv_var, mask = row_mask)
+ normed = X_rows * inv_var[:, None]
+ if GEMMA:
+ output = normed * (W_row[None, :] + 1.0)
+ else:
+ normed = normed.to(W_row.dtype) # Exact copy from HF
+ output = normed * W_row[None, :]
+ tl.store(Y + rows[:, None] * Y_row_stride + col_offsets[None, :], output, mask = mask)
+
+
+def _rms_layernorm_backward_rows(
+ dY,
+ dY_row_stride: tl.constexpr,
+ dX,
+ dX_row_stride: tl.constexpr,
+ X,
+ X_row_stride: tl.constexpr,
+ W,
+ r,
+ n_rows,
+ n_cols: tl.constexpr,
+ eps: tl.constexpr,
+ GEMMA: tl.constexpr,
+ BLOCK_SIZE: tl.constexpr,
+ ROWS: tl.constexpr,
+ WARPS: tl.constexpr,
+):
+ rows = tl.program_id(0).to(tl.int64) * ROWS + tl.arange(0, ROWS)
+ col_offsets = tl.arange(0, BLOCK_SIZE)
+ row_mask = rows < n_rows
+ col_mask = col_offsets < n_cols
+ mask = row_mask[:, None] & col_mask[None, :]
+
+ dY_rows = tl.load(
+ dY + rows[:, None] * dY_row_stride + col_offsets[None, :], mask = mask, other = 0
+ ).to(tl.float32)
+ X_rows = tl.load(
+ X + rows[:, None] * X_row_stride + col_offsets[None, :], mask = mask, other = 0
+ ).to(tl.float32)
+ W_row = tl.load(W + col_offsets, mask = col_mask, other = 0).to(tl.float32)
+
+ inv_var = tl.load(r + rows, mask = row_mask, other = 0).to(tl.float32)
+ normed = X_rows * inv_var[:, None]
+
+ if GEMMA:
+ dY_W = dY_rows * (W_row[None, :] + 1.0)
+ else:
+ dY_W = dY_rows * W_row[None, :]
+
+ rowsum_dY_normed = _row_dot_in_row_kernel_order(dY_W, normed, ROWS, WARPS)
+ # Spelled as in the one-row kernel: an explicit fma here turns -0 into +0 on sm75.
+ output = (inv_var / n_cols)[:, None] * (n_cols * dY_W - normed * rowsum_dY_normed[:, None])
+ if GEMMA:
+ tl.store(dX + rows[:, None] * dX_row_stride + col_offsets[None, :], output, mask = mask)
+ else:
+ tl.store(dY + rows[:, None] * dY_row_stride + col_offsets[None, :], output, mask = mask)
+
+
+# Off on ROCm (64-lane wavefronts reduce in another order) and before Triton 3.6 (slower backward).
+def _triton_at_least(major, minor):
+ try:
+ return tuple(int(v) for v in triton.__version__.split(".")[:2]) >= (major, minor)
+ except Exception:
+ return False
+
+
+_MULTIROW = (
+ os.environ.get("UNSLOTH_RMSNORM_MULTIROW", "1") != "0"
+ and not torch.version.hip
+ and _triton_at_least(3, 6)
+)
+_MULTIROW_ELEMENTS = 4096
+_MULTIROW_NUM_WARPS = 4
+
+
+def _multirow_settings(n_cols):
+ # Only widths 33..128 (one element per lane) can replay the one-row order; others stay one-row.
+ if not _MULTIROW:
+ return None
BLOCK_SIZE, num_warps = calculate_settings(n_cols)
+ if num_warps != 4 or not (64 <= BLOCK_SIZE <= 128):
+ return None
+ return BLOCK_SIZE, _MULTIROW_ELEMENTS // BLOCK_SIZE, BLOCK_SIZE // 32, _MULTIROW_NUM_WARPS
+
+
+_MULTIROW_CHECKED = {}
+
+
+def _multirow_disable(reason):
+ global _MULTIROW
+ if _MULTIROW:
+ _MULTIROW = False
+ warnings.warn(
+ f"Unsloth: narrow-row RMSNorm kernels disabled ({reason}); using one row per program."
+ )
+
+
+def _multirow_must_raise(error, wrap):
+ # Never fall back while torch.compile traces, nor on OOM or dynamo / inductor errors.
+ return (
+ wrap is not _eager_kernel
+ or isinstance(error, torch.cuda.OutOfMemoryError)
+ or type(error).__module__.startswith(("torch._dynamo", "torch._inductor"))
+ or torch.compiler.is_compiling()
+ )
+
+
+def _bits(t):
+ return t.view({2: torch.int16, 4: torch.int32}[t.element_size()])
+
+
+def _multirow_self_check(device, dtype, W_dtype, n_cols, eps, gemma, multirow):
+ from torch.utils._python_dispatch import _disable_current_modes
+ with _disable_current_modes(), torch.no_grad(), torch_gpu_device(device):
+ g = torch.Generator(device = device).manual_seed(3407)
+ shape = (2 * multirow[1] + 3, n_cols)
+ X = torch.randn(shape, device = device, generator = g)
+ X = X * torch.exp2(torch.randint(-12, 13, shape, device = device, generator = g).float())
+ dY = torch.randn(shape, device = device, generator = g)
+ X[0], dY[0, ::2] = 0, 0 # all-zero row: signed zeros in dX
+ X, dY = X.to(dtype), dY.to(dtype)
+ W = torch.randn(n_cols, device = device, generator = g).to(W_dtype)
+ results = []
+ for rows in (False, multirow):
+ Y, r = _rms_forward(X, W, eps, gemma, _eager_kernel, rows)
+ dX = dY.clone()
+ _rms_backward(dY.clone() if gemma else dX, dX, X, W, r, eps, gemma, _eager_kernel, rows)
+ results.append((Y, r, dX))
+ return all(torch.equal(_bits(a), _bits(b)) for a, b in zip(*results))
+
+
+def _multirow_checked(X, W, eps, gemma):
+ n_cols = X.shape[1]
+ multirow = _multirow_settings(n_cols)
+ if multirow is None:
+ return None
+ key = (X.device, X.dtype, W.dtype, int(n_cols), float(eps), gemma)
+ verdict = _MULTIROW_CHECKED.get(key)
+ if verdict is None:
+ if X.device.type == "cuda" and torch.cuda.is_current_stream_capturing():
+ return None # check on a later uncaptured call
+ try:
+ verdict = _multirow_self_check(X.device, X.dtype, W.dtype, n_cols, eps, gemma, multirow)
+ reason = f"self-check mismatch for {key}"
+ except torch.cuda.OutOfMemoryError:
+ raise
+ except Exception as error:
+ # Its launches re-raise while torch.compile traces; this call then runs one row.
+ verdict, reason = False, f"self-check failed for {key}: {error!r}"
+ _MULTIROW_CHECKED[key] = verdict
+ if not verdict:
+ _multirow_disable(reason)
+ return multirow if verdict and _MULTIROW else None
+
+
+def _rms_forward(
+ X,
+ W,
+ eps,
+ gemma,
+ wrap,
+ multirow = None,
+):
+ # multirow: None picks (self-checked), False forces one row, settings force multi-row.
+ n_rows, n_cols = X.shape
Y = torch.empty((n_rows, n_cols), dtype = X.dtype, device = X.device)
r = torch.empty(n_rows, dtype = torch.float32, device = X.device)
+ if multirow is None:
+ multirow = _multirow_checked(X, W, eps, gemma)
+ if multirow:
+ BLOCK_SIZE, ROWS, WARPS, num_warps = multirow
+ try:
+ wrap(_rms_layernorm_forward_rows)[((n_rows + ROWS - 1) // ROWS,)](
+ Y,
+ Y.stride(0),
+ X,
+ X.stride(0),
+ W,
+ r,
+ n_rows,
+ n_cols,
+ eps,
+ GEMMA = gemma,
+ BLOCK_SIZE = BLOCK_SIZE,
+ ROWS = ROWS,
+ WARPS = WARPS,
+ num_warps = num_warps,
+ )
+ return Y, r
+ except Exception as error:
+ if _multirow_must_raise(error, wrap):
+ raise
+ _multirow_disable(f"launch failed: {error!r}")
+ BLOCK_SIZE, num_warps = calculate_settings(n_cols)
fx = _gemma_rms_layernorm_forward if gemma else _rms_layernorm_forward
wrap(fx)[(n_rows,)](
Y,
@@ -180,9 +428,47 @@ def _rms_forward(X, W, eps, gemma, wrap):
return Y, r
-def _rms_backward(dY, dX, X, W, r, eps, gemma, wrap):
+def _rms_backward(
+ dY,
+ dX,
+ X,
+ W,
+ r,
+ eps,
+ gemma,
+ wrap,
+ multirow = None,
+):
# Non-Gemma writes dX over dY (the kernel ignores dX); Gemma writes into dX.
n_rows, n_cols = dY.shape
+ if multirow is None:
+ multirow = _multirow_checked(X, W, eps, gemma)
+ if multirow:
+ BLOCK_SIZE, ROWS, WARPS, num_warps = multirow
+ try:
+ wrap(_rms_layernorm_backward_rows)[((n_rows + ROWS - 1) // ROWS,)](
+ dY,
+ dY.stride(0),
+ dX,
+ dX.stride(0),
+ X,
+ X.stride(0),
+ W,
+ r,
+ n_rows,
+ n_cols,
+ eps,
+ GEMMA = gemma,
+ BLOCK_SIZE = BLOCK_SIZE,
+ ROWS = ROWS,
+ WARPS = WARPS,
+ num_warps = num_warps,
+ )
+ return
+ except Exception as error:
+ if _multirow_must_raise(error, wrap):
+ raise
+ _multirow_disable(f"launch failed: {error!r}")
BLOCK_SIZE, num_warps = calculate_settings(n_cols)
wrap(_rms_layernorm_backward)[(n_rows,)](
dY,
@@ -283,7 +569,9 @@ if _TRACEABLE:
def _rms_layernorm_op(
X: torch.Tensor, W: torch.Tensor, eps: float, gemma: bool
) -> Tuple[torch.Tensor, torch.Tensor]:
- return _rms_forward(X, W, eps, gemma, _traced_kernel)
+ # Called eagerly (outside a compiled graph) the launch needs X's device made current.
+ with torch_gpu_device(X.device):
+ return _rms_forward(X, W, eps, gemma, _traced_kernel)
@torch.library.triton_op("unsloth::rms_layernorm_backward", mutates_args = ())
def _rms_layernorm_backward_op(
@@ -291,7 +579,8 @@ if _TRACEABLE:
) -> torch.Tensor:
dX = torch.empty_like(dY)
# Non-Gemma rewrites its dY argument, so it gets a copy.
- _rms_backward(dY if gemma else dX.copy_(dY), dX, X, W, r, eps, gemma, _traced_kernel)
+ with torch_gpu_device(dY.device):
+ _rms_backward(dY if gemma else dX.copy_(dY), dX, X, W, r, eps, gemma, _traced_kernel)
return dX
def _rms_setup_context(ctx, inputs, output):