Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Run narrow RMSNorm rows several per program, bit-identical (Qwen3 q/k norms) (#12754)
Daniel Hanb945fd7e8c2d
diff --git a/tests/test_rms_layernorm_multirow.py b/tests/test_rms_layernorm_multirow.pynew file mode 100644index 00000000000..515f056d967--- /dev/null+++ b/tests/test_rms_layernorm_multirow.py@@ -0,0 +1,325 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.++"""Narrow-row RMSNorm (several rows per program) is byte-identical to the one-row kernels."""++import pytest+import torch++from real_accelerator import has_real_cuda++# The CUDA spoof patches torch.cuda probes to True process-wide; ask the pre-spoof answer.+if not has_real_cuda():+ pytest.skip("needs a CUDA device", allow_module_level = True)++pytest.importorskip("triton")++import unsloth # noqa: F401 (patches first)+import torch._dynamo.utils as dynamo_utils+from unsloth.kernels import rms_layernorm+from unsloth.kernels.rms_layernorm import fast_rms_layernorm++if torch.version.hip:+ pytest.skip("the multi-row kernels are off on ROCm", allow_module_level = True)++BF16_OK = torch.cuda.get_device_capability()[0] >= 8+DTYPES = [torch.float16, torch.float32] + ([torch.bfloat16] if BF16_OK else [])+NARROW = [33, 64, 96, 100, 127, 128]+_eager = rms_layernorm._eager_kernel+++def multirow(monkeypatch):+ monkeypatch.setattr(rms_layernorm, "_MULTIROW", True)+ monkeypatch.setattr(rms_layernorm, "_MULTIROW_CHECKED", {})+ return monkeypatch+++def _bits(t):+ return t.contiguous().view({2: torch.int16, 4: torch.int32}[t.element_size()])+++def _inputs(+ n_rows,+ n_cols,+ dtype,+ kind,+ seed = 0,+):+ g = torch.Generator(device = "cuda").manual_seed(seed)+ X = torch.randn(n_rows, n_cols, device = "cuda", generator = g)+ dY = torch.randn(n_rows, n_cols, device = "cuda", generator = g)+ if kind == "wide":+ # Magnitudes spread over 2^+-12 make every change in the order of adds visible.+ span = lambda: torch.randint(-12, 13, (n_rows, n_cols), device = "cuda", generator = g)+ X = X * torch.exp2(span().float())+ dY = dY * torch.exp2(span().float())+ elif kind == "offset":+ X = X + 30.0+ X, dY = X.to(dtype), dY.to(dtype)+ if kind == "zero_rows":+ # All-zero rows with zero upstream grads: dX must keep the one-row kernel's signed zeros.+ X[::3] = 0+ dY[::3, ::2] = 0+ W = torch.randn(n_cols, device = "cuda", generator = g).to(dtype)+ return X, W, dY+++def _run(+ X,+ W,+ dY,+ gemma,+ rows = None,+):+ Y, r = rms_layernorm._rms_forward(X, W, 1e-6, gemma, _eager, rows)+ dY = dY.clone()+ dX = torch.empty_like(dY) if gemma else dY+ rms_layernorm._rms_backward(dY, dX, X, W, r, 1e-6, gemma, _eager, rows)+ return Y, r, dX+++def _both(monkeypatch, X, W, dY, gemma):+ # Forced past the self-check: a mismatch there would fall back and compare one row to itself.+ settings = rms_layernorm._multirow_settings(X.shape[1])+ assert settings is not None+ return _run(X, W, dY, gemma, False), _run(X, W, dY, gemma, settings)++[email protected]("kind", ["normal", "wide", "offset", "zero_rows"])[email protected]("n_rows", [1, 3, 129, 4099])[email protected]("n_cols", NARROW)[email protected]("gemma", [False, True], ids = ["llama", "gemma"])[email protected]("dtype", DTYPES, ids = str)+def test_multirow_matches_one_row_kernel(multirow, dtype, gemma, n_cols, n_rows, kind):+ """Y, the saved rstd and dX are bit-identical to one row per program."""+ assert rms_layernorm._multirow_settings(n_cols) is not None+ X, W, dY = _inputs(n_rows, n_cols, dtype, kind)+ one_row, many_rows = _both(multirow, X, W, dY, gemma)+ for a, b, name in zip(one_row, many_rows, ("Y", "r", "dX")):+ assert torch.equal(_bits(a), _bits(b)), name++[email protected]("elements,num_warps", [(512, 1), (1024, 1), (4096, 4), (8192, 8)])+def test_multirow_bits_do_not_depend_on_tile_shape(multirow, elements, num_warps):+ """The row sum spells out every add, so other tile shapes give the same bytes."""+ multirow.setattr(rms_layernorm, "_MULTIROW_ELEMENTS", elements)+ multirow.setattr(rms_layernorm, "_MULTIROW_NUM_WARPS", num_warps)+ dtype = torch.bfloat16 if BF16_OK else torch.float16+ for gemma in (False, True):+ X, W, dY = _inputs(2051, 128, dtype, "wide", seed = 1)+ one_row, many_rows = _both(multirow, X, W, dY, gemma)+ for a, b, name in zip(one_row, many_rows, ("Y", "r", "dX")):+ assert torch.equal(_bits(a), _bits(b)), name+++def test_multirow_selection(multirow):+ """Only rows of 33 to 128 columns (one element per lane in the one-row kernels) qualify."""+ for n_cols in (1, 16, 32, 129, 256, 1024, 4096):+ assert rms_layernorm._multirow_settings(n_cols) is None, n_cols+ for n_cols in (33, 64, 128):+ assert rms_layernorm._multirow_settings(n_cols) is not None, n_cols+ multirow.setattr(rms_layernorm, "_MULTIROW", False)+ assert rms_layernorm._multirow_settings(128) is None+++class _Norm(torch.nn.Module):+ def __init__(self, dim, dtype):+ super().__init__()+ g = torch.Generator(device = "cuda").manual_seed(1)+ self.weight = torch.nn.Parameter((torch.randn(dim, device = "cuda", generator = g)).to(dtype))+ self.variance_epsilon = 1e-6+++def _qk_norm_case(fn, dtype, gemma):+ g = torch.Generator(device = "cuda").manual_seed(0)+ norm = _Norm(128, dtype)+ X = torch.randn(2, 37, 16, 128, device = "cuda", generator = g).to(dtype).requires_grad_(True)+ dY = torch.randn(2, 37, 16, 128, device = "cuda", generator = g).to(dtype)+ Y = fn(norm, X, gemma)+ Y.backward(dY)+ return Y.detach(), X.grad++[email protected]("gemma", [False, True], ids = ["llama", "gemma"])+def test_multirow_compiled_matches_one_row_eager(multirow, gemma):+ """Under torch.compile (fullgraph, no graph break) the bytes match the one-row eager kernel."""+ if not getattr(torch.library, "triton_op", None):+ pytest.skip("needs torch.library.triton_op")+ dtype = torch.bfloat16 if rms_layernorm._BF16_TRACEABLE else torch.float16+ multirow.setattr(rms_layernorm, "_MULTIROW", False)+ ref = _qk_norm_case(fast_rms_layernorm, dtype, gemma)+ multirow.setattr(rms_layernorm, "_MULTIROW", True)+ torch._dynamo.reset()+ dynamo_utils.counters.clear()+ compiled = torch.compile(lambda n, x, gm: fast_rms_layernorm(n, x, gm), fullgraph = True)+ got = _qk_norm_case(compiled, dtype, gemma)+ torch._dynamo.reset()+ assert sum(dynamo_utils.counters["graph_break"].values()) == 0+ assert list(rms_layernorm._MULTIROW_CHECKED.values()) == [True]+ assert rms_layernorm._MULTIROW+ for a, b, name in zip(ref, got, ("Y", "dX")):+ assert torch.equal(_bits(a), _bits(b)), name+++class _Raising:+ def __init__(self, error):+ self.error = error++ def __getitem__(self, grid):+ def launch(*args, **kwargs):+ raise self.error++ return launch+++class _OffByOneUlp:+ def __init__(self, kernel):+ self.kernel = kernel++ def __getitem__(self, grid):+ def launch(Y, *args, **kwargs):+ self.kernel[grid](Y, *args, **kwargs)+ _bits(Y).view(-1)[0] ^= 1++ return launch+++def _narrow_case(monkeypatch, gemma = False):+ dtype = torch.bfloat16 if BF16_OK else torch.float16+ X, W, dY = _inputs(129, 128, dtype, "wide", seed = 2)+ monkeypatch.setattr(rms_layernorm, "_MULTIROW", False)+ one_row = _run(X, W, dY, gemma)+ monkeypatch.setattr(rms_layernorm, "_MULTIROW", True)+ return X, W, dY, one_row+++def _assert_bits(one_row, got):+ for a, b, name in zip(one_row, got, ("Y", "r", "dX")):+ assert torch.equal(_bits(a), _bits(b)), name++[email protected]("kernel", ["_rms_layernorm_forward_rows", "_rms_layernorm_backward_rows"])+def test_multirow_launch_failure_falls_back(multirow, kernel):+ """A failed multi-row launch runs the one-row kernel and turns the lever off process-wide."""+ X, W, dY, one_row = _narrow_case(multirow)+ multirow.setitem(+ rms_layernorm._MULTIROW_CHECKED, (X.device, X.dtype, W.dtype, 128, 1e-6, False), True+ )+ multirow.setattr(rms_layernorm, kernel, _Raising(RuntimeError("PTX JIT compilation failed")))+ with pytest.warns(UserWarning, match = "launch failed"):+ got = _run(X, W, dY, False)+ assert not rms_layernorm._MULTIROW+ _assert_bits(one_row, got)+++def test_multirow_self_check_runs_per_eps(multirow):+ """eps is a constexpr, so every distinct eps gets its own self-check."""+ X, W, dY, _ = _narrow_case(multirow)+ for eps in (1e-6, 1e-5, 1e-6):+ rms_layernorm._rms_forward(X, W, eps, False, _eager)+ assert sorted(key[4] for key in rms_layernorm._MULTIROW_CHECKED) == [1e-6, 1e-5]+ assert all(rms_layernorm._MULTIROW_CHECKED.values())+++def test_multirow_self_check_failure_under_compile_falls_back(multirow):+ """The first call's self-check launches re-raise while tracing; it must not escape."""+ X, W, dY, one_row = _narrow_case(multirow)+ multirow.setattr(+ rms_layernorm,+ "_rms_layernorm_forward_rows",+ _Raising(RuntimeError("PTX JIT compilation failed")),+ )+ multirow.setattr(torch.compiler, "is_compiling", lambda: True)+ with pytest.warns(UserWarning, match = "self-check failed"):+ Y, r = rms_layernorm._rms_forward(X, W, 1e-6, False, _eager)+ assert not rms_layernorm._MULTIROW+ assert torch.equal(_bits(one_row[0]), _bits(Y))+++ "error",+ [+ torch.cuda.OutOfMemoryError("CUDA out of memory"),+ torch._dynamo.exc.TorchRuntimeError("traced"),+ ],+ ids = ["oom", "dynamo"],+)+def test_multirow_launch_failure_reraises(multirow, error):+ """OOM and dynamo errors are not launch failures: they propagate and keep the lever on."""+ X, W, dY, _ = _narrow_case(multirow)+ multirow.setitem(+ rms_layernorm._MULTIROW_CHECKED, (X.device, X.dtype, W.dtype, 128, 1e-6, False), True+ )+ multirow.setattr(rms_layernorm, "_rms_layernorm_forward_rows", _Raising(error))+ with pytest.raises(type(error)):+ _run(X, W, dY, False)+ with pytest.raises(RuntimeError):+ rms_layernorm._rms_forward(X, W, 1e-6, False, lambda kernel: _Raising(RuntimeError("x")))+ assert rms_layernorm._MULTIROW++[email protected]("gemma", [False, True], ids = ["llama", "gemma"])+def test_multirow_self_check_disables_on_mismatch(multirow, gemma):+ """A multi-row kernel that differs by one ulp is caught before use: one-row bytes, lever off."""+ X, W, dY, one_row = _narrow_case(multirow, gemma)+ multirow.setattr(+ rms_layernorm,+ "_rms_layernorm_forward_rows",+ _OffByOneUlp(rms_layernorm._rms_layernorm_forward_rows),+ )+ with pytest.warns(UserWarning, match = "self-check mismatch"):+ got = _run(X, W, dY, gemma)+ assert not rms_layernorm._MULTIROW+ assert list(rms_layernorm._MULTIROW_CHECKED.values()) == [False]+ _assert_bits(one_row, got)+++def test_multirow_self_check_runs_once(multirow):+ """One check per (device, dtypes, width, variant); later calls only read the verdict."""+ X, W, dY, one_row = _narrow_case(multirow)+ calls = []+ check = rms_layernorm._multirow_self_check+ multirow.setattr(rms_layernorm, "_multirow_self_check", lambda *a: calls.append(a) or check(*a))+ launches = []+ rows_kernel = rms_layernorm._rms_layernorm_forward_rows++ class _Counting:+ def __getitem__(self, grid):+ launches.append(grid)+ return rows_kernel[grid]++ multirow.setattr(rms_layernorm, "_rms_layernorm_forward_rows", _Counting())+ for _ in range(3):+ _assert_bits(one_row, _run(X, W, dY, False))+ assert len(calls) == 1 and rms_layernorm._MULTIROW+ assert len(launches) == 1 + 3++[email protected](not has_real_cuda() or torch.cuda.device_count() < 2, reason = "needs two GPUs")[email protected]("path", ["eager", "compiled", "op"])+def test_multirow_launches_on_the_tensors_device(multirow, path):+ """With cuda:0 current, cuda:1 inputs give the one-row bytes (launches use the tensor's device)."""+ if path != "eager" and not getattr(torch.library, "triton_op", None):+ pytest.skip("needs torch.library.triton_op")+ dtype = torch.bfloat16 if BF16_OK else torch.float16+ g = torch.Generator(device = "cuda:1").manual_seed(0)+ X = torch.randn(1184, 128, device = "cuda:1", generator = g).to(dtype)+ W = torch.randn(128, device = "cuda:1", generator = g).to(dtype)+ multirow.setattr(rms_layernorm, "_MULTIROW", False)+ with torch.cuda.device(1):+ ref, _ = rms_layernorm._rms_forward(X, W, 1e-6, False, _eager)+ multirow.setattr(rms_layernorm, "_MULTIROW", True)+ norm = torch.nn.Module()+ norm.weight, norm.variance_epsilon = torch.nn.Parameter(W), 1e-6+ with torch.cuda.device(0):+ if path == "op":+ got, _ = torch.ops.unsloth.rms_layernorm(X, W, 1e-6, False)+ else:+ fn = lambda n, x: fast_rms_layernorm(n, x)+ got = (torch.compile(fn, fullgraph = True) if path == "compiled" else fn)(+ norm, X+ ).detach()+ torch._dynamo.reset()+ assert torch.equal(_bits(ref), _bits(got))+ assert any(rms_layernorm._MULTIROW_CHECKED.values()) and rms_layernorm._MULTIROWdiff --git a/unsloth/kernels/rms_layernorm.py b/unsloth/kernels/rms_layernorm.pyindex fa4940c5f49..d9ea06a3cce 100644--- a/unsloth/kernels/rms_layernorm.py+++ b/unsloth/kernels/rms_layernorm.py@@ -11,6 +11,7 @@import hashlibimport os+import warningsimport tritonimport triton.language as tlimport torch@@ -157,11 +158,258 @@ def _gemma_rms_layernorm_forward(tl.store(Y + col_offsets, output, mask = mask)-def _rms_forward(X, W, eps, gemma, wrap):- n_rows, n_cols = X.shape+def _fold_lanes(x, ROWS: tl.constexpr, WARPS: tl.constexpr, HALF: tl.constexpr):+ return tl.sum(tl.reshape(x, (ROWS, WARPS, 2, HALF)), axis = 2)+++def _row_dot_in_row_kernel_order(a, b, ROWS: tl.constexpr, WARPS: tl.constexpr):+ # Bit-exact replay of the one-row kernels' tl.sum (4 warps, 64 <= BLOCK_SIZE <= 128):+ # fma(a_l, b_l, a_(l+16) * b_(l+16)), lane xor 8, 4, 2, 1, warp xor 2, 1; explicit steps only.+ a = tl.permute(tl.reshape(a, (ROWS, WARPS, 2, 16)), (0, 1, 3, 2))+ b = tl.permute(tl.reshape(b, (ROWS, WARPS, 2, 16)), (0, 1, 3, 2))+ a_lo, a_hi = tl.split(a)+ b_lo, b_hi = tl.split(b)+ acc = tl.fma(a_lo, b_lo, a_hi * b_hi)+ acc = _fold_lanes(acc, ROWS, WARPS, 8)+ acc = _fold_lanes(acc, ROWS, WARPS, 4)+ acc = _fold_lanes(acc, ROWS, WARPS, 2)+ acc = _fold_lanes(acc, ROWS, WARPS, 1)+ acc = tl.reshape(acc, (ROWS, WARPS))+ if WARPS == 4:+ acc = tl.sum(tl.reshape(acc, (ROWS, 2, 2)), axis = 1)+ return tl.sum(acc, axis = 1)+++def _rms_layernorm_forward_rows(+ Y,+ Y_row_stride: tl.constexpr,+ X,+ X_row_stride: tl.constexpr,+ W,+ r,+ n_rows,+ n_cols: tl.constexpr,+ eps: tl.constexpr,+ GEMMA: tl.constexpr,+ BLOCK_SIZE: tl.constexpr,+ ROWS: tl.constexpr,+ WARPS: tl.constexpr,+):+ rows = tl.program_id(0).to(tl.int64) * ROWS + tl.arange(0, ROWS)+ col_offsets = tl.arange(0, BLOCK_SIZE)+ row_mask = rows < n_rows+ col_mask = col_offsets < n_cols+ mask = row_mask[:, None] & col_mask[None, :]++ X_rows = tl.load(+ X + rows[:, None] * X_row_stride + col_offsets[None, :], mask = mask, other = 0+ ).to(tl.float32)+ if GEMMA:+ W_row = tl.load(W + col_offsets, mask = col_mask, other = 0).to(tl.float32)+ else:+ W_row = tl.load(W + col_offsets, mask = col_mask, other = 0)++ row_var = _row_dot_in_row_kernel_order(X_rows, X_rows, ROWS, WARPS) / n_cols+ eps_f32 = tl.full((), eps, tl.float32)+ inv_var = tl.math.rsqrt(row_var + eps_f32)+ tl.store(r + rows, inv_var, mask = row_mask)+ normed = X_rows * inv_var[:, None]+ if GEMMA:+ output = normed * (W_row[None, :] + 1.0)+ else:+ normed = normed.to(W_row.dtype) # Exact copy from HF+ output = normed * W_row[None, :]+ tl.store(Y + rows[:, None] * Y_row_stride + col_offsets[None, :], output, mask = mask)+++def _rms_layernorm_backward_rows(+ dY,+ dY_row_stride: tl.constexpr,+ dX,+ dX_row_stride: tl.constexpr,+ X,+ X_row_stride: tl.constexpr,+ W,+ r,+ n_rows,+ n_cols: tl.constexpr,+ eps: tl.constexpr,+ GEMMA: tl.constexpr,+ BLOCK_SIZE: tl.constexpr,+ ROWS: tl.constexpr,+ WARPS: tl.constexpr,+):+ rows = tl.program_id(0).to(tl.int64) * ROWS + tl.arange(0, ROWS)+ col_offsets = tl.arange(0, BLOCK_SIZE)+ row_mask = rows < n_rows+ col_mask = col_offsets < n_cols+ mask = row_mask[:, None] & col_mask[None, :]++ dY_rows = tl.load(+ dY + rows[:, None] * dY_row_stride + col_offsets[None, :], mask = mask, other = 0+ ).to(tl.float32)+ X_rows = tl.load(+ X + rows[:, None] * X_row_stride + col_offsets[None, :], mask = mask, other = 0+ ).to(tl.float32)+ W_row = tl.load(W + col_offsets, mask = col_mask, other = 0).to(tl.float32)++ inv_var = tl.load(r + rows, mask = row_mask, other = 0).to(tl.float32)+ normed = X_rows * inv_var[:, None]++ if GEMMA:+ dY_W = dY_rows * (W_row[None, :] + 1.0)+ else:+ dY_W = dY_rows * W_row[None, :]++ rowsum_dY_normed = _row_dot_in_row_kernel_order(dY_W, normed, ROWS, WARPS)+ # Spelled as in the one-row kernel: an explicit fma here turns -0 into +0 on sm75.+ output = (inv_var / n_cols)[:, None] * (n_cols * dY_W - normed * rowsum_dY_normed[:, None])+ if GEMMA:+ tl.store(dX + rows[:, None] * dX_row_stride + col_offsets[None, :], output, mask = mask)+ else:+ tl.store(dY + rows[:, None] * dY_row_stride + col_offsets[None, :], output, mask = mask)+++# Off on ROCm (64-lane wavefronts reduce in another order) and before Triton 3.6 (slower backward).+def _triton_at_least(major, minor):+ try:+ return tuple(int(v) for v in triton.__version__.split(".")[:2]) >= (major, minor)+ except Exception:+ return False+++_MULTIROW = (+ os.environ.get("UNSLOTH_RMSNORM_MULTIROW", "1") != "0"+ and not torch.version.hip+ and _triton_at_least(3, 6)+)+_MULTIROW_ELEMENTS = 4096+_MULTIROW_NUM_WARPS = 4+++def _multirow_settings(n_cols):+ # Only widths 33..128 (one element per lane) can replay the one-row order; others stay one-row.+ if not _MULTIROW:+ return NoneBLOCK_SIZE, num_warps = calculate_settings(n_cols)+ if num_warps != 4 or not (64 <= BLOCK_SIZE <= 128):+ return None+ return BLOCK_SIZE, _MULTIROW_ELEMENTS // BLOCK_SIZE, BLOCK_SIZE // 32, _MULTIROW_NUM_WARPS+++_MULTIROW_CHECKED = {}+++def _multirow_disable(reason):+ global _MULTIROW+ if _MULTIROW:+ _MULTIROW = False+ warnings.warn(+ f"Unsloth: narrow-row RMSNorm kernels disabled ({reason}); using one row per program."+ )+++def _multirow_must_raise(error, wrap):+ # Never fall back while torch.compile traces, nor on OOM or dynamo / inductor errors.+ return (+ wrap is not _eager_kernel+ or isinstance(error, torch.cuda.OutOfMemoryError)+ or type(error).__module__.startswith(("torch._dynamo", "torch._inductor"))+ or torch.compiler.is_compiling()+ )+++def _bits(t):+ return t.view({2: torch.int16, 4: torch.int32}[t.element_size()])+++def _multirow_self_check(device, dtype, W_dtype, n_cols, eps, gemma, multirow):+ from torch.utils._python_dispatch import _disable_current_modes+ with _disable_current_modes(), torch.no_grad(), torch_gpu_device(device):+ g = torch.Generator(device = device).manual_seed(3407)+ shape = (2 * multirow[1] + 3, n_cols)+ X = torch.randn(shape, device = device, generator = g)+ X = X * torch.exp2(torch.randint(-12, 13, shape, device = device, generator = g).float())+ dY = torch.randn(shape, device = device, generator = g)+ X[0], dY[0, ::2] = 0, 0 # all-zero row: signed zeros in dX+ X, dY = X.to(dtype), dY.to(dtype)+ W = torch.randn(n_cols, device = device, generator = g).to(W_dtype)+ results = []+ for rows in (False, multirow):+ Y, r = _rms_forward(X, W, eps, gemma, _eager_kernel, rows)+ dX = dY.clone()+ _rms_backward(dY.clone() if gemma else dX, dX, X, W, r, eps, gemma, _eager_kernel, rows)+ results.append((Y, r, dX))+ return all(torch.equal(_bits(a), _bits(b)) for a, b in zip(*results))+++def _multirow_checked(X, W, eps, gemma):+ n_cols = X.shape[1]+ multirow = _multirow_settings(n_cols)+ if multirow is None:+ return None+ key = (X.device, X.dtype, W.dtype, int(n_cols), float(eps), gemma)+ verdict = _MULTIROW_CHECKED.get(key)+ if verdict is None:+ if X.device.type == "cuda" and torch.cuda.is_current_stream_capturing():+ return None # check on a later uncaptured call+ try:+ verdict = _multirow_self_check(X.device, X.dtype, W.dtype, n_cols, eps, gemma, multirow)+ reason = f"self-check mismatch for {key}"+ except torch.cuda.OutOfMemoryError:+ raise+ except Exception as error:+ # Its launches re-raise while torch.compile traces; this call then runs one row.+ verdict, reason = False, f"self-check failed for {key}: {error!r}"+ _MULTIROW_CHECKED[key] = verdict+ if not verdict:+ _multirow_disable(reason)+ return multirow if verdict and _MULTIROW else None+++def _rms_forward(+ X,+ W,+ eps,+ gemma,+ wrap,+ multirow = None,+):+ # multirow: None picks (self-checked), False forces one row, settings force multi-row.+ n_rows, n_cols = X.shapeY = torch.empty((n_rows, n_cols), dtype = X.dtype, device = X.device)r = torch.empty(n_rows, dtype = torch.float32, device = X.device)+ if multirow is None:+ multirow = _multirow_checked(X, W, eps, gemma)+ if multirow:+ BLOCK_SIZE, ROWS, WARPS, num_warps = multirow+ try:+ wrap(_rms_layernorm_forward_rows)[((n_rows + ROWS - 1) // ROWS,)](+ Y,+ Y.stride(0),+ X,+ X.stride(0),+ W,+ r,+ n_rows,+ n_cols,+ eps,+ GEMMA = gemma,+ BLOCK_SIZE = BLOCK_SIZE,+ ROWS = ROWS,+ WARPS = WARPS,+ num_warps = num_warps,+ )+ return Y, r+ except Exception as error:+ if _multirow_must_raise(error, wrap):+ raise+ _multirow_disable(f"launch failed: {error!r}")+ BLOCK_SIZE, num_warps = calculate_settings(n_cols)fx = _gemma_rms_layernorm_forward if gemma else _rms_layernorm_forwardwrap(fx)[(n_rows,)](Y,@@ -180,9 +428,47 @@ def _rms_forward(X, W, eps, gemma, wrap):return Y, r-def _rms_backward(dY, dX, X, W, r, eps, gemma, wrap):+def _rms_backward(+ dY,+ dX,+ X,+ W,+ r,+ eps,+ gemma,+ wrap,+ multirow = None,+):# Non-Gemma writes dX over dY (the kernel ignores dX); Gemma writes into dX.n_rows, n_cols = dY.shape+ if multirow is None:+ multirow = _multirow_checked(X, W, eps, gemma)+ if multirow:+ BLOCK_SIZE, ROWS, WARPS, num_warps = multirow+ try:+ wrap(_rms_layernorm_backward_rows)[((n_rows + ROWS - 1) // ROWS,)](+ dY,+ dY.stride(0),+ dX,+ dX.stride(0),+ X,+ X.stride(0),+ W,+ r,+ n_rows,+ n_cols,+ eps,+ GEMMA = gemma,+ BLOCK_SIZE = BLOCK_SIZE,+ ROWS = ROWS,+ WARPS = WARPS,+ num_warps = num_warps,+ )+ return+ except Exception as error:+ if _multirow_must_raise(error, wrap):+ raise+ _multirow_disable(f"launch failed: {error!r}")BLOCK_SIZE, num_warps = calculate_settings(n_cols)wrap(_rms_layernorm_backward)[(n_rows,)](dY,@@ -283,7 +569,9 @@ if _TRACEABLE:def _rms_layernorm_op(X: torch.Tensor, W: torch.Tensor, eps: float, gemma: bool) -> Tuple[torch.Tensor, torch.Tensor]:- return _rms_forward(X, W, eps, gemma, _traced_kernel)+ # Called eagerly (outside a compiled graph) the launch needs X's device made current.+ with torch_gpu_device(X.device):+ return _rms_forward(X, W, eps, gemma, _traced_kernel)@torch.library.triton_op("unsloth::rms_layernorm_backward", mutates_args = ())def _rms_layernorm_backward_op(@@ -291,7 +579,8 @@ if _TRACEABLE:) -> torch.Tensor:dX = torch.empty_like(dY)# Non-Gemma rewrites its dY argument, so it gets a copy.- _rms_backward(dY if gemma else dX.copy_(dY), dX, X, W, r, eps, gemma, _traced_kernel)+ with torch_gpu_device(dY.device):+ _rms_backward(dY if gemma else dX.copy_(dY), dX, X, W, r, eps, gemma, _traced_kernel)return dXdef _rms_setup_context(ctx, inputs, output):