Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

RMSNorm kernels: do not specialize on the row count and row stride (#14786)

Daniel Han64c7a18a1cdd
diff --git a/tests/test_triton_variant_budget.py b/tests/test_triton_variant_budget.py
new file mode 100644
index 00000000000..3e5ffd75dad
--- /dev/null
+++ b/tests/test_triton_variant_budget.py
@@ -0,0 +1,64 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team.
+
+# Row counts are runtime arguments that Triton would otherwise specialize
+# (== 1, divisible by 16): every new token count could compile another copy of the same kernel.
+
+import pytest
+import torch
+
+from real_accelerator import has_real_cuda
+
+pytestmark = pytest.mark.skipif(not has_real_cuda(), reason = "needs a CUDA GPU")
+triton = pytest.importorskip("triton")
+
+import unsloth # noqa: F401 (patches first)
+from unsloth.kernels import rms_layernorm
+from unsloth.kernels.rms_layernorm import fast_rms_layernorm
+
+# Varied lengths: odd, divisible by 16, a single token.
+_SHAPES = ((1, 100), (1, 511), (1, 512), (2, 2048), (4, 333), (3, 1))
+
+
+def _variants(kernel):
+ while not isinstance(kernel, triton.runtime.jit.JITFunction):
+ kernel = kernel.fn
+ caches = getattr(kernel, "device_caches", None)
+ if caches is not None:
+ return sum(len(v[0]) for v in caches.values())
+ return sum(len(v) for v in kernel.cache.values())
+
+
+class _Norm(torch.nn.Module):
+ def __init__(self, n):
+ super().__init__()
+ self.weight = torch.nn.Parameter(torch.randn(n, device = "cuda", dtype = torch.bfloat16))
+ self.variance_epsilon = 1e-6
+
+
+def _reset(*kernels):
+ for kernel in kernels:
+ while not isinstance(kernel, triton.runtime.jit.JITFunction):
+ kernel = kernel.fn
+ caches = getattr(kernel, "device_caches", None)
+ if caches is not None:
+ for v in caches.values():
+ v[0].clear()
+ else:
+ for v in kernel.cache.values():
+ v.clear()
+ from unsloth.kernels import triton_launch
+
+ triton_launch._CACHE.clear()
+
+
+def test_rms_layernorm_backward_one_variant_across_token_counts():
+ kernels = (rms_layernorm._rms_layernorm_backward, rms_layernorm._rms_sum_dw_partials)
+ _reset(*kernels)
+ norm = _Norm(1024)
+ for batch, seq in _SHAPES:
+ X = torch.randn(batch, seq, 1024, device = "cuda", dtype = torch.bfloat16, requires_grad = True)
+ fast_rms_layernorm(norm, X).sum().backward()
+ torch.cuda.synchronize()
+ assert [_variants(k) for k in kernels] == [1, 1]
+
diff --git a/unsloth/kernels/rms_layernorm.py b/unsloth/kernels/rms_layernorm.py
index ac4b64bfcef..f841e3565ee 100644
--- a/unsloth/kernels/rms_layernorm.py
+++ b/unsloth/kernels/rms_layernorm.py
@@ -201,7 +201,9 @@ def _rms_layernorm_backward(
)
-_rms_layernorm_backward = triton.jit(_rms_layernorm_backward)
+_rms_layernorm_backward = triton.jit(
+ _rms_layernorm_backward, do_not_specialize = ("n_rows", "rows_per_program")
+)
_rms_layernorm_backward = triton.heuristics(
{
"GEMMA": lambda args: bool(args["GEMMA"]),
@@ -313,7 +315,7 @@ def _row_dot_in_row_kernel_order(a, b, ROWS: tl.constexpr, WARPS: tl.constexpr):
return total
[email protected](do_not_specialize = ("n_rows",))
def _rms_layernorm_forward_rows(
Y,
X,
@@ -415,7 +417,7 @@ def _rms_backward_rows_block(
return tl.sum(dY_rows * normed, axis = 0)
[email protected](do_not_specialize = ("n_rows", "blocks_per_program"))
def _rms_layernorm_backward_rows(
dY,
dX,
@@ -487,7 +489,7 @@ def _rms_layernorm_backward_rows(
)
[email protected](do_not_specialize = ("n_programs",))
def _rms_sum_dw_partials(
partials, dW, n_programs, n_cols, BLOCK_ROWS: tl.constexpr, BLOCK_COLS: tl.constexpr
):