Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
RMSNorm kernels: do not specialize on the row count and row stride (#14786)
Daniel Han64c7a18a1cdd
diff --git a/tests/test_triton_variant_budget.py b/tests/test_triton_variant_budget.pynew file mode 100644index 00000000000..3e5ffd75dad--- /dev/null+++ b/tests/test_triton_variant_budget.py@@ -0,0 +1,64 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team.++# Row counts are runtime arguments that Triton would otherwise specialize+# (== 1, divisible by 16): every new token count could compile another copy of the same kernel.++import pytest+import torch++from real_accelerator import has_real_cuda++pytestmark = pytest.mark.skipif(not has_real_cuda(), reason = "needs a CUDA GPU")+triton = pytest.importorskip("triton")++import unsloth # noqa: F401 (patches first)+from unsloth.kernels import rms_layernorm+from unsloth.kernels.rms_layernorm import fast_rms_layernorm++# Varied lengths: odd, divisible by 16, a single token.+_SHAPES = ((1, 100), (1, 511), (1, 512), (2, 2048), (4, 333), (3, 1))+++def _variants(kernel):+ while not isinstance(kernel, triton.runtime.jit.JITFunction):+ kernel = kernel.fn+ caches = getattr(kernel, "device_caches", None)+ if caches is not None:+ return sum(len(v[0]) for v in caches.values())+ return sum(len(v) for v in kernel.cache.values())+++class _Norm(torch.nn.Module):+ def __init__(self, n):+ super().__init__()+ self.weight = torch.nn.Parameter(torch.randn(n, device = "cuda", dtype = torch.bfloat16))+ self.variance_epsilon = 1e-6+++def _reset(*kernels):+ for kernel in kernels:+ while not isinstance(kernel, triton.runtime.jit.JITFunction):+ kernel = kernel.fn+ caches = getattr(kernel, "device_caches", None)+ if caches is not None:+ for v in caches.values():+ v[0].clear()+ else:+ for v in kernel.cache.values():+ v.clear()+ from unsloth.kernels import triton_launch++ triton_launch._CACHE.clear()+++def test_rms_layernorm_backward_one_variant_across_token_counts():+ kernels = (rms_layernorm._rms_layernorm_backward, rms_layernorm._rms_sum_dw_partials)+ _reset(*kernels)+ norm = _Norm(1024)+ for batch, seq in _SHAPES:+ X = torch.randn(batch, seq, 1024, device = "cuda", dtype = torch.bfloat16, requires_grad = True)+ fast_rms_layernorm(norm, X).sum().backward()+ torch.cuda.synchronize()+ assert [_variants(k) for k in kernels] == [1, 1]+diff --git a/unsloth/kernels/rms_layernorm.py b/unsloth/kernels/rms_layernorm.pyindex ac4b64bfcef..f841e3565ee 100644--- a/unsloth/kernels/rms_layernorm.py+++ b/unsloth/kernels/rms_layernorm.py@@ -201,7 +201,9 @@ def _rms_layernorm_backward()-_rms_layernorm_backward = triton.jit(_rms_layernorm_backward)+_rms_layernorm_backward = triton.jit(+ _rms_layernorm_backward, do_not_specialize = ("n_rows", "rows_per_program")+)_rms_layernorm_backward = triton.heuristics({"GEMMA": lambda args: bool(args["GEMMA"]),@@ -313,7 +315,7 @@ def _row_dot_in_row_kernel_order(a, b, ROWS: tl.constexpr, WARPS: tl.constexpr):return total[email protected](do_not_specialize = ("n_rows",))def _rms_layernorm_forward_rows(Y,X,@@ -415,7 +417,7 @@ def _rms_backward_rows_block(return tl.sum(dY_rows * normed, axis = 0)[email protected](do_not_specialize = ("n_rows", "blocks_per_program"))def _rms_layernorm_backward_rows(dY,dX,@@ -487,7 +489,7 @@ def _rms_layernorm_backward_rows()[email protected](do_not_specialize = ("n_programs",))def _rms_sum_dw_partials(partials, dW, n_programs, n_cols, BLOCK_ROWS: tl.constexpr, BLOCK_COLS: tl.constexpr):