Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Kernels: keep per-call sizes out of the Triton compile key (layernorm dW, fp8 block matmul, nvfp4 split sum) (#16584)

Daniel Hanc15435d41f87
diff --git a/tests/test_triton_runtime_size_args.py b/tests/test_triton_runtime_size_args.py
new file mode 100644
index 00000000000..f8124e40b29
--- /dev/null
+++ b/tests/test_triton_runtime_size_args.py
@@ -0,0 +1,55 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""Per-call sizes (token rows, split counts) stay runtime Triton arguments with do_not_specialize, so a
+kernel does not compile a new variant per prompt length, batch size or split count."""
+
+import ast
+import pathlib
+
+import pytest
+
+KERNELS = pathlib.Path(__file__).resolve().parents[1] / "unsloth" / "kernels"
+
+CASES = {
+ "layernorm.py": {
+ "layernorm_backward_dw": ("n_rows", "rows_per_program"),
+ "_layernorm_sum_partials": ("n_programs",),
+ },
+ "fp8.py": {"_w8a8_block_fp8_matmul": ("M",)},
+}
+
+
+def _defs(path):
+ tree = ast.parse(path.read_text(encoding = "utf-8"))
+ return {n.name: n for n in ast.walk(tree) if isinstance(n, ast.FunctionDef)}
+
+
+def _do_not_specialize(fn):
+ for dec in fn.decorator_list:
+ if isinstance(dec, ast.Call) and ast.unparse(dec.func).endswith("jit"):
+ for kw in dec.keywords:
+ if kw.arg == "do_not_specialize":
+ return {e.value for e in kw.value.elts}
+ return set()
+
+
[email protected]("rel", sorted(CASES))
+def test_runtime_sizes_are_not_specialized(rel):
+ defs = _defs(KERNELS / rel)
+ for name, args in CASES[rel].items():
+ fn = defs[name]
+ params = {a.arg: a for a in fn.args.args}
+ dns = _do_not_specialize(fn)
+ for arg in args:
+ assert arg in params, (name, arg)
+ ann = params[arg].annotation
+ assert ann is None or "constexpr" not in ast.unparse(ann), (name, arg)
+ assert arg in dns, (name, arg, sorted(dns))
+
+
+def test_nvfp4_split_sum_keeps_split_constexpr():
+ # The unrolled static_range over SPLIT (2, 4, 8 or 16) is 13-34% faster than a runtime loop.
+ fn = _defs(KERNELS / "nvfp4.py")["_split_sum_kernel"]
+ ann = {a.arg: a.annotation for a in fn.args.args}["SPLIT"]
+ assert ann is not None and "constexpr" in ast.unparse(ann)
diff --git a/unsloth/kernels/fp8.py b/unsloth/kernels/fp8.py
index 00593ffefd4..f3757e8fe4e 100644
--- a/unsloth/kernels/fp8.py
+++ b/unsloth/kernels/fp8.py
@@ -425,7 +425,8 @@ def act_quant(x: torch.Tensor, block_size: int = 128) -> tuple[torch.Tensor, tor
# Adapted from sgl-project/sglang python/sglang/srt/layers/quantization/fp8_kernel.py
+# M (token rows) runtime: BLOCK_SIZE_M already buckets it; no extra %16 / == 1 variant per prompt length.
[email protected](do_not_specialize = ["M"])
def _w8a8_block_fp8_matmul(
A,
B,
diff --git a/unsloth/kernels/layernorm.py b/unsloth/kernels/layernorm.py
index 21768362cc2..1536df0aecd 100644
--- a/unsloth/kernels/layernorm.py
+++ b/unsloth/kernels/layernorm.py
@@ -133,7 +133,7 @@ def layernorm_backward(
# MIT 2023-2026 Songlin Yang et al.: dW/dB follow flash-linear-attention's layer_norm_bwd_kernel_row
[email protected](do_not_specialize = ("n_rows", "rows_per_program"))
def layernorm_backward_dw(
dY,
dX,
@@ -187,7 +187,7 @@ def layernorm_backward_dw(
tl.store(dB + pid * n_cols + col_offsets, dB_acc, mask = mask)
[email protected](do_not_specialize = ("n_programs",))
def _layernorm_sum_partials(
partials,
out0,
diff --git a/unsloth/kernels/nvfp4.py b/unsloth/kernels/nvfp4.py
index 871d4c03d60..2d3016bdb2b 100644
--- a/unsloth/kernels/nvfp4.py
+++ b/unsloth/kernels/nvfp4.py
@@ -558,7 +558,8 @@ def _fused_config(
def _split_sum_kernel(
partial_ptr, out_ptr, numel, SPLIT: tl.constexpr, BLOCK: tl.constexpr
): # SPLIT split-K partials, BLOCK block size
- # Fixed-order sum of the split partials (determinism), one cast.
+ # Fixed-order sum of the split partials (determinism), one cast. SPLIT stays constexpr: the unrolled
+ # static_range is 13-34% faster than a runtime loop, and SPLIT is only ever 2, 4, 8 or 16.
offs = tl.program_id(0).to(tl.int64) * BLOCK + tl.arange(0, BLOCK)
mask = offs < numel
acc = tl.load(partial_ptr + offs, mask = mask, other = 0.0)