Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Kernels: keep per-call sizes out of the Triton compile key (layernorm dW, fp8 block matmul, nvfp4 split sum) (#16584)
Daniel Hanc15435d41f87
diff --git a/tests/test_triton_runtime_size_args.py b/tests/test_triton_runtime_size_args.pynew file mode 100644index 00000000000..f8124e40b29--- /dev/null+++ b/tests/test_triton_runtime_size_args.py@@ -0,0 +1,55 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""Per-call sizes (token rows, split counts) stay runtime Triton arguments with do_not_specialize, so a+kernel does not compile a new variant per prompt length, batch size or split count."""++import ast+import pathlib++import pytest++KERNELS = pathlib.Path(__file__).resolve().parents[1] / "unsloth" / "kernels"++CASES = {+ "layernorm.py": {+ "layernorm_backward_dw": ("n_rows", "rows_per_program"),+ "_layernorm_sum_partials": ("n_programs",),+ },+ "fp8.py": {"_w8a8_block_fp8_matmul": ("M",)},+}+++def _defs(path):+ tree = ast.parse(path.read_text(encoding = "utf-8"))+ return {n.name: n for n in ast.walk(tree) if isinstance(n, ast.FunctionDef)}+++def _do_not_specialize(fn):+ for dec in fn.decorator_list:+ if isinstance(dec, ast.Call) and ast.unparse(dec.func).endswith("jit"):+ for kw in dec.keywords:+ if kw.arg == "do_not_specialize":+ return {e.value for e in kw.value.elts}+ return set()++[email protected]("rel", sorted(CASES))+def test_runtime_sizes_are_not_specialized(rel):+ defs = _defs(KERNELS / rel)+ for name, args in CASES[rel].items():+ fn = defs[name]+ params = {a.arg: a for a in fn.args.args}+ dns = _do_not_specialize(fn)+ for arg in args:+ assert arg in params, (name, arg)+ ann = params[arg].annotation+ assert ann is None or "constexpr" not in ast.unparse(ann), (name, arg)+ assert arg in dns, (name, arg, sorted(dns))+++def test_nvfp4_split_sum_keeps_split_constexpr():+ # The unrolled static_range over SPLIT (2, 4, 8 or 16) is 13-34% faster than a runtime loop.+ fn = _defs(KERNELS / "nvfp4.py")["_split_sum_kernel"]+ ann = {a.arg: a.annotation for a in fn.args.args}["SPLIT"]+ assert ann is not None and "constexpr" in ast.unparse(ann)diff --git a/unsloth/kernels/fp8.py b/unsloth/kernels/fp8.pyindex 00593ffefd4..f3757e8fe4e 100644--- a/unsloth/kernels/fp8.py+++ b/unsloth/kernels/fp8.py@@ -425,7 +425,8 @@ def act_quant(x: torch.Tensor, block_size: int = 128) -> tuple[torch.Tensor, tor# Adapted from sgl-project/sglang python/sglang/srt/layers/quantization/fp8_kernel.py+# M (token rows) runtime: BLOCK_SIZE_M already buckets it; no extra %16 / == 1 variant per prompt length.[email protected](do_not_specialize = ["M"])def _w8a8_block_fp8_matmul(A,B,diff --git a/unsloth/kernels/layernorm.py b/unsloth/kernels/layernorm.pyindex 21768362cc2..1536df0aecd 100644--- a/unsloth/kernels/layernorm.py+++ b/unsloth/kernels/layernorm.py@@ -133,7 +133,7 @@ def layernorm_backward(# MIT 2023-2026 Songlin Yang et al.: dW/dB follow flash-linear-attention's layer_norm_bwd_kernel_row[email protected](do_not_specialize = ("n_rows", "rows_per_program"))def layernorm_backward_dw(dY,dX,@@ -187,7 +187,7 @@ def layernorm_backward_dw(tl.store(dB + pid * n_cols + col_offsets, dB_acc, mask = mask)[email protected](do_not_specialize = ("n_programs",))def _layernorm_sum_partials(partials,out0,diff --git a/unsloth/kernels/nvfp4.py b/unsloth/kernels/nvfp4.pyindex 871d4c03d60..2d3016bdb2b 100644--- a/unsloth/kernels/nvfp4.py+++ b/unsloth/kernels/nvfp4.py@@ -558,7 +558,8 @@ def _fused_config(def _split_sum_kernel(partial_ptr, out_ptr, numel, SPLIT: tl.constexpr, BLOCK: tl.constexpr): # SPLIT split-K partials, BLOCK block size- # Fixed-order sum of the split partials (determinism), one cast.+ # Fixed-order sum of the split partials (determinism), one cast. SPLIT stays constexpr: the unrolled+ # static_range is 13-34% faster than a runtime loop, and SPLIT is only ever 2, 4, 8 or 16.offs = tl.program_id(0).to(tl.int64) * BLOCK + tl.arange(0, BLOCK)mask = offs < numelacc = tl.load(partial_ptr + offs, mask = mask, other = 0.0)