Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Launch eager RMSNorm and LayerNorm kernels through triton_launch (#13889)

Daniel Hanb67f20f599dd
diff --git a/tests/test_norm_fast_launch.py b/tests/test_norm_fast_launch.py
new file mode 100644
index 00000000000..3fcbcee3904
--- /dev/null
+++ b/tests/test_norm_fast_launch.py
@@ -0,0 +1,185 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
+
+"""Eager RMSNorm / LayerNorm launches go through triton_launch (no Triton dispatch once warm), keep
+kernel[grid] under torch.compile, and stay correct under CUDA graph capture, on a side stream and on a
+device that is not the current one."""
+
+import pathlib
+import sys
+
+import pytest
+import torch
+
+sys.path.insert(0, str(pathlib.Path(__file__).parent / "_shared"))
+try:
+ from real_accelerator import has_real_cuda
+except Exception:
+ has_real_cuda = torch.cuda.is_available
+
+if not has_real_cuda():
+ pytest.skip("needs a CUDA GPU", allow_module_level = True)
+
+import triton
+from unsloth.kernels import layernorm, rms_layernorm, triton_launch
+
+BF16 = torch.cuda.get_device_capability()[0] >= 8
+DTYPE = torch.bfloat16 if BF16 else torch.float16
+FAST = triton_launch._ENABLED # off on HIP and with UNSLOTH_TRITON_FAST_LAUNCH=0
+# (kind, width, gemma): 128 takes the multi-row kernels where enabled, 4096 / 1000 the one-row ones.
+CASES = [
+ ("rms", 128, False),
+ ("rms", 128, True),
+ ("rms", 4096, False),
+ ("rms", 4096, True),
+ ("ln", 1000, False),
+ ("ln", 4096, False),
+]
+
+
+def _module(kind, width, train, device = "cuda", bias = True):
+ g = torch.Generator().manual_seed(width)
+ weight = torch.nn.Parameter(torch.randn(width, generator = g).to(device, DTYPE), requires_grad = train)
+ b = torch.nn.Parameter(torch.randn(width, generator = g).to(device, DTYPE), requires_grad = train)
+ return weight, (b if kind == "ln" and bias else None)
+
+
+def _step(kind, gemma, x, weight, bias, dy):
+ x = x.detach().requires_grad_()
+ if kind == "rms":
+ y = rms_layernorm.Fast_RMS_Layernorm.apply(x, weight, 1e-6, gemma)
+ else:
+ y = layernorm.Fast_Layernorm.apply(x, weight, bias, 1e-5)
+ y.backward(dy.clone()) # non-Gemma RMSNorm writes dX over its dY
+ grads = [p.grad for p in (weight, bias) if p is not None and p.requires_grad]
+ return [y.detach(), x.grad] + grads
+
+
+def _clear(weight, bias):
+ for p in (weight, bias):
+ if p is not None:
+ p.grad = None
+
+
+def _inputs(width, rows = 300, device = "cuda", seed = 0):
+ g = torch.Generator().manual_seed(seed)
+ return (
+ torch.randn(2, rows, width, generator = g).to(device, DTYPE),
+ torch.randn(2, rows, width, generator = g).to(device, DTYPE),
+ )
+
+
[email protected](not FAST, reason = "fast launch disabled (HIP or UNSLOTH_TRITON_FAST_LAUNCH=0)")
[email protected]("train", [False, True], ids = ["frozen", "trainable"])
[email protected]("kind,width,gemma", CASES)
+def test_warm_eager_call_skips_triton_dispatch(monkeypatch, kind, width, gemma, train):
+ weight, bias = _module(kind, width, train)
+ x, dy = _inputs(width)
+ ref = _step(kind, gemma, x, weight, bias, dy)
+ _clear(weight, bias)
+ calls = []
+ run = triton.runtime.jit.JITFunction.run
+ monkeypatch.setattr(
+ triton.runtime.jit.JITFunction, "run", lambda self, *a, **k: calls.append(self) or run(self, *a, **k)
+ )
+ got = _step(kind, gemma, x, weight, bias, dy)
+ assert calls == [], [c.__name__ for c in calls]
+ for a, b in zip(ref, got):
+ assert torch.equal(a, b)
+
+
[email protected](not getattr(torch.library, "triton_op", None), reason = "needs triton_op")
[email protected]("kind,width,gemma", [("rms", 128, False), ("rms", 4096, True), ("ln", 1000, False)])
+def test_compiled_calls_never_use_fast_launch(monkeypatch, kind, width, gemma):
+ calls = []
+ for module in (rms_layernorm, layernorm):
+ real = module.launch
+ monkeypatch.setattr(module, "launch", lambda *a, real = real, **k: calls.append(a[0]) or real(*a, **k))
+ # fp32: T4 cannot trace bf16 Triton.
+ weight = torch.nn.Parameter(torch.randn(width, device = "cuda"))
+ bias = torch.nn.Parameter(torch.randn(width, device = "cuda"))
+ norm = torch.nn.Module()
+ norm.weight, norm.bias, norm.eps, norm.variance_epsilon = weight, bias, 1e-5, 1e-6
+ norm.elementwise_affine = True
+ if kind == "rms":
+ fn = lambda x: rms_layernorm.fast_rms_layernorm(norm, x, gemma = gemma)
+ else:
+ fn = lambda x: layernorm.fast_layernorm(norm, x)
+ torch._dynamo.reset()
+ x = torch.randn(4, 64, width, device = "cuda", requires_grad = True)
+ torch.compile(fn, fullgraph = True)(x).sum().backward()
+ assert calls == []
+ assert x.grad is not None and weight.grad is not None
+ fn(x.detach().requires_grad_()).sum().backward() # eager: same module, launch() taken
+ assert calls
+
+
[email protected]("train", [False, True], ids = ["frozen", "trainable"])
[email protected]("kind,width,gemma", CASES)
+def test_cuda_graph_capture_replays(kind, width, gemma, train):
+ weight, bias = _module(kind, width, train)
+ x, dy = _inputs(width, seed = 1)
+ for _ in range(2): # warm: multi-row self-check, compile, fast-launch entries
+ _step(kind, gemma, x, weight, bias, dy)
+ _clear(weight, bias)
+ static_x, static_dy = x.clone(), dy.clone()
+ stream = torch.cuda.Stream()
+ stream.wait_stream(torch.cuda.current_stream())
+ with torch.cuda.stream(stream):
+ _step(kind, gemma, static_x, weight, bias, static_dy)
+ _clear(weight, bias)
+ torch.cuda.current_stream().wait_stream(stream)
+ graph = torch.cuda.CUDAGraph()
+ with torch.cuda.graph(graph):
+ static_out = _step(kind, gemma, static_x, weight, bias, static_dy)
+ for seed in (2, 3):
+ new_x, new_dy = _inputs(width, seed = seed)
+ static_x.copy_(new_x)
+ static_dy.copy_(new_dy)
+ graph.replay()
+ torch.cuda.synchronize()
+ _clear(weight, bias)
+ ref = _step(kind, gemma, new_x, weight, bias, new_dy)
+ _clear(weight, bias)
+ for a, b in zip(ref, static_out):
+ assert torch.equal(a, b)
+
+
[email protected]("train", [False, True], ids = ["frozen", "trainable"])
[email protected]("kind,width,gemma", CASES)
+def test_launches_follow_the_current_stream(kind, width, gemma, train):
+ weight, bias = _module(kind, width, train)
+ x, dy = _inputs(width, seed = 4)
+ ref = _step(kind, gemma, x, weight, bias, dy)
+ _clear(weight, bias)
+ _step(kind, gemma, x, weight, bias, dy)
+ _clear(weight, bias)
+ side = torch.cuda.Stream()
+ late_x = torch.zeros_like(x)
+ torch.cuda.synchronize()
+ with torch.cuda.stream(side):
+ # A launch on any other stream would read late_x before the copy lands.
+ torch.cuda._sleep(200_000_000)
+ late_x.copy_(x)
+ got = _step(kind, gemma, late_x, weight, bias, dy)
+ side.synchronize()
+ for a, b in zip(ref, got):
+ assert torch.equal(a, b)
+
+
[email protected](torch.cuda.device_count() < 2, reason = "needs two GPUs")
[email protected]("train", [False, True], ids = ["frozen", "trainable"])
[email protected]("kind,width,gemma", CASES)
+def test_second_device_with_first_current(kind, width, gemma, train):
+ weight, bias = _module(kind, width, train, device = "cuda:1")
+ x, dy = _inputs(width, device = "cuda:1", seed = 5)
+ with torch.cuda.device(1):
+ ref = [t.clone() for t in _step(kind, gemma, x, weight, bias, dy)]
+ _clear(weight, bias)
+ for _ in range(2): # the second call is the warm fast-launch one
+ with torch.cuda.device(0):
+ got = _step(kind, gemma, x, weight, bias, dy)
+ _clear(weight, bias)
+ torch.cuda.synchronize(1)
+ for a, b in zip(ref, got):
+ assert a.device == b.device and torch.equal(a, b)
diff --git a/tests/test_triton_fp_fusion.py b/tests/test_triton_fp_fusion.py
index 171005d150f..dbca7905bac 100644
--- a/tests/test_triton_fp_fusion.py
+++ b/tests/test_triton_fp_fusion.py
@@ -25,7 +25,11 @@ _NOT_LAUNCHES = {
"utils.py": {"plan"},
"nf4.py": {"kernel"},
"triton_launch.py": {"kernel"},
+ "layernorm.py": {"kernel"},
+ "rms_layernorm.py": {"wrap(kernel)"},
}
+# Launch helpers forwarding options to the excluded sites above: each call is checked instead.
+_LAUNCH_HELPERS = {"layernorm.py": "_launch", "rms_layernorm.py": "_launch"}
def _launch_sites():
@@ -37,6 +41,9 @@ def _launch_sites():
if "triton" not in src:
continue
for node in ast.walk(ast.parse(src)):
+ helper = _LAUNCH_HELPERS.get(path.name)
+ if isinstance(node, ast.Call) and isinstance(node.func, ast.Name) and node.func.id == helper:
+ yield f"{rel}:{node.lineno}:{helper}", node
if isinstance(node, ast.Call) and isinstance(node.func, ast.Subscript):
name = ast.get_source_segment(src, node.func.value)
if name in _NOT_LAUNCHES.get(path.name, ()):
diff --git a/unsloth/kernels/layernorm.py b/unsloth/kernels/layernorm.py
index d693493c2e6..de5fde44044 100644
--- a/unsloth/kernels/layernorm.py
+++ b/unsloth/kernels/layernorm.py
@@ -15,6 +15,7 @@ import triton
import triton.language as tl
import torch
from .utils import calculate_settings, long_indexing, torch_gpu_device
+from .triton_launch import launch
from unsloth_zoo.patching_utils import (
patch_layernorm,
)
@@ -23,13 +24,13 @@ from unsloth_zoo.patching_utils import (
@triton.jit
def layernorm_forward(
Y,
- Y_row_stride: tl.constexpr, # n_cols
X,
- X_row_stride: tl.constexpr, # n_cols
W,
b,
r,
mu,
+ Y_row_stride: tl.constexpr, # n_cols
+ X_row_stride: tl.constexpr, # n_cols
n_cols: tl.constexpr,
eps: tl.constexpr,
LONG_INDEXING: tl.constexpr,
@@ -86,13 +87,13 @@ def _layernorm_backward_row(dY, X, W_row, r, mu, row_idx, col_offsets, mask, n_c
@triton.jit
def layernorm_backward(
dY,
- dY_row_stride: tl.constexpr, # n_cols
X,
- X_row_stride: tl.constexpr, # n_cols
W,
b,
r,
mu,
+ dY_row_stride: tl.constexpr, # n_cols
+ X_row_stride: tl.constexpr, # n_cols
n_cols: tl.constexpr,
eps: tl.constexpr,
LONG_INDEXING: tl.constexpr,
@@ -122,9 +123,7 @@ def layernorm_backward(
@triton.jit
def layernorm_backward_dw(
dY,
- dY_row_stride: tl.constexpr,
X,
- X_row_stride: tl.constexpr,
W,
r,
mu,
@@ -132,6 +131,8 @@ def layernorm_backward_dw(
dB,
n_rows,
rows_per_program,
+ dY_row_stride: tl.constexpr,
+ X_row_stride: tl.constexpr,
n_cols: tl.constexpr,
LONG_INDEXING: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
@@ -203,6 +204,24 @@ def _dw_slots(device):
return 128, 32
+_cached_settings = functools.lru_cache(maxsize = None)(calculate_settings)
+
+
+def _settings(n_cols):
+ # Cached per width for eager calls (host bound at small shapes); tracing calls it directly.
+ if torch.compiler.is_compiling():
+ return calculate_settings(n_cols)
+ return _cached_settings(n_cols)
+
+
+def _launch(kernel, grid, args, n_tensors, constexprs, device, **options):
+ # Eager CUDA skips Triton's arg binding (triton_launch.py); compiling calls keep kernel[grid].
+ if device.type == "cuda" and not torch.compiler.is_compiling():
+ launch(kernel, grid, args, n_tensors, constexprs, device.index, **options)
+ else:
+ kernel[grid](*args, **constexprs, **options)
+
+
class Fast_Layernorm(torch.autograd.Function):
@staticmethod
def forward(ctx, X, W, b, eps):
@@ -214,27 +233,28 @@ class Fast_Layernorm(torch.autograd.Function):
has_bias = b is not None
b = b.contiguous() if has_bias else W
n_rows, n_cols = X.shape
- BLOCK_SIZE, num_warps = calculate_settings(n_cols)
+ BLOCK_SIZE, num_warps = _settings(n_cols)
device = X.device
Y = torch.empty((n_rows, n_cols), dtype = X.dtype, device = device)
r = torch.empty(n_rows, dtype = torch.float32, device = device)
mu = torch.empty(n_rows, dtype = torch.float32, device = device)
with torch_gpu_device(device):
- layernorm_forward[(n_rows,)](
- Y,
- Y.stride(0),
- X,
- X.stride(0),
- W,
- b,
- r,
- mu,
- n_cols,
- eps,
- LONG_INDEXING = long_indexing(X, Y, block = BLOCK_SIZE),
- BLOCK_SIZE = BLOCK_SIZE,
- HAS_BIAS = has_bias,
+ _launch(
+ layernorm_forward,
+ (n_rows,),
+ (Y, X, W, b, r, mu),
+ 6,
+ dict(
+ Y_row_stride = Y.stride(0),
+ X_row_stride = X.stride(0),
+ n_cols = n_cols,
+ eps = eps,
+ LONG_INDEXING = long_indexing(X, Y, block = BLOCK_SIZE),
+ BLOCK_SIZE = BLOCK_SIZE,
+ HAS_BIAS = has_bias,
+ ),
+ device,
num_warps = num_warps,
enable_fp_fusion = True,
)
@@ -251,77 +271,79 @@ class Fast_Layernorm(torch.autograd.Function):
dY = dY.reshape(-1, dim).contiguous()
X, W, b, r, mu = ctx.saved_tensors
n_rows, n_cols = dY.shape
+ device = dY.device
train_w, train_b = ctx.needs_input_grad[1], ctx.needs_input_grad[2]
LONG_INDEXING = long_indexing(dY, X, block = ctx.BLOCK_SIZE)
if not (train_w or train_b):
- with torch_gpu_device(dY.device):
- layernorm_backward[(n_rows,)](
- dY,
- dY.stride(0),
- X,
- X.stride(0),
- W,
- b,
- r,
- mu,
- n_cols,
- ctx.eps,
- LONG_INDEXING = LONG_INDEXING,
- BLOCK_SIZE = ctx.BLOCK_SIZE,
+ with torch_gpu_device(device):
+ _launch(
+ layernorm_backward,
+ (n_rows,),
+ (dY, X, W, b, r, mu),
+ 6,
+ dict(
+ dY_row_stride = dY.stride(0),
+ X_row_stride = X.stride(0),
+ n_cols = n_cols,
+ eps = ctx.eps,
+ LONG_INDEXING = LONG_INDEXING,
+ BLOCK_SIZE = ctx.BLOCK_SIZE,
+ ),
+ device,
num_warps = ctx.num_warps,
enable_fp_fusion = True,
)
return dY.view(*shape), None, None, None
- sms, warps_per_sm = _dw_slots(dY.device)
- programs = max(1, min(n_rows, sms * max(1, warps_per_sm // ctx.num_warps)))
- rows_per_program = max(1, triton.cdiv(n_rows, programs))
- programs = max(1, triton.cdiv(n_rows, rows_per_program))
+ programs, rows_per_program = _dw_programs(device, n_rows, ctx.num_warps)
# Every program stores its partial row, so empty suffices unless nothing launches.
alloc = torch.empty if n_rows > 0 else torch.zeros
partials = alloc(
- (train_w + train_b, programs, n_cols), dtype = torch.float32, device = dY.device
+ (train_w + train_b, programs, n_cols), dtype = torch.float32, device = device
)
- if n_rows > 0:
- with torch_gpu_device(dY.device):
- layernorm_backward_dw[(programs,)](
- dY,
- dY.stride(0),
- X,
- X.stride(0),
- W,
- r,
- mu,
- partials[0],
- partials[-1],
- n_rows,
- rows_per_program,
- n_cols,
- LONG_INDEXING = LONG_INDEXING,
- BLOCK_SIZE = ctx.BLOCK_SIZE,
- TRAIN_W = train_w,
- TRAIN_B = train_b,
+ dW = torch.empty(n_cols, dtype = W.dtype, device = device) if train_w else None
+ dB = torch.empty(n_cols, dtype = b.dtype, device = device) if train_b else None
+ outs = [t for t in (dW, dB) if t is not None]
+ with torch_gpu_device(device):
+ if n_rows > 0:
+ _launch(
+ layernorm_backward_dw,
+ (programs,),
+ (dY, X, W, r, mu, partials[0], partials[-1], n_rows, rows_per_program),
+ 7,
+ dict(
+ dY_row_stride = dY.stride(0),
+ X_row_stride = X.stride(0),
+ n_cols = n_cols,
+ LONG_INDEXING = LONG_INDEXING,
+ BLOCK_SIZE = ctx.BLOCK_SIZE,
+ TRAIN_W = train_w,
+ TRAIN_B = train_b,
+ ),
+ device,
num_warps = ctx.num_warps,
enable_fp_fusion = True,
)
- dW = torch.empty(n_cols, dtype = W.dtype, device = dY.device) if train_w else None
- dB = torch.empty(n_cols, dtype = b.dtype, device = dY.device) if train_b else None
- outs = [t for t in (dW, dB) if t is not None]
- with torch_gpu_device(dY.device):
- _layernorm_sum_partials[(triton.cdiv(n_cols, 64), len(outs))](
- partials,
- outs[0],
- outs[-1],
- programs,
- n_cols,
- BLOCK_ROWS = 64,
- BLOCK_COLS = 64,
+ _launch(
+ _layernorm_sum_partials,
+ ((n_cols + 63) // 64, len(outs)),
+ (partials, outs[0], outs[-1], programs, n_cols),
+ 3,
+ dict(BLOCK_ROWS = 64, BLOCK_COLS = 64),
+ device,
enable_fp_fusion = True,
)
return dY.view(*shape), dW, dB, None
+def _dw_programs(device, n_rows, num_warps):
+ sms, warps_per_sm = _dw_slots(device)
+ programs = max(1, min(n_rows, sms * max(1, warps_per_sm // num_warps)))
+ rows_per_program = max(1, (n_rows + programs - 1) // programs)
+ return max(1, (n_rows + rows_per_program - 1) // rows_per_program), rows_per_program
+
+
def fast_layernorm(layernorm, X):
assert layernorm.elementwise_affine is True
W = layernorm.weight
diff --git a/unsloth/kernels/rms_layernorm.py b/unsloth/kernels/rms_layernorm.py
index 038cfd5f21c..2bd30a98c14 100644
--- a/unsloth/kernels/rms_layernorm.py
+++ b/unsloth/kernels/rms_layernorm.py
@@ -18,17 +18,18 @@ import triton.language as tl
import torch
from typing import Tuple
from .utils import calculate_settings, long_indexing, torch_gpu_device
+from .triton_launch import launch
@triton.jit
def _rms_layernorm_forward(
Y,
- Y_row_stride: tl.constexpr,
X,
- X_row_stride: tl.constexpr,
W,
- W_row_stride: tl.constexpr,
r,
+ Y_row_stride: tl.constexpr,
+ X_row_stride: tl.constexpr,
+ W_row_stride: tl.constexpr,
r_row_stride: tl.constexpr,
n_cols: tl.constexpr,
eps: tl.constexpr,
@@ -96,7 +97,6 @@ def _rms_backward_row(
X_row = tl.load(X + col_offsets, mask = mask, other = 0).to(tl.float32)
W_row = tl.load(W + col_offsets, mask = mask, other = 0).to(tl.float32)
- # Get saved row variance
inv_var = tl.load(r).to(tl.float32)
normed = X_row * inv_var
if TRAIN_W:
@@ -120,18 +120,18 @@ def _rms_backward_row(
def _rms_layernorm_backward(
dY,
- dY_row_stride: tl.constexpr,
dX,
- dX_row_stride: tl.constexpr,
X,
- X_row_stride: tl.constexpr,
W,
- W_row_stride: tl.constexpr,
r,
- r_row_stride: tl.constexpr,
dW,
n_rows,
rows_per_program,
+ dY_row_stride: tl.constexpr,
+ dX_row_stride: tl.constexpr,
+ X_row_stride: tl.constexpr,
+ W_row_stride: tl.constexpr,
+ r_row_stride: tl.constexpr,
n_cols: tl.constexpr,
eps: tl.constexpr,
GEMMA: tl.constexpr,
@@ -208,12 +208,12 @@ _rms_layernorm_backward = triton.heuristics(
@triton.jit
def _gemma_rms_layernorm_forward(
Y,
- Y_row_stride: tl.constexpr,
X,
- X_row_stride: tl.constexpr,
W,
- W_row_stride: tl.constexpr,
r,
+ Y_row_stride: tl.constexpr,
+ X_row_stride: tl.constexpr,
+ W_row_stride: tl.constexpr,
r_row_stride: tl.constexpr,
n_cols: tl.constexpr,
eps: tl.constexpr,
@@ -273,12 +273,12 @@ def _row_dot_in_row_kernel_order(a, b, ROWS: tl.constexpr, WARPS: tl.constexpr):
@triton.jit
def _rms_layernorm_forward_rows(
Y,
- Y_row_stride: tl.constexpr,
X,
- X_row_stride: tl.constexpr,
W,
r,
n_rows,
+ Y_row_stride: tl.constexpr,
+ X_row_stride: tl.constexpr,
n_cols: tl.constexpr,
eps: tl.constexpr,
GEMMA: tl.constexpr,
@@ -373,16 +373,16 @@ def _rms_backward_rows_block(
@triton.jit
def _rms_layernorm_backward_rows(
dY,
- dY_row_stride: tl.constexpr,
dX,
- dX_row_stride: tl.constexpr,
X,
- X_row_stride: tl.constexpr,
W,
r,
dW,
n_rows,
blocks_per_program,
+ dY_row_stride: tl.constexpr,
+ dX_row_stride: tl.constexpr,
+ X_row_stride: tl.constexpr,
n_cols: tl.constexpr,
eps: tl.constexpr,
GEMMA: tl.constexpr,
@@ -454,6 +454,36 @@ def _rms_sum_dw_partials(
tl.store(dW + cols, tl.sum(acc, axis = 0).to(dW.dtype.element_ty), mask = col_mask)
+def _cdiv(a, b):
+ # triton.cdiv's formula without its call overhead (a ConstexprFunction on recent Triton).
+ return (a + b - 1) // b
+
+
+_cached_settings = functools.lru_cache(maxsize = None)(calculate_settings)
+
+
+def _settings(n_cols):
+ # Cached per width for eager calls (host bound at small shapes); tracing calls it directly.
+ if torch.compiler.is_compiling():
+ return calculate_settings(n_cols)
+ return _cached_settings(n_cols)
+
+
+def _launch(wrap, kernel, grid, args, n_tensors, constexprs, device, **options):
+ # Eager CUDA skips Triton's arg binding (triton_launch.py); traced / non-Triton keep kernel[grid].
+ if wrap is _eager_kernel and device.type == "cuda" and not torch.compiler.is_compiling():
+ fn = kernel
+ if isinstance(kernel, triton.runtime.autotuner.Heuristics):
+ # What Heuristics.run does; these heuristics only read their own constexpr.
+ fn, constexprs = kernel.fn, dict(constexprs)
+ for name, heuristic in kernel.values.items():
+ constexprs[name] = heuristic(constexprs)
+ if isinstance(fn, triton.runtime.jit.JITFunction):
+ launch(fn, grid, args, n_tensors, constexprs, device.index, **options)
+ return
+ wrap(kernel)[grid](*args, **constexprs, **options)
+
+
# Off on ROCm (64-lane wavefronts reduce in another order) and before Triton 3.6 (slower backward).
def _triton_at_least(major, minor):
try:
@@ -475,7 +505,7 @@ def _multirow_settings(n_cols):
# Only widths 33..128 (one element per lane) can replay the one-row order; others stay one-row.
if not _MULTIROW:
return None
- BLOCK_SIZE, num_warps = calculate_settings(n_cols)
+ BLOCK_SIZE, num_warps = _settings(n_cols)
if num_warps != 4 or not (64 <= BLOCK_SIZE <= 128):
return None
return BLOCK_SIZE, _MULTIROW_ELEMENTS // BLOCK_SIZE, BLOCK_SIZE // 32, _MULTIROW_NUM_WARPS
@@ -565,30 +595,34 @@ def _rms_forward(
):
# multirow: None picks (self-checked), False forces one row, settings force multi-row.
n_rows, n_cols = X.shape
- Y = torch.empty((n_rows, n_cols), dtype = X.dtype, device = X.device)
- r = torch.empty(n_rows, dtype = torch.float32, device = X.device)
+ device = X.device
+ Y = torch.empty((n_rows, n_cols), dtype = X.dtype, device = device)
+ r = torch.empty(n_rows, dtype = torch.float32, device = device)
if multirow is None:
multirow = _multirow_checked(X, W, eps, gemma)
long = long_indexing(X, Y, block = _INDEX_MARGIN)
if multirow:
BLOCK_SIZE, ROWS, WARPS, num_warps = multirow
try:
- wrap(_rms_layernorm_forward_rows)[((n_rows + ROWS - 1) // ROWS,)](
- Y,
- Y.stride(0),
- X,
- X.stride(0),
- W,
- r,
- n_rows,
- n_cols,
- eps,
- GEMMA = gemma,
- BLOCK_SIZE = BLOCK_SIZE,
- ROWS = ROWS,
- WARPS = WARPS,
- # int32 rows measured slower only for the Gemma multi-row forward: keep main's int64.
- LONG_INDEXING = long or gemma,
+ _launch(
+ wrap,
+ _rms_layernorm_forward_rows,
+ ((n_rows + ROWS - 1) // ROWS,),
+ (Y, X, W, r, n_rows),
+ 4,
+ dict(
+ Y_row_stride = Y.stride(0),
+ X_row_stride = X.stride(0),
+ n_cols = n_cols,
+ eps = eps,
+ GEMMA = gemma,
+ BLOCK_SIZE = BLOCK_SIZE,
+ ROWS = ROWS,
+ WARPS = WARPS,
+ # int32 rows measured slower only for the Gemma multi-row forward: keep main's int64.
+ LONG_INDEXING = long or gemma,
+ ),
+ device,
num_warps = num_warps,
enable_fp_fusion = True,
)
@@ -597,21 +631,24 @@ def _rms_forward(
if _multirow_must_raise(error, wrap):
raise
_multirow_disable(f"launch failed: {error!r}")
- BLOCK_SIZE, num_warps = calculate_settings(n_cols)
- fx = _gemma_rms_layernorm_forward if gemma else _rms_layernorm_forward
- wrap(fx)[(n_rows,)](
- Y,
- Y.stride(0),
- X,
- X.stride(0),
- W,
- W.stride(0),
- r,
- r.stride(0),
- n_cols,
- eps,
- LONG_INDEXING = long,
- BLOCK_SIZE = BLOCK_SIZE,
+ BLOCK_SIZE, num_warps = _settings(n_cols)
+ _launch(
+ wrap,
+ _gemma_rms_layernorm_forward if gemma else _rms_layernorm_forward,
+ (n_rows,),
+ (Y, X, W, r),
+ 4,
+ dict(
+ Y_row_stride = Y.stride(0),
+ X_row_stride = X.stride(0),
+ W_row_stride = W.stride(0),
+ r_row_stride = r.stride(0),
+ n_cols = n_cols,
+ eps = eps,
+ LONG_INDEXING = long,
+ BLOCK_SIZE = BLOCK_SIZE,
+ ),
+ device,
num_warps = num_warps,
enable_fp_fusion = True,
)
@@ -638,8 +675,8 @@ def _max_sm_threads(device):
def _dw_split(units, max_programs):
- per_program = max(1, triton.cdiv(units, max(1, min(units, max_programs))))
- return max(1, triton.cdiv(units, per_program)), per_program
+ per_program = max(1, _cdiv(units, max(1, min(units, max_programs))))
+ return max(1, _cdiv(units, per_program)), per_program
def _dw_launch(units, device):
@@ -648,18 +685,17 @@ def _dw_launch(units, device):
@functools.lru_cache(maxsize = None)
def _dw_row_config(device, n_cols):
- # Cached (backward is host bound at small shapes). 64-warp SMs, <= 16 elems/lane: cap registers at 64
- # so all programs stay resident; uncapped, wide rows kept one program per SM. T4: no measured gain.
- BLOCK_SIZE, num_warps = calculate_settings(n_cols)
- per_sm, launch = _DW_PROGRAMS_PER_SM, {}
+ # Cached (host bound). 64-warp SMs, <= 16 elems/lane: cap registers at 64 so all programs stay resident.
+ BLOCK_SIZE, num_warps = _settings(n_cols)
+ per_sm, options = _DW_PROGRAMS_PER_SM, {}
if (
device.type == "cuda"
and not torch.version.hip
and BLOCK_SIZE <= 512 * num_warps
and _max_sm_threads(device) >= 2048
):
- per_sm, launch = max(1, 32 // num_warps), {"maxnreg": 64}
- return BLOCK_SIZE, num_warps, _sm_count(device) * per_sm, launch
+ per_sm, options = max(1, 32 // num_warps), {"maxnreg": 64}
+ return BLOCK_SIZE, num_warps, _sm_count(device) * per_sm, options
def _rms_backward(
@@ -676,37 +712,38 @@ def _rms_backward(
):
# Non-Gemma writes dX over dY (the kernel ignores dX); Gemma writes into dX.
n_rows, n_cols = dY.shape
+ device = dY.device
if multirow is None:
multirow = _multirow_checked(X, W, eps, gemma)
long = long_indexing(dY, dX, X, block = _INDEX_MARGIN)
partials = None
if multirow:
BLOCK_SIZE, ROWS, WARPS, num_warps = multirow
- programs, per_program = triton.cdiv(n_rows, ROWS), 1
+ programs, per_program = _cdiv(n_rows, ROWS), 1
if train_w:
- programs, per_program = _dw_launch(programs, dY.device)
- partials = torch.empty((programs, n_cols), dtype = torch.float32, device = dY.device)
+ programs, per_program = _dw_launch(programs, device)
+ partials = torch.empty((programs, n_cols), dtype = torch.float32, device = device)
try:
- wrap(_rms_layernorm_backward_rows)[(programs,)](
- dY,
- dY.stride(0),
- dX,
- dX.stride(0),
- X,
- X.stride(0),
- W,
- r,
- r if partials is None else partials,
- n_rows,
- per_program,
- n_cols,
- eps,
- GEMMA = gemma,
- BLOCK_SIZE = BLOCK_SIZE,
- ROWS = ROWS,
- WARPS = WARPS,
- LONG_INDEXING = long,
- TRAIN_W = train_w,
+ _launch(
+ wrap,
+ _rms_layernorm_backward_rows,
+ (programs,),
+ (dY, dX, X, W, r, r if partials is None else partials, n_rows, per_program),
+ 6,
+ dict(
+ dY_row_stride = dY.stride(0),
+ dX_row_stride = dX.stride(0),
+ X_row_stride = X.stride(0),
+ n_cols = n_cols,
+ eps = eps,
+ GEMMA = gemma,
+ BLOCK_SIZE = BLOCK_SIZE,
+ ROWS = ROWS,
+ WARPS = WARPS,
+ LONG_INDEXING = long,
+ TRAIN_W = train_w,
+ ),
+ device,
num_warps = num_warps,
enable_fp_fusion = True,
)
@@ -716,35 +753,35 @@ def _rms_backward(
_multirow_disable(f"launch failed: {error!r}")
else:
return _rms_sum_partials(partials, W, wrap)
- programs, per_program, launch = n_rows, 1, {}
+ programs, per_program, options = n_rows, 1, {}
if train_w:
- BLOCK_SIZE, num_warps, max_programs, launch = _dw_row_config(dY.device, int(n_cols))
+ BLOCK_SIZE, num_warps, max_programs, options = _dw_row_config(device, int(n_cols))
programs, per_program = _dw_split(n_rows, max_programs)
- partials = torch.empty((programs, n_cols), dtype = torch.float32, device = dY.device)
+ partials = torch.empty((programs, n_cols), dtype = torch.float32, device = device)
else:
- BLOCK_SIZE, num_warps = calculate_settings(n_cols)
- wrap(_rms_layernorm_backward)[(programs,)](
- dY,
- dY.stride(0),
- dX,
- dX.stride(0),
- X,
- X.stride(0),
- W,
- W.stride(0),
- r,
- r.stride(0),
- r if partials is None else partials,
- n_rows,
- per_program,
- n_cols,
- eps,
- GEMMA = gemma,
- LONG_INDEXING = long,
- BLOCK_SIZE = BLOCK_SIZE,
- TRAIN_W = train_w,
+ BLOCK_SIZE, num_warps = _settings(n_cols)
+ _launch(
+ wrap,
+ _rms_layernorm_backward,
+ (programs,),
+ (dY, dX, X, W, r, r if partials is None else partials, n_rows, per_program),
+ 6,
+ dict(
+ dY_row_stride = dY.stride(0),
+ dX_row_stride = dX.stride(0),
+ X_row_stride = X.stride(0),
+ W_row_stride = W.stride(0),
+ r_row_stride = r.stride(0),
+ n_cols = n_cols,
+ eps = eps,
+ GEMMA = gemma,
+ LONG_INDEXING = long,
+ BLOCK_SIZE = BLOCK_SIZE,
+ TRAIN_W = train_w,
+ ),
+ device,
num_warps = num_warps,
- **launch,
+ **options,
enable_fp_fusion = True,
)
return _rms_sum_partials(partials, W, wrap)
@@ -755,13 +792,14 @@ def _rms_sum_partials(partials, W, wrap):
return None
programs, n_cols = partials.shape
dW = torch.empty(n_cols, dtype = W.dtype, device = partials.device)
- wrap(_rms_sum_dw_partials)[(triton.cdiv(n_cols, 64),)](
- partials,
- dW,
- programs,
- n_cols,
- BLOCK_ROWS = 64,
- BLOCK_COLS = 64,
+ _launch(
+ wrap,
+ _rms_sum_dw_partials,
+ (_cdiv(n_cols, 64),),
+ (partials, dW, programs, n_cols),
+ 2,
+ dict(BLOCK_ROWS = 64, BLOCK_COLS = 64),
+ partials.device,
enable_fp_fusion = True,
)
return dW