Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Launch eager RMSNorm and LayerNorm kernels through triton_launch (#13889)
Daniel Hanb67f20f599dd
diff --git a/tests/test_norm_fast_launch.py b/tests/test_norm_fast_launch.pynew file mode 100644index 00000000000..3fcbcee3904--- /dev/null+++ b/tests/test_norm_fast_launch.py@@ -0,0 +1,185 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.++"""Eager RMSNorm / LayerNorm launches go through triton_launch (no Triton dispatch once warm), keep+kernel[grid] under torch.compile, and stay correct under CUDA graph capture, on a side stream and on a+device that is not the current one."""++import pathlib+import sys++import pytest+import torch++sys.path.insert(0, str(pathlib.Path(__file__).parent / "_shared"))+try:+ from real_accelerator import has_real_cuda+except Exception:+ has_real_cuda = torch.cuda.is_available++if not has_real_cuda():+ pytest.skip("needs a CUDA GPU", allow_module_level = True)++import triton+from unsloth.kernels import layernorm, rms_layernorm, triton_launch++BF16 = torch.cuda.get_device_capability()[0] >= 8+DTYPE = torch.bfloat16 if BF16 else torch.float16+FAST = triton_launch._ENABLED # off on HIP and with UNSLOTH_TRITON_FAST_LAUNCH=0+# (kind, width, gemma): 128 takes the multi-row kernels where enabled, 4096 / 1000 the one-row ones.+CASES = [+ ("rms", 128, False),+ ("rms", 128, True),+ ("rms", 4096, False),+ ("rms", 4096, True),+ ("ln", 1000, False),+ ("ln", 4096, False),+]+++def _module(kind, width, train, device = "cuda", bias = True):+ g = torch.Generator().manual_seed(width)+ weight = torch.nn.Parameter(torch.randn(width, generator = g).to(device, DTYPE), requires_grad = train)+ b = torch.nn.Parameter(torch.randn(width, generator = g).to(device, DTYPE), requires_grad = train)+ return weight, (b if kind == "ln" and bias else None)+++def _step(kind, gemma, x, weight, bias, dy):+ x = x.detach().requires_grad_()+ if kind == "rms":+ y = rms_layernorm.Fast_RMS_Layernorm.apply(x, weight, 1e-6, gemma)+ else:+ y = layernorm.Fast_Layernorm.apply(x, weight, bias, 1e-5)+ y.backward(dy.clone()) # non-Gemma RMSNorm writes dX over its dY+ grads = [p.grad for p in (weight, bias) if p is not None and p.requires_grad]+ return [y.detach(), x.grad] + grads+++def _clear(weight, bias):+ for p in (weight, bias):+ if p is not None:+ p.grad = None+++def _inputs(width, rows = 300, device = "cuda", seed = 0):+ g = torch.Generator().manual_seed(seed)+ return (+ torch.randn(2, rows, width, generator = g).to(device, DTYPE),+ torch.randn(2, rows, width, generator = g).to(device, DTYPE),+ )++[email protected](not FAST, reason = "fast launch disabled (HIP or UNSLOTH_TRITON_FAST_LAUNCH=0)")[email protected]("train", [False, True], ids = ["frozen", "trainable"])[email protected]("kind,width,gemma", CASES)+def test_warm_eager_call_skips_triton_dispatch(monkeypatch, kind, width, gemma, train):+ weight, bias = _module(kind, width, train)+ x, dy = _inputs(width)+ ref = _step(kind, gemma, x, weight, bias, dy)+ _clear(weight, bias)+ calls = []+ run = triton.runtime.jit.JITFunction.run+ monkeypatch.setattr(+ triton.runtime.jit.JITFunction, "run", lambda self, *a, **k: calls.append(self) or run(self, *a, **k)+ )+ got = _step(kind, gemma, x, weight, bias, dy)+ assert calls == [], [c.__name__ for c in calls]+ for a, b in zip(ref, got):+ assert torch.equal(a, b)++[email protected](not getattr(torch.library, "triton_op", None), reason = "needs triton_op")[email protected]("kind,width,gemma", [("rms", 128, False), ("rms", 4096, True), ("ln", 1000, False)])+def test_compiled_calls_never_use_fast_launch(monkeypatch, kind, width, gemma):+ calls = []+ for module in (rms_layernorm, layernorm):+ real = module.launch+ monkeypatch.setattr(module, "launch", lambda *a, real = real, **k: calls.append(a[0]) or real(*a, **k))+ # fp32: T4 cannot trace bf16 Triton.+ weight = torch.nn.Parameter(torch.randn(width, device = "cuda"))+ bias = torch.nn.Parameter(torch.randn(width, device = "cuda"))+ norm = torch.nn.Module()+ norm.weight, norm.bias, norm.eps, norm.variance_epsilon = weight, bias, 1e-5, 1e-6+ norm.elementwise_affine = True+ if kind == "rms":+ fn = lambda x: rms_layernorm.fast_rms_layernorm(norm, x, gemma = gemma)+ else:+ fn = lambda x: layernorm.fast_layernorm(norm, x)+ torch._dynamo.reset()+ x = torch.randn(4, 64, width, device = "cuda", requires_grad = True)+ torch.compile(fn, fullgraph = True)(x).sum().backward()+ assert calls == []+ assert x.grad is not None and weight.grad is not None+ fn(x.detach().requires_grad_()).sum().backward() # eager: same module, launch() taken+ assert calls++[email protected]("train", [False, True], ids = ["frozen", "trainable"])[email protected]("kind,width,gemma", CASES)+def test_cuda_graph_capture_replays(kind, width, gemma, train):+ weight, bias = _module(kind, width, train)+ x, dy = _inputs(width, seed = 1)+ for _ in range(2): # warm: multi-row self-check, compile, fast-launch entries+ _step(kind, gemma, x, weight, bias, dy)+ _clear(weight, bias)+ static_x, static_dy = x.clone(), dy.clone()+ stream = torch.cuda.Stream()+ stream.wait_stream(torch.cuda.current_stream())+ with torch.cuda.stream(stream):+ _step(kind, gemma, static_x, weight, bias, static_dy)+ _clear(weight, bias)+ torch.cuda.current_stream().wait_stream(stream)+ graph = torch.cuda.CUDAGraph()+ with torch.cuda.graph(graph):+ static_out = _step(kind, gemma, static_x, weight, bias, static_dy)+ for seed in (2, 3):+ new_x, new_dy = _inputs(width, seed = seed)+ static_x.copy_(new_x)+ static_dy.copy_(new_dy)+ graph.replay()+ torch.cuda.synchronize()+ _clear(weight, bias)+ ref = _step(kind, gemma, new_x, weight, bias, new_dy)+ _clear(weight, bias)+ for a, b in zip(ref, static_out):+ assert torch.equal(a, b)++[email protected]("train", [False, True], ids = ["frozen", "trainable"])[email protected]("kind,width,gemma", CASES)+def test_launches_follow_the_current_stream(kind, width, gemma, train):+ weight, bias = _module(kind, width, train)+ x, dy = _inputs(width, seed = 4)+ ref = _step(kind, gemma, x, weight, bias, dy)+ _clear(weight, bias)+ _step(kind, gemma, x, weight, bias, dy)+ _clear(weight, bias)+ side = torch.cuda.Stream()+ late_x = torch.zeros_like(x)+ torch.cuda.synchronize()+ with torch.cuda.stream(side):+ # A launch on any other stream would read late_x before the copy lands.+ torch.cuda._sleep(200_000_000)+ late_x.copy_(x)+ got = _step(kind, gemma, late_x, weight, bias, dy)+ side.synchronize()+ for a, b in zip(ref, got):+ assert torch.equal(a, b)++[email protected](torch.cuda.device_count() < 2, reason = "needs two GPUs")[email protected]("train", [False, True], ids = ["frozen", "trainable"])[email protected]("kind,width,gemma", CASES)+def test_second_device_with_first_current(kind, width, gemma, train):+ weight, bias = _module(kind, width, train, device = "cuda:1")+ x, dy = _inputs(width, device = "cuda:1", seed = 5)+ with torch.cuda.device(1):+ ref = [t.clone() for t in _step(kind, gemma, x, weight, bias, dy)]+ _clear(weight, bias)+ for _ in range(2): # the second call is the warm fast-launch one+ with torch.cuda.device(0):+ got = _step(kind, gemma, x, weight, bias, dy)+ _clear(weight, bias)+ torch.cuda.synchronize(1)+ for a, b in zip(ref, got):+ assert a.device == b.device and torch.equal(a, b)diff --git a/tests/test_triton_fp_fusion.py b/tests/test_triton_fp_fusion.pyindex 171005d150f..dbca7905bac 100644--- a/tests/test_triton_fp_fusion.py+++ b/tests/test_triton_fp_fusion.py@@ -25,7 +25,11 @@ _NOT_LAUNCHES = {"utils.py": {"plan"},"nf4.py": {"kernel"},"triton_launch.py": {"kernel"},+ "layernorm.py": {"kernel"},+ "rms_layernorm.py": {"wrap(kernel)"},}+# Launch helpers forwarding options to the excluded sites above: each call is checked instead.+_LAUNCH_HELPERS = {"layernorm.py": "_launch", "rms_layernorm.py": "_launch"}def _launch_sites():@@ -37,6 +41,9 @@ def _launch_sites():if "triton" not in src:continuefor node in ast.walk(ast.parse(src)):+ helper = _LAUNCH_HELPERS.get(path.name)+ if isinstance(node, ast.Call) and isinstance(node.func, ast.Name) and node.func.id == helper:+ yield f"{rel}:{node.lineno}:{helper}", nodeif isinstance(node, ast.Call) and isinstance(node.func, ast.Subscript):name = ast.get_source_segment(src, node.func.value)if name in _NOT_LAUNCHES.get(path.name, ()):diff --git a/unsloth/kernels/layernorm.py b/unsloth/kernels/layernorm.pyindex d693493c2e6..de5fde44044 100644--- a/unsloth/kernels/layernorm.py+++ b/unsloth/kernels/layernorm.py@@ -15,6 +15,7 @@ import tritonimport triton.language as tlimport torchfrom .utils import calculate_settings, long_indexing, torch_gpu_device+from .triton_launch import launchfrom unsloth_zoo.patching_utils import (patch_layernorm,)@@ -23,13 +24,13 @@ from unsloth_zoo.patching_utils import (@triton.jitdef layernorm_forward(Y,- Y_row_stride: tl.constexpr, # n_colsX,- X_row_stride: tl.constexpr, # n_colsW,b,r,mu,+ Y_row_stride: tl.constexpr, # n_cols+ X_row_stride: tl.constexpr, # n_colsn_cols: tl.constexpr,eps: tl.constexpr,LONG_INDEXING: tl.constexpr,@@ -86,13 +87,13 @@ def _layernorm_backward_row(dY, X, W_row, r, mu, row_idx, col_offsets, mask, n_c@triton.jitdef layernorm_backward(dY,- dY_row_stride: tl.constexpr, # n_colsX,- X_row_stride: tl.constexpr, # n_colsW,b,r,mu,+ dY_row_stride: tl.constexpr, # n_cols+ X_row_stride: tl.constexpr, # n_colsn_cols: tl.constexpr,eps: tl.constexpr,LONG_INDEXING: tl.constexpr,@@ -122,9 +123,7 @@ def layernorm_backward(@triton.jitdef layernorm_backward_dw(dY,- dY_row_stride: tl.constexpr,X,- X_row_stride: tl.constexpr,W,r,mu,@@ -132,6 +131,8 @@ def layernorm_backward_dw(dB,n_rows,rows_per_program,+ dY_row_stride: tl.constexpr,+ X_row_stride: tl.constexpr,n_cols: tl.constexpr,LONG_INDEXING: tl.constexpr,BLOCK_SIZE: tl.constexpr,@@ -203,6 +204,24 @@ def _dw_slots(device):return 128, 32+_cached_settings = functools.lru_cache(maxsize = None)(calculate_settings)+++def _settings(n_cols):+ # Cached per width for eager calls (host bound at small shapes); tracing calls it directly.+ if torch.compiler.is_compiling():+ return calculate_settings(n_cols)+ return _cached_settings(n_cols)+++def _launch(kernel, grid, args, n_tensors, constexprs, device, **options):+ # Eager CUDA skips Triton's arg binding (triton_launch.py); compiling calls keep kernel[grid].+ if device.type == "cuda" and not torch.compiler.is_compiling():+ launch(kernel, grid, args, n_tensors, constexprs, device.index, **options)+ else:+ kernel[grid](*args, **constexprs, **options)++class Fast_Layernorm(torch.autograd.Function):@staticmethoddef forward(ctx, X, W, b, eps):@@ -214,27 +233,28 @@ class Fast_Layernorm(torch.autograd.Function):has_bias = b is not Noneb = b.contiguous() if has_bias else Wn_rows, n_cols = X.shape- BLOCK_SIZE, num_warps = calculate_settings(n_cols)+ BLOCK_SIZE, num_warps = _settings(n_cols)device = X.deviceY = torch.empty((n_rows, n_cols), dtype = X.dtype, device = device)r = torch.empty(n_rows, dtype = torch.float32, device = device)mu = torch.empty(n_rows, dtype = torch.float32, device = device)with torch_gpu_device(device):- layernorm_forward[(n_rows,)](- Y,- Y.stride(0),- X,- X.stride(0),- W,- b,- r,- mu,- n_cols,- eps,- LONG_INDEXING = long_indexing(X, Y, block = BLOCK_SIZE),- BLOCK_SIZE = BLOCK_SIZE,- HAS_BIAS = has_bias,+ _launch(+ layernorm_forward,+ (n_rows,),+ (Y, X, W, b, r, mu),+ 6,+ dict(+ Y_row_stride = Y.stride(0),+ X_row_stride = X.stride(0),+ n_cols = n_cols,+ eps = eps,+ LONG_INDEXING = long_indexing(X, Y, block = BLOCK_SIZE),+ BLOCK_SIZE = BLOCK_SIZE,+ HAS_BIAS = has_bias,+ ),+ device,num_warps = num_warps,enable_fp_fusion = True,)@@ -251,77 +271,79 @@ class Fast_Layernorm(torch.autograd.Function):dY = dY.reshape(-1, dim).contiguous()X, W, b, r, mu = ctx.saved_tensorsn_rows, n_cols = dY.shape+ device = dY.devicetrain_w, train_b = ctx.needs_input_grad[1], ctx.needs_input_grad[2]LONG_INDEXING = long_indexing(dY, X, block = ctx.BLOCK_SIZE)if not (train_w or train_b):- with torch_gpu_device(dY.device):- layernorm_backward[(n_rows,)](- dY,- dY.stride(0),- X,- X.stride(0),- W,- b,- r,- mu,- n_cols,- ctx.eps,- LONG_INDEXING = LONG_INDEXING,- BLOCK_SIZE = ctx.BLOCK_SIZE,+ with torch_gpu_device(device):+ _launch(+ layernorm_backward,+ (n_rows,),+ (dY, X, W, b, r, mu),+ 6,+ dict(+ dY_row_stride = dY.stride(0),+ X_row_stride = X.stride(0),+ n_cols = n_cols,+ eps = ctx.eps,+ LONG_INDEXING = LONG_INDEXING,+ BLOCK_SIZE = ctx.BLOCK_SIZE,+ ),+ device,num_warps = ctx.num_warps,enable_fp_fusion = True,)return dY.view(*shape), None, None, None- sms, warps_per_sm = _dw_slots(dY.device)- programs = max(1, min(n_rows, sms * max(1, warps_per_sm // ctx.num_warps)))- rows_per_program = max(1, triton.cdiv(n_rows, programs))- programs = max(1, triton.cdiv(n_rows, rows_per_program))+ programs, rows_per_program = _dw_programs(device, n_rows, ctx.num_warps)# Every program stores its partial row, so empty suffices unless nothing launches.alloc = torch.empty if n_rows > 0 else torch.zerospartials = alloc(- (train_w + train_b, programs, n_cols), dtype = torch.float32, device = dY.device+ (train_w + train_b, programs, n_cols), dtype = torch.float32, device = device)- if n_rows > 0:- with torch_gpu_device(dY.device):- layernorm_backward_dw[(programs,)](- dY,- dY.stride(0),- X,- X.stride(0),- W,- r,- mu,- partials[0],- partials[-1],- n_rows,- rows_per_program,- n_cols,- LONG_INDEXING = LONG_INDEXING,- BLOCK_SIZE = ctx.BLOCK_SIZE,- TRAIN_W = train_w,- TRAIN_B = train_b,+ dW = torch.empty(n_cols, dtype = W.dtype, device = device) if train_w else None+ dB = torch.empty(n_cols, dtype = b.dtype, device = device) if train_b else None+ outs = [t for t in (dW, dB) if t is not None]+ with torch_gpu_device(device):+ if n_rows > 0:+ _launch(+ layernorm_backward_dw,+ (programs,),+ (dY, X, W, r, mu, partials[0], partials[-1], n_rows, rows_per_program),+ 7,+ dict(+ dY_row_stride = dY.stride(0),+ X_row_stride = X.stride(0),+ n_cols = n_cols,+ LONG_INDEXING = LONG_INDEXING,+ BLOCK_SIZE = ctx.BLOCK_SIZE,+ TRAIN_W = train_w,+ TRAIN_B = train_b,+ ),+ device,num_warps = ctx.num_warps,enable_fp_fusion = True,)- dW = torch.empty(n_cols, dtype = W.dtype, device = dY.device) if train_w else None- dB = torch.empty(n_cols, dtype = b.dtype, device = dY.device) if train_b else None- outs = [t for t in (dW, dB) if t is not None]- with torch_gpu_device(dY.device):- _layernorm_sum_partials[(triton.cdiv(n_cols, 64), len(outs))](- partials,- outs[0],- outs[-1],- programs,- n_cols,- BLOCK_ROWS = 64,- BLOCK_COLS = 64,+ _launch(+ _layernorm_sum_partials,+ ((n_cols + 63) // 64, len(outs)),+ (partials, outs[0], outs[-1], programs, n_cols),+ 3,+ dict(BLOCK_ROWS = 64, BLOCK_COLS = 64),+ device,enable_fp_fusion = True,)return dY.view(*shape), dW, dB, None+def _dw_programs(device, n_rows, num_warps):+ sms, warps_per_sm = _dw_slots(device)+ programs = max(1, min(n_rows, sms * max(1, warps_per_sm // num_warps)))+ rows_per_program = max(1, (n_rows + programs - 1) // programs)+ return max(1, (n_rows + rows_per_program - 1) // rows_per_program), rows_per_program++def fast_layernorm(layernorm, X):assert layernorm.elementwise_affine is TrueW = layernorm.weightdiff --git a/unsloth/kernels/rms_layernorm.py b/unsloth/kernels/rms_layernorm.pyindex 038cfd5f21c..2bd30a98c14 100644--- a/unsloth/kernels/rms_layernorm.py+++ b/unsloth/kernels/rms_layernorm.py@@ -18,17 +18,18 @@ import triton.language as tlimport torchfrom typing import Tuplefrom .utils import calculate_settings, long_indexing, torch_gpu_device+from .triton_launch import launch@triton.jitdef _rms_layernorm_forward(Y,- Y_row_stride: tl.constexpr,X,- X_row_stride: tl.constexpr,W,- W_row_stride: tl.constexpr,r,+ Y_row_stride: tl.constexpr,+ X_row_stride: tl.constexpr,+ W_row_stride: tl.constexpr,r_row_stride: tl.constexpr,n_cols: tl.constexpr,eps: tl.constexpr,@@ -96,7 +97,6 @@ def _rms_backward_row(X_row = tl.load(X + col_offsets, mask = mask, other = 0).to(tl.float32)W_row = tl.load(W + col_offsets, mask = mask, other = 0).to(tl.float32)- # Get saved row varianceinv_var = tl.load(r).to(tl.float32)normed = X_row * inv_varif TRAIN_W:@@ -120,18 +120,18 @@ def _rms_backward_row(def _rms_layernorm_backward(dY,- dY_row_stride: tl.constexpr,dX,- dX_row_stride: tl.constexpr,X,- X_row_stride: tl.constexpr,W,- W_row_stride: tl.constexpr,r,- r_row_stride: tl.constexpr,dW,n_rows,rows_per_program,+ dY_row_stride: tl.constexpr,+ dX_row_stride: tl.constexpr,+ X_row_stride: tl.constexpr,+ W_row_stride: tl.constexpr,+ r_row_stride: tl.constexpr,n_cols: tl.constexpr,eps: tl.constexpr,GEMMA: tl.constexpr,@@ -208,12 +208,12 @@ _rms_layernorm_backward = triton.heuristics(@triton.jitdef _gemma_rms_layernorm_forward(Y,- Y_row_stride: tl.constexpr,X,- X_row_stride: tl.constexpr,W,- W_row_stride: tl.constexpr,r,+ Y_row_stride: tl.constexpr,+ X_row_stride: tl.constexpr,+ W_row_stride: tl.constexpr,r_row_stride: tl.constexpr,n_cols: tl.constexpr,eps: tl.constexpr,@@ -273,12 +273,12 @@ def _row_dot_in_row_kernel_order(a, b, ROWS: tl.constexpr, WARPS: tl.constexpr):@triton.jitdef _rms_layernorm_forward_rows(Y,- Y_row_stride: tl.constexpr,X,- X_row_stride: tl.constexpr,W,r,n_rows,+ Y_row_stride: tl.constexpr,+ X_row_stride: tl.constexpr,n_cols: tl.constexpr,eps: tl.constexpr,GEMMA: tl.constexpr,@@ -373,16 +373,16 @@ def _rms_backward_rows_block(@triton.jitdef _rms_layernorm_backward_rows(dY,- dY_row_stride: tl.constexpr,dX,- dX_row_stride: tl.constexpr,X,- X_row_stride: tl.constexpr,W,r,dW,n_rows,blocks_per_program,+ dY_row_stride: tl.constexpr,+ dX_row_stride: tl.constexpr,+ X_row_stride: tl.constexpr,n_cols: tl.constexpr,eps: tl.constexpr,GEMMA: tl.constexpr,@@ -454,6 +454,36 @@ def _rms_sum_dw_partials(tl.store(dW + cols, tl.sum(acc, axis = 0).to(dW.dtype.element_ty), mask = col_mask)+def _cdiv(a, b):+ # triton.cdiv's formula without its call overhead (a ConstexprFunction on recent Triton).+ return (a + b - 1) // b+++_cached_settings = functools.lru_cache(maxsize = None)(calculate_settings)+++def _settings(n_cols):+ # Cached per width for eager calls (host bound at small shapes); tracing calls it directly.+ if torch.compiler.is_compiling():+ return calculate_settings(n_cols)+ return _cached_settings(n_cols)+++def _launch(wrap, kernel, grid, args, n_tensors, constexprs, device, **options):+ # Eager CUDA skips Triton's arg binding (triton_launch.py); traced / non-Triton keep kernel[grid].+ if wrap is _eager_kernel and device.type == "cuda" and not torch.compiler.is_compiling():+ fn = kernel+ if isinstance(kernel, triton.runtime.autotuner.Heuristics):+ # What Heuristics.run does; these heuristics only read their own constexpr.+ fn, constexprs = kernel.fn, dict(constexprs)+ for name, heuristic in kernel.values.items():+ constexprs[name] = heuristic(constexprs)+ if isinstance(fn, triton.runtime.jit.JITFunction):+ launch(fn, grid, args, n_tensors, constexprs, device.index, **options)+ return+ wrap(kernel)[grid](*args, **constexprs, **options)++# Off on ROCm (64-lane wavefronts reduce in another order) and before Triton 3.6 (slower backward).def _triton_at_least(major, minor):try:@@ -475,7 +505,7 @@ def _multirow_settings(n_cols):# Only widths 33..128 (one element per lane) can replay the one-row order; others stay one-row.if not _MULTIROW:return None- BLOCK_SIZE, num_warps = calculate_settings(n_cols)+ BLOCK_SIZE, num_warps = _settings(n_cols)if num_warps != 4 or not (64 <= BLOCK_SIZE <= 128):return Nonereturn BLOCK_SIZE, _MULTIROW_ELEMENTS // BLOCK_SIZE, BLOCK_SIZE // 32, _MULTIROW_NUM_WARPS@@ -565,30 +595,34 @@ def _rms_forward():# multirow: None picks (self-checked), False forces one row, settings force multi-row.n_rows, n_cols = X.shape- Y = torch.empty((n_rows, n_cols), dtype = X.dtype, device = X.device)- r = torch.empty(n_rows, dtype = torch.float32, device = X.device)+ device = X.device+ Y = torch.empty((n_rows, n_cols), dtype = X.dtype, device = device)+ r = torch.empty(n_rows, dtype = torch.float32, device = device)if multirow is None:multirow = _multirow_checked(X, W, eps, gemma)long = long_indexing(X, Y, block = _INDEX_MARGIN)if multirow:BLOCK_SIZE, ROWS, WARPS, num_warps = multirowtry:- wrap(_rms_layernorm_forward_rows)[((n_rows + ROWS - 1) // ROWS,)](- Y,- Y.stride(0),- X,- X.stride(0),- W,- r,- n_rows,- n_cols,- eps,- GEMMA = gemma,- BLOCK_SIZE = BLOCK_SIZE,- ROWS = ROWS,- WARPS = WARPS,- # int32 rows measured slower only for the Gemma multi-row forward: keep main's int64.- LONG_INDEXING = long or gemma,+ _launch(+ wrap,+ _rms_layernorm_forward_rows,+ ((n_rows + ROWS - 1) // ROWS,),+ (Y, X, W, r, n_rows),+ 4,+ dict(+ Y_row_stride = Y.stride(0),+ X_row_stride = X.stride(0),+ n_cols = n_cols,+ eps = eps,+ GEMMA = gemma,+ BLOCK_SIZE = BLOCK_SIZE,+ ROWS = ROWS,+ WARPS = WARPS,+ # int32 rows measured slower only for the Gemma multi-row forward: keep main's int64.+ LONG_INDEXING = long or gemma,+ ),+ device,num_warps = num_warps,enable_fp_fusion = True,)@@ -597,21 +631,24 @@ def _rms_forward(if _multirow_must_raise(error, wrap):raise_multirow_disable(f"launch failed: {error!r}")- BLOCK_SIZE, num_warps = calculate_settings(n_cols)- fx = _gemma_rms_layernorm_forward if gemma else _rms_layernorm_forward- wrap(fx)[(n_rows,)](- Y,- Y.stride(0),- X,- X.stride(0),- W,- W.stride(0),- r,- r.stride(0),- n_cols,- eps,- LONG_INDEXING = long,- BLOCK_SIZE = BLOCK_SIZE,+ BLOCK_SIZE, num_warps = _settings(n_cols)+ _launch(+ wrap,+ _gemma_rms_layernorm_forward if gemma else _rms_layernorm_forward,+ (n_rows,),+ (Y, X, W, r),+ 4,+ dict(+ Y_row_stride = Y.stride(0),+ X_row_stride = X.stride(0),+ W_row_stride = W.stride(0),+ r_row_stride = r.stride(0),+ n_cols = n_cols,+ eps = eps,+ LONG_INDEXING = long,+ BLOCK_SIZE = BLOCK_SIZE,+ ),+ device,num_warps = num_warps,enable_fp_fusion = True,)@@ -638,8 +675,8 @@ def _max_sm_threads(device):def _dw_split(units, max_programs):- per_program = max(1, triton.cdiv(units, max(1, min(units, max_programs))))- return max(1, triton.cdiv(units, per_program)), per_program+ per_program = max(1, _cdiv(units, max(1, min(units, max_programs))))+ return max(1, _cdiv(units, per_program)), per_programdef _dw_launch(units, device):@@ -648,18 +685,17 @@ def _dw_launch(units, device):@functools.lru_cache(maxsize = None)def _dw_row_config(device, n_cols):- # Cached (backward is host bound at small shapes). 64-warp SMs, <= 16 elems/lane: cap registers at 64- # so all programs stay resident; uncapped, wide rows kept one program per SM. T4: no measured gain.- BLOCK_SIZE, num_warps = calculate_settings(n_cols)- per_sm, launch = _DW_PROGRAMS_PER_SM, {}+ # Cached (host bound). 64-warp SMs, <= 16 elems/lane: cap registers at 64 so all programs stay resident.+ BLOCK_SIZE, num_warps = _settings(n_cols)+ per_sm, options = _DW_PROGRAMS_PER_SM, {}if (device.type == "cuda"and not torch.version.hipand BLOCK_SIZE <= 512 * num_warpsand _max_sm_threads(device) >= 2048):- per_sm, launch = max(1, 32 // num_warps), {"maxnreg": 64}- return BLOCK_SIZE, num_warps, _sm_count(device) * per_sm, launch+ per_sm, options = max(1, 32 // num_warps), {"maxnreg": 64}+ return BLOCK_SIZE, num_warps, _sm_count(device) * per_sm, optionsdef _rms_backward(@@ -676,37 +712,38 @@ def _rms_backward():# Non-Gemma writes dX over dY (the kernel ignores dX); Gemma writes into dX.n_rows, n_cols = dY.shape+ device = dY.deviceif multirow is None:multirow = _multirow_checked(X, W, eps, gemma)long = long_indexing(dY, dX, X, block = _INDEX_MARGIN)partials = Noneif multirow:BLOCK_SIZE, ROWS, WARPS, num_warps = multirow- programs, per_program = triton.cdiv(n_rows, ROWS), 1+ programs, per_program = _cdiv(n_rows, ROWS), 1if train_w:- programs, per_program = _dw_launch(programs, dY.device)- partials = torch.empty((programs, n_cols), dtype = torch.float32, device = dY.device)+ programs, per_program = _dw_launch(programs, device)+ partials = torch.empty((programs, n_cols), dtype = torch.float32, device = device)try:- wrap(_rms_layernorm_backward_rows)[(programs,)](- dY,- dY.stride(0),- dX,- dX.stride(0),- X,- X.stride(0),- W,- r,- r if partials is None else partials,- n_rows,- per_program,- n_cols,- eps,- GEMMA = gemma,- BLOCK_SIZE = BLOCK_SIZE,- ROWS = ROWS,- WARPS = WARPS,- LONG_INDEXING = long,- TRAIN_W = train_w,+ _launch(+ wrap,+ _rms_layernorm_backward_rows,+ (programs,),+ (dY, dX, X, W, r, r if partials is None else partials, n_rows, per_program),+ 6,+ dict(+ dY_row_stride = dY.stride(0),+ dX_row_stride = dX.stride(0),+ X_row_stride = X.stride(0),+ n_cols = n_cols,+ eps = eps,+ GEMMA = gemma,+ BLOCK_SIZE = BLOCK_SIZE,+ ROWS = ROWS,+ WARPS = WARPS,+ LONG_INDEXING = long,+ TRAIN_W = train_w,+ ),+ device,num_warps = num_warps,enable_fp_fusion = True,)@@ -716,35 +753,35 @@ def _rms_backward(_multirow_disable(f"launch failed: {error!r}")else:return _rms_sum_partials(partials, W, wrap)- programs, per_program, launch = n_rows, 1, {}+ programs, per_program, options = n_rows, 1, {}if train_w:- BLOCK_SIZE, num_warps, max_programs, launch = _dw_row_config(dY.device, int(n_cols))+ BLOCK_SIZE, num_warps, max_programs, options = _dw_row_config(device, int(n_cols))programs, per_program = _dw_split(n_rows, max_programs)- partials = torch.empty((programs, n_cols), dtype = torch.float32, device = dY.device)+ partials = torch.empty((programs, n_cols), dtype = torch.float32, device = device)else:- BLOCK_SIZE, num_warps = calculate_settings(n_cols)- wrap(_rms_layernorm_backward)[(programs,)](- dY,- dY.stride(0),- dX,- dX.stride(0),- X,- X.stride(0),- W,- W.stride(0),- r,- r.stride(0),- r if partials is None else partials,- n_rows,- per_program,- n_cols,- eps,- GEMMA = gemma,- LONG_INDEXING = long,- BLOCK_SIZE = BLOCK_SIZE,- TRAIN_W = train_w,+ BLOCK_SIZE, num_warps = _settings(n_cols)+ _launch(+ wrap,+ _rms_layernorm_backward,+ (programs,),+ (dY, dX, X, W, r, r if partials is None else partials, n_rows, per_program),+ 6,+ dict(+ dY_row_stride = dY.stride(0),+ dX_row_stride = dX.stride(0),+ X_row_stride = X.stride(0),+ W_row_stride = W.stride(0),+ r_row_stride = r.stride(0),+ n_cols = n_cols,+ eps = eps,+ GEMMA = gemma,+ LONG_INDEXING = long,+ BLOCK_SIZE = BLOCK_SIZE,+ TRAIN_W = train_w,+ ),+ device,num_warps = num_warps,- **launch,+ **options,enable_fp_fusion = True,)return _rms_sum_partials(partials, W, wrap)@@ -755,13 +792,14 @@ def _rms_sum_partials(partials, W, wrap):return Noneprograms, n_cols = partials.shapedW = torch.empty(n_cols, dtype = W.dtype, device = partials.device)- wrap(_rms_sum_dw_partials)[(triton.cdiv(n_cols, 64),)](- partials,- dW,- programs,- n_cols,- BLOCK_ROWS = 64,- BLOCK_COLS = 64,+ _launch(+ wrap,+ _rms_sum_dw_partials,+ (_cdiv(n_cols, 64),),+ (partials, dW, programs, n_cols),+ 2,+ dict(BLOCK_ROWS = 64, BLOCK_COLS = 64),+ partials.device,enable_fp_fusion = True,)return dW