Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Return weight and bias gradients from the fast LayerNorm, and support bias=False (#13801)

Daniel Hand13e2400dcf4
diff --git a/tests/test_layernorm_weight_grad.py b/tests/test_layernorm_weight_grad.py
new file mode 100644
index 00000000000..0dc7e5316cb
--- /dev/null
+++ b/tests/test_layernorm_weight_grad.py
@@ -0,0 +1,162 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+import pytest
+from real_accelerator import (
+ has_real_cuda,
+)
+import torch
+
+pytestmark = [
+ pytest.mark.gpu,
+ pytest.mark.skipif(not has_real_cuda(), reason = "CUDA Triton kernels required"),
+]
+
+TOL = {torch.float32: 1e-4, torch.float16: 2e-3, torch.bfloat16: 1.6e-2}
+
+
+def _layer(
+ cols,
+ dtype,
+ bias = True,
+ trainable = True,
+):
+ layer = torch.nn.LayerNorm(cols, bias = bias, device = "cuda", dtype = dtype)
+ with torch.no_grad():
+ layer.weight.uniform_(0.5, 1.5)
+ if bias:
+ layer.bias.uniform_(-0.5, 0.5)
+ layer.requires_grad_(trainable)
+ return layer
+
+
+def _fast(layer, inputs, grad):
+ from unsloth.kernels.layernorm import fast_layernorm
+
+ x = inputs.clone().requires_grad_()
+ y = fast_layernorm(layer, x)
+ y.backward(grad.clone())
+ return y.detach(), x.grad, layer.weight.grad, None if layer.bias is None else layer.bias.grad
+
+
+def _exact(layer, inputs, grad):
+ x = inputs.double().requires_grad_()
+ w = layer.weight.detach().double().requires_grad_()
+ b = None if layer.bias is None else layer.bias.detach().double().requires_grad_()
+ y = torch.nn.functional.layer_norm(x, (inputs.shape[-1],), w, b, layer.eps)
+ y.backward(grad.double())
+ return y, x.grad, w.grad, None if b is None else b.grad
+
+
+def _close(actual, exact, dtype):
+ err = (actual.double() - exact).norm() / exact.norm().clamp_min(1e-12)
+ assert err < TOL[dtype], err.item()
+
+
+# 3 rows is below the program count, 9000 above; 1000 is not a power of two.
[email protected]("bias", [True, False], ids = ["bias", "no_bias"])
[email protected]("dtype", [torch.float32, torch.float16, torch.bfloat16])
[email protected]("cols", [128, 384, 1000, 4096])
[email protected]("rows", [3, 9000])
+def test_layernorm_trainable_weight_and_bias_get_grads(rows, cols, dtype, bias):
+ torch.manual_seed(3407)
+ layer = _layer(cols, dtype, bias)
+ inputs = torch.randn(rows, cols, device = "cuda", dtype = dtype)
+ grad = torch.randn(rows, cols, device = "cuda", dtype = dtype)
+ y, dx, dw, db = _fast(layer, inputs, grad)
+ ey, edx, edw, edb = _exact(layer, inputs, grad)
+ assert dw is not None and dw.dtype == dtype and dw.shape == (cols,)
+ _close(y, ey, dtype)
+ _close(dx, edx, dtype)
+ _close(dw, edw, dtype)
+ if bias:
+ assert db is not None and db.dtype == dtype and db.shape == (cols,)
+ _close(db, edb, dtype)
+
+
[email protected]("dtype", [torch.float16, torch.bfloat16])
[email protected]("cols", [128, 1000, 4096])
+def test_layernorm_trainable_matches_frozen_y_and_dx(cols, dtype):
+ torch.manual_seed(0)
+ inputs = torch.randn(2050, cols, device = "cuda", dtype = dtype)
+ grad = torch.randn(2050, cols, device = "cuda", dtype = dtype)
+ layer = _layer(cols, dtype)
+ y, dx, _, _ = _fast(layer, inputs, grad)
+ layer.zero_grad(set_to_none = True)
+ layer.requires_grad_(False)
+ fy, fdx, fdw, fdb = _fast(layer, inputs, grad)
+ assert fdw is None and fdb is None
+ assert torch.equal(y, fy)
+ # Persistent loop contracts some products differently: <= 1 ulp.
+ ulp = 2**-7 if dtype == torch.bfloat16 else 2**-10
+ torch.testing.assert_close(dx.float(), fdx.float(), rtol = ulp, atol = 1e-6)
+
+
+def test_layernorm_only_bias_trainable():
+ torch.manual_seed(1)
+ layer = _layer(512, torch.bfloat16)
+ layer.weight.requires_grad_(False)
+ inputs = torch.randn(700, 512, device = "cuda", dtype = torch.bfloat16)
+ grad = torch.randn(700, 512, device = "cuda", dtype = torch.bfloat16)
+ _, _, dw, db = _fast(layer, inputs, grad)
+ assert dw is None
+ _close(db, grad.double().sum(0), torch.bfloat16)
+
+
+def test_layernorm_only_weight_trainable():
+ torch.manual_seed(5)
+ layer = _layer(512, torch.bfloat16)
+ layer.bias.requires_grad_(False)
+ inputs = torch.randn(700, 512, device = "cuda", dtype = torch.bfloat16)
+ grad = torch.randn(700, 512, device = "cuda", dtype = torch.bfloat16)
+ _, _, dw, db = _fast(layer, inputs, grad)
+ assert db is None
+ _close(dw, _exact(layer, inputs, grad)[2], torch.bfloat16)
+
+
+def test_layernorm_compiled_matches_eager_bytes():
+ from unsloth.kernels.layernorm import fast_layernorm
+
+ torch.manual_seed(6)
+ inputs = torch.randn(3000, 1000, device = "cuda", dtype = torch.bfloat16)
+ grad = torch.randn(3000, 1000, device = "cuda", dtype = torch.bfloat16)
+ layer = _layer(1000, torch.bfloat16)
+ eager = _fast(layer, inputs, grad)
+ layer.zero_grad(set_to_none = True)
+ compiled = torch.compile(lambda x: fast_layernorm(layer, x), fullgraph = True)
+ x = inputs.clone().requires_grad_()
+ y = compiled(x)
+ y.backward(grad.clone())
+ for a, b in zip(eager, (y.detach(), x.grad, layer.weight.grad, layer.bias.grad)):
+ assert torch.equal(a, b)
+
+
+def test_layernorm_dw_is_deterministic():
+ torch.manual_seed(2)
+ inputs = torch.randn(33000, 1024, device = "cuda", dtype = torch.bfloat16)
+ grad = torch.randn(33000, 1024, device = "cuda", dtype = torch.bfloat16)
+ runs = []
+ for _ in range(3):
+ layer = _layer(1024, torch.bfloat16)
+ torch.manual_seed(2)
+ runs.append(_fast(layer, inputs, grad)[2:])
+ for dw, db in runs[1:]:
+ assert torch.equal(dw, runs[0][0]) and torch.equal(db, runs[0][1])
+
+
+def test_layernorm_empty_input_gives_zero_grads():
+ layer = _layer(256, torch.float16)
+ inputs = torch.randn(0, 256, device = "cuda", dtype = torch.float16)
+ y, dx, dw, db = _fast(layer, inputs, torch.randn(0, 256, device = "cuda", dtype = torch.float16))
+ assert y.shape == (0, 256) and dx.shape == (0, 256)
+ assert dw is not None and not dw.any() and db is not None and not db.any()
+
+
+def test_layernorm_no_bias_forward_matches_torch():
+ torch.manual_seed(4)
+ layer = _layer(768, torch.float32, bias = False, trainable = False)
+ from unsloth.kernels.layernorm import fast_layernorm
+
+ inputs = torch.randn(4, 9, 768, device = "cuda")
+ expected = torch.nn.functional.layer_norm(inputs, (768,), layer.weight, None, layer.eps)
+ torch.testing.assert_close(fast_layernorm(layer, inputs), expected, rtol = 1e-5, atol = 1e-5)
diff --git a/unsloth/kernels/layernorm.py b/unsloth/kernels/layernorm.py
index 24f1a813943..a9a33c0e32f 100644
--- a/unsloth/kernels/layernorm.py
+++ b/unsloth/kernels/layernorm.py
@@ -10,6 +10,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+import functools
import triton
import triton.language as tl
import torch
@@ -33,6 +34,7 @@ def layernorm_forward(
eps: tl.constexpr,
LONG_INDEXING: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
+ HAS_BIAS: tl.constexpr = True,
):
row_idx = tl.program_id(0)
if LONG_INDEXING:
@@ -48,7 +50,6 @@ def layernorm_forward(
# Per torchtune's Fp32LayerNorm, all modules are in float32.
X_row = tl.load(X + col_offsets, mask = mask, other = 0).to(tl.float32)
W_row = tl.load(W + col_offsets, mask = mask, other = 0).to(tl.float32)
- b_row = tl.load(b + col_offsets, mask = mask, other = 0).to(tl.float32)
mean_X = tl.sum(X_row, axis = 0) / n_cols
# (X[0] - mean) == -mean, so mask it out.
@@ -59,10 +60,29 @@ def layernorm_forward(
inv_var = tl.math.rsqrt(row_var + eps_f32)
tl.store(r, inv_var)
tl.store(mu, mean_X)
- output = (XX * inv_var) * W_row + b_row
+ output = (XX * inv_var) * W_row
+ if HAS_BIAS:
+ output += tl.load(b + col_offsets, mask = mask, other = 0).to(tl.float32)
tl.store(Y + col_offsets, output, mask = mask)
+def _layernorm_backward_row(dY, X, W_row, r, mu, row_idx, col_offsets, mask, n_cols: tl.constexpr):
+ # Approximately follows karpathy/llm.c doc/layernorm/layernorm.md
+ # Per torchtune's Fp32LayerNorm, all modules are in float32.
+ dY_row = tl.load(dY + col_offsets, mask = mask, other = 0).to(tl.float32)
+ X_row = tl.load(X + col_offsets, mask = mask, other = 0).to(tl.float32)
+
+ inv_var = tl.load(r + row_idx).to(tl.float32)
+ mean = tl.load(mu + row_idx).to(tl.float32)
+ normed = (X_row - mean) * inv_var
+ dY_W = dY_row * W_row
+ dX_row = dY_W - tl.sum(dY_W, axis = 0) / n_cols - normed * tl.sum(dY_W * normed, axis = 0) / n_cols
+ dX_row = dX_row * inv_var
+ tl.store(dY + col_offsets, dX_row, mask = mask)
+ return dY_row, normed
+
+
@triton.jit
def layernorm_backward(
dY,
@@ -78,31 +98,109 @@ def layernorm_backward(
LONG_INDEXING: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
):
- # Approximately follows karpathy/llm.c doc/layernorm/layernorm.md
row_idx = tl.program_id(0)
if LONG_INDEXING:
row_idx = row_idx.to(tl.int64)
col_offsets = tl.arange(0, BLOCK_SIZE)
mask = col_offsets < n_cols
+ W_row = tl.load(W + col_offsets, mask = mask, other = 0).to(tl.float32)
+ _layernorm_backward_row(
+ dY + row_idx * dY_row_stride,
+ X + row_idx * X_row_stride,
+ W_row,
+ r,
+ mu,
+ row_idx,
+ col_offsets,
+ mask,
+ n_cols,
+ )
- dY += row_idx * dY_row_stride
- X += row_idx * X_row_stride
- r += row_idx
- mu += row_idx
- # Per torchtune's Fp32LayerNorm, all modules are in float32.
- dY_row = tl.load(dY + col_offsets, mask = mask, other = 0).to(tl.float32)
- X_row = tl.load(X + col_offsets, mask = mask, other = 0).to(tl.float32)
+# dW / dB follow flash-linear-attention's layer_norm_bwd_kernel_row (fla/modules/norm/layernorm.py,
+# MIT licence, Copyright (c) 2023-2026 Songlin Yang, Yu Zhang, Zhiyuan Li).
+def layernorm_backward_dw(
+ dY,
+ dY_row_stride: tl.constexpr,
+ X,
+ X_row_stride: tl.constexpr,
+ W,
+ r,
+ mu,
+ dW,
+ dB,
+ n_rows,
+ rows_per_program,
+ n_cols: tl.constexpr,
+ LONG_INDEXING: tl.constexpr,
+ BLOCK_SIZE: tl.constexpr,
+ TRAIN_W: tl.constexpr,
+ TRAIN_B: tl.constexpr,
+):
+ pid = tl.program_id(0)
+ col_offsets = tl.arange(0, BLOCK_SIZE)
+ mask = col_offsets < n_cols
W_row = tl.load(W + col_offsets, mask = mask, other = 0).to(tl.float32)
- b_row = tl.load(b + col_offsets, mask = mask, other = 0).to(tl.float32)
+ dW_acc = tl.zeros((BLOCK_SIZE,), dtype = tl.float32)
+ dB_acc = tl.zeros((BLOCK_SIZE,), dtype = tl.float32)
+ row_start = pid * rows_per_program
+ for i in range(row_start, tl.minimum(row_start + rows_per_program, n_rows)):
+ row_idx = i
+ if LONG_INDEXING:
+ row_idx = i.to(tl.int64)
+ dY_row, normed = _layernorm_backward_row(
+ dY + row_idx * dY_row_stride,
+ X + row_idx * X_row_stride,
+ W_row,
+ r,
+ mu,
+ row_idx,
+ col_offsets,
+ mask,
+ n_cols,
+ )
+ if TRAIN_W:
+ dW_acc += dY_row * normed
+ if TRAIN_B:
+ dB_acc += dY_row
+ if TRAIN_W:
+ tl.store(dW + pid * n_cols + col_offsets, dW_acc, mask = mask)
+ if TRAIN_B:
+ tl.store(dB + pid * n_cols + col_offsets, dB_acc, mask = mask)
- inv_var = tl.load(r).to(tl.float32)
- mean = tl.load(mu).to(tl.float32)
- normed = (X_row - mean) * inv_var
- dY_W = dY_row * W_row
- dX_row = dY_W - tl.sum(dY_W, axis = 0) / n_cols - normed * tl.sum(dY_W * normed, axis = 0) / n_cols
- dX_row = dX_row * inv_var
- tl.store(dY + col_offsets, dX_row, mask = mask)
+
+def _layernorm_sum_partials(
+ partials, out0, out1, n_programs, n_cols, BLOCK_ROWS: tl.constexpr, BLOCK_COLS: tl.constexpr
+):
+ # Not torch.sum: Inductor reorders it, so eager and compiled bytes would differ.
+ which = tl.program_id(1)
+ cols = tl.program_id(0) * BLOCK_COLS + tl.arange(0, BLOCK_COLS)
+ col_mask = cols < n_cols
+ partials += which * n_programs * n_cols
+ acc = tl.zeros((BLOCK_ROWS, BLOCK_COLS), dtype = tl.float32)
+ for start in tl.range(0, n_programs, BLOCK_ROWS, num_stages = 3):
+ rows = start + tl.arange(0, BLOCK_ROWS)
+ mask = (rows < n_programs)[:, None] & col_mask[None, :]
+ acc += tl.load(partials + rows[:, None] * n_cols + cols[None, :], mask = mask, other = 0)
+ total = tl.sum(acc, axis = 0)
+ if which == 0:
+ tl.store(out0 + cols, total.to(out0.dtype.element_ty), mask = col_mask)
+ else:
+ tl.store(out1 + cols, total.to(out1.dtype.element_ty), mask = col_mask)
+
+
+# Half of each SM's warps (not fla's one program per SM) measured best on A100 and T4.
[email protected]_cache(maxsize = None)
+def _dw_slots(device):
+ try:
+ props = torch.cuda.get_device_properties(device)
+ return int(props.multi_processor_count), max(
+ 1, int(props.max_threads_per_multi_processor) // 64
+ )
+ except Exception:
+ return 128, 32
class Fast_Layernorm(torch.autograd.Function):
@@ -113,7 +211,8 @@ class Fast_Layernorm(torch.autograd.Function):
X = X.reshape(-1, dim).contiguous()
# The kernels read W and b at unit stride, and these are the ones saved for backward.
W = W.contiguous()
- b = b.contiguous()
+ has_bias = b is not None
+ b = b.contiguous() if has_bias else W
n_rows, n_cols = X.shape
BLOCK_SIZE, num_warps = calculate_settings(n_cols)
device = X.device
@@ -135,6 +234,7 @@ class Fast_Layernorm(torch.autograd.Function):
eps,
LONG_INDEXING = long_indexing(X, Y, block = BLOCK_SIZE),
BLOCK_SIZE = BLOCK_SIZE,
+ HAS_BIAS = has_bias,
num_warps = num_warps,
)
ctx.eps = eps
@@ -150,25 +250,72 @@ class Fast_Layernorm(torch.autograd.Function):
dY = dY.reshape(-1, dim).contiguous()
X, W, b, r, mu = ctx.saved_tensors
n_rows, n_cols = dY.shape
+ train_w, train_b = ctx.needs_input_grad[1], ctx.needs_input_grad[2]
+ LONG_INDEXING = long_indexing(dY, X, block = ctx.BLOCK_SIZE)
+
+ if not (train_w or train_b):
+ with torch_gpu_device(dY.device):
+ layernorm_backward[(n_rows,)](
+ dY,
+ dY.stride(0),
+ X,
+ X.stride(0),
+ W,
+ b,
+ r,
+ mu,
+ n_cols,
+ ctx.eps,
+ LONG_INDEXING = LONG_INDEXING,
+ BLOCK_SIZE = ctx.BLOCK_SIZE,
+ num_warps = ctx.num_warps,
+ )
+ return dY.view(*shape), None, None, None
+ sms, warps_per_sm = _dw_slots(dY.device)
+ programs = max(1, min(n_rows, sms * max(1, warps_per_sm // ctx.num_warps)))
+ rows_per_program = max(1, triton.cdiv(n_rows, programs))
+ programs = max(1, triton.cdiv(n_rows, rows_per_program))
+ # Every program stores its partial row, so empty suffices unless nothing launches.
+ alloc = torch.empty if n_rows > 0 else torch.zeros
+ partials = alloc(
+ (train_w + train_b, programs, n_cols), dtype = torch.float32, device = dY.device
+ )
+ if n_rows > 0:
+ with torch_gpu_device(dY.device):
+ layernorm_backward_dw[(programs,)](
+ dY,
+ dY.stride(0),
+ X,
+ X.stride(0),
+ W,
+ r,
+ mu,
+ partials[0],
+ partials[-1],
+ n_rows,
+ rows_per_program,
+ n_cols,
+ LONG_INDEXING = LONG_INDEXING,
+ BLOCK_SIZE = ctx.BLOCK_SIZE,
+ TRAIN_W = train_w,
+ TRAIN_B = train_b,
+ num_warps = ctx.num_warps,
+ )
+ dW = torch.empty(n_cols, dtype = W.dtype, device = dY.device) if train_w else None
+ dB = torch.empty(n_cols, dtype = b.dtype, device = dY.device) if train_b else None
+ outs = [t for t in (dW, dB) if t is not None]
with torch_gpu_device(dY.device):
- layernorm_backward[(n_rows,)](
- dY,
- dY.stride(0),
- X,
- X.stride(0),
- W,
- b,
- r,
- mu,
+ _layernorm_sum_partials[(triton.cdiv(n_cols, 64), len(outs))](
+ partials,
+ outs[0],
+ outs[-1],
+ programs,
n_cols,
- ctx.eps,
- LONG_INDEXING = long_indexing(dY, X, block = ctx.BLOCK_SIZE),
- BLOCK_SIZE = ctx.BLOCK_SIZE,
- num_warps = ctx.num_warps,
+ BLOCK_ROWS = 64,
+ BLOCK_COLS = 64,
)
- dX = dY.view(*shape)
- return dX, None, None, None, None
+ return dY.view(*shape), dW, dB, None
def fast_layernorm(layernorm, X):