Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Return weight and bias gradients from the fast LayerNorm, and support bias=False (#13801)
Daniel Hand13e2400dcf4
diff --git a/tests/test_layernorm_weight_grad.py b/tests/test_layernorm_weight_grad.pynew file mode 100644index 00000000000..0dc7e5316cb--- /dev/null+++ b/tests/test_layernorm_weight_grad.py@@ -0,0 +1,162 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++import pytest+from real_accelerator import (+ has_real_cuda,+)+import torch++pytestmark = [+ pytest.mark.gpu,+ pytest.mark.skipif(not has_real_cuda(), reason = "CUDA Triton kernels required"),+]++TOL = {torch.float32: 1e-4, torch.float16: 2e-3, torch.bfloat16: 1.6e-2}+++def _layer(+ cols,+ dtype,+ bias = True,+ trainable = True,+):+ layer = torch.nn.LayerNorm(cols, bias = bias, device = "cuda", dtype = dtype)+ with torch.no_grad():+ layer.weight.uniform_(0.5, 1.5)+ if bias:+ layer.bias.uniform_(-0.5, 0.5)+ layer.requires_grad_(trainable)+ return layer+++def _fast(layer, inputs, grad):+ from unsloth.kernels.layernorm import fast_layernorm++ x = inputs.clone().requires_grad_()+ y = fast_layernorm(layer, x)+ y.backward(grad.clone())+ return y.detach(), x.grad, layer.weight.grad, None if layer.bias is None else layer.bias.grad+++def _exact(layer, inputs, grad):+ x = inputs.double().requires_grad_()+ w = layer.weight.detach().double().requires_grad_()+ b = None if layer.bias is None else layer.bias.detach().double().requires_grad_()+ y = torch.nn.functional.layer_norm(x, (inputs.shape[-1],), w, b, layer.eps)+ y.backward(grad.double())+ return y, x.grad, w.grad, None if b is None else b.grad+++def _close(actual, exact, dtype):+ err = (actual.double() - exact).norm() / exact.norm().clamp_min(1e-12)+ assert err < TOL[dtype], err.item()+++# 3 rows is below the program count, 9000 above; 1000 is not a power of two.[email protected]("bias", [True, False], ids = ["bias", "no_bias"])[email protected]("dtype", [torch.float32, torch.float16, torch.bfloat16])[email protected]("cols", [128, 384, 1000, 4096])[email protected]("rows", [3, 9000])+def test_layernorm_trainable_weight_and_bias_get_grads(rows, cols, dtype, bias):+ torch.manual_seed(3407)+ layer = _layer(cols, dtype, bias)+ inputs = torch.randn(rows, cols, device = "cuda", dtype = dtype)+ grad = torch.randn(rows, cols, device = "cuda", dtype = dtype)+ y, dx, dw, db = _fast(layer, inputs, grad)+ ey, edx, edw, edb = _exact(layer, inputs, grad)+ assert dw is not None and dw.dtype == dtype and dw.shape == (cols,)+ _close(y, ey, dtype)+ _close(dx, edx, dtype)+ _close(dw, edw, dtype)+ if bias:+ assert db is not None and db.dtype == dtype and db.shape == (cols,)+ _close(db, edb, dtype)++[email protected]("dtype", [torch.float16, torch.bfloat16])[email protected]("cols", [128, 1000, 4096])+def test_layernorm_trainable_matches_frozen_y_and_dx(cols, dtype):+ torch.manual_seed(0)+ inputs = torch.randn(2050, cols, device = "cuda", dtype = dtype)+ grad = torch.randn(2050, cols, device = "cuda", dtype = dtype)+ layer = _layer(cols, dtype)+ y, dx, _, _ = _fast(layer, inputs, grad)+ layer.zero_grad(set_to_none = True)+ layer.requires_grad_(False)+ fy, fdx, fdw, fdb = _fast(layer, inputs, grad)+ assert fdw is None and fdb is None+ assert torch.equal(y, fy)+ # Persistent loop contracts some products differently: <= 1 ulp.+ ulp = 2**-7 if dtype == torch.bfloat16 else 2**-10+ torch.testing.assert_close(dx.float(), fdx.float(), rtol = ulp, atol = 1e-6)+++def test_layernorm_only_bias_trainable():+ torch.manual_seed(1)+ layer = _layer(512, torch.bfloat16)+ layer.weight.requires_grad_(False)+ inputs = torch.randn(700, 512, device = "cuda", dtype = torch.bfloat16)+ grad = torch.randn(700, 512, device = "cuda", dtype = torch.bfloat16)+ _, _, dw, db = _fast(layer, inputs, grad)+ assert dw is None+ _close(db, grad.double().sum(0), torch.bfloat16)+++def test_layernorm_only_weight_trainable():+ torch.manual_seed(5)+ layer = _layer(512, torch.bfloat16)+ layer.bias.requires_grad_(False)+ inputs = torch.randn(700, 512, device = "cuda", dtype = torch.bfloat16)+ grad = torch.randn(700, 512, device = "cuda", dtype = torch.bfloat16)+ _, _, dw, db = _fast(layer, inputs, grad)+ assert db is None+ _close(dw, _exact(layer, inputs, grad)[2], torch.bfloat16)+++def test_layernorm_compiled_matches_eager_bytes():+ from unsloth.kernels.layernorm import fast_layernorm++ torch.manual_seed(6)+ inputs = torch.randn(3000, 1000, device = "cuda", dtype = torch.bfloat16)+ grad = torch.randn(3000, 1000, device = "cuda", dtype = torch.bfloat16)+ layer = _layer(1000, torch.bfloat16)+ eager = _fast(layer, inputs, grad)+ layer.zero_grad(set_to_none = True)+ compiled = torch.compile(lambda x: fast_layernorm(layer, x), fullgraph = True)+ x = inputs.clone().requires_grad_()+ y = compiled(x)+ y.backward(grad.clone())+ for a, b in zip(eager, (y.detach(), x.grad, layer.weight.grad, layer.bias.grad)):+ assert torch.equal(a, b)+++def test_layernorm_dw_is_deterministic():+ torch.manual_seed(2)+ inputs = torch.randn(33000, 1024, device = "cuda", dtype = torch.bfloat16)+ grad = torch.randn(33000, 1024, device = "cuda", dtype = torch.bfloat16)+ runs = []+ for _ in range(3):+ layer = _layer(1024, torch.bfloat16)+ torch.manual_seed(2)+ runs.append(_fast(layer, inputs, grad)[2:])+ for dw, db in runs[1:]:+ assert torch.equal(dw, runs[0][0]) and torch.equal(db, runs[0][1])+++def test_layernorm_empty_input_gives_zero_grads():+ layer = _layer(256, torch.float16)+ inputs = torch.randn(0, 256, device = "cuda", dtype = torch.float16)+ y, dx, dw, db = _fast(layer, inputs, torch.randn(0, 256, device = "cuda", dtype = torch.float16))+ assert y.shape == (0, 256) and dx.shape == (0, 256)+ assert dw is not None and not dw.any() and db is not None and not db.any()+++def test_layernorm_no_bias_forward_matches_torch():+ torch.manual_seed(4)+ layer = _layer(768, torch.float32, bias = False, trainable = False)+ from unsloth.kernels.layernorm import fast_layernorm++ inputs = torch.randn(4, 9, 768, device = "cuda")+ expected = torch.nn.functional.layer_norm(inputs, (768,), layer.weight, None, layer.eps)+ torch.testing.assert_close(fast_layernorm(layer, inputs), expected, rtol = 1e-5, atol = 1e-5)diff --git a/unsloth/kernels/layernorm.py b/unsloth/kernels/layernorm.pyindex 24f1a813943..a9a33c0e32f 100644--- a/unsloth/kernels/layernorm.py+++ b/unsloth/kernels/layernorm.py@@ -10,6 +10,7 @@# See the License for the specific language governing permissions and# limitations under the License.+import functoolsimport tritonimport triton.language as tlimport torch@@ -33,6 +34,7 @@ def layernorm_forward(eps: tl.constexpr,LONG_INDEXING: tl.constexpr,BLOCK_SIZE: tl.constexpr,+ HAS_BIAS: tl.constexpr = True,):row_idx = tl.program_id(0)if LONG_INDEXING:@@ -48,7 +50,6 @@ def layernorm_forward(# Per torchtune's Fp32LayerNorm, all modules are in float32.X_row = tl.load(X + col_offsets, mask = mask, other = 0).to(tl.float32)W_row = tl.load(W + col_offsets, mask = mask, other = 0).to(tl.float32)- b_row = tl.load(b + col_offsets, mask = mask, other = 0).to(tl.float32)mean_X = tl.sum(X_row, axis = 0) / n_cols# (X[0] - mean) == -mean, so mask it out.@@ -59,10 +60,29 @@ def layernorm_forward(inv_var = tl.math.rsqrt(row_var + eps_f32)tl.store(r, inv_var)tl.store(mu, mean_X)- output = (XX * inv_var) * W_row + b_row+ output = (XX * inv_var) * W_row+ if HAS_BIAS:+ output += tl.load(b + col_offsets, mask = mask, other = 0).to(tl.float32)tl.store(Y + col_offsets, output, mask = mask)+def _layernorm_backward_row(dY, X, W_row, r, mu, row_idx, col_offsets, mask, n_cols: tl.constexpr):+ # Approximately follows karpathy/llm.c doc/layernorm/layernorm.md+ # Per torchtune's Fp32LayerNorm, all modules are in float32.+ dY_row = tl.load(dY + col_offsets, mask = mask, other = 0).to(tl.float32)+ X_row = tl.load(X + col_offsets, mask = mask, other = 0).to(tl.float32)++ inv_var = tl.load(r + row_idx).to(tl.float32)+ mean = tl.load(mu + row_idx).to(tl.float32)+ normed = (X_row - mean) * inv_var+ dY_W = dY_row * W_row+ dX_row = dY_W - tl.sum(dY_W, axis = 0) / n_cols - normed * tl.sum(dY_W * normed, axis = 0) / n_cols+ dX_row = dX_row * inv_var+ tl.store(dY + col_offsets, dX_row, mask = mask)+ return dY_row, normed++@triton.jitdef layernorm_backward(dY,@@ -78,31 +98,109 @@ def layernorm_backward(LONG_INDEXING: tl.constexpr,BLOCK_SIZE: tl.constexpr,):- # Approximately follows karpathy/llm.c doc/layernorm/layernorm.mdrow_idx = tl.program_id(0)if LONG_INDEXING:row_idx = row_idx.to(tl.int64)col_offsets = tl.arange(0, BLOCK_SIZE)mask = col_offsets < n_cols+ W_row = tl.load(W + col_offsets, mask = mask, other = 0).to(tl.float32)+ _layernorm_backward_row(+ dY + row_idx * dY_row_stride,+ X + row_idx * X_row_stride,+ W_row,+ r,+ mu,+ row_idx,+ col_offsets,+ mask,+ n_cols,+ )- dY += row_idx * dY_row_stride- X += row_idx * X_row_stride- r += row_idx- mu += row_idx- # Per torchtune's Fp32LayerNorm, all modules are in float32.- dY_row = tl.load(dY + col_offsets, mask = mask, other = 0).to(tl.float32)- X_row = tl.load(X + col_offsets, mask = mask, other = 0).to(tl.float32)+# dW / dB follow flash-linear-attention's layer_norm_bwd_kernel_row (fla/modules/norm/layernorm.py,+# MIT licence, Copyright (c) 2023-2026 Songlin Yang, Yu Zhang, Zhiyuan Li).+def layernorm_backward_dw(+ dY,+ dY_row_stride: tl.constexpr,+ X,+ X_row_stride: tl.constexpr,+ W,+ r,+ mu,+ dW,+ dB,+ n_rows,+ rows_per_program,+ n_cols: tl.constexpr,+ LONG_INDEXING: tl.constexpr,+ BLOCK_SIZE: tl.constexpr,+ TRAIN_W: tl.constexpr,+ TRAIN_B: tl.constexpr,+):+ pid = tl.program_id(0)+ col_offsets = tl.arange(0, BLOCK_SIZE)+ mask = col_offsets < n_colsW_row = tl.load(W + col_offsets, mask = mask, other = 0).to(tl.float32)- b_row = tl.load(b + col_offsets, mask = mask, other = 0).to(tl.float32)+ dW_acc = tl.zeros((BLOCK_SIZE,), dtype = tl.float32)+ dB_acc = tl.zeros((BLOCK_SIZE,), dtype = tl.float32)+ row_start = pid * rows_per_program+ for i in range(row_start, tl.minimum(row_start + rows_per_program, n_rows)):+ row_idx = i+ if LONG_INDEXING:+ row_idx = i.to(tl.int64)+ dY_row, normed = _layernorm_backward_row(+ dY + row_idx * dY_row_stride,+ X + row_idx * X_row_stride,+ W_row,+ r,+ mu,+ row_idx,+ col_offsets,+ mask,+ n_cols,+ )+ if TRAIN_W:+ dW_acc += dY_row * normed+ if TRAIN_B:+ dB_acc += dY_row+ if TRAIN_W:+ tl.store(dW + pid * n_cols + col_offsets, dW_acc, mask = mask)+ if TRAIN_B:+ tl.store(dB + pid * n_cols + col_offsets, dB_acc, mask = mask)- inv_var = tl.load(r).to(tl.float32)- mean = tl.load(mu).to(tl.float32)- normed = (X_row - mean) * inv_var- dY_W = dY_row * W_row- dX_row = dY_W - tl.sum(dY_W, axis = 0) / n_cols - normed * tl.sum(dY_W * normed, axis = 0) / n_cols- dX_row = dX_row * inv_var- tl.store(dY + col_offsets, dX_row, mask = mask)++def _layernorm_sum_partials(+ partials, out0, out1, n_programs, n_cols, BLOCK_ROWS: tl.constexpr, BLOCK_COLS: tl.constexpr+):+ # Not torch.sum: Inductor reorders it, so eager and compiled bytes would differ.+ which = tl.program_id(1)+ cols = tl.program_id(0) * BLOCK_COLS + tl.arange(0, BLOCK_COLS)+ col_mask = cols < n_cols+ partials += which * n_programs * n_cols+ acc = tl.zeros((BLOCK_ROWS, BLOCK_COLS), dtype = tl.float32)+ for start in tl.range(0, n_programs, BLOCK_ROWS, num_stages = 3):+ rows = start + tl.arange(0, BLOCK_ROWS)+ mask = (rows < n_programs)[:, None] & col_mask[None, :]+ acc += tl.load(partials + rows[:, None] * n_cols + cols[None, :], mask = mask, other = 0)+ total = tl.sum(acc, axis = 0)+ if which == 0:+ tl.store(out0 + cols, total.to(out0.dtype.element_ty), mask = col_mask)+ else:+ tl.store(out1 + cols, total.to(out1.dtype.element_ty), mask = col_mask)+++# Half of each SM's warps (not fla's one program per SM) measured best on A100 and T4.[email protected]_cache(maxsize = None)+def _dw_slots(device):+ try:+ props = torch.cuda.get_device_properties(device)+ return int(props.multi_processor_count), max(+ 1, int(props.max_threads_per_multi_processor) // 64+ )+ except Exception:+ return 128, 32class Fast_Layernorm(torch.autograd.Function):@@ -113,7 +211,8 @@ class Fast_Layernorm(torch.autograd.Function):X = X.reshape(-1, dim).contiguous()# The kernels read W and b at unit stride, and these are the ones saved for backward.W = W.contiguous()- b = b.contiguous()+ has_bias = b is not None+ b = b.contiguous() if has_bias else Wn_rows, n_cols = X.shapeBLOCK_SIZE, num_warps = calculate_settings(n_cols)device = X.device@@ -135,6 +234,7 @@ class Fast_Layernorm(torch.autograd.Function):eps,LONG_INDEXING = long_indexing(X, Y, block = BLOCK_SIZE),BLOCK_SIZE = BLOCK_SIZE,+ HAS_BIAS = has_bias,num_warps = num_warps,)ctx.eps = eps@@ -150,25 +250,72 @@ class Fast_Layernorm(torch.autograd.Function):dY = dY.reshape(-1, dim).contiguous()X, W, b, r, mu = ctx.saved_tensorsn_rows, n_cols = dY.shape+ train_w, train_b = ctx.needs_input_grad[1], ctx.needs_input_grad[2]+ LONG_INDEXING = long_indexing(dY, X, block = ctx.BLOCK_SIZE)++ if not (train_w or train_b):+ with torch_gpu_device(dY.device):+ layernorm_backward[(n_rows,)](+ dY,+ dY.stride(0),+ X,+ X.stride(0),+ W,+ b,+ r,+ mu,+ n_cols,+ ctx.eps,+ LONG_INDEXING = LONG_INDEXING,+ BLOCK_SIZE = ctx.BLOCK_SIZE,+ num_warps = ctx.num_warps,+ )+ return dY.view(*shape), None, None, None+ sms, warps_per_sm = _dw_slots(dY.device)+ programs = max(1, min(n_rows, sms * max(1, warps_per_sm // ctx.num_warps)))+ rows_per_program = max(1, triton.cdiv(n_rows, programs))+ programs = max(1, triton.cdiv(n_rows, rows_per_program))+ # Every program stores its partial row, so empty suffices unless nothing launches.+ alloc = torch.empty if n_rows > 0 else torch.zeros+ partials = alloc(+ (train_w + train_b, programs, n_cols), dtype = torch.float32, device = dY.device+ )+ if n_rows > 0:+ with torch_gpu_device(dY.device):+ layernorm_backward_dw[(programs,)](+ dY,+ dY.stride(0),+ X,+ X.stride(0),+ W,+ r,+ mu,+ partials[0],+ partials[-1],+ n_rows,+ rows_per_program,+ n_cols,+ LONG_INDEXING = LONG_INDEXING,+ BLOCK_SIZE = ctx.BLOCK_SIZE,+ TRAIN_W = train_w,+ TRAIN_B = train_b,+ num_warps = ctx.num_warps,+ )+ dW = torch.empty(n_cols, dtype = W.dtype, device = dY.device) if train_w else None+ dB = torch.empty(n_cols, dtype = b.dtype, device = dY.device) if train_b else None+ outs = [t for t in (dW, dB) if t is not None]with torch_gpu_device(dY.device):- layernorm_backward[(n_rows,)](- dY,- dY.stride(0),- X,- X.stride(0),- W,- b,- r,- mu,+ _layernorm_sum_partials[(triton.cdiv(n_cols, 64), len(outs))](+ partials,+ outs[0],+ outs[-1],+ programs,n_cols,- ctx.eps,- LONG_INDEXING = long_indexing(dY, X, block = ctx.BLOCK_SIZE),- BLOCK_SIZE = ctx.BLOCK_SIZE,- num_warps = ctx.num_warps,+ BLOCK_ROWS = 64,+ BLOCK_COLS = 64,)- dX = dY.view(*shape)- return dX, None, None, None, None+ return dY.view(*shape), dW, dB, Nonedef fast_layernorm(layernorm, X):