Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Fast LayerNorm backward: no aliased gradient buffers when only one of weight / bias trains (#14609)

Daniel Han72da63a6e7fd
diff --git a/tests/test_layernorm_weight_grad.py b/tests/test_layernorm_weight_grad.py
index bc101b3e0ab..5905b539ec2 100644
--- a/tests/test_layernorm_weight_grad.py
+++ b/tests/test_layernorm_weight_grad.py
@@ -135,6 +135,38 @@ def test_layernorm_compiled_matches_eager_bytes(dtype):
assert torch.equal(a, b)
+# One trainable of the two: the unused gradient slot must not alias the used one under compile.
+ "bias, train_w, train_b",
+ [(True, False, True), (True, True, False), (False, True, False)],
+ ids = ["only_bias", "only_weight", "no_bias"],
+)
[email protected]("dtype", [torch.float32, torch.float16])
+def test_layernorm_compiled_single_trainable_matches_eager(dtype, bias, train_w, train_b):
+ from unsloth.kernels.layernorm import fast_layernorm
+
+ torch.manual_seed(7)
+ inputs = torch.randn(2, 300, 1000, device = "cuda", dtype = dtype)
+ grad = torch.randn(2, 300, 1000, device = "cuda", dtype = dtype)
+ layer = _layer(1000, dtype, bias = bias)
+ layer.weight.requires_grad_(train_w)
+ if bias:
+ layer.bias.requires_grad_(train_b)
+ eager = _fast(layer, inputs, grad)
+ layer.zero_grad(set_to_none = True)
+ torch._dynamo.reset()
+ compiled = torch.compile(lambda x: fast_layernorm(layer, x), fullgraph = True)
+ x = inputs.clone().requires_grad_()
+ y = compiled(x)
+ y.backward(grad.clone())
+ got = (y.detach(), x.grad, layer.weight.grad, None if layer.bias is None else layer.bias.grad)
+ torch._dynamo.reset()
+ for a, b in zip(eager, got):
+ assert (a is None) == (b is None)
+ if a is not None:
+ assert torch.equal(a, b)
+
+
def test_layernorm_dw_is_deterministic():
torch.manual_seed(2)
inputs = torch.randn(33000, 1024, device = "cuda", dtype = torch.bfloat16)
diff --git a/unsloth/kernels/layernorm.py b/unsloth/kernels/layernorm.py
index 756a9e964b9..70772ab6078 100644
--- a/unsloth/kernels/layernorm.py
+++ b/unsloth/kernels/layernorm.py
@@ -314,12 +314,17 @@ class Fast_Layernorm(torch.autograd.Function):
dW = torch.empty(n_cols, dtype = W.dtype, device = device) if train_w else None
dB = torch.empty(n_cols, dtype = b.dtype, device = device) if train_b else None
outs = [t for t in (dW, dB) if t is not None]
+ # Unused slot must not alias the used one: torch.compile drops one of two writes to a buffer.
+ spare = partials.new_empty(1) if len(outs) == 1 else None
+ partial_w = partials[0] if train_w else spare
+ partial_b = partials[-1] if train_b else spare
+ out1 = outs[-1] if len(outs) == 2 else outs[0].new_empty(1)
with torch_gpu_device(device):
if n_rows > 0:
_launch(
layernorm_backward_dw,
(programs,),
- (dY, dX, X, W, r, mu, partials[0], partials[-1], n_rows, rows_per_program),
+ (dY, dX, X, W, r, mu, partial_w, partial_b, n_rows, rows_per_program),
8,
dict(
dY_row_stride = dY.stride(0),
@@ -338,7 +343,7 @@ class Fast_Layernorm(torch.autograd.Function):
_launch(
_layernorm_sum_partials,
((n_cols + 63) // 64, len(outs)),
- (partials, outs[0], outs[-1], programs, n_cols),
+ (partials, outs[0], out1, programs, n_cols),
3,
dict(BLOCK_ROWS = 64, BLOCK_COLS = 64),
device,