Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Fast LayerNorm backward: no aliased gradient buffers when only one of weight / bias trains (#14609)
Daniel Han72da63a6e7fd
diff --git a/tests/test_layernorm_weight_grad.py b/tests/test_layernorm_weight_grad.pyindex bc101b3e0ab..5905b539ec2 100644--- a/tests/test_layernorm_weight_grad.py+++ b/tests/test_layernorm_weight_grad.py@@ -135,6 +135,38 @@ def test_layernorm_compiled_matches_eager_bytes(dtype):assert torch.equal(a, b)+# One trainable of the two: the unused gradient slot must not alias the used one under compile.+ "bias, train_w, train_b",+ [(True, False, True), (True, True, False), (False, True, False)],+ ids = ["only_bias", "only_weight", "no_bias"],+)[email protected]("dtype", [torch.float32, torch.float16])+def test_layernorm_compiled_single_trainable_matches_eager(dtype, bias, train_w, train_b):+ from unsloth.kernels.layernorm import fast_layernorm++ torch.manual_seed(7)+ inputs = torch.randn(2, 300, 1000, device = "cuda", dtype = dtype)+ grad = torch.randn(2, 300, 1000, device = "cuda", dtype = dtype)+ layer = _layer(1000, dtype, bias = bias)+ layer.weight.requires_grad_(train_w)+ if bias:+ layer.bias.requires_grad_(train_b)+ eager = _fast(layer, inputs, grad)+ layer.zero_grad(set_to_none = True)+ torch._dynamo.reset()+ compiled = torch.compile(lambda x: fast_layernorm(layer, x), fullgraph = True)+ x = inputs.clone().requires_grad_()+ y = compiled(x)+ y.backward(grad.clone())+ got = (y.detach(), x.grad, layer.weight.grad, None if layer.bias is None else layer.bias.grad)+ torch._dynamo.reset()+ for a, b in zip(eager, got):+ assert (a is None) == (b is None)+ if a is not None:+ assert torch.equal(a, b)++def test_layernorm_dw_is_deterministic():torch.manual_seed(2)inputs = torch.randn(33000, 1024, device = "cuda", dtype = torch.bfloat16)diff --git a/unsloth/kernels/layernorm.py b/unsloth/kernels/layernorm.pyindex 756a9e964b9..70772ab6078 100644--- a/unsloth/kernels/layernorm.py+++ b/unsloth/kernels/layernorm.py@@ -314,12 +314,17 @@ class Fast_Layernorm(torch.autograd.Function):dW = torch.empty(n_cols, dtype = W.dtype, device = device) if train_w else NonedB = torch.empty(n_cols, dtype = b.dtype, device = device) if train_b else Noneouts = [t for t in (dW, dB) if t is not None]+ # Unused slot must not alias the used one: torch.compile drops one of two writes to a buffer.+ spare = partials.new_empty(1) if len(outs) == 1 else None+ partial_w = partials[0] if train_w else spare+ partial_b = partials[-1] if train_b else spare+ out1 = outs[-1] if len(outs) == 2 else outs[0].new_empty(1)with torch_gpu_device(device):if n_rows > 0:_launch(layernorm_backward_dw,(programs,),- (dY, dX, X, W, r, mu, partials[0], partials[-1], n_rows, rows_per_program),+ (dY, dX, X, W, r, mu, partial_w, partial_b, n_rows, rows_per_program),8,dict(dY_row_stride = dY.stride(0),@@ -338,7 +343,7 @@ class Fast_Layernorm(torch.autograd.Function):_launch(_layernorm_sum_partials,((n_cols + 63) // 64, len(outs)),- (partials, outs[0], outs[-1], programs, n_cols),+ (partials, outs[0], out1, programs, n_cols),3,dict(BLOCK_ROWS = 64, BLOCK_COLS = 64),device,