Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
fix: materialize strided gradients before RMSNorm backward (#10617)
Chenghao Liu8a374359a18b
diff --git a/tests/test_rmsnorm_gradient_layout.py b/tests/test_rmsnorm_gradient_layout.pynew file mode 100644index 00000000000..e687c140782--- /dev/null+++ b/tests/test_rmsnorm_gradient_layout.py@@ -0,0 +1,112 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++import pytest+import torch++pytestmark = pytest.mark.gpu++[email protected](not torch.cuda.is_available(), reason = "CUDA Triton kernels required")[email protected]("gemma", [False, True])[email protected]("layout", ["contiguous", "columns", "rows", "expanded"])[email protected]("dtype", [torch.float32, torch.float16, torch.bfloat16])+def test_rmsnorm_backward_gradient_layout(gemma, layout, dtype):+ from unsloth.kernels.rms_layernorm import Fast_RMS_Layernorm++ torch.manual_seed(42)+ shape = (1, 4, 64)+ inputs = torch.randn(shape, device = "cuda", dtype = dtype)+ weights = torch.rand(shape[-1], device = "cuda", dtype = dtype)+ if layout == "columns":+ grad = torch.randn(1, 4, 128, device = "cuda", dtype = dtype)[..., ::2]+ elif layout == "rows":+ grad = torch.randn(1, 8, 64, device = "cuda", dtype = dtype)[:, ::2]+ elif layout == "expanded":+ grad = torch.ones(1, device = "cuda", dtype = dtype).expand(shape)+ else:+ grad = torch.randn(shape, device = "cuda", dtype = dtype)+ assert grad.is_contiguous() == (layout == "contiguous")+ reference_inputs = inputs.clone().requires_grad_()+ normalized = reference_inputs.float() * torch.rsqrt(+ reference_inputs.float().square().mean(-1, keepdim = True) + 1e-6+ )+ if gemma:+ expected = (normalized * (weights.float() + 1.0)).to(dtype)+ else:+ expected = normalized.to(dtype) * weights+ expected.backward(grad.clone())++ actual_inputs = inputs.clone().requires_grad_()+ actual = Fast_RMS_Layernorm.apply(actual_inputs, weights, 1e-6, gemma)+ actual.backward(grad)+ torch.testing.assert_close(actual, expected, rtol = 1e-2, atol = 1e-3)+ torch.testing.assert_close(actual_inputs.grad, reference_inputs.grad, rtol = 2e-2, atol = 1e-2)++[email protected](not torch.cuda.is_available(), reason = "CUDA Triton kernels required")[email protected]("gemma", [False, True])[email protected]("dtype", [torch.float32, torch.float16, torch.bfloat16])+def test_rmsnorm_forward_column_strided_input(gemma, dtype):+ """`x[..., ::2]` reshapes to a stride-2 view, not a copy, so the forward reads it wrong too."""+ from unsloth.kernels.rms_layernorm import Fast_RMS_Layernorm++ torch.manual_seed(42)+ source = torch.randn(1, 4, 128, device = "cuda", dtype = dtype, requires_grad = True)+ strided = source[..., ::2]+ assert not strided.is_contiguous()+ weights = torch.rand(strided.shape[-1], device = "cuda", dtype = dtype)++ reference_source = source.detach().clone().requires_grad_()+ reference_inputs = reference_source[..., ::2]+ normalized = reference_inputs.float() * torch.rsqrt(+ reference_inputs.float().square().mean(-1, keepdim = True) + 1e-6+ )+ if gemma:+ expected = (normalized * (weights.float() + 1.0)).to(dtype)+ else:+ expected = normalized.to(dtype) * weights+ expected.backward(torch.ones_like(expected))++ actual = Fast_RMS_Layernorm.apply(strided, weights, 1e-6, gemma)+ actual.backward(torch.ones_like(actual))+ torch.testing.assert_close(actual, expected, rtol = 1e-2, atol = 1e-3)+ torch.testing.assert_close(source.grad, reference_source.grad, rtol = 2e-2, atol = 1e-2)++[email protected](not torch.cuda.is_available(), reason = "CUDA Triton kernels required")[email protected]("gemma", [False, True])[email protected]("layout", ["contiguous", "columns", "expanded"])[email protected]("dtype", [torch.float32, torch.float16, torch.bfloat16])+def test_rmsnorm_weight_layout(gemma, layout, dtype):+ """The kernels read the norm weight with unit stride, so it needs materializing as well."""+ from unsloth.kernels.rms_layernorm import Fast_RMS_Layernorm++ torch.manual_seed(42)+ shape = (1, 4, 64)+ dim = shape[-1]+ inputs = torch.randn(shape, device = "cuda", dtype = dtype)+ if layout == "columns":+ weights = torch.rand(2 * dim, device = "cuda", dtype = dtype)[::2]+ elif layout == "expanded":+ weights = torch.rand(1, device = "cuda", dtype = dtype).expand(dim)+ else:+ weights = torch.rand(dim, device = "cuda", dtype = dtype)+ assert weights.is_contiguous() == (layout == "contiguous")+ grad = torch.randn(shape, device = "cuda", dtype = dtype)++ reference_inputs = inputs.clone().requires_grad_()+ normalized = reference_inputs.float() * torch.rsqrt(+ reference_inputs.float().square().mean(-1, keepdim = True) + 1e-6+ )+ if gemma:+ expected = (normalized * (weights.float() + 1.0)).to(dtype)+ else:+ expected = normalized.to(dtype) * weights+ expected.backward(grad.clone())++ actual_inputs = inputs.clone().requires_grad_()+ actual = Fast_RMS_Layernorm.apply(actual_inputs, weights, 1e-6, gemma)+ actual.backward(grad)+ torch.testing.assert_close(actual, expected, rtol = 1e-2, atol = 1e-3)+ torch.testing.assert_close(actual_inputs.grad, reference_inputs.grad, rtol = 2e-2, atol = 1e-2)diff --git a/unsloth/kernels/rms_layernorm.py b/unsloth/kernels/rms_layernorm.pyindex 13c8863332d..302dd7ddad2 100644--- a/unsloth/kernels/rms_layernorm.py+++ b/unsloth/kernels/rms_layernorm.py@@ -89,7 +89,7 @@ def _rms_layernorm_backward(r += row_idx * r_row_strideif GEMMA:- dX += row_idx * dY_row_stride+ dX += row_idx * dX_row_strideelse:dX = dY@@ -168,7 +168,9 @@ class Fast_RMS_Layernorm(torch.autograd.Function):):shape = X.shapedim: int = shape[-1]- X = X.reshape(-1, dim)+ X = X.reshape(-1, dim).contiguous()+ # kernels read W at unit stride, and this W is the one saved for backward.+ W = W.contiguous()n_rows: intn_cols: intn_rows, n_cols = X.shape@@ -207,7 +209,7 @@ class Fast_RMS_Layernorm(torch.autograd.Function):def backward(ctx, dY: torch.Tensor):shape = dY.shapedim: int = shape[-1]- dY = dY.reshape(-1, dim)+ dY = dY.reshape(-1, dim).contiguous()X, W, r = ctx.saved_tensorsn_rows: intn_cols: int