Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

fix: materialize strided gradients before RMSNorm backward (#10617)

Chenghao Liu8a374359a18b
diff --git a/tests/test_rmsnorm_gradient_layout.py b/tests/test_rmsnorm_gradient_layout.py
new file mode 100644
index 00000000000..e687c140782
--- /dev/null
+++ b/tests/test_rmsnorm_gradient_layout.py
@@ -0,0 +1,112 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+import pytest
+import torch
+
+pytestmark = pytest.mark.gpu
+
+
[email protected](not torch.cuda.is_available(), reason = "CUDA Triton kernels required")
[email protected]("gemma", [False, True])
[email protected]("layout", ["contiguous", "columns", "rows", "expanded"])
[email protected]("dtype", [torch.float32, torch.float16, torch.bfloat16])
+def test_rmsnorm_backward_gradient_layout(gemma, layout, dtype):
+ from unsloth.kernels.rms_layernorm import Fast_RMS_Layernorm
+
+ torch.manual_seed(42)
+ shape = (1, 4, 64)
+ inputs = torch.randn(shape, device = "cuda", dtype = dtype)
+ weights = torch.rand(shape[-1], device = "cuda", dtype = dtype)
+ if layout == "columns":
+ grad = torch.randn(1, 4, 128, device = "cuda", dtype = dtype)[..., ::2]
+ elif layout == "rows":
+ grad = torch.randn(1, 8, 64, device = "cuda", dtype = dtype)[:, ::2]
+ elif layout == "expanded":
+ grad = torch.ones(1, device = "cuda", dtype = dtype).expand(shape)
+ else:
+ grad = torch.randn(shape, device = "cuda", dtype = dtype)
+ assert grad.is_contiguous() == (layout == "contiguous")
+ reference_inputs = inputs.clone().requires_grad_()
+ normalized = reference_inputs.float() * torch.rsqrt(
+ reference_inputs.float().square().mean(-1, keepdim = True) + 1e-6
+ )
+ if gemma:
+ expected = (normalized * (weights.float() + 1.0)).to(dtype)
+ else:
+ expected = normalized.to(dtype) * weights
+ expected.backward(grad.clone())
+
+ actual_inputs = inputs.clone().requires_grad_()
+ actual = Fast_RMS_Layernorm.apply(actual_inputs, weights, 1e-6, gemma)
+ actual.backward(grad)
+ torch.testing.assert_close(actual, expected, rtol = 1e-2, atol = 1e-3)
+ torch.testing.assert_close(actual_inputs.grad, reference_inputs.grad, rtol = 2e-2, atol = 1e-2)
+
+
[email protected](not torch.cuda.is_available(), reason = "CUDA Triton kernels required")
[email protected]("gemma", [False, True])
[email protected]("dtype", [torch.float32, torch.float16, torch.bfloat16])
+def test_rmsnorm_forward_column_strided_input(gemma, dtype):
+ """`x[..., ::2]` reshapes to a stride-2 view, not a copy, so the forward reads it wrong too."""
+ from unsloth.kernels.rms_layernorm import Fast_RMS_Layernorm
+
+ torch.manual_seed(42)
+ source = torch.randn(1, 4, 128, device = "cuda", dtype = dtype, requires_grad = True)
+ strided = source[..., ::2]
+ assert not strided.is_contiguous()
+ weights = torch.rand(strided.shape[-1], device = "cuda", dtype = dtype)
+
+ reference_source = source.detach().clone().requires_grad_()
+ reference_inputs = reference_source[..., ::2]
+ normalized = reference_inputs.float() * torch.rsqrt(
+ reference_inputs.float().square().mean(-1, keepdim = True) + 1e-6
+ )
+ if gemma:
+ expected = (normalized * (weights.float() + 1.0)).to(dtype)
+ else:
+ expected = normalized.to(dtype) * weights
+ expected.backward(torch.ones_like(expected))
+
+ actual = Fast_RMS_Layernorm.apply(strided, weights, 1e-6, gemma)
+ actual.backward(torch.ones_like(actual))
+ torch.testing.assert_close(actual, expected, rtol = 1e-2, atol = 1e-3)
+ torch.testing.assert_close(source.grad, reference_source.grad, rtol = 2e-2, atol = 1e-2)
+
+
[email protected](not torch.cuda.is_available(), reason = "CUDA Triton kernels required")
[email protected]("gemma", [False, True])
[email protected]("layout", ["contiguous", "columns", "expanded"])
[email protected]("dtype", [torch.float32, torch.float16, torch.bfloat16])
+def test_rmsnorm_weight_layout(gemma, layout, dtype):
+ """The kernels read the norm weight with unit stride, so it needs materializing as well."""
+ from unsloth.kernels.rms_layernorm import Fast_RMS_Layernorm
+
+ torch.manual_seed(42)
+ shape = (1, 4, 64)
+ dim = shape[-1]
+ inputs = torch.randn(shape, device = "cuda", dtype = dtype)
+ if layout == "columns":
+ weights = torch.rand(2 * dim, device = "cuda", dtype = dtype)[::2]
+ elif layout == "expanded":
+ weights = torch.rand(1, device = "cuda", dtype = dtype).expand(dim)
+ else:
+ weights = torch.rand(dim, device = "cuda", dtype = dtype)
+ assert weights.is_contiguous() == (layout == "contiguous")
+ grad = torch.randn(shape, device = "cuda", dtype = dtype)
+
+ reference_inputs = inputs.clone().requires_grad_()
+ normalized = reference_inputs.float() * torch.rsqrt(
+ reference_inputs.float().square().mean(-1, keepdim = True) + 1e-6
+ )
+ if gemma:
+ expected = (normalized * (weights.float() + 1.0)).to(dtype)
+ else:
+ expected = normalized.to(dtype) * weights
+ expected.backward(grad.clone())
+
+ actual_inputs = inputs.clone().requires_grad_()
+ actual = Fast_RMS_Layernorm.apply(actual_inputs, weights, 1e-6, gemma)
+ actual.backward(grad)
+ torch.testing.assert_close(actual, expected, rtol = 1e-2, atol = 1e-3)
+ torch.testing.assert_close(actual_inputs.grad, reference_inputs.grad, rtol = 2e-2, atol = 1e-2)
diff --git a/unsloth/kernels/rms_layernorm.py b/unsloth/kernels/rms_layernorm.py
index 13c8863332d..302dd7ddad2 100644
--- a/unsloth/kernels/rms_layernorm.py
+++ b/unsloth/kernels/rms_layernorm.py
@@ -89,7 +89,7 @@ def _rms_layernorm_backward(
r += row_idx * r_row_stride
if GEMMA:
- dX += row_idx * dY_row_stride
+ dX += row_idx * dX_row_stride
else:
dX = dY
@@ -168,7 +168,9 @@ class Fast_RMS_Layernorm(torch.autograd.Function):
):
shape = X.shape
dim: int = shape[-1]
- X = X.reshape(-1, dim)
+ X = X.reshape(-1, dim).contiguous()
+ # kernels read W at unit stride, and this W is the one saved for backward.
+ W = W.contiguous()
n_rows: int
n_cols: int
n_rows, n_cols = X.shape
@@ -207,7 +209,7 @@ class Fast_RMS_Layernorm(torch.autograd.Function):
def backward(ctx, dY: torch.Tensor):
shape = dY.shape
dim: int = shape[-1]
- dY = dY.reshape(-1, dim)
+ dY = dY.reshape(-1, dim).contiguous()
X, W, r = ctx.saved_tensors
n_rows: int
n_cols: int