Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

fix: reshape noncontiguous gradients in fused LoRA backward (#10712)

Chenghao Liu8d4d52b748ad
diff --git a/tests/test_lora_gradient_layout.py b/tests/test_lora_gradient_layout.py
new file mode 100644
index 00000000000..341237d10ef
--- /dev/null
+++ b/tests/test_lora_gradient_layout.py
@@ -0,0 +1,104 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+import pytest
+import torch
+import torch.nn.functional as F
+from real_accelerator import has_real_cuda
+
+pytestmark = pytest.mark.gpu
+
+
[email protected](not has_real_cuda(), reason = "CUDA Triton kernels required")
[email protected]("kernel", ["qkv", "swiglu", "geglu", "geglu_approx"])
[email protected]("layout", ["contiguous", "transposed", "columns"])
[email protected]("dtype", [torch.float32, torch.float16, torch.bfloat16])
[email protected]("inplace", [False, True])
+def test_lora_backward_gradient_layout(kernel, layout, dtype, inplace):
+ # Triton rejects bf16 kernels below sm_80 (`.bf16 requires .target sm_80`), e.g. T4.
+ if (
+ dtype is torch.bfloat16
+ and not torch.version.hip
+ and torch.cuda.get_device_capability()[0] < 8
+ ):
+ pytest.skip("bf16 Triton kernels need sm_80+")
+ from unsloth.kernels.fast_lora import LoRA_MLP, LoRA_QKV
+ from unsloth.kernels.geglu import (
+ geglu_approx_backward_kernel,
+ geglu_approx_forward_kernel,
+ geglu_exact_backward_kernel,
+ geglu_exact_forward_kernel,
+ )
+ from unsloth.kernels.swiglu import swiglu_DWf_DW_dfg_kernel, swiglu_fg_kernel
+
+ torch.manual_seed(42)
+ inputs = torch.randn(2, 4, 64, device = "cuda", dtype = dtype)
+ widths = [(64, 64)] * 3 if kernel == "qkv" else [(64, 96), (64, 96), (96, 64)]
+ weights = []
+ adapters = []
+ scales = [0.25, 0.5, 0.75]
+ for in_features, out_features in widths:
+ weights.append(torch.randn(out_features, in_features, device = "cuda", dtype = dtype) * 0.05)
+ adapters.extend(
+ [
+ (torch.randn(8, in_features, device = "cuda", dtype = dtype) * 0.05).requires_grad_(),
+ (torch.randn(out_features, 8, device = "cuda", dtype = dtype) * 0.05).requires_grad_(),
+ ]
+ )
+ reference_adapters = [value.detach().clone().requires_grad_() for value in adapters]
+ reference_inputs = inputs.clone().requires_grad_()
+
+ def project(value, index):
+ a, b = reference_adapters[2 * index : 2 * index + 2]
+ return F.linear(value, weights[index]) + F.linear(F.linear(value, a), b) * scales[index]
+
+ if kernel == "qkv":
+ expected = tuple(project(reference_inputs, i) for i in range(3))
+ else:
+ gate, up = project(reference_inputs, 0), project(reference_inputs, 1)
+ if kernel == "swiglu":
+ activated = F.silu(gate.float()).to(dtype)
+ forward, backward = swiglu_fg_kernel, swiglu_DWf_DW_dfg_kernel
+ elif kernel == "geglu":
+ activated = F.gelu(gate.float()).to(dtype)
+ forward, backward = geglu_exact_forward_kernel, geglu_exact_backward_kernel
+ else:
+ activated = F.gelu(gate.float(), approximate = "tanh").to(dtype)
+ forward, backward = geglu_approx_forward_kernel, geglu_approx_backward_kernel
+ expected = (project(activated * up, 2),)
+
+ gradients = []
+ for output in expected:
+ if layout == "transposed":
+ gradient = torch.randn(4, 2, output.shape[-1], device = "cuda", dtype = dtype).transpose(
+ 0, 1
+ )
+ elif layout == "columns":
+ gradient = torch.randn(2, 4, output.shape[-1] * 2, device = "cuda", dtype = dtype)[..., ::2]
+ else:
+ gradient = torch.randn_like(output)
+ gradients.append(gradient)
+ torch.autograd.backward(expected, [gradient.clone() for gradient in gradients])
+
+ actual_inputs = inputs.clone().requires_grad_()
+ arguments = []
+ for i in range(3):
+ arguments.extend([weights[i], None, *adapters[2 * i : 2 * i + 2], scales[i]])
+ if kernel == "qkv":
+ actual = LoRA_QKV.apply(actual_inputs, *arguments, inplace)
+ else:
+ actual = (LoRA_MLP.apply(actual_inputs, *arguments, forward, backward, inplace),)
+ torch.autograd.backward(actual, gradients)
+
+ tolerance = {
+ torch.float32: (1e-4, 1e-5),
+ torch.float16: (1e-2, 1e-3),
+ torch.bfloat16: (5e-2, 1e-2),
+ }
+ rtol, atol = tolerance[dtype]
+ for result, reference in zip(actual, expected):
+ torch.testing.assert_close(result, reference, rtol = rtol, atol = atol)
+ for result, reference in zip(
+ [actual_inputs, *adapters], [reference_inputs, *reference_adapters]
+ ):
+ torch.testing.assert_close(result.grad, reference.grad, rtol = rtol, atol = atol)
diff --git a/unsloth/kernels/fast_lora.py b/unsloth/kernels/fast_lora.py
index c2784269a8e..7530948d1b8 100644
--- a/unsloth/kernels/fast_lora.py
+++ b/unsloth/kernels/fast_lora.py
@@ -151,7 +151,7 @@ class LoRA_MLP(torch.autograd.Function):
gateA, gateB, upA, upB, downA, downB, X, e, g = ctx.saved_tensors
batch, seq_len, hd = X.shape
- dY = dY.view(-1, dY.shape[-1])
+ dY = dY.reshape(-1, dY.shape[-1])
X = X.view(-1, X.shape[-1])
e = e.view(-1, e.shape[-1])
g = g.view(-1, g.shape[-1])
@@ -472,9 +472,9 @@ class LoRA_QKV(torch.autograd.Function):
) = ctx.saved_tensors
batch, seq_len, hd = X.shape
- dQ = dQ.view(-1, dQ.shape[-1])
+ dQ = dQ.reshape(-1, dQ.shape[-1])
dK = dK.reshape(-1, dK.shape[-1]) # view doesn't work on K.T
- dV = dV.view(-1, dV.shape[-1])
+ dV = dV.reshape(-1, dV.shape[-1])
X = X.view(-1, X.shape[-1])
dtype = X.dtype