Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
fix: reshape noncontiguous gradients in fused LoRA backward (#10712)
Chenghao Liu8d4d52b748ad
diff --git a/tests/test_lora_gradient_layout.py b/tests/test_lora_gradient_layout.pynew file mode 100644index 00000000000..341237d10ef--- /dev/null+++ b/tests/test_lora_gradient_layout.py@@ -0,0 +1,104 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++import pytest+import torch+import torch.nn.functional as F+from real_accelerator import has_real_cuda++pytestmark = pytest.mark.gpu++[email protected](not has_real_cuda(), reason = "CUDA Triton kernels required")[email protected]("kernel", ["qkv", "swiglu", "geglu", "geglu_approx"])[email protected]("layout", ["contiguous", "transposed", "columns"])[email protected]("dtype", [torch.float32, torch.float16, torch.bfloat16])[email protected]("inplace", [False, True])+def test_lora_backward_gradient_layout(kernel, layout, dtype, inplace):+ # Triton rejects bf16 kernels below sm_80 (`.bf16 requires .target sm_80`), e.g. T4.+ if (+ dtype is torch.bfloat16+ and not torch.version.hip+ and torch.cuda.get_device_capability()[0] < 8+ ):+ pytest.skip("bf16 Triton kernels need sm_80+")+ from unsloth.kernels.fast_lora import LoRA_MLP, LoRA_QKV+ from unsloth.kernels.geglu import (+ geglu_approx_backward_kernel,+ geglu_approx_forward_kernel,+ geglu_exact_backward_kernel,+ geglu_exact_forward_kernel,+ )+ from unsloth.kernels.swiglu import swiglu_DWf_DW_dfg_kernel, swiglu_fg_kernel++ torch.manual_seed(42)+ inputs = torch.randn(2, 4, 64, device = "cuda", dtype = dtype)+ widths = [(64, 64)] * 3 if kernel == "qkv" else [(64, 96), (64, 96), (96, 64)]+ weights = []+ adapters = []+ scales = [0.25, 0.5, 0.75]+ for in_features, out_features in widths:+ weights.append(torch.randn(out_features, in_features, device = "cuda", dtype = dtype) * 0.05)+ adapters.extend(+ [+ (torch.randn(8, in_features, device = "cuda", dtype = dtype) * 0.05).requires_grad_(),+ (torch.randn(out_features, 8, device = "cuda", dtype = dtype) * 0.05).requires_grad_(),+ ]+ )+ reference_adapters = [value.detach().clone().requires_grad_() for value in adapters]+ reference_inputs = inputs.clone().requires_grad_()++ def project(value, index):+ a, b = reference_adapters[2 * index : 2 * index + 2]+ return F.linear(value, weights[index]) + F.linear(F.linear(value, a), b) * scales[index]++ if kernel == "qkv":+ expected = tuple(project(reference_inputs, i) for i in range(3))+ else:+ gate, up = project(reference_inputs, 0), project(reference_inputs, 1)+ if kernel == "swiglu":+ activated = F.silu(gate.float()).to(dtype)+ forward, backward = swiglu_fg_kernel, swiglu_DWf_DW_dfg_kernel+ elif kernel == "geglu":+ activated = F.gelu(gate.float()).to(dtype)+ forward, backward = geglu_exact_forward_kernel, geglu_exact_backward_kernel+ else:+ activated = F.gelu(gate.float(), approximate = "tanh").to(dtype)+ forward, backward = geglu_approx_forward_kernel, geglu_approx_backward_kernel+ expected = (project(activated * up, 2),)++ gradients = []+ for output in expected:+ if layout == "transposed":+ gradient = torch.randn(4, 2, output.shape[-1], device = "cuda", dtype = dtype).transpose(+ 0, 1+ )+ elif layout == "columns":+ gradient = torch.randn(2, 4, output.shape[-1] * 2, device = "cuda", dtype = dtype)[..., ::2]+ else:+ gradient = torch.randn_like(output)+ gradients.append(gradient)+ torch.autograd.backward(expected, [gradient.clone() for gradient in gradients])++ actual_inputs = inputs.clone().requires_grad_()+ arguments = []+ for i in range(3):+ arguments.extend([weights[i], None, *adapters[2 * i : 2 * i + 2], scales[i]])+ if kernel == "qkv":+ actual = LoRA_QKV.apply(actual_inputs, *arguments, inplace)+ else:+ actual = (LoRA_MLP.apply(actual_inputs, *arguments, forward, backward, inplace),)+ torch.autograd.backward(actual, gradients)++ tolerance = {+ torch.float32: (1e-4, 1e-5),+ torch.float16: (1e-2, 1e-3),+ torch.bfloat16: (5e-2, 1e-2),+ }+ rtol, atol = tolerance[dtype]+ for result, reference in zip(actual, expected):+ torch.testing.assert_close(result, reference, rtol = rtol, atol = atol)+ for result, reference in zip(+ [actual_inputs, *adapters], [reference_inputs, *reference_adapters]+ ):+ torch.testing.assert_close(result.grad, reference.grad, rtol = rtol, atol = atol)diff --git a/unsloth/kernels/fast_lora.py b/unsloth/kernels/fast_lora.pyindex c2784269a8e..7530948d1b8 100644--- a/unsloth/kernels/fast_lora.py+++ b/unsloth/kernels/fast_lora.py@@ -151,7 +151,7 @@ class LoRA_MLP(torch.autograd.Function):gateA, gateB, upA, upB, downA, downB, X, e, g = ctx.saved_tensorsbatch, seq_len, hd = X.shape- dY = dY.view(-1, dY.shape[-1])+ dY = dY.reshape(-1, dY.shape[-1])X = X.view(-1, X.shape[-1])e = e.view(-1, e.shape[-1])g = g.view(-1, g.shape[-1])@@ -472,9 +472,9 @@ class LoRA_QKV(torch.autograd.Function):) = ctx.saved_tensorsbatch, seq_len, hd = X.shape- dQ = dQ.view(-1, dQ.shape[-1])+ dQ = dQ.reshape(-1, dQ.shape[-1])dK = dK.reshape(-1, dK.shape[-1]) # view doesn't work on K.T- dV = dV.view(-1, dV.shape[-1])+ dV = dV.reshape(-1, dV.shape[-1])X = X.view(-1, X.shape[-1])dtype = X.dtype