Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Fused LoRA backward: skip the gradient GEMMs of a frozen A or B (#16752)
Daniel Han16060aba73c6
diff --git a/tests/test_fused_lora_frozen_factor_grads.py b/tests/test_fused_lora_frozen_factor_grads.pynew file mode 100644index 00000000000..4835d8e61ff--- /dev/null+++ b/tests/test_fused_lora_frozen_factor_grads.py@@ -0,0 +1,149 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""Fused LoRA backward skips a frozen A / B's GEMMs; dX and the other gradients stay bit for bit."""++import itertools++import pytest+import torch+import unsloth # noqa: F401+from torch.utils._python_dispatch import TorchDispatchMode++from unsloth.kernels.fast_lora import LoRA_QKV, LoRA_W, _shared_lora_backward++_GEMMS = {"mm", "addmm", "addmm_", "bmm", "matmul"}+++class _CountGemms(TorchDispatchMode):+ def __init__(self):+ super().__init__()+ self.count = 0++ def __torch_dispatch__(+ self,+ func,+ types,+ args = (),+ kwargs = None,+ ):+ if func.overloadpacket.__name__ in _GEMMS:+ self.count += 1+ return func(*args, **(kwargs or {}))+++def _tensors(+ n_proj,+ dtype,+ seed = 0,+):+ g = torch.Generator().manual_seed(seed)+ X = torch.randn(2, 5, 48, generator = g).to(dtype)+ weights, factors = [], []+ for _ in range(n_proj):+ weights.append((torch.randn(40, 48, generator = g) * 0.05).to(dtype))+ factors.append((torch.randn(8, 48, generator = g) * 0.1).to(dtype))+ factors.append((torch.randn(40, 8, generator = g) * 0.1).to(dtype))+ grads = [torch.randn(2, 5, 40, generator = g).to(dtype) for _ in range(n_proj)]+ return X, weights, factors, grads+++def _run(+ cls,+ X,+ weights,+ factors,+ grads,+ trainable,+ dropout_p = None,+):+ X = X.clone().requires_grad_(True)+ factors = [f.clone().requires_grad_(t) for f, t in zip(factors, trainable)]+ scales = [0.5, 0.25, 0.75]+ if cls is LoRA_W:+ args = (X, weights[0], None, factors[0], factors[1], scales[0])+ if dropout_p is not None:+ args = args + (dropout_p,)+ else:+ args = [X]+ for i in range(3):+ args += [weights[i], None, factors[2 * i], factors[2 * i + 1], scales[0]]+ args += [False]+ if dropout_p is not None:+ args += [(dropout_p,) * 3]+ torch.manual_seed(1) # same dropout mask in every run+ outs = cls.apply(*args)+ outs = outs if isinstance(outs, tuple) else (outs,)+ with _CountGemms() as counter:+ torch.autograd.backward(outs, grads)+ return X.grad, [f.grad for f in factors], counter.count++[email protected]("dtype", [torch.float32, torch.bfloat16])[email protected]("dropout_p", [None, 0.25])+def test_lora_w_frozen_factor_skips_its_gemm_and_keeps_the_rest(dtype, dropout_p):+ X, weights, factors, grads = _tensors(1, dtype)+ dX_ref, (dA_ref, dB_ref), n_ref = _run(+ LoRA_W, X, weights, factors, grads, (True, True), dropout_p+ )+ for trainable in [(False, True), (True, False), (False, False)]:+ dX, (dA, dB), n = _run(LoRA_W, X, weights, factors, grads, trainable, dropout_p)+ assert torch.equal(dX, dX_ref)+ assert (dA is None) != trainable[0] and (dB is None) != trainable[1]+ if trainable[0]:+ assert torch.equal(dA, dA_ref)+ if trainable[1]:+ assert torch.equal(dB, dB_ref)+ assert n == n_ref - (0 if trainable[0] else 1) - (0 if trainable[1] else 2), (+ trainable,+ n,+ n_ref,+ )++[email protected]("dtype", [torch.float32, torch.bfloat16])+def test_lora_qkv_frozen_factors_keep_the_rest_bit_for_bit(dtype):+ X, weights, factors, grads = _tensors(3, dtype)+ dX_ref, f_ref, n_ref = _run(LoRA_QKV, X, weights, factors, grads, (True,) * 6)+ for trainable in [+ (False, True) * 3,+ (True, False) * 3,+ (False, True, True, True, True, False),+ (False,) * 6,+ ]:+ dX, f, n = _run(LoRA_QKV, X, weights, factors, grads, trainable)+ assert torch.equal(dX, dX_ref)+ for got, ref, t in zip(f, f_ref, trainable):+ assert (got is None) != t+ if t:+ assert torch.equal(got, ref)+ assert n < n_ref, (trainable, n, n_ref)+++ "need_A, need_B",+ list(+ itertools.product(+ [(True, True, True), (False, False, False), (True, False, True)],+ [(True, True, True), (False, False, False), (False, True, False)],+ )+ ),+)+def test_shared_backward_skips_frozen_factor_gemms(need_A, need_B):+ g = torch.Generator().manual_seed(3)+ X = torch.randn(10, 48, generator = g)+ As = [torch.randn(48, 8, generator = g) for _ in range(3)]+ Bs = [torch.randn(8, 40, generator = g) for _ in range(3)]+ dYs = [torch.randn(10, 40, generator = g) for _ in range(3)]+ ref_As, ref_Bs, ref_dX = _shared_lora_backward(X, dYs, As, Bs, 0.5, None)+ with _CountGemms() as counter:+ d_As, d_Bs, dX = _shared_lora_backward(+ X, dYs, As, Bs, 0.5, None, need_A = need_A, need_B = need_B+ )+ assert torch.equal(dX, ref_dX)+ for got, ref, need in zip(d_As + d_Bs, ref_As + ref_Bs, need_A + need_B):+ assert (got is None) != need+ if need:+ assert torch.equal(got, ref)+ expected = 3 + 1 + (1 if any(need_A) else 0) + ((1 + sum(need_B)) if any(need_B) else 0)+ assert counter.count == expected, (counter.count, expected)diff --git a/unsloth/kernels/fast_lora.py b/unsloth/kernels/fast_lora.pyindex b61333c1261..664d1e116c7 100644--- a/unsloth/kernels/fast_lora.py+++ b/unsloth/kernels/fast_lora.py@@ -336,6 +336,31 @@ def _add_lora_dX(dX, y_dB, A, S, mask, p):dX.add_(_dropout_replay(y_dB @ A.t(), mask, p, S))+def _grad_needed(ctx, index):+ # A ctx without the flags (handcrafted) computes every gradient.+ needs = getattr(ctx, "needs_input_grad", None)+ return needs is None or index >= len(needs) or bool(needs[index])+++def _t(x):+ return None if x is None else x.t()+++def _lora_AB_grads(X, mask, p, y_dB, A, B, dY, S, need_A, need_B):+ # A, B transposed; Xd = X through the LoRA dropout mask. None for a frozen factor.+ if not (need_A or need_B):+ return None, None+ d_A = d_B = None+ Xd = X if mask is None else _dropout_replay(X, mask, p)+ if need_A:+ d_A = torch.empty_like(A)+ d_A.addmm_(Xd.t(), y_dB, alpha = S, beta = 0)+ if need_B:+ d_B = torch.empty_like(B)+ d_B.addmm_(A.t() @ Xd.t(), dY, alpha = S, beta = 0)+ return d_A, d_B++def _shared_XA(X, As, dtype):# One X @ [A_1; A_2; ...].t() instead of one pass over X per projection; per-projection column views.# Compressed sparse adapters cannot be concatenated: one product each, as before.@@ -346,16 +371,29 @@ def _shared_XA(X, As, dtype):return XA.split_with_sizes([A.shape[0] for A in As], 1)-def _shared_lora_backward(X, dYs, As, Bs, S, dX_out):+def _shared_lora_backward(+ X,+ dYs,+ As,+ Bs,+ S,+ dX_out,+ need_A = None,+ need_B = None,+):# LoRA grads of projections sharing X and S: X is read once for all d_A and A @ X.t(), and dX starts# as S * dB @ A.t() for the base GEMMs to accumulate onto. Each d_A.t() is a row block of one buffer.+ need_A = (True,) * len(As) if need_A is None else need_A+ need_B = (True,) * len(Bs) if need_B is None else need_BA_rows = torch.cat([A.t() for A in As], 0)rs = [B.shape[0] for B in Bs]- d_Bs = []- for AX_, dY, B in zip((A_rows @ X.t()).split_with_sizes(rs, 0), dYs, Bs):- d_Bs.append(torch.empty_like(B).addmm_(AX_, dY, alpha = S, beta = 0))- # The last view keeps the whole A @ X.t() buffer alive otherwise.- del AX_+ d_Bs = [None] * len(Bs)+ if any(need_B):+ for i, (AX_, dY, B) in enumerate(zip((A_rows @ X.t()).split_with_sizes(rs, 0), dYs, Bs)):+ if need_B[i]:+ d_Bs[i] = torch.empty_like(B).addmm_(AX_, dY, alpha = S, beta = 0)+ # The last view keeps the whole A @ X.t() buffer alive otherwise.+ del AX_if _is_compiling() or torch.is_grad_enabled():# Traced, Inductor plans the buffers; slice writes would only add scatter copies. A recorded# backward (create_graph) cannot write through sibling split views either.@@ -364,10 +402,12 @@ def _shared_lora_backward(X, dYs, As, Bs, S, dX_out):dB = dYs[0].new_empty((X.shape[0], A_rows.shape[0]))for dB_, dY, B in zip(dB.split_with_sizes(rs, 1), dYs, Bs):dB_.addmm_(dY, B.t(), beta = 0)- dA = dB.new_empty((A_rows.shape[0], X.shape[1]))- dA.addmm_(dB.t(), X, alpha = S, beta = 0)- d_As = [dA_.t() for dA_ in dA.split_with_sizes(rs, 0)]- del dA+ d_As = [None] * len(As)+ if any(need_A):+ dA = dB.new_empty((A_rows.shape[0], X.shape[1]))+ dA.addmm_(dB.t(), X, alpha = S, beta = 0)+ d_As = [dA_.t() if need else None for dA_, need in zip(dA.split_with_sizes(rs, 0), need_A)]+ del dAif dX_out is None and _is_compiling():dX = torch.addmm(dB.new_zeros(1), dB, A_rows, beta = 0, alpha = S)elif dX_out is None:@@ -654,12 +694,19 @@ class LoRA_MLP(torch.autograd.Function):if down_mask is not None:h = _dropout_replay(h, down_mask, down_p)- d_downA = torch.empty_like(downA)- d_downB = torch.empty_like(downB)+ # Forward input indices of gate / up / down A, B.+ need_gateA, need_gateB, need_upA, need_upB, need_downA, need_downB = (+ _grad_needed(ctx, i) for i in (3, 4, 8, 9, 13, 14)+ )# d_downA = h.t() @ (dY @ downB.t()), d_downB = (downA.t() @ h.t()) @ dY, both scaled by downS.- d_downA.addmm_(h.t(), dY @ downB.t(), alpha = downS, beta = 0)- d_downB.addmm_(downA.t() @ h.t(), dY, alpha = downS, beta = 0)+ d_downA = d_downB = None+ if need_downA:+ d_downA = torch.empty_like(downA)+ d_downA.addmm_(h.t(), dY @ downB.t(), alpha = downS, beta = 0)+ if need_downB:+ d_downB = torch.empty_like(downB)+ d_downB.addmm_(downA.t() @ h.t(), dY, alpha = downS, beta = 0)if fused and gateS == upS and df.dtype == de.dtype == dY.dtype:(d_gateA, d_upA), (d_gateB, d_upB), dX = _shared_lora_backward(@@ -669,6 +716,8 @@ class LoRA_MLP(torch.autograd.Function):(gateB, upB),gateS,X if ctx.inplace and not _is_compiling() else None,+ need_A = (need_gateA, need_upA),+ need_B = (need_gateB, need_upB),)if has_fp8_dgrad(upW_quant):dX = upW_quant.dgrad_accum(df, upW, dX)@@ -688,18 +737,18 @@ class LoRA_MLP(torch.autograd.Function):dX.view(batch, seq_len, hd),None,None,- d_gateA.t(),- d_gateB.t(),+ _t(d_gateA),+ _t(d_gateB),None,None,None,- d_upA.t(),- d_upB.t(),+ _t(d_upA),+ _t(d_upB),None,None,None,- d_downA.t(),- d_downB.t(),+ _t(d_downA),+ _t(d_downB),None,None,None,@@ -710,10 +759,6 @@ class LoRA_MLP(torch.autograd.Function):d_downBias,)- d_gateA = torch.empty_like(gateA)- d_gateB = torch.empty_like(gateB)- d_upA = torch.empty_like(upA)- d_upB = torch.empty_like(upB)up_dB = df @ upB.t()gate_dB = de @ gateB.t()@@ -723,15 +768,13 @@ class LoRA_MLP(torch.autograd.Function):gate_mask = gate_mask.view(-1, hd)# d_upA = X.t() @ (df @ upB.t()), d_upB = (upA.t() @ X.t()) @ df, both scaled by upS.- Xd = X if up_mask is None else _dropout_replay(X, up_mask, up_p)- d_upA.addmm_(Xd.t(), up_dB, alpha = upS, beta = 0)- d_upB.addmm_(upA.t() @ Xd.t(), df, alpha = upS, beta = 0)-+ d_upA, d_upB = _lora_AB_grads(+ X, up_mask, up_p, up_dB, upA, upB, df, upS, need_upA, need_upB+ )# d_gateA = X.t() @ (de @ gateB.t()), d_gateB = (gateA.t() @ X.t()) @ de, both scaled by gateS.- Xd = X if gate_mask is None else _dropout_replay(X, gate_mask, gate_p)- d_gateA.addmm_(Xd.t(), gate_dB, alpha = gateS, beta = 0)- d_gateB.addmm_(gateA.t() @ Xd.t(), de, alpha = gateS, beta = 0)- del Xd+ d_gateA, d_gateB = _lora_AB_grads(+ X, gate_mask, gate_p, gate_dB, gateA, gateB, de, gateS, need_gateA, need_gateB+ )gate_has_bias, up_has_bias, down_has_bias = ctx.biasesd_gateBias = _lora_bias_grad(de, gateS) if gate_has_bias else None@@ -760,18 +803,18 @@ class LoRA_MLP(torch.autograd.Function):dX.view(batch, seq_len, hd),None,None,- d_gateA.t(),- d_gateB.t(),+ _t(d_gateA),+ _t(d_gateB),None,None,None,- d_upA.t(),- d_upB.t(),+ _t(d_upA),+ _t(d_upB),None,None,None,- d_downA.t(),- d_downB.t(),+ _t(d_downA),+ _t(d_downB),None,None,None,@@ -1103,6 +1146,11 @@ class LoRA_QKV(torch.autograd.Function):QA, QB, KA, KB, VA, VB = QA.t(), QB.t(), KA.t(), KB.t(), VA.t(), VB.t()+ # Forward input indices of Q / K / V A, B.+ need_QA, need_QB, need_KA, need_KB, need_VA, need_VB = (+ _grad_needed(ctx, i) for i in (3, 4, 8, 9, 13, 14)+ )+# Weight projection LoRA derivatives; see the Unsloth blogpost.if (getattr(ctx, "fused", False)@@ -1120,6 +1168,8 @@ class LoRA_QKV(torch.autograd.Function):(QB, KB, VB),QS,X if ctx.inplace and not _is_compiling() else None,+ need_A = (need_QA, need_KA, need_VA),+ need_B = (need_QB, need_KB, need_VB),)for W_, W_quant_, dY_ in ((QW, QW_quant, dQ), (KW, KW_quant, dK), (VW, VW_quant, dV)):if has_fp8_dgrad(W_quant_):@@ -1136,18 +1186,18 @@ class LoRA_QKV(torch.autograd.Function):dX.view(batch, seq_len, hd),None,None,- d_QA.t(),- d_QB.t(),+ _t(d_QA),+ _t(d_QB),None,None,None,- d_KA.t(),- d_KB.t(),+ _t(d_KA),+ _t(d_KB),None,None,None,- d_VA.t(),- d_VB.t(),+ _t(d_VA),+ _t(d_VB),None,None,None,@@ -1156,12 +1206,6 @@ class LoRA_QKV(torch.autograd.Function):None,)- d_QA = torch.empty_like(QA)- d_QB = torch.empty_like(QB)- d_KA = torch.empty_like(KA)- d_KB = torch.empty_like(KB)- d_VA = torch.empty_like(VA)- d_VB = torch.empty_like(VB)q_dB = dQ @ QB.t()k_dB = dK @ KB.t()v_dB = dV @ VB.t()@@ -1172,18 +1216,9 @@ class LoRA_QKV(torch.autograd.Function):# d_QA = X.t() @ (dQ @ QB.t()), d_QB = (QA.t() @ X.t()) @ dQ, both scaled by QS; K and V below are# identical with their own scales.- Xd = X if Q_mask is None else _dropout_replay(X, Q_mask, Q_p)- d_QA.addmm_(Xd.t(), q_dB, alpha = QS, beta = 0)- d_QB.addmm_(QA.t() @ Xd.t(), dQ, alpha = QS, beta = 0)-- Xd = X if K_mask is None else _dropout_replay(X, K_mask, K_p)- d_KA.addmm_(Xd.t(), k_dB, alpha = KS, beta = 0)- d_KB.addmm_(KA.t() @ Xd.t(), dK, alpha = KS, beta = 0)-- Xd = X if V_mask is None else _dropout_replay(X, V_mask, V_p)- d_VA.addmm_(Xd.t(), v_dB, alpha = VS, beta = 0)- d_VB.addmm_(VA.t() @ Xd.t(), dV, alpha = VS, beta = 0)- del Xd+ d_QA, d_QB = _lora_AB_grads(X, Q_mask, Q_p, q_dB, QA, QB, dQ, QS, need_QA, need_QB)+ d_KA, d_KB = _lora_AB_grads(X, K_mask, K_p, k_dB, KA, KB, dK, KS, need_KA, need_KB)+ d_VA, d_VB = _lora_AB_grads(X, V_mask, V_p, v_dB, VA, VB, dV, VS, need_VA, need_VB)Q_has_bias, K_has_bias, V_has_bias = ctx.biasesd_QBias = _lora_bias_grad(dQ, QS) if Q_has_bias else None@@ -1219,18 +1254,18 @@ class LoRA_QKV(torch.autograd.Function):dX.view(batch, seq_len, hd),None,None,- d_QA.t(),- d_QB.t(),+ _t(d_QA),+ _t(d_QB),None,None,None,- d_KA.t(),- d_KB.t(),+ _t(d_KA),+ _t(d_KB),None,None,None,- d_VA.t(),- d_VB.t(),+ _t(d_VA),+ _t(d_VB),None,None,None,@@ -1368,16 +1403,11 @@ class LoRA_W(torch.autograd.Function):A, B = A.t(), B.t()- d_A = torch.empty_like(A)- d_B = torch.empty_like(B)-y_dB = dY @ B.t()- # d_A = X.t() @ (dY @ B.t()), d_B = (A.t() @ X.t()) @ dY, both scaled by S.- Xd = X if mask is None else _dropout_replay(X, mask, ctx.dropout_p)- d_A.addmm_(Xd.t(), y_dB, alpha = S, beta = 0)- d_B.addmm_(A.t() @ Xd.t(), dY, alpha = S, beta = 0)- del Xd+ d_A, d_B = _lora_AB_grads(+ X, mask, ctx.dropout_p, y_dB, A, B, dY, S, _grad_needed(ctx, 3), _grad_needed(ctx, 4)+ )d_bias = _lora_bias_grad(dY, S) if ctx.has_bias else Noneif getattr(ctx, "fused", False) and (@@ -1405,7 +1435,7 @@ class LoRA_W(torch.autograd.Function):del W_add_lora_dX(dX, y_dB, A, S, mask, ctx.dropout_p)- return dX.view(batch, seq_len, hd), None, None, d_A.t(), d_B.t(), None, None, d_bias+ return dX.view(batch, seq_len, hd), None, None, _t(d_A), _t(d_B), None, None, d_biasdef apply_lora_o(self, X):