Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Fused LoRA backward: skip the gradient GEMMs of a frozen A or B (#16752)

Daniel Han16060aba73c6
diff --git a/tests/test_fused_lora_frozen_factor_grads.py b/tests/test_fused_lora_frozen_factor_grads.py
new file mode 100644
index 00000000000..4835d8e61ff
--- /dev/null
+++ b/tests/test_fused_lora_frozen_factor_grads.py
@@ -0,0 +1,149 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""Fused LoRA backward skips a frozen A / B's GEMMs; dX and the other gradients stay bit for bit."""
+
+import itertools
+
+import pytest
+import torch
+import unsloth # noqa: F401
+from torch.utils._python_dispatch import TorchDispatchMode
+
+from unsloth.kernels.fast_lora import LoRA_QKV, LoRA_W, _shared_lora_backward
+
+_GEMMS = {"mm", "addmm", "addmm_", "bmm", "matmul"}
+
+
+class _CountGemms(TorchDispatchMode):
+ def __init__(self):
+ super().__init__()
+ self.count = 0
+
+ def __torch_dispatch__(
+ self,
+ func,
+ types,
+ args = (),
+ kwargs = None,
+ ):
+ if func.overloadpacket.__name__ in _GEMMS:
+ self.count += 1
+ return func(*args, **(kwargs or {}))
+
+
+def _tensors(
+ n_proj,
+ dtype,
+ seed = 0,
+):
+ g = torch.Generator().manual_seed(seed)
+ X = torch.randn(2, 5, 48, generator = g).to(dtype)
+ weights, factors = [], []
+ for _ in range(n_proj):
+ weights.append((torch.randn(40, 48, generator = g) * 0.05).to(dtype))
+ factors.append((torch.randn(8, 48, generator = g) * 0.1).to(dtype))
+ factors.append((torch.randn(40, 8, generator = g) * 0.1).to(dtype))
+ grads = [torch.randn(2, 5, 40, generator = g).to(dtype) for _ in range(n_proj)]
+ return X, weights, factors, grads
+
+
+def _run(
+ cls,
+ X,
+ weights,
+ factors,
+ grads,
+ trainable,
+ dropout_p = None,
+):
+ X = X.clone().requires_grad_(True)
+ factors = [f.clone().requires_grad_(t) for f, t in zip(factors, trainable)]
+ scales = [0.5, 0.25, 0.75]
+ if cls is LoRA_W:
+ args = (X, weights[0], None, factors[0], factors[1], scales[0])
+ if dropout_p is not None:
+ args = args + (dropout_p,)
+ else:
+ args = [X]
+ for i in range(3):
+ args += [weights[i], None, factors[2 * i], factors[2 * i + 1], scales[0]]
+ args += [False]
+ if dropout_p is not None:
+ args += [(dropout_p,) * 3]
+ torch.manual_seed(1) # same dropout mask in every run
+ outs = cls.apply(*args)
+ outs = outs if isinstance(outs, tuple) else (outs,)
+ with _CountGemms() as counter:
+ torch.autograd.backward(outs, grads)
+ return X.grad, [f.grad for f in factors], counter.count
+
+
[email protected]("dtype", [torch.float32, torch.bfloat16])
[email protected]("dropout_p", [None, 0.25])
+def test_lora_w_frozen_factor_skips_its_gemm_and_keeps_the_rest(dtype, dropout_p):
+ X, weights, factors, grads = _tensors(1, dtype)
+ dX_ref, (dA_ref, dB_ref), n_ref = _run(
+ LoRA_W, X, weights, factors, grads, (True, True), dropout_p
+ )
+ for trainable in [(False, True), (True, False), (False, False)]:
+ dX, (dA, dB), n = _run(LoRA_W, X, weights, factors, grads, trainable, dropout_p)
+ assert torch.equal(dX, dX_ref)
+ assert (dA is None) != trainable[0] and (dB is None) != trainable[1]
+ if trainable[0]:
+ assert torch.equal(dA, dA_ref)
+ if trainable[1]:
+ assert torch.equal(dB, dB_ref)
+ assert n == n_ref - (0 if trainable[0] else 1) - (0 if trainable[1] else 2), (
+ trainable,
+ n,
+ n_ref,
+ )
+
+
[email protected]("dtype", [torch.float32, torch.bfloat16])
+def test_lora_qkv_frozen_factors_keep_the_rest_bit_for_bit(dtype):
+ X, weights, factors, grads = _tensors(3, dtype)
+ dX_ref, f_ref, n_ref = _run(LoRA_QKV, X, weights, factors, grads, (True,) * 6)
+ for trainable in [
+ (False, True) * 3,
+ (True, False) * 3,
+ (False, True, True, True, True, False),
+ (False,) * 6,
+ ]:
+ dX, f, n = _run(LoRA_QKV, X, weights, factors, grads, trainable)
+ assert torch.equal(dX, dX_ref)
+ for got, ref, t in zip(f, f_ref, trainable):
+ assert (got is None) != t
+ if t:
+ assert torch.equal(got, ref)
+ assert n < n_ref, (trainable, n, n_ref)
+
+
+ "need_A, need_B",
+ list(
+ itertools.product(
+ [(True, True, True), (False, False, False), (True, False, True)],
+ [(True, True, True), (False, False, False), (False, True, False)],
+ )
+ ),
+)
+def test_shared_backward_skips_frozen_factor_gemms(need_A, need_B):
+ g = torch.Generator().manual_seed(3)
+ X = torch.randn(10, 48, generator = g)
+ As = [torch.randn(48, 8, generator = g) for _ in range(3)]
+ Bs = [torch.randn(8, 40, generator = g) for _ in range(3)]
+ dYs = [torch.randn(10, 40, generator = g) for _ in range(3)]
+ ref_As, ref_Bs, ref_dX = _shared_lora_backward(X, dYs, As, Bs, 0.5, None)
+ with _CountGemms() as counter:
+ d_As, d_Bs, dX = _shared_lora_backward(
+ X, dYs, As, Bs, 0.5, None, need_A = need_A, need_B = need_B
+ )
+ assert torch.equal(dX, ref_dX)
+ for got, ref, need in zip(d_As + d_Bs, ref_As + ref_Bs, need_A + need_B):
+ assert (got is None) != need
+ if need:
+ assert torch.equal(got, ref)
+ expected = 3 + 1 + (1 if any(need_A) else 0) + ((1 + sum(need_B)) if any(need_B) else 0)
+ assert counter.count == expected, (counter.count, expected)
diff --git a/unsloth/kernels/fast_lora.py b/unsloth/kernels/fast_lora.py
index b61333c1261..664d1e116c7 100644
--- a/unsloth/kernels/fast_lora.py
+++ b/unsloth/kernels/fast_lora.py
@@ -336,6 +336,31 @@ def _add_lora_dX(dX, y_dB, A, S, mask, p):
dX.add_(_dropout_replay(y_dB @ A.t(), mask, p, S))
+def _grad_needed(ctx, index):
+ # A ctx without the flags (handcrafted) computes every gradient.
+ needs = getattr(ctx, "needs_input_grad", None)
+ return needs is None or index >= len(needs) or bool(needs[index])
+
+
+def _t(x):
+ return None if x is None else x.t()
+
+
+def _lora_AB_grads(X, mask, p, y_dB, A, B, dY, S, need_A, need_B):
+ # A, B transposed; Xd = X through the LoRA dropout mask. None for a frozen factor.
+ if not (need_A or need_B):
+ return None, None
+ d_A = d_B = None
+ Xd = X if mask is None else _dropout_replay(X, mask, p)
+ if need_A:
+ d_A = torch.empty_like(A)
+ d_A.addmm_(Xd.t(), y_dB, alpha = S, beta = 0)
+ if need_B:
+ d_B = torch.empty_like(B)
+ d_B.addmm_(A.t() @ Xd.t(), dY, alpha = S, beta = 0)
+ return d_A, d_B
+
+
def _shared_XA(X, As, dtype):
# One X @ [A_1; A_2; ...].t() instead of one pass over X per projection; per-projection column views.
# Compressed sparse adapters cannot be concatenated: one product each, as before.
@@ -346,16 +371,29 @@ def _shared_XA(X, As, dtype):
return XA.split_with_sizes([A.shape[0] for A in As], 1)
-def _shared_lora_backward(X, dYs, As, Bs, S, dX_out):
+def _shared_lora_backward(
+ X,
+ dYs,
+ As,
+ Bs,
+ S,
+ dX_out,
+ need_A = None,
+ need_B = None,
+):
# LoRA grads of projections sharing X and S: X is read once for all d_A and A @ X.t(), and dX starts
# as S * dB @ A.t() for the base GEMMs to accumulate onto. Each d_A.t() is a row block of one buffer.
+ need_A = (True,) * len(As) if need_A is None else need_A
+ need_B = (True,) * len(Bs) if need_B is None else need_B
A_rows = torch.cat([A.t() for A in As], 0)
rs = [B.shape[0] for B in Bs]
- d_Bs = []
- for AX_, dY, B in zip((A_rows @ X.t()).split_with_sizes(rs, 0), dYs, Bs):
- d_Bs.append(torch.empty_like(B).addmm_(AX_, dY, alpha = S, beta = 0))
- # The last view keeps the whole A @ X.t() buffer alive otherwise.
- del AX_
+ d_Bs = [None] * len(Bs)
+ if any(need_B):
+ for i, (AX_, dY, B) in enumerate(zip((A_rows @ X.t()).split_with_sizes(rs, 0), dYs, Bs)):
+ if need_B[i]:
+ d_Bs[i] = torch.empty_like(B).addmm_(AX_, dY, alpha = S, beta = 0)
+ # The last view keeps the whole A @ X.t() buffer alive otherwise.
+ del AX_
if _is_compiling() or torch.is_grad_enabled():
# Traced, Inductor plans the buffers; slice writes would only add scatter copies. A recorded
# backward (create_graph) cannot write through sibling split views either.
@@ -364,10 +402,12 @@ def _shared_lora_backward(X, dYs, As, Bs, S, dX_out):
dB = dYs[0].new_empty((X.shape[0], A_rows.shape[0]))
for dB_, dY, B in zip(dB.split_with_sizes(rs, 1), dYs, Bs):
dB_.addmm_(dY, B.t(), beta = 0)
- dA = dB.new_empty((A_rows.shape[0], X.shape[1]))
- dA.addmm_(dB.t(), X, alpha = S, beta = 0)
- d_As = [dA_.t() for dA_ in dA.split_with_sizes(rs, 0)]
- del dA
+ d_As = [None] * len(As)
+ if any(need_A):
+ dA = dB.new_empty((A_rows.shape[0], X.shape[1]))
+ dA.addmm_(dB.t(), X, alpha = S, beta = 0)
+ d_As = [dA_.t() if need else None for dA_, need in zip(dA.split_with_sizes(rs, 0), need_A)]
+ del dA
if dX_out is None and _is_compiling():
dX = torch.addmm(dB.new_zeros(1), dB, A_rows, beta = 0, alpha = S)
elif dX_out is None:
@@ -654,12 +694,19 @@ class LoRA_MLP(torch.autograd.Function):
if down_mask is not None:
h = _dropout_replay(h, down_mask, down_p)
- d_downA = torch.empty_like(downA)
- d_downB = torch.empty_like(downB)
+ # Forward input indices of gate / up / down A, B.
+ need_gateA, need_gateB, need_upA, need_upB, need_downA, need_downB = (
+ _grad_needed(ctx, i) for i in (3, 4, 8, 9, 13, 14)
+ )
# d_downA = h.t() @ (dY @ downB.t()), d_downB = (downA.t() @ h.t()) @ dY, both scaled by downS.
- d_downA.addmm_(h.t(), dY @ downB.t(), alpha = downS, beta = 0)
- d_downB.addmm_(downA.t() @ h.t(), dY, alpha = downS, beta = 0)
+ d_downA = d_downB = None
+ if need_downA:
+ d_downA = torch.empty_like(downA)
+ d_downA.addmm_(h.t(), dY @ downB.t(), alpha = downS, beta = 0)
+ if need_downB:
+ d_downB = torch.empty_like(downB)
+ d_downB.addmm_(downA.t() @ h.t(), dY, alpha = downS, beta = 0)
if fused and gateS == upS and df.dtype == de.dtype == dY.dtype:
(d_gateA, d_upA), (d_gateB, d_upB), dX = _shared_lora_backward(
@@ -669,6 +716,8 @@ class LoRA_MLP(torch.autograd.Function):
(gateB, upB),
gateS,
X if ctx.inplace and not _is_compiling() else None,
+ need_A = (need_gateA, need_upA),
+ need_B = (need_gateB, need_upB),
)
if has_fp8_dgrad(upW_quant):
dX = upW_quant.dgrad_accum(df, upW, dX)
@@ -688,18 +737,18 @@ class LoRA_MLP(torch.autograd.Function):
dX.view(batch, seq_len, hd),
None,
None,
- d_gateA.t(),
- d_gateB.t(),
+ _t(d_gateA),
+ _t(d_gateB),
None,
None,
None,
- d_upA.t(),
- d_upB.t(),
+ _t(d_upA),
+ _t(d_upB),
None,
None,
None,
- d_downA.t(),
- d_downB.t(),
+ _t(d_downA),
+ _t(d_downB),
None,
None,
None,
@@ -710,10 +759,6 @@ class LoRA_MLP(torch.autograd.Function):
d_downBias,
)
- d_gateA = torch.empty_like(gateA)
- d_gateB = torch.empty_like(gateB)
- d_upA = torch.empty_like(upA)
- d_upB = torch.empty_like(upB)
up_dB = df @ upB.t()
gate_dB = de @ gateB.t()
@@ -723,15 +768,13 @@ class LoRA_MLP(torch.autograd.Function):
gate_mask = gate_mask.view(-1, hd)
# d_upA = X.t() @ (df @ upB.t()), d_upB = (upA.t() @ X.t()) @ df, both scaled by upS.
- Xd = X if up_mask is None else _dropout_replay(X, up_mask, up_p)
- d_upA.addmm_(Xd.t(), up_dB, alpha = upS, beta = 0)
- d_upB.addmm_(upA.t() @ Xd.t(), df, alpha = upS, beta = 0)
-
+ d_upA, d_upB = _lora_AB_grads(
+ X, up_mask, up_p, up_dB, upA, upB, df, upS, need_upA, need_upB
+ )
# d_gateA = X.t() @ (de @ gateB.t()), d_gateB = (gateA.t() @ X.t()) @ de, both scaled by gateS.
- Xd = X if gate_mask is None else _dropout_replay(X, gate_mask, gate_p)
- d_gateA.addmm_(Xd.t(), gate_dB, alpha = gateS, beta = 0)
- d_gateB.addmm_(gateA.t() @ Xd.t(), de, alpha = gateS, beta = 0)
- del Xd
+ d_gateA, d_gateB = _lora_AB_grads(
+ X, gate_mask, gate_p, gate_dB, gateA, gateB, de, gateS, need_gateA, need_gateB
+ )
gate_has_bias, up_has_bias, down_has_bias = ctx.biases
d_gateBias = _lora_bias_grad(de, gateS) if gate_has_bias else None
@@ -760,18 +803,18 @@ class LoRA_MLP(torch.autograd.Function):
dX.view(batch, seq_len, hd),
None,
None,
- d_gateA.t(),
- d_gateB.t(),
+ _t(d_gateA),
+ _t(d_gateB),
None,
None,
None,
- d_upA.t(),
- d_upB.t(),
+ _t(d_upA),
+ _t(d_upB),
None,
None,
None,
- d_downA.t(),
- d_downB.t(),
+ _t(d_downA),
+ _t(d_downB),
None,
None,
None,
@@ -1103,6 +1146,11 @@ class LoRA_QKV(torch.autograd.Function):
QA, QB, KA, KB, VA, VB = QA.t(), QB.t(), KA.t(), KB.t(), VA.t(), VB.t()
+ # Forward input indices of Q / K / V A, B.
+ need_QA, need_QB, need_KA, need_KB, need_VA, need_VB = (
+ _grad_needed(ctx, i) for i in (3, 4, 8, 9, 13, 14)
+ )
+
# Weight projection LoRA derivatives; see the Unsloth blogpost.
if (
getattr(ctx, "fused", False)
@@ -1120,6 +1168,8 @@ class LoRA_QKV(torch.autograd.Function):
(QB, KB, VB),
QS,
X if ctx.inplace and not _is_compiling() else None,
+ need_A = (need_QA, need_KA, need_VA),
+ need_B = (need_QB, need_KB, need_VB),
)
for W_, W_quant_, dY_ in ((QW, QW_quant, dQ), (KW, KW_quant, dK), (VW, VW_quant, dV)):
if has_fp8_dgrad(W_quant_):
@@ -1136,18 +1186,18 @@ class LoRA_QKV(torch.autograd.Function):
dX.view(batch, seq_len, hd),
None,
None,
- d_QA.t(),
- d_QB.t(),
+ _t(d_QA),
+ _t(d_QB),
None,
None,
None,
- d_KA.t(),
- d_KB.t(),
+ _t(d_KA),
+ _t(d_KB),
None,
None,
None,
- d_VA.t(),
- d_VB.t(),
+ _t(d_VA),
+ _t(d_VB),
None,
None,
None,
@@ -1156,12 +1206,6 @@ class LoRA_QKV(torch.autograd.Function):
None,
)
- d_QA = torch.empty_like(QA)
- d_QB = torch.empty_like(QB)
- d_KA = torch.empty_like(KA)
- d_KB = torch.empty_like(KB)
- d_VA = torch.empty_like(VA)
- d_VB = torch.empty_like(VB)
q_dB = dQ @ QB.t()
k_dB = dK @ KB.t()
v_dB = dV @ VB.t()
@@ -1172,18 +1216,9 @@ class LoRA_QKV(torch.autograd.Function):
# d_QA = X.t() @ (dQ @ QB.t()), d_QB = (QA.t() @ X.t()) @ dQ, both scaled by QS; K and V below are
# identical with their own scales.
- Xd = X if Q_mask is None else _dropout_replay(X, Q_mask, Q_p)
- d_QA.addmm_(Xd.t(), q_dB, alpha = QS, beta = 0)
- d_QB.addmm_(QA.t() @ Xd.t(), dQ, alpha = QS, beta = 0)
-
- Xd = X if K_mask is None else _dropout_replay(X, K_mask, K_p)
- d_KA.addmm_(Xd.t(), k_dB, alpha = KS, beta = 0)
- d_KB.addmm_(KA.t() @ Xd.t(), dK, alpha = KS, beta = 0)
-
- Xd = X if V_mask is None else _dropout_replay(X, V_mask, V_p)
- d_VA.addmm_(Xd.t(), v_dB, alpha = VS, beta = 0)
- d_VB.addmm_(VA.t() @ Xd.t(), dV, alpha = VS, beta = 0)
- del Xd
+ d_QA, d_QB = _lora_AB_grads(X, Q_mask, Q_p, q_dB, QA, QB, dQ, QS, need_QA, need_QB)
+ d_KA, d_KB = _lora_AB_grads(X, K_mask, K_p, k_dB, KA, KB, dK, KS, need_KA, need_KB)
+ d_VA, d_VB = _lora_AB_grads(X, V_mask, V_p, v_dB, VA, VB, dV, VS, need_VA, need_VB)
Q_has_bias, K_has_bias, V_has_bias = ctx.biases
d_QBias = _lora_bias_grad(dQ, QS) if Q_has_bias else None
@@ -1219,18 +1254,18 @@ class LoRA_QKV(torch.autograd.Function):
dX.view(batch, seq_len, hd),
None,
None,
- d_QA.t(),
- d_QB.t(),
+ _t(d_QA),
+ _t(d_QB),
None,
None,
None,
- d_KA.t(),
- d_KB.t(),
+ _t(d_KA),
+ _t(d_KB),
None,
None,
None,
- d_VA.t(),
- d_VB.t(),
+ _t(d_VA),
+ _t(d_VB),
None,
None,
None,
@@ -1368,16 +1403,11 @@ class LoRA_W(torch.autograd.Function):
A, B = A.t(), B.t()
- d_A = torch.empty_like(A)
- d_B = torch.empty_like(B)
-
y_dB = dY @ B.t()
- # d_A = X.t() @ (dY @ B.t()), d_B = (A.t() @ X.t()) @ dY, both scaled by S.
- Xd = X if mask is None else _dropout_replay(X, mask, ctx.dropout_p)
- d_A.addmm_(Xd.t(), y_dB, alpha = S, beta = 0)
- d_B.addmm_(A.t() @ Xd.t(), dY, alpha = S, beta = 0)
- del Xd
+ d_A, d_B = _lora_AB_grads(
+ X, mask, ctx.dropout_p, y_dB, A, B, dY, S, _grad_needed(ctx, 3), _grad_needed(ctx, 4)
+ )
d_bias = _lora_bias_grad(dY, S) if ctx.has_bias else None
if getattr(ctx, "fused", False) and (
@@ -1405,7 +1435,7 @@ class LoRA_W(torch.autograd.Function):
del W
_add_lora_dX(dX, y_dB, A, S, mask, ctx.dropout_p)
- return dX.view(batch, seq_len, hd), None, None, d_A.t(), d_B.t(), None, None, d_bias
+ return dX.view(batch, seq_len, hd), None, None, _t(d_A), _t(d_B), None, None, d_bias
def apply_lora_o(self, X):