Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Reuse the dY @ B.t() product in the fused LoRA backward (#12212)
Daniel Hanae7ed4095594
diff --git a/unsloth/kernels/fast_lora.py b/unsloth/kernels/fast_lora.pyindex 1e3b61be567..26c321ff091 100644--- a/unsloth/kernels/fast_lora.py+++ b/unsloth/kernels/fast_lora.py@@ -165,24 +165,27 @@ class LoRA_MLP(torch.autograd.Function):d_downA.addmm_(h.t(), dY @ downB.t(), alpha = downS, beta = 0)d_downB.addmm_(downA.t() @ h.t(), dY, alpha = downS, beta = 0)+ up_dB = df @ upB.t()+ gate_dB = de @ gateB.t()+# d_upA = X.t() @ (df @ upB.t()), d_upB = (upA.t() @ X.t()) @ df, both scaled by upS.- d_upA.addmm_(X.t(), df @ upB.t(), alpha = upS, beta = 0)+ d_upA.addmm_(X.t(), up_dB, alpha = upS, beta = 0)d_upB.addmm_(upA.t() @ X.t(), df, alpha = upS, beta = 0)# d_gateA = X.t() @ (de @ gateB.t()), d_gateB = (gateA.t() @ X.t()) @ de, both scaled by gateS.- d_gateA.addmm_(X.t(), de @ gateB.t(), alpha = gateS, beta = 0)+ d_gateA.addmm_(X.t(), gate_dB, alpha = gateS, beta = 0)d_gateB.addmm_(gateA.t() @ X.t(), de, alpha = gateS, beta = 0)# dX = matmul_lora(df, upW.t(), ...) + matmul_lora(de, gateW.t(), ...), expanded below.upW = fast_dequantize(upW.t(), upW_quant)dX = torch.matmul(df, upW.t(), out = X if ctx.inplace else None)del upW- dX.addmm_(df @ upB.t(), upA.t(), alpha = upS)+ dX.addmm_(up_dB, upA.t(), alpha = upS)gateW = fast_dequantize(gateW.t(), gateW_quant)dX.addmm_(de, gateW.t())del gateW- dX.addmm_(de @ gateB.t(), gateA.t(), alpha = gateS)+ dX.addmm_(gate_dB, gateA.t(), alpha = gateS)return (dX.view(batch, seq_len, hd),@@ -444,32 +447,36 @@ class LoRA_QKV(torch.autograd.Function):d_VA = torch.empty_like(VA)d_VB = torch.empty_like(VB)+ q_dB = dQ @ QB.t()+ k_dB = dK @ KB.t()+ v_dB = dV @ VB.t()+# d_QA = X.t() @ (dQ @ QB.t()), d_QB = (QA.t() @ X.t()) @ dQ, both scaled by QS; K and V below are# identical with their own scales.- d_QA.addmm_(X.t(), dQ @ QB.t(), alpha = QS, beta = 0)+ d_QA.addmm_(X.t(), q_dB, alpha = QS, beta = 0)d_QB.addmm_(QA.t() @ X.t(), dQ, alpha = QS, beta = 0)- d_KA.addmm_(X.t(), dK @ KB.t(), alpha = KS, beta = 0)+ d_KA.addmm_(X.t(), k_dB, alpha = KS, beta = 0)d_KB.addmm_(KA.t() @ X.t(), dK, alpha = KS, beta = 0)- d_VA.addmm_(X.t(), dV @ VB.t(), alpha = VS, beta = 0)+ d_VA.addmm_(X.t(), v_dB, alpha = VS, beta = 0)d_VB.addmm_(VA.t() @ X.t(), dV, alpha = VS, beta = 0)# Combine the per-projection derivatives into dX.QW = fast_dequantize(QW.t(), QW_quant)dX = torch.matmul(dQ, QW.t(), out = X if ctx.inplace else None)del QW- dX.addmm_(dQ @ QB.t(), QA.t(), alpha = QS)+ dX.addmm_(q_dB, QA.t(), alpha = QS)KW = fast_dequantize(KW.t(), KW_quant)dX.addmm_(dK, KW.t())del KW- dX.addmm_(dK @ KB.t(), KA.t(), alpha = KS)+ dX.addmm_(k_dB, KA.t(), alpha = KS)VW = fast_dequantize(VW.t(), VW_quant)dX.addmm_(dV, VW.t())del VW- dX.addmm_(dV @ VB.t(), VA.t(), alpha = VS)+ dX.addmm_(v_dB, VA.t(), alpha = VS)return (dX.view(batch, seq_len, hd),@@ -582,15 +589,17 @@ class LoRA_W(torch.autograd.Function):d_A = torch.empty_like(A)d_B = torch.empty_like(B)+ y_dB = dY @ B.t()+# d_A = X.t() @ (dY @ B.t()), d_B = (A.t() @ X.t()) @ dY, both scaled by S.- d_A.addmm_(X.t(), dY @ B.t(), alpha = S, beta = 0)+ d_A.addmm_(X.t(), y_dB, alpha = S, beta = 0)d_B.addmm_(A.t() @ X.t(), dY, alpha = S, beta = 0)# Get derivative for dXW = fast_dequantize(W.t(), W_quant)dX = dY @ W.t()del W- dX.addmm_(dY @ B.t(), A.t(), alpha = S)+ dX.addmm_(y_dB, A.t(), alpha = S)return dX.view(batch, seq_len, hd), None, None, d_A.t(), d_B.t(), None