Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Keep every active LoRA adapter in the fused LoRA paths (#12214)

cherry77-cloudb70bd2860803
diff --git a/tests/test_multiple_active_lora_adapters.py b/tests/test_multiple_active_lora_adapters.py
new file mode 100644
index 00000000000..04d3ad6b279
--- /dev/null
+++ b/tests/test_multiple_active_lora_adapters.py
@@ -0,0 +1,115 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""The fused LoRA paths must sum every active adapter like PEFT, not just active_adapters[0]."""
+
+from __future__ import annotations
+
+import pytest
+import torch
+import unsloth # noqa: F401
+
+from real_accelerator import has_real_cuda
+
+pytestmark = [
+ pytest.mark.gpu,
+ pytest.mark.skipif(not has_real_cuda(), reason = "needs a real CUDA device"),
+]
+
+H, I = 64, 128
+
+
+class _Block(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.q_proj = torch.nn.Linear(H, H, bias = False)
+ self.k_proj = torch.nn.Linear(H, H, bias = False)
+ self.v_proj = torch.nn.Linear(H, H, bias = False)
+ self.o_proj = torch.nn.Linear(H, H, bias = False)
+ self.gate_proj = torch.nn.Linear(H, I, bias = False)
+ self.up_proj = torch.nn.Linear(H, I, bias = False)
+ self.down_proj = torch.nn.Linear(I, H, bias = False)
+ self.act_fn = torch.nn.SiLU()
+
+
+def _peft_block(adapters):
+ from peft import LoraConfig, get_peft_model
+
+ torch.manual_seed(3407)
+ targets = ["q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj"]
+ cfg = lambda r: LoraConfig(
+ r = r, lora_alpha = 2 * r, target_modules = targets, init_lora_weights = False
+ )
+ model = get_peft_model(_Block(), cfg(2), adapter_name = "a")
+ if adapters > 1:
+ model.add_adapter("b", cfg(3))
+ model.base_model.set_adapter(["a", "b"])
+ model = model.to("cuda", torch.bfloat16)
+ for name, p in model.named_parameters():
+ p.requires_grad_("lora_" in name)
+ return model, model.base_model.model
+
+
+def _lora_b_grads(model):
+ return {
+ n: p.grad.clone()
+ for n, p in model.named_parameters()
+ if "lora_B" in n and p.grad is not None
+ }
+
+
+def _check(fast, ref, model):
+ X = torch.randn(2, 5, H, device = "cuda", dtype = torch.bfloat16)
+ out = fast(X.clone()) # the fused kernels may write into X
+ sum(o.float().sum() for o in (out if isinstance(out, tuple) else (out,))).backward()
+ fast_grads = _lora_b_grads(model)
+ model.zero_grad(set_to_none = True)
+ expected = ref(X)
+ sum(
+ o.float().sum() for o in (expected if isinstance(expected, tuple) else (expected,))
+ ).backward()
+ ref_grads = _lora_b_grads(model)
+ model.zero_grad(set_to_none = True)
+ for o, e in zip(
+ out if isinstance(out, tuple) else (out,),
+ expected if isinstance(expected, tuple) else (expected,),
+ ):
+ torch.testing.assert_close(o, e, rtol = 2e-2, atol = 2e-2)
+ assert fast_grads.keys() == ref_grads.keys()
+ for n in ref_grads:
+ torch.testing.assert_close(fast_grads[n], ref_grads[n], rtol = 5e-2, atol = 5e-2)
+ return ref_grads
+
+
[email protected]("adapters", [1, 2])
+def test_qkv_o_mlp_match_peft(adapters):
+ from unsloth.kernels import apply_lora_mlp_swiglu, apply_lora_o, apply_lora_qkv
+
+ model, block = _peft_block(adapters)
+ qkv = _check(
+ lambda X: apply_lora_qkv(block, X),
+ lambda X: (block.q_proj(X), block.k_proj(X), block.v_proj(X)),
+ model,
+ )
+ _check(lambda X: apply_lora_o(block, X), block.o_proj, model)
+ mlp = _check(
+ lambda X: apply_lora_mlp_swiglu(block, X),
+ lambda X: block.down_proj(block.act_fn(block.gate_proj(X)) * block.up_proj(X)),
+ model,
+ )
+ if adapters > 1:
+ # The second adapter must be in the graph, not just the first.
+ assert any(".b." in n for n in qkv) and any(".b." in n for n in mlp)
+
+
[email protected]("adapters", [1, 2])
+def test_fast_linear_forward_decode_matches_peft(adapters):
+ from unsloth.kernels import fast_linear_forward
+
+ _, block = _peft_block(adapters)
+ X = torch.randn(1, 1, H, device = "cuda", dtype = torch.bfloat16)
+ with torch.no_grad():
+ out = torch.empty(1, 1, H, device = "cuda", dtype = torch.bfloat16)
+ got = fast_linear_forward(block.q_proj, X, out = out)
+ torch.testing.assert_close(got, block.q_proj(X), rtol = 2e-2, atol = 2e-2)
+ assert got.data_ptr() == out.data_ptr()
diff --git a/unsloth/kernels/fast_lora.py b/unsloth/kernels/fast_lora.py
index 26c321ff091..50e44fc9ca2 100644
--- a/unsloth/kernels/fast_lora.py
+++ b/unsloth/kernels/fast_lora.py
@@ -11,6 +11,7 @@
import torch
from .utils import (
+ _has_multiple_active_adapters,
_maybe_fake_quantize_activations,
fast_dequantize,
QUANT_STATE,
@@ -218,6 +219,11 @@ def apply_lora_mlp_swiglu(
X,
inplace = True,
):
+ if any(
+ _has_multiple_active_adapters(proj)
+ for proj in (self.gate_proj, self.up_proj, self.down_proj)
+ ):
+ return self.down_proj(self.act_fn(self.gate_proj(X)) * self.up_proj(X))
X = _maybe_fake_quantize_activations(X, self.gate_proj)
gateW, gateW_quant, gateA, gateB, gateS = get_lora_parameters(self.gate_proj)
upW, upW_quant, upA, upB, upS = get_lora_parameters(self.up_proj)
@@ -254,6 +260,11 @@ def apply_lora_mlp_geglu_exact(
X,
inplace = True,
):
+ if any(
+ _has_multiple_active_adapters(proj)
+ for proj in (self.gate_proj, self.up_proj, self.down_proj)
+ ):
+ return self.down_proj(self.act_fn(self.gate_proj(X)) * self.up_proj(X))
X = _maybe_fake_quantize_activations(X, self.gate_proj)
gateW, gateW_quant, gateA, gateB, gateS = get_lora_parameters(self.gate_proj)
upW, upW_quant, upA, upB, upS = get_lora_parameters(self.up_proj)
@@ -286,6 +297,11 @@ from .geglu import geglu_approx_forward_kernel, geglu_approx_backward_kernel
def apply_lora_mlp_geglu_approx(self, X):
+ if any(
+ _has_multiple_active_adapters(proj)
+ for proj in (self.gate_proj, self.up_proj, self.down_proj)
+ ):
+ return self.down_proj(self.act_fn(self.gate_proj(X)) * self.up_proj(X))
X = _maybe_fake_quantize_activations(X, self.gate_proj)
gateW, gateW_quant, gateA, gateB, gateS = get_lora_parameters(self.gate_proj)
upW, upW_quant, upA, upB, upS = get_lora_parameters(self.up_proj)
@@ -504,6 +520,8 @@ def apply_lora_qkv(
X,
inplace = True,
):
+ if any(_has_multiple_active_adapters(proj) for proj in (self.q_proj, self.k_proj, self.v_proj)):
+ return self.q_proj(X), self.k_proj(X), self.v_proj(X)
X = _maybe_fake_quantize_activations(X, self.q_proj)
QW, QW_quant, QA, QB, QS = get_lora_parameters(self.q_proj)
KW, KW_quant, KA, KB, KS = get_lora_parameters(self.k_proj)
@@ -605,6 +623,8 @@ class LoRA_W(torch.autograd.Function):
def apply_lora_o(self, X):
+ if _has_multiple_active_adapters(self.o_proj):
+ return self.o_proj(X)
X = _maybe_fake_quantize_activations(X, self.o_proj)
OW, OW_quant, OA, OB, OS = get_lora_parameters(self.o_proj)
O = LoRA_W.apply(X, OW, OW_quant, OA, OB, OS)
diff --git a/unsloth/kernels/utils.py b/unsloth/kernels/utils.py
index 1c0d9578ea7..ad0b2568a08 100644
--- a/unsloth/kernels/utils.py
+++ b/unsloth/kernels/utils.py
@@ -339,6 +339,14 @@ _FP8_WEIGHT_DTYPES = tuple(
)
+def _has_multiple_active_adapters(proj):
+ # Adapter activation can change after the single-adapter fast paths are installed.
+ adapters = getattr(proj, "active_adapters", None)
+ if adapters is None:
+ adapters = getattr(proj, "active_adapter", ())
+ return not isinstance(adapters, str) and len(adapters) > 1
+
+
def has_mxfp4_base(*projs):
"""A packed MXFP4 base that cannot hand its bytes to the fused LoRA kernels (they would then hold a 16-bit
weight until backward): use PEFT. Bases with ``mxfp4_quant_state`` pass ``weight_packed`` + that state instead."""
@@ -405,7 +413,10 @@ def get_lora_parameters(proj):
adapter = getattr(proj, "active_adapters", None)
if adapter is None:
adapter = getattr(proj, "active_adapter", ("default"))
- adapter = adapter[0]
+ if not isinstance(adapter, str):
+ if len(adapter) > 1:
+ raise ValueError("Unsloth: LoRA parameter extraction requires a single active adapter.")
+ adapter = adapter[0]
# Optionally apply fake quantization to lora weights for QAT.
lora_A_linear = proj.lora_A[adapter]
@@ -460,7 +471,10 @@ def get_lora_parameters_bias(proj):
adapter = getattr(proj, "active_adapters", None)
if adapter is None:
adapter = getattr(proj, "active_adapter", ("default"))
- adapter = adapter[0]
+ if not isinstance(adapter, str):
+ if len(adapter) > 1:
+ raise ValueError("Unsloth: LoRA parameter extraction requires a single active adapter.")
+ adapter = adapter[0]
return (
W,
@@ -1142,6 +1156,9 @@ def fast_linear_forward(
temp_lora = None,
out = None,
):
+ if _has_multiple_active_adapters(proj):
+ result = proj(X)
+ return result if out is None else out.copy_(result)
W, W_quant, lora_A, lora_B, lora_S, bias = get_lora_parameters_bias(proj)
bsz, q_len, in_dim = X.shape
if q_len != 1: