Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Keep every active LoRA adapter in the fused LoRA paths (#12214)
cherry77-cloudb70bd2860803
diff --git a/tests/test_multiple_active_lora_adapters.py b/tests/test_multiple_active_lora_adapters.pynew file mode 100644index 00000000000..04d3ad6b279--- /dev/null+++ b/tests/test_multiple_active_lora_adapters.py@@ -0,0 +1,115 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""The fused LoRA paths must sum every active adapter like PEFT, not just active_adapters[0]."""++from __future__ import annotations++import pytest+import torch+import unsloth # noqa: F401++from real_accelerator import has_real_cuda++pytestmark = [+ pytest.mark.gpu,+ pytest.mark.skipif(not has_real_cuda(), reason = "needs a real CUDA device"),+]++H, I = 64, 128+++class _Block(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.q_proj = torch.nn.Linear(H, H, bias = False)+ self.k_proj = torch.nn.Linear(H, H, bias = False)+ self.v_proj = torch.nn.Linear(H, H, bias = False)+ self.o_proj = torch.nn.Linear(H, H, bias = False)+ self.gate_proj = torch.nn.Linear(H, I, bias = False)+ self.up_proj = torch.nn.Linear(H, I, bias = False)+ self.down_proj = torch.nn.Linear(I, H, bias = False)+ self.act_fn = torch.nn.SiLU()+++def _peft_block(adapters):+ from peft import LoraConfig, get_peft_model++ torch.manual_seed(3407)+ targets = ["q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj"]+ cfg = lambda r: LoraConfig(+ r = r, lora_alpha = 2 * r, target_modules = targets, init_lora_weights = False+ )+ model = get_peft_model(_Block(), cfg(2), adapter_name = "a")+ if adapters > 1:+ model.add_adapter("b", cfg(3))+ model.base_model.set_adapter(["a", "b"])+ model = model.to("cuda", torch.bfloat16)+ for name, p in model.named_parameters():+ p.requires_grad_("lora_" in name)+ return model, model.base_model.model+++def _lora_b_grads(model):+ return {+ n: p.grad.clone()+ for n, p in model.named_parameters()+ if "lora_B" in n and p.grad is not None+ }+++def _check(fast, ref, model):+ X = torch.randn(2, 5, H, device = "cuda", dtype = torch.bfloat16)+ out = fast(X.clone()) # the fused kernels may write into X+ sum(o.float().sum() for o in (out if isinstance(out, tuple) else (out,))).backward()+ fast_grads = _lora_b_grads(model)+ model.zero_grad(set_to_none = True)+ expected = ref(X)+ sum(+ o.float().sum() for o in (expected if isinstance(expected, tuple) else (expected,))+ ).backward()+ ref_grads = _lora_b_grads(model)+ model.zero_grad(set_to_none = True)+ for o, e in zip(+ out if isinstance(out, tuple) else (out,),+ expected if isinstance(expected, tuple) else (expected,),+ ):+ torch.testing.assert_close(o, e, rtol = 2e-2, atol = 2e-2)+ assert fast_grads.keys() == ref_grads.keys()+ for n in ref_grads:+ torch.testing.assert_close(fast_grads[n], ref_grads[n], rtol = 5e-2, atol = 5e-2)+ return ref_grads++[email protected]("adapters", [1, 2])+def test_qkv_o_mlp_match_peft(adapters):+ from unsloth.kernels import apply_lora_mlp_swiglu, apply_lora_o, apply_lora_qkv++ model, block = _peft_block(adapters)+ qkv = _check(+ lambda X: apply_lora_qkv(block, X),+ lambda X: (block.q_proj(X), block.k_proj(X), block.v_proj(X)),+ model,+ )+ _check(lambda X: apply_lora_o(block, X), block.o_proj, model)+ mlp = _check(+ lambda X: apply_lora_mlp_swiglu(block, X),+ lambda X: block.down_proj(block.act_fn(block.gate_proj(X)) * block.up_proj(X)),+ model,+ )+ if adapters > 1:+ # The second adapter must be in the graph, not just the first.+ assert any(".b." in n for n in qkv) and any(".b." in n for n in mlp)++[email protected]("adapters", [1, 2])+def test_fast_linear_forward_decode_matches_peft(adapters):+ from unsloth.kernels import fast_linear_forward++ _, block = _peft_block(adapters)+ X = torch.randn(1, 1, H, device = "cuda", dtype = torch.bfloat16)+ with torch.no_grad():+ out = torch.empty(1, 1, H, device = "cuda", dtype = torch.bfloat16)+ got = fast_linear_forward(block.q_proj, X, out = out)+ torch.testing.assert_close(got, block.q_proj(X), rtol = 2e-2, atol = 2e-2)+ assert got.data_ptr() == out.data_ptr()diff --git a/unsloth/kernels/fast_lora.py b/unsloth/kernels/fast_lora.pyindex 26c321ff091..50e44fc9ca2 100644--- a/unsloth/kernels/fast_lora.py+++ b/unsloth/kernels/fast_lora.py@@ -11,6 +11,7 @@import torchfrom .utils import (+ _has_multiple_active_adapters,_maybe_fake_quantize_activations,fast_dequantize,QUANT_STATE,@@ -218,6 +219,11 @@ def apply_lora_mlp_swiglu(X,inplace = True,):+ if any(+ _has_multiple_active_adapters(proj)+ for proj in (self.gate_proj, self.up_proj, self.down_proj)+ ):+ return self.down_proj(self.act_fn(self.gate_proj(X)) * self.up_proj(X))X = _maybe_fake_quantize_activations(X, self.gate_proj)gateW, gateW_quant, gateA, gateB, gateS = get_lora_parameters(self.gate_proj)upW, upW_quant, upA, upB, upS = get_lora_parameters(self.up_proj)@@ -254,6 +260,11 @@ def apply_lora_mlp_geglu_exact(X,inplace = True,):+ if any(+ _has_multiple_active_adapters(proj)+ for proj in (self.gate_proj, self.up_proj, self.down_proj)+ ):+ return self.down_proj(self.act_fn(self.gate_proj(X)) * self.up_proj(X))X = _maybe_fake_quantize_activations(X, self.gate_proj)gateW, gateW_quant, gateA, gateB, gateS = get_lora_parameters(self.gate_proj)upW, upW_quant, upA, upB, upS = get_lora_parameters(self.up_proj)@@ -286,6 +297,11 @@ from .geglu import geglu_approx_forward_kernel, geglu_approx_backward_kerneldef apply_lora_mlp_geglu_approx(self, X):+ if any(+ _has_multiple_active_adapters(proj)+ for proj in (self.gate_proj, self.up_proj, self.down_proj)+ ):+ return self.down_proj(self.act_fn(self.gate_proj(X)) * self.up_proj(X))X = _maybe_fake_quantize_activations(X, self.gate_proj)gateW, gateW_quant, gateA, gateB, gateS = get_lora_parameters(self.gate_proj)upW, upW_quant, upA, upB, upS = get_lora_parameters(self.up_proj)@@ -504,6 +520,8 @@ def apply_lora_qkv(X,inplace = True,):+ if any(_has_multiple_active_adapters(proj) for proj in (self.q_proj, self.k_proj, self.v_proj)):+ return self.q_proj(X), self.k_proj(X), self.v_proj(X)X = _maybe_fake_quantize_activations(X, self.q_proj)QW, QW_quant, QA, QB, QS = get_lora_parameters(self.q_proj)KW, KW_quant, KA, KB, KS = get_lora_parameters(self.k_proj)@@ -605,6 +623,8 @@ class LoRA_W(torch.autograd.Function):def apply_lora_o(self, X):+ if _has_multiple_active_adapters(self.o_proj):+ return self.o_proj(X)X = _maybe_fake_quantize_activations(X, self.o_proj)OW, OW_quant, OA, OB, OS = get_lora_parameters(self.o_proj)O = LoRA_W.apply(X, OW, OW_quant, OA, OB, OS)diff --git a/unsloth/kernels/utils.py b/unsloth/kernels/utils.pyindex 1c0d9578ea7..ad0b2568a08 100644--- a/unsloth/kernels/utils.py+++ b/unsloth/kernels/utils.py@@ -339,6 +339,14 @@ _FP8_WEIGHT_DTYPES = tuple()+def _has_multiple_active_adapters(proj):+ # Adapter activation can change after the single-adapter fast paths are installed.+ adapters = getattr(proj, "active_adapters", None)+ if adapters is None:+ adapters = getattr(proj, "active_adapter", ())+ return not isinstance(adapters, str) and len(adapters) > 1++def has_mxfp4_base(*projs):"""A packed MXFP4 base that cannot hand its bytes to the fused LoRA kernels (they would then hold a 16-bitweight until backward): use PEFT. Bases with ``mxfp4_quant_state`` pass ``weight_packed`` + that state instead."""@@ -405,7 +413,10 @@ def get_lora_parameters(proj):adapter = getattr(proj, "active_adapters", None)if adapter is None:adapter = getattr(proj, "active_adapter", ("default"))- adapter = adapter[0]+ if not isinstance(adapter, str):+ if len(adapter) > 1:+ raise ValueError("Unsloth: LoRA parameter extraction requires a single active adapter.")+ adapter = adapter[0]# Optionally apply fake quantization to lora weights for QAT.lora_A_linear = proj.lora_A[adapter]@@ -460,7 +471,10 @@ def get_lora_parameters_bias(proj):adapter = getattr(proj, "active_adapters", None)if adapter is None:adapter = getattr(proj, "active_adapter", ("default"))- adapter = adapter[0]+ if not isinstance(adapter, str):+ if len(adapter) > 1:+ raise ValueError("Unsloth: LoRA parameter extraction requires a single active adapter.")+ adapter = adapter[0]return (W,@@ -1142,6 +1156,9 @@ def fast_linear_forward(temp_lora = None,out = None,):+ if _has_multiple_active_adapters(proj):+ result = proj(X)+ return result if out is None else out.copy_(result)W, W_quant, lora_A, lora_B, lora_S, bias = get_lora_parameters_bias(proj)bsz, q_len, in_dim = X.shapeif q_len != 1: