Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Skip the fused LoRA kernels when a DoRA adapter is active (#13254)

Vineeth Sai Varikuntla2308f6ef7cdc
diff --git a/tests/test_multiple_active_lora_adapters.py b/tests/test_multiple_active_lora_adapters.py
index 26299f0597c..9c0faa1339d 100644
--- a/tests/test_multiple_active_lora_adapters.py
+++ b/tests/test_multiple_active_lora_adapters.py
@@ -126,6 +126,37 @@ def test_qkv_o_mlp_match_peft_with_lora_bias():
)
+def test_qkv_o_mlp_match_peft_with_active_dora_adapter():
+ from peft import LoraConfig
+ from unsloth.kernels import apply_lora_mlp_swiglu, apply_lora_o, apply_lora_qkv
+
+ model, block = _peft_block(1)
+ targets = ["q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj"]
+ cfg = LoraConfig(
+ r = 3, lora_alpha = 6, target_modules = targets, init_lora_weights = False, use_dora = True
+ )
+ model.add_adapter("b", cfg)
+ model.base_model.set_adapter("b")
+ model = model.to("cuda", torch.bfloat16)
+ for name, p in model.named_parameters():
+ p.requires_grad_("lora_" in name)
+ with torch.no_grad():
+ # a trained magnitude differs from the initial DoRA row norms
+ for proj in targets:
+ block.get_submodule(proj).lora_magnitude_vector["b"].weight.mul_(1.5)
+ _check(
+ lambda X: apply_lora_qkv(block, X),
+ lambda X: (block.q_proj(X), block.k_proj(X), block.v_proj(X)),
+ model,
+ )
+ _check(lambda X: apply_lora_o(block, X), block.o_proj, model)
+ _check(
+ lambda X: apply_lora_mlp_swiglu(block, X),
+ lambda X: block.down_proj(block.act_fn(block.gate_proj(X)) * block.up_proj(X)),
+ model,
+ )
+
+
@pytest.mark.parametrize("adapters", [1, 2])
def test_fast_linear_forward_decode_matches_peft(adapters):
from unsloth.kernels import fast_linear_forward
diff --git a/unsloth/kernels/fast_lora.py b/unsloth/kernels/fast_lora.py
index 91ddec29aef..c2784269a8e 100644
--- a/unsloth/kernels/fast_lora.py
+++ b/unsloth/kernels/fast_lora.py
@@ -12,6 +12,7 @@
import torch
from unsloth_zoo.utils import Version
from .utils import (
+ _has_active_dora_adapter,
_has_active_lora_bias,
_has_multiple_active_adapters,
addmm_,
@@ -245,6 +246,7 @@ def apply_lora_mlp_swiglu(
):
if any(
_has_multiple_active_adapters(proj)
+ or _has_active_dora_adapter(proj)
or _has_active_lora_bias(proj)
or _has_activation_fake_quantizer(proj)
for proj in (self.gate_proj, self.up_proj, self.down_proj)
@@ -288,6 +290,7 @@ def apply_lora_mlp_geglu_exact(
):
if any(
_has_multiple_active_adapters(proj)
+ or _has_active_dora_adapter(proj)
or _has_active_lora_bias(proj)
or _has_activation_fake_quantizer(proj)
for proj in (self.gate_proj, self.up_proj, self.down_proj)
@@ -327,6 +330,7 @@ from .geglu import geglu_approx_forward_kernel, geglu_approx_backward_kernel
def apply_lora_mlp_geglu_approx(self, X):
if any(
_has_multiple_active_adapters(proj)
+ or _has_active_dora_adapter(proj)
or _has_active_lora_bias(proj)
or _has_activation_fake_quantizer(proj)
for proj in (self.gate_proj, self.up_proj, self.down_proj)
@@ -552,6 +556,7 @@ def apply_lora_qkv(
):
if any(
_has_multiple_active_adapters(proj)
+ or _has_active_dora_adapter(proj)
or _has_active_lora_bias(proj)
or _has_activation_fake_quantizer(proj)
for proj in (self.q_proj, self.k_proj, self.v_proj)
@@ -659,6 +664,7 @@ class LoRA_W(torch.autograd.Function):
def apply_lora_o(self, X):
if (
_has_multiple_active_adapters(self.o_proj)
+ or _has_active_dora_adapter(self.o_proj)
or _has_active_lora_bias(self.o_proj)
or _has_activation_fake_quantizer(self.o_proj)
):