Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Skip the fused LoRA kernels when a DoRA adapter is active (#13254)
Vineeth Sai Varikuntla2308f6ef7cdc
diff --git a/tests/test_multiple_active_lora_adapters.py b/tests/test_multiple_active_lora_adapters.pyindex 26299f0597c..9c0faa1339d 100644--- a/tests/test_multiple_active_lora_adapters.py+++ b/tests/test_multiple_active_lora_adapters.py@@ -126,6 +126,37 @@ def test_qkv_o_mlp_match_peft_with_lora_bias():)+def test_qkv_o_mlp_match_peft_with_active_dora_adapter():+ from peft import LoraConfig+ from unsloth.kernels import apply_lora_mlp_swiglu, apply_lora_o, apply_lora_qkv++ model, block = _peft_block(1)+ targets = ["q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj"]+ cfg = LoraConfig(+ r = 3, lora_alpha = 6, target_modules = targets, init_lora_weights = False, use_dora = True+ )+ model.add_adapter("b", cfg)+ model.base_model.set_adapter("b")+ model = model.to("cuda", torch.bfloat16)+ for name, p in model.named_parameters():+ p.requires_grad_("lora_" in name)+ with torch.no_grad():+ # a trained magnitude differs from the initial DoRA row norms+ for proj in targets:+ block.get_submodule(proj).lora_magnitude_vector["b"].weight.mul_(1.5)+ _check(+ lambda X: apply_lora_qkv(block, X),+ lambda X: (block.q_proj(X), block.k_proj(X), block.v_proj(X)),+ model,+ )+ _check(lambda X: apply_lora_o(block, X), block.o_proj, model)+ _check(+ lambda X: apply_lora_mlp_swiglu(block, X),+ lambda X: block.down_proj(block.act_fn(block.gate_proj(X)) * block.up_proj(X)),+ model,+ )++@pytest.mark.parametrize("adapters", [1, 2])def test_fast_linear_forward_decode_matches_peft(adapters):from unsloth.kernels import fast_linear_forwarddiff --git a/unsloth/kernels/fast_lora.py b/unsloth/kernels/fast_lora.pyindex 91ddec29aef..c2784269a8e 100644--- a/unsloth/kernels/fast_lora.py+++ b/unsloth/kernels/fast_lora.py@@ -12,6 +12,7 @@import torchfrom unsloth_zoo.utils import Versionfrom .utils import (+ _has_active_dora_adapter,_has_active_lora_bias,_has_multiple_active_adapters,addmm_,@@ -245,6 +246,7 @@ def apply_lora_mlp_swiglu():if any(_has_multiple_active_adapters(proj)+ or _has_active_dora_adapter(proj)or _has_active_lora_bias(proj)or _has_activation_fake_quantizer(proj)for proj in (self.gate_proj, self.up_proj, self.down_proj)@@ -288,6 +290,7 @@ def apply_lora_mlp_geglu_exact():if any(_has_multiple_active_adapters(proj)+ or _has_active_dora_adapter(proj)or _has_active_lora_bias(proj)or _has_activation_fake_quantizer(proj)for proj in (self.gate_proj, self.up_proj, self.down_proj)@@ -327,6 +330,7 @@ from .geglu import geglu_approx_forward_kernel, geglu_approx_backward_kerneldef apply_lora_mlp_geglu_approx(self, X):if any(_has_multiple_active_adapters(proj)+ or _has_active_dora_adapter(proj)or _has_active_lora_bias(proj)or _has_activation_fake_quantizer(proj)for proj in (self.gate_proj, self.up_proj, self.down_proj)@@ -552,6 +556,7 @@ def apply_lora_qkv():if any(_has_multiple_active_adapters(proj)+ or _has_active_dora_adapter(proj)or _has_active_lora_bias(proj)or _has_activation_fake_quantizer(proj)for proj in (self.q_proj, self.k_proj, self.v_proj)@@ -659,6 +664,7 @@ class LoRA_W(torch.autograd.Function):def apply_lora_o(self, X):if (_has_multiple_active_adapters(self.o_proj)+ or _has_active_dora_adapter(self.o_proj)or _has_active_lora_bias(self.o_proj)or _has_activation_fake_quantizer(self.o_proj)):