Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Skip the fast LoRA paths when lora_B has a bias (#12981)
Vineeth Sai Varikuntlad8ad0b080654
diff --git a/tests/test_multiple_active_lora_adapters.py b/tests/test_multiple_active_lora_adapters.pyindex 97f36e74c79..26299f0597c 100644--- a/tests/test_multiple_active_lora_adapters.py+++ b/tests/test_multiple_active_lora_adapters.py@@ -32,13 +32,17 @@ class _Block(torch.nn.Module):self.act_fn = torch.nn.SiLU()-def _peft_block(adapters):+def _peft_block(adapters, lora_bias = False):from peft import LoraConfig, get_peft_modeltorch.manual_seed(3407)targets = ["q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj"]cfg = lambda r: LoraConfig(- r = r, lora_alpha = 2 * r, target_modules = targets, init_lora_weights = False+ r = r,+ lora_alpha = 2 * r,+ target_modules = targets,+ init_lora_weights = False,+ lora_bias = lora_bias,)model = get_peft_model(_Block(), cfg(2), adapter_name = "a")if adapters > 1:@@ -47,6 +51,10 @@ def _peft_block(adapters):model = model.to("cuda", torch.bfloat16)for name, p in model.named_parameters():p.requires_grad_("lora_" in name)+ if lora_bias:+ with torch.no_grad():+ for proj in targets:+ model.base_model.model.get_submodule(proj).lora_B["a"].bias.normal_()return model, model.base_model.model@@ -98,10 +106,26 @@ def test_qkv_o_mlp_match_peft(adapters):model,)if adapters > 1:- # The second adapter must be in the graph, not just the first.assert any(".b." in n for n in qkv) and any(".b." in n for n in mlp)+def test_qkv_o_mlp_match_peft_with_lora_bias():+ from unsloth.kernels import apply_lora_mlp_swiglu, apply_lora_o, apply_lora_qkv++ model, block = _peft_block(1, lora_bias = True)+ _check(+ lambda X: apply_lora_qkv(block, X),+ lambda X: (block.q_proj(X), block.k_proj(X), block.v_proj(X)),+ model,+ )+ _check(lambda X: apply_lora_o(block, X), block.o_proj, model)+ _check(+ lambda X: apply_lora_mlp_swiglu(block, X),+ lambda X: block.down_proj(block.act_fn(block.gate_proj(X)) * block.up_proj(X)),+ model,+ )++@pytest.mark.parametrize("adapters", [1, 2])def test_fast_linear_forward_decode_matches_peft(adapters):from unsloth.kernels import fast_linear_forward@@ -127,8 +151,27 @@ def test_fast_linear_forward_applies_dora_magnitude(q_len):block = get_peft_model(_Block(), cfg).to("cuda", torch.bfloat16).base_model.modelX = torch.randn(1, q_len, H, device = "cuda", dtype = torch.bfloat16)with torch.no_grad():- # A trained magnitude no longer equals the row norms DoRA starts from.+ # a trained magnitude differs from the initial DoRA row normsblock.q_proj.lora_magnitude_vector["default"].weight.mul_(1.5)torch.testing.assert_close(fast_linear_forward(block.q_proj, X), block.q_proj(X), rtol = 2e-2, atol = 2e-2)++[email protected]("q_len", [1, 5])+def test_fast_linear_forward_adds_lora_bias(q_len):+ from peft import LoraConfig, get_peft_model+ from unsloth.kernels import fast_linear_forward++ torch.manual_seed(3407)+ cfg = LoraConfig(+ r = 4, lora_alpha = 8, target_modules = ["q_proj"], init_lora_weights = False, lora_bias = True+ )+ block = get_peft_model(_Block(), cfg).to("cuda", torch.bfloat16).base_model.model+ X = torch.randn(1, q_len, H, device = "cuda", dtype = torch.bfloat16)+ with torch.no_grad():+ # a trained lora_B bias differs from its zero initialization+ block.q_proj.lora_B["default"].bias.normal_()+ torch.testing.assert_close(+ fast_linear_forward(block.q_proj, X), block.q_proj(X), rtol = 2e-2, atol = 2e-2+ )diff --git a/unsloth/kernels/fast_lora.py b/unsloth/kernels/fast_lora.pyindex 19a4f0a393a..91ddec29aef 100644--- a/unsloth/kernels/fast_lora.py+++ b/unsloth/kernels/fast_lora.py@@ -12,6 +12,7 @@import torchfrom unsloth_zoo.utils import Versionfrom .utils import (+ _has_active_lora_bias,_has_multiple_active_adapters,addmm_,fast_dequantize,@@ -25,7 +26,7 @@ from .utils import (_is_compiling = torch.compiler.is_compiling-# Inductor before torch 2.11 miscompiles these Functions' traced backward (wrong LoRA gradients).+# Inductor before torch 2.11 miscompiles traced backward passes and produces wrong LoRA gradients.TRACE_LORA_FUNCTIONS = Version(torch.__version__) >= Version("2.11.0")@@ -231,7 +232,7 @@ class LoRA_MLP(torch.autograd.Function):None,None,None,- ) # _backward and _forward and inplace+ )from .swiglu import swiglu_fg_kernel, swiglu_DWf_DW_dfg_kernel@@ -243,7 +244,9 @@ def apply_lora_mlp_swiglu(inplace = True,):if any(- _has_multiple_active_adapters(proj) or _has_activation_fake_quantizer(proj)+ _has_multiple_active_adapters(proj)+ or _has_active_lora_bias(proj)+ or _has_activation_fake_quantizer(proj)for proj in (self.gate_proj, self.up_proj, self.down_proj)):return self.down_proj(self.act_fn(self.gate_proj(X)) * self.up_proj(X))@@ -284,7 +287,9 @@ def apply_lora_mlp_geglu_exact(inplace = True,):if any(- _has_multiple_active_adapters(proj) or _has_activation_fake_quantizer(proj)+ _has_multiple_active_adapters(proj)+ or _has_active_lora_bias(proj)+ or _has_activation_fake_quantizer(proj)for proj in (self.gate_proj, self.up_proj, self.down_proj)):return self.down_proj(self.act_fn(self.gate_proj(X)) * self.up_proj(X))@@ -321,7 +326,9 @@ from .geglu import geglu_approx_forward_kernel, geglu_approx_backward_kerneldef apply_lora_mlp_geglu_approx(self, X):if any(- _has_multiple_active_adapters(proj) or _has_activation_fake_quantizer(proj)+ _has_multiple_active_adapters(proj)+ or _has_active_lora_bias(proj)+ or _has_activation_fake_quantizer(proj)for proj in (self.gate_proj, self.up_proj, self.down_proj)):return self.down_proj(self.act_fn(self.gate_proj(X)) * self.up_proj(X))@@ -544,7 +551,9 @@ def apply_lora_qkv(inplace = True,):if any(- _has_multiple_active_adapters(proj) or _has_activation_fake_quantizer(proj)+ _has_multiple_active_adapters(proj)+ or _has_active_lora_bias(proj)+ or _has_activation_fake_quantizer(proj)for proj in (self.q_proj, self.k_proj, self.v_proj)):return self.q_proj(X), self.k_proj(X), self.v_proj(X)@@ -639,7 +648,6 @@ class LoRA_W(torch.autograd.Function):d_A.addmm_(X.t(), y_dB, alpha = S, beta = 0)d_B.addmm_(A.t() @ X.t(), dY, alpha = S, beta = 0)- # Get derivative for dXW = fast_dequantize(W.t(), W_quant)dX = dY @ W.t()del W@@ -649,7 +657,11 @@ class LoRA_W(torch.autograd.Function):def apply_lora_o(self, X):- if _has_multiple_active_adapters(self.o_proj) or _has_activation_fake_quantizer(self.o_proj):+ if (+ _has_multiple_active_adapters(self.o_proj)+ or _has_active_lora_bias(self.o_proj)+ or _has_activation_fake_quantizer(self.o_proj)+ ):return self.o_proj(X)OW, OW_quant, OA, OB, OS = get_lora_parameters(self.o_proj)O = _apply(LoRA_W, X, OW, OW_quant, OA, OB, OS)diff --git a/unsloth/kernels/utils.py b/unsloth/kernels/utils.pyindex 433b3dcf129..78c1e5ee8b3 100644--- a/unsloth/kernels/utils.py+++ b/unsloth/kernels/utils.py@@ -361,6 +361,19 @@ def _has_active_dora_adapter(proj):return any(adapter in magnitude for adapter in adapters)+def _has_active_lora_bias(proj):+ # PEFT lora_bias=True gives lora_B a bias, which the fast LoRA paths never add.+ lora_B = getattr(proj, "lora_B", None)+ if not lora_B or getattr(proj, "disable_adapters", True) or getattr(proj, "merged", False):+ return False+ adapters = getattr(proj, "active_adapters", None)+ if adapters is None:+ adapters = getattr(proj, "active_adapter", ())+ if isinstance(adapters, str):+ adapters = (adapters,)+ return any(adapter in lora_B and lora_B[adapter].bias is not None for adapter in adapters)++def has_mxfp4_base(*projs):"""A packed MXFP4 base that cannot hand its bytes to the fused LoRA kernels (they would then hold a 16-bitweight until backward): use PEFT. Bases with ``mxfp4_quant_state`` pass ``weight_packed`` + that state instead."""@@ -1389,7 +1402,11 @@ def fast_linear_forward(temp_lora = None,out = None,):- if _has_multiple_active_adapters(proj) or _has_active_dora_adapter(proj):+ if (+ _has_multiple_active_adapters(proj)+ or _has_active_dora_adapter(proj)+ or _has_active_lora_bias(proj)+ ):result = proj(X)return result if out is None else out.copy_(result)W, W_quant, lora_A, lora_B, lora_S, bias = get_lora_parameters_bias(proj)diff --git a/unsloth/models/llama.py b/unsloth/models/llama.pyindex 1fb4e3cda58..76b02e5307a 100644--- a/unsloth/models/llama.py+++ b/unsloth/models/llama.py@@ -116,7 +116,7 @@ from unsloth.models._attn_mask_compat import ()from transformers.cache_utils import DynamicCache, Cachefrom ..kernels import *-from ..kernels.utils import has_mxfp4_base+from ..kernels.utils import _has_active_lora_bias, has_mxfp4_basefrom ..kernels.bnb_override import install_bnb_nf4_override as _install_bnb_nf4_overridefrom ..tokenizer_utils import *from .vision import FastBaseModel, _is_text_seq2seq_config@@ -4355,10 +4355,13 @@ class FastLlamaModel:and (len(getattr(up_proj, "lora_magnitude_vector", []) or []) == 0)and (len(getattr(down_proj, "lora_magnitude_vector", []) or []) == 0)and not has_mxfp4_base(gate_proj, up_proj, down_proj)+ and not _has_active_lora_bias(gate_proj)+ and not _has_active_lora_bias(up_proj)+ and not _has_active_lora_bias(down_proj)):- # See stackoverflow.com/questions/50599045 on replacing a function within a class of a module.+ # MethodType binds the replacement; see stackoverflow.com/questions/50599045.if hasattr(mlp_module, "_unsloth_forward"):- # Then the mlp has been patched to use TiledMLP.+ # _unsloth_forward identifies an existing TiledMLP patch.mlp_module._unsloth_forward = types.MethodType(_apply_lora_mlp, mlp_module)@@ -4385,6 +4388,9 @@ class FastLlamaModel:and (len(getattr(k_proj, "lora_magnitude_vector", []) or []) == 0)and (len(getattr(v_proj, "lora_magnitude_vector", []) or []) == 0)and not has_mxfp4_base(q_proj, k_proj, v_proj)+ and not _has_active_lora_bias(q_proj)+ and not _has_active_lora_bias(k_proj)+ and not _has_active_lora_bias(v_proj)):layer.self_attn.apply_qkv = apply_lora_qkvn_qkv += 1@@ -4403,6 +4409,7 @@ class FastLlamaModel:and (getattr(o_proj, "base_layer", o_proj).bias is None)and (len(getattr(o_proj, "lora_magnitude_vector", []) or []) == 0)and not has_mxfp4_base(o_proj)+ and not _has_active_lora_bias(o_proj)):layer.self_attn.apply_o = apply_lora_on_o += 1@@ -4412,7 +4419,6 @@ class FastLlamaModel:"are not enabled or a bias term (like in Qwen) is used.")- # A zero count reads as a failure, so say why the fused kernels were skipped.unfused_reason = _fused_lora_skip_reason(lora_dropout, bias, float32_base, fsdp = fused_lora_declined_for_fsdp)