Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Fused LoRA kernels: support lora_dropout and lora_bias (#13898)
Daniel Handf63cab4b846
diff --git a/tests/test_fp8_down_proj_bf16_backward.py b/tests/test_fp8_down_proj_bf16_backward.pyindex 9ad2fe33701..39e9f579231 100644--- a/tests/test_fp8_down_proj_bf16_backward.py+++ b/tests/test_fp8_down_proj_bf16_backward.py@@ -84,8 +84,10 @@ def _backward(downW, downW_quant):2.0,backward_function,),- saved_tensors = (gateA, gateB, upA, upB, downA, downB, X, e, gg),+ saved_tensors = (gateA, gateB, upA, upB, downA, downB, X, e, gg, None, None, None),inplace = False,+ dropout_p = (0.0, 0.0, 0.0),+ biases = (False, False, False),_fwd_used_autocast = False,_dtype = torch.float32,)diff --git a/tests/test_fused_lora_dropout_bias.py b/tests/test_fused_lora_dropout_bias.pynew file mode 100644index 00000000000..060ef09a1f2--- /dev/null+++ b/tests/test_fused_lora_dropout_bias.py@@ -0,0 +1,364 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""lora_dropout > 0 and lora_bias = True run inside the fused LoRA kernels with PEFT's semantics.++PEFT computes base(X) + lora_B(lora_A(dropout(X))) * scaling: only the LoRA branch sees the dropped+out input, every projection draws its own mask, and lora_B may carry a bias. The fused kernels used to+hand these configs back to PEFT. They now draw the same masks from the same RNG stream in the same+order, so outputs and gradients match PEFT run from the same seed, and RNG state afterwards is equal.+"""++from __future__ import annotations++import pytest+import torch+import unsloth # noqa: F401++from real_accelerator import has_real_cuda+from unsloth.kernels import fast_lora++H, I = 64, 96+TARGETS = ["q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj"]++DEVICES = ["cpu"] + (["cuda"] if has_real_cuda() else [])+++class _Block(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.q_proj = torch.nn.Linear(H, H, bias = False)+ self.k_proj = torch.nn.Linear(H, H, bias = False)+ self.v_proj = torch.nn.Linear(H, H, bias = False)+ self.o_proj = torch.nn.Linear(H, H, bias = False)+ self.gate_proj = torch.nn.Linear(H, I, bias = False)+ self.up_proj = torch.nn.Linear(H, I, bias = False)+ self.down_proj = torch.nn.Linear(I, H, bias = False)+ self.act_fn = torch.nn.SiLU()+++def _peft_block(+ device,+ dtype,+ dropout,+ lora_bias = False,+):+ from peft import LoraConfig, get_peft_model++ torch.manual_seed(3407)+ cfg = LoraConfig(+ r = 4,+ lora_alpha = 8,+ lora_dropout = dropout,+ target_modules = TARGETS,+ init_lora_weights = False,+ lora_bias = lora_bias,+ )+ model = get_peft_model(_Block(), cfg).to(device, dtype)+ for name, p in model.named_parameters():+ p.requires_grad_("lora_" in name)+ if lora_bias:+ with torch.no_grad():+ for proj in TARGETS:+ model.base_model.model.get_submodule(proj).lora_B["default"].bias.normal_()+ model.train()+ return model, model.base_model.model+++PATHS = {+ "qkv": (+ lambda b: (lambda X: fast_lora.apply_lora_qkv(b, X)),+ lambda b: (lambda X: (b.q_proj(X), b.k_proj(X), b.v_proj(X))),+ ),+ "o": (+ lambda b: (lambda X: fast_lora.apply_lora_o(b, X)),+ lambda b: b.o_proj,+ ),+ "mlp": (+ lambda b: (lambda X: fast_lora.apply_lora_mlp_swiglu(b, X)),+ lambda b: (lambda X: b.down_proj(b.act_fn(b.gate_proj(X)) * b.up_proj(X))),+ ),+}+++def _skip_triton_on_cpu(device, path):+ if device == "cpu" and path == "mlp":+ pytest.skip("the SwiGLU kernels are Triton")+++def _rng_state(device):+ return torch.cuda.get_rng_state() if device == "cuda" else torch.get_rng_state()+++def _run(fn, model, X0, seed, device):+ torch.manual_seed(seed)+ X = X0.detach().clone().requires_grad_()+ out = fn(X * 1) # a non-leaf, since the fused backward may write dX into its input buffer+ outs = out if isinstance(out, tuple) else (out,)+ gen = torch.Generator(device = "cpu").manual_seed(seed + 1)+ loss = sum((o.float() * torch.randn(o.shape, generator = gen).to(device)).sum() for o in outs)+ loss.backward()+ grads = {n: p.grad.clone() for n, p in model.named_parameters() if p.grad is not None}+ model.zero_grad(set_to_none = True)+ return [o.detach() for o in outs], X.grad.clone(), grads, _rng_state(device)+++class _Spy:+ def __init__(self, monkeypatch):+ self.calls = []+ real = fast_lora._apply++ def spy(function, *args):+ self.calls.append((function.__name__, args))+ return real(function, *args)++ monkeypatch.setattr(fast_lora, "_apply", spy)+++class _FixedDropout(torch.nn.Module):+ def __init__(self, mask, p):+ super().__init__()+ self.mask, self.scale = mask, 1.0 / (1.0 - p)++ def forward(self, x):+ return x * self.mask.to(x.dtype) * self.scale+++def _fp64_oracle(model, path, X0, seed, device, masks, p):+ import copy++ ref_model = copy.deepcopy(model).double()+ ref_block = ref_model.base_model.model+ for name, mask in masks.items():+ ref_block.get_submodule(name).lora_dropout["default"] = _FixedDropout(mask, p)+ return _run(PATHS[path][1](ref_block), ref_model, X0.double(), seed, device)+++class _Recorder(torch.nn.Module):+ # nn.Dropout under autograd is native_dropout on CUDA and bernoulli_ noise on CPU; recording the mask+ # as `output != 0` would also catch inputs that are exactly zero.+ def __init__(self, dropout, masks, name):+ super().__init__()+ self.dropout, self.masks, self.name = dropout, masks, name++ def forward(self, x):+ if x.device.type == "cpu":+ out = self.dropout(x)+ self.masks[self.name] = (out != 0) | (+ x == 0+ ) # zeros never matter on the reference path+ return out+ out, self.masks[self.name] = torch.native_dropout(x, self.dropout.p, True)+ return out+++def _record_masks(block):+ masks, restore = {}, []+ for name in TARGETS:+ holder = block.get_submodule(name).lora_dropout+ dropout = holder["default"]+ if isinstance(dropout, torch.nn.Dropout):+ holder["default"] = _Recorder(dropout, masks, name)+ restore.append((holder, dropout))+ return masks, restore+++def _rel_err(x, ref):+ return ((x.double() - ref).norm() / ref.norm().clamp_min(1e-30)).item()++[email protected]("device", DEVICES)[email protected]("dtype", [torch.float32, torch.bfloat16, torch.float16])[email protected]("path", list(PATHS))[email protected]("dropout,lora_bias", [(0.3, False), (0.0, True), (0.3, True)])+def test_fused_matches_peft(monkeypatch, device, dtype, path, dropout, lora_bias):+ """Fused vs PEFT from the same seed, each scored against an fp64 PEFT run on PEFT's masks."""+ _skip_triton_on_cpu(device, path)+ if dtype == torch.float16 and device == "cpu":+ pytest.skip("fp16 matmuls on CPU")+ model, block = _peft_block(device, dtype, dropout, lora_bias)+ fused, ref = (make(block) for make in PATHS[path])+ X0 = torch.randn(2, 7, H, device = device, dtype = dtype)+ spy = _Spy(monkeypatch)++ f_out, f_dX, f_grads, f_rng = _run(fused, model, X0, 11, device)+ assert len(spy.calls) == 1, "the fused kernel did not run"+ masks, restore = _record_masks(block)+ r_out, r_dX, r_grads, r_rng = _run(ref, model, X0, 11, device)+ for holder, module in restore:+ holder["default"] = module+ assert len(spy.calls) == 1+ assert torch.equal(f_rng, r_rng), "different number of RNG draws than PEFT"+ o_out, o_dX, o_grads, _ = _fp64_oracle(model, path, X0, 11, device, masks, dropout)++ assert f_grads.keys() == r_grads.keys()+ if lora_bias:+ assert any(n.endswith("lora_B.default.bias") for n in f_grads)+ pairs = list(zip(f_out, r_out, o_out)) + [(f_dX, r_dX, o_dX)]+ pairs += [(f_grads[n], r_grads[n], o_grads[n]) for n in r_grads]+ # Same masks, so fused is within a few roundings of PEFT, and no further from fp64 than PEFT is+ # (with one dtype step of slack: the two paths round in a different order).+ eps = torch.finfo(dtype).eps+ for f, r, o in pairs:+ e_fused, e_peft = _rel_err(f, o), _rel_err(r, o)+ assert e_fused <= 1.5 * e_peft + 2 * eps, (e_fused, e_peft)++[email protected]("device", DEVICES)[email protected]("path", list(PATHS))+def test_masks_are_bitwise_peft_masks(monkeypatch, device, path):+ _skip_triton_on_cpu(device, path)+ # p = 0.5 with fp32: a single different mask element would move outputs by O(1).+ model, block = _peft_block(device, torch.float32, 0.5)+ fused, ref = (make(block) for make in PATHS[path])+ X0 = torch.randn(3, 5, H, device = device)+ for seed in range(3):+ f_out = _run(fused, model, X0, seed, device)[0]+ r_out = _run(ref, model, X0, seed, device)[0]+ for f, r in zip(f_out, r_out):+ torch.testing.assert_close(f, r, rtol = 1e-5, atol = 1e-5)++[email protected]("device", DEVICES)+def test_eval_mode_skips_dropout_like_peft(monkeypatch, device):+ model, block = _peft_block(device, torch.float32, 0.3)+ model.eval()+ spy = _Spy(monkeypatch)+ X = torch.randn(2, 5, H, device = device)+ with torch.no_grad():+ got = fast_lora.apply_lora_qkv(block, X.clone())+ expected = (block.q_proj(X), block.k_proj(X), block.v_proj(X))+ for g, e in zip(got, expected):+ torch.testing.assert_close(g, e)+ name, args = spy.calls[0]+ assert name == "LoRA_QKV" and args[-4:] == (None, None, None, None)++[email protected]("device", DEVICES)[email protected]("path", list(PATHS))+def test_checkpoint_recompute_replays_the_same_masks(device, path):+ from torch.utils.checkpoint import checkpoint++ _skip_triton_on_cpu(device, path)+ model, block = _peft_block(device, torch.float32, 0.3)+ fused = PATHS[path][0](block)+ X0 = torch.randn(2, 5, H, device = device)+ plain = _run(fused, model, X0, 5, device)+ ckpt = _run(+ lambda X: checkpoint(fused, X, use_reentrant = True, preserve_rng_state = True),+ model,+ X0,+ 5,+ device,+ )+ torch.testing.assert_close(plain[1], ckpt[1], rtol = 0, atol = 0)+ for n in plain[2]:+ torch.testing.assert_close(plain[2][n], ckpt[2][n], rtol = 0, atol = 0)+++ "dropout,lora_bias,flag", [(0.3, False, "FUSED_LORA_DROPOUT"), (0.0, True, "FUSED_LORA_BIAS")]+)+def test_kill_switch_restores_the_peft_fallback(monkeypatch, dropout, lora_bias, flag):+ model, block = _peft_block("cpu", torch.float32, dropout, lora_bias)+ monkeypatch.setattr(fast_lora, flag, False)+ spy = _Spy(monkeypatch)+ X = torch.randn(2, 5, H)+ fast_lora.apply_lora_qkv(block, X)+ fast_lora.apply_lora_o(block, X)+ if has_real_cuda():+ model, block = _peft_block("cuda", torch.float32, dropout, lora_bias)+ fast_lora.apply_lora_mlp_swiglu(block, X.cuda())+ assert spy.calls == []+++def test_unknown_dropout_module_falls_back(monkeypatch):+ model, block = _peft_block("cpu", torch.float32, 0.3)+ block.o_proj.lora_dropout["default"] = torch.nn.AlphaDropout(0.3)+ spy = _Spy(monkeypatch)+ fast_lora.apply_lora_o(block, torch.randn(2, 5, H))+ assert spy.calls == []+++def test_zero_dropout_no_bias_call_is_unchanged(monkeypatch):+ # The pre-existing fused call shape: no dropout / bias arguments reach LoRA_W.+ model, block = _peft_block("cpu", torch.float32, 0.0)+ spy = _Spy(monkeypatch)+ fast_lora.apply_lora_o(block, torch.randn(2, 5, H))+ assert [len(args) for _, args in spy.calls] == [6]++[email protected]("dropout,lora_bias", [(0.0, False), (0.3, False), (0.0, True)])+def test_active_adapter_absent_from_projection_falls_back(monkeypatch, dropout, lora_bias):+ # Adapters with different target modules: PEFT runs the base layer where the active one is missing.+ from peft import LoraConfig++ model, block = _peft_block("cpu", torch.float32, dropout, lora_bias)+ model.add_adapter(+ "other",+ LoraConfig(+ r = 4, lora_alpha = 8, lora_dropout = dropout, target_modules = ["q_proj"], lora_bias = lora_bias+ ),+ )+ model.set_adapter("other")+ model.train()+ spy = _Spy(monkeypatch)+ X = torch.randn(2, 5, H)+ torch.testing.assert_close(fast_lora.apply_lora_o(block, X), block.o_proj.base_layer(X))+ fast_lora.apply_lora_qkv(block, X)+ assert spy.calls == []++[email protected]("lora_bias", [False, True])+def test_routing_resolves_each_active_adapter_once(monkeypatch, lora_bias):+ # The routing runs on every fused LoRA call: the default config (no dropout, no bias) never reaches the+ # general adapter resolution, the rest resolve each projection's adapter once, and the lora_bias scan+ # is skipped unless UNSLOTH_FUSED_LORA_BIAS=0.+ model, block = _peft_block("cpu", torch.float32, 0.0, lora_bias)+ projs = (block.q_proj, block.k_proj, block.v_proj)+ real = fast_lora._active_lora_adapter+ seen = []+ monkeypatch.setattr(+ fast_lora, "_active_lora_adapter", lambda proj: seen.append(proj) or real(proj)+ )+ monkeypatch.setattr(+ fast_lora,+ "_has_active_lora_bias",+ lambda proj: pytest.fail("lora_bias scanned with fused bias on"),+ )+ dropout_p, biases = fast_lora._lora_dropout_and_bias(projs, torch.randn(2, 5, H))+ assert dropout_p is None+ assert len(seen) == (len(projs) if lora_bias else 0)+ assert all((b is not None) == lora_bias for b in biases)+++ "state", ["train", "eval", "disabled", "merged", "absent", "list_adapter", "dropout", "bias"]+)+def test_plain_lora_shortcut_matches_general_routing(monkeypatch, state):+ # The default-config shortcut must give the general routing's answer in every adapter state.+ from peft import LoraConfig++ model, block = _peft_block(+ "cpu", torch.float32, 0.3 if state == "dropout" else 0.0, state == "bias"+ )+ if state == "eval":+ model.eval()+ elif state == "disabled":+ model.base_model.disable_adapter_layers()+ elif state == "merged":+ block.q_proj.merge()+ elif state == "absent":+ model.add_adapter("other", LoraConfig(r = 4, lora_alpha = 8, target_modules = ["o_proj"]))+ model.set_adapter("other")+ elif state == "list_adapter":+ for proj in (block.q_proj, block.k_proj, block.v_proj):+ proj._active_adapter = ["default"]+ projs = (block.q_proj, block.k_proj, block.v_proj)+ X = torch.randn(2, 5, H)+ plain = [fast_lora._is_plain_lora(proj) for proj in projs]+ fast = fast_lora._lora_dropout_and_bias(projs, X)+ monkeypatch.setattr(fast_lora, "_is_plain_lora", lambda proj: False)+ general = fast_lora._lora_dropout_and_bias(projs, X)+ assert fast == general+ assert all(plain) == (state in ("train", "eval", "list_adapter"))diff --git a/tests/test_lora_dropout_patch_summary.py b/tests/test_lora_dropout_patch_summary.pyindex 16515407334..6227e77d9a9 100644--- a/tests/test_lora_dropout_patch_summary.py+++ b/tests/test_lora_dropout_patch_summary.py@@ -1,7 +1,11 @@# SPDX-License-Identifier: AGPL-3.0-only# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.-"""Fused LoRA kernels are skipped when lora_dropout or bias is set, and the summary must say why."""+"""Fused LoRA kernels are skipped when lora_dropout or bias is set, and the summary must say why.++The fused kernels now run lora_dropout > 0 themselves, so the dropout clause only appears under+the UNSLOTH_FUSED_LORA_DROPOUT=0 kill switch, which these tests set.+"""from __future__ import annotations@@ -14,10 +18,17 @@ import pytestimport unsloth # noqa: F401from real_accelerator import has_real_accelerator+from unsloth.kernels import fast_lorafrom unsloth.models import llama as llama_modulefrom unsloth.models.llama import _fused_lora_skip_reason[email protected](autouse = True)+def _fused_dropout_off(monkeypatch):+ # lora_dropout > 0 only closes the fused gate under the UNSLOTH_FUSED_LORA_DROPOUT=0 kill switch.+ monkeypatch.setattr(fast_lora, "FUSED_LORA_DROPOUT", False)++def test_no_reason_when_the_fused_kernels_were_installed():assert _fused_lora_skip_reason(0, "none") == ""assert _fused_lora_skip_reason(0.0, "none") == ""@@ -26,7 +37,7 @@ def test_no_reason_when_the_fused_kernels_were_installed():def test_lora_dropout_is_named_with_its_value():reason = _fused_lora_skip_reason(0.1, "none")assert "lora_dropout = 0.1" in reason- assert "bias" not in reason+ assert "bias =" not in reasonassert "Training is unaffected." in reason@@ -69,6 +80,12 @@ def test_reason_is_non_empty_exactly_when_the_fused_gate_is_closed(assert bool(reason) is not fused_installed[email protected]("lora_dropout", [0, 0.1, 0.5])+def test_lora_dropout_keeps_the_fused_gate_open_by_default(monkeypatch, lora_dropout):+ monkeypatch.setattr(fast_lora, "FUSED_LORA_DROPOUT", True)+ assert _fused_lora_skip_reason(lora_dropout, "none") == ""++def _gate_tests(source: str) -> list[str]:"""Every `if` test in patch_peft_model that reads both lora_dropout and bias."""tree = ast.parse(textwrap.dedent(source))@@ -86,7 +103,7 @@ def test_patch_peft_model_still_gates_the_fused_kernels_on_the_same_values():"""Every gate condition needs a matching _fused_lora_skip_reason clause."""source = inspect.getsource(llama_module.FastLlamaModel.patch_peft_model)assert _gate_tests(source) == [- "lora_dropout == 0 and bias == 'none' and (not float32_base) and (not fused_lora_declined_for_fsdp)"+ "(lora_dropout == 0 or _fast_lora.FUSED_LORA_DROPOUT) and bias == 'none' and (not float32_base) and (not fused_lora_declined_for_fsdp)"]diff --git a/unsloth/kernels/fast_lora.py b/unsloth/kernels/fast_lora.pyindex 685d6d14297..fca52dadc9d 100644--- a/unsloth/kernels/fast_lora.py+++ b/unsloth/kernels/fast_lora.py@@ -9,6 +9,7 @@# See the License for the specific language governing permissions and# limitations under the License.+import osimport sysimport torchfrom unsloth_zoo.utils import Version@@ -75,6 +76,167 @@ def _has_activation_fake_quantizer(proj):return False+# lora_dropout > 0 and lora_bias = True run inside the fused kernels; "0" restores the PEFT fallback.+FUSED_LORA_DROPOUT = os.environ.get("UNSLOTH_FUSED_LORA_DROPOUT", "1") != "0"+FUSED_LORA_BIAS = os.environ.get("UNSLOTH_FUSED_LORA_BIAS", "1") != "0"+++def _active_lora_adapter(proj):+ if getattr(proj, "disable_adapters", True) or getattr(proj, "merged", False):+ return None+ adapter = getattr(proj, "active_adapters", None)+ if adapter is None:+ adapter = getattr(proj, "active_adapter", "default")+ if not isinstance(adapter, str):+ adapter = adapter[0]+ return adapter if adapter in proj.lora_A else None+++def _lora_dropout_p(+ proj,+ X,+ adapter = False,+):+ """Dropout probability of the active adapter, 0.0 when it is a no-op, None when only PEFT can run it."""+ if adapter is False:+ adapter = _active_lora_adapter(proj)+ dropout = getattr(proj, "lora_dropout", None)+ if adapter is None or dropout is None or adapter not in dropout:+ return 0.0+ dropout = dropout[adapter]+ if isinstance(dropout, torch.nn.Identity) or not dropout.training:+ return 0.0+ if (+ type(dropout) is not torch.nn.Dropout+ or dropout.inplace+ or not FUSED_LORA_DROPOUT+ or X.device.type not in ("cuda", "xpu", "cpu")+ ):+ return None+ p = float(dropout.p)+ if p == 0:+ return 0.0+ return p if p < 1 else None+++def _lora_B_bias(proj):+ adapter = _active_lora_adapter(proj)+ if adapter is None:+ return None+ return proj.lora_B[adapter].bias+++def lora_bias_blocks_fused(proj):+ return not FUSED_LORA_BIAS and _has_active_lora_bias(proj)+++def _is_plain_lora(proj):+ # One enabled, unmerged adapter with no dropout module and no lora_B bias (the default LoRA config).+ # Reads PEFT's instance state directly: getattr through nn.Module / properties costs more than the+ # routing it guards. Anything else (or other PEFT internals) takes the general path below.+ state = proj.__dict__+ adapter = state.get("_active_adapter")+ if type(adapter) is not str:+ if type(adapter) is not list or len(adapter) != 1:+ return False+ adapter = adapter[0]+ if state.get("_disable_adapters", True) or state.get("merged_adapters"):+ return False+ modules = proj._modules+ dropout = modules.get("lora_dropout")+ lora_B = modules.get("lora_B")+ if dropout is None or lora_B is None:+ return False+ lora_B = lora_B._modules.get(adapter)+ return (+ type(dropout._modules.get(adapter)) is torch.nn.Identity+ and lora_B is not None+ and lora_B._parameters.get("bias") is None+ )+++def _lora_dropout_and_bias(projs, X):+ """(dropout probabilities or None, lora_B biases) for the fused kernels, or None to use PEFT."""+ for proj in projs:+ if not _is_plain_lora(proj):+ break+ else:+ return None, (None,) * len(projs)+ dropout_p, biases = [], []+ for proj in projs:+ adapter = _active_lora_adapter(proj)+ if adapter is None:+ # Enabled adapter missing from this projection: PEFT runs the base layer alone.+ if not getattr(proj, "disable_adapters", True) and not getattr(proj, "merged", False):+ return None+ dropout_p.append(0.0)+ biases.append(None)+ continue+ p = _lora_dropout_p(proj, X, adapter)+ if p is None:+ return None+ dropout_p.append(p)+ biases.append(proj.lora_B[adapter].bias)+ if any(lora_bias_blocks_fused(proj) for proj in projs):+ return None+ return (tuple(dropout_p) if any(dropout_p) else None), tuple(biases)+++def _dropout_forward(X, p):+ # Same RNG draws and rounding as torch.nn.Dropout under autograd: native_dropout on CUDA / XPU+ # (ATen's fused path), a bernoulli_ noise tensor in X's dtype on CPU.+ if not p:+ return X, None+ if X.device.type == "cpu":+ noise = torch.empty_like(X).bernoulli_(1 - p)+ mask = noise.bool()+ return X * noise.div_(1 - p), mask+ return torch.native_dropout(X, p, True)+++def _dropout_replay(+ T,+ mask,+ p,+ S = 1.0,+):+ # S * T * mask / (1 - p): with S = 1 exactly the dropout forward's output, and the dropout backward.+ if mask.device.type == "cpu":+ T = T * mask.to(T.dtype).div_(1 - p)+ return T if S == 1 else T.mul_(S)+ return torch.ops.aten.native_dropout_backward(T, mask, S / (1.0 - p))+++def _matmul_lora_ext(X, X_lora, W, W_quant, A, B, S, bias):+ # matmul_lora with the LoRA branch reading X_lora (X after dropout, or X itself) plus lora_B's bias.+ if A is None or (X_lora is X and bias is None):+ return matmul_lora(X, W, W_quant, A, B, S)+ out = matmul_lora(X, W, W_quant, None, None, None)+ dtype = X.dtype+ XA = torch.matmul(X_lora.reshape(-1, X_lora.shape[-1]), A.to(dtype).t())+ out2d = out.view(-1, out.shape[-1])+ if bias is None:+ addmm_(out2d, XA, B.to(dtype).t(), alpha = S)+ else:+ # PEFT's rounding order: (lora_B(x) + bias) * scaling, then + base. No add(alpha = S): Inductor+ # (torch 2.13) miscompiles it traced inside these Functions.+ out2d.add_(torch.addmm(bias.to(dtype), XA, B.to(dtype).t()).mul_(S))+ return out+++def _add_lora_dX(dX, y_dB, A, S, mask, p):+ # dX += S * y_dB @ A.t(), through the dropout mask when the LoRA branch saw a dropped out X.+ if mask is None:+ addmm_(dX, y_dB, A.t(), alpha = S)+ else:+ dX.add_(_dropout_replay(y_dB @ A.t(), mask, p, S))+++def _lora_bias_grad(dY, S):+ # Autograd casts it to the bias dtype.+ return dY.sum(0, dtype = torch.float32).mul_(S)++def _may_overwrite_input(X):# Backward writes dX into X; not when an RMSNorm saved X (its output) for its own backward,# including through a view of it (tiled MLP chunks).@@ -149,13 +311,32 @@ class LoRA_MLP(torch.autograd.Function):_forward_function,_backward_function,inplace = True,+ dropout_p = None,+ gateBias = None,+ upBias = None,+ downBias = None,):dtype = X.dtype- e = matmul_lora(X, gateW, gateW_quant, gateA, gateB, gateS)- g = matmul_lora(X, upW, upW_quant, upA, upB, upS)- h = _forward_function(e, g)- i = matmul_lora(h, downW, downW_quant, downA, downB, downS)+ if dropout_p is None and gateBias is None and upBias is None and downBias is None:+ e = matmul_lora(X, gateW, gateW_quant, gateA, gateB, gateS)+ g = matmul_lora(X, upW, upW_quant, upA, upB, upS)+ h = _forward_function(e, g)+ i = matmul_lora(h, downW, downW_quant, downA, downB, downS)+ masks = (None, None, None)+ else:+ # Masks are drawn gate, up, down: the order PEFT's down(act(gate(X)) * up(X)) draws them in.+ gate_p, up_p, down_p = dropout_p or (0.0, 0.0, 0.0)+ Xd, gate_mask = _dropout_forward(X, gate_p)+ e = _matmul_lora_ext(X, Xd, gateW, gateW_quant, gateA, gateB, gateS, gateBias)+ Xd, up_mask = _dropout_forward(X, up_p)+ g = _matmul_lora_ext(X, Xd, upW, upW_quant, upA, upB, upS, upBias)+ del Xd+ h = _forward_function(e, g)+ hd, down_mask = _dropout_forward(h, down_p)+ i = _matmul_lora_ext(h, hd, downW, downW_quant, downA, downB, downS, downBias)+ del hd+ masks = (gate_mask, up_mask, down_mask)ctx.custom_saved_tensors = (gateW,@@ -169,9 +350,11 @@ class LoRA_MLP(torch.autograd.Function):downS,_backward_function,)- ctx.save_for_backward(gateA, gateB, upA, upB, downA, downB, X, e, g)+ ctx.save_for_backward(gateA, gateB, upA, upB, downA, downB, X, e, g, *masks)# Only an opt-in UNSLOTH_RMSNORM_SAVE_OUTPUT norm marks its output; skip the check otherwise.ctx.inplace = inplace and (not _rms_layernorm._SAVE_OUTPUT or _may_overwrite_input(X))+ ctx.dropout_p = dropout_p or (0.0, 0.0, 0.0)+ ctx.biases = (gateBias is not None, upBias is not None, downBias is not None)return i@staticmethod@@ -189,7 +372,10 @@ class LoRA_MLP(torch.autograd.Function):downS,_backward_function,) = ctx.custom_saved_tensors- gateA, gateB, upA, upB, downA, downB, X, e, g = ctx.saved_tensors+ gateA, gateB, upA, upB, downA, downB, X, e, g, gate_mask, up_mask, down_mask = (+ ctx.saved_tensors+ )+ gate_p, up_p, down_p = ctx.dropout_pbatch, seq_len, hd = X.shapedY = dY.reshape(-1, dY.shape[-1])@@ -218,11 +404,21 @@ class LoRA_MLP(torch.autograd.Function):if downW.dtype == torch.float8_e4m3fn:# Dequantize like gate / up: matmul_lora would send the FP8 weight to fp8_linear, quantizing dY to FP8.- DW = matmul_lora(dY, fast_dequantize(downW.t(), downW_quant), None, downB, downA, downS)+ downW, downW_quant = fast_dequantize(downW.t(), downW_quant), None+ else:+ downW = downW.t()+ if down_mask is None:+ DW = matmul_lora(dY, downW, downW_quant, downB, downA, downS)else:- DW = matmul_lora(dY, downW.t(), downW_quant, downB, downA, downS)+ # Only the LoRA branch saw the dropped out h, so only its gradient goes through the mask.+ DW = matmul_lora(dY, downW, downW_quant, None, None, None)+ down_mask = down_mask.view(-1, down_mask.shape[-1])+ DW.add_(_dropout_replay((dY @ downB.t()) @ downA.t(), down_mask, down_p, downS))+ del downWDW, e, g = _backward_function(DW, e, g)h, df, de = DW, e, g+ if down_mask is not None:+ h = _dropout_replay(h, down_mask, down_p)d_downA = torch.empty_like(downA)d_downB = torch.empty_like(downB)@@ -238,25 +434,38 @@ class LoRA_MLP(torch.autograd.Function):up_dB = df @ upB.t()gate_dB = de @ gateB.t()+ if up_mask is not None:+ up_mask = up_mask.view(-1, hd)+ if gate_mask is not None:+ gate_mask = gate_mask.view(-1, hd)+# d_upA = X.t() @ (df @ upB.t()), d_upB = (upA.t() @ X.t()) @ df, both scaled by upS.- d_upA.addmm_(X.t(), up_dB, alpha = upS, beta = 0)- d_upB.addmm_(upA.t() @ X.t(), df, alpha = upS, beta = 0)+ Xd = X if up_mask is None else _dropout_replay(X, up_mask, up_p)+ d_upA.addmm_(Xd.t(), up_dB, alpha = upS, beta = 0)+ d_upB.addmm_(upA.t() @ Xd.t(), df, alpha = upS, beta = 0)# d_gateA = X.t() @ (de @ gateB.t()), d_gateB = (gateA.t() @ X.t()) @ de, both scaled by gateS.- d_gateA.addmm_(X.t(), gate_dB, alpha = gateS, beta = 0)- d_gateB.addmm_(gateA.t() @ X.t(), de, alpha = gateS, beta = 0)+ Xd = X if gate_mask is None else _dropout_replay(X, gate_mask, gate_p)+ d_gateA.addmm_(Xd.t(), gate_dB, alpha = gateS, beta = 0)+ d_gateB.addmm_(gateA.t() @ Xd.t(), de, alpha = gateS, beta = 0)+ del Xd++ gate_has_bias, up_has_bias, down_has_bias = ctx.biases+ d_gateBias = _lora_bias_grad(de, gateS) if gate_has_bias else None+ d_upBias = _lora_bias_grad(df, upS) if up_has_bias else None+ d_downBias = _lora_bias_grad(dY, downS) if down_has_bias else None# dX = matmul_lora(df, upW.t(), ...) + matmul_lora(de, gateW.t(), ...), expanded below.upW = fast_dequantize(upW.t(), upW_quant)# Eager only: AOT autograd rejects a backward mutating a forward input that requires grad.dX = torch.matmul(df, upW.t(), out = X if ctx.inplace and not _is_compiling() else None)del upW- addmm_(dX, up_dB, upA.t(), alpha = upS)+ _add_lora_dX(dX, up_dB, upA, upS, up_mask, up_p)gateW = fast_dequantize(gateW.t(), gateW_quant)addmm_(dX, de, gateW.t())del gateW- addmm_(dX, gate_dB, gateA.t(), alpha = gateS)+ _add_lora_dX(dX, gate_dB, gateA, gateS, gate_mask, gate_p)return (dX.view(batch, seq_len, hd),@@ -278,6 +487,10 @@ class LoRA_MLP(torch.autograd.Function):None,None,None,+ None,+ d_gateBias,+ d_upBias,+ d_downBias,)@@ -289,14 +502,18 @@ def apply_lora_mlp_swiglu(X,inplace = True,):- if any(+ projs = (self.gate_proj, self.up_proj, self.down_proj)+ extras = None+ if not any(_has_multiple_active_adapters(proj)or _has_active_dora_adapter(proj)- or _has_active_lora_bias(proj)or _has_activation_fake_quantizer(proj)- for proj in (self.gate_proj, self.up_proj, self.down_proj)+ for proj in projs):+ extras = _lora_dropout_and_bias(projs, X)+ if extras is None:return self.down_proj(self.act_fn(self.gate_proj(X)) * self.up_proj(X))+ dropout_p, biases = extrasgateW, gateW_quant, gateA, gateB, gateS = get_lora_parameters(self.gate_proj)upW, upW_quant, upA, upB, upS = get_lora_parameters(self.up_proj)downW, downW_quant, downA, downB, downS = get_lora_parameters(self.down_proj)@@ -321,6 +538,8 @@ def apply_lora_mlp_swiglu(swiglu_fg_kernel,swiglu_DWf_DW_dfg_kernel,inplace,+ dropout_p,+ *biases,)return out@@ -333,14 +552,18 @@ def apply_lora_mlp_geglu_exact(X,inplace = True,):- if any(+ projs = (self.gate_proj, self.up_proj, self.down_proj)+ extras = None+ if not any(_has_multiple_active_adapters(proj)or _has_active_dora_adapter(proj)- or _has_active_lora_bias(proj)or _has_activation_fake_quantizer(proj)- for proj in (self.gate_proj, self.up_proj, self.down_proj)+ for proj in projs):+ extras = _lora_dropout_and_bias(projs, X)+ if extras is None:return self.down_proj(self.act_fn(self.gate_proj(X)) * self.up_proj(X))+ dropout_p, biases = extrasgateW, gateW_quant, gateA, gateB, gateS = get_lora_parameters(self.gate_proj)upW, upW_quant, upA, upB, upS = get_lora_parameters(self.up_proj)downW, downW_quant, downA, downB, downS = get_lora_parameters(self.down_proj)@@ -365,6 +588,8 @@ def apply_lora_mlp_geglu_exact(geglu_exact_forward_kernel,geglu_exact_backward_kernel,inplace,+ dropout_p,+ *biases,)return out@@ -373,14 +598,18 @@ from .geglu import geglu_approx_forward_kernel, geglu_approx_backward_kerneldef apply_lora_mlp_geglu_approx(self, X):- if any(+ projs = (self.gate_proj, self.up_proj, self.down_proj)+ extras = None+ if not any(_has_multiple_active_adapters(proj)or _has_active_dora_adapter(proj)- or _has_active_lora_bias(proj)or _has_activation_fake_quantizer(proj)- for proj in (self.gate_proj, self.up_proj, self.down_proj)+ for proj in projs):+ extras = _lora_dropout_and_bias(projs, X)+ if extras is None:return self.down_proj(self.act_fn(self.gate_proj(X)) * self.up_proj(X))+ dropout_p, biases = extrasgateW, gateW_quant, gateA, gateB, gateS = get_lora_parameters(self.gate_proj)upW, upW_quant, upA, upB, upS = get_lora_parameters(self.up_proj)downW, downW_quant, downA, downB, downS = get_lora_parameters(self.down_proj)@@ -404,6 +633,9 @@ def apply_lora_mlp_geglu_approx(self, X):downS,geglu_approx_forward_kernel,geglu_approx_backward_kernel,+ True,+ dropout_p,+ *biases,)return out@@ -460,6 +692,10 @@ class LoRA_QKV(torch.autograd.Function):VB,VS,inplace = True,+ dropout_p = None,+ QBias = None,+ KBias = None,+ VBias = None,):dtype = X.dtype@@ -469,9 +705,21 @@ class LoRA_QKV(torch.autograd.Function):X_for_matmul = Xif X.dim() == 3:X_for_matmul = X.view(-1, X.shape[-1])- Q = matmul_lora(X_for_matmul, QW, QW_quant, QA, QB, QS)- K = matmul_lora(X_for_matmul, KW, KW_quant, KA, KB, KS)- V = matmul_lora(X_for_matmul, VW, VW_quant, VA, VB, VS)+ if dropout_p is None and QBias is None and KBias is None and VBias is None:+ Q = matmul_lora(X_for_matmul, QW, QW_quant, QA, QB, QS)+ K = matmul_lora(X_for_matmul, KW, KW_quant, KA, KB, KS)+ V = matmul_lora(X_for_matmul, VW, VW_quant, VA, VB, VS)+ masks = (None, None, None)+ else:+ Q_p, K_p, V_p = dropout_p or (0.0, 0.0, 0.0)+ Xd, Q_mask = _dropout_forward(X_for_matmul, Q_p)+ Q = _matmul_lora_ext(X_for_matmul, Xd, QW, QW_quant, QA, QB, QS, QBias)+ Xd, K_mask = _dropout_forward(X_for_matmul, K_p)+ K = _matmul_lora_ext(X_for_matmul, Xd, KW, KW_quant, KA, KB, KS, KBias)+ Xd, V_mask = _dropout_forward(X_for_matmul, V_p)+ V = _matmul_lora_ext(X_for_matmul, Xd, VW, VW_quant, VA, VB, VS, VBias)+ del Xd+ masks = (Q_mask, K_mask, V_mask)# Restore original shape after matmulif len(orig_shape) == 3:@@ -498,9 +746,12 @@ class LoRA_QKV(torch.autograd.Function):KB,VA,VB,+ *masks,)# Only an opt-in UNSLOTH_RMSNORM_SAVE_OUTPUT norm marks its output; skip the check otherwise.ctx.inplace = inplace and (not _rms_layernorm._SAVE_OUTPUT or _may_overwrite_input(X))+ ctx.dropout_p = dropout_p or (0.0, 0.0, 0.0)+ ctx.biases = (QBias is not None, KBias is not None, VBias is not None)return Q, K, V@staticmethod@@ -515,7 +766,11 @@ class LoRA_QKV(torch.autograd.Function):KB,VA,VB,+ Q_mask,+ K_mask,+ V_mask,) = ctx.saved_tensors+ Q_p, K_p, V_p = ctx.dropout_pbatch, seq_len, hd = X.shapedQ = dQ.reshape(-1, dQ.shape[-1])@@ -547,32 +802,45 @@ class LoRA_QKV(torch.autograd.Function):k_dB = dK @ KB.t()v_dB = dV @ VB.t()+ Q_mask, K_mask, V_mask = (+ None if mask is None else mask.view(-1, hd) for mask in (Q_mask, K_mask, V_mask)+ )+# d_QA = X.t() @ (dQ @ QB.t()), d_QB = (QA.t() @ X.t()) @ dQ, both scaled by QS; K and V below are# identical with their own scales.- d_QA.addmm_(X.t(), q_dB, alpha = QS, beta = 0)- d_QB.addmm_(QA.t() @ X.t(), dQ, alpha = QS, beta = 0)+ Xd = X if Q_mask is None else _dropout_replay(X, Q_mask, Q_p)+ d_QA.addmm_(Xd.t(), q_dB, alpha = QS, beta = 0)+ d_QB.addmm_(QA.t() @ Xd.t(), dQ, alpha = QS, beta = 0)++ Xd = X if K_mask is None else _dropout_replay(X, K_mask, K_p)+ d_KA.addmm_(Xd.t(), k_dB, alpha = KS, beta = 0)+ d_KB.addmm_(KA.t() @ Xd.t(), dK, alpha = KS, beta = 0)- d_KA.addmm_(X.t(), k_dB, alpha = KS, beta = 0)- d_KB.addmm_(KA.t() @ X.t(), dK, alpha = KS, beta = 0)+ Xd = X if V_mask is None else _dropout_replay(X, V_mask, V_p)+ d_VA.addmm_(Xd.t(), v_dB, alpha = VS, beta = 0)+ d_VB.addmm_(VA.t() @ Xd.t(), dV, alpha = VS, beta = 0)+ del Xd- d_VA.addmm_(X.t(), v_dB, alpha = VS, beta = 0)- d_VB.addmm_(VA.t() @ X.t(), dV, alpha = VS, beta = 0)+ Q_has_bias, K_has_bias, V_has_bias = ctx.biases+ d_QBias = _lora_bias_grad(dQ, QS) if Q_has_bias else None+ d_KBias = _lora_bias_grad(dK, KS) if K_has_bias else None+ d_VBias = _lora_bias_grad(dV, VS) if V_has_bias else None# Combine the per-projection derivatives into dX.QW = fast_dequantize(QW.t(), QW_quant)dX = torch.matmul(dQ, QW.t(), out = X if ctx.inplace and not _is_compiling() else None)del QW- addmm_(dX, q_dB, QA.t(), alpha = QS)+ _add_lora_dX(dX, q_dB, QA, QS, Q_mask, Q_p)KW = fast_dequantize(KW.t(), KW_quant)addmm_(dX, dK, KW.t())del KW- addmm_(dX, k_dB, KA.t(), alpha = KS)+ _add_lora_dX(dX, k_dB, KA, KS, K_mask, K_p)VW = fast_dequantize(VW.t(), VW_quant)addmm_(dX, dV, VW.t())del VW- addmm_(dX, v_dB, VA.t(), alpha = VS)+ _add_lora_dX(dX, v_dB, VA, VS, V_mask, V_p)return (dX.view(batch, seq_len, hd),@@ -592,6 +860,10 @@ class LoRA_QKV(torch.autograd.Function):d_VB.t(),None,None,+ None,+ d_QBias,+ d_KBias,+ d_VBias,)@@ -600,14 +872,18 @@ def apply_lora_qkv(X,inplace = True,):- if any(+ projs = (self.q_proj, self.k_proj, self.v_proj)+ extras = None+ if not any(_has_multiple_active_adapters(proj)or _has_active_dora_adapter(proj)- or _has_active_lora_bias(proj)or _has_activation_fake_quantizer(proj)- for proj in (self.q_proj, self.k_proj, self.v_proj)+ for proj in projs):+ extras = _lora_dropout_and_bias(projs, X)+ if extras is None:return self.q_proj(X), self.k_proj(X), self.v_proj(X)+ dropout_p, biases = extrasQW, QW_quant, QA, QB, QS = get_lora_parameters(self.q_proj)KW, KW_quant, KA, KB, KS = get_lora_parameters(self.k_proj)VW, VW_quant, VA, VB, VS = get_lora_parameters(self.v_proj)@@ -630,6 +906,8 @@ def apply_lora_qkv(VB,VS,inplace,+ dropout_p,+ *biases,)return Q, K, V@@ -664,26 +942,46 @@ class LoRA_W(torch.autograd.Function):@staticmethod@torch_amp_custom_fwd- def forward(ctx, X: torch.Tensor, W, W_quant, A, B, S):+ def forward(+ ctx,+ X: torch.Tensor,+ W,+ W_quant,+ A,+ B,+ S,+ dropout_p = None,+ bias = None,+ ):dtype = X.dtype- XW = matmul_lora(X, W, W_quant, A, B, S)+ if dropout_p is None and bias is None:+ XW = matmul_lora(X, W, W_quant, A, B, S)+ mask = None+ else:+ Xd, mask = _dropout_forward(X, dropout_p or 0.0)+ XW = _matmul_lora_ext(X, Xd, W, W_quant, A, B, S, bias)+ del Xdctx.custom_saved_tensors = (W,W_quant,S,)- ctx.save_for_backward(A, B, X)+ ctx.save_for_backward(A, B, X, mask)+ ctx.dropout_p = dropout_p+ ctx.has_bias = bias is not Nonereturn XW@staticmethod@torch_amp_custom_bwddef backward(ctx, dY: torch.Tensor):W, W_quant, S = ctx.custom_saved_tensors- A, B, X = ctx.saved_tensors+ A, B, X, mask = ctx.saved_tensorsbatch, seq_len, hd = X.shapedY = dY.reshape(-1, dY.shape[-1]) # Must be reshapeX = X.reshape(-1, X.shape[-1]) # Must be reshape+ if mask is not None:+ mask = mask.reshape(-1, hd)dtype = X.dtypeA, B = A.to(dtype), B.to(dtype)@@ -696,28 +994,35 @@ class LoRA_W(torch.autograd.Function):y_dB = dY @ B.t()# d_A = X.t() @ (dY @ B.t()), d_B = (A.t() @ X.t()) @ dY, both scaled by S.- d_A.addmm_(X.t(), y_dB, alpha = S, beta = 0)- d_B.addmm_(A.t() @ X.t(), dY, alpha = S, beta = 0)+ Xd = X if mask is None else _dropout_replay(X, mask, ctx.dropout_p)+ d_A.addmm_(Xd.t(), y_dB, alpha = S, beta = 0)+ d_B.addmm_(A.t() @ Xd.t(), dY, alpha = S, beta = 0)+ del Xd+ d_bias = _lora_bias_grad(dY, S) if ctx.has_bias else NoneW = fast_dequantize(W.t(), W_quant)dX = dY @ W.t()del W- addmm_(dX, y_dB, A.t(), alpha = S)+ _add_lora_dX(dX, y_dB, A, S, mask, ctx.dropout_p)- return dX.view(batch, seq_len, hd), None, None, d_A.t(), d_B.t(), None+ return dX.view(batch, seq_len, hd), None, None, d_A.t(), d_B.t(), None, None, d_biasdef apply_lora_o(self, X):- if (+ extras = None+ if not (_has_multiple_active_adapters(self.o_proj)or _has_active_dora_adapter(self.o_proj)- or _has_active_lora_bias(self.o_proj)or _has_activation_fake_quantizer(self.o_proj)):+ extras = _lora_dropout_and_bias((self.o_proj,), X)+ if extras is None:return self.o_proj(X)+ dropout_p, (bias,) = extrasOW, OW_quant, OA, OB, OS = get_lora_parameters(self.o_proj)- O = _apply(LoRA_W, X, OW, OW_quant, OA, OB, OS)- return O+ if dropout_p is None and bias is None:+ return _apply(LoRA_W, X, OW, OW_quant, OA, OB, OS)+ return _apply(LoRA_W, X, OW, OW_quant, OA, OB, OS, dropout_p and dropout_p[0], bias)IDENTITY_DROPOUT = torch.nn.Identitydiff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.pyindex 4d7a486cf00..2d20604c749 100644--- a/unsloth/models/_utils.py+++ b/unsloth/models/_utils.py@@ -7354,7 +7354,9 @@ def validate_loftq_config(signature = str(inspect.signature(LoraConfig))SUPPORTS_LOFTQ = "loftq_config" in signature- if lora_dropout != 0:+ from ..kernels.fast_lora import FUSED_LORA_DROPOUT++ if lora_dropout != 0 and not FUSED_LORA_DROPOUT:logger.warning_once(f"Unsloth: Dropout = 0 is supported for fast patching. You are using dropout = {lora_dropout}.\n"f"Unsloth will patch all other layers, except LoRA matrices, causing a performance hit."diff --git a/unsloth/models/llama.py b/unsloth/models/llama.pyindex ccca75cfd83..54078768963 100644--- a/unsloth/models/llama.py+++ b/unsloth/models/llama.py@@ -146,7 +146,8 @@ from unsloth.models._attn_mask_compat import ()from transformers.cache_utils import DynamicCache, Cachefrom ..kernels import *-from ..kernels.utils import _has_active_lora_bias, has_mxfp4_base+from ..kernels.utils import has_mxfp4_base+from ..kernels import fast_lora as _fast_lorafrom ..kernels.bnb_override import install_bnb_nf4_override as _install_bnb_nf4_overridefrom ..tokenizer_utils import *from ..context_parallel import get_cp_manager@@ -2742,8 +2743,8 @@ def _fused_lora_skip_reason(conditions mirror the fused-kernel gate in patch_peft_model."""reasons = []- if lora_dropout != 0:- reasons.append(f"lora_dropout = {lora_dropout}")+ if lora_dropout != 0 and not _fast_lora.FUSED_LORA_DROPOUT:+ reasons.append(f"lora_dropout = {lora_dropout} with UNSLOTH_FUSED_LORA_DROPOUT=0")if bias != "none":reasons.append(f"bias = '{bias}'")if float32_base:@@ -4306,7 +4307,7 @@ class FastLlamaModel:SUPPORTS_LOFTQ = "loftq_config" in signatureSUPPORTS_RSLORA = "use_rslora" in signature- if lora_dropout != 0:+ if lora_dropout != 0 and not _fast_lora.FUSED_LORA_DROPOUT:logger.warning_once(f"Unsloth: Dropout = 0 is supported for fast patching. You are using dropout = {lora_dropout}.\n"f"Unsloth will patch all other layers, except LoRA matrices, causing a performance hit."@@ -4819,7 +4820,7 @@ class FastLlamaModel:fused_lora_declined_for_fsdp = fsdp_will_wrap() and not fsdp2_keeps_fused_lora()if (- lora_dropout == 0+ (lora_dropout == 0 or _fast_lora.FUSED_LORA_DROPOUT)and bias == "none"and not float32_baseand not fused_lora_declined_for_fsdp@@ -4845,9 +4846,9 @@ class FastLlamaModel:and (len(getattr(down_proj, "lora_magnitude_vector", []) or []) == 0)and not has_mxfp4_base(gate_proj, up_proj, down_proj)and not _has_packed_base(gate_proj, up_proj, down_proj)- and not _has_active_lora_bias(gate_proj)- and not _has_active_lora_bias(up_proj)- and not _has_active_lora_bias(down_proj)+ and not _fast_lora.lora_bias_blocks_fused(gate_proj)+ and not _fast_lora.lora_bias_blocks_fused(up_proj)+ and not _fast_lora.lora_bias_blocks_fused(down_proj)):# MethodType binds the replacement; see stackoverflow.com/questions/50599045.if hasattr(mlp_module, "_unsloth_forward"):@@ -4879,9 +4880,9 @@ class FastLlamaModel:and (len(getattr(v_proj, "lora_magnitude_vector", []) or []) == 0)and not has_mxfp4_base(q_proj, k_proj, v_proj)and not _has_packed_base(q_proj, k_proj, v_proj)- and not _has_active_lora_bias(q_proj)- and not _has_active_lora_bias(k_proj)- and not _has_active_lora_bias(v_proj)+ and not _fast_lora.lora_bias_blocks_fused(q_proj)+ and not _fast_lora.lora_bias_blocks_fused(k_proj)+ and not _fast_lora.lora_bias_blocks_fused(v_proj)):layer.self_attn.apply_qkv = apply_lora_qkvn_qkv += 1@@ -4901,7 +4902,7 @@ class FastLlamaModel:and (len(getattr(o_proj, "lora_magnitude_vector", []) or []) == 0)and not has_mxfp4_base(o_proj)and not _has_packed_base(o_proj)- and not _has_active_lora_bias(o_proj)+ and not _fast_lora.lora_bias_blocks_fused(o_proj)):layer.self_attn.apply_o = apply_lora_on_o += 1