Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Fused LoRA kernels: support lora_dropout and lora_bias (#13898)

Daniel Handf63cab4b846
diff --git a/tests/test_fp8_down_proj_bf16_backward.py b/tests/test_fp8_down_proj_bf16_backward.py
index 9ad2fe33701..39e9f579231 100644
--- a/tests/test_fp8_down_proj_bf16_backward.py
+++ b/tests/test_fp8_down_proj_bf16_backward.py
@@ -84,8 +84,10 @@ def _backward(downW, downW_quant):
2.0,
backward_function,
),
- saved_tensors = (gateA, gateB, upA, upB, downA, downB, X, e, gg),
+ saved_tensors = (gateA, gateB, upA, upB, downA, downB, X, e, gg, None, None, None),
inplace = False,
+ dropout_p = (0.0, 0.0, 0.0),
+ biases = (False, False, False),
_fwd_used_autocast = False,
_dtype = torch.float32,
)
diff --git a/tests/test_fused_lora_dropout_bias.py b/tests/test_fused_lora_dropout_bias.py
new file mode 100644
index 00000000000..060ef09a1f2
--- /dev/null
+++ b/tests/test_fused_lora_dropout_bias.py
@@ -0,0 +1,364 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""lora_dropout > 0 and lora_bias = True run inside the fused LoRA kernels with PEFT's semantics.
+
+PEFT computes base(X) + lora_B(lora_A(dropout(X))) * scaling: only the LoRA branch sees the dropped
+out input, every projection draws its own mask, and lora_B may carry a bias. The fused kernels used to
+hand these configs back to PEFT. They now draw the same masks from the same RNG stream in the same
+order, so outputs and gradients match PEFT run from the same seed, and RNG state afterwards is equal.
+"""
+
+from __future__ import annotations
+
+import pytest
+import torch
+import unsloth # noqa: F401
+
+from real_accelerator import has_real_cuda
+from unsloth.kernels import fast_lora
+
+H, I = 64, 96
+TARGETS = ["q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj"]
+
+DEVICES = ["cpu"] + (["cuda"] if has_real_cuda() else [])
+
+
+class _Block(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.q_proj = torch.nn.Linear(H, H, bias = False)
+ self.k_proj = torch.nn.Linear(H, H, bias = False)
+ self.v_proj = torch.nn.Linear(H, H, bias = False)
+ self.o_proj = torch.nn.Linear(H, H, bias = False)
+ self.gate_proj = torch.nn.Linear(H, I, bias = False)
+ self.up_proj = torch.nn.Linear(H, I, bias = False)
+ self.down_proj = torch.nn.Linear(I, H, bias = False)
+ self.act_fn = torch.nn.SiLU()
+
+
+def _peft_block(
+ device,
+ dtype,
+ dropout,
+ lora_bias = False,
+):
+ from peft import LoraConfig, get_peft_model
+
+ torch.manual_seed(3407)
+ cfg = LoraConfig(
+ r = 4,
+ lora_alpha = 8,
+ lora_dropout = dropout,
+ target_modules = TARGETS,
+ init_lora_weights = False,
+ lora_bias = lora_bias,
+ )
+ model = get_peft_model(_Block(), cfg).to(device, dtype)
+ for name, p in model.named_parameters():
+ p.requires_grad_("lora_" in name)
+ if lora_bias:
+ with torch.no_grad():
+ for proj in TARGETS:
+ model.base_model.model.get_submodule(proj).lora_B["default"].bias.normal_()
+ model.train()
+ return model, model.base_model.model
+
+
+PATHS = {
+ "qkv": (
+ lambda b: (lambda X: fast_lora.apply_lora_qkv(b, X)),
+ lambda b: (lambda X: (b.q_proj(X), b.k_proj(X), b.v_proj(X))),
+ ),
+ "o": (
+ lambda b: (lambda X: fast_lora.apply_lora_o(b, X)),
+ lambda b: b.o_proj,
+ ),
+ "mlp": (
+ lambda b: (lambda X: fast_lora.apply_lora_mlp_swiglu(b, X)),
+ lambda b: (lambda X: b.down_proj(b.act_fn(b.gate_proj(X)) * b.up_proj(X))),
+ ),
+}
+
+
+def _skip_triton_on_cpu(device, path):
+ if device == "cpu" and path == "mlp":
+ pytest.skip("the SwiGLU kernels are Triton")
+
+
+def _rng_state(device):
+ return torch.cuda.get_rng_state() if device == "cuda" else torch.get_rng_state()
+
+
+def _run(fn, model, X0, seed, device):
+ torch.manual_seed(seed)
+ X = X0.detach().clone().requires_grad_()
+ out = fn(X * 1) # a non-leaf, since the fused backward may write dX into its input buffer
+ outs = out if isinstance(out, tuple) else (out,)
+ gen = torch.Generator(device = "cpu").manual_seed(seed + 1)
+ loss = sum((o.float() * torch.randn(o.shape, generator = gen).to(device)).sum() for o in outs)
+ loss.backward()
+ grads = {n: p.grad.clone() for n, p in model.named_parameters() if p.grad is not None}
+ model.zero_grad(set_to_none = True)
+ return [o.detach() for o in outs], X.grad.clone(), grads, _rng_state(device)
+
+
+class _Spy:
+ def __init__(self, monkeypatch):
+ self.calls = []
+ real = fast_lora._apply
+
+ def spy(function, *args):
+ self.calls.append((function.__name__, args))
+ return real(function, *args)
+
+ monkeypatch.setattr(fast_lora, "_apply", spy)
+
+
+class _FixedDropout(torch.nn.Module):
+ def __init__(self, mask, p):
+ super().__init__()
+ self.mask, self.scale = mask, 1.0 / (1.0 - p)
+
+ def forward(self, x):
+ return x * self.mask.to(x.dtype) * self.scale
+
+
+def _fp64_oracle(model, path, X0, seed, device, masks, p):
+ import copy
+
+ ref_model = copy.deepcopy(model).double()
+ ref_block = ref_model.base_model.model
+ for name, mask in masks.items():
+ ref_block.get_submodule(name).lora_dropout["default"] = _FixedDropout(mask, p)
+ return _run(PATHS[path][1](ref_block), ref_model, X0.double(), seed, device)
+
+
+class _Recorder(torch.nn.Module):
+ # nn.Dropout under autograd is native_dropout on CUDA and bernoulli_ noise on CPU; recording the mask
+ # as `output != 0` would also catch inputs that are exactly zero.
+ def __init__(self, dropout, masks, name):
+ super().__init__()
+ self.dropout, self.masks, self.name = dropout, masks, name
+
+ def forward(self, x):
+ if x.device.type == "cpu":
+ out = self.dropout(x)
+ self.masks[self.name] = (out != 0) | (
+ x == 0
+ ) # zeros never matter on the reference path
+ return out
+ out, self.masks[self.name] = torch.native_dropout(x, self.dropout.p, True)
+ return out
+
+
+def _record_masks(block):
+ masks, restore = {}, []
+ for name in TARGETS:
+ holder = block.get_submodule(name).lora_dropout
+ dropout = holder["default"]
+ if isinstance(dropout, torch.nn.Dropout):
+ holder["default"] = _Recorder(dropout, masks, name)
+ restore.append((holder, dropout))
+ return masks, restore
+
+
+def _rel_err(x, ref):
+ return ((x.double() - ref).norm() / ref.norm().clamp_min(1e-30)).item()
+
+
[email protected]("device", DEVICES)
[email protected]("dtype", [torch.float32, torch.bfloat16, torch.float16])
[email protected]("path", list(PATHS))
[email protected]("dropout,lora_bias", [(0.3, False), (0.0, True), (0.3, True)])
+def test_fused_matches_peft(monkeypatch, device, dtype, path, dropout, lora_bias):
+ """Fused vs PEFT from the same seed, each scored against an fp64 PEFT run on PEFT's masks."""
+ _skip_triton_on_cpu(device, path)
+ if dtype == torch.float16 and device == "cpu":
+ pytest.skip("fp16 matmuls on CPU")
+ model, block = _peft_block(device, dtype, dropout, lora_bias)
+ fused, ref = (make(block) for make in PATHS[path])
+ X0 = torch.randn(2, 7, H, device = device, dtype = dtype)
+ spy = _Spy(monkeypatch)
+
+ f_out, f_dX, f_grads, f_rng = _run(fused, model, X0, 11, device)
+ assert len(spy.calls) == 1, "the fused kernel did not run"
+ masks, restore = _record_masks(block)
+ r_out, r_dX, r_grads, r_rng = _run(ref, model, X0, 11, device)
+ for holder, module in restore:
+ holder["default"] = module
+ assert len(spy.calls) == 1
+ assert torch.equal(f_rng, r_rng), "different number of RNG draws than PEFT"
+ o_out, o_dX, o_grads, _ = _fp64_oracle(model, path, X0, 11, device, masks, dropout)
+
+ assert f_grads.keys() == r_grads.keys()
+ if lora_bias:
+ assert any(n.endswith("lora_B.default.bias") for n in f_grads)
+ pairs = list(zip(f_out, r_out, o_out)) + [(f_dX, r_dX, o_dX)]
+ pairs += [(f_grads[n], r_grads[n], o_grads[n]) for n in r_grads]
+ # Same masks, so fused is within a few roundings of PEFT, and no further from fp64 than PEFT is
+ # (with one dtype step of slack: the two paths round in a different order).
+ eps = torch.finfo(dtype).eps
+ for f, r, o in pairs:
+ e_fused, e_peft = _rel_err(f, o), _rel_err(r, o)
+ assert e_fused <= 1.5 * e_peft + 2 * eps, (e_fused, e_peft)
+
+
[email protected]("device", DEVICES)
[email protected]("path", list(PATHS))
+def test_masks_are_bitwise_peft_masks(monkeypatch, device, path):
+ _skip_triton_on_cpu(device, path)
+ # p = 0.5 with fp32: a single different mask element would move outputs by O(1).
+ model, block = _peft_block(device, torch.float32, 0.5)
+ fused, ref = (make(block) for make in PATHS[path])
+ X0 = torch.randn(3, 5, H, device = device)
+ for seed in range(3):
+ f_out = _run(fused, model, X0, seed, device)[0]
+ r_out = _run(ref, model, X0, seed, device)[0]
+ for f, r in zip(f_out, r_out):
+ torch.testing.assert_close(f, r, rtol = 1e-5, atol = 1e-5)
+
+
[email protected]("device", DEVICES)
+def test_eval_mode_skips_dropout_like_peft(monkeypatch, device):
+ model, block = _peft_block(device, torch.float32, 0.3)
+ model.eval()
+ spy = _Spy(monkeypatch)
+ X = torch.randn(2, 5, H, device = device)
+ with torch.no_grad():
+ got = fast_lora.apply_lora_qkv(block, X.clone())
+ expected = (block.q_proj(X), block.k_proj(X), block.v_proj(X))
+ for g, e in zip(got, expected):
+ torch.testing.assert_close(g, e)
+ name, args = spy.calls[0]
+ assert name == "LoRA_QKV" and args[-4:] == (None, None, None, None)
+
+
[email protected]("device", DEVICES)
[email protected]("path", list(PATHS))
+def test_checkpoint_recompute_replays_the_same_masks(device, path):
+ from torch.utils.checkpoint import checkpoint
+
+ _skip_triton_on_cpu(device, path)
+ model, block = _peft_block(device, torch.float32, 0.3)
+ fused = PATHS[path][0](block)
+ X0 = torch.randn(2, 5, H, device = device)
+ plain = _run(fused, model, X0, 5, device)
+ ckpt = _run(
+ lambda X: checkpoint(fused, X, use_reentrant = True, preserve_rng_state = True),
+ model,
+ X0,
+ 5,
+ device,
+ )
+ torch.testing.assert_close(plain[1], ckpt[1], rtol = 0, atol = 0)
+ for n in plain[2]:
+ torch.testing.assert_close(plain[2][n], ckpt[2][n], rtol = 0, atol = 0)
+
+
+ "dropout,lora_bias,flag", [(0.3, False, "FUSED_LORA_DROPOUT"), (0.0, True, "FUSED_LORA_BIAS")]
+)
+def test_kill_switch_restores_the_peft_fallback(monkeypatch, dropout, lora_bias, flag):
+ model, block = _peft_block("cpu", torch.float32, dropout, lora_bias)
+ monkeypatch.setattr(fast_lora, flag, False)
+ spy = _Spy(monkeypatch)
+ X = torch.randn(2, 5, H)
+ fast_lora.apply_lora_qkv(block, X)
+ fast_lora.apply_lora_o(block, X)
+ if has_real_cuda():
+ model, block = _peft_block("cuda", torch.float32, dropout, lora_bias)
+ fast_lora.apply_lora_mlp_swiglu(block, X.cuda())
+ assert spy.calls == []
+
+
+def test_unknown_dropout_module_falls_back(monkeypatch):
+ model, block = _peft_block("cpu", torch.float32, 0.3)
+ block.o_proj.lora_dropout["default"] = torch.nn.AlphaDropout(0.3)
+ spy = _Spy(monkeypatch)
+ fast_lora.apply_lora_o(block, torch.randn(2, 5, H))
+ assert spy.calls == []
+
+
+def test_zero_dropout_no_bias_call_is_unchanged(monkeypatch):
+ # The pre-existing fused call shape: no dropout / bias arguments reach LoRA_W.
+ model, block = _peft_block("cpu", torch.float32, 0.0)
+ spy = _Spy(monkeypatch)
+ fast_lora.apply_lora_o(block, torch.randn(2, 5, H))
+ assert [len(args) for _, args in spy.calls] == [6]
+
+
[email protected]("dropout,lora_bias", [(0.0, False), (0.3, False), (0.0, True)])
+def test_active_adapter_absent_from_projection_falls_back(monkeypatch, dropout, lora_bias):
+ # Adapters with different target modules: PEFT runs the base layer where the active one is missing.
+ from peft import LoraConfig
+
+ model, block = _peft_block("cpu", torch.float32, dropout, lora_bias)
+ model.add_adapter(
+ "other",
+ LoraConfig(
+ r = 4, lora_alpha = 8, lora_dropout = dropout, target_modules = ["q_proj"], lora_bias = lora_bias
+ ),
+ )
+ model.set_adapter("other")
+ model.train()
+ spy = _Spy(monkeypatch)
+ X = torch.randn(2, 5, H)
+ torch.testing.assert_close(fast_lora.apply_lora_o(block, X), block.o_proj.base_layer(X))
+ fast_lora.apply_lora_qkv(block, X)
+ assert spy.calls == []
+
+
[email protected]("lora_bias", [False, True])
+def test_routing_resolves_each_active_adapter_once(monkeypatch, lora_bias):
+ # The routing runs on every fused LoRA call: the default config (no dropout, no bias) never reaches the
+ # general adapter resolution, the rest resolve each projection's adapter once, and the lora_bias scan
+ # is skipped unless UNSLOTH_FUSED_LORA_BIAS=0.
+ model, block = _peft_block("cpu", torch.float32, 0.0, lora_bias)
+ projs = (block.q_proj, block.k_proj, block.v_proj)
+ real = fast_lora._active_lora_adapter
+ seen = []
+ monkeypatch.setattr(
+ fast_lora, "_active_lora_adapter", lambda proj: seen.append(proj) or real(proj)
+ )
+ monkeypatch.setattr(
+ fast_lora,
+ "_has_active_lora_bias",
+ lambda proj: pytest.fail("lora_bias scanned with fused bias on"),
+ )
+ dropout_p, biases = fast_lora._lora_dropout_and_bias(projs, torch.randn(2, 5, H))
+ assert dropout_p is None
+ assert len(seen) == (len(projs) if lora_bias else 0)
+ assert all((b is not None) == lora_bias for b in biases)
+
+
+ "state", ["train", "eval", "disabled", "merged", "absent", "list_adapter", "dropout", "bias"]
+)
+def test_plain_lora_shortcut_matches_general_routing(monkeypatch, state):
+ # The default-config shortcut must give the general routing's answer in every adapter state.
+ from peft import LoraConfig
+
+ model, block = _peft_block(
+ "cpu", torch.float32, 0.3 if state == "dropout" else 0.0, state == "bias"
+ )
+ if state == "eval":
+ model.eval()
+ elif state == "disabled":
+ model.base_model.disable_adapter_layers()
+ elif state == "merged":
+ block.q_proj.merge()
+ elif state == "absent":
+ model.add_adapter("other", LoraConfig(r = 4, lora_alpha = 8, target_modules = ["o_proj"]))
+ model.set_adapter("other")
+ elif state == "list_adapter":
+ for proj in (block.q_proj, block.k_proj, block.v_proj):
+ proj._active_adapter = ["default"]
+ projs = (block.q_proj, block.k_proj, block.v_proj)
+ X = torch.randn(2, 5, H)
+ plain = [fast_lora._is_plain_lora(proj) for proj in projs]
+ fast = fast_lora._lora_dropout_and_bias(projs, X)
+ monkeypatch.setattr(fast_lora, "_is_plain_lora", lambda proj: False)
+ general = fast_lora._lora_dropout_and_bias(projs, X)
+ assert fast == general
+ assert all(plain) == (state in ("train", "eval", "list_adapter"))
diff --git a/tests/test_lora_dropout_patch_summary.py b/tests/test_lora_dropout_patch_summary.py
index 16515407334..6227e77d9a9 100644
--- a/tests/test_lora_dropout_patch_summary.py
+++ b/tests/test_lora_dropout_patch_summary.py
@@ -1,7 +1,11 @@
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
-"""Fused LoRA kernels are skipped when lora_dropout or bias is set, and the summary must say why."""
+"""Fused LoRA kernels are skipped when lora_dropout or bias is set, and the summary must say why.
+
+The fused kernels now run lora_dropout > 0 themselves, so the dropout clause only appears under
+the UNSLOTH_FUSED_LORA_DROPOUT=0 kill switch, which these tests set.
+"""
from __future__ import annotations
@@ -14,10 +18,17 @@ import pytest
import unsloth # noqa: F401
from real_accelerator import has_real_accelerator
+from unsloth.kernels import fast_lora
from unsloth.models import llama as llama_module
from unsloth.models.llama import _fused_lora_skip_reason
[email protected](autouse = True)
+def _fused_dropout_off(monkeypatch):
+ # lora_dropout > 0 only closes the fused gate under the UNSLOTH_FUSED_LORA_DROPOUT=0 kill switch.
+ monkeypatch.setattr(fast_lora, "FUSED_LORA_DROPOUT", False)
+
+
def test_no_reason_when_the_fused_kernels_were_installed():
assert _fused_lora_skip_reason(0, "none") == ""
assert _fused_lora_skip_reason(0.0, "none") == ""
@@ -26,7 +37,7 @@ def test_no_reason_when_the_fused_kernels_were_installed():
def test_lora_dropout_is_named_with_its_value():
reason = _fused_lora_skip_reason(0.1, "none")
assert "lora_dropout = 0.1" in reason
- assert "bias" not in reason
+ assert "bias =" not in reason
assert "Training is unaffected." in reason
@@ -69,6 +80,12 @@ def test_reason_is_non_empty_exactly_when_the_fused_gate_is_closed(
assert bool(reason) is not fused_installed
[email protected]("lora_dropout", [0, 0.1, 0.5])
+def test_lora_dropout_keeps_the_fused_gate_open_by_default(monkeypatch, lora_dropout):
+ monkeypatch.setattr(fast_lora, "FUSED_LORA_DROPOUT", True)
+ assert _fused_lora_skip_reason(lora_dropout, "none") == ""
+
+
def _gate_tests(source: str) -> list[str]:
"""Every `if` test in patch_peft_model that reads both lora_dropout and bias."""
tree = ast.parse(textwrap.dedent(source))
@@ -86,7 +103,7 @@ def test_patch_peft_model_still_gates_the_fused_kernels_on_the_same_values():
"""Every gate condition needs a matching _fused_lora_skip_reason clause."""
source = inspect.getsource(llama_module.FastLlamaModel.patch_peft_model)
assert _gate_tests(source) == [
- "lora_dropout == 0 and bias == 'none' and (not float32_base) and (not fused_lora_declined_for_fsdp)"
+ "(lora_dropout == 0 or _fast_lora.FUSED_LORA_DROPOUT) and bias == 'none' and (not float32_base) and (not fused_lora_declined_for_fsdp)"
]
diff --git a/unsloth/kernels/fast_lora.py b/unsloth/kernels/fast_lora.py
index 685d6d14297..fca52dadc9d 100644
--- a/unsloth/kernels/fast_lora.py
+++ b/unsloth/kernels/fast_lora.py
@@ -9,6 +9,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+import os
import sys
import torch
from unsloth_zoo.utils import Version
@@ -75,6 +76,167 @@ def _has_activation_fake_quantizer(proj):
return False
+# lora_dropout > 0 and lora_bias = True run inside the fused kernels; "0" restores the PEFT fallback.
+FUSED_LORA_DROPOUT = os.environ.get("UNSLOTH_FUSED_LORA_DROPOUT", "1") != "0"
+FUSED_LORA_BIAS = os.environ.get("UNSLOTH_FUSED_LORA_BIAS", "1") != "0"
+
+
+def _active_lora_adapter(proj):
+ if getattr(proj, "disable_adapters", True) or getattr(proj, "merged", False):
+ return None
+ adapter = getattr(proj, "active_adapters", None)
+ if adapter is None:
+ adapter = getattr(proj, "active_adapter", "default")
+ if not isinstance(adapter, str):
+ adapter = adapter[0]
+ return adapter if adapter in proj.lora_A else None
+
+
+def _lora_dropout_p(
+ proj,
+ X,
+ adapter = False,
+):
+ """Dropout probability of the active adapter, 0.0 when it is a no-op, None when only PEFT can run it."""
+ if adapter is False:
+ adapter = _active_lora_adapter(proj)
+ dropout = getattr(proj, "lora_dropout", None)
+ if adapter is None or dropout is None or adapter not in dropout:
+ return 0.0
+ dropout = dropout[adapter]
+ if isinstance(dropout, torch.nn.Identity) or not dropout.training:
+ return 0.0
+ if (
+ type(dropout) is not torch.nn.Dropout
+ or dropout.inplace
+ or not FUSED_LORA_DROPOUT
+ or X.device.type not in ("cuda", "xpu", "cpu")
+ ):
+ return None
+ p = float(dropout.p)
+ if p == 0:
+ return 0.0
+ return p if p < 1 else None
+
+
+def _lora_B_bias(proj):
+ adapter = _active_lora_adapter(proj)
+ if adapter is None:
+ return None
+ return proj.lora_B[adapter].bias
+
+
+def lora_bias_blocks_fused(proj):
+ return not FUSED_LORA_BIAS and _has_active_lora_bias(proj)
+
+
+def _is_plain_lora(proj):
+ # One enabled, unmerged adapter with no dropout module and no lora_B bias (the default LoRA config).
+ # Reads PEFT's instance state directly: getattr through nn.Module / properties costs more than the
+ # routing it guards. Anything else (or other PEFT internals) takes the general path below.
+ state = proj.__dict__
+ adapter = state.get("_active_adapter")
+ if type(adapter) is not str:
+ if type(adapter) is not list or len(adapter) != 1:
+ return False
+ adapter = adapter[0]
+ if state.get("_disable_adapters", True) or state.get("merged_adapters"):
+ return False
+ modules = proj._modules
+ dropout = modules.get("lora_dropout")
+ lora_B = modules.get("lora_B")
+ if dropout is None or lora_B is None:
+ return False
+ lora_B = lora_B._modules.get(adapter)
+ return (
+ type(dropout._modules.get(adapter)) is torch.nn.Identity
+ and lora_B is not None
+ and lora_B._parameters.get("bias") is None
+ )
+
+
+def _lora_dropout_and_bias(projs, X):
+ """(dropout probabilities or None, lora_B biases) for the fused kernels, or None to use PEFT."""
+ for proj in projs:
+ if not _is_plain_lora(proj):
+ break
+ else:
+ return None, (None,) * len(projs)
+ dropout_p, biases = [], []
+ for proj in projs:
+ adapter = _active_lora_adapter(proj)
+ if adapter is None:
+ # Enabled adapter missing from this projection: PEFT runs the base layer alone.
+ if not getattr(proj, "disable_adapters", True) and not getattr(proj, "merged", False):
+ return None
+ dropout_p.append(0.0)
+ biases.append(None)
+ continue
+ p = _lora_dropout_p(proj, X, adapter)
+ if p is None:
+ return None
+ dropout_p.append(p)
+ biases.append(proj.lora_B[adapter].bias)
+ if any(lora_bias_blocks_fused(proj) for proj in projs):
+ return None
+ return (tuple(dropout_p) if any(dropout_p) else None), tuple(biases)
+
+
+def _dropout_forward(X, p):
+ # Same RNG draws and rounding as torch.nn.Dropout under autograd: native_dropout on CUDA / XPU
+ # (ATen's fused path), a bernoulli_ noise tensor in X's dtype on CPU.
+ if not p:
+ return X, None
+ if X.device.type == "cpu":
+ noise = torch.empty_like(X).bernoulli_(1 - p)
+ mask = noise.bool()
+ return X * noise.div_(1 - p), mask
+ return torch.native_dropout(X, p, True)
+
+
+def _dropout_replay(
+ T,
+ mask,
+ p,
+ S = 1.0,
+):
+ # S * T * mask / (1 - p): with S = 1 exactly the dropout forward's output, and the dropout backward.
+ if mask.device.type == "cpu":
+ T = T * mask.to(T.dtype).div_(1 - p)
+ return T if S == 1 else T.mul_(S)
+ return torch.ops.aten.native_dropout_backward(T, mask, S / (1.0 - p))
+
+
+def _matmul_lora_ext(X, X_lora, W, W_quant, A, B, S, bias):
+ # matmul_lora with the LoRA branch reading X_lora (X after dropout, or X itself) plus lora_B's bias.
+ if A is None or (X_lora is X and bias is None):
+ return matmul_lora(X, W, W_quant, A, B, S)
+ out = matmul_lora(X, W, W_quant, None, None, None)
+ dtype = X.dtype
+ XA = torch.matmul(X_lora.reshape(-1, X_lora.shape[-1]), A.to(dtype).t())
+ out2d = out.view(-1, out.shape[-1])
+ if bias is None:
+ addmm_(out2d, XA, B.to(dtype).t(), alpha = S)
+ else:
+ # PEFT's rounding order: (lora_B(x) + bias) * scaling, then + base. No add(alpha = S): Inductor
+ # (torch 2.13) miscompiles it traced inside these Functions.
+ out2d.add_(torch.addmm(bias.to(dtype), XA, B.to(dtype).t()).mul_(S))
+ return out
+
+
+def _add_lora_dX(dX, y_dB, A, S, mask, p):
+ # dX += S * y_dB @ A.t(), through the dropout mask when the LoRA branch saw a dropped out X.
+ if mask is None:
+ addmm_(dX, y_dB, A.t(), alpha = S)
+ else:
+ dX.add_(_dropout_replay(y_dB @ A.t(), mask, p, S))
+
+
+def _lora_bias_grad(dY, S):
+ # Autograd casts it to the bias dtype.
+ return dY.sum(0, dtype = torch.float32).mul_(S)
+
+
def _may_overwrite_input(X):
# Backward writes dX into X; not when an RMSNorm saved X (its output) for its own backward,
# including through a view of it (tiled MLP chunks).
@@ -149,13 +311,32 @@ class LoRA_MLP(torch.autograd.Function):
_forward_function,
_backward_function,
inplace = True,
+ dropout_p = None,
+ gateBias = None,
+ upBias = None,
+ downBias = None,
):
dtype = X.dtype
- e = matmul_lora(X, gateW, gateW_quant, gateA, gateB, gateS)
- g = matmul_lora(X, upW, upW_quant, upA, upB, upS)
- h = _forward_function(e, g)
- i = matmul_lora(h, downW, downW_quant, downA, downB, downS)
+ if dropout_p is None and gateBias is None and upBias is None and downBias is None:
+ e = matmul_lora(X, gateW, gateW_quant, gateA, gateB, gateS)
+ g = matmul_lora(X, upW, upW_quant, upA, upB, upS)
+ h = _forward_function(e, g)
+ i = matmul_lora(h, downW, downW_quant, downA, downB, downS)
+ masks = (None, None, None)
+ else:
+ # Masks are drawn gate, up, down: the order PEFT's down(act(gate(X)) * up(X)) draws them in.
+ gate_p, up_p, down_p = dropout_p or (0.0, 0.0, 0.0)
+ Xd, gate_mask = _dropout_forward(X, gate_p)
+ e = _matmul_lora_ext(X, Xd, gateW, gateW_quant, gateA, gateB, gateS, gateBias)
+ Xd, up_mask = _dropout_forward(X, up_p)
+ g = _matmul_lora_ext(X, Xd, upW, upW_quant, upA, upB, upS, upBias)
+ del Xd
+ h = _forward_function(e, g)
+ hd, down_mask = _dropout_forward(h, down_p)
+ i = _matmul_lora_ext(h, hd, downW, downW_quant, downA, downB, downS, downBias)
+ del hd
+ masks = (gate_mask, up_mask, down_mask)
ctx.custom_saved_tensors = (
gateW,
@@ -169,9 +350,11 @@ class LoRA_MLP(torch.autograd.Function):
downS,
_backward_function,
)
- ctx.save_for_backward(gateA, gateB, upA, upB, downA, downB, X, e, g)
+ ctx.save_for_backward(gateA, gateB, upA, upB, downA, downB, X, e, g, *masks)
# Only an opt-in UNSLOTH_RMSNORM_SAVE_OUTPUT norm marks its output; skip the check otherwise.
ctx.inplace = inplace and (not _rms_layernorm._SAVE_OUTPUT or _may_overwrite_input(X))
+ ctx.dropout_p = dropout_p or (0.0, 0.0, 0.0)
+ ctx.biases = (gateBias is not None, upBias is not None, downBias is not None)
return i
@staticmethod
@@ -189,7 +372,10 @@ class LoRA_MLP(torch.autograd.Function):
downS,
_backward_function,
) = ctx.custom_saved_tensors
- gateA, gateB, upA, upB, downA, downB, X, e, g = ctx.saved_tensors
+ gateA, gateB, upA, upB, downA, downB, X, e, g, gate_mask, up_mask, down_mask = (
+ ctx.saved_tensors
+ )
+ gate_p, up_p, down_p = ctx.dropout_p
batch, seq_len, hd = X.shape
dY = dY.reshape(-1, dY.shape[-1])
@@ -218,11 +404,21 @@ class LoRA_MLP(torch.autograd.Function):
if downW.dtype == torch.float8_e4m3fn:
# Dequantize like gate / up: matmul_lora would send the FP8 weight to fp8_linear, quantizing dY to FP8.
- DW = matmul_lora(dY, fast_dequantize(downW.t(), downW_quant), None, downB, downA, downS)
+ downW, downW_quant = fast_dequantize(downW.t(), downW_quant), None
+ else:
+ downW = downW.t()
+ if down_mask is None:
+ DW = matmul_lora(dY, downW, downW_quant, downB, downA, downS)
else:
- DW = matmul_lora(dY, downW.t(), downW_quant, downB, downA, downS)
+ # Only the LoRA branch saw the dropped out h, so only its gradient goes through the mask.
+ DW = matmul_lora(dY, downW, downW_quant, None, None, None)
+ down_mask = down_mask.view(-1, down_mask.shape[-1])
+ DW.add_(_dropout_replay((dY @ downB.t()) @ downA.t(), down_mask, down_p, downS))
+ del downW
DW, e, g = _backward_function(DW, e, g)
h, df, de = DW, e, g
+ if down_mask is not None:
+ h = _dropout_replay(h, down_mask, down_p)
d_downA = torch.empty_like(downA)
d_downB = torch.empty_like(downB)
@@ -238,25 +434,38 @@ class LoRA_MLP(torch.autograd.Function):
up_dB = df @ upB.t()
gate_dB = de @ gateB.t()
+ if up_mask is not None:
+ up_mask = up_mask.view(-1, hd)
+ if gate_mask is not None:
+ gate_mask = gate_mask.view(-1, hd)
+
# d_upA = X.t() @ (df @ upB.t()), d_upB = (upA.t() @ X.t()) @ df, both scaled by upS.
- d_upA.addmm_(X.t(), up_dB, alpha = upS, beta = 0)
- d_upB.addmm_(upA.t() @ X.t(), df, alpha = upS, beta = 0)
+ Xd = X if up_mask is None else _dropout_replay(X, up_mask, up_p)
+ d_upA.addmm_(Xd.t(), up_dB, alpha = upS, beta = 0)
+ d_upB.addmm_(upA.t() @ Xd.t(), df, alpha = upS, beta = 0)
# d_gateA = X.t() @ (de @ gateB.t()), d_gateB = (gateA.t() @ X.t()) @ de, both scaled by gateS.
- d_gateA.addmm_(X.t(), gate_dB, alpha = gateS, beta = 0)
- d_gateB.addmm_(gateA.t() @ X.t(), de, alpha = gateS, beta = 0)
+ Xd = X if gate_mask is None else _dropout_replay(X, gate_mask, gate_p)
+ d_gateA.addmm_(Xd.t(), gate_dB, alpha = gateS, beta = 0)
+ d_gateB.addmm_(gateA.t() @ Xd.t(), de, alpha = gateS, beta = 0)
+ del Xd
+
+ gate_has_bias, up_has_bias, down_has_bias = ctx.biases
+ d_gateBias = _lora_bias_grad(de, gateS) if gate_has_bias else None
+ d_upBias = _lora_bias_grad(df, upS) if up_has_bias else None
+ d_downBias = _lora_bias_grad(dY, downS) if down_has_bias else None
# dX = matmul_lora(df, upW.t(), ...) + matmul_lora(de, gateW.t(), ...), expanded below.
upW = fast_dequantize(upW.t(), upW_quant)
# Eager only: AOT autograd rejects a backward mutating a forward input that requires grad.
dX = torch.matmul(df, upW.t(), out = X if ctx.inplace and not _is_compiling() else None)
del upW
- addmm_(dX, up_dB, upA.t(), alpha = upS)
+ _add_lora_dX(dX, up_dB, upA, upS, up_mask, up_p)
gateW = fast_dequantize(gateW.t(), gateW_quant)
addmm_(dX, de, gateW.t())
del gateW
- addmm_(dX, gate_dB, gateA.t(), alpha = gateS)
+ _add_lora_dX(dX, gate_dB, gateA, gateS, gate_mask, gate_p)
return (
dX.view(batch, seq_len, hd),
@@ -278,6 +487,10 @@ class LoRA_MLP(torch.autograd.Function):
None,
None,
None,
+ None,
+ d_gateBias,
+ d_upBias,
+ d_downBias,
)
@@ -289,14 +502,18 @@ def apply_lora_mlp_swiglu(
X,
inplace = True,
):
- if any(
+ projs = (self.gate_proj, self.up_proj, self.down_proj)
+ extras = None
+ if not any(
_has_multiple_active_adapters(proj)
or _has_active_dora_adapter(proj)
- or _has_active_lora_bias(proj)
or _has_activation_fake_quantizer(proj)
- for proj in (self.gate_proj, self.up_proj, self.down_proj)
+ for proj in projs
):
+ extras = _lora_dropout_and_bias(projs, X)
+ if extras is None:
return self.down_proj(self.act_fn(self.gate_proj(X)) * self.up_proj(X))
+ dropout_p, biases = extras
gateW, gateW_quant, gateA, gateB, gateS = get_lora_parameters(self.gate_proj)
upW, upW_quant, upA, upB, upS = get_lora_parameters(self.up_proj)
downW, downW_quant, downA, downB, downS = get_lora_parameters(self.down_proj)
@@ -321,6 +538,8 @@ def apply_lora_mlp_swiglu(
swiglu_fg_kernel,
swiglu_DWf_DW_dfg_kernel,
inplace,
+ dropout_p,
+ *biases,
)
return out
@@ -333,14 +552,18 @@ def apply_lora_mlp_geglu_exact(
X,
inplace = True,
):
- if any(
+ projs = (self.gate_proj, self.up_proj, self.down_proj)
+ extras = None
+ if not any(
_has_multiple_active_adapters(proj)
or _has_active_dora_adapter(proj)
- or _has_active_lora_bias(proj)
or _has_activation_fake_quantizer(proj)
- for proj in (self.gate_proj, self.up_proj, self.down_proj)
+ for proj in projs
):
+ extras = _lora_dropout_and_bias(projs, X)
+ if extras is None:
return self.down_proj(self.act_fn(self.gate_proj(X)) * self.up_proj(X))
+ dropout_p, biases = extras
gateW, gateW_quant, gateA, gateB, gateS = get_lora_parameters(self.gate_proj)
upW, upW_quant, upA, upB, upS = get_lora_parameters(self.up_proj)
downW, downW_quant, downA, downB, downS = get_lora_parameters(self.down_proj)
@@ -365,6 +588,8 @@ def apply_lora_mlp_geglu_exact(
geglu_exact_forward_kernel,
geglu_exact_backward_kernel,
inplace,
+ dropout_p,
+ *biases,
)
return out
@@ -373,14 +598,18 @@ from .geglu import geglu_approx_forward_kernel, geglu_approx_backward_kernel
def apply_lora_mlp_geglu_approx(self, X):
- if any(
+ projs = (self.gate_proj, self.up_proj, self.down_proj)
+ extras = None
+ if not any(
_has_multiple_active_adapters(proj)
or _has_active_dora_adapter(proj)
- or _has_active_lora_bias(proj)
or _has_activation_fake_quantizer(proj)
- for proj in (self.gate_proj, self.up_proj, self.down_proj)
+ for proj in projs
):
+ extras = _lora_dropout_and_bias(projs, X)
+ if extras is None:
return self.down_proj(self.act_fn(self.gate_proj(X)) * self.up_proj(X))
+ dropout_p, biases = extras
gateW, gateW_quant, gateA, gateB, gateS = get_lora_parameters(self.gate_proj)
upW, upW_quant, upA, upB, upS = get_lora_parameters(self.up_proj)
downW, downW_quant, downA, downB, downS = get_lora_parameters(self.down_proj)
@@ -404,6 +633,9 @@ def apply_lora_mlp_geglu_approx(self, X):
downS,
geglu_approx_forward_kernel,
geglu_approx_backward_kernel,
+ True,
+ dropout_p,
+ *biases,
)
return out
@@ -460,6 +692,10 @@ class LoRA_QKV(torch.autograd.Function):
VB,
VS,
inplace = True,
+ dropout_p = None,
+ QBias = None,
+ KBias = None,
+ VBias = None,
):
dtype = X.dtype
@@ -469,9 +705,21 @@ class LoRA_QKV(torch.autograd.Function):
X_for_matmul = X
if X.dim() == 3:
X_for_matmul = X.view(-1, X.shape[-1])
- Q = matmul_lora(X_for_matmul, QW, QW_quant, QA, QB, QS)
- K = matmul_lora(X_for_matmul, KW, KW_quant, KA, KB, KS)
- V = matmul_lora(X_for_matmul, VW, VW_quant, VA, VB, VS)
+ if dropout_p is None and QBias is None and KBias is None and VBias is None:
+ Q = matmul_lora(X_for_matmul, QW, QW_quant, QA, QB, QS)
+ K = matmul_lora(X_for_matmul, KW, KW_quant, KA, KB, KS)
+ V = matmul_lora(X_for_matmul, VW, VW_quant, VA, VB, VS)
+ masks = (None, None, None)
+ else:
+ Q_p, K_p, V_p = dropout_p or (0.0, 0.0, 0.0)
+ Xd, Q_mask = _dropout_forward(X_for_matmul, Q_p)
+ Q = _matmul_lora_ext(X_for_matmul, Xd, QW, QW_quant, QA, QB, QS, QBias)
+ Xd, K_mask = _dropout_forward(X_for_matmul, K_p)
+ K = _matmul_lora_ext(X_for_matmul, Xd, KW, KW_quant, KA, KB, KS, KBias)
+ Xd, V_mask = _dropout_forward(X_for_matmul, V_p)
+ V = _matmul_lora_ext(X_for_matmul, Xd, VW, VW_quant, VA, VB, VS, VBias)
+ del Xd
+ masks = (Q_mask, K_mask, V_mask)
# Restore original shape after matmul
if len(orig_shape) == 3:
@@ -498,9 +746,12 @@ class LoRA_QKV(torch.autograd.Function):
KB,
VA,
VB,
+ *masks,
)
# Only an opt-in UNSLOTH_RMSNORM_SAVE_OUTPUT norm marks its output; skip the check otherwise.
ctx.inplace = inplace and (not _rms_layernorm._SAVE_OUTPUT or _may_overwrite_input(X))
+ ctx.dropout_p = dropout_p or (0.0, 0.0, 0.0)
+ ctx.biases = (QBias is not None, KBias is not None, VBias is not None)
return Q, K, V
@staticmethod
@@ -515,7 +766,11 @@ class LoRA_QKV(torch.autograd.Function):
KB,
VA,
VB,
+ Q_mask,
+ K_mask,
+ V_mask,
) = ctx.saved_tensors
+ Q_p, K_p, V_p = ctx.dropout_p
batch, seq_len, hd = X.shape
dQ = dQ.reshape(-1, dQ.shape[-1])
@@ -547,32 +802,45 @@ class LoRA_QKV(torch.autograd.Function):
k_dB = dK @ KB.t()
v_dB = dV @ VB.t()
+ Q_mask, K_mask, V_mask = (
+ None if mask is None else mask.view(-1, hd) for mask in (Q_mask, K_mask, V_mask)
+ )
+
# d_QA = X.t() @ (dQ @ QB.t()), d_QB = (QA.t() @ X.t()) @ dQ, both scaled by QS; K and V below are
# identical with their own scales.
- d_QA.addmm_(X.t(), q_dB, alpha = QS, beta = 0)
- d_QB.addmm_(QA.t() @ X.t(), dQ, alpha = QS, beta = 0)
+ Xd = X if Q_mask is None else _dropout_replay(X, Q_mask, Q_p)
+ d_QA.addmm_(Xd.t(), q_dB, alpha = QS, beta = 0)
+ d_QB.addmm_(QA.t() @ Xd.t(), dQ, alpha = QS, beta = 0)
+
+ Xd = X if K_mask is None else _dropout_replay(X, K_mask, K_p)
+ d_KA.addmm_(Xd.t(), k_dB, alpha = KS, beta = 0)
+ d_KB.addmm_(KA.t() @ Xd.t(), dK, alpha = KS, beta = 0)
- d_KA.addmm_(X.t(), k_dB, alpha = KS, beta = 0)
- d_KB.addmm_(KA.t() @ X.t(), dK, alpha = KS, beta = 0)
+ Xd = X if V_mask is None else _dropout_replay(X, V_mask, V_p)
+ d_VA.addmm_(Xd.t(), v_dB, alpha = VS, beta = 0)
+ d_VB.addmm_(VA.t() @ Xd.t(), dV, alpha = VS, beta = 0)
+ del Xd
- d_VA.addmm_(X.t(), v_dB, alpha = VS, beta = 0)
- d_VB.addmm_(VA.t() @ X.t(), dV, alpha = VS, beta = 0)
+ Q_has_bias, K_has_bias, V_has_bias = ctx.biases
+ d_QBias = _lora_bias_grad(dQ, QS) if Q_has_bias else None
+ d_KBias = _lora_bias_grad(dK, KS) if K_has_bias else None
+ d_VBias = _lora_bias_grad(dV, VS) if V_has_bias else None
# Combine the per-projection derivatives into dX.
QW = fast_dequantize(QW.t(), QW_quant)
dX = torch.matmul(dQ, QW.t(), out = X if ctx.inplace and not _is_compiling() else None)
del QW
- addmm_(dX, q_dB, QA.t(), alpha = QS)
+ _add_lora_dX(dX, q_dB, QA, QS, Q_mask, Q_p)
KW = fast_dequantize(KW.t(), KW_quant)
addmm_(dX, dK, KW.t())
del KW
- addmm_(dX, k_dB, KA.t(), alpha = KS)
+ _add_lora_dX(dX, k_dB, KA, KS, K_mask, K_p)
VW = fast_dequantize(VW.t(), VW_quant)
addmm_(dX, dV, VW.t())
del VW
- addmm_(dX, v_dB, VA.t(), alpha = VS)
+ _add_lora_dX(dX, v_dB, VA, VS, V_mask, V_p)
return (
dX.view(batch, seq_len, hd),
@@ -592,6 +860,10 @@ class LoRA_QKV(torch.autograd.Function):
d_VB.t(),
None,
None,
+ None,
+ d_QBias,
+ d_KBias,
+ d_VBias,
)
@@ -600,14 +872,18 @@ def apply_lora_qkv(
X,
inplace = True,
):
- if any(
+ projs = (self.q_proj, self.k_proj, self.v_proj)
+ extras = None
+ if not any(
_has_multiple_active_adapters(proj)
or _has_active_dora_adapter(proj)
- or _has_active_lora_bias(proj)
or _has_activation_fake_quantizer(proj)
- for proj in (self.q_proj, self.k_proj, self.v_proj)
+ for proj in projs
):
+ extras = _lora_dropout_and_bias(projs, X)
+ if extras is None:
return self.q_proj(X), self.k_proj(X), self.v_proj(X)
+ dropout_p, biases = extras
QW, QW_quant, QA, QB, QS = get_lora_parameters(self.q_proj)
KW, KW_quant, KA, KB, KS = get_lora_parameters(self.k_proj)
VW, VW_quant, VA, VB, VS = get_lora_parameters(self.v_proj)
@@ -630,6 +906,8 @@ def apply_lora_qkv(
VB,
VS,
inplace,
+ dropout_p,
+ *biases,
)
return Q, K, V
@@ -664,26 +942,46 @@ class LoRA_W(torch.autograd.Function):
@staticmethod
@torch_amp_custom_fwd
- def forward(ctx, X: torch.Tensor, W, W_quant, A, B, S):
+ def forward(
+ ctx,
+ X: torch.Tensor,
+ W,
+ W_quant,
+ A,
+ B,
+ S,
+ dropout_p = None,
+ bias = None,
+ ):
dtype = X.dtype
- XW = matmul_lora(X, W, W_quant, A, B, S)
+ if dropout_p is None and bias is None:
+ XW = matmul_lora(X, W, W_quant, A, B, S)
+ mask = None
+ else:
+ Xd, mask = _dropout_forward(X, dropout_p or 0.0)
+ XW = _matmul_lora_ext(X, Xd, W, W_quant, A, B, S, bias)
+ del Xd
ctx.custom_saved_tensors = (
W,
W_quant,
S,
)
- ctx.save_for_backward(A, B, X)
+ ctx.save_for_backward(A, B, X, mask)
+ ctx.dropout_p = dropout_p
+ ctx.has_bias = bias is not None
return XW
@staticmethod
@torch_amp_custom_bwd
def backward(ctx, dY: torch.Tensor):
W, W_quant, S = ctx.custom_saved_tensors
- A, B, X = ctx.saved_tensors
+ A, B, X, mask = ctx.saved_tensors
batch, seq_len, hd = X.shape
dY = dY.reshape(-1, dY.shape[-1]) # Must be reshape
X = X.reshape(-1, X.shape[-1]) # Must be reshape
+ if mask is not None:
+ mask = mask.reshape(-1, hd)
dtype = X.dtype
A, B = A.to(dtype), B.to(dtype)
@@ -696,28 +994,35 @@ class LoRA_W(torch.autograd.Function):
y_dB = dY @ B.t()
# d_A = X.t() @ (dY @ B.t()), d_B = (A.t() @ X.t()) @ dY, both scaled by S.
- d_A.addmm_(X.t(), y_dB, alpha = S, beta = 0)
- d_B.addmm_(A.t() @ X.t(), dY, alpha = S, beta = 0)
+ Xd = X if mask is None else _dropout_replay(X, mask, ctx.dropout_p)
+ d_A.addmm_(Xd.t(), y_dB, alpha = S, beta = 0)
+ d_B.addmm_(A.t() @ Xd.t(), dY, alpha = S, beta = 0)
+ del Xd
+ d_bias = _lora_bias_grad(dY, S) if ctx.has_bias else None
W = fast_dequantize(W.t(), W_quant)
dX = dY @ W.t()
del W
- addmm_(dX, y_dB, A.t(), alpha = S)
+ _add_lora_dX(dX, y_dB, A, S, mask, ctx.dropout_p)
- return dX.view(batch, seq_len, hd), None, None, d_A.t(), d_B.t(), None
+ return dX.view(batch, seq_len, hd), None, None, d_A.t(), d_B.t(), None, None, d_bias
def apply_lora_o(self, X):
- if (
+ extras = None
+ if not (
_has_multiple_active_adapters(self.o_proj)
or _has_active_dora_adapter(self.o_proj)
- or _has_active_lora_bias(self.o_proj)
or _has_activation_fake_quantizer(self.o_proj)
):
+ extras = _lora_dropout_and_bias((self.o_proj,), X)
+ if extras is None:
return self.o_proj(X)
+ dropout_p, (bias,) = extras
OW, OW_quant, OA, OB, OS = get_lora_parameters(self.o_proj)
- O = _apply(LoRA_W, X, OW, OW_quant, OA, OB, OS)
- return O
+ if dropout_p is None and bias is None:
+ return _apply(LoRA_W, X, OW, OW_quant, OA, OB, OS)
+ return _apply(LoRA_W, X, OW, OW_quant, OA, OB, OS, dropout_p and dropout_p[0], bias)
IDENTITY_DROPOUT = torch.nn.Identity
diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py
index 4d7a486cf00..2d20604c749 100644
--- a/unsloth/models/_utils.py
+++ b/unsloth/models/_utils.py
@@ -7354,7 +7354,9 @@ def validate_loftq_config(
signature = str(inspect.signature(LoraConfig))
SUPPORTS_LOFTQ = "loftq_config" in signature
- if lora_dropout != 0:
+ from ..kernels.fast_lora import FUSED_LORA_DROPOUT
+
+ if lora_dropout != 0 and not FUSED_LORA_DROPOUT:
logger.warning_once(
f"Unsloth: Dropout = 0 is supported for fast patching. You are using dropout = {lora_dropout}.\n"
f"Unsloth will patch all other layers, except LoRA matrices, causing a performance hit."
diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py
index ccca75cfd83..54078768963 100644
--- a/unsloth/models/llama.py
+++ b/unsloth/models/llama.py
@@ -146,7 +146,8 @@ from unsloth.models._attn_mask_compat import (
)
from transformers.cache_utils import DynamicCache, Cache
from ..kernels import *
-from ..kernels.utils import _has_active_lora_bias, has_mxfp4_base
+from ..kernels.utils import has_mxfp4_base
+from ..kernels import fast_lora as _fast_lora
from ..kernels.bnb_override import install_bnb_nf4_override as _install_bnb_nf4_override
from ..tokenizer_utils import *
from ..context_parallel import get_cp_manager
@@ -2742,8 +2743,8 @@ def _fused_lora_skip_reason(
conditions mirror the fused-kernel gate in patch_peft_model.
"""
reasons = []
- if lora_dropout != 0:
- reasons.append(f"lora_dropout = {lora_dropout}")
+ if lora_dropout != 0 and not _fast_lora.FUSED_LORA_DROPOUT:
+ reasons.append(f"lora_dropout = {lora_dropout} with UNSLOTH_FUSED_LORA_DROPOUT=0")
if bias != "none":
reasons.append(f"bias = '{bias}'")
if float32_base:
@@ -4306,7 +4307,7 @@ class FastLlamaModel:
SUPPORTS_LOFTQ = "loftq_config" in signature
SUPPORTS_RSLORA = "use_rslora" in signature
- if lora_dropout != 0:
+ if lora_dropout != 0 and not _fast_lora.FUSED_LORA_DROPOUT:
logger.warning_once(
f"Unsloth: Dropout = 0 is supported for fast patching. You are using dropout = {lora_dropout}.\n"
f"Unsloth will patch all other layers, except LoRA matrices, causing a performance hit."
@@ -4819,7 +4820,7 @@ class FastLlamaModel:
fused_lora_declined_for_fsdp = fsdp_will_wrap() and not fsdp2_keeps_fused_lora()
if (
- lora_dropout == 0
+ (lora_dropout == 0 or _fast_lora.FUSED_LORA_DROPOUT)
and bias == "none"
and not float32_base
and not fused_lora_declined_for_fsdp
@@ -4845,9 +4846,9 @@ class FastLlamaModel:
and (len(getattr(down_proj, "lora_magnitude_vector", []) or []) == 0)
and not has_mxfp4_base(gate_proj, up_proj, down_proj)
and not _has_packed_base(gate_proj, up_proj, down_proj)
- and not _has_active_lora_bias(gate_proj)
- and not _has_active_lora_bias(up_proj)
- and not _has_active_lora_bias(down_proj)
+ and not _fast_lora.lora_bias_blocks_fused(gate_proj)
+ and not _fast_lora.lora_bias_blocks_fused(up_proj)
+ and not _fast_lora.lora_bias_blocks_fused(down_proj)
):
# MethodType binds the replacement; see stackoverflow.com/questions/50599045.
if hasattr(mlp_module, "_unsloth_forward"):
@@ -4879,9 +4880,9 @@ class FastLlamaModel:
and (len(getattr(v_proj, "lora_magnitude_vector", []) or []) == 0)
and not has_mxfp4_base(q_proj, k_proj, v_proj)
and not _has_packed_base(q_proj, k_proj, v_proj)
- and not _has_active_lora_bias(q_proj)
- and not _has_active_lora_bias(k_proj)
- and not _has_active_lora_bias(v_proj)
+ and not _fast_lora.lora_bias_blocks_fused(q_proj)
+ and not _fast_lora.lora_bias_blocks_fused(k_proj)
+ and not _fast_lora.lora_bias_blocks_fused(v_proj)
):
layer.self_attn.apply_qkv = apply_lora_qkv
n_qkv += 1
@@ -4901,7 +4902,7 @@ class FastLlamaModel:
and (len(getattr(o_proj, "lora_magnitude_vector", []) or []) == 0)
and not has_mxfp4_base(o_proj)
and not _has_packed_base(o_proj)
- and not _has_active_lora_bias(o_proj)
+ and not _fast_lora.lora_bias_blocks_fused(o_proj)
):
layer.self_attn.apply_o = apply_lora_o
n_o += 1