Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Fused LoRA: fall back to PEFT for variants, disabled or merged adapters, trainable bases and autocast dtype mismatches (#14806)

Daniel Hane228cbc45052
diff --git a/tests/test_fused_lora_peft_fallbacks.py b/tests/test_fused_lora_peft_fallbacks.py
new file mode 100644
index 00000000000..a4e4caff307
--- /dev/null
+++ b/tests/test_fused_lora_peft_fallbacks.py
@@ -0,0 +1,311 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""Configurations the fused LoRA kernels cannot run hand the projection back to PEFT's forward.
+
+The fused Functions compute base(X) + S * lora_B(lora_A(dropout(X))) [+ bias] with frozen base weights in
+X's dtype. A PEFT LoRA variant (its own forward), a disabled or merged adapter (base layer only: the
+fused backward has no LoRA weights to use), a trainable base weight (its gradient) and an autocast dtype
+other than X's (the in-place LoRA accumulation is not autocast) used to run fused anyway, training plain
+LoRA, dropping gradients or crashing. Each is checked against PEFT run in float64 (CPU) / float32 (CUDA).
+A disabled or merged adapter with no gradient needed (the DPO / GRPO reference forward) stays fused.
+"""
+
+from __future__ import annotations
+
+import os
+import sys
+
+import pytest
+import torch
+import unsloth # noqa: F401
+
+from real_accelerator import has_real_cuda
+from unsloth.kernels import fast_lora
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from test_fused_lora_dropout_bias import H, PATHS, TARGETS, _Block, _peft_block, _Spy # noqa: E402
+
+CONFIGS = [(0.0, False), (0.3, False), (0.0, True), (0.3, True)]
+
+
+def _paths(device):
+ # The SwiGLU kernels are Triton.
+ return ["o", "qkv"] + (["mlp"] if device == "cuda" else [])
+
+
+CASES = [("cpu", torch.float64, path) for path in _paths("cpu")]
+if has_real_cuda():
+ CASES += [("cuda", torch.float32, path) for path in _paths("cuda")]
+
+
+def _run(fn, model, X0, autocast_dtype = None):
+ model.zero_grad(set_to_none = True)
+ torch.manual_seed(7)
+ X = X0.detach().clone().requires_grad_()
+ if autocast_dtype is None:
+ out = fn(X * 1)
+ else:
+ with torch.autocast(X0.device.type, dtype = autocast_dtype):
+ out = fn(X * 1)
+ outs = out if isinstance(out, tuple) else (out,)
+ gen = torch.Generator(device = "cpu").manual_seed(11)
+ loss = sum((o.double() * torch.randn(o.shape, generator = gen, dtype = torch.float64).to(o.device)).sum() for o in outs)
+ loss.backward()
+ grads = {n: p.grad.clone() for n, p in model.named_parameters() if p.grad is not None}
+ model.zero_grad(set_to_none = True)
+ return [o.detach() for o in outs], X.grad.clone(), grads
+
+
+def _assert_falls_back_to_peft(monkeypatch, model, block, path, X0, autocast_dtype = None, exact = True):
+ spy = _Spy(monkeypatch)
+ fused = _run(PATHS[path][0](block), model, X0, autocast_dtype)
+ assert spy.calls == [], "a fused LoRA Function ran"
+ ref = _run(PATHS[path][1](block), model, X0, autocast_dtype)
+ tol = dict(rtol = 0, atol = 0) if exact else {}
+ for a, b in zip(fused[0], ref[0]):
+ torch.testing.assert_close(a, b, **tol)
+ torch.testing.assert_close(fused[1], ref[1], **tol)
+ assert fused[2].keys() == ref[2].keys()
+ for name in ref[2]:
+ torch.testing.assert_close(fused[2][name], ref[2][name], **tol)
+ return ref
+
+
+class _DoubledLoRA:
+ # Stand-in variant: PEFT dispatches on `active_adapter in self.lora_variant`, then calls forward.
+ @staticmethod
+ def forward(module, active_adapter, x, result, **kwargs):
+ lora_A, lora_B = module.lora_A[active_adapter], module.lora_B[active_adapter]
+ dropout, scaling = module.lora_dropout[active_adapter], module.scaling[active_adapter]
+ return result + 2.0 * lora_B(lora_A(dropout(x))) * scaling
+
+
[email protected]("device,dtype,path", CASES)
[email protected]("dropout,lora_bias", CONFIGS)
+def test_lora_variant_runs_peft_forward(monkeypatch, device, dtype, path, dropout, lora_bias):
+ model, block = _peft_block(device, dtype, dropout, lora_bias)
+ for name in TARGETS:
+ block.get_submodule(name).lora_variant["default"] = _DoubledLoRA()
+ X0 = torch.randn(2, 5, H, device = device, dtype = dtype)
+ _assert_falls_back_to_peft(monkeypatch, model, block, path, X0)
+
+
[email protected]("dropout", [0.0, 0.3])
+def test_real_peft_variant_runs_peft_forward(monkeypatch, dropout):
+ peft_config = pytest.importorskip("peft.tuners.lora.config")
+ if not hasattr(peft_config, "KasaConfig"):
+ pytest.skip("PEFT without KaSA")
+ from peft import LoraConfig, get_peft_model
+
+
+ torch.manual_seed(3407)
+ cfg = LoraConfig(
+ r = 4,
+ lora_alpha = 8,
+ lora_dropout = dropout,
+ target_modules = TARGETS,
+ kasa_config = peft_config.KasaConfig(),
+ )
+ model = get_peft_model(_Block(), cfg).double()
+ block = model.base_model.model
+ assert "default" in block.o_proj.lora_variant
+ with torch.no_grad():
+ for name in TARGETS:
+ block.get_submodule(name).lora_B["default"].weight.normal_()
+ model.train()
+ X0 = torch.randn(2, 5, H, dtype = torch.float64)
+ for path in _paths("cpu"):
+ _assert_falls_back_to_peft(monkeypatch, model, block, path, X0)
+
+
[email protected]("device,dtype,path", CASES)
[email protected]("state", ["disabled", "merged"])
[email protected]("dropout,lora_bias", CONFIGS)
+def test_disabled_or_merged_adapter_backward_matches_peft(monkeypatch, device, dtype, path, state, dropout, lora_bias):
+ if state == "merged" and lora_bias:
+ pytest.skip("PEFT refuses to merge a lora_B bias into a base layer without one")
+ model, block = _peft_block(device, dtype, dropout, lora_bias)
+ if state == "disabled":
+ model.base_model.disable_adapter_layers()
+ else:
+ for name in TARGETS:
+ block.get_submodule(name).merge()
+ X0 = torch.randn(2, 5, H, device = device, dtype = dtype)
+ _assert_falls_back_to_peft(monkeypatch, model, block, path, X0)
+
+
[email protected]("device,dtype,path", CASES)
[email protected]("dropout,lora_bias", CONFIGS)
+def test_trainable_base_weight_gets_its_gradient(monkeypatch, device, dtype, path, dropout, lora_bias):
+ model, block = _peft_block(device, dtype, dropout, lora_bias)
+ for p in model.parameters():
+ p.requires_grad_(True)
+ X0 = torch.randn(2, 5, H, device = device, dtype = dtype)
+ ref = _assert_falls_back_to_peft(monkeypatch, model, block, path, X0)
+ assert any("base_layer.weight" in name for name in ref[2])
+
+
[email protected](not has_real_cuda(), reason = "CUDA autocast")
[email protected]("path", _paths("cuda"))
[email protected]("dropout,lora_bias", CONFIGS)
+ "x_dtype,autocast_dtype",
+ [(torch.float16, torch.bfloat16), (torch.bfloat16, torch.float16), (torch.float32, torch.float16)],
+)
+def test_autocast_dtype_other_than_input_runs_peft(monkeypatch, path, dropout, lora_bias, x_dtype, autocast_dtype):
+ model, block = _peft_block("cuda", torch.bfloat16, dropout, lora_bias)
+ X0 = torch.randn(2, 5, H, device = "cuda", dtype = x_dtype)
+ _assert_falls_back_to_peft(monkeypatch, model, block, path, X0, autocast_dtype)
+
+
[email protected]("device", ["cpu"] + (["cuda"] if has_real_cuda() else []))
[email protected]("dropout,lora_bias", CONFIGS)
+def test_eligible_configs_still_fuse(monkeypatch, device, dropout, lora_bias):
+ # Frozen base, one enabled adapter, no variant: fused, including under autocast in X's own dtype.
+ dtype = torch.bfloat16
+ model, block = _peft_block(device, dtype, dropout, lora_bias)
+ X = torch.randn(2, 5, H, device = device, dtype = dtype)
+ spy = _Spy(monkeypatch)
+ fast_lora.apply_lora_o(block, X)
+ fast_lora.apply_lora_qkv(block, X)
+ with torch.autocast(device, dtype = dtype):
+ fast_lora.apply_lora_o(block, X)
+ fast_lora.apply_lora_qkv(block, X)
+ assert [name for name, _ in spy.calls] == ["LoRA_W", "LoRA_QKV"] * 2
+ if dropout == 0 and not lora_bias:
+ # The default config keeps its pre-existing six-argument call.
+ assert len(spy.calls[0][1]) == 6
+
+
+FUSED_NAMES = {"o": ["LoRA_W"], "qkv": ["LoRA_QKV"], "mlp": ["LoRA_MLP"]}
+
+
+def _disable_or_merge(model, block, state):
+ if state == "disabled":
+ model.base_model.disable_adapter_layers()
+ else:
+ for name in TARGETS:
+ block.get_submodule(name).merge()
+
+
[email protected]("device,dtype,path", CASES)
[email protected]("state", ["disabled", "merged"])
[email protected]("grad_mode", ["no_grad", "inference_mode", "grad_frozen_input"])
[email protected]("dropout", [0.0, 0.3])
+def test_disabled_or_merged_adapter_without_grad_stays_fused(monkeypatch, device, dtype, path, state, grad_mode, dropout):
+ # The reference forward: no gradient flows, so the fused base matmul runs and equals PEFT bitwise.
+ model, block = _peft_block(device, dtype, dropout)
+ _disable_or_merge(model, block, state)
+ X = torch.randn(2, 5, H, device = device, dtype = dtype)
+ spy = _Spy(monkeypatch)
+ ctx = {"no_grad": torch.no_grad, "inference_mode": torch.inference_mode}.get(grad_mode, torch.enable_grad)
+ with ctx():
+ fused = PATHS[path][0](block)(X)
+ assert [name for name, _ in spy.calls] == FUSED_NAMES[path]
+ args = spy.calls[0][1]
+ if path == "o":
+ # The default config's six-argument call.
+ assert len(args) == 6
+ else:
+ # No LoRA weights, no dropout masks, no lora_B biases: the base matmul only.
+ assert args[3] is None and args[-4] is None and all(b is None for b in args[-3:])
+ with ctx():
+ ref = PATHS[path][1](block)(X)
+ # Each projection is PEFT's base_layer(X) bitwise; the MLP's Triton SwiGLU rounds its float32
+ # activation 1 ulp differently from torch's SiLU * up.
+ tol = {} if path == "mlp" else dict(rtol = 0, atol = 0)
+ for a, b in zip(fused if isinstance(fused, tuple) else (fused,), ref if isinstance(ref, tuple) else (ref,)):
+ assert not a.requires_grad
+ torch.testing.assert_close(a, b, **tol)
+
+
[email protected](not has_real_cuda(), reason = "bitsandbytes 4-bit needs CUDA")
[email protected]("path", _paths("cuda"))
+def test_disabled_adapter_without_grad_on_4bit_base_stays_fused(monkeypatch, path):
+ bnb = pytest.importorskip("bitsandbytes")
+ from peft import LoraConfig, get_peft_model
+
+ class _Block4bit(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ for name, (i, o) in {
+ "q_proj": (H, H), "k_proj": (H, H), "v_proj": (H, H), "o_proj": (H, H),
+ "gate_proj": (H, 96), "up_proj": (H, 96), "down_proj": (96, H),
+ }.items():
+ layer = bnb.nn.Linear4bit(i, o, bias = False, compute_dtype = torch.bfloat16, quant_type = "nf4")
+ layer.weight = bnb.nn.Params4bit(
+ layer.weight.data.to(torch.bfloat16), requires_grad = False, quant_type = "nf4"
+ )
+ setattr(self, name, layer)
+ self.act_fn = torch.nn.SiLU()
+
+ torch.manual_seed(3407)
+ base = _Block4bit().to("cuda")
+ cfg = LoraConfig(r = 4, lora_alpha = 8, lora_dropout = 0.1, target_modules = TARGETS, init_lora_weights = False)
+ model = get_peft_model(base, cfg)
+ for name, p in model.named_parameters():
+ if "lora_" in name:
+ p.data = p.data.to(torch.bfloat16)
+ block = model.base_model.model
+ assert getattr(block.o_proj.base_layer.weight, "quant_state", None) is not None
+ model.train()
+ model.base_model.disable_adapter_layers()
+ X = torch.randn(2, 7, H, device = "cuda", dtype = torch.bfloat16)
+ spy = _Spy(monkeypatch)
+ with torch.no_grad():
+ fused = PATHS[path][0](block)(X)
+ ref = PATHS[path][1](block)(X)
+ assert [name for name, _ in spy.calls] == FUSED_NAMES[path]
+ for a, b in zip(fused if isinstance(fused, tuple) else (fused,), ref if isinstance(ref, tuple) else (ref,)):
+ torch.testing.assert_close(a, b, rtol = 0, atol = 0)
+
+
+def test_patch_summary_names_a_lora_variant():
+ from unsloth.models.llama import _lora_variant_note
+
+ model, block = _peft_block("cpu", torch.float32, 0.0)
+ assert _lora_variant_note(model, "default") == ""
+ block.o_proj.lora_variant["default"] = _DoubledLoRA()
+ note = _lora_variant_note(model, "default")
+ assert "_DoubledLoRA" in note and "PEFT's own forward" in note
+
+
+def test_routing_compiles_without_graph_breaks(monkeypatch):
+ # The added routing checks are Python attribute reads and grad / autocast mode queries: fullgraph traces
+ # the fused, variant, trainable base, autocast and disabled (with and without grad) routes.
+ if not fast_lora.TRACE_LORA_FUNCTIONS:
+ pytest.skip("fused Functions run Dynamo-disabled before torch 2.11")
+ import torch._dynamo
+
+ model, block = _peft_block("cpu", torch.float32, 0.3, True)
+ X = torch.randn(2, 5, H)
+
+ def fn(X):
+ return fast_lora.apply_lora_o(block, X), fast_lora.apply_lora_qkv(block, X)
+
+ def check(X, autocast_dtype = None, grad = True):
+ torch._dynamo.reset()
+ compiled = torch.compile(fn, backend = "eager", fullgraph = True)
+ with torch.set_grad_enabled(grad), torch.autocast("cpu", dtype = autocast_dtype or torch.bfloat16, enabled = autocast_dtype is not None):
+ got = compiled(X)
+ want = fn(X)
+ torch.testing.assert_close(got[0], want[0])
+ for a, b in zip(got[1], want[1]):
+ torch.testing.assert_close(a, b)
+
+ model.eval() # deterministic dropout, so compiled and eager agree
+ check(X)
+ check(X, torch.bfloat16) # autocast in another dtype: PEFT
+ check(X.requires_grad_(), grad = True)
+ for name in TARGETS:
+ block.get_submodule(name).lora_variant["default"] = _DoubledLoRA()
+ check(X)
+ for name in TARGETS:
+ del block.get_submodule(name).lora_variant["default"]
+ block.o_proj.base_layer.weight.requires_grad_(True)
+ check(X)
+ block.o_proj.base_layer.weight.requires_grad_(False)
+ model.base_model.disable_adapter_layers()
+ check(X.detach(), grad = False) # fused
+ check(X.detach().requires_grad_(), grad = True) # PEFT
diff --git a/unsloth/kernels/fast_lora.py b/unsloth/kernels/fast_lora.py
index 6fe2185b457..661604a4e9f 100644
--- a/unsloth/kernels/fast_lora.py
+++ b/unsloth/kernels/fast_lora.py
@@ -129,6 +129,35 @@ def lora_bias_blocks_fused(proj):
return not FUSED_LORA_BIAS and _has_active_lora_bias(proj)
+def _needs_peft_forward(proj, adapter):
+ # A PEFT LoRA variant (DoRA, MoNTeCLoRA, VeLoRA, BD-LoRA, ...) replaces the LoRA branch in PEFT's
+ # forward, and a trainable base weight needs its gradient: the fused Functions compute neither.
+ variants = proj.__dict__.get("lora_variant")
+ if variants and adapter in variants:
+ return True
+ return _base_weight_requires_grad(proj)
+
+
+def _base_weight_requires_grad(proj):
+ base = proj._modules.get("base_layer")
+ W = None if base is None else base._parameters.get("weight")
+ return W is not None and W.requires_grad
+
+
+_is_any_autocast_enabled = getattr(torch._C, "_is_any_autocast_enabled", None)
+
+
+def _autocast_dtype_mismatch(X):
+ # Under autocast the fused matmuls return the autocast dtype but the in-place LoRA accumulation runs
+ # in X's dtype (in-place ops are not autocast), so a different autocast dtype cannot be fused.
+ # Mode flags only, no tensor work. Eager first asks whether any autocast is on (a private query, so not
+ # under Dynamo, which traces the public calls below as constants).
+ if _is_any_autocast_enabled is not None and not _is_compiling() and not _is_any_autocast_enabled():
+ return False
+ device_type = "cuda" if X.is_cuda else X.device.type
+ return torch.is_autocast_enabled(device_type) and torch.get_autocast_dtype(device_type) != X.dtype
+
+
def _is_plain_lora(proj):
# One enabled, unmerged adapter with no dropout module and no lora_B bias (the default LoRA config).
# Reads PEFT's instance state directly: getattr through nn.Module / properties costs more than the
@@ -141,7 +170,14 @@ def _is_plain_lora(proj):
adapter = adapter[0]
if state.get("_disable_adapters", True) or state.get("merged_adapters"):
return False
+ variants = state.get("lora_variant")
+ if variants and adapter in variants:
+ return False
modules = proj._modules
+ base = modules.get("base_layer")
+ W = None if base is None else base._parameters.get("weight")
+ if W is not None and W.requires_grad:
+ return False
dropout = modules.get("lora_dropout")
lora_B = modules.get("lora_B")
if dropout is None or lora_B is None:
@@ -156,6 +192,8 @@ def _is_plain_lora(proj):
def _lora_dropout_and_bias(projs, X):
"""(dropout probabilities or None, lora_B biases) for the fused kernels, or None to use PEFT."""
+ if _autocast_dtype_mismatch(X):
+ return None
for proj in projs:
if not _is_plain_lora(proj):
break
@@ -166,6 +204,7 @@ def _lora_dropout_and_bias(projs, X):
# module or lora_B bias keeps PEFT's traceable forward, as before they were fused, not a graph break.
peft_when_unplain = not TRACE_LORA_FUNCTIONS and _is_compiling()
has_dropout_module = False
+ grad_enabled = None
for proj in projs:
if peft_when_unplain and not _is_plain_lora(proj):
return None
@@ -174,9 +213,18 @@ def _lora_dropout_and_bias(projs, X):
# Enabled adapter missing from this projection: PEFT runs the base layer alone.
if not getattr(proj, "disable_adapters", True) and not getattr(proj, "merged", False):
return None
+ # Disabled or merged: the fused Functions run the base matmul alone (the no-grad reference
+ # forward of DPO / GRPO), but their backward needs LoRA weights, so PEFT runs it under grad.
+ if grad_enabled is None:
+ grad_enabled = torch.is_grad_enabled()
+ if grad_enabled and (X.requires_grad or _base_weight_requires_grad(proj)):
+ return None
dropout_p.append(0.0)
biases.append(None)
continue
+ if _needs_peft_forward(proj, adapter):
+ # A LoRA variant, or a trainable base weight.
+ return None
dropout = getattr(proj, "lora_dropout", None)
has_dropout_module |= (
dropout is not None
diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py
index e4225421596..f46f1b1cc44 100644
--- a/unsloth/models/llama.py
+++ b/unsloth/models/llama.py
@@ -2894,6 +2894,19 @@ def _vllm_will_load_weights(fast_inference, num_labels = None):
return True
+def _lora_variant_note(model, adapter) -> str:
+ """Summary note when the adapter is a PEFT LoRA variant: its layers run PEFT's forward at runtime."""
+ for module in model.modules():
+ variants = module.__dict__.get("lora_variant")
+ if variants and adapter in variants:
+ name = type(variants[adapter]).__name__
+ return (
+ f" The adapter is a PEFT LoRA variant ({name}), so its layers run PEFT's own forward "
+ "instead of the fused LoRA kernels."
+ )
+ return ""
+
+
def _fused_lora_skip_reason(
lora_dropout,
bias,
@@ -5215,7 +5228,7 @@ class FastLlamaModel:
unfused_reason = _fused_lora_skip_reason(
lora_dropout, bias, float32_base, fsdp = fused_lora_declined_for_fsdp
- )
+ ) + _lora_variant_note(model, active_adapter)
logger.warning_once(
f"Unsloth {__version__} patched {len(model.model.model.layers)} layers with "
f"{n_qkv} QKV layers, {n_o} O layers and {n_mlp} MLP layers.{unfused_reason}",