Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Fused LoRA: fall back to PEFT for variants, disabled or merged adapters, trainable bases and autocast dtype mismatches (#14806)
Daniel Hane228cbc45052
diff --git a/tests/test_fused_lora_peft_fallbacks.py b/tests/test_fused_lora_peft_fallbacks.pynew file mode 100644index 00000000000..a4e4caff307--- /dev/null+++ b/tests/test_fused_lora_peft_fallbacks.py@@ -0,0 +1,311 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""Configurations the fused LoRA kernels cannot run hand the projection back to PEFT's forward.++The fused Functions compute base(X) + S * lora_B(lora_A(dropout(X))) [+ bias] with frozen base weights in+X's dtype. A PEFT LoRA variant (its own forward), a disabled or merged adapter (base layer only: the+fused backward has no LoRA weights to use), a trainable base weight (its gradient) and an autocast dtype+other than X's (the in-place LoRA accumulation is not autocast) used to run fused anyway, training plain+LoRA, dropping gradients or crashing. Each is checked against PEFT run in float64 (CPU) / float32 (CUDA).+A disabled or merged adapter with no gradient needed (the DPO / GRPO reference forward) stays fused.+"""++from __future__ import annotations++import os+import sys++import pytest+import torch+import unsloth # noqa: F401++from real_accelerator import has_real_cuda+from unsloth.kernels import fast_lora++sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))+from test_fused_lora_dropout_bias import H, PATHS, TARGETS, _Block, _peft_block, _Spy # noqa: E402++CONFIGS = [(0.0, False), (0.3, False), (0.0, True), (0.3, True)]+++def _paths(device):+ # The SwiGLU kernels are Triton.+ return ["o", "qkv"] + (["mlp"] if device == "cuda" else [])+++CASES = [("cpu", torch.float64, path) for path in _paths("cpu")]+if has_real_cuda():+ CASES += [("cuda", torch.float32, path) for path in _paths("cuda")]+++def _run(fn, model, X0, autocast_dtype = None):+ model.zero_grad(set_to_none = True)+ torch.manual_seed(7)+ X = X0.detach().clone().requires_grad_()+ if autocast_dtype is None:+ out = fn(X * 1)+ else:+ with torch.autocast(X0.device.type, dtype = autocast_dtype):+ out = fn(X * 1)+ outs = out if isinstance(out, tuple) else (out,)+ gen = torch.Generator(device = "cpu").manual_seed(11)+ loss = sum((o.double() * torch.randn(o.shape, generator = gen, dtype = torch.float64).to(o.device)).sum() for o in outs)+ loss.backward()+ grads = {n: p.grad.clone() for n, p in model.named_parameters() if p.grad is not None}+ model.zero_grad(set_to_none = True)+ return [o.detach() for o in outs], X.grad.clone(), grads+++def _assert_falls_back_to_peft(monkeypatch, model, block, path, X0, autocast_dtype = None, exact = True):+ spy = _Spy(monkeypatch)+ fused = _run(PATHS[path][0](block), model, X0, autocast_dtype)+ assert spy.calls == [], "a fused LoRA Function ran"+ ref = _run(PATHS[path][1](block), model, X0, autocast_dtype)+ tol = dict(rtol = 0, atol = 0) if exact else {}+ for a, b in zip(fused[0], ref[0]):+ torch.testing.assert_close(a, b, **tol)+ torch.testing.assert_close(fused[1], ref[1], **tol)+ assert fused[2].keys() == ref[2].keys()+ for name in ref[2]:+ torch.testing.assert_close(fused[2][name], ref[2][name], **tol)+ return ref+++class _DoubledLoRA:+ # Stand-in variant: PEFT dispatches on `active_adapter in self.lora_variant`, then calls forward.+ @staticmethod+ def forward(module, active_adapter, x, result, **kwargs):+ lora_A, lora_B = module.lora_A[active_adapter], module.lora_B[active_adapter]+ dropout, scaling = module.lora_dropout[active_adapter], module.scaling[active_adapter]+ return result + 2.0 * lora_B(lora_A(dropout(x))) * scaling++[email protected]("device,dtype,path", CASES)[email protected]("dropout,lora_bias", CONFIGS)+def test_lora_variant_runs_peft_forward(monkeypatch, device, dtype, path, dropout, lora_bias):+ model, block = _peft_block(device, dtype, dropout, lora_bias)+ for name in TARGETS:+ block.get_submodule(name).lora_variant["default"] = _DoubledLoRA()+ X0 = torch.randn(2, 5, H, device = device, dtype = dtype)+ _assert_falls_back_to_peft(monkeypatch, model, block, path, X0)++[email protected]("dropout", [0.0, 0.3])+def test_real_peft_variant_runs_peft_forward(monkeypatch, dropout):+ peft_config = pytest.importorskip("peft.tuners.lora.config")+ if not hasattr(peft_config, "KasaConfig"):+ pytest.skip("PEFT without KaSA")+ from peft import LoraConfig, get_peft_model+++ torch.manual_seed(3407)+ cfg = LoraConfig(+ r = 4,+ lora_alpha = 8,+ lora_dropout = dropout,+ target_modules = TARGETS,+ kasa_config = peft_config.KasaConfig(),+ )+ model = get_peft_model(_Block(), cfg).double()+ block = model.base_model.model+ assert "default" in block.o_proj.lora_variant+ with torch.no_grad():+ for name in TARGETS:+ block.get_submodule(name).lora_B["default"].weight.normal_()+ model.train()+ X0 = torch.randn(2, 5, H, dtype = torch.float64)+ for path in _paths("cpu"):+ _assert_falls_back_to_peft(monkeypatch, model, block, path, X0)++[email protected]("device,dtype,path", CASES)[email protected]("state", ["disabled", "merged"])[email protected]("dropout,lora_bias", CONFIGS)+def test_disabled_or_merged_adapter_backward_matches_peft(monkeypatch, device, dtype, path, state, dropout, lora_bias):+ if state == "merged" and lora_bias:+ pytest.skip("PEFT refuses to merge a lora_B bias into a base layer without one")+ model, block = _peft_block(device, dtype, dropout, lora_bias)+ if state == "disabled":+ model.base_model.disable_adapter_layers()+ else:+ for name in TARGETS:+ block.get_submodule(name).merge()+ X0 = torch.randn(2, 5, H, device = device, dtype = dtype)+ _assert_falls_back_to_peft(monkeypatch, model, block, path, X0)++[email protected]("device,dtype,path", CASES)[email protected]("dropout,lora_bias", CONFIGS)+def test_trainable_base_weight_gets_its_gradient(monkeypatch, device, dtype, path, dropout, lora_bias):+ model, block = _peft_block(device, dtype, dropout, lora_bias)+ for p in model.parameters():+ p.requires_grad_(True)+ X0 = torch.randn(2, 5, H, device = device, dtype = dtype)+ ref = _assert_falls_back_to_peft(monkeypatch, model, block, path, X0)+ assert any("base_layer.weight" in name for name in ref[2])++[email protected](not has_real_cuda(), reason = "CUDA autocast")[email protected]("path", _paths("cuda"))[email protected]("dropout,lora_bias", CONFIGS)+ "x_dtype,autocast_dtype",+ [(torch.float16, torch.bfloat16), (torch.bfloat16, torch.float16), (torch.float32, torch.float16)],+)+def test_autocast_dtype_other_than_input_runs_peft(monkeypatch, path, dropout, lora_bias, x_dtype, autocast_dtype):+ model, block = _peft_block("cuda", torch.bfloat16, dropout, lora_bias)+ X0 = torch.randn(2, 5, H, device = "cuda", dtype = x_dtype)+ _assert_falls_back_to_peft(monkeypatch, model, block, path, X0, autocast_dtype)++[email protected]("device", ["cpu"] + (["cuda"] if has_real_cuda() else []))[email protected]("dropout,lora_bias", CONFIGS)+def test_eligible_configs_still_fuse(monkeypatch, device, dropout, lora_bias):+ # Frozen base, one enabled adapter, no variant: fused, including under autocast in X's own dtype.+ dtype = torch.bfloat16+ model, block = _peft_block(device, dtype, dropout, lora_bias)+ X = torch.randn(2, 5, H, device = device, dtype = dtype)+ spy = _Spy(monkeypatch)+ fast_lora.apply_lora_o(block, X)+ fast_lora.apply_lora_qkv(block, X)+ with torch.autocast(device, dtype = dtype):+ fast_lora.apply_lora_o(block, X)+ fast_lora.apply_lora_qkv(block, X)+ assert [name for name, _ in spy.calls] == ["LoRA_W", "LoRA_QKV"] * 2+ if dropout == 0 and not lora_bias:+ # The default config keeps its pre-existing six-argument call.+ assert len(spy.calls[0][1]) == 6+++FUSED_NAMES = {"o": ["LoRA_W"], "qkv": ["LoRA_QKV"], "mlp": ["LoRA_MLP"]}+++def _disable_or_merge(model, block, state):+ if state == "disabled":+ model.base_model.disable_adapter_layers()+ else:+ for name in TARGETS:+ block.get_submodule(name).merge()++[email protected]("device,dtype,path", CASES)[email protected]("state", ["disabled", "merged"])[email protected]("grad_mode", ["no_grad", "inference_mode", "grad_frozen_input"])[email protected]("dropout", [0.0, 0.3])+def test_disabled_or_merged_adapter_without_grad_stays_fused(monkeypatch, device, dtype, path, state, grad_mode, dropout):+ # The reference forward: no gradient flows, so the fused base matmul runs and equals PEFT bitwise.+ model, block = _peft_block(device, dtype, dropout)+ _disable_or_merge(model, block, state)+ X = torch.randn(2, 5, H, device = device, dtype = dtype)+ spy = _Spy(monkeypatch)+ ctx = {"no_grad": torch.no_grad, "inference_mode": torch.inference_mode}.get(grad_mode, torch.enable_grad)+ with ctx():+ fused = PATHS[path][0](block)(X)+ assert [name for name, _ in spy.calls] == FUSED_NAMES[path]+ args = spy.calls[0][1]+ if path == "o":+ # The default config's six-argument call.+ assert len(args) == 6+ else:+ # No LoRA weights, no dropout masks, no lora_B biases: the base matmul only.+ assert args[3] is None and args[-4] is None and all(b is None for b in args[-3:])+ with ctx():+ ref = PATHS[path][1](block)(X)+ # Each projection is PEFT's base_layer(X) bitwise; the MLP's Triton SwiGLU rounds its float32+ # activation 1 ulp differently from torch's SiLU * up.+ tol = {} if path == "mlp" else dict(rtol = 0, atol = 0)+ for a, b in zip(fused if isinstance(fused, tuple) else (fused,), ref if isinstance(ref, tuple) else (ref,)):+ assert not a.requires_grad+ torch.testing.assert_close(a, b, **tol)++[email protected](not has_real_cuda(), reason = "bitsandbytes 4-bit needs CUDA")[email protected]("path", _paths("cuda"))+def test_disabled_adapter_without_grad_on_4bit_base_stays_fused(monkeypatch, path):+ bnb = pytest.importorskip("bitsandbytes")+ from peft import LoraConfig, get_peft_model++ class _Block4bit(torch.nn.Module):+ def __init__(self):+ super().__init__()+ for name, (i, o) in {+ "q_proj": (H, H), "k_proj": (H, H), "v_proj": (H, H), "o_proj": (H, H),+ "gate_proj": (H, 96), "up_proj": (H, 96), "down_proj": (96, H),+ }.items():+ layer = bnb.nn.Linear4bit(i, o, bias = False, compute_dtype = torch.bfloat16, quant_type = "nf4")+ layer.weight = bnb.nn.Params4bit(+ layer.weight.data.to(torch.bfloat16), requires_grad = False, quant_type = "nf4"+ )+ setattr(self, name, layer)+ self.act_fn = torch.nn.SiLU()++ torch.manual_seed(3407)+ base = _Block4bit().to("cuda")+ cfg = LoraConfig(r = 4, lora_alpha = 8, lora_dropout = 0.1, target_modules = TARGETS, init_lora_weights = False)+ model = get_peft_model(base, cfg)+ for name, p in model.named_parameters():+ if "lora_" in name:+ p.data = p.data.to(torch.bfloat16)+ block = model.base_model.model+ assert getattr(block.o_proj.base_layer.weight, "quant_state", None) is not None+ model.train()+ model.base_model.disable_adapter_layers()+ X = torch.randn(2, 7, H, device = "cuda", dtype = torch.bfloat16)+ spy = _Spy(monkeypatch)+ with torch.no_grad():+ fused = PATHS[path][0](block)(X)+ ref = PATHS[path][1](block)(X)+ assert [name for name, _ in spy.calls] == FUSED_NAMES[path]+ for a, b in zip(fused if isinstance(fused, tuple) else (fused,), ref if isinstance(ref, tuple) else (ref,)):+ torch.testing.assert_close(a, b, rtol = 0, atol = 0)+++def test_patch_summary_names_a_lora_variant():+ from unsloth.models.llama import _lora_variant_note++ model, block = _peft_block("cpu", torch.float32, 0.0)+ assert _lora_variant_note(model, "default") == ""+ block.o_proj.lora_variant["default"] = _DoubledLoRA()+ note = _lora_variant_note(model, "default")+ assert "_DoubledLoRA" in note and "PEFT's own forward" in note+++def test_routing_compiles_without_graph_breaks(monkeypatch):+ # The added routing checks are Python attribute reads and grad / autocast mode queries: fullgraph traces+ # the fused, variant, trainable base, autocast and disabled (with and without grad) routes.+ if not fast_lora.TRACE_LORA_FUNCTIONS:+ pytest.skip("fused Functions run Dynamo-disabled before torch 2.11")+ import torch._dynamo++ model, block = _peft_block("cpu", torch.float32, 0.3, True)+ X = torch.randn(2, 5, H)++ def fn(X):+ return fast_lora.apply_lora_o(block, X), fast_lora.apply_lora_qkv(block, X)++ def check(X, autocast_dtype = None, grad = True):+ torch._dynamo.reset()+ compiled = torch.compile(fn, backend = "eager", fullgraph = True)+ with torch.set_grad_enabled(grad), torch.autocast("cpu", dtype = autocast_dtype or torch.bfloat16, enabled = autocast_dtype is not None):+ got = compiled(X)+ want = fn(X)+ torch.testing.assert_close(got[0], want[0])+ for a, b in zip(got[1], want[1]):+ torch.testing.assert_close(a, b)++ model.eval() # deterministic dropout, so compiled and eager agree+ check(X)+ check(X, torch.bfloat16) # autocast in another dtype: PEFT+ check(X.requires_grad_(), grad = True)+ for name in TARGETS:+ block.get_submodule(name).lora_variant["default"] = _DoubledLoRA()+ check(X)+ for name in TARGETS:+ del block.get_submodule(name).lora_variant["default"]+ block.o_proj.base_layer.weight.requires_grad_(True)+ check(X)+ block.o_proj.base_layer.weight.requires_grad_(False)+ model.base_model.disable_adapter_layers()+ check(X.detach(), grad = False) # fused+ check(X.detach().requires_grad_(), grad = True) # PEFTdiff --git a/unsloth/kernels/fast_lora.py b/unsloth/kernels/fast_lora.pyindex 6fe2185b457..661604a4e9f 100644--- a/unsloth/kernels/fast_lora.py+++ b/unsloth/kernels/fast_lora.py@@ -129,6 +129,35 @@ def lora_bias_blocks_fused(proj):return not FUSED_LORA_BIAS and _has_active_lora_bias(proj)+def _needs_peft_forward(proj, adapter):+ # A PEFT LoRA variant (DoRA, MoNTeCLoRA, VeLoRA, BD-LoRA, ...) replaces the LoRA branch in PEFT's+ # forward, and a trainable base weight needs its gradient: the fused Functions compute neither.+ variants = proj.__dict__.get("lora_variant")+ if variants and adapter in variants:+ return True+ return _base_weight_requires_grad(proj)+++def _base_weight_requires_grad(proj):+ base = proj._modules.get("base_layer")+ W = None if base is None else base._parameters.get("weight")+ return W is not None and W.requires_grad+++_is_any_autocast_enabled = getattr(torch._C, "_is_any_autocast_enabled", None)+++def _autocast_dtype_mismatch(X):+ # Under autocast the fused matmuls return the autocast dtype but the in-place LoRA accumulation runs+ # in X's dtype (in-place ops are not autocast), so a different autocast dtype cannot be fused.+ # Mode flags only, no tensor work. Eager first asks whether any autocast is on (a private query, so not+ # under Dynamo, which traces the public calls below as constants).+ if _is_any_autocast_enabled is not None and not _is_compiling() and not _is_any_autocast_enabled():+ return False+ device_type = "cuda" if X.is_cuda else X.device.type+ return torch.is_autocast_enabled(device_type) and torch.get_autocast_dtype(device_type) != X.dtype++def _is_plain_lora(proj):# One enabled, unmerged adapter with no dropout module and no lora_B bias (the default LoRA config).# Reads PEFT's instance state directly: getattr through nn.Module / properties costs more than the@@ -141,7 +170,14 @@ def _is_plain_lora(proj):adapter = adapter[0]if state.get("_disable_adapters", True) or state.get("merged_adapters"):return False+ variants = state.get("lora_variant")+ if variants and adapter in variants:+ return Falsemodules = proj._modules+ base = modules.get("base_layer")+ W = None if base is None else base._parameters.get("weight")+ if W is not None and W.requires_grad:+ return Falsedropout = modules.get("lora_dropout")lora_B = modules.get("lora_B")if dropout is None or lora_B is None:@@ -156,6 +192,8 @@ def _is_plain_lora(proj):def _lora_dropout_and_bias(projs, X):"""(dropout probabilities or None, lora_B biases) for the fused kernels, or None to use PEFT."""+ if _autocast_dtype_mismatch(X):+ return Nonefor proj in projs:if not _is_plain_lora(proj):break@@ -166,6 +204,7 @@ def _lora_dropout_and_bias(projs, X):# module or lora_B bias keeps PEFT's traceable forward, as before they were fused, not a graph break.peft_when_unplain = not TRACE_LORA_FUNCTIONS and _is_compiling()has_dropout_module = False+ grad_enabled = Nonefor proj in projs:if peft_when_unplain and not _is_plain_lora(proj):return None@@ -174,9 +213,18 @@ def _lora_dropout_and_bias(projs, X):# Enabled adapter missing from this projection: PEFT runs the base layer alone.if not getattr(proj, "disable_adapters", True) and not getattr(proj, "merged", False):return None+ # Disabled or merged: the fused Functions run the base matmul alone (the no-grad reference+ # forward of DPO / GRPO), but their backward needs LoRA weights, so PEFT runs it under grad.+ if grad_enabled is None:+ grad_enabled = torch.is_grad_enabled()+ if grad_enabled and (X.requires_grad or _base_weight_requires_grad(proj)):+ return Nonedropout_p.append(0.0)biases.append(None)continue+ if _needs_peft_forward(proj, adapter):+ # A LoRA variant, or a trainable base weight.+ return Nonedropout = getattr(proj, "lora_dropout", None)has_dropout_module |= (dropout is not Nonediff --git a/unsloth/models/llama.py b/unsloth/models/llama.pyindex e4225421596..f46f1b1cc44 100644--- a/unsloth/models/llama.py+++ b/unsloth/models/llama.py@@ -2894,6 +2894,19 @@ def _vllm_will_load_weights(fast_inference, num_labels = None):return True+def _lora_variant_note(model, adapter) -> str:+ """Summary note when the adapter is a PEFT LoRA variant: its layers run PEFT's forward at runtime."""+ for module in model.modules():+ variants = module.__dict__.get("lora_variant")+ if variants and adapter in variants:+ name = type(variants[adapter]).__name__+ return (+ f" The adapter is a PEFT LoRA variant ({name}), so its layers run PEFT's own forward "+ "instead of the fused LoRA kernels."+ )+ return ""++def _fused_lora_skip_reason(lora_dropout,bias,@@ -5215,7 +5228,7 @@ class FastLlamaModel:unfused_reason = _fused_lora_skip_reason(lora_dropout, bias, float32_base, fsdp = fused_lora_declined_for_fsdp- )+ ) + _lora_variant_note(model, active_adapter)logger.warning_once(f"Unsloth {__version__} patched {len(model.model.model.layers)} layers with "f"{n_qkv} QKV layers, {n_o} O layers and {n_mlp} MLP layers.{unfused_reason}",