Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

LoRA: add the rank-r term inside the base GEMM so small updates are not rounded away (#16071)

Daniel Han91989b1a9bda
diff --git a/tests/test_fp8_down_proj_bf16_backward.py b/tests/test_fp8_down_proj_bf16_backward.py
index 39e9f579231..af6650ca3ec 100644
--- a/tests/test_fp8_down_proj_bf16_backward.py
+++ b/tests/test_fp8_down_proj_bf16_backward.py
@@ -108,5 +108,6 @@ def test_bf16_down_proj_backward_unchanged(fp8_calls):
downW = torch.randn(HIDDEN, INTER, generator = torch.Generator().manual_seed(5))
DW, dY, downA, downB = _backward(downW, None)
assert fp8_calls == []
- old = kernel_utils.matmul_lora(dY, downW.t(), None, downB.t(), downA.t(), 2.0)
+ # The module holds FP8 gate / up weights, so it keeps the base-then-LoRA order.
+ old = kernel_utils.matmul_lora(dY, downW.t(), None, downB.t(), downA.t(), 2.0, fused = False)
assert torch.equal(DW, old)
diff --git a/tests/test_lora_first_addmm_op.py b/tests/test_lora_first_addmm_op.py
new file mode 100644
index 00000000000..0b707b17f4a
--- /dev/null
+++ b/tests/test_lora_first_addmm_op.py
@@ -0,0 +1,419 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""Traced fused LoRA order runs as one opaque op (unsloth::lora_first_addmm): same bits as the eager sequence,
+no graph break, and the kill switch keeps the old order."""
+
+import pytest
+import torch
+import unsloth # noqa: F401
+from unsloth.kernels import utils as KU
+
+# Gated on the API, not on the registration result, so a failed registration on a capable torch is a failure.
+pytestmark = pytest.mark.skipif(
+ not hasattr(torch.library, "custom_op"), reason = "needs torch.library.custom_op"
+)
+# Compiled code uses the op only when torch.export's flag tells strict export apart (torch 2.7+).
+_OP_COMPILED = hasattr(torch.compiler, "_is_exporting_flag")
+
+
+def _mk(
+ M = 64,
+ K = 96,
+ N = 80,
+ r = 8,
+ dtype = torch.bfloat16,
+):
+ g = torch.Generator().manual_seed(0)
+ X = torch.randn(M, K, generator = g).to(dtype)
+ W = (torch.randn(N, K, generator = g) * 0.02).to(dtype)
+ A = (torch.randn(r, K, generator = g) * 0.1).to(dtype)
+ B = (torch.randn(N, r, generator = g) * 2e-3).to(dtype)
+ return X, W, A, B
+
+
+def test_op_matches_eager_sequence_bitwise():
+ X, W, A, B = _mk()
+ XA = X @ A.t()
+ ref = torch.addmm(XA.new_zeros(1), XA, B.t(), beta = 0, alpha = 0.5)
+ ref.addmm_(X, W.t())
+ got = torch.ops.unsloth.lora_first_addmm(X, W.t(), XA, B.t(), 0.5)
+ assert torch.equal(got, ref)
+
+
+def test_opcheck():
+ X, W, A, B = _mk(dtype = torch.float32)
+ XA = X @ A.t()
+ torch.library.opcheck(
+ torch.ops.unsloth.lora_first_addmm.default,
+ (X, W.t(), XA, B.t(), 0.5),
+ test_utils = ("test_schema", "test_faketensor", "test_aot_dispatch_static"),
+ )
+
+
[email protected]("dtype", [torch.bfloat16, torch.float32])
+def test_compiled_matmul_lora_uses_op_fullgraph(dtype):
+ X, W, A, B = _mk(dtype = dtype)
+ eager = KU.matmul_lora(X, W, None, A, B, 0.5)
+ import torch._dynamo
+
+ torch._dynamo.reset()
+ seen = []
+ orig = torch.ops.unsloth.lora_first_addmm
+
+ def backend(gm, inputs):
+ seen.extend(str(n.target) for n in gm.graph.nodes if n.op == "call_function")
+ return gm.forward
+
+ f = torch.compile(
+ lambda X, W, A, B: KU.matmul_lora(X, W, None, A, B, 0.5), backend = backend, fullgraph = True
+ )
+ got = f(X, W, A, B)
+ assert any("lora_first_addmm" in s for s in seen) == _OP_COMPILED, seen
+ assert torch.equal(got, eager)
+ f2 = torch.compile(
+ lambda X, W, A, B: KU.matmul_lora(X, W, None, A, B, 0.5),
+ backend = "aot_eager",
+ fullgraph = True,
+ )
+ assert torch.equal(f2(X, W, A, B), eager)
+
+
+def test_kill_switch_keeps_old_order_compiled(monkeypatch):
+ X, W, A, B = _mk()
+ monkeypatch.setattr(KU, "LORA_FUSED_PASSES", False)
+ seen = []
+
+ def backend(gm, inputs):
+ seen.extend(str(n.target) for n in gm.graph.nodes if n.op == "call_function")
+ return gm.forward
+
+ import torch._dynamo
+
+ torch._dynamo.reset()
+ torch.compile(
+ lambda X, W, A, B: KU.matmul_lora(X, W, None, A, B, 0.5), backend = backend, fullgraph = True
+ )(X, W, A, B)
+ assert not any("lora_first_addmm" in s for s in seen)
+
+
+def test_without_the_op_traced_falls_back_to_the_functional_order(monkeypatch):
+ # Registration failed (or an older torch.library): the traced fused branch keeps the two functional addmms.
+ X, W, A, B = _mk(dtype = torch.float32)
+ eager = KU.matmul_lora(X, W, None, A, B, 0.5)
+ monkeypatch.setattr(KU, "_HAS_LORA_FIRST_OP", False)
+ seen = []
+
+ def backend(gm, inputs):
+ seen.extend(str(n.target) for n in gm.graph.nodes if n.op == "call_function")
+ return gm.forward
+
+ torch._dynamo.reset()
+ got = torch.compile(
+ lambda X, W, A, B: KU.matmul_lora(X, W, None, A, B, 0.5), backend = backend, fullgraph = True
+ )(X, W, A, B)
+ assert not any("lora_first_addmm" in s for s in seen)
+ torch.testing.assert_close(got, eager, rtol = 1e-6, atol = 1e-6)
+
+
+def test_opcheck_autograd():
+ X, W, A, B = _mk(dtype = torch.float32)
+ XA = X @ A.t()
+ args = [t.clone().requires_grad_(True) for t in (X, W.t(), XA, B.t())] + [0.5]
+ torch.library.opcheck(
+ torch.ops.unsloth.lora_first_addmm.default,
+ tuple(args),
+ test_utils = ("test_schema", "test_autograd_registration", "test_aot_dispatch_dynamic"),
+ )
+
+
[email protected]("dtype", [torch.bfloat16, torch.float32])
+def test_compiled_differentiable_matmul_lora_matches_eager_grads(dtype):
+ # A compiled caller that differentiates through matmul_lora gets the addmm gradients.
+ import torch._dynamo
+ from functorch.compile import make_boxed_func
+ from torch._dynamo.backends.common import aot_autograd
+
+ X, W, A, B = _mk(dtype = dtype)
+
+ def run(fn):
+ x, a, b = (t.clone().requires_grad_(True) for t in (X, A, B))
+ out = fn(x, W, a, b)
+ out.float().square().sum().backward()
+ return out.detach(), x.grad, a.grad, b.grad
+
+ f = lambda X, W, A, B: KU.matmul_lora(X, W, None, A, B, 0.5) # noqa: E731
+ eager = run(f)
+ torch._dynamo.reset()
+ seen = []
+ boxed = lambda g, i: make_boxed_func(g.forward) # noqa: E731
+
+ def backend(gm, inputs):
+ seen.extend(str(n.target) for n in gm.graph.nodes if n.op == "call_function")
+ return aot_autograd(fw_compiler = boxed, bw_compiler = boxed)(gm, inputs)
+
+ compiled = run(torch.compile(f, backend = backend, fullgraph = True))
+ assert any("lora_first_addmm" in s for s in seen) == _OP_COMPILED, seen
+ tol = dict(rtol = 1e-5, atol = 1e-5) if dtype == torch.float32 else dict(rtol = 2e-2, atol = 2e-2)
+ assert torch.equal(compiled[0], eager[0])
+ for c, e in zip(compiled[1:], eager[1:]):
+ torch.testing.assert_close(c, e, **tol)
+
+
[email protected]("mapped", ["X", "W"])
+def test_eager_helper_under_vmap(mapped):
+ X, W, A, B = _mk(dtype = torch.float32)
+ Xs, Ws = torch.stack([X, X * 2]), torch.stack([W, W * 2])
+
+ def ref(X, W):
+ return X @ W.t() + 0.5 * (X @ A.t()) @ B.t()
+
+ if mapped == "X":
+ got = torch.vmap(lambda x: KU.matmul_lora(x, W, None, A, B, 0.5))(Xs)
+ exp = torch.stack([ref(x, W) for x in Xs])
+ else:
+ got = torch.vmap(lambda w: KU.matmul_lora(X, w, None, A, B, 0.5))(Ws)
+ exp = torch.stack([ref(X, w) for w in Ws])
+ torch.testing.assert_close(got, exp, rtol = 1e-5, atol = 1e-5)
+
+
+def _transform_cases():
+ def f(x, W, A, B):
+ return KU.matmul_lora(x, W, None, A, B, 0.5)
+
+ def jvp(X, W, A, B):
+ return torch.func.jvp(lambda x: f(x, W, A, B), (X,), (torch.ones_like(X),))[1]
+
+ def grad_(X, W, A, B):
+ return torch.func.grad(lambda x: f(x, W, A, B).float().square().sum())(X)
+
+ def jacrev(X, W, A, B):
+ return torch.func.jacrev(lambda x: f(x, W, A, B).float().sum(0))(X)
+
+ def jacfwd(X, W, A, B):
+ return torch.func.jacfwd(lambda x: f(x, W, A, B).float().sum(0))(X)
+
+ def vmap_(X, W, A, B):
+ return torch.func.vmap(lambda x: f(x, W, A, B))(torch.stack([X, X * 2]))
+
+ return {"jvp": jvp, "grad": grad_, "jacrev": jacrev, "jacfwd": jacfwd, "vmap": vmap_}
+
+
[email protected]("backend", ["eager", "aot_eager"])
[email protected]("dtype", [torch.bfloat16, torch.float32])
[email protected]("name", ["jvp", "grad", "jacrev", "jacfwd", "vmap"])
+def test_compiled_function_transforms_match_eager(name, dtype, backend):
+ # Compiled inside torch.func transforms, matmul_lora keeps the base-then-LoRA order: the op has no
+ # forward-mode or batching rule, so compiled jvp / jacfwd returned zeros or failed to trace.
+ import torch._dynamo
+
+ X, W, A, B = _mk(M = 8, K = 16, N = 12, r = 4, dtype = dtype)
+ fn = _transform_cases()[name]
+ eager = fn(X, W, A, B)
+ assert eager.abs().max() > 0
+ torch._dynamo.reset()
+ got = torch.compile(fn, backend = backend, fullgraph = True)(X, W, A, B)
+ tol = dict(rtol = 1e-5, atol = 1e-5) if dtype == torch.float32 else dict(rtol = 2e-2, atol = 2e-2)
+ torch.testing.assert_close(got, eager, **tol)
+
+
+def test_custom_op_saves_only_what_the_requested_grads_need():
+ # Frozen X and a base created under inference_mode (cannot be saved for backward), trainable adapters.
+ X, W, A, B = _mk(dtype = torch.float32)
+ with torch.inference_mode():
+ Wi = W.clone()
+ a, b = A.clone().requires_grad_(True), B.clone().requires_grad_(True)
+ XA = X @ a.t()
+ out = torch.ops.unsloth.lora_first_addmm(X, Wi.t(), XA, b.t(), 0.5)
+ out.square().sum().backward()
+ a2, b2 = A.clone().requires_grad_(True), B.clone().requires_grad_(True)
+ (X @ W.t() + 0.5 * (X @ a2.t()) @ b2.t()).square().sum().backward()
+ torch.testing.assert_close(a.grad, a2.grad, rtol = 1e-5, atol = 1e-5)
+ torch.testing.assert_close(b.grad, b2.grad, rtol = 1e-5, atol = 1e-5)
+
+
[email protected]("backend", ["eager", "aot_eager"])
[email protected]("dtype", [torch.bfloat16, torch.float32])
+def test_compiled_native_forward_ad_keeps_the_tangent(dtype, backend):
+ # Native forward AD (dual_level / make_dual) inside a compiled function: the op has no forward-mode
+ # rule, so matmul_lora keeps the base-then-LoRA order and the tangent survives.
+ import torch._dynamo
+ import torch.autograd.forward_ad as fwAD
+
+ X, W, A, B = _mk(M = 8, K = 16, N = 12, r = 4, dtype = dtype)
+
+ def tangent(x):
+ with fwAD.dual_level():
+ y = KU.matmul_lora(fwAD.make_dual(x, torch.ones_like(x)), W, None, A, B, 0.5)
+ return fwAD.unpack_dual(y).tangent
+
+ eager = tangent(X)
+ assert eager is not None and eager.abs().max() > 0
+ torch._dynamo.reset()
+ got = torch.compile(tangent, backend = backend, fullgraph = True)(X)
+ assert got is not None
+ tol = dict(rtol = 1e-5, atol = 1e-5) if dtype == torch.float32 else dict(rtol = 2e-2, atol = 2e-2)
+ torch.testing.assert_close(got, eager, **tol)
+
+
[email protected]("strict", [False, True])
+def test_exported_program_stays_transformable(strict):
+ # Export (non-strict outside Dynamo, strict through it) keeps the functional form: torch.func on the
+ # exported module has no rule for the opaque op (grad raised, jvp returned zeros).
+ if not hasattr(torch, "export"):
+ pytest.skip("torch.export is not available")
+
+ class M(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ # Buffers, not closed-over tensors: strict export on older torch rejects free tensors.
+ self.register_buffer("W", torch.eye(3))
+ self.register_buffer("A", torch.ones(1, 3))
+ self.register_buffer("B", torch.ones(3, 1))
+
+ def forward(self, x):
+ return KU.matmul_lora(x, self.W, None, self.A, self.B, 0.5)
+
+ x = torch.ones(2, 3)
+ ep = torch.export.export(M(), (x,), strict = strict)
+ assert not any("lora_first_addmm" in str(n.target) for n in ep.graph.nodes)
+ m = ep.module()
+ _, tangent = torch.func.jvp(m, (x,), (torch.ones_like(x),))
+ torch.testing.assert_close(tangent, torch.full((2, 3), 2.5))
+ g = torch.func.grad(lambda x: m(x).sum())(x)
+ torch.testing.assert_close(g, torch.full((2, 3), 2.5))
+
+
+def test_compiled_lora_forward_and_backward_have_no_graph_break():
+ # A compiled LoRA layer (forward + its backward) traces as one graph and still uses the op. On torch
+ # 2.10 / 2.11 Dynamo folds is_exporting() to True, which once reached a banned `layout` read.
+ import torch._dynamo
+ from functorch.compile import make_boxed_func
+ from torch._dynamo.backends.common import aot_autograd
+ from unsloth.kernels import fast_lora
+
+ X, W, A, B = _mk(M = 8, K = 16, N = 12, r = 4, dtype = torch.float32)
+ a, b = torch.nn.Parameter(A.clone()), torch.nn.Parameter(B.clone())
+ seen = []
+
+ def boxed(g, i):
+ # The op sits in the flattened forward / backward graphs AOTAutograd hands to the compilers.
+ seen.extend(str(n.target) for n in g.graph.nodes if n.op == "call_function")
+ return make_boxed_func(g.forward)
+
+ def backend(gm, inputs):
+ return aot_autograd(fw_compiler = boxed, bw_compiler = boxed)(gm, inputs)
+
+ def f(x):
+ return fast_lora.LoRA_W.apply(x.view(1, 8, 16), W, None, a, b, 0.5)
+
+ torch._dynamo.reset()
+ x = X.clone().requires_grad_(True)
+ out = torch.compile(f, backend = backend, fullgraph = True)(x)
+ out.square().sum().backward()
+ assert any("lora_first_addmm" in s for s in seen) == _OP_COMPILED, seen
+ assert x.grad is not None and a.grad is not None and b.grad is not None
+
+
+def test_exporting_reads_the_export_flag_not_the_folded_query(monkeypatch):
+ # torch 2.10 / 2.11 Dynamo folds is_exporting() to True in every compiled region, so plain compiles
+ # would lose the op; the flag torch.export sets is exact. Without the flag, assume export.
+ if hasattr(torch.compiler, "_is_exporting_flag"):
+ monkeypatch.setattr(torch.compiler, "_is_exporting_flag", False)
+ assert KU._exporting() is False
+ monkeypatch.setattr(torch.compiler, "_is_exporting_flag", True)
+ assert KU._exporting() is True
+ monkeypatch.delattr(torch.compiler, "_is_exporting_flag")
+ assert KU._exporting() is True
+
+
+_PARTIAL_REGISTRATION_CASE = """
+import torch
+from torch._library.custom_ops import CustomOpDef
+orig = CustomOpDef.register_autograd
+def boom(self, *a, **k):
+ if self._qualname == "unsloth::lora_first_addmm":
+ raise RuntimeError("injected registration failure")
+ return orig(self, *a, **k)
+CustomOpDef.register_autograd = boom
+import unsloth # noqa: F401
+from unsloth.kernels import utils as KU
+assert hasattr(torch.ops.unsloth, "lora_first_addmm"), "the schema should still be registered"
+print("HAS_OP", KU._HAS_LORA_FIRST_OP)
+"""
+
+
+def test_a_partly_registered_op_is_not_used():
+ # Registration fails after the schema exists (autograd here): the op must stay disabled rather than be
+ # enabled because its name resolves. A fresh interpreter, since this process registered it already.
+ import os
+ import subprocess
+ import sys
+ from pathlib import Path
+
+ root = Path(__file__).resolve().parents[1]
+ env = dict(
+ os.environ,
+ PYTHONPATH = os.pathsep.join([str(root), os.environ.get("PYTHONPATH", "")]),
+ UNSLOTH_DISABLE_CPU_FALLBACK = "0",
+ )
+ p = subprocess.run(
+ [sys.executable, "-c", _PARTIAL_REGISTRATION_CASE],
+ cwd = root,
+ env = env,
+ capture_output = True,
+ text = True,
+ timeout = 900,
+ )
+ assert p.returncode == 0 and "HAS_OP False" in p.stdout, (p.stdout + p.stderr)[-3000:]
+
+
+_REEXECUTED_MODULE_CASE = """
+import importlib
+import torch
+import unsloth # noqa: F401
+from unsloth.kernels import utils as KU
+assert KU._HAS_LORA_FIRST_OP is True
+calls = []
+orig = torch.library.custom_op
+def counting(name, *a, **k):
+ if name == "unsloth::lora_first_addmm":
+ calls.append(name)
+ return orig(name, *a, **k)
+torch.library.custom_op = counting
+KU = importlib.reload(KU)
+print("REREGISTERED", len(calls), "HAS_OP", KU._HAS_LORA_FIRST_OP)
+"""
+
+
+def test_a_reexecuted_module_reuses_the_completed_registration():
+ # Re-registering over a working op could leave it half replaced if a later step failed; a re-executed
+ # module must reuse the ready op instead of defining it again.
+ import os
+ import subprocess
+ import sys
+ from pathlib import Path
+
+ root = Path(__file__).resolve().parents[1]
+ env = dict(
+ os.environ,
+ PYTHONPATH = os.pathsep.join([str(root), os.environ.get("PYTHONPATH", "")]),
+ UNSLOTH_DISABLE_CPU_FALLBACK = "0",
+ )
+ p = subprocess.run(
+ [sys.executable, "-c", _REEXECUTED_MODULE_CASE],
+ cwd = root,
+ env = env,
+ capture_output = True,
+ text = True,
+ timeout = 900,
+ )
+ assert p.returncode == 0 and "REREGISTERED 0 HAS_OP True" in p.stdout, (p.stdout + p.stderr)[
+ -3000:
+ ]
+
+
+def test_a_completely_registered_op_is_marked_ready():
+ # The marker a re-executed module reads is set only once every registration step succeeded.
+ assert torch.ops.unsloth.__dict__.get("_lora_first_addmm_ready") is True
+ assert KU._HAS_LORA_FIRST_OP is True
diff --git a/tests/test_lora_fused_passes.py b/tests/test_lora_fused_passes.py
new file mode 100644
index 00000000000..0504cd4d2b0
--- /dev/null
+++ b/tests/test_lora_fused_passes.py
@@ -0,0 +1,899 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""UNSLOTH_LORA_FUSED_PASSES: the fused LoRA kernels write the rank-r term first and let the base GEMM
+accumulate onto it, share X @ A across projections reading the same X, and read X once for every
+projection's LoRA gradients. Outputs and gradients match the base-then-LoRA order and an fp64 PEFT
+oracle; the shared paths are taken only when they apply."""
+
+from __future__ import annotations
+
+import copy
+
+import pytest
+import torch
+import unsloth # noqa: F401
+
+from real_accelerator import has_real_cuda
+from unsloth.kernels import fast_lora
+from unsloth.kernels import utils as kernel_utils
+
+H, I = 64, 96
+TARGETS = ["q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj"]
+DEVICES = ["cpu"] + (["cuda"] if has_real_cuda() else [])
+
+
+class _Block(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.q_proj = torch.nn.Linear(H, H, bias = False)
+ self.k_proj = torch.nn.Linear(H, H // 2, bias = False)
+ self.v_proj = torch.nn.Linear(H, H // 2, bias = False)
+ self.o_proj = torch.nn.Linear(H, H, bias = False)
+ self.gate_proj = torch.nn.Linear(H, I, bias = False)
+ self.up_proj = torch.nn.Linear(H, I, bias = False)
+ self.down_proj = torch.nn.Linear(I, H, bias = False)
+ self.act_fn = torch.nn.SiLU()
+
+
+def _model(
+ device,
+ dtype,
+ alpha_pattern = None,
+ dropout = 0.0,
+ lora_bias = False,
+):
+ from peft import LoraConfig, get_peft_model
+
+ torch.manual_seed(3407)
+ cfg = LoraConfig(
+ r = 4,
+ lora_alpha = 8,
+ lora_dropout = dropout,
+ target_modules = TARGETS,
+ init_lora_weights = False,
+ alpha_pattern = alpha_pattern or {},
+ lora_bias = lora_bias,
+ )
+ model = get_peft_model(_Block(), cfg).to(device, dtype)
+ for name, p in model.named_parameters():
+ p.requires_grad_("lora_" in name)
+ if lora_bias:
+ with torch.no_grad():
+ for name, p in model.named_parameters():
+ if "lora_B" in name and name.endswith("bias"):
+ p.normal_()
+ model.train()
+ return model, model.base_model.model
+
+
+PATHS = {
+ "qkv": (
+ lambda b: (lambda X: fast_lora.apply_lora_qkv(b, X)),
+ lambda b: (lambda X: (b.q_proj(X), b.k_proj(X), b.v_proj(X))),
+ ),
+ "o": (lambda b: (lambda X: fast_lora.apply_lora_o(b, X)), lambda b: b.o_proj),
+ "mlp": (
+ lambda b: (lambda X: fast_lora.apply_lora_mlp_swiglu(b, X)),
+ lambda b: (lambda X: b.down_proj(b.act_fn(b.gate_proj(X)) * b.up_proj(X))),
+ ),
+}
+
+
+def _run(fn, model, X0):
+ X = X0.detach().clone().requires_grad_()
+ out = fn(X * 1) # a non-leaf: the fused backward may write dX into its input buffer
+ outs = out if isinstance(out, tuple) else (out,)
+ gen = torch.Generator(device = "cpu").manual_seed(7)
+ loss = sum(
+ (o.double() * torch.randn(o.shape, generator = gen, dtype = torch.float64).to(o.device)).sum()
+ for o in outs
+ )
+ loss.backward()
+ grads = {n: p.grad.clone() for n, p in model.named_parameters() if p.grad is not None}
+ model.zero_grad(set_to_none = True)
+ return [o.detach() for o in outs], X.grad.clone(), grads
+
+
+def _skip_without_native_bf16(device, dtype = torch.bfloat16):
+ # Same rule as test_lora_gradient_layout: Triton's bf16 kernels need sm_80+, and is_bf16_supported() also
+ # reports emulated bf16 on older GPUs.
+ if (
+ device == "cuda"
+ and dtype is torch.bfloat16
+ and (
+ not torch.cuda.is_bf16_supported()
+ or (not torch.version.hip and torch.cuda.get_device_capability()[0] < 8)
+ )
+ ):
+ pytest.skip("bf16 kernels need native bf16 (sm_80+)")
+
+
+def _set(monkeypatch, on):
+ monkeypatch.setattr(kernel_utils, "LORA_FUSED_PASSES", on)
+
+
+def _err(a, b):
+ return float((a.double() - b.double()).norm() / b.double().norm().clamp_min(1e-30))
+
+
[email protected]("device", DEVICES)
[email protected]("dtype", [torch.float32, torch.bfloat16])
[email protected]("path", ["qkv", "o", "mlp"])
[email protected]("alphas", ["equal", "mixed"])
+def test_fused_passes_match_the_unfused_order_and_an_fp64_oracle(
+ device, dtype, path, alphas, monkeypatch
+):
+ if device == "cpu" and path == "mlp":
+ pytest.skip("the SwiGLU kernels are Triton")
+ if device == "cpu" and dtype is torch.bfloat16:
+ pytest.skip("bf16 GEMMs on CPU are slow and not what runs in training")
+ _skip_without_native_bf16(device, dtype)
+ pattern = {"k_proj": 16, "up_proj": 16} if alphas == "mixed" else None
+ model, block = _model(device, dtype, pattern)
+ X0 = torch.randn(3, 5, H, device = device, dtype = dtype)
+ _set(monkeypatch, False)
+ ref_out, ref_dX, ref_g = _run(PATHS[path][0](block), model, X0)
+ _set(monkeypatch, True)
+ out, dX, g = _run(PATHS[path][0](block), model, X0)
+ oracle_model = copy.deepcopy(model).double()
+ o_out, o_dX, o_g = _run(
+ PATHS[path][1](oracle_model.base_model.model), oracle_model, X0.double()
+ )
+ tol = 1e-5 if dtype is torch.float32 else 2e-2
+ for a, b, o in zip(out, ref_out, o_out):
+ assert _err(a, o) <= max(tol, 1.5 * _err(b, o)), (_err(a, o), _err(b, o))
+ assert _err(dX, o_dX) <= max(tol, 1.5 * _err(ref_dX, o_dX)), (
+ _err(dX, o_dX),
+ _err(ref_dX, o_dX),
+ )
+ assert set(g) == set(ref_g) == set(o_g)
+ for n in g:
+ assert g[n].shape == ref_g[n].shape
+ assert _err(g[n], o_g[n]) <= max(tol, 1.5 * _err(ref_g[n], o_g[n])), (
+ n,
+ _err(g[n], o_g[n]),
+ _err(ref_g[n], o_g[n]),
+ )
+
+
+def test_shared_paths_engage_only_when_they_apply(monkeypatch):
+ calls = {"xa": 0, "bwd": 0}
+ real_xa, real_bwd = fast_lora._shared_XA, fast_lora._shared_lora_backward
+
+ def xa(*a, **k):
+ calls["xa"] += 1
+ return real_xa(*a, **k)
+
+ def bwd(*a, **k):
+ calls["bwd"] += 1
+ return real_bwd(*a, **k)
+
+ monkeypatch.setattr(fast_lora, "_shared_XA", xa)
+ monkeypatch.setattr(fast_lora, "_shared_lora_backward", bwd)
+ X0 = torch.randn(2, 3, H)
+ for alphas, on, want in (
+ ("equal", True, (1, 1)),
+ ("mixed", True, (1, 0)),
+ ("equal", False, (0, 0)),
+ ):
+ _set(monkeypatch, on)
+ calls.update(xa = 0, bwd = 0)
+ model, block = _model("cpu", torch.float32, {"k_proj": 16} if alphas == "mixed" else None)
+ _run(PATHS["qkv"][0](block), model, X0)
+ assert (calls["xa"], calls["bwd"]) == want, (alphas, on, calls)
+
+
[email protected]("device", DEVICES)
[email protected]("dtype", [torch.float32, torch.bfloat16])
+def test_kill_switch_is_the_base_then_lora_order(device, dtype, monkeypatch):
+ # UNSLOTH_LORA_FUSED_PASSES=0: base GEMM, then the rank-r term added in place, bit for bit.
+ if device == "cpu" and dtype is torch.bfloat16:
+ pytest.skip("bf16 GEMMs on CPU are slow and not what runs in training")
+ _skip_without_native_bf16(device, dtype)
+ _set(monkeypatch, False)
+ g = torch.Generator(device = "cpu").manual_seed(0)
+ X, W = torch.randn(7, H, generator = g), torch.randn(I, H, generator = g)
+ A, B = torch.randn(4, H, generator = g), torch.randn(I, 4, generator = g)
+ X, W, A, B = (t.to(device, dtype) for t in (X, W, A, B))
+ want = X @ W.t()
+ kernel_utils.addmm_(want, X @ A.t(), B.t(), alpha = 2.0)
+ assert torch.equal(kernel_utils.matmul_lora(X, W, None, A, B, 2.0), want)
+
+
+def _drop_lora_bias(block, keep):
+ # PEFT puts lora_bias on every target; keep it only on `keep` (a down-only bias, for example).
+ for name in TARGETS:
+ if name not in keep:
+ getattr(block, name).lora_B["default"].bias = None
+
+
[email protected]("device", DEVICES)
[email protected]("dtype", [torch.float16, torch.bfloat16, torch.float32])
[email protected]("path", ["qkv", "o", "mlp"])
[email protected]("config", ["plain", "dropout", "lora_bias", "down_bias"])
+def test_exclusions_keep_the_base_then_lora_order(device, dtype, path, config, monkeypatch):
+ # Every configuration the fused order excludes (fp16 at all, dropout, lora_bias anywhere in the
+ # module) gives the kill switch's outputs and gradients bit for bit, forward and backward.
+ if device == "cpu" and path == "mlp":
+ pytest.skip("the SwiGLU kernels are Triton")
+ if config == "down_bias" and path != "mlp":
+ pytest.skip("a down-only bias is an MLP case")
+ if config == "plain" and dtype is not torch.float16:
+ pytest.skip("plain bf16 / fp32 take the fused order")
+ _skip_without_native_bf16(device, dtype)
+ kwargs = {"dropout": 0.1} if config == "dropout" else {}
+ kwargs["lora_bias"] = config in ("lora_bias", "down_bias")
+ model, block = _model(device, dtype, **kwargs)
+ if config == "down_bias":
+ _drop_lora_bias(block, keep = ("down_proj",))
+ X0 = torch.randn(3, 5, H, device = device, dtype = dtype)
+ runs = []
+ for on in (False, True):
+ _set(monkeypatch, on)
+ torch.manual_seed(11)
+ runs.append(_run(PATHS[path][0](block), model, X0))
+ (out0, dX0, g0), (out1, dX1, g1) = runs
+ assert all(torch.equal(a, b) for a, b in zip(out0, out1))
+ assert torch.equal(dX0, dX1)
+ assert set(g0) == set(g1) and all(torch.equal(g0[n], g1[n]) for n in g0)
+
+
[email protected]("device", DEVICES)
+def test_fp16_rank_r_term_cannot_overflow_before_the_base_cancels_it(device, monkeypatch):
+ # The rank-r term alone is ~70000 (past fp16's 65504) but the output is ~10000. The base-then-LoRA
+ # order adds it in the GEMM's fp32 epilogue; writing it first would store inf.
+ X = torch.ones(1, H, dtype = torch.float16, device = device)
+ W = torch.zeros(I, H, dtype = torch.float16, device = device)
+ W[0] = -60000 / H
+ A = torch.zeros(4, H, dtype = torch.float16, device = device)
+ A[0, 0] = 1
+ B = torch.zeros(I, 4, dtype = torch.float16, device = device)
+ B[0, 0] = 35000
+ outs = {}
+ for on in (True, False):
+ _set(monkeypatch, on)
+ outs[on] = kernel_utils.matmul_lora(X, W, None, A, B, 2.0)
+ assert torch.isfinite(outs[True]).all() and abs(float(outs[True][0, 0]) - 10000) <= 32
+ assert torch.equal(outs[True], outs[False])
+
+
+def test_fused_eligibility_is_one_rule(monkeypatch):
+ _set(monkeypatch, True)
+ from types import SimpleNamespace
+
+ from unsloth.kernels.int4_packed import Int4QuantState
+
+ ok = kernel_utils._lora_fused_ok
+ dense = lambda dtype: torch.empty(4, 4, dtype = dtype) # noqa: E731
+ u8 = torch.empty(4, 4, dtype = torch.uint8)
+ assert ok(dense(torch.bfloat16), None, torch.bfloat16) and ok(
+ dense(torch.float32), None, torch.float32
+ )
+ assert not ok(dense(torch.float16), None, torch.float16) # fp16 keeps the old order
+ assert not ok(dense(torch.float32), None, torch.bfloat16) # fp32 weight under bf16 autocast
+ # fast_dequantize returns a 4-bit state's own dtype, not the one asked for.
+ assert ok(u8, SimpleNamespace(dtype = torch.bfloat16), torch.bfloat16)
+ assert not ok(u8, SimpleNamespace(dtype = torch.float16), torch.bfloat16)
+ assert not ok(u8, [None, None, torch.float16], torch.bfloat16)
+ assert not ok(dense(torch.float8_e4m3fn), torch.ones(1), torch.bfloat16)
+ int4 = object.__new__(Int4QuantState)
+ w = torch.empty(8, 4, dtype = torch.int8)
+ assert not ok(w, int4, torch.bfloat16) and not ok(
+ w.t(), int4, torch.bfloat16
+ ) # either orientation
+ packed = type("Packed", (), {"_unsloth_packed_weight_state": True, "dtype": torch.bfloat16})()
+ assert not ok(u8, packed, torch.bfloat16)
+
+
+def test_a_caller_buffer_that_is_not_the_output_keeps_main_behavior(monkeypatch):
+ # torch.matmul(..., out = empty) resizes; the fused order writes the rank-r term first, so it is
+ # taken only when the buffer is already the output.
+ g = torch.Generator(device = "cpu").manual_seed(0)
+ X, W = torch.randn(7, H, generator = g), torch.randn(I, H, generator = g)
+ A, B = torch.randn(4, H, generator = g), torch.randn(I, 4, generator = g)
+ outs = {}
+ for on in (True, False):
+ _set(monkeypatch, on)
+ out = torch.empty(0)
+ assert kernel_utils.matmul_lora(X, W, None, A, B, 2.0, out = out) is not None
+ outs[on] = out
+ assert outs[True].shape == (7, I) and torch.equal(outs[True], outs[False])
+
+
[email protected]("device", DEVICES)
+def test_fp32_base_under_autocast_keeps_the_base_then_lora_order(device, monkeypatch):
+ # addmm_ is not autocast: an fp32 base weight with bf16 activations takes the old order.
+ _skip_without_native_bf16(device)
+ half = torch.bfloat16
+ g = torch.Generator(device = "cpu").manual_seed(0)
+ X, W = torch.randn(7, H, generator = g), torch.randn(I, H, generator = g)
+ A, B = torch.randn(4, H, generator = g), torch.randn(I, 4, generator = g)
+ X, W, A, B = X.to(device, half), W.to(device), A.to(device), B.to(device)
+ model, block = _model(device, torch.float32)
+ X0 = torch.randn(2, 5, H, generator = g).to(device, half)
+ outs, runs = {}, {}
+ for on in (True, False):
+ _set(monkeypatch, on)
+ with torch.autocast(device, dtype = half):
+ outs[on] = kernel_utils.matmul_lora(X, W, None, A, B, 2.0)
+ runs[on] = _run(PATHS["o"][0](block), model, X0)
+ assert torch.equal(outs[True], outs[False])
+ (out_on, dX_on, g_on), (out_off, dX_off, g_off) = runs[True], runs[False]
+ assert torch.equal(out_on[0], out_off[0]) and torch.equal(dX_on, dX_off)
+ assert g_on.keys() == g_off.keys() and all(torch.equal(g_on[k], g_off[k]) for k in g_on)
+
+
[email protected](not has_real_cuda(), reason = "needs bitsandbytes 4-bit on a real GPU")
+def test_fp16_nf4_state_under_bf16_autocast_keeps_the_base_then_lora_order(monkeypatch):
+ # fast_dequantize returns the state's fp16, so the in-place base accumulate would mix dtypes.
+ bnb = pytest.importorskip("bitsandbytes")
+ _skip_without_native_bf16("cuda")
+ lin = bnb.nn.Linear4bit(H, I, bias = False, compute_dtype = torch.float16, quant_type = "nf4")
+ lin.weight = bnb.nn.Params4bit(
+ torch.randn(I, H, dtype = torch.float16), requires_grad = False, quant_type = "nf4"
+ )
+ lin = lin.cuda()
+ W, Wq = lin.weight, lin.weight.quant_state
+ assert Wq.dtype == torch.float16
+ g = torch.Generator(device = "cpu").manual_seed(0)
+ X = torch.randn(7, H, generator = g).cuda().bfloat16()
+ A, B = torch.randn(4, H, generator = g).cuda(), torch.randn(I, 4, generator = g).cuda()
+ outs = {}
+ for on in (True, False):
+ _set(monkeypatch, on)
+ with torch.autocast("cuda", dtype = torch.bfloat16):
+ outs[on] = kernel_utils.matmul_lora(X, W, Wq, A, B, 2.0)
+ assert torch.equal(outs[True], outs[False])
+
+
+def test_small_update_survives_the_single_rounding(monkeypatch):
+ # bf16 base output 1.0 plus a rank-r term of 1/128 rounds once to 1.0078125; the old order rounds the
+ # base output first and the separate add then loses everything below half a bf16 step.
+ x = torch.tensor([[1.0, 1.0]], dtype = torch.bfloat16)
+ w = torch.tensor([[1.0, 1 / 256]], dtype = torch.bfloat16)
+ a = torch.tensor([[1.0, 0.0]], dtype = torch.bfloat16)
+ b = torch.tensor([[1 / 512]], dtype = torch.bfloat16)
+ exact = (x.double() @ w.double().t() + (x.double() @ a.double().t()) @ b.double().t()).to(
+ torch.bfloat16
+ )
+ _set(monkeypatch, True)
+ got = kernel_utils.matmul_lora(x, w, None, a, b, 1.0)
+ assert torch.equal(got, exact) and got.item() == 1.0078125
+ _set(monkeypatch, False)
+ assert kernel_utils.matmul_lora(x, w, None, a, b, 1.0).item() == 1.0
+
+
+def test_a_caller_buffer_sharing_an_operand_keeps_main_behavior(monkeypatch):
+ # Writing the rank-r term first would overwrite a base operand the GEMM has not read yet.
+ _set(monkeypatch, True)
+ g = torch.Generator(device = "cpu").manual_seed(0)
+ X, W = torch.randn(H, H, generator = g), torch.randn(H, H, generator = g)
+ A, B = torch.randn(4, H, generator = g), torch.randn(H, 4, generator = g)
+ assert kernel_utils._shares_storage(W, X, W)
+ assert not kernel_utils._shares_storage(torch.empty(H, H), X, W, A, B, None)
+ # The branch taken is checked, not the values: matmul with out= overlapping an input is undefined on
+ # main too (on CPU a 64 x 64 result differs from call to call).
+ betas = []
+ real = kernel_utils.addmm_
+ monkeypatch.setattr(
+ kernel_utils, "addmm_", lambda *a, **k: betas.append(k.get("beta", 1)) or real(*a, **k)
+ )
+ out = W.clone()
+ kernel_utils.matmul_lora(X, out, None, A, B, 2.0, out = out)
+ assert betas == [1], "a buffer sharing W must take the base-then-LoRA order"
+ betas.clear()
+ kernel_utils.matmul_lora(X, W, None, A, B, 2.0, out = torch.empty(H, H))
+ assert betas == [0], "a separate buffer takes the fused order"
+
+
+def test_shared_lora_a_grads_have_the_parameter_layout(monkeypatch):
+ # The shared backward hands back d_A whose .t() is contiguous like the [r, K] parameter, so autograd
+ # does not copy it into a new layout.
+ _set(monkeypatch, True)
+ X = torch.randn(10, H)
+ As = (torch.randn(4, H).t(), torch.randn(4, H).t())
+ Bs = (torch.randn(H, 4).t(), torch.randn(I, 4).t())
+ dYs = (torch.randn(10, H), torch.randn(10, I))
+ d_As, _, _ = fast_lora._shared_lora_backward(X, dYs, As, Bs, 2.0, None)
+ for dA in d_As:
+ assert dA.t().is_contiguous(), dA.stride()
+
+
+def test_shared_lora_a_grads_are_views_of_one_buffer(monkeypatch):
+ # Each d_A is a row block of one [sum r, K] buffer: no per-projection copy into the parameter layout.
+ _set(monkeypatch, True)
+ X = torch.randn(10, H)
+ As = (torch.randn(4, H).t(), torch.randn(3, H).t())
+ Bs = (torch.randn(H, 4).t(), torch.randn(I, 3).t())
+ dYs = (torch.randn(10, H), torch.randn(10, I))
+ d_As, d_Bs, dX = fast_lora._shared_lora_backward(X, dYs, As, Bs, 2.0, None)
+ ptrs = {dA.untyped_storage().data_ptr() for dA in d_As}
+ assert len(ptrs) == 1, "d_A blocks were copied out of the shared buffer"
+ X64 = X.double()
+ for dA, dB_, A, B, dY in zip(d_As, d_Bs, As, Bs, dYs):
+ y_dB = dY.double() @ B.double().t()
+ torch.testing.assert_close(dA.double(), 2.0 * X64.t() @ y_dB, rtol = 1e-4, atol = 1e-4)
+ torch.testing.assert_close(
+ dB_.double(), 2.0 * (A.double().t() @ X64.t()) @ dY.double(), rtol = 1e-4, atol = 1e-4
+ )
+
+
+def _small_update(dtype = torch.bfloat16):
+ # Base output 1 + 1/256 plus a rank-r term 1/512: the fused order rounds once to 1.0078125, the
+ # base-then-LoRA order rounds the base to 1.0 first and then loses the update.
+ x = torch.tensor([[1.0, 1.0]], dtype = dtype)
+ w = torch.tensor([[1.0, 1 / 256]], dtype = dtype)
+ a = torch.tensor([[1.0, 0.0]], dtype = dtype)
+ b = torch.tensor([[1 / 512]], dtype = dtype)
+ return x, w, a, b
+
+
+def test_compiled_caller_buffer_keeps_main_behavior(monkeypatch):
+ # Traced, a caller's buffer cannot be checked for aliasing, so it keeps the base-then-LoRA order.
+ _set(monkeypatch, True)
+ torch._dynamo.reset()
+ x, w, a, b = _small_update()
+
+ def f(x, w, a, b):
+ out = torch.empty(1, 1, dtype = x.dtype)
+ return kernel_utils.matmul_lora(x, w, None, a, b, 1.0, out = out)
+
+ got = torch.compile(f, backend = "aot_eager", fullgraph = True)(x, w, a, b)
+ assert got.item() == 1.0
+ # A buffer that aliases W: the old order reads W before the output is written.
+ g = torch.Generator(device = "cpu").manual_seed(1)
+ X, W = torch.randn(4, 4, generator = g), torch.randn(4, 4, generator = g)
+ A, B = torch.randn(4, 4, generator = g), torch.randn(4, 4, generator = g)
+ ref = X @ W.t() + 2.0 * (X @ A.t()) @ B.t()
+
+ def g_(X, W, A, B):
+ out = W.clone()
+ return kernel_utils.matmul_lora(X, out, None, A, B, 2.0, out = out)
+
+ torch._dynamo.reset()
+ got = torch.compile(g_, backend = "aot_eager", fullgraph = True)(X, W, A, B)
+ torch.testing.assert_close(got, ref, rtol = 1e-5, atol = 1e-5)
+
+
+def test_fake_tracing_with_a_caller_buffer(monkeypatch):
+ # make_fx fake tensors have no data pointer: the buffer is treated as shared, not inspected.
+ from torch.fx.experimental.proxy_tensor import make_fx
+
+ _set(monkeypatch, True)
+ X, W, A, B = torch.randn(3, H), torch.randn(H, H), torch.randn(4, H), torch.randn(H, 4)
+
+ def f(X, W, A, B, out):
+ return kernel_utils.matmul_lora(X, W, None, A, B, 2.0, out = out)
+
+ gm = make_fx(f, tracing_mode = "fake")(X, W, A, B, torch.empty(3, H))
+ out = torch.empty(3, H)
+ torch.testing.assert_close(
+ gm(X, W, A, B, out), X @ W.t() + 2.0 * (X @ A.t()) @ B.t(), rtol = 1e-5, atol = 1e-5
+ )
+
+
[email protected]("fused", [True, False])
+def test_a_precomputed_xa_sharing_the_caller_buffer_is_not_reused(fused, monkeypatch):
+ # The base GEMM writes the buffer before the rank-r term reads XA: XA is recomputed instead.
+ _set(monkeypatch, True)
+ g = torch.Generator(device = "cpu").manual_seed(3)
+ X, W = torch.randn(3, 4, generator = g), torch.randn(2, 4, generator = g)
+ A, B = torch.randn(2, 4, generator = g), torch.randn(2, 2, generator = g)
+ ref = X @ W.t() + 2.0 * (X @ A.t()) @ B.t()
+ XA = X @ A.t()
+ got = kernel_utils.matmul_lora(X, W, None, A, B, 2.0, XA = XA, out = XA, fused = fused)
+ torch.testing.assert_close(got, ref, rtol = 1e-5, atol = 1e-5)
+
+
+def test_sparse_base_weight_keeps_the_base_then_lora_order(monkeypatch):
+ # addmm_ cannot accumulate a dense X @ sparse W.t() in place; main's matmul handles it.
+ _set(monkeypatch, True)
+ X, A, B = torch.ones(1, 2), torch.ones(1, 2), torch.ones(2, 1)
+ W = torch.eye(2).to_sparse_coo()
+ assert not kernel_utils._lora_fused_ok(W, None, torch.float32)
+ torch.testing.assert_close(
+ kernel_utils.matmul_lora(X, W, None, A, B, 1.0), torch.full((1, 2), 3.0)
+ )
+
+
[email protected]("dropout", [0.0, 0.1])
+def test_multi_token_inference_follows_the_training_order(dropout, monkeypatch):
+ # fast_linear_forward with q_len > 1 takes the order the fused training kernels use: a configured
+ # lora_dropout (even in eval) keeps base-then-LoRA, a plain adapter takes the fused order.
+ _skip_without_native_bf16("cpu")
+ from peft import LoraConfig, get_peft_model
+
+ _set(monkeypatch, True)
+ lin = torch.nn.Sequential(torch.nn.Linear(2, 1, bias = False))
+ model = get_peft_model(
+ lin, LoraConfig(r = 1, lora_alpha = 1, lora_dropout = dropout, target_modules = ["0"])
+ )
+ model = model.to(torch.bfloat16).eval()
+ proj = model.base_model.model[0]
+ x, w, a, b = _small_update()
+ with torch.no_grad():
+ proj.base_layer.weight.copy_(w)
+ proj.lora_A["default"].weight.copy_(a)
+ proj.lora_B["default"].weight.copy_(b)
+ X = x.view(1, 1, 2).expand(1, 3, 2).contiguous()
+ got = kernel_utils.fast_linear_forward(proj, X)
+ assert got[0, 0, 0].item() == (1.0 if dropout else 1.0078125)
+
+
+_DYNAMIC_RANK_CASE = """
+import sys
+import torch
+import unsloth # noqa: F401
+from unsloth.kernels import fast_lora
+
+torch.manual_seed(0)
+weights = []
+for n, r in zip((16, 12, 12), (2, 3, 4)):
+ W = torch.randn(n, 16) * 0.1
+ A, B = torch.randn(r, 16) * 0.1, torch.randn(n, r) * 0.1
+ if sys.argv[1] == "param":
+ A, B = torch.nn.Parameter(A), torch.nn.Parameter(B)
+ else:
+ A, B = A.requires_grad_(), B.requires_grad_()
+ weights.append((W, A, B))
+(QW, QA, QB), (KW, KA, KB), (VW, VA, VB) = weights
+
+
+def f(X, QA, QB, KA, KB, VA, VB):
+ q, k, v = fast_lora.LoRA_QKV.apply(
+ X, QW, None, QA, QB, 2.0, KW, None, KA, KB, 2.0, VW, None, VA, VB, 2.0, False
+ )
+ return q.sum() + k.sum() + v.sum()
+
+
+g = torch.compile(f, backend = "aot_eager", fullgraph = True, dynamic = True)
+for seq in (3, 5, 7):
+ X = torch.randn(2, seq, 16, requires_grad = True)
+ g(X, QA, QB, KA, KB, VA, VB).backward()
+ assert X.grad.shape == X.shape
+print("OK")
+"""
+
+
[email protected]("param", [False, True])
+def test_dynamic_compile_with_a_rank_equal_to_the_first_length(param):
+ # torch 2.14: dynamic-shape plain-tensor LoRA weights once recompiled to a width-0 dQ; they keep the
+ # base-then-LoRA order when traced. Fresh interpreter: depends on Dynamo state.
+ import os
+ import subprocess
+ import sys
+ from pathlib import Path
+
+ root = Path(__file__).resolve().parents[1]
+ env = dict(
+ os.environ,
+ PYTHONPATH = os.pathsep.join([str(root), os.environ.get("PYTHONPATH", "")]),
+ UNSLOTH_DISABLE_CPU_FALLBACK = "0",
+ )
+ p = subprocess.run(
+ [sys.executable, "-c", _DYNAMIC_RANK_CASE, "param" if param else "plain"],
+ cwd = root,
+ env = env,
+ capture_output = True,
+ text = True,
+ timeout = 900,
+ )
+ assert p.returncode == 0 and "OK" in p.stdout, (p.stdout + p.stderr)[-3000:]
+
+
[email protected]("param", [False, True])
+def test_traced_order_depends_on_parameter_lora_weights(param, monkeypatch):
+ _skip_without_native_bf16("cpu")
+ _set(monkeypatch, True)
+ torch._dynamo.reset()
+ x, w, a, b = _small_update()
+ if param:
+ a, b = torch.nn.Parameter(a), torch.nn.Parameter(b)
+ else:
+ a, b = a.requires_grad_(), b.requires_grad_()
+
+ def f(x, a, b):
+ return fast_lora.LoRA_W.apply(x.view(1, 1, 2), w, None, a, b, 1.0)
+
+ got = torch.compile(f, backend = "aot_eager", fullgraph = True)(x, a, b)
+ assert got.item() == (1.0078125 if param else 1.0)
+
+
+class _FakeF8(torch.Tensor):
+ pass
+
+
[email protected]("backend", ["packed", "int4", "float8"])
+def test_a_precomputed_xa_sharing_the_buffer_is_not_reused_by_any_dispatch(backend, monkeypatch):
+ # Packed, INT4 and Float8Tensor bases write the caller's buffer too before the rank-r term reads XA.
+ _set(monkeypatch, True)
+ g = torch.Generator(device = "cpu").manual_seed(3)
+ X, W = torch.randn(3, 4, generator = g), torch.randn(2, 4, generator = g)
+ A, B = torch.randn(2, 4, generator = g), torch.randn(2, 2, generator = g)
+ ref = X @ W.t() + 2.0 * (X @ A.t()) @ B.t()
+ W_quant = None
+ if backend == "packed":
+ W_quant = type(
+ "Packed",
+ (),
+ {
+ "_unsloth_packed_weight_state": True,
+ "dtype": torch.float32,
+ "matmul": lambda self, X, W, out = None: torch.matmul(X, W.t(), out = out),
+ },
+ )()
+ monkeypatch.setattr(kernel_utils, "_is_packed_state", lambda q: q is W_quant)
+ elif backend == "int4":
+ Int4 = type("Int4QuantState", (), {})
+ monkeypatch.setattr(kernel_utils, "Int4QuantState", Int4)
+ monkeypatch.setattr(
+ kernel_utils, "_int4_matmul", lambda X, W, q, out = None: torch.matmul(X, W.t(), out = out)
+ )
+ W_quant = Int4()
+ else:
+ monkeypatch.setattr(kernel_utils, "Float8Tensor", _FakeF8)
+ W = W.as_subclass(_FakeF8)
+ W.block_size = (1, W.shape[1]) # not rowwise-after-transpose: the forward GEMM
+ XA = X @ A.t()
+ got = kernel_utils.matmul_lora(X, W, W_quant, A, B, 2.0, XA = XA, out = XA)
+ torch.testing.assert_close(got.as_subclass(torch.Tensor), ref, rtol = 1e-5, atol = 1e-5)
+
+
[email protected]("on", [True, False])
+def test_a_3d_precomputed_xa_is_flattened_with_x(on, monkeypatch):
+ _set(monkeypatch, on)
+ g = torch.Generator(device = "cpu").manual_seed(5)
+ X, W = torch.randn(2, 3, H, generator = g), torch.randn(I, H, generator = g)
+ A, B = torch.randn(4, H, generator = g), torch.randn(I, 4, generator = g)
+ ref = kernel_utils.matmul_lora(X, W, None, A, B, 2.0)
+ got = kernel_utils.matmul_lora(X, W, None, A, B, 2.0, XA = X @ A.t())
+ assert got.shape == ref.shape
+ torch.testing.assert_close(got, ref, rtol = 1e-5, atol = 1e-5)
+
+
+def test_sparse_base_weight_keeps_the_base_then_lora_order_under_export(monkeypatch):
+ # torch.export traces with is_compiling() True: the layout check must still run there.
+ _set(monkeypatch, True)
+ if not hasattr(torch, "export"):
+ pytest.skip("torch.export is not available")
+
+ class M(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.W = torch.eye(2).to_sparse_coo()
+ self.A, self.B = torch.ones(1, 2), torch.ones(2, 1)
+
+ def forward(self, x):
+ return kernel_utils.matmul_lora(x, self.W, None, self.A, self.B, 1.0)
+
+ x = torch.ones(1, 2)
+ ep = torch.export.export(M(), (x,))
+ torch.testing.assert_close(ep.module()(x), torch.full((1, 2), 3.0))
+
+
+def one_rank_mesh(tmp_path):
+ # One-rank gloo group on a file store (no TCP / libuv rendezvous, no environment variables); only a
+ # group this fixture created is destroyed.
+ if not isinstance(kernel_utils._LORA_DTENSOR, type):
+ pytest.skip("torch.distributed.tensor is not available")
+ import torch.distributed as dist
+
+ if not dist.is_available() or not dist.is_gloo_available():
+ pytest.skip("gloo is not available")
+ from torch.distributed.device_mesh import init_device_mesh
+
+ created = not dist.is_initialized()
+ if created:
+ store = dist.FileStore(str(tmp_path / "store"), 1)
+ dist.init_process_group("gloo", store = store, rank = 0, world_size = 1)
+ try:
+ yield init_device_mesh("cpu", (1,))
+ finally:
+ if created:
+ dist.destroy_process_group()
+
+
+def test_dtensor_base_weight_keeps_the_base_then_lora_order(one_rank_mesh, monkeypatch):
+ # A DTensor carries placements the plain accumulator would drop: keep main's order.
+ from torch.distributed.tensor import Replicate, distribute_tensor
+
+ _set(monkeypatch, True)
+ W = distribute_tensor(torch.randn(4, 3), one_rank_mesh, [Replicate()])
+ assert not kernel_utils._lora_fused_ok(W, None, torch.float32)
+
+
[email protected]("placement", ["replicate", "shard0", "shard1"])
+def test_dtensor_activation_keeps_the_base_then_lora_order(placement, one_rank_mesh, monkeypatch):
+ # A DTensor X with plain W / A / B (implicit replication) keeps its value and placement.
+ from torch.distributed.tensor import Replicate, Shard, distribute_tensor
+ from torch.distributed.tensor.experimental import implicit_replication
+
+ g = torch.Generator(device = "cpu").manual_seed(7)
+ X, W = torch.randn(4, 6, generator = g), torch.randn(5, 6, generator = g)
+ A, B = torch.randn(2, 6, generator = g), torch.randn(5, 2, generator = g)
+ pl = {"replicate": Replicate(), "shard0": Shard(0), "shard1": Shard(1)}[placement]
+ Xd = distribute_tensor(X, one_rank_mesh, [pl])
+ out = {}
+ for on in (False, True):
+ _set(monkeypatch, on)
+ with implicit_replication():
+ try:
+ out[on] = kernel_utils.matmul_lora(Xd, W, None, A, B, 2.0)
+ except NotImplementedError as e:
+ if on:
+ raise
+ pytest.skip(f"main's order needs DTensor ops this torch lacks: {e}"[:200])
+ assert out[True].placements == out[False].placements
+ torch.testing.assert_close(out[True].full_tensor(), out[False].full_tensor())
+
+
+def _dtensor_lora_inputs(mesh, which, placement):
+ from torch.distributed.tensor import Replicate, Shard, distribute_tensor
+
+ g = torch.Generator().manual_seed(0)
+ X, W = torch.randn(2, 4, 4, generator = g), torch.randn(4, 4, generator = g)
+ A, B = torch.randn(2, 4, generator = g), torch.randn(4, 2, generator = g)
+ pl = {"replicate": Replicate(), "shard0": Shard(0), "shard1": Shard(1)}[placement]
+ # Mixed plain / DTensor LoRA weights fail on main as well (in-place gradient writes), so the adapters
+ # are either both plain or both DTensor.
+ X = distribute_tensor(X, mesh, [pl])
+ if which == "XAB":
+ A, B = (distribute_tensor(t, mesh, [Replicate()]) for t in (A, B))
+ return X.requires_grad_(), W, A.requires_grad_(), B.requires_grad_()
+
+
+def _apply_lora_function(name, X, W, A, B):
+ from unsloth.kernels.swiglu import _DWf_DW_dfg_traced
+
+ if name == "W":
+ return (fast_lora.LoRA_W.apply(X, W, None, A, B, 1.0),)
+ if name == "QKV":
+ return fast_lora.LoRA_QKV.apply(
+ X, W, None, A, B, 1.0, W, None, A, B, 1.0, W, None, A, B, 1.0
+ )
+ silu_mul = lambda e, g: torch.nn.functional.silu(e) * g
+ return (
+ fast_lora.LoRA_MLP.apply(
+ X,
+ W,
+ None,
+ A,
+ B,
+ 1.0,
+ W,
+ None,
+ A,
+ B,
+ 1.0,
+ W,
+ None,
+ A,
+ B,
+ 1.0,
+ silu_mul,
+ _DWf_DW_dfg_traced,
+ False,
+ ),
+ )
+
+
+def _run_dtensor_lora(name, which, placement, mesh, monkeypatch, on):
+ # Main's order is the reference: where this torch's DTensor cannot run it (older releases lack the
+ # sharding rules for views of a sharded dim / in-place addmm), there is nothing to compare against.
+ from torch.distributed.tensor.experimental import implicit_replication
+
+ _set(monkeypatch, on)
+ X, W, A, B = _dtensor_lora_inputs(mesh, which, placement)
+ try:
+ with implicit_replication():
+ out = _apply_lora_function(name, X, W, A, B)
+ sum(o.sum() for o in out).backward()
+ except Exception as e: # noqa: BLE001
+ if on:
+ raise
+ pytest.skip(f"main's order needs DTensor ops this torch lacks: {e}"[:200])
+ return out, X.grad
+
+
[email protected]("name", ["W", "QKV", "MLP"])
[email protected]("placement", ["replicate", "shard0", "shard1"])
+def test_dtensor_operands_keep_the_base_then_lora_order(
+ name, placement, one_rank_mesh, monkeypatch
+):
+ # matmul_lora falls back for a DTensor X / A / B, so the decision the backward reads must agree: the
+ # shared and fused backward paths would otherwise redistribute the input gradient.
+ _, ref = _run_dtensor_lora(name, "XAB", placement, one_rank_mesh, monkeypatch, False)
+ out, got = _run_dtensor_lora(name, "XAB", placement, one_rank_mesh, monkeypatch, True)
+ assert out[0].grad_fn.fused is False
+ assert got.placements == ref.placements
+ torch.testing.assert_close(got.full_tensor(), ref.full_tensor())
+
+
[email protected]("name", ["W", "QKV", "MLP"])
+def test_a_dtensor_activation_records_the_base_then_lora_order(name, one_rank_mesh, monkeypatch):
+ # With plain adapters the backward cannot run under DTensor on main either, so only the recorded
+ # forward decision is checked here.
+ from torch.distributed.tensor.experimental import implicit_replication
+
+ X, W, A, B = _dtensor_lora_inputs(one_rank_mesh, "X", "replicate")
+ for on in (False, True):
+ _set(monkeypatch, on)
+ try:
+ with implicit_replication():
+ out = _apply_lora_function(name, X, W, A, B)
+ except Exception as e: # noqa: BLE001
+ if on:
+ raise
+ pytest.skip(f"main's order needs DTensor ops this torch lacks: {e}"[:200])
+ assert out[0].grad_fn.fused is False
+
+
+def test_a_wrapper_subclass_base_keeps_the_base_then_lora_order(monkeypatch):
+ # TorchAO's NF4Tensor reports bf16 over packed storage and implements mm but not addmm_: with no quant
+ # state it must not qualify for the in-place accumulate.
+ nf4 = pytest.importorskip("torchao.quantization.quantize_.workflows.nf4.nf4_tensor")
+ _set(monkeypatch, True)
+ g = torch.Generator().manual_seed(0)
+ W = nf4.to_nf4(torch.randn(128, 128, generator = g).to(torch.bfloat16))
+ X = torch.randn(8, 128, generator = g).to(torch.bfloat16)
+ A = (torch.randn(4, 128, generator = g) * 0.1).to(torch.bfloat16)
+ B = (torch.randn(128, 4, generator = g) * 0.1).to(torch.bfloat16)
+ assert not kernel_utils._lora_fused_ok(W, None, torch.bfloat16)
+ got = kernel_utils.matmul_lora(X, W, None, A, B, 0.5)
+ _set(monkeypatch, False)
+ torch.testing.assert_close(got, kernel_utils.matmul_lora(X, W, None, A, B, 0.5), rtol = 0, atol = 0)
+
+
[email protected]("layout", ["csr", "csc", "bsr"])
+def test_compressed_sparse_lora_a_takes_one_product_per_projection(layout):
+ # A compressed sparse A cannot be concatenated for the shared X @ [A_1; A_2].t(): one product each.
+ if layout == "bsr" and not torch.backends.mkl.is_available():
+ # CPU dense @ block-sparse needs MKL (the ROCm wheels ship without it); main's product fails the same way.
+ pytest.skip("CPU dense @ block-sparse needs MKL")
+ g = torch.Generator().manual_seed(0)
+ X = torch.randn(4, 6, generator = g)
+ dense = [torch.randn(3, 6, generator = g) for _ in range(2)]
+ to = {
+ "csr": lambda t: t.to_sparse_csr(),
+ "csc": lambda t: t.to_sparse_csc(),
+ "bsr": lambda t: t.to_sparse_bsr((3, 3)),
+ }[layout]
+ views = fast_lora._shared_XA(X, [to(t) for t in dense], torch.float32)
+ for v, t in zip(views, dense):
+ v = v.to_dense() if v.layout != torch.strided else v
+ torch.testing.assert_close(v, X @ t.t(), rtol = 1e-5, atol = 1e-5)
+
+
+def test_a_caller_out_sharing_the_quant_state_is_written_after_dequantizing(monkeypatch):
+ # The caller's buffer may share storage with the quant state (absmax): the weight is dequantized from
+ # it before the rank-r term is written into the buffer.
+ _set(monkeypatch, True)
+
+ class State:
+ pass
+
+ def dequant(
+ W,
+ W_quant,
+ use_global_buffer = True,
+ dtype = None,
+ ):
+ return (W * W_quant.absmax.sum()).to(dtype)
+
+ monkeypatch.setattr(kernel_utils, "fast_dequantize", dequant)
+ monkeypatch.setattr(kernel_utils, "_quant_state_dtype", lambda q: torch.float32)
+ g = torch.Generator().manual_seed(0)
+ X = torch.randn(2, 8, generator = g)
+ W = torch.randn(4, 8, generator = g)
+ A, B = torch.randn(2, 8, generator = g), torch.randn(4, 2, generator = g)
+
+ def run(shared):
+ state = State()
+ buf = torch.empty(2, 4)
+ state.absmax = buf.view(-1)[:3] if shared else torch.empty(3)
+ state.absmax.copy_(torch.tensor([0.5, 0.25, 0.25]))
+ return kernel_utils.matmul_lora(X, W, state, A, B, 0.5, out = buf).clone()
+
+ torch.testing.assert_close(run(True), run(False), rtol = 0, atol = 0)
diff --git a/tests/test_tp_native_gloo.py b/tests/test_tp_native_gloo.py
index bae14bf0200..5bb8df5bb9d 100644
--- a/tests/test_tp_native_gloo.py
+++ b/tests/test_tp_native_gloo.py
@@ -1073,8 +1073,15 @@ def test_fused_lora_functions_read_local_shards():
)
assert set(fused["grads"]) == set(plain["grads"]) and plain["grads"]
for name in plain["grads"]:
+ # The fused order sums in a different order than PEFT, so as for O / dX an element that cancels out of
+ # large terms needs an atol at the tensor's scale; a wrong shard would be off by O(1).
+ ref = plain["grads"][name]
torch.testing.assert_close(
- fused["grads"][name], plain["grads"][name], rtol = 1e-4, atol = 1e-6, msg = name
+ fused["grads"][name],
+ ref,
+ rtol = 1e-4,
+ atol = max(1e-6, 1e-5 * ref.abs().max().item()),
+ msg = lambda m, name = name: f"{name}: {m}",
)
diff --git a/unsloth/kernels/fast_lora.py b/unsloth/kernels/fast_lora.py
index 3d85c747e6c..a528bf8ae52 100644
--- a/unsloth/kernels/fast_lora.py
+++ b/unsloth/kernels/fast_lora.py
@@ -14,6 +14,7 @@ import sys
import torch
from unsloth_zoo.utils import Version
from .utils import (
+ _LORA_DTENSOR,
_has_active_dora_adapter,
_has_active_lora_bias,
_has_multiple_active_adapters,
@@ -22,6 +23,7 @@ from .utils import (
_base_quant_state,
get_lora_parameters,
get_lora_parameters_bias,
+ _lora_fused_ok,
matmul_lora,
torch_amp_custom_fwd,
torch_amp_custom_bwd,
@@ -305,7 +307,8 @@ def _matmul_lora_ext(X, X_lora, W, W_quant, A, B, S, bias):
# matmul_lora with the LoRA branch reading X_lora (X after dropout, or X itself) plus lora_B's bias.
# No global dequantization scratch: these configurations ran PEFT, which keeps no dense weight resident.
if A is None or (X_lora is X and bias is None):
- return matmul_lora(X, W, W_quant, A, B, S, use_global_buffer = False)
+ # fused = False: dropout or a lora_bias elsewhere in the module keeps the whole module on one order.
+ return matmul_lora(X, W, W_quant, A, B, S, use_global_buffer = False, fused = False)
out = matmul_lora(X, W, W_quant, None, None, None, use_global_buffer = False)
dtype = X.dtype
XA = torch.matmul(X_lora.reshape(-1, X_lora.shape[-1]), A.to(dtype).t())
@@ -327,6 +330,91 @@ def _add_lora_dX(dX, y_dB, A, S, mask, p):
dX.add_(_dropout_replay(y_dB @ A.t(), mask, p, S))
+def _dense(t):
+ # Dynamo bans reading `layout` in a compiled backward, so traced code checks is_sparse / is_sparse_csr.
+ if t.is_sparse or t.is_sparse_csr:
+ return False
+ return torch.compiler.is_compiling() or t.layout == torch.strided
+
+
+def _shared_XA(X, As, dtype):
+ # One X @ [A_1; A_2; ...].t() instead of one pass over X per projection; per-projection column views.
+ # Compressed sparse adapters cannot be concatenated: one product each, as before.
+ if not all(_dense(A) for A in As):
+ return [torch.matmul(X, A.to(dtype).t()) for A in As]
+ XA = torch.matmul(X, torch.cat([A.to(dtype) for A in As], 0).t())
+ views, start = [], 0
+ for A in As:
+ views.append(XA[:, start : start + A.shape[0]])
+ start += A.shape[0]
+ return views
+
+
+def _shared_lora_backward(X, dYs, As, Bs, S, dX_out):
+ # LoRA grads of projections sharing X and S: X is read once for all d_A and A @ X.t(), and dX starts
+ # as S * dB @ A.t() for the base GEMMs to accumulate onto. Each d_A.t() is a row block of one buffer.
+ A_rows = torch.cat([A.t() for A in As], 0)
+ AX = A_rows @ X.t()
+ d_Bs, start = [], 0
+ for dY, B in zip(dYs, Bs):
+ r = B.shape[0]
+ d_B = torch.empty_like(B)
+ d_B.addmm_(AX[start : start + r], dY, alpha = S, beta = 0)
+ d_Bs.append(d_B)
+ start += r
+ del AX
+ if _is_compiling():
+ # Traced, Inductor plans the buffers; slice writes would only add scatter copies.
+ dB = torch.cat([dY @ B.t() for dY, B in zip(dYs, Bs)], 1)
+ else:
+ dB = dYs[0].new_empty((X.shape[0], A_rows.shape[0]))
+ start = 0
+ for dY, B in zip(dYs, Bs):
+ r = B.shape[0]
+ dB[:, start : start + r].addmm_(dY, B.t(), beta = 0)
+ start += r
+ dA = dB.new_empty((A_rows.shape[0], X.shape[1]))
+ dA.addmm_(dB.t(), X, alpha = S, beta = 0)
+ d_As, start = [], 0
+ for B in Bs:
+ r = B.shape[0]
+ d_As.append(dA[start : start + r].t())
+ start += r
+ del dA
+ if dX_out is None:
+ dX = torch.addmm(dB.new_zeros(1), dB, A_rows, beta = 0, alpha = S)
+ else:
+ dX = dX_out
+ addmm_(dX, dB, A_rows, alpha = S, beta = 0)
+ return d_As, d_Bs, dX
+
+
+def _lora_traced_ok(*lora_weights, X = None):
+ # A DTensor X or LoRA weight keeps the base-then-LoRA order: matmul_lora falls back for it, and the
+ # recorded decision must match so the backward does not take the shared / fused path either.
+ if any(isinstance(t, _LORA_DTENSOR) for t in (X, *lora_weights)):
+ return False
+ # Traced, the fused order needs nn.Parameter LoRA weights: plain tensors of dynamic shape can
+ # recompile with a zero output width on torch 2.14.
+ return not _is_compiling() or all(
+ t is None or isinstance(t, torch.nn.Parameter) for t in lora_weights
+ )
+
+
+def _mlp_fused_ok(gateW, gateW_quant, upW, upW_quant, downW, downW_quant, dtype):
+ # All or none: the forward and backward of one module use one order.
+ return all(
+ _lora_fused_ok(W, Q, dtype)
+ for W, Q in ((gateW, gateW_quant), (upW, upW_quant), (downW, downW_quant))
+ )
+
+
+def _qkv_fused_ok(QW, QW_quant, KW, KW_quant, VW, VW_quant, dtype):
+ return all(
+ _lora_fused_ok(W, Q, dtype) for W, Q in ((QW, QW_quant), (KW, KW_quant), (VW, VW_quant))
+ )
+
+
def _lora_bias_grad(dY, S):
# Autograd casts it to the bias dtype.
return dY.sum(0, dtype = torch.float32).mul_(S)
@@ -415,12 +503,21 @@ class LoRA_MLP(torch.autograd.Function):
plain = dropout_p is None and gateBias is None and upBias is None and downBias is None
if plain:
- e = matmul_lora(X, gateW, gateW_quant, gateA, gateB, gateS)
- g = matmul_lora(X, upW, upW_quant, upA, upB, upS)
+ fused = _mlp_fused_ok(
+ gateW, gateW_quant, upW, upW_quant, downW, downW_quant, dtype
+ ) and _lora_traced_ok(gateA, gateB, upA, upB, downA, downB, X = X)
+ gateXA = upXA = None
+ # Eager only: traced, the shared intermediate fails the block compile (saved output aliasing).
+ if fused and not _is_compiling() and gateA is not None and upA is not None:
+ gateXA, upXA = _shared_XA(X.view(-1, X.shape[-1]), (gateA, upA), dtype)
+ e = matmul_lora(X, gateW, gateW_quant, gateA, gateB, gateS, XA = gateXA, fused = fused)
+ g = matmul_lora(X, upW, upW_quant, upA, upB, upS, XA = upXA, fused = fused)
+ del gateXA, upXA
h = _forward_function(e, g)
- i = matmul_lora(h, downW, downW_quant, downA, downB, downS)
+ i = matmul_lora(h, downW, downW_quant, downA, downB, downS, fused = fused)
masks = (None, None, None)
else:
+ fused = False
# Masks are drawn gate, up, down: the order PEFT's down(act(gate(X)) * up(X)) draws them in.
gate_p, up_p, down_p = dropout_p or (0.0, 0.0, 0.0)
Xd, gate_mask = _dropout_forward(X, gate_p)
@@ -451,6 +548,7 @@ class LoRA_MLP(torch.autograd.Function):
ctx.inplace = inplace and (not _rms_layernorm._SAVE_OUTPUT or _may_overwrite_input(X))
ctx.dropout_p = dropout_p or (0.0, 0.0, 0.0)
ctx.biases = (gateBias is not None, upBias is not None, downBias is not None)
+ ctx.fused = fused
ctx.global_buffer = plain
return i
@@ -499,6 +597,11 @@ class LoRA_MLP(torch.autograd.Function):
downB.t(),
)
+ # Decided on the stored weights, before the FP8 down weight below is converted to a dense one.
+ # The forward's order; a handcrafted or older ctx without `fused` keeps the base-then-LoRA order.
+ fused = getattr(ctx, "fused", False) and _mlp_fused_ok(
+ gateW, gateW_quant, upW, upW_quant, downW, downW_quant, dY.dtype
+ )
if downW.dtype == torch.float8_e4m3fn:
# Dequantize like gate / up: matmul_lora would send the FP8 weight to fp8_linear, quantizing dY to FP8.
downW, downW_quant = fast_dequantize(downW.t(), downW_quant, dtype = dY.dtype), None
@@ -513,6 +616,7 @@ class LoRA_MLP(torch.autograd.Function):
downA,
downS,
use_global_buffer = getattr(ctx, "global_buffer", True),
+ fused = fused,
)
else:
# Only the LoRA branch saw the dropped out h, so only its gradient goes through the mask.
@@ -527,15 +631,58 @@ class LoRA_MLP(torch.autograd.Function):
d_downA = torch.empty_like(downA)
d_downB = torch.empty_like(downB)
- d_gateA = torch.empty_like(gateA)
- d_gateB = torch.empty_like(gateB)
- d_upA = torch.empty_like(upA)
- d_upB = torch.empty_like(upB)
# d_downA = h.t() @ (dY @ downB.t()), d_downB = (downA.t() @ h.t()) @ dY, both scaled by downS.
d_downA.addmm_(h.t(), dY @ downB.t(), alpha = downS, beta = 0)
d_downB.addmm_(downA.t() @ h.t(), dY, alpha = downS, beta = 0)
+ if fused and gateS == upS and df.dtype == de.dtype == dY.dtype:
+ (d_gateA, d_upA), (d_gateB, d_upB), dX = _shared_lora_backward(
+ X,
+ (de, df),
+ (gateA, upA),
+ (gateB, upB),
+ gateS,
+ X if ctx.inplace and not _is_compiling() else None,
+ )
+ upW = fast_dequantize(upW.t(), upW_quant, dtype = df.dtype)
+ addmm_(dX, df, upW.t())
+ del upW
+ gateW = fast_dequantize(gateW.t(), gateW_quant, dtype = de.dtype)
+ addmm_(dX, de, gateW.t())
+ del gateW
+ d_gateBias = d_upBias = None
+ d_downBias = _lora_bias_grad(dY, downS) if ctx.biases[2] else None
+ return (
+ dX.view(batch, seq_len, hd),
+ None,
+ None,
+ d_gateA.t(),
+ d_gateB.t(),
+ None,
+ None,
+ None,
+ d_upA.t(),
+ d_upB.t(),
+ None,
+ None,
+ None,
+ d_downA.t(),
+ d_downB.t(),
+ None,
+ None,
+ None,
+ None,
+ None,
+ d_gateBias,
+ d_upBias,
+ d_downBias,
+ )
+
+ d_gateA = torch.empty_like(gateA)
+ d_gateB = torch.empty_like(gateB)
+ d_upA = torch.empty_like(upA)
+ d_upB = torch.empty_like(upB)
up_dB = df @ upB.t()
gate_dB = de @ gateB.t()
@@ -811,11 +958,26 @@ class LoRA_QKV(torch.autograd.Function):
if X.dim() == 3:
X_for_matmul = X.view(-1, X.shape[-1])
if dropout_p is None and QBias is None and KBias is None and VBias is None:
- Q = matmul_lora(X_for_matmul, QW, QW_quant, QA, QB, QS)
- K = matmul_lora(X_for_matmul, KW, KW_quant, KA, KB, KS)
- V = matmul_lora(X_for_matmul, VW, VW_quant, VA, VB, VS)
+ fused = _qkv_fused_ok(
+ QW, QW_quant, KW, KW_quant, VW, VW_quant, dtype
+ ) and _lora_traced_ok(QA, QB, KA, KB, VA, VB, X = X)
+ QXA = KXA = VXA = None
+ # Eager only, as in LoRA_MLP.
+ if (
+ fused
+ and not _is_compiling()
+ and QA is not None
+ and KA is not None
+ and VA is not None
+ ):
+ QXA, KXA, VXA = _shared_XA(X_for_matmul, (QA, KA, VA), dtype)
+ Q = matmul_lora(X_for_matmul, QW, QW_quant, QA, QB, QS, XA = QXA, fused = fused)
+ K = matmul_lora(X_for_matmul, KW, KW_quant, KA, KB, KS, XA = KXA, fused = fused)
+ V = matmul_lora(X_for_matmul, VW, VW_quant, VA, VB, VS, XA = VXA, fused = fused)
+ del QXA, KXA, VXA
masks = (None, None, None)
else:
+ fused = False
Q_p, K_p, V_p = dropout_p or (0.0, 0.0, 0.0)
Xd, Q_mask = _dropout_forward(X_for_matmul, Q_p)
Q = _matmul_lora_ext(X_for_matmul, Xd, QW, QW_quant, QA, QB, QS, QBias)
@@ -857,6 +1019,7 @@ class LoRA_QKV(torch.autograd.Function):
ctx.inplace = inplace and (not _rms_layernorm._SAVE_OUTPUT or _may_overwrite_input(X))
ctx.dropout_p = dropout_p or (0.0, 0.0, 0.0)
ctx.biases = (QBias is not None, KBias is not None, VBias is not None)
+ ctx.fused = fused
return Q, K, V
@staticmethod
@@ -896,13 +1059,54 @@ class LoRA_QKV(torch.autograd.Function):
QA, QB, KA, KB, VA, VB = QA.t(), QB.t(), KA.t(), KB.t(), VA.t(), VB.t()
# Weight projection LoRA derivatives; see the Unsloth blogpost.
+ if (
+ getattr(ctx, "fused", False)
+ and QS == KS == VS
+ and _qkv_fused_ok(QW, QW_quant, KW, KW_quant, VW, VW_quant, dQ.dtype)
+ and dQ.dtype == dK.dtype == dV.dtype
+ ):
+ (d_QA, d_KA, d_VA), (d_QB, d_KB, d_VB), dX = _shared_lora_backward(
+ X,
+ (dQ, dK, dV),
+ (QA, KA, VA),
+ (QB, KB, VB),
+ QS,
+ X if ctx.inplace and not _is_compiling() else None,
+ )
+ for W_, W_quant_, dY_ in ((QW, QW_quant, dQ), (KW, KW_quant, dK), (VW, VW_quant, dV)):
+ W_ = fast_dequantize(W_.t(), W_quant_, dtype = dY_.dtype)
+ addmm_(dX, dY_, W_.t())
+ del W_
+ return (
+ dX.view(batch, seq_len, hd),
+ None,
+ None,
+ d_QA.t(),
+ d_QB.t(),
+ None,
+ None,
+ None,
+ d_KA.t(),
+ d_KB.t(),
+ None,
+ None,
+ None,
+ d_VA.t(),
+ d_VB.t(),
+ None,
+ None,
+ None,
+ None,
+ None,
+ None,
+ )
+
d_QA = torch.empty_like(QA)
d_QB = torch.empty_like(QB)
d_KA = torch.empty_like(KA)
d_KB = torch.empty_like(KB)
d_VA = torch.empty_like(VA)
d_VB = torch.empty_like(VB)
-
q_dB = dQ @ QB.t()
k_dB = dK @ KB.t()
v_dB = dV @ VB.t()
@@ -1059,8 +1263,10 @@ class LoRA_W(torch.autograd.Function):
bias = None,
):
dtype = X.dtype
+ fused = False
if dropout_p is None and bias is None:
- XW = matmul_lora(X, W, W_quant, A, B, S)
+ fused = _lora_traced_ok(A, B, X = X)
+ XW = matmul_lora(X, W, W_quant, A, B, S, fused = fused)
mask = None
else:
Xd, mask = _dropout_forward(X, dropout_p or 0.0)
@@ -1074,6 +1280,7 @@ class LoRA_W(torch.autograd.Function):
ctx.save_for_backward(A, B, X, mask)
ctx.dropout_p = dropout_p
ctx.has_bias = bias is not None
+ ctx.fused = fused
return XW
@staticmethod
@@ -1105,10 +1312,17 @@ class LoRA_W(torch.autograd.Function):
del Xd
d_bias = _lora_bias_grad(dY, S) if ctx.has_bias else None
- W = fast_dequantize(W.t(), W_quant, dtype = dY.dtype)
- dX = dY @ W.t()
- del W
- _add_lora_dX(dX, y_dB, A, S, mask, ctx.dropout_p)
+ if getattr(ctx, "fused", False) and _lora_fused_ok(W, W_quant, dY.dtype):
+ # dX starts as the rank-r term; the base GEMM accumulates onto it in its epilogue.
+ dX = torch.addmm(y_dB.new_zeros(1), y_dB, A.t(), beta = 0, alpha = S)
+ W = fast_dequantize(W.t(), W_quant, dtype = dY.dtype)
+ addmm_(dX, dY, W.t())
+ del W
+ else:
+ W = fast_dequantize(W.t(), W_quant, dtype = dY.dtype)
+ dX = dY @ W.t()
+ del W
+ _add_lora_dX(dX, y_dB, A, S, mask, ctx.dropout_p)
return dX.view(batch, seq_len, hd), None, None, d_A.t(), d_B.t(), None, None, d_bias
diff --git a/unsloth/kernels/utils.py b/unsloth/kernels/utils.py
index 27c6e372753..75a34542104 100644
--- a/unsloth/kernels/utils.py
+++ b/unsloth/kernels/utils.py
@@ -437,6 +437,21 @@ def _has_active_lora_bias(proj):
return any(adapter in lora_B and lora_B[adapter].bias is not None for adapter in adapters)
+def _has_lora_dropout_module(proj):
+ dropout = getattr(proj, "lora_dropout", None)
+ if not dropout:
+ return False
+ adapters = getattr(proj, "active_adapters", None)
+ if adapters is None:
+ adapters = getattr(proj, "active_adapter", ())
+ if isinstance(adapters, str):
+ adapters = (adapters,)
+ return any(
+ adapter in dropout and type(dropout[adapter]) is not torch.nn.Identity
+ for adapter in adapters
+ )
+
+
def has_mxfp4_base(*projs):
"""A packed MXFP4 base that cannot hand its bytes to the fused LoRA kernels (they would then hold a 16-bit
weight until backward): use PEFT. Bases with ``mxfp4_quant_state`` pass ``weight_packed`` + that state instead."""
@@ -1767,7 +1782,10 @@ def fast_linear_forward(
W, W_quant, lora_A, lora_B, lora_S, bias = get_lora_parameters_bias(proj)
bsz, q_len, in_dim = X.shape
if q_len != 1:
- return matmul_lora(X, W, W_quant, lora_A, lora_B, lora_S)
+ # A configured lora_dropout keeps the base-then-LoRA order, as the fused training kernels do.
+ return matmul_lora(
+ X, W, W_quant, lora_A, lora_B, lora_S, fused = not _has_lora_dropout_module(proj)
+ )
if W_quant is None:
out = torch_matmul(X, W.t(), out = out)
@@ -1845,8 +1863,26 @@ def fast_linear_forward(
return out
+def _functorch_wrapped(*tensors):
+ # vmap / grad transforms wrap tensors; an in-place write from them into a plain buffer is not allowed.
+ check = getattr(torch._C._functorch, "is_functorch_wrapped_tensor", None)
+ return check is not None and any(check(t) for t in tensors)
+
+
+def _lora_first_addmm(X, Wt, XA, Bt, alpha):
+ # alpha * XA @ Bt into a fresh buffer (beta 0 never reads it), then the base GEMM accumulates onto it.
+ if _functorch_wrapped(X, Wt, XA, Bt):
+ out = torch.addmm(XA.new_zeros(1), XA, Bt, beta = 0, alpha = alpha)
+ return torch.addmm(out, X, Wt)
+ out = torch.empty((X.shape[0], Wt.shape[1]), dtype = X.dtype, device = X.device)
+ out.addmm_(XA, Bt, alpha = alpha, beta = 0)
+ out.addmm_(X, Wt)
+ return out
+
+
# Traced, x.addmm_ copies x first; this op stays in place. No derivative: grad off only.
_HAS_ADDMM_OP = hasattr(torch.library, "custom_op")
+_HAS_LORA_FIRST_OP = False
if _HAS_ADDMM_OP:
@torch.library.custom_op("unsloth::addmm_", mutates_args = ["out"])
@@ -1855,6 +1891,55 @@ if _HAS_ADDMM_OP:
) -> None:
out.addmm_(A, B, alpha = alpha, beta = beta)
+ # Traced fused LoRA order as one opaque op, so Inductor does not copy the rank-r term into a new
+ # output before the base GEMM. Returns a new tensor, so a saved output aliases nothing.
+ if torch.ops.unsloth.__dict__.get("_lora_first_addmm_ready", False):
+ _HAS_LORA_FIRST_OP = True
+ else:
+ try:
+
+ @torch.library.custom_op("unsloth::lora_first_addmm", mutates_args = ())
+ def _lora_first_addmm_op(
+ X: torch.Tensor, Wt: torch.Tensor, XA: torch.Tensor, Bt: torch.Tensor, alpha: float
+ ) -> torch.Tensor:
+ return _lora_first_addmm(X, Wt, XA, Bt, alpha)
+
+ @_lora_first_addmm_op.register_fake
+ def _(X, Wt, XA, Bt, alpha):
+ return X.new_empty((X.shape[0], Wt.shape[1]))
+
+ # out = alpha * XA @ Bt + X @ Wt, so a compiled differentiable caller gets the addmm gradients.
+ def _lora_first_addmm_setup(ctx, inputs, output):
+ # Save only what the requested gradients read (a frozen inference-mode base cannot be saved).
+ X, Wt, XA, Bt, alpha = inputs
+ ctx.alpha = alpha
+ need = ctx.needs_input_grad
+ ctx.save_for_backward(
+ X if need[1] else None,
+ Wt if need[0] else None,
+ XA if need[3] else None,
+ Bt if need[2] else None,
+ )
+
+ def _lora_first_addmm_backward(ctx, grad):
+ X, Wt, XA, Bt = ctx.saved_tensors
+ need = ctx.needs_input_grad
+ dX = grad @ Wt.t() if need[0] else None
+ dWt = X.t() @ grad if need[1] else None
+ dXA = (grad @ Bt.t()) * ctx.alpha if need[2] else None
+ dBt = (XA.t() @ grad) * ctx.alpha if need[3] else None
+ return dX, dWt, dXA, dBt, None
+
+ _lora_first_addmm_op.register_autograd(
+ _lora_first_addmm_backward, setup_context = _lora_first_addmm_setup
+ )
+ # Marked only after every registration succeeded, so a re-executed module can trust it.
+ torch.ops.unsloth.__dict__["_lora_first_addmm_ready"] = True
+ _HAS_LORA_FIRST_OP = True
+ except Exception:
+ # An older torch.library, or a registration that failed partway: the functional path below.
+ _HAS_LORA_FIRST_OP = False
+
def addmm_(
out,
@@ -1874,6 +1959,73 @@ def addmm_(
return out.addmm_(A, B, alpha = alpha, beta = beta)
+# The rank-r product is written first and the base GEMM accumulates onto it (cuBLAS reads it in the
+# epilogue of a compute-bound GEMM), instead of a separate read-modify-write pass over the whole output.
+# UNSLOTH_LORA_FUSED_PASSES=0 restores the base-then-LoRA order everywhere.
+LORA_FUSED_PASSES = os.environ.get("UNSLOTH_LORA_FUSED_PASSES", "1") != "0"
+
+try:
+ from torch.distributed.tensor import DTensor as _LORA_DTENSOR
+except Exception:
+ _LORA_DTENSOR = ()
+from torch.utils._python_dispatch import is_traceable_wrapper_subclass as _is_wrapper_subclass
+
+_functorch_active = getattr(torch._C, "_are_functorch_transforms_active", lambda: False)
+_is_dynamo_compiling = getattr(torch.compiler, "is_dynamo_compiling", lambda: False)
+
+
+def _exporting():
+ # Set by torch.export (2.7+), strict included; is_exporting() folds to True under 2.10 / 2.11 Dynamo.
+ # Without the flag (torch 2.6) strict and plain compiles look the same, so assume export.
+ return getattr(torch.compiler, "_is_exporting_flag", True)
+
+
+def _forward_ad_active():
+ # Inside torch.autograd.forward_ad.dual_level(); a plain module global, so Dynamo reads it as a constant.
+ return getattr(torch.autograd.forward_ad, "_current_level", -1) >= 0
+
+
+def _lora_fused_ok(W, W_quant, dtype):
+ # fp16 keeps the old order: a rank-r term rounded on its own can overflow. FP8, INT4 and packed
+ # weights have their own GEMMs. addmm_ is not autocast, so the weight must dequantize to `dtype`.
+ if not LORA_FUSED_PASSES or dtype not in (torch.bfloat16, torch.float32):
+ return False
+ if isinstance(W, Float8Tensor) or W.dtype == torch.float8_e4m3fn:
+ return False
+ # A DTensor's placements would be lost in the plain accumulator.
+ if isinstance(W, _LORA_DTENSOR):
+ return False
+ # A sparse base cannot be the dense operand of addmm_. Traced code avoids `layout` (Dynamo bans it).
+ if W.is_sparse or W.is_sparse_csr:
+ return False
+ if not _is_compiling() and W.layout != torch.strided:
+ return False
+ if type(W_quant) is Int4QuantState or _is_packed_state(W_quant):
+ return False
+ if W_quant is None:
+ # A wrapper subclass (TorchAO NF4Tensor, ...) can report a float dtype over packed storage and only
+ # implement its own mm.
+ if _is_wrapper_subclass(W):
+ return False
+ return W.dtype == dtype
+ return _quant_state_dtype(W_quant) == dtype
+
+
+def _shares_storage(out, *tensors):
+ # True when `out` may alias an operand. Traced (Dynamo, make_fx fake tensors) storage cannot be
+ # inspected, so a caller's buffer is then treated as shared and keeps the base-then-LoRA order.
+ if _is_compiling():
+ return True
+ try:
+ ptr = out.untyped_storage().data_ptr()
+ return any(
+ t is not None and isinstance(t, torch_Tensor) and t.untyped_storage().data_ptr() == ptr
+ for t in tensors
+ )
+ except RuntimeError:
+ return True
+
+
def matmul_lora(
X,
W,
@@ -1883,7 +2035,12 @@ def matmul_lora(
s,
out = None,
use_global_buffer = True,
+ XA = None,
+ fused = True,
):
+ # XA: X @ A.t() already computed by the caller (shared across projections reading the same X).
+ # fused = False: the caller's module needs the base-then-LoRA order (dropout, lora_bias, a weight
+ # converted before this call); _lora_fused_ok decides the rest.
dtype = X.dtype
if X.dim() == 3:
@@ -1892,6 +2049,11 @@ def matmul_lora(
reshape = True
else:
reshape = False
+ if XA is not None and XA.dim() == 3:
+ XA = XA.reshape(-1, XA.shape[-1])
+ # Every branch below writes `out` before the LoRA term reads XA: recompute a caller's XA that shares it.
+ if XA is not None and out is not None and _shares_storage(out, XA):
+ XA = None
if isinstance(W, Float8Tensor):
assert W.ndim == 2
@@ -1908,6 +2070,51 @@ def matmul_lora(
out = _int4_matmul(X, W, W_quant, out = out)
elif _is_packed_state(W_quant):
out = W_quant.matmul(X, W, out = out)
+ elif (
+ A is not None
+ and fused
+ # Compiled inside torch.func transforms or native forward-mode AD: the op has no forward-mode or
+ # batching rule, and a traced beta=0 addmm under jvp crashes, so keep the base-then-LoRA order there.
+ and not (_is_compiling() and (_functorch_active() or _forward_ad_active()))
+ # A DTensor operand's placements would be lost in the plain accumulator (W is checked below).
+ and not any(isinstance(t, _LORA_DTENSOR) for t in (X, A, B, XA))
+ and _lora_fused_ok(W, W_quant, dtype)
+ # A caller's buffer must already be the output and must not share storage with a base operand.
+ and (
+ out is None
+ or (
+ out.shape == (X.shape[0], B.shape[0])
+ and out.dtype == dtype
+ and not _shares_storage(out, W, X, A, B, XA)
+ )
+ )
+ ):
+ if XA is None:
+ XA = torch_matmul(X, A.t().to(dtype))
+ B = B.t().to(dtype)
+ if out is not None:
+ # Eager only: a traced caller buffer cannot be checked for aliasing (_shares_storage). Dequantize
+ # first: the caller's buffer may share storage with the quant state (absmax, code, ...).
+ W = fast_dequantize(W, W_quant, use_global_buffer = use_global_buffer, dtype = dtype)
+ addmm_(out, XA, B, alpha = s, beta = 0)
+ out.addmm_(X, W.t())
+ else:
+ W = fast_dequantize(W, W_quant, use_global_buffer = use_global_buffer, dtype = dtype)
+ if not _is_compiling():
+ out = _lora_first_addmm(X, W.t(), XA, B, s)
+ elif (
+ _HAS_LORA_FIRST_OP
+ and type(s) in (int, float)
+ and _is_dynamo_compiling()
+ and not _exporting()
+ ):
+ out = torch.ops.unsloth.lora_first_addmm(X, W.t(), XA, B, s)
+ else:
+ # Traced without the op: functional, since an in-place accumulate onto a saved buffer breaks the
+ # block compile. Export lands here too, so torch.func works on the exported program.
+ out = torch.addmm(XA.new_zeros(1), XA, B, beta = 0, alpha = s)
+ out = torch.addmm(out, X, W.t())
+ A = None
else:
W = fast_dequantize(W, W_quant, use_global_buffer = use_global_buffer, dtype = dtype)
out = torch_matmul(X, W.t(), out = out)
@@ -1916,7 +2123,8 @@ def matmul_lora(
if A is not None:
A, B = A.t(), B.t()
- XA = torch_matmul(X, A.to(dtype))
+ if XA is None:
+ XA = torch_matmul(X, A.to(dtype))
addmm_(out, XA, B.to(dtype), alpha = s)
return out.view(batch, seq_len, -1) if reshape else out