Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
LoRA: add the rank-r term inside the base GEMM so small updates are not rounded away (#16071)
Daniel Han91989b1a9bda
diff --git a/tests/test_fp8_down_proj_bf16_backward.py b/tests/test_fp8_down_proj_bf16_backward.pyindex 39e9f579231..af6650ca3ec 100644--- a/tests/test_fp8_down_proj_bf16_backward.py+++ b/tests/test_fp8_down_proj_bf16_backward.py@@ -108,5 +108,6 @@ def test_bf16_down_proj_backward_unchanged(fp8_calls):downW = torch.randn(HIDDEN, INTER, generator = torch.Generator().manual_seed(5))DW, dY, downA, downB = _backward(downW, None)assert fp8_calls == []- old = kernel_utils.matmul_lora(dY, downW.t(), None, downB.t(), downA.t(), 2.0)+ # The module holds FP8 gate / up weights, so it keeps the base-then-LoRA order.+ old = kernel_utils.matmul_lora(dY, downW.t(), None, downB.t(), downA.t(), 2.0, fused = False)assert torch.equal(DW, old)diff --git a/tests/test_lora_first_addmm_op.py b/tests/test_lora_first_addmm_op.pynew file mode 100644index 00000000000..0b707b17f4a--- /dev/null+++ b/tests/test_lora_first_addmm_op.py@@ -0,0 +1,419 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""Traced fused LoRA order runs as one opaque op (unsloth::lora_first_addmm): same bits as the eager sequence,+no graph break, and the kill switch keeps the old order."""++import pytest+import torch+import unsloth # noqa: F401+from unsloth.kernels import utils as KU++# Gated on the API, not on the registration result, so a failed registration on a capable torch is a failure.+pytestmark = pytest.mark.skipif(+ not hasattr(torch.library, "custom_op"), reason = "needs torch.library.custom_op"+)+# Compiled code uses the op only when torch.export's flag tells strict export apart (torch 2.7+).+_OP_COMPILED = hasattr(torch.compiler, "_is_exporting_flag")+++def _mk(+ M = 64,+ K = 96,+ N = 80,+ r = 8,+ dtype = torch.bfloat16,+):+ g = torch.Generator().manual_seed(0)+ X = torch.randn(M, K, generator = g).to(dtype)+ W = (torch.randn(N, K, generator = g) * 0.02).to(dtype)+ A = (torch.randn(r, K, generator = g) * 0.1).to(dtype)+ B = (torch.randn(N, r, generator = g) * 2e-3).to(dtype)+ return X, W, A, B+++def test_op_matches_eager_sequence_bitwise():+ X, W, A, B = _mk()+ XA = X @ A.t()+ ref = torch.addmm(XA.new_zeros(1), XA, B.t(), beta = 0, alpha = 0.5)+ ref.addmm_(X, W.t())+ got = torch.ops.unsloth.lora_first_addmm(X, W.t(), XA, B.t(), 0.5)+ assert torch.equal(got, ref)+++def test_opcheck():+ X, W, A, B = _mk(dtype = torch.float32)+ XA = X @ A.t()+ torch.library.opcheck(+ torch.ops.unsloth.lora_first_addmm.default,+ (X, W.t(), XA, B.t(), 0.5),+ test_utils = ("test_schema", "test_faketensor", "test_aot_dispatch_static"),+ )++[email protected]("dtype", [torch.bfloat16, torch.float32])+def test_compiled_matmul_lora_uses_op_fullgraph(dtype):+ X, W, A, B = _mk(dtype = dtype)+ eager = KU.matmul_lora(X, W, None, A, B, 0.5)+ import torch._dynamo++ torch._dynamo.reset()+ seen = []+ orig = torch.ops.unsloth.lora_first_addmm++ def backend(gm, inputs):+ seen.extend(str(n.target) for n in gm.graph.nodes if n.op == "call_function")+ return gm.forward++ f = torch.compile(+ lambda X, W, A, B: KU.matmul_lora(X, W, None, A, B, 0.5), backend = backend, fullgraph = True+ )+ got = f(X, W, A, B)+ assert any("lora_first_addmm" in s for s in seen) == _OP_COMPILED, seen+ assert torch.equal(got, eager)+ f2 = torch.compile(+ lambda X, W, A, B: KU.matmul_lora(X, W, None, A, B, 0.5),+ backend = "aot_eager",+ fullgraph = True,+ )+ assert torch.equal(f2(X, W, A, B), eager)+++def test_kill_switch_keeps_old_order_compiled(monkeypatch):+ X, W, A, B = _mk()+ monkeypatch.setattr(KU, "LORA_FUSED_PASSES", False)+ seen = []++ def backend(gm, inputs):+ seen.extend(str(n.target) for n in gm.graph.nodes if n.op == "call_function")+ return gm.forward++ import torch._dynamo++ torch._dynamo.reset()+ torch.compile(+ lambda X, W, A, B: KU.matmul_lora(X, W, None, A, B, 0.5), backend = backend, fullgraph = True+ )(X, W, A, B)+ assert not any("lora_first_addmm" in s for s in seen)+++def test_without_the_op_traced_falls_back_to_the_functional_order(monkeypatch):+ # Registration failed (or an older torch.library): the traced fused branch keeps the two functional addmms.+ X, W, A, B = _mk(dtype = torch.float32)+ eager = KU.matmul_lora(X, W, None, A, B, 0.5)+ monkeypatch.setattr(KU, "_HAS_LORA_FIRST_OP", False)+ seen = []++ def backend(gm, inputs):+ seen.extend(str(n.target) for n in gm.graph.nodes if n.op == "call_function")+ return gm.forward++ torch._dynamo.reset()+ got = torch.compile(+ lambda X, W, A, B: KU.matmul_lora(X, W, None, A, B, 0.5), backend = backend, fullgraph = True+ )(X, W, A, B)+ assert not any("lora_first_addmm" in s for s in seen)+ torch.testing.assert_close(got, eager, rtol = 1e-6, atol = 1e-6)+++def test_opcheck_autograd():+ X, W, A, B = _mk(dtype = torch.float32)+ XA = X @ A.t()+ args = [t.clone().requires_grad_(True) for t in (X, W.t(), XA, B.t())] + [0.5]+ torch.library.opcheck(+ torch.ops.unsloth.lora_first_addmm.default,+ tuple(args),+ test_utils = ("test_schema", "test_autograd_registration", "test_aot_dispatch_dynamic"),+ )++[email protected]("dtype", [torch.bfloat16, torch.float32])+def test_compiled_differentiable_matmul_lora_matches_eager_grads(dtype):+ # A compiled caller that differentiates through matmul_lora gets the addmm gradients.+ import torch._dynamo+ from functorch.compile import make_boxed_func+ from torch._dynamo.backends.common import aot_autograd++ X, W, A, B = _mk(dtype = dtype)++ def run(fn):+ x, a, b = (t.clone().requires_grad_(True) for t in (X, A, B))+ out = fn(x, W, a, b)+ out.float().square().sum().backward()+ return out.detach(), x.grad, a.grad, b.grad++ f = lambda X, W, A, B: KU.matmul_lora(X, W, None, A, B, 0.5) # noqa: E731+ eager = run(f)+ torch._dynamo.reset()+ seen = []+ boxed = lambda g, i: make_boxed_func(g.forward) # noqa: E731++ def backend(gm, inputs):+ seen.extend(str(n.target) for n in gm.graph.nodes if n.op == "call_function")+ return aot_autograd(fw_compiler = boxed, bw_compiler = boxed)(gm, inputs)++ compiled = run(torch.compile(f, backend = backend, fullgraph = True))+ assert any("lora_first_addmm" in s for s in seen) == _OP_COMPILED, seen+ tol = dict(rtol = 1e-5, atol = 1e-5) if dtype == torch.float32 else dict(rtol = 2e-2, atol = 2e-2)+ assert torch.equal(compiled[0], eager[0])+ for c, e in zip(compiled[1:], eager[1:]):+ torch.testing.assert_close(c, e, **tol)++[email protected]("mapped", ["X", "W"])+def test_eager_helper_under_vmap(mapped):+ X, W, A, B = _mk(dtype = torch.float32)+ Xs, Ws = torch.stack([X, X * 2]), torch.stack([W, W * 2])++ def ref(X, W):+ return X @ W.t() + 0.5 * (X @ A.t()) @ B.t()++ if mapped == "X":+ got = torch.vmap(lambda x: KU.matmul_lora(x, W, None, A, B, 0.5))(Xs)+ exp = torch.stack([ref(x, W) for x in Xs])+ else:+ got = torch.vmap(lambda w: KU.matmul_lora(X, w, None, A, B, 0.5))(Ws)+ exp = torch.stack([ref(X, w) for w in Ws])+ torch.testing.assert_close(got, exp, rtol = 1e-5, atol = 1e-5)+++def _transform_cases():+ def f(x, W, A, B):+ return KU.matmul_lora(x, W, None, A, B, 0.5)++ def jvp(X, W, A, B):+ return torch.func.jvp(lambda x: f(x, W, A, B), (X,), (torch.ones_like(X),))[1]++ def grad_(X, W, A, B):+ return torch.func.grad(lambda x: f(x, W, A, B).float().square().sum())(X)++ def jacrev(X, W, A, B):+ return torch.func.jacrev(lambda x: f(x, W, A, B).float().sum(0))(X)++ def jacfwd(X, W, A, B):+ return torch.func.jacfwd(lambda x: f(x, W, A, B).float().sum(0))(X)++ def vmap_(X, W, A, B):+ return torch.func.vmap(lambda x: f(x, W, A, B))(torch.stack([X, X * 2]))++ return {"jvp": jvp, "grad": grad_, "jacrev": jacrev, "jacfwd": jacfwd, "vmap": vmap_}++[email protected]("backend", ["eager", "aot_eager"])[email protected]("dtype", [torch.bfloat16, torch.float32])[email protected]("name", ["jvp", "grad", "jacrev", "jacfwd", "vmap"])+def test_compiled_function_transforms_match_eager(name, dtype, backend):+ # Compiled inside torch.func transforms, matmul_lora keeps the base-then-LoRA order: the op has no+ # forward-mode or batching rule, so compiled jvp / jacfwd returned zeros or failed to trace.+ import torch._dynamo++ X, W, A, B = _mk(M = 8, K = 16, N = 12, r = 4, dtype = dtype)+ fn = _transform_cases()[name]+ eager = fn(X, W, A, B)+ assert eager.abs().max() > 0+ torch._dynamo.reset()+ got = torch.compile(fn, backend = backend, fullgraph = True)(X, W, A, B)+ tol = dict(rtol = 1e-5, atol = 1e-5) if dtype == torch.float32 else dict(rtol = 2e-2, atol = 2e-2)+ torch.testing.assert_close(got, eager, **tol)+++def test_custom_op_saves_only_what_the_requested_grads_need():+ # Frozen X and a base created under inference_mode (cannot be saved for backward), trainable adapters.+ X, W, A, B = _mk(dtype = torch.float32)+ with torch.inference_mode():+ Wi = W.clone()+ a, b = A.clone().requires_grad_(True), B.clone().requires_grad_(True)+ XA = X @ a.t()+ out = torch.ops.unsloth.lora_first_addmm(X, Wi.t(), XA, b.t(), 0.5)+ out.square().sum().backward()+ a2, b2 = A.clone().requires_grad_(True), B.clone().requires_grad_(True)+ (X @ W.t() + 0.5 * (X @ a2.t()) @ b2.t()).square().sum().backward()+ torch.testing.assert_close(a.grad, a2.grad, rtol = 1e-5, atol = 1e-5)+ torch.testing.assert_close(b.grad, b2.grad, rtol = 1e-5, atol = 1e-5)++[email protected]("backend", ["eager", "aot_eager"])[email protected]("dtype", [torch.bfloat16, torch.float32])+def test_compiled_native_forward_ad_keeps_the_tangent(dtype, backend):+ # Native forward AD (dual_level / make_dual) inside a compiled function: the op has no forward-mode+ # rule, so matmul_lora keeps the base-then-LoRA order and the tangent survives.+ import torch._dynamo+ import torch.autograd.forward_ad as fwAD++ X, W, A, B = _mk(M = 8, K = 16, N = 12, r = 4, dtype = dtype)++ def tangent(x):+ with fwAD.dual_level():+ y = KU.matmul_lora(fwAD.make_dual(x, torch.ones_like(x)), W, None, A, B, 0.5)+ return fwAD.unpack_dual(y).tangent++ eager = tangent(X)+ assert eager is not None and eager.abs().max() > 0+ torch._dynamo.reset()+ got = torch.compile(tangent, backend = backend, fullgraph = True)(X)+ assert got is not None+ tol = dict(rtol = 1e-5, atol = 1e-5) if dtype == torch.float32 else dict(rtol = 2e-2, atol = 2e-2)+ torch.testing.assert_close(got, eager, **tol)++[email protected]("strict", [False, True])+def test_exported_program_stays_transformable(strict):+ # Export (non-strict outside Dynamo, strict through it) keeps the functional form: torch.func on the+ # exported module has no rule for the opaque op (grad raised, jvp returned zeros).+ if not hasattr(torch, "export"):+ pytest.skip("torch.export is not available")++ class M(torch.nn.Module):+ def __init__(self):+ super().__init__()+ # Buffers, not closed-over tensors: strict export on older torch rejects free tensors.+ self.register_buffer("W", torch.eye(3))+ self.register_buffer("A", torch.ones(1, 3))+ self.register_buffer("B", torch.ones(3, 1))++ def forward(self, x):+ return KU.matmul_lora(x, self.W, None, self.A, self.B, 0.5)++ x = torch.ones(2, 3)+ ep = torch.export.export(M(), (x,), strict = strict)+ assert not any("lora_first_addmm" in str(n.target) for n in ep.graph.nodes)+ m = ep.module()+ _, tangent = torch.func.jvp(m, (x,), (torch.ones_like(x),))+ torch.testing.assert_close(tangent, torch.full((2, 3), 2.5))+ g = torch.func.grad(lambda x: m(x).sum())(x)+ torch.testing.assert_close(g, torch.full((2, 3), 2.5))+++def test_compiled_lora_forward_and_backward_have_no_graph_break():+ # A compiled LoRA layer (forward + its backward) traces as one graph and still uses the op. On torch+ # 2.10 / 2.11 Dynamo folds is_exporting() to True, which once reached a banned `layout` read.+ import torch._dynamo+ from functorch.compile import make_boxed_func+ from torch._dynamo.backends.common import aot_autograd+ from unsloth.kernels import fast_lora++ X, W, A, B = _mk(M = 8, K = 16, N = 12, r = 4, dtype = torch.float32)+ a, b = torch.nn.Parameter(A.clone()), torch.nn.Parameter(B.clone())+ seen = []++ def boxed(g, i):+ # The op sits in the flattened forward / backward graphs AOTAutograd hands to the compilers.+ seen.extend(str(n.target) for n in g.graph.nodes if n.op == "call_function")+ return make_boxed_func(g.forward)++ def backend(gm, inputs):+ return aot_autograd(fw_compiler = boxed, bw_compiler = boxed)(gm, inputs)++ def f(x):+ return fast_lora.LoRA_W.apply(x.view(1, 8, 16), W, None, a, b, 0.5)++ torch._dynamo.reset()+ x = X.clone().requires_grad_(True)+ out = torch.compile(f, backend = backend, fullgraph = True)(x)+ out.square().sum().backward()+ assert any("lora_first_addmm" in s for s in seen) == _OP_COMPILED, seen+ assert x.grad is not None and a.grad is not None and b.grad is not None+++def test_exporting_reads_the_export_flag_not_the_folded_query(monkeypatch):+ # torch 2.10 / 2.11 Dynamo folds is_exporting() to True in every compiled region, so plain compiles+ # would lose the op; the flag torch.export sets is exact. Without the flag, assume export.+ if hasattr(torch.compiler, "_is_exporting_flag"):+ monkeypatch.setattr(torch.compiler, "_is_exporting_flag", False)+ assert KU._exporting() is False+ monkeypatch.setattr(torch.compiler, "_is_exporting_flag", True)+ assert KU._exporting() is True+ monkeypatch.delattr(torch.compiler, "_is_exporting_flag")+ assert KU._exporting() is True+++_PARTIAL_REGISTRATION_CASE = """+import torch+from torch._library.custom_ops import CustomOpDef+orig = CustomOpDef.register_autograd+def boom(self, *a, **k):+ if self._qualname == "unsloth::lora_first_addmm":+ raise RuntimeError("injected registration failure")+ return orig(self, *a, **k)+CustomOpDef.register_autograd = boom+import unsloth # noqa: F401+from unsloth.kernels import utils as KU+assert hasattr(torch.ops.unsloth, "lora_first_addmm"), "the schema should still be registered"+print("HAS_OP", KU._HAS_LORA_FIRST_OP)+"""+++def test_a_partly_registered_op_is_not_used():+ # Registration fails after the schema exists (autograd here): the op must stay disabled rather than be+ # enabled because its name resolves. A fresh interpreter, since this process registered it already.+ import os+ import subprocess+ import sys+ from pathlib import Path++ root = Path(__file__).resolve().parents[1]+ env = dict(+ os.environ,+ PYTHONPATH = os.pathsep.join([str(root), os.environ.get("PYTHONPATH", "")]),+ UNSLOTH_DISABLE_CPU_FALLBACK = "0",+ )+ p = subprocess.run(+ [sys.executable, "-c", _PARTIAL_REGISTRATION_CASE],+ cwd = root,+ env = env,+ capture_output = True,+ text = True,+ timeout = 900,+ )+ assert p.returncode == 0 and "HAS_OP False" in p.stdout, (p.stdout + p.stderr)[-3000:]+++_REEXECUTED_MODULE_CASE = """+import importlib+import torch+import unsloth # noqa: F401+from unsloth.kernels import utils as KU+assert KU._HAS_LORA_FIRST_OP is True+calls = []+orig = torch.library.custom_op+def counting(name, *a, **k):+ if name == "unsloth::lora_first_addmm":+ calls.append(name)+ return orig(name, *a, **k)+torch.library.custom_op = counting+KU = importlib.reload(KU)+print("REREGISTERED", len(calls), "HAS_OP", KU._HAS_LORA_FIRST_OP)+"""+++def test_a_reexecuted_module_reuses_the_completed_registration():+ # Re-registering over a working op could leave it half replaced if a later step failed; a re-executed+ # module must reuse the ready op instead of defining it again.+ import os+ import subprocess+ import sys+ from pathlib import Path++ root = Path(__file__).resolve().parents[1]+ env = dict(+ os.environ,+ PYTHONPATH = os.pathsep.join([str(root), os.environ.get("PYTHONPATH", "")]),+ UNSLOTH_DISABLE_CPU_FALLBACK = "0",+ )+ p = subprocess.run(+ [sys.executable, "-c", _REEXECUTED_MODULE_CASE],+ cwd = root,+ env = env,+ capture_output = True,+ text = True,+ timeout = 900,+ )+ assert p.returncode == 0 and "REREGISTERED 0 HAS_OP True" in p.stdout, (p.stdout + p.stderr)[+ -3000:+ ]+++def test_a_completely_registered_op_is_marked_ready():+ # The marker a re-executed module reads is set only once every registration step succeeded.+ assert torch.ops.unsloth.__dict__.get("_lora_first_addmm_ready") is True+ assert KU._HAS_LORA_FIRST_OP is Truediff --git a/tests/test_lora_fused_passes.py b/tests/test_lora_fused_passes.pynew file mode 100644index 00000000000..0504cd4d2b0--- /dev/null+++ b/tests/test_lora_fused_passes.py@@ -0,0 +1,899 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""UNSLOTH_LORA_FUSED_PASSES: the fused LoRA kernels write the rank-r term first and let the base GEMM+accumulate onto it, share X @ A across projections reading the same X, and read X once for every+projection's LoRA gradients. Outputs and gradients match the base-then-LoRA order and an fp64 PEFT+oracle; the shared paths are taken only when they apply."""++from __future__ import annotations++import copy++import pytest+import torch+import unsloth # noqa: F401++from real_accelerator import has_real_cuda+from unsloth.kernels import fast_lora+from unsloth.kernels import utils as kernel_utils++H, I = 64, 96+TARGETS = ["q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj"]+DEVICES = ["cpu"] + (["cuda"] if has_real_cuda() else [])+++class _Block(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.q_proj = torch.nn.Linear(H, H, bias = False)+ self.k_proj = torch.nn.Linear(H, H // 2, bias = False)+ self.v_proj = torch.nn.Linear(H, H // 2, bias = False)+ self.o_proj = torch.nn.Linear(H, H, bias = False)+ self.gate_proj = torch.nn.Linear(H, I, bias = False)+ self.up_proj = torch.nn.Linear(H, I, bias = False)+ self.down_proj = torch.nn.Linear(I, H, bias = False)+ self.act_fn = torch.nn.SiLU()+++def _model(+ device,+ dtype,+ alpha_pattern = None,+ dropout = 0.0,+ lora_bias = False,+):+ from peft import LoraConfig, get_peft_model++ torch.manual_seed(3407)+ cfg = LoraConfig(+ r = 4,+ lora_alpha = 8,+ lora_dropout = dropout,+ target_modules = TARGETS,+ init_lora_weights = False,+ alpha_pattern = alpha_pattern or {},+ lora_bias = lora_bias,+ )+ model = get_peft_model(_Block(), cfg).to(device, dtype)+ for name, p in model.named_parameters():+ p.requires_grad_("lora_" in name)+ if lora_bias:+ with torch.no_grad():+ for name, p in model.named_parameters():+ if "lora_B" in name and name.endswith("bias"):+ p.normal_()+ model.train()+ return model, model.base_model.model+++PATHS = {+ "qkv": (+ lambda b: (lambda X: fast_lora.apply_lora_qkv(b, X)),+ lambda b: (lambda X: (b.q_proj(X), b.k_proj(X), b.v_proj(X))),+ ),+ "o": (lambda b: (lambda X: fast_lora.apply_lora_o(b, X)), lambda b: b.o_proj),+ "mlp": (+ lambda b: (lambda X: fast_lora.apply_lora_mlp_swiglu(b, X)),+ lambda b: (lambda X: b.down_proj(b.act_fn(b.gate_proj(X)) * b.up_proj(X))),+ ),+}+++def _run(fn, model, X0):+ X = X0.detach().clone().requires_grad_()+ out = fn(X * 1) # a non-leaf: the fused backward may write dX into its input buffer+ outs = out if isinstance(out, tuple) else (out,)+ gen = torch.Generator(device = "cpu").manual_seed(7)+ loss = sum(+ (o.double() * torch.randn(o.shape, generator = gen, dtype = torch.float64).to(o.device)).sum()+ for o in outs+ )+ loss.backward()+ grads = {n: p.grad.clone() for n, p in model.named_parameters() if p.grad is not None}+ model.zero_grad(set_to_none = True)+ return [o.detach() for o in outs], X.grad.clone(), grads+++def _skip_without_native_bf16(device, dtype = torch.bfloat16):+ # Same rule as test_lora_gradient_layout: Triton's bf16 kernels need sm_80+, and is_bf16_supported() also+ # reports emulated bf16 on older GPUs.+ if (+ device == "cuda"+ and dtype is torch.bfloat16+ and (+ not torch.cuda.is_bf16_supported()+ or (not torch.version.hip and torch.cuda.get_device_capability()[0] < 8)+ )+ ):+ pytest.skip("bf16 kernels need native bf16 (sm_80+)")+++def _set(monkeypatch, on):+ monkeypatch.setattr(kernel_utils, "LORA_FUSED_PASSES", on)+++def _err(a, b):+ return float((a.double() - b.double()).norm() / b.double().norm().clamp_min(1e-30))++[email protected]("device", DEVICES)[email protected]("dtype", [torch.float32, torch.bfloat16])[email protected]("path", ["qkv", "o", "mlp"])[email protected]("alphas", ["equal", "mixed"])+def test_fused_passes_match_the_unfused_order_and_an_fp64_oracle(+ device, dtype, path, alphas, monkeypatch+):+ if device == "cpu" and path == "mlp":+ pytest.skip("the SwiGLU kernels are Triton")+ if device == "cpu" and dtype is torch.bfloat16:+ pytest.skip("bf16 GEMMs on CPU are slow and not what runs in training")+ _skip_without_native_bf16(device, dtype)+ pattern = {"k_proj": 16, "up_proj": 16} if alphas == "mixed" else None+ model, block = _model(device, dtype, pattern)+ X0 = torch.randn(3, 5, H, device = device, dtype = dtype)+ _set(monkeypatch, False)+ ref_out, ref_dX, ref_g = _run(PATHS[path][0](block), model, X0)+ _set(monkeypatch, True)+ out, dX, g = _run(PATHS[path][0](block), model, X0)+ oracle_model = copy.deepcopy(model).double()+ o_out, o_dX, o_g = _run(+ PATHS[path][1](oracle_model.base_model.model), oracle_model, X0.double()+ )+ tol = 1e-5 if dtype is torch.float32 else 2e-2+ for a, b, o in zip(out, ref_out, o_out):+ assert _err(a, o) <= max(tol, 1.5 * _err(b, o)), (_err(a, o), _err(b, o))+ assert _err(dX, o_dX) <= max(tol, 1.5 * _err(ref_dX, o_dX)), (+ _err(dX, o_dX),+ _err(ref_dX, o_dX),+ )+ assert set(g) == set(ref_g) == set(o_g)+ for n in g:+ assert g[n].shape == ref_g[n].shape+ assert _err(g[n], o_g[n]) <= max(tol, 1.5 * _err(ref_g[n], o_g[n])), (+ n,+ _err(g[n], o_g[n]),+ _err(ref_g[n], o_g[n]),+ )+++def test_shared_paths_engage_only_when_they_apply(monkeypatch):+ calls = {"xa": 0, "bwd": 0}+ real_xa, real_bwd = fast_lora._shared_XA, fast_lora._shared_lora_backward++ def xa(*a, **k):+ calls["xa"] += 1+ return real_xa(*a, **k)++ def bwd(*a, **k):+ calls["bwd"] += 1+ return real_bwd(*a, **k)++ monkeypatch.setattr(fast_lora, "_shared_XA", xa)+ monkeypatch.setattr(fast_lora, "_shared_lora_backward", bwd)+ X0 = torch.randn(2, 3, H)+ for alphas, on, want in (+ ("equal", True, (1, 1)),+ ("mixed", True, (1, 0)),+ ("equal", False, (0, 0)),+ ):+ _set(monkeypatch, on)+ calls.update(xa = 0, bwd = 0)+ model, block = _model("cpu", torch.float32, {"k_proj": 16} if alphas == "mixed" else None)+ _run(PATHS["qkv"][0](block), model, X0)+ assert (calls["xa"], calls["bwd"]) == want, (alphas, on, calls)++[email protected]("device", DEVICES)[email protected]("dtype", [torch.float32, torch.bfloat16])+def test_kill_switch_is_the_base_then_lora_order(device, dtype, monkeypatch):+ # UNSLOTH_LORA_FUSED_PASSES=0: base GEMM, then the rank-r term added in place, bit for bit.+ if device == "cpu" and dtype is torch.bfloat16:+ pytest.skip("bf16 GEMMs on CPU are slow and not what runs in training")+ _skip_without_native_bf16(device, dtype)+ _set(monkeypatch, False)+ g = torch.Generator(device = "cpu").manual_seed(0)+ X, W = torch.randn(7, H, generator = g), torch.randn(I, H, generator = g)+ A, B = torch.randn(4, H, generator = g), torch.randn(I, 4, generator = g)+ X, W, A, B = (t.to(device, dtype) for t in (X, W, A, B))+ want = X @ W.t()+ kernel_utils.addmm_(want, X @ A.t(), B.t(), alpha = 2.0)+ assert torch.equal(kernel_utils.matmul_lora(X, W, None, A, B, 2.0), want)+++def _drop_lora_bias(block, keep):+ # PEFT puts lora_bias on every target; keep it only on `keep` (a down-only bias, for example).+ for name in TARGETS:+ if name not in keep:+ getattr(block, name).lora_B["default"].bias = None++[email protected]("device", DEVICES)[email protected]("dtype", [torch.float16, torch.bfloat16, torch.float32])[email protected]("path", ["qkv", "o", "mlp"])[email protected]("config", ["plain", "dropout", "lora_bias", "down_bias"])+def test_exclusions_keep_the_base_then_lora_order(device, dtype, path, config, monkeypatch):+ # Every configuration the fused order excludes (fp16 at all, dropout, lora_bias anywhere in the+ # module) gives the kill switch's outputs and gradients bit for bit, forward and backward.+ if device == "cpu" and path == "mlp":+ pytest.skip("the SwiGLU kernels are Triton")+ if config == "down_bias" and path != "mlp":+ pytest.skip("a down-only bias is an MLP case")+ if config == "plain" and dtype is not torch.float16:+ pytest.skip("plain bf16 / fp32 take the fused order")+ _skip_without_native_bf16(device, dtype)+ kwargs = {"dropout": 0.1} if config == "dropout" else {}+ kwargs["lora_bias"] = config in ("lora_bias", "down_bias")+ model, block = _model(device, dtype, **kwargs)+ if config == "down_bias":+ _drop_lora_bias(block, keep = ("down_proj",))+ X0 = torch.randn(3, 5, H, device = device, dtype = dtype)+ runs = []+ for on in (False, True):+ _set(monkeypatch, on)+ torch.manual_seed(11)+ runs.append(_run(PATHS[path][0](block), model, X0))+ (out0, dX0, g0), (out1, dX1, g1) = runs+ assert all(torch.equal(a, b) for a, b in zip(out0, out1))+ assert torch.equal(dX0, dX1)+ assert set(g0) == set(g1) and all(torch.equal(g0[n], g1[n]) for n in g0)++[email protected]("device", DEVICES)+def test_fp16_rank_r_term_cannot_overflow_before_the_base_cancels_it(device, monkeypatch):+ # The rank-r term alone is ~70000 (past fp16's 65504) but the output is ~10000. The base-then-LoRA+ # order adds it in the GEMM's fp32 epilogue; writing it first would store inf.+ X = torch.ones(1, H, dtype = torch.float16, device = device)+ W = torch.zeros(I, H, dtype = torch.float16, device = device)+ W[0] = -60000 / H+ A = torch.zeros(4, H, dtype = torch.float16, device = device)+ A[0, 0] = 1+ B = torch.zeros(I, 4, dtype = torch.float16, device = device)+ B[0, 0] = 35000+ outs = {}+ for on in (True, False):+ _set(monkeypatch, on)+ outs[on] = kernel_utils.matmul_lora(X, W, None, A, B, 2.0)+ assert torch.isfinite(outs[True]).all() and abs(float(outs[True][0, 0]) - 10000) <= 32+ assert torch.equal(outs[True], outs[False])+++def test_fused_eligibility_is_one_rule(monkeypatch):+ _set(monkeypatch, True)+ from types import SimpleNamespace++ from unsloth.kernels.int4_packed import Int4QuantState++ ok = kernel_utils._lora_fused_ok+ dense = lambda dtype: torch.empty(4, 4, dtype = dtype) # noqa: E731+ u8 = torch.empty(4, 4, dtype = torch.uint8)+ assert ok(dense(torch.bfloat16), None, torch.bfloat16) and ok(+ dense(torch.float32), None, torch.float32+ )+ assert not ok(dense(torch.float16), None, torch.float16) # fp16 keeps the old order+ assert not ok(dense(torch.float32), None, torch.bfloat16) # fp32 weight under bf16 autocast+ # fast_dequantize returns a 4-bit state's own dtype, not the one asked for.+ assert ok(u8, SimpleNamespace(dtype = torch.bfloat16), torch.bfloat16)+ assert not ok(u8, SimpleNamespace(dtype = torch.float16), torch.bfloat16)+ assert not ok(u8, [None, None, torch.float16], torch.bfloat16)+ assert not ok(dense(torch.float8_e4m3fn), torch.ones(1), torch.bfloat16)+ int4 = object.__new__(Int4QuantState)+ w = torch.empty(8, 4, dtype = torch.int8)+ assert not ok(w, int4, torch.bfloat16) and not ok(+ w.t(), int4, torch.bfloat16+ ) # either orientation+ packed = type("Packed", (), {"_unsloth_packed_weight_state": True, "dtype": torch.bfloat16})()+ assert not ok(u8, packed, torch.bfloat16)+++def test_a_caller_buffer_that_is_not_the_output_keeps_main_behavior(monkeypatch):+ # torch.matmul(..., out = empty) resizes; the fused order writes the rank-r term first, so it is+ # taken only when the buffer is already the output.+ g = torch.Generator(device = "cpu").manual_seed(0)+ X, W = torch.randn(7, H, generator = g), torch.randn(I, H, generator = g)+ A, B = torch.randn(4, H, generator = g), torch.randn(I, 4, generator = g)+ outs = {}+ for on in (True, False):+ _set(monkeypatch, on)+ out = torch.empty(0)+ assert kernel_utils.matmul_lora(X, W, None, A, B, 2.0, out = out) is not None+ outs[on] = out+ assert outs[True].shape == (7, I) and torch.equal(outs[True], outs[False])++[email protected]("device", DEVICES)+def test_fp32_base_under_autocast_keeps_the_base_then_lora_order(device, monkeypatch):+ # addmm_ is not autocast: an fp32 base weight with bf16 activations takes the old order.+ _skip_without_native_bf16(device)+ half = torch.bfloat16+ g = torch.Generator(device = "cpu").manual_seed(0)+ X, W = torch.randn(7, H, generator = g), torch.randn(I, H, generator = g)+ A, B = torch.randn(4, H, generator = g), torch.randn(I, 4, generator = g)+ X, W, A, B = X.to(device, half), W.to(device), A.to(device), B.to(device)+ model, block = _model(device, torch.float32)+ X0 = torch.randn(2, 5, H, generator = g).to(device, half)+ outs, runs = {}, {}+ for on in (True, False):+ _set(monkeypatch, on)+ with torch.autocast(device, dtype = half):+ outs[on] = kernel_utils.matmul_lora(X, W, None, A, B, 2.0)+ runs[on] = _run(PATHS["o"][0](block), model, X0)+ assert torch.equal(outs[True], outs[False])+ (out_on, dX_on, g_on), (out_off, dX_off, g_off) = runs[True], runs[False]+ assert torch.equal(out_on[0], out_off[0]) and torch.equal(dX_on, dX_off)+ assert g_on.keys() == g_off.keys() and all(torch.equal(g_on[k], g_off[k]) for k in g_on)++[email protected](not has_real_cuda(), reason = "needs bitsandbytes 4-bit on a real GPU")+def test_fp16_nf4_state_under_bf16_autocast_keeps_the_base_then_lora_order(monkeypatch):+ # fast_dequantize returns the state's fp16, so the in-place base accumulate would mix dtypes.+ bnb = pytest.importorskip("bitsandbytes")+ _skip_without_native_bf16("cuda")+ lin = bnb.nn.Linear4bit(H, I, bias = False, compute_dtype = torch.float16, quant_type = "nf4")+ lin.weight = bnb.nn.Params4bit(+ torch.randn(I, H, dtype = torch.float16), requires_grad = False, quant_type = "nf4"+ )+ lin = lin.cuda()+ W, Wq = lin.weight, lin.weight.quant_state+ assert Wq.dtype == torch.float16+ g = torch.Generator(device = "cpu").manual_seed(0)+ X = torch.randn(7, H, generator = g).cuda().bfloat16()+ A, B = torch.randn(4, H, generator = g).cuda(), torch.randn(I, 4, generator = g).cuda()+ outs = {}+ for on in (True, False):+ _set(monkeypatch, on)+ with torch.autocast("cuda", dtype = torch.bfloat16):+ outs[on] = kernel_utils.matmul_lora(X, W, Wq, A, B, 2.0)+ assert torch.equal(outs[True], outs[False])+++def test_small_update_survives_the_single_rounding(monkeypatch):+ # bf16 base output 1.0 plus a rank-r term of 1/128 rounds once to 1.0078125; the old order rounds the+ # base output first and the separate add then loses everything below half a bf16 step.+ x = torch.tensor([[1.0, 1.0]], dtype = torch.bfloat16)+ w = torch.tensor([[1.0, 1 / 256]], dtype = torch.bfloat16)+ a = torch.tensor([[1.0, 0.0]], dtype = torch.bfloat16)+ b = torch.tensor([[1 / 512]], dtype = torch.bfloat16)+ exact = (x.double() @ w.double().t() + (x.double() @ a.double().t()) @ b.double().t()).to(+ torch.bfloat16+ )+ _set(monkeypatch, True)+ got = kernel_utils.matmul_lora(x, w, None, a, b, 1.0)+ assert torch.equal(got, exact) and got.item() == 1.0078125+ _set(monkeypatch, False)+ assert kernel_utils.matmul_lora(x, w, None, a, b, 1.0).item() == 1.0+++def test_a_caller_buffer_sharing_an_operand_keeps_main_behavior(monkeypatch):+ # Writing the rank-r term first would overwrite a base operand the GEMM has not read yet.+ _set(monkeypatch, True)+ g = torch.Generator(device = "cpu").manual_seed(0)+ X, W = torch.randn(H, H, generator = g), torch.randn(H, H, generator = g)+ A, B = torch.randn(4, H, generator = g), torch.randn(H, 4, generator = g)+ assert kernel_utils._shares_storage(W, X, W)+ assert not kernel_utils._shares_storage(torch.empty(H, H), X, W, A, B, None)+ # The branch taken is checked, not the values: matmul with out= overlapping an input is undefined on+ # main too (on CPU a 64 x 64 result differs from call to call).+ betas = []+ real = kernel_utils.addmm_+ monkeypatch.setattr(+ kernel_utils, "addmm_", lambda *a, **k: betas.append(k.get("beta", 1)) or real(*a, **k)+ )+ out = W.clone()+ kernel_utils.matmul_lora(X, out, None, A, B, 2.0, out = out)+ assert betas == [1], "a buffer sharing W must take the base-then-LoRA order"+ betas.clear()+ kernel_utils.matmul_lora(X, W, None, A, B, 2.0, out = torch.empty(H, H))+ assert betas == [0], "a separate buffer takes the fused order"+++def test_shared_lora_a_grads_have_the_parameter_layout(monkeypatch):+ # The shared backward hands back d_A whose .t() is contiguous like the [r, K] parameter, so autograd+ # does not copy it into a new layout.+ _set(monkeypatch, True)+ X = torch.randn(10, H)+ As = (torch.randn(4, H).t(), torch.randn(4, H).t())+ Bs = (torch.randn(H, 4).t(), torch.randn(I, 4).t())+ dYs = (torch.randn(10, H), torch.randn(10, I))+ d_As, _, _ = fast_lora._shared_lora_backward(X, dYs, As, Bs, 2.0, None)+ for dA in d_As:+ assert dA.t().is_contiguous(), dA.stride()+++def test_shared_lora_a_grads_are_views_of_one_buffer(monkeypatch):+ # Each d_A is a row block of one [sum r, K] buffer: no per-projection copy into the parameter layout.+ _set(monkeypatch, True)+ X = torch.randn(10, H)+ As = (torch.randn(4, H).t(), torch.randn(3, H).t())+ Bs = (torch.randn(H, 4).t(), torch.randn(I, 3).t())+ dYs = (torch.randn(10, H), torch.randn(10, I))+ d_As, d_Bs, dX = fast_lora._shared_lora_backward(X, dYs, As, Bs, 2.0, None)+ ptrs = {dA.untyped_storage().data_ptr() for dA in d_As}+ assert len(ptrs) == 1, "d_A blocks were copied out of the shared buffer"+ X64 = X.double()+ for dA, dB_, A, B, dY in zip(d_As, d_Bs, As, Bs, dYs):+ y_dB = dY.double() @ B.double().t()+ torch.testing.assert_close(dA.double(), 2.0 * X64.t() @ y_dB, rtol = 1e-4, atol = 1e-4)+ torch.testing.assert_close(+ dB_.double(), 2.0 * (A.double().t() @ X64.t()) @ dY.double(), rtol = 1e-4, atol = 1e-4+ )+++def _small_update(dtype = torch.bfloat16):+ # Base output 1 + 1/256 plus a rank-r term 1/512: the fused order rounds once to 1.0078125, the+ # base-then-LoRA order rounds the base to 1.0 first and then loses the update.+ x = torch.tensor([[1.0, 1.0]], dtype = dtype)+ w = torch.tensor([[1.0, 1 / 256]], dtype = dtype)+ a = torch.tensor([[1.0, 0.0]], dtype = dtype)+ b = torch.tensor([[1 / 512]], dtype = dtype)+ return x, w, a, b+++def test_compiled_caller_buffer_keeps_main_behavior(monkeypatch):+ # Traced, a caller's buffer cannot be checked for aliasing, so it keeps the base-then-LoRA order.+ _set(monkeypatch, True)+ torch._dynamo.reset()+ x, w, a, b = _small_update()++ def f(x, w, a, b):+ out = torch.empty(1, 1, dtype = x.dtype)+ return kernel_utils.matmul_lora(x, w, None, a, b, 1.0, out = out)++ got = torch.compile(f, backend = "aot_eager", fullgraph = True)(x, w, a, b)+ assert got.item() == 1.0+ # A buffer that aliases W: the old order reads W before the output is written.+ g = torch.Generator(device = "cpu").manual_seed(1)+ X, W = torch.randn(4, 4, generator = g), torch.randn(4, 4, generator = g)+ A, B = torch.randn(4, 4, generator = g), torch.randn(4, 4, generator = g)+ ref = X @ W.t() + 2.0 * (X @ A.t()) @ B.t()++ def g_(X, W, A, B):+ out = W.clone()+ return kernel_utils.matmul_lora(X, out, None, A, B, 2.0, out = out)++ torch._dynamo.reset()+ got = torch.compile(g_, backend = "aot_eager", fullgraph = True)(X, W, A, B)+ torch.testing.assert_close(got, ref, rtol = 1e-5, atol = 1e-5)+++def test_fake_tracing_with_a_caller_buffer(monkeypatch):+ # make_fx fake tensors have no data pointer: the buffer is treated as shared, not inspected.+ from torch.fx.experimental.proxy_tensor import make_fx++ _set(monkeypatch, True)+ X, W, A, B = torch.randn(3, H), torch.randn(H, H), torch.randn(4, H), torch.randn(H, 4)++ def f(X, W, A, B, out):+ return kernel_utils.matmul_lora(X, W, None, A, B, 2.0, out = out)++ gm = make_fx(f, tracing_mode = "fake")(X, W, A, B, torch.empty(3, H))+ out = torch.empty(3, H)+ torch.testing.assert_close(+ gm(X, W, A, B, out), X @ W.t() + 2.0 * (X @ A.t()) @ B.t(), rtol = 1e-5, atol = 1e-5+ )++[email protected]("fused", [True, False])+def test_a_precomputed_xa_sharing_the_caller_buffer_is_not_reused(fused, monkeypatch):+ # The base GEMM writes the buffer before the rank-r term reads XA: XA is recomputed instead.+ _set(monkeypatch, True)+ g = torch.Generator(device = "cpu").manual_seed(3)+ X, W = torch.randn(3, 4, generator = g), torch.randn(2, 4, generator = g)+ A, B = torch.randn(2, 4, generator = g), torch.randn(2, 2, generator = g)+ ref = X @ W.t() + 2.0 * (X @ A.t()) @ B.t()+ XA = X @ A.t()+ got = kernel_utils.matmul_lora(X, W, None, A, B, 2.0, XA = XA, out = XA, fused = fused)+ torch.testing.assert_close(got, ref, rtol = 1e-5, atol = 1e-5)+++def test_sparse_base_weight_keeps_the_base_then_lora_order(monkeypatch):+ # addmm_ cannot accumulate a dense X @ sparse W.t() in place; main's matmul handles it.+ _set(monkeypatch, True)+ X, A, B = torch.ones(1, 2), torch.ones(1, 2), torch.ones(2, 1)+ W = torch.eye(2).to_sparse_coo()+ assert not kernel_utils._lora_fused_ok(W, None, torch.float32)+ torch.testing.assert_close(+ kernel_utils.matmul_lora(X, W, None, A, B, 1.0), torch.full((1, 2), 3.0)+ )++[email protected]("dropout", [0.0, 0.1])+def test_multi_token_inference_follows_the_training_order(dropout, monkeypatch):+ # fast_linear_forward with q_len > 1 takes the order the fused training kernels use: a configured+ # lora_dropout (even in eval) keeps base-then-LoRA, a plain adapter takes the fused order.+ _skip_without_native_bf16("cpu")+ from peft import LoraConfig, get_peft_model++ _set(monkeypatch, True)+ lin = torch.nn.Sequential(torch.nn.Linear(2, 1, bias = False))+ model = get_peft_model(+ lin, LoraConfig(r = 1, lora_alpha = 1, lora_dropout = dropout, target_modules = ["0"])+ )+ model = model.to(torch.bfloat16).eval()+ proj = model.base_model.model[0]+ x, w, a, b = _small_update()+ with torch.no_grad():+ proj.base_layer.weight.copy_(w)+ proj.lora_A["default"].weight.copy_(a)+ proj.lora_B["default"].weight.copy_(b)+ X = x.view(1, 1, 2).expand(1, 3, 2).contiguous()+ got = kernel_utils.fast_linear_forward(proj, X)+ assert got[0, 0, 0].item() == (1.0 if dropout else 1.0078125)+++_DYNAMIC_RANK_CASE = """+import sys+import torch+import unsloth # noqa: F401+from unsloth.kernels import fast_lora++torch.manual_seed(0)+weights = []+for n, r in zip((16, 12, 12), (2, 3, 4)):+ W = torch.randn(n, 16) * 0.1+ A, B = torch.randn(r, 16) * 0.1, torch.randn(n, r) * 0.1+ if sys.argv[1] == "param":+ A, B = torch.nn.Parameter(A), torch.nn.Parameter(B)+ else:+ A, B = A.requires_grad_(), B.requires_grad_()+ weights.append((W, A, B))+(QW, QA, QB), (KW, KA, KB), (VW, VA, VB) = weights+++def f(X, QA, QB, KA, KB, VA, VB):+ q, k, v = fast_lora.LoRA_QKV.apply(+ X, QW, None, QA, QB, 2.0, KW, None, KA, KB, 2.0, VW, None, VA, VB, 2.0, False+ )+ return q.sum() + k.sum() + v.sum()+++g = torch.compile(f, backend = "aot_eager", fullgraph = True, dynamic = True)+for seq in (3, 5, 7):+ X = torch.randn(2, seq, 16, requires_grad = True)+ g(X, QA, QB, KA, KB, VA, VB).backward()+ assert X.grad.shape == X.shape+print("OK")+"""++[email protected]("param", [False, True])+def test_dynamic_compile_with_a_rank_equal_to_the_first_length(param):+ # torch 2.14: dynamic-shape plain-tensor LoRA weights once recompiled to a width-0 dQ; they keep the+ # base-then-LoRA order when traced. Fresh interpreter: depends on Dynamo state.+ import os+ import subprocess+ import sys+ from pathlib import Path++ root = Path(__file__).resolve().parents[1]+ env = dict(+ os.environ,+ PYTHONPATH = os.pathsep.join([str(root), os.environ.get("PYTHONPATH", "")]),+ UNSLOTH_DISABLE_CPU_FALLBACK = "0",+ )+ p = subprocess.run(+ [sys.executable, "-c", _DYNAMIC_RANK_CASE, "param" if param else "plain"],+ cwd = root,+ env = env,+ capture_output = True,+ text = True,+ timeout = 900,+ )+ assert p.returncode == 0 and "OK" in p.stdout, (p.stdout + p.stderr)[-3000:]++[email protected]("param", [False, True])+def test_traced_order_depends_on_parameter_lora_weights(param, monkeypatch):+ _skip_without_native_bf16("cpu")+ _set(monkeypatch, True)+ torch._dynamo.reset()+ x, w, a, b = _small_update()+ if param:+ a, b = torch.nn.Parameter(a), torch.nn.Parameter(b)+ else:+ a, b = a.requires_grad_(), b.requires_grad_()++ def f(x, a, b):+ return fast_lora.LoRA_W.apply(x.view(1, 1, 2), w, None, a, b, 1.0)++ got = torch.compile(f, backend = "aot_eager", fullgraph = True)(x, a, b)+ assert got.item() == (1.0078125 if param else 1.0)+++class _FakeF8(torch.Tensor):+ pass++[email protected]("backend", ["packed", "int4", "float8"])+def test_a_precomputed_xa_sharing_the_buffer_is_not_reused_by_any_dispatch(backend, monkeypatch):+ # Packed, INT4 and Float8Tensor bases write the caller's buffer too before the rank-r term reads XA.+ _set(monkeypatch, True)+ g = torch.Generator(device = "cpu").manual_seed(3)+ X, W = torch.randn(3, 4, generator = g), torch.randn(2, 4, generator = g)+ A, B = torch.randn(2, 4, generator = g), torch.randn(2, 2, generator = g)+ ref = X @ W.t() + 2.0 * (X @ A.t()) @ B.t()+ W_quant = None+ if backend == "packed":+ W_quant = type(+ "Packed",+ (),+ {+ "_unsloth_packed_weight_state": True,+ "dtype": torch.float32,+ "matmul": lambda self, X, W, out = None: torch.matmul(X, W.t(), out = out),+ },+ )()+ monkeypatch.setattr(kernel_utils, "_is_packed_state", lambda q: q is W_quant)+ elif backend == "int4":+ Int4 = type("Int4QuantState", (), {})+ monkeypatch.setattr(kernel_utils, "Int4QuantState", Int4)+ monkeypatch.setattr(+ kernel_utils, "_int4_matmul", lambda X, W, q, out = None: torch.matmul(X, W.t(), out = out)+ )+ W_quant = Int4()+ else:+ monkeypatch.setattr(kernel_utils, "Float8Tensor", _FakeF8)+ W = W.as_subclass(_FakeF8)+ W.block_size = (1, W.shape[1]) # not rowwise-after-transpose: the forward GEMM+ XA = X @ A.t()+ got = kernel_utils.matmul_lora(X, W, W_quant, A, B, 2.0, XA = XA, out = XA)+ torch.testing.assert_close(got.as_subclass(torch.Tensor), ref, rtol = 1e-5, atol = 1e-5)++[email protected]("on", [True, False])+def test_a_3d_precomputed_xa_is_flattened_with_x(on, monkeypatch):+ _set(monkeypatch, on)+ g = torch.Generator(device = "cpu").manual_seed(5)+ X, W = torch.randn(2, 3, H, generator = g), torch.randn(I, H, generator = g)+ A, B = torch.randn(4, H, generator = g), torch.randn(I, 4, generator = g)+ ref = kernel_utils.matmul_lora(X, W, None, A, B, 2.0)+ got = kernel_utils.matmul_lora(X, W, None, A, B, 2.0, XA = X @ A.t())+ assert got.shape == ref.shape+ torch.testing.assert_close(got, ref, rtol = 1e-5, atol = 1e-5)+++def test_sparse_base_weight_keeps_the_base_then_lora_order_under_export(monkeypatch):+ # torch.export traces with is_compiling() True: the layout check must still run there.+ _set(monkeypatch, True)+ if not hasattr(torch, "export"):+ pytest.skip("torch.export is not available")++ class M(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.W = torch.eye(2).to_sparse_coo()+ self.A, self.B = torch.ones(1, 2), torch.ones(2, 1)++ def forward(self, x):+ return kernel_utils.matmul_lora(x, self.W, None, self.A, self.B, 1.0)++ x = torch.ones(1, 2)+ ep = torch.export.export(M(), (x,))+ torch.testing.assert_close(ep.module()(x), torch.full((1, 2), 3.0))+++def one_rank_mesh(tmp_path):+ # One-rank gloo group on a file store (no TCP / libuv rendezvous, no environment variables); only a+ # group this fixture created is destroyed.+ if not isinstance(kernel_utils._LORA_DTENSOR, type):+ pytest.skip("torch.distributed.tensor is not available")+ import torch.distributed as dist++ if not dist.is_available() or not dist.is_gloo_available():+ pytest.skip("gloo is not available")+ from torch.distributed.device_mesh import init_device_mesh++ created = not dist.is_initialized()+ if created:+ store = dist.FileStore(str(tmp_path / "store"), 1)+ dist.init_process_group("gloo", store = store, rank = 0, world_size = 1)+ try:+ yield init_device_mesh("cpu", (1,))+ finally:+ if created:+ dist.destroy_process_group()+++def test_dtensor_base_weight_keeps_the_base_then_lora_order(one_rank_mesh, monkeypatch):+ # A DTensor carries placements the plain accumulator would drop: keep main's order.+ from torch.distributed.tensor import Replicate, distribute_tensor++ _set(monkeypatch, True)+ W = distribute_tensor(torch.randn(4, 3), one_rank_mesh, [Replicate()])+ assert not kernel_utils._lora_fused_ok(W, None, torch.float32)++[email protected]("placement", ["replicate", "shard0", "shard1"])+def test_dtensor_activation_keeps_the_base_then_lora_order(placement, one_rank_mesh, monkeypatch):+ # A DTensor X with plain W / A / B (implicit replication) keeps its value and placement.+ from torch.distributed.tensor import Replicate, Shard, distribute_tensor+ from torch.distributed.tensor.experimental import implicit_replication++ g = torch.Generator(device = "cpu").manual_seed(7)+ X, W = torch.randn(4, 6, generator = g), torch.randn(5, 6, generator = g)+ A, B = torch.randn(2, 6, generator = g), torch.randn(5, 2, generator = g)+ pl = {"replicate": Replicate(), "shard0": Shard(0), "shard1": Shard(1)}[placement]+ Xd = distribute_tensor(X, one_rank_mesh, [pl])+ out = {}+ for on in (False, True):+ _set(monkeypatch, on)+ with implicit_replication():+ try:+ out[on] = kernel_utils.matmul_lora(Xd, W, None, A, B, 2.0)+ except NotImplementedError as e:+ if on:+ raise+ pytest.skip(f"main's order needs DTensor ops this torch lacks: {e}"[:200])+ assert out[True].placements == out[False].placements+ torch.testing.assert_close(out[True].full_tensor(), out[False].full_tensor())+++def _dtensor_lora_inputs(mesh, which, placement):+ from torch.distributed.tensor import Replicate, Shard, distribute_tensor++ g = torch.Generator().manual_seed(0)+ X, W = torch.randn(2, 4, 4, generator = g), torch.randn(4, 4, generator = g)+ A, B = torch.randn(2, 4, generator = g), torch.randn(4, 2, generator = g)+ pl = {"replicate": Replicate(), "shard0": Shard(0), "shard1": Shard(1)}[placement]+ # Mixed plain / DTensor LoRA weights fail on main as well (in-place gradient writes), so the adapters+ # are either both plain or both DTensor.+ X = distribute_tensor(X, mesh, [pl])+ if which == "XAB":+ A, B = (distribute_tensor(t, mesh, [Replicate()]) for t in (A, B))+ return X.requires_grad_(), W, A.requires_grad_(), B.requires_grad_()+++def _apply_lora_function(name, X, W, A, B):+ from unsloth.kernels.swiglu import _DWf_DW_dfg_traced++ if name == "W":+ return (fast_lora.LoRA_W.apply(X, W, None, A, B, 1.0),)+ if name == "QKV":+ return fast_lora.LoRA_QKV.apply(+ X, W, None, A, B, 1.0, W, None, A, B, 1.0, W, None, A, B, 1.0+ )+ silu_mul = lambda e, g: torch.nn.functional.silu(e) * g+ return (+ fast_lora.LoRA_MLP.apply(+ X,+ W,+ None,+ A,+ B,+ 1.0,+ W,+ None,+ A,+ B,+ 1.0,+ W,+ None,+ A,+ B,+ 1.0,+ silu_mul,+ _DWf_DW_dfg_traced,+ False,+ ),+ )+++def _run_dtensor_lora(name, which, placement, mesh, monkeypatch, on):+ # Main's order is the reference: where this torch's DTensor cannot run it (older releases lack the+ # sharding rules for views of a sharded dim / in-place addmm), there is nothing to compare against.+ from torch.distributed.tensor.experimental import implicit_replication++ _set(monkeypatch, on)+ X, W, A, B = _dtensor_lora_inputs(mesh, which, placement)+ try:+ with implicit_replication():+ out = _apply_lora_function(name, X, W, A, B)+ sum(o.sum() for o in out).backward()+ except Exception as e: # noqa: BLE001+ if on:+ raise+ pytest.skip(f"main's order needs DTensor ops this torch lacks: {e}"[:200])+ return out, X.grad++[email protected]("name", ["W", "QKV", "MLP"])[email protected]("placement", ["replicate", "shard0", "shard1"])+def test_dtensor_operands_keep_the_base_then_lora_order(+ name, placement, one_rank_mesh, monkeypatch+):+ # matmul_lora falls back for a DTensor X / A / B, so the decision the backward reads must agree: the+ # shared and fused backward paths would otherwise redistribute the input gradient.+ _, ref = _run_dtensor_lora(name, "XAB", placement, one_rank_mesh, monkeypatch, False)+ out, got = _run_dtensor_lora(name, "XAB", placement, one_rank_mesh, monkeypatch, True)+ assert out[0].grad_fn.fused is False+ assert got.placements == ref.placements+ torch.testing.assert_close(got.full_tensor(), ref.full_tensor())++[email protected]("name", ["W", "QKV", "MLP"])+def test_a_dtensor_activation_records_the_base_then_lora_order(name, one_rank_mesh, monkeypatch):+ # With plain adapters the backward cannot run under DTensor on main either, so only the recorded+ # forward decision is checked here.+ from torch.distributed.tensor.experimental import implicit_replication++ X, W, A, B = _dtensor_lora_inputs(one_rank_mesh, "X", "replicate")+ for on in (False, True):+ _set(monkeypatch, on)+ try:+ with implicit_replication():+ out = _apply_lora_function(name, X, W, A, B)+ except Exception as e: # noqa: BLE001+ if on:+ raise+ pytest.skip(f"main's order needs DTensor ops this torch lacks: {e}"[:200])+ assert out[0].grad_fn.fused is False+++def test_a_wrapper_subclass_base_keeps_the_base_then_lora_order(monkeypatch):+ # TorchAO's NF4Tensor reports bf16 over packed storage and implements mm but not addmm_: with no quant+ # state it must not qualify for the in-place accumulate.+ nf4 = pytest.importorskip("torchao.quantization.quantize_.workflows.nf4.nf4_tensor")+ _set(monkeypatch, True)+ g = torch.Generator().manual_seed(0)+ W = nf4.to_nf4(torch.randn(128, 128, generator = g).to(torch.bfloat16))+ X = torch.randn(8, 128, generator = g).to(torch.bfloat16)+ A = (torch.randn(4, 128, generator = g) * 0.1).to(torch.bfloat16)+ B = (torch.randn(128, 4, generator = g) * 0.1).to(torch.bfloat16)+ assert not kernel_utils._lora_fused_ok(W, None, torch.bfloat16)+ got = kernel_utils.matmul_lora(X, W, None, A, B, 0.5)+ _set(monkeypatch, False)+ torch.testing.assert_close(got, kernel_utils.matmul_lora(X, W, None, A, B, 0.5), rtol = 0, atol = 0)++[email protected]("layout", ["csr", "csc", "bsr"])+def test_compressed_sparse_lora_a_takes_one_product_per_projection(layout):+ # A compressed sparse A cannot be concatenated for the shared X @ [A_1; A_2].t(): one product each.+ if layout == "bsr" and not torch.backends.mkl.is_available():+ # CPU dense @ block-sparse needs MKL (the ROCm wheels ship without it); main's product fails the same way.+ pytest.skip("CPU dense @ block-sparse needs MKL")+ g = torch.Generator().manual_seed(0)+ X = torch.randn(4, 6, generator = g)+ dense = [torch.randn(3, 6, generator = g) for _ in range(2)]+ to = {+ "csr": lambda t: t.to_sparse_csr(),+ "csc": lambda t: t.to_sparse_csc(),+ "bsr": lambda t: t.to_sparse_bsr((3, 3)),+ }[layout]+ views = fast_lora._shared_XA(X, [to(t) for t in dense], torch.float32)+ for v, t in zip(views, dense):+ v = v.to_dense() if v.layout != torch.strided else v+ torch.testing.assert_close(v, X @ t.t(), rtol = 1e-5, atol = 1e-5)+++def test_a_caller_out_sharing_the_quant_state_is_written_after_dequantizing(monkeypatch):+ # The caller's buffer may share storage with the quant state (absmax): the weight is dequantized from+ # it before the rank-r term is written into the buffer.+ _set(monkeypatch, True)++ class State:+ pass++ def dequant(+ W,+ W_quant,+ use_global_buffer = True,+ dtype = None,+ ):+ return (W * W_quant.absmax.sum()).to(dtype)++ monkeypatch.setattr(kernel_utils, "fast_dequantize", dequant)+ monkeypatch.setattr(kernel_utils, "_quant_state_dtype", lambda q: torch.float32)+ g = torch.Generator().manual_seed(0)+ X = torch.randn(2, 8, generator = g)+ W = torch.randn(4, 8, generator = g)+ A, B = torch.randn(2, 8, generator = g), torch.randn(4, 2, generator = g)++ def run(shared):+ state = State()+ buf = torch.empty(2, 4)+ state.absmax = buf.view(-1)[:3] if shared else torch.empty(3)+ state.absmax.copy_(torch.tensor([0.5, 0.25, 0.25]))+ return kernel_utils.matmul_lora(X, W, state, A, B, 0.5, out = buf).clone()++ torch.testing.assert_close(run(True), run(False), rtol = 0, atol = 0)diff --git a/tests/test_tp_native_gloo.py b/tests/test_tp_native_gloo.pyindex bae14bf0200..5bb8df5bb9d 100644--- a/tests/test_tp_native_gloo.py+++ b/tests/test_tp_native_gloo.py@@ -1073,8 +1073,15 @@ def test_fused_lora_functions_read_local_shards():)assert set(fused["grads"]) == set(plain["grads"]) and plain["grads"]for name in plain["grads"]:+ # The fused order sums in a different order than PEFT, so as for O / dX an element that cancels out of+ # large terms needs an atol at the tensor's scale; a wrong shard would be off by O(1).+ ref = plain["grads"][name]torch.testing.assert_close(- fused["grads"][name], plain["grads"][name], rtol = 1e-4, atol = 1e-6, msg = name+ fused["grads"][name],+ ref,+ rtol = 1e-4,+ atol = max(1e-6, 1e-5 * ref.abs().max().item()),+ msg = lambda m, name = name: f"{name}: {m}",)diff --git a/unsloth/kernels/fast_lora.py b/unsloth/kernels/fast_lora.pyindex 3d85c747e6c..a528bf8ae52 100644--- a/unsloth/kernels/fast_lora.py+++ b/unsloth/kernels/fast_lora.py@@ -14,6 +14,7 @@ import sysimport torchfrom unsloth_zoo.utils import Versionfrom .utils import (+ _LORA_DTENSOR,_has_active_dora_adapter,_has_active_lora_bias,_has_multiple_active_adapters,@@ -22,6 +23,7 @@ from .utils import (_base_quant_state,get_lora_parameters,get_lora_parameters_bias,+ _lora_fused_ok,matmul_lora,torch_amp_custom_fwd,torch_amp_custom_bwd,@@ -305,7 +307,8 @@ def _matmul_lora_ext(X, X_lora, W, W_quant, A, B, S, bias):# matmul_lora with the LoRA branch reading X_lora (X after dropout, or X itself) plus lora_B's bias.# No global dequantization scratch: these configurations ran PEFT, which keeps no dense weight resident.if A is None or (X_lora is X and bias is None):- return matmul_lora(X, W, W_quant, A, B, S, use_global_buffer = False)+ # fused = False: dropout or a lora_bias elsewhere in the module keeps the whole module on one order.+ return matmul_lora(X, W, W_quant, A, B, S, use_global_buffer = False, fused = False)out = matmul_lora(X, W, W_quant, None, None, None, use_global_buffer = False)dtype = X.dtypeXA = torch.matmul(X_lora.reshape(-1, X_lora.shape[-1]), A.to(dtype).t())@@ -327,6 +330,91 @@ def _add_lora_dX(dX, y_dB, A, S, mask, p):dX.add_(_dropout_replay(y_dB @ A.t(), mask, p, S))+def _dense(t):+ # Dynamo bans reading `layout` in a compiled backward, so traced code checks is_sparse / is_sparse_csr.+ if t.is_sparse or t.is_sparse_csr:+ return False+ return torch.compiler.is_compiling() or t.layout == torch.strided+++def _shared_XA(X, As, dtype):+ # One X @ [A_1; A_2; ...].t() instead of one pass over X per projection; per-projection column views.+ # Compressed sparse adapters cannot be concatenated: one product each, as before.+ if not all(_dense(A) for A in As):+ return [torch.matmul(X, A.to(dtype).t()) for A in As]+ XA = torch.matmul(X, torch.cat([A.to(dtype) for A in As], 0).t())+ views, start = [], 0+ for A in As:+ views.append(XA[:, start : start + A.shape[0]])+ start += A.shape[0]+ return views+++def _shared_lora_backward(X, dYs, As, Bs, S, dX_out):+ # LoRA grads of projections sharing X and S: X is read once for all d_A and A @ X.t(), and dX starts+ # as S * dB @ A.t() for the base GEMMs to accumulate onto. Each d_A.t() is a row block of one buffer.+ A_rows = torch.cat([A.t() for A in As], 0)+ AX = A_rows @ X.t()+ d_Bs, start = [], 0+ for dY, B in zip(dYs, Bs):+ r = B.shape[0]+ d_B = torch.empty_like(B)+ d_B.addmm_(AX[start : start + r], dY, alpha = S, beta = 0)+ d_Bs.append(d_B)+ start += r+ del AX+ if _is_compiling():+ # Traced, Inductor plans the buffers; slice writes would only add scatter copies.+ dB = torch.cat([dY @ B.t() for dY, B in zip(dYs, Bs)], 1)+ else:+ dB = dYs[0].new_empty((X.shape[0], A_rows.shape[0]))+ start = 0+ for dY, B in zip(dYs, Bs):+ r = B.shape[0]+ dB[:, start : start + r].addmm_(dY, B.t(), beta = 0)+ start += r+ dA = dB.new_empty((A_rows.shape[0], X.shape[1]))+ dA.addmm_(dB.t(), X, alpha = S, beta = 0)+ d_As, start = [], 0+ for B in Bs:+ r = B.shape[0]+ d_As.append(dA[start : start + r].t())+ start += r+ del dA+ if dX_out is None:+ dX = torch.addmm(dB.new_zeros(1), dB, A_rows, beta = 0, alpha = S)+ else:+ dX = dX_out+ addmm_(dX, dB, A_rows, alpha = S, beta = 0)+ return d_As, d_Bs, dX+++def _lora_traced_ok(*lora_weights, X = None):+ # A DTensor X or LoRA weight keeps the base-then-LoRA order: matmul_lora falls back for it, and the+ # recorded decision must match so the backward does not take the shared / fused path either.+ if any(isinstance(t, _LORA_DTENSOR) for t in (X, *lora_weights)):+ return False+ # Traced, the fused order needs nn.Parameter LoRA weights: plain tensors of dynamic shape can+ # recompile with a zero output width on torch 2.14.+ return not _is_compiling() or all(+ t is None or isinstance(t, torch.nn.Parameter) for t in lora_weights+ )+++def _mlp_fused_ok(gateW, gateW_quant, upW, upW_quant, downW, downW_quant, dtype):+ # All or none: the forward and backward of one module use one order.+ return all(+ _lora_fused_ok(W, Q, dtype)+ for W, Q in ((gateW, gateW_quant), (upW, upW_quant), (downW, downW_quant))+ )+++def _qkv_fused_ok(QW, QW_quant, KW, KW_quant, VW, VW_quant, dtype):+ return all(+ _lora_fused_ok(W, Q, dtype) for W, Q in ((QW, QW_quant), (KW, KW_quant), (VW, VW_quant))+ )++def _lora_bias_grad(dY, S):# Autograd casts it to the bias dtype.return dY.sum(0, dtype = torch.float32).mul_(S)@@ -415,12 +503,21 @@ class LoRA_MLP(torch.autograd.Function):plain = dropout_p is None and gateBias is None and upBias is None and downBias is Noneif plain:- e = matmul_lora(X, gateW, gateW_quant, gateA, gateB, gateS)- g = matmul_lora(X, upW, upW_quant, upA, upB, upS)+ fused = _mlp_fused_ok(+ gateW, gateW_quant, upW, upW_quant, downW, downW_quant, dtype+ ) and _lora_traced_ok(gateA, gateB, upA, upB, downA, downB, X = X)+ gateXA = upXA = None+ # Eager only: traced, the shared intermediate fails the block compile (saved output aliasing).+ if fused and not _is_compiling() and gateA is not None and upA is not None:+ gateXA, upXA = _shared_XA(X.view(-1, X.shape[-1]), (gateA, upA), dtype)+ e = matmul_lora(X, gateW, gateW_quant, gateA, gateB, gateS, XA = gateXA, fused = fused)+ g = matmul_lora(X, upW, upW_quant, upA, upB, upS, XA = upXA, fused = fused)+ del gateXA, upXAh = _forward_function(e, g)- i = matmul_lora(h, downW, downW_quant, downA, downB, downS)+ i = matmul_lora(h, downW, downW_quant, downA, downB, downS, fused = fused)masks = (None, None, None)else:+ fused = False# Masks are drawn gate, up, down: the order PEFT's down(act(gate(X)) * up(X)) draws them in.gate_p, up_p, down_p = dropout_p or (0.0, 0.0, 0.0)Xd, gate_mask = _dropout_forward(X, gate_p)@@ -451,6 +548,7 @@ class LoRA_MLP(torch.autograd.Function):ctx.inplace = inplace and (not _rms_layernorm._SAVE_OUTPUT or _may_overwrite_input(X))ctx.dropout_p = dropout_p or (0.0, 0.0, 0.0)ctx.biases = (gateBias is not None, upBias is not None, downBias is not None)+ ctx.fused = fusedctx.global_buffer = plainreturn i@@ -499,6 +597,11 @@ class LoRA_MLP(torch.autograd.Function):downB.t(),)+ # Decided on the stored weights, before the FP8 down weight below is converted to a dense one.+ # The forward's order; a handcrafted or older ctx without `fused` keeps the base-then-LoRA order.+ fused = getattr(ctx, "fused", False) and _mlp_fused_ok(+ gateW, gateW_quant, upW, upW_quant, downW, downW_quant, dY.dtype+ )if downW.dtype == torch.float8_e4m3fn:# Dequantize like gate / up: matmul_lora would send the FP8 weight to fp8_linear, quantizing dY to FP8.downW, downW_quant = fast_dequantize(downW.t(), downW_quant, dtype = dY.dtype), None@@ -513,6 +616,7 @@ class LoRA_MLP(torch.autograd.Function):downA,downS,use_global_buffer = getattr(ctx, "global_buffer", True),+ fused = fused,)else:# Only the LoRA branch saw the dropped out h, so only its gradient goes through the mask.@@ -527,15 +631,58 @@ class LoRA_MLP(torch.autograd.Function):d_downA = torch.empty_like(downA)d_downB = torch.empty_like(downB)- d_gateA = torch.empty_like(gateA)- d_gateB = torch.empty_like(gateB)- d_upA = torch.empty_like(upA)- d_upB = torch.empty_like(upB)# d_downA = h.t() @ (dY @ downB.t()), d_downB = (downA.t() @ h.t()) @ dY, both scaled by downS.d_downA.addmm_(h.t(), dY @ downB.t(), alpha = downS, beta = 0)d_downB.addmm_(downA.t() @ h.t(), dY, alpha = downS, beta = 0)+ if fused and gateS == upS and df.dtype == de.dtype == dY.dtype:+ (d_gateA, d_upA), (d_gateB, d_upB), dX = _shared_lora_backward(+ X,+ (de, df),+ (gateA, upA),+ (gateB, upB),+ gateS,+ X if ctx.inplace and not _is_compiling() else None,+ )+ upW = fast_dequantize(upW.t(), upW_quant, dtype = df.dtype)+ addmm_(dX, df, upW.t())+ del upW+ gateW = fast_dequantize(gateW.t(), gateW_quant, dtype = de.dtype)+ addmm_(dX, de, gateW.t())+ del gateW+ d_gateBias = d_upBias = None+ d_downBias = _lora_bias_grad(dY, downS) if ctx.biases[2] else None+ return (+ dX.view(batch, seq_len, hd),+ None,+ None,+ d_gateA.t(),+ d_gateB.t(),+ None,+ None,+ None,+ d_upA.t(),+ d_upB.t(),+ None,+ None,+ None,+ d_downA.t(),+ d_downB.t(),+ None,+ None,+ None,+ None,+ None,+ d_gateBias,+ d_upBias,+ d_downBias,+ )++ d_gateA = torch.empty_like(gateA)+ d_gateB = torch.empty_like(gateB)+ d_upA = torch.empty_like(upA)+ d_upB = torch.empty_like(upB)up_dB = df @ upB.t()gate_dB = de @ gateB.t()@@ -811,11 +958,26 @@ class LoRA_QKV(torch.autograd.Function):if X.dim() == 3:X_for_matmul = X.view(-1, X.shape[-1])if dropout_p is None and QBias is None and KBias is None and VBias is None:- Q = matmul_lora(X_for_matmul, QW, QW_quant, QA, QB, QS)- K = matmul_lora(X_for_matmul, KW, KW_quant, KA, KB, KS)- V = matmul_lora(X_for_matmul, VW, VW_quant, VA, VB, VS)+ fused = _qkv_fused_ok(+ QW, QW_quant, KW, KW_quant, VW, VW_quant, dtype+ ) and _lora_traced_ok(QA, QB, KA, KB, VA, VB, X = X)+ QXA = KXA = VXA = None+ # Eager only, as in LoRA_MLP.+ if (+ fused+ and not _is_compiling()+ and QA is not None+ and KA is not None+ and VA is not None+ ):+ QXA, KXA, VXA = _shared_XA(X_for_matmul, (QA, KA, VA), dtype)+ Q = matmul_lora(X_for_matmul, QW, QW_quant, QA, QB, QS, XA = QXA, fused = fused)+ K = matmul_lora(X_for_matmul, KW, KW_quant, KA, KB, KS, XA = KXA, fused = fused)+ V = matmul_lora(X_for_matmul, VW, VW_quant, VA, VB, VS, XA = VXA, fused = fused)+ del QXA, KXA, VXAmasks = (None, None, None)else:+ fused = FalseQ_p, K_p, V_p = dropout_p or (0.0, 0.0, 0.0)Xd, Q_mask = _dropout_forward(X_for_matmul, Q_p)Q = _matmul_lora_ext(X_for_matmul, Xd, QW, QW_quant, QA, QB, QS, QBias)@@ -857,6 +1019,7 @@ class LoRA_QKV(torch.autograd.Function):ctx.inplace = inplace and (not _rms_layernorm._SAVE_OUTPUT or _may_overwrite_input(X))ctx.dropout_p = dropout_p or (0.0, 0.0, 0.0)ctx.biases = (QBias is not None, KBias is not None, VBias is not None)+ ctx.fused = fusedreturn Q, K, V@staticmethod@@ -896,13 +1059,54 @@ class LoRA_QKV(torch.autograd.Function):QA, QB, KA, KB, VA, VB = QA.t(), QB.t(), KA.t(), KB.t(), VA.t(), VB.t()# Weight projection LoRA derivatives; see the Unsloth blogpost.+ if (+ getattr(ctx, "fused", False)+ and QS == KS == VS+ and _qkv_fused_ok(QW, QW_quant, KW, KW_quant, VW, VW_quant, dQ.dtype)+ and dQ.dtype == dK.dtype == dV.dtype+ ):+ (d_QA, d_KA, d_VA), (d_QB, d_KB, d_VB), dX = _shared_lora_backward(+ X,+ (dQ, dK, dV),+ (QA, KA, VA),+ (QB, KB, VB),+ QS,+ X if ctx.inplace and not _is_compiling() else None,+ )+ for W_, W_quant_, dY_ in ((QW, QW_quant, dQ), (KW, KW_quant, dK), (VW, VW_quant, dV)):+ W_ = fast_dequantize(W_.t(), W_quant_, dtype = dY_.dtype)+ addmm_(dX, dY_, W_.t())+ del W_+ return (+ dX.view(batch, seq_len, hd),+ None,+ None,+ d_QA.t(),+ d_QB.t(),+ None,+ None,+ None,+ d_KA.t(),+ d_KB.t(),+ None,+ None,+ None,+ d_VA.t(),+ d_VB.t(),+ None,+ None,+ None,+ None,+ None,+ None,+ )+d_QA = torch.empty_like(QA)d_QB = torch.empty_like(QB)d_KA = torch.empty_like(KA)d_KB = torch.empty_like(KB)d_VA = torch.empty_like(VA)d_VB = torch.empty_like(VB)-q_dB = dQ @ QB.t()k_dB = dK @ KB.t()v_dB = dV @ VB.t()@@ -1059,8 +1263,10 @@ class LoRA_W(torch.autograd.Function):bias = None,):dtype = X.dtype+ fused = Falseif dropout_p is None and bias is None:- XW = matmul_lora(X, W, W_quant, A, B, S)+ fused = _lora_traced_ok(A, B, X = X)+ XW = matmul_lora(X, W, W_quant, A, B, S, fused = fused)mask = Noneelse:Xd, mask = _dropout_forward(X, dropout_p or 0.0)@@ -1074,6 +1280,7 @@ class LoRA_W(torch.autograd.Function):ctx.save_for_backward(A, B, X, mask)ctx.dropout_p = dropout_pctx.has_bias = bias is not None+ ctx.fused = fusedreturn XW@staticmethod@@ -1105,10 +1312,17 @@ class LoRA_W(torch.autograd.Function):del Xdd_bias = _lora_bias_grad(dY, S) if ctx.has_bias else None- W = fast_dequantize(W.t(), W_quant, dtype = dY.dtype)- dX = dY @ W.t()- del W- _add_lora_dX(dX, y_dB, A, S, mask, ctx.dropout_p)+ if getattr(ctx, "fused", False) and _lora_fused_ok(W, W_quant, dY.dtype):+ # dX starts as the rank-r term; the base GEMM accumulates onto it in its epilogue.+ dX = torch.addmm(y_dB.new_zeros(1), y_dB, A.t(), beta = 0, alpha = S)+ W = fast_dequantize(W.t(), W_quant, dtype = dY.dtype)+ addmm_(dX, dY, W.t())+ del W+ else:+ W = fast_dequantize(W.t(), W_quant, dtype = dY.dtype)+ dX = dY @ W.t()+ del W+ _add_lora_dX(dX, y_dB, A, S, mask, ctx.dropout_p)return dX.view(batch, seq_len, hd), None, None, d_A.t(), d_B.t(), None, None, d_biasdiff --git a/unsloth/kernels/utils.py b/unsloth/kernels/utils.pyindex 27c6e372753..75a34542104 100644--- a/unsloth/kernels/utils.py+++ b/unsloth/kernels/utils.py@@ -437,6 +437,21 @@ def _has_active_lora_bias(proj):return any(adapter in lora_B and lora_B[adapter].bias is not None for adapter in adapters)+def _has_lora_dropout_module(proj):+ dropout = getattr(proj, "lora_dropout", None)+ if not dropout:+ return False+ adapters = getattr(proj, "active_adapters", None)+ if adapters is None:+ adapters = getattr(proj, "active_adapter", ())+ if isinstance(adapters, str):+ adapters = (adapters,)+ return any(+ adapter in dropout and type(dropout[adapter]) is not torch.nn.Identity+ for adapter in adapters+ )++def has_mxfp4_base(*projs):"""A packed MXFP4 base that cannot hand its bytes to the fused LoRA kernels (they would then hold a 16-bitweight until backward): use PEFT. Bases with ``mxfp4_quant_state`` pass ``weight_packed`` + that state instead."""@@ -1767,7 +1782,10 @@ def fast_linear_forward(W, W_quant, lora_A, lora_B, lora_S, bias = get_lora_parameters_bias(proj)bsz, q_len, in_dim = X.shapeif q_len != 1:- return matmul_lora(X, W, W_quant, lora_A, lora_B, lora_S)+ # A configured lora_dropout keeps the base-then-LoRA order, as the fused training kernels do.+ return matmul_lora(+ X, W, W_quant, lora_A, lora_B, lora_S, fused = not _has_lora_dropout_module(proj)+ )if W_quant is None:out = torch_matmul(X, W.t(), out = out)@@ -1845,8 +1863,26 @@ def fast_linear_forward(return out+def _functorch_wrapped(*tensors):+ # vmap / grad transforms wrap tensors; an in-place write from them into a plain buffer is not allowed.+ check = getattr(torch._C._functorch, "is_functorch_wrapped_tensor", None)+ return check is not None and any(check(t) for t in tensors)+++def _lora_first_addmm(X, Wt, XA, Bt, alpha):+ # alpha * XA @ Bt into a fresh buffer (beta 0 never reads it), then the base GEMM accumulates onto it.+ if _functorch_wrapped(X, Wt, XA, Bt):+ out = torch.addmm(XA.new_zeros(1), XA, Bt, beta = 0, alpha = alpha)+ return torch.addmm(out, X, Wt)+ out = torch.empty((X.shape[0], Wt.shape[1]), dtype = X.dtype, device = X.device)+ out.addmm_(XA, Bt, alpha = alpha, beta = 0)+ out.addmm_(X, Wt)+ return out++# Traced, x.addmm_ copies x first; this op stays in place. No derivative: grad off only._HAS_ADDMM_OP = hasattr(torch.library, "custom_op")+_HAS_LORA_FIRST_OP = Falseif _HAS_ADDMM_OP:@torch.library.custom_op("unsloth::addmm_", mutates_args = ["out"])@@ -1855,6 +1891,55 @@ if _HAS_ADDMM_OP:) -> None:out.addmm_(A, B, alpha = alpha, beta = beta)+ # Traced fused LoRA order as one opaque op, so Inductor does not copy the rank-r term into a new+ # output before the base GEMM. Returns a new tensor, so a saved output aliases nothing.+ if torch.ops.unsloth.__dict__.get("_lora_first_addmm_ready", False):+ _HAS_LORA_FIRST_OP = True+ else:+ try:++ @torch.library.custom_op("unsloth::lora_first_addmm", mutates_args = ())+ def _lora_first_addmm_op(+ X: torch.Tensor, Wt: torch.Tensor, XA: torch.Tensor, Bt: torch.Tensor, alpha: float+ ) -> torch.Tensor:+ return _lora_first_addmm(X, Wt, XA, Bt, alpha)++ @_lora_first_addmm_op.register_fake+ def _(X, Wt, XA, Bt, alpha):+ return X.new_empty((X.shape[0], Wt.shape[1]))++ # out = alpha * XA @ Bt + X @ Wt, so a compiled differentiable caller gets the addmm gradients.+ def _lora_first_addmm_setup(ctx, inputs, output):+ # Save only what the requested gradients read (a frozen inference-mode base cannot be saved).+ X, Wt, XA, Bt, alpha = inputs+ ctx.alpha = alpha+ need = ctx.needs_input_grad+ ctx.save_for_backward(+ X if need[1] else None,+ Wt if need[0] else None,+ XA if need[3] else None,+ Bt if need[2] else None,+ )++ def _lora_first_addmm_backward(ctx, grad):+ X, Wt, XA, Bt = ctx.saved_tensors+ need = ctx.needs_input_grad+ dX = grad @ Wt.t() if need[0] else None+ dWt = X.t() @ grad if need[1] else None+ dXA = (grad @ Bt.t()) * ctx.alpha if need[2] else None+ dBt = (XA.t() @ grad) * ctx.alpha if need[3] else None+ return dX, dWt, dXA, dBt, None++ _lora_first_addmm_op.register_autograd(+ _lora_first_addmm_backward, setup_context = _lora_first_addmm_setup+ )+ # Marked only after every registration succeeded, so a re-executed module can trust it.+ torch.ops.unsloth.__dict__["_lora_first_addmm_ready"] = True+ _HAS_LORA_FIRST_OP = True+ except Exception:+ # An older torch.library, or a registration that failed partway: the functional path below.+ _HAS_LORA_FIRST_OP = False+def addmm_(out,@@ -1874,6 +1959,73 @@ def addmm_(return out.addmm_(A, B, alpha = alpha, beta = beta)+# The rank-r product is written first and the base GEMM accumulates onto it (cuBLAS reads it in the+# epilogue of a compute-bound GEMM), instead of a separate read-modify-write pass over the whole output.+# UNSLOTH_LORA_FUSED_PASSES=0 restores the base-then-LoRA order everywhere.+LORA_FUSED_PASSES = os.environ.get("UNSLOTH_LORA_FUSED_PASSES", "1") != "0"++try:+ from torch.distributed.tensor import DTensor as _LORA_DTENSOR+except Exception:+ _LORA_DTENSOR = ()+from torch.utils._python_dispatch import is_traceable_wrapper_subclass as _is_wrapper_subclass++_functorch_active = getattr(torch._C, "_are_functorch_transforms_active", lambda: False)+_is_dynamo_compiling = getattr(torch.compiler, "is_dynamo_compiling", lambda: False)+++def _exporting():+ # Set by torch.export (2.7+), strict included; is_exporting() folds to True under 2.10 / 2.11 Dynamo.+ # Without the flag (torch 2.6) strict and plain compiles look the same, so assume export.+ return getattr(torch.compiler, "_is_exporting_flag", True)+++def _forward_ad_active():+ # Inside torch.autograd.forward_ad.dual_level(); a plain module global, so Dynamo reads it as a constant.+ return getattr(torch.autograd.forward_ad, "_current_level", -1) >= 0+++def _lora_fused_ok(W, W_quant, dtype):+ # fp16 keeps the old order: a rank-r term rounded on its own can overflow. FP8, INT4 and packed+ # weights have their own GEMMs. addmm_ is not autocast, so the weight must dequantize to `dtype`.+ if not LORA_FUSED_PASSES or dtype not in (torch.bfloat16, torch.float32):+ return False+ if isinstance(W, Float8Tensor) or W.dtype == torch.float8_e4m3fn:+ return False+ # A DTensor's placements would be lost in the plain accumulator.+ if isinstance(W, _LORA_DTENSOR):+ return False+ # A sparse base cannot be the dense operand of addmm_. Traced code avoids `layout` (Dynamo bans it).+ if W.is_sparse or W.is_sparse_csr:+ return False+ if not _is_compiling() and W.layout != torch.strided:+ return False+ if type(W_quant) is Int4QuantState or _is_packed_state(W_quant):+ return False+ if W_quant is None:+ # A wrapper subclass (TorchAO NF4Tensor, ...) can report a float dtype over packed storage and only+ # implement its own mm.+ if _is_wrapper_subclass(W):+ return False+ return W.dtype == dtype+ return _quant_state_dtype(W_quant) == dtype+++def _shares_storage(out, *tensors):+ # True when `out` may alias an operand. Traced (Dynamo, make_fx fake tensors) storage cannot be+ # inspected, so a caller's buffer is then treated as shared and keeps the base-then-LoRA order.+ if _is_compiling():+ return True+ try:+ ptr = out.untyped_storage().data_ptr()+ return any(+ t is not None and isinstance(t, torch_Tensor) and t.untyped_storage().data_ptr() == ptr+ for t in tensors+ )+ except RuntimeError:+ return True++def matmul_lora(X,W,@@ -1883,7 +2035,12 @@ def matmul_lora(s,out = None,use_global_buffer = True,+ XA = None,+ fused = True,):+ # XA: X @ A.t() already computed by the caller (shared across projections reading the same X).+ # fused = False: the caller's module needs the base-then-LoRA order (dropout, lora_bias, a weight+ # converted before this call); _lora_fused_ok decides the rest.dtype = X.dtypeif X.dim() == 3:@@ -1892,6 +2049,11 @@ def matmul_lora(reshape = Trueelse:reshape = False+ if XA is not None and XA.dim() == 3:+ XA = XA.reshape(-1, XA.shape[-1])+ # Every branch below writes `out` before the LoRA term reads XA: recompute a caller's XA that shares it.+ if XA is not None and out is not None and _shares_storage(out, XA):+ XA = Noneif isinstance(W, Float8Tensor):assert W.ndim == 2@@ -1908,6 +2070,51 @@ def matmul_lora(out = _int4_matmul(X, W, W_quant, out = out)elif _is_packed_state(W_quant):out = W_quant.matmul(X, W, out = out)+ elif (+ A is not None+ and fused+ # Compiled inside torch.func transforms or native forward-mode AD: the op has no forward-mode or+ # batching rule, and a traced beta=0 addmm under jvp crashes, so keep the base-then-LoRA order there.+ and not (_is_compiling() and (_functorch_active() or _forward_ad_active()))+ # A DTensor operand's placements would be lost in the plain accumulator (W is checked below).+ and not any(isinstance(t, _LORA_DTENSOR) for t in (X, A, B, XA))+ and _lora_fused_ok(W, W_quant, dtype)+ # A caller's buffer must already be the output and must not share storage with a base operand.+ and (+ out is None+ or (+ out.shape == (X.shape[0], B.shape[0])+ and out.dtype == dtype+ and not _shares_storage(out, W, X, A, B, XA)+ )+ )+ ):+ if XA is None:+ XA = torch_matmul(X, A.t().to(dtype))+ B = B.t().to(dtype)+ if out is not None:+ # Eager only: a traced caller buffer cannot be checked for aliasing (_shares_storage). Dequantize+ # first: the caller's buffer may share storage with the quant state (absmax, code, ...).+ W = fast_dequantize(W, W_quant, use_global_buffer = use_global_buffer, dtype = dtype)+ addmm_(out, XA, B, alpha = s, beta = 0)+ out.addmm_(X, W.t())+ else:+ W = fast_dequantize(W, W_quant, use_global_buffer = use_global_buffer, dtype = dtype)+ if not _is_compiling():+ out = _lora_first_addmm(X, W.t(), XA, B, s)+ elif (+ _HAS_LORA_FIRST_OP+ and type(s) in (int, float)+ and _is_dynamo_compiling()+ and not _exporting()+ ):+ out = torch.ops.unsloth.lora_first_addmm(X, W.t(), XA, B, s)+ else:+ # Traced without the op: functional, since an in-place accumulate onto a saved buffer breaks the+ # block compile. Export lands here too, so torch.func works on the exported program.+ out = torch.addmm(XA.new_zeros(1), XA, B, beta = 0, alpha = s)+ out = torch.addmm(out, X, W.t())+ A = Noneelse:W = fast_dequantize(W, W_quant, use_global_buffer = use_global_buffer, dtype = dtype)out = torch_matmul(X, W.t(), out = out)@@ -1916,7 +2123,8 @@ def matmul_lora(if A is not None:A, B = A.t(), B.t()- XA = torch_matmul(X, A.to(dtype))+ if XA is None:+ XA = torch_matmul(X, A.to(dtype))addmm_(out, XA, B.to(dtype), alpha = s)return out.view(batch, seq_len, -1) if reshape else out