Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
"""UNSLOTH_LORA_FUSED_PASSES: the fused LoRA kernels write the rank-r term first and let the base GEMM
accumulate onto it, share X @ A across projections reading the same X, and read X once for every
projection's LoRA gradients. Outputs and gradients match the base-then-LoRA order and an fp64 PEFT
oracle; the shared paths are taken only when they apply."""
from __future__ import annotations
import copy
import pytest
import torch
import unsloth # noqa: F401
from real_accelerator import has_real_cuda
from unsloth.kernels import fast_lora
from unsloth.kernels import utils as kernel_utils
H, I = 64, 96
TARGETS = ["q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj"]
DEVICES = ["cpu"] + (["cuda"] if has_real_cuda() else [])
class _Block(torch.nn.Module):
def __init__(self):
super().__init__()
self.q_proj = torch.nn.Linear(H, H, bias = False)
self.k_proj = torch.nn.Linear(H, H // 2, bias = False)
self.v_proj = torch.nn.Linear(H, H // 2, bias = False)
self.o_proj = torch.nn.Linear(H, H, bias = False)
self.gate_proj = torch.nn.Linear(H, I, bias = False)
self.up_proj = torch.nn.Linear(H, I, bias = False)
self.down_proj = torch.nn.Linear(I, H, bias = False)
self.act_fn = torch.nn.SiLU()
def _model(
device,
dtype,
alpha_pattern = None,
dropout = 0.0,
lora_bias = False,
):
from peft import LoraConfig, get_peft_model
torch.manual_seed(3407)
cfg = LoraConfig(
r = 4,
lora_alpha = 8,
lora_dropout = dropout,
target_modules = TARGETS,
init_lora_weights = False,
alpha_pattern = alpha_pattern or {},
lora_bias = lora_bias,
)
model = get_peft_model(_Block(), cfg).to(device, dtype)
for name, p in model.named_parameters():
p.requires_grad_("lora_" in name)
if lora_bias:
with torch.no_grad():
for name, p in model.named_parameters():
if "lora_B" in name and name.endswith("bias"):
p.normal_()
model.train()
return model, model.base_model.model
PATHS = {
"qkv": (
lambda b: (lambda X: fast_lora.apply_lora_qkv(b, X)),
lambda b: (lambda X: (b.q_proj(X), b.k_proj(X), b.v_proj(X))),
),
"o": (lambda b: (lambda X: fast_lora.apply_lora_o(b, X)), lambda b: b.o_proj),
"mlp": (
lambda b: (lambda X: fast_lora.apply_lora_mlp_swiglu(b, X)),
lambda b: (lambda X: b.down_proj(b.act_fn(b.gate_proj(X)) * b.up_proj(X))),
),
}
def _run(fn, model, X0):
X = X0.detach().clone().requires_grad_()
out = fn(X * 1) # a non-leaf: the fused backward may write dX into its input buffer
outs = out if isinstance(out, tuple) else (out,)
gen = torch.Generator(device = "cpu").manual_seed(7)
loss = sum(
(o.double() * torch.randn(o.shape, generator = gen, dtype = torch.float64).to(o.device)).sum()
for o in outs
)
loss.backward()
grads = {n: p.grad.clone() for n, p in model.named_parameters() if p.grad is not None}
model.zero_grad(set_to_none = True)
return [o.detach() for o in outs], X.grad.clone(), grads
def _skip_without_native_bf16(device, dtype = torch.bfloat16):
# Same rule as test_lora_gradient_layout: Triton's bf16 kernels need sm_80+, and is_bf16_supported() also
# reports emulated bf16 on older GPUs.
if (
device == "cuda"
and dtype is torch.bfloat16
and (
not torch.cuda.is_bf16_supported()
or (not torch.version.hip and torch.cuda.get_device_capability()[0] < 8)
)
):
pytest.skip("bf16 kernels need native bf16 (sm_80+)")
def _set(monkeypatch, on):
monkeypatch.setattr(kernel_utils, "LORA_FUSED_PASSES", on)
def _err(a, b):
return float((a.double() - b.double()).norm() / b.double().norm().clamp_min(1e-30))
@pytest.mark.parametrize("device", DEVICES)
@pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16])
@pytest.mark.parametrize("path", ["qkv", "o", "mlp"])
@pytest.mark.parametrize("alphas", ["equal", "mixed"])
def test_fused_passes_match_the_unfused_order_and_an_fp64_oracle(
device, dtype, path, alphas, monkeypatch
):
if device == "cpu" and path == "mlp":
pytest.skip("the SwiGLU kernels are Triton")
if device == "cpu" and dtype is torch.bfloat16:
pytest.skip("bf16 GEMMs on CPU are slow and not what runs in training")
_skip_without_native_bf16(device, dtype)
pattern = {"k_proj": 16, "up_proj": 16} if alphas == "mixed" else None
model, block = _model(device, dtype, pattern)
X0 = torch.randn(3, 5, H, device = device, dtype = dtype)
_set(monkeypatch, False)
ref_out, ref_dX, ref_g = _run(PATHS[path][0](block), model, X0)
_set(monkeypatch, True)
out, dX, g = _run(PATHS[path][0](block), model, X0)
oracle_model = copy.deepcopy(model).double()
o_out, o_dX, o_g = _run(
PATHS[path][1](oracle_model.base_model.model), oracle_model, X0.double()
)
tol = 1e-5 if dtype is torch.float32 else 2e-2
for a, b, o in zip(out, ref_out, o_out):
assert _err(a, o) <= max(tol, 1.5 * _err(b, o)), (_err(a, o), _err(b, o))
assert _err(dX, o_dX) <= max(tol, 1.5 * _err(ref_dX, o_dX)), (
_err(dX, o_dX),
_err(ref_dX, o_dX),
)
assert set(g) == set(ref_g) == set(o_g)
for n in g:
assert g[n].shape == ref_g[n].shape
assert _err(g[n], o_g[n]) <= max(tol, 1.5 * _err(ref_g[n], o_g[n])), (
n,
_err(g[n], o_g[n]),
_err(ref_g[n], o_g[n]),
)
def test_shared_paths_engage_only_when_they_apply(monkeypatch):
calls = {"xa": 0, "bwd": 0}
real_xa, real_bwd = fast_lora._shared_XA, fast_lora._shared_lora_backward
def xa(*a, **k):
calls["xa"] += 1
return real_xa(*a, **k)
def bwd(*a, **k):
calls["bwd"] += 1
return real_bwd(*a, **k)
monkeypatch.setattr(fast_lora, "_shared_XA", xa)
monkeypatch.setattr(fast_lora, "_shared_lora_backward", bwd)
X0 = torch.randn(2, 3, H)
for alphas, on, want in (
("equal", True, (1, 1)),
("mixed", True, (1, 0)),
("equal", False, (0, 0)),
):
_set(monkeypatch, on)
calls.update(xa = 0, bwd = 0)
model, block = _model("cpu", torch.float32, {"k_proj": 16} if alphas == "mixed" else None)
_run(PATHS["qkv"][0](block), model, X0)
assert (calls["xa"], calls["bwd"]) == want, (alphas, on, calls)
@pytest.mark.parametrize("device", DEVICES)
@pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16])
def test_kill_switch_is_the_base_then_lora_order(device, dtype, monkeypatch):
# UNSLOTH_LORA_FUSED_PASSES=0: base GEMM, then the rank-r term added in place, bit for bit.
if device == "cpu" and dtype is torch.bfloat16:
pytest.skip("bf16 GEMMs on CPU are slow and not what runs in training")
_skip_without_native_bf16(device, dtype)
_set(monkeypatch, False)
g = torch.Generator(device = "cpu").manual_seed(0)
X, W = torch.randn(7, H, generator = g), torch.randn(I, H, generator = g)
A, B = torch.randn(4, H, generator = g), torch.randn(I, 4, generator = g)
X, W, A, B = (t.to(device, dtype) for t in (X, W, A, B))
want = X @ W.t()
kernel_utils.addmm_(want, X @ A.t(), B.t(), alpha = 2.0)
assert torch.equal(kernel_utils.matmul_lora(X, W, None, A, B, 2.0), want)
def _drop_lora_bias(block, keep):
# PEFT puts lora_bias on every target; keep it only on `keep` (a down-only bias, for example).
for name in TARGETS:
if name not in keep:
getattr(block, name).lora_B["default"].bias = None
@pytest.mark.parametrize("device", DEVICES)
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16, torch.float32])
@pytest.mark.parametrize("path", ["qkv", "o", "mlp"])
@pytest.mark.parametrize("config", ["plain", "dropout", "lora_bias", "down_bias"])
def test_exclusions_keep_the_base_then_lora_order(device, dtype, path, config, monkeypatch):
# Every configuration the fused order excludes (fp16 at all, dropout, lora_bias anywhere in the
# module) gives the kill switch's outputs and gradients bit for bit, forward and backward.
if device == "cpu" and path == "mlp":
pytest.skip("the SwiGLU kernels are Triton")
if config == "down_bias" and path != "mlp":
pytest.skip("a down-only bias is an MLP case")
if config == "plain" and dtype is not torch.float16:
pytest.skip("plain bf16 / fp32 take the fused order")
_skip_without_native_bf16(device, dtype)
kwargs = {"dropout": 0.1} if config == "dropout" else {}
kwargs["lora_bias"] = config in ("lora_bias", "down_bias")
model, block = _model(device, dtype, **kwargs)
if config == "down_bias":
_drop_lora_bias(block, keep = ("down_proj",))
X0 = torch.randn(3, 5, H, device = device, dtype = dtype)
runs = []
for on in (False, True):
_set(monkeypatch, on)
torch.manual_seed(11)
runs.append(_run(PATHS[path][0](block), model, X0))
(out0, dX0, g0), (out1, dX1, g1) = runs
assert all(torch.equal(a, b) for a, b in zip(out0, out1))
assert torch.equal(dX0, dX1)
assert set(g0) == set(g1) and all(torch.equal(g0[n], g1[n]) for n in g0)
@pytest.mark.parametrize("device", DEVICES)
def test_fp16_rank_r_term_cannot_overflow_before_the_base_cancels_it(device, monkeypatch):
# The rank-r term alone is ~70000 (past fp16's 65504) but the output is ~10000. The base-then-LoRA
# order adds it in the GEMM's fp32 epilogue; writing it first would store inf.
X = torch.ones(1, H, dtype = torch.float16, device = device)
W = torch.zeros(I, H, dtype = torch.float16, device = device)
W[0] = -60000 / H
A = torch.zeros(4, H, dtype = torch.float16, device = device)
A[0, 0] = 1
B = torch.zeros(I, 4, dtype = torch.float16, device = device)
B[0, 0] = 35000
outs = {}
for on in (True, False):
_set(monkeypatch, on)
outs[on] = kernel_utils.matmul_lora(X, W, None, A, B, 2.0)
assert torch.isfinite(outs[True]).all() and abs(float(outs[True][0, 0]) - 10000) <= 32
assert torch.equal(outs[True], outs[False])
def test_fused_eligibility_is_one_rule(monkeypatch):
_set(monkeypatch, True)
from types import SimpleNamespace
from unsloth.kernels.int4_packed import Int4QuantState
ok = kernel_utils._lora_fused_ok
dense = lambda dtype: torch.empty(4, 4, dtype = dtype) # noqa: E731
u8 = torch.empty(4, 4, dtype = torch.uint8)
assert ok(dense(torch.bfloat16), None, torch.bfloat16) and ok(
dense(torch.float32), None, torch.float32
)
assert not ok(dense(torch.float16), None, torch.float16) # fp16 keeps the old order
assert not ok(dense(torch.float32), None, torch.bfloat16) # fp32 weight under bf16 autocast
# fast_dequantize returns a 4-bit state's own dtype, not the one asked for.
assert ok(u8, SimpleNamespace(dtype = torch.bfloat16), torch.bfloat16)
assert not ok(u8, SimpleNamespace(dtype = torch.float16), torch.bfloat16)
assert not ok(u8, [None, None, torch.float16], torch.bfloat16)
assert not ok(dense(torch.float8_e4m3fn), torch.ones(1), torch.bfloat16)
int4 = object.__new__(Int4QuantState)
w = torch.empty(8, 4, dtype = torch.int8)
assert not ok(w, int4, torch.bfloat16) and not ok(
w.t(), int4, torch.bfloat16
) # either orientation
packed = type("Packed", (), {"_unsloth_packed_weight_state": True, "dtype": torch.bfloat16})()
assert not ok(u8, packed, torch.bfloat16)
def test_a_caller_buffer_that_is_not_the_output_keeps_main_behavior(monkeypatch):
# torch.matmul(..., out = empty) resizes; the fused order writes the rank-r term first, so it is
# taken only when the buffer is already the output.
g = torch.Generator(device = "cpu").manual_seed(0)
X, W = torch.randn(7, H, generator = g), torch.randn(I, H, generator = g)
A, B = torch.randn(4, H, generator = g), torch.randn(I, 4, generator = g)
outs = {}
for on in (True, False):
_set(monkeypatch, on)
out = torch.empty(0)
assert kernel_utils.matmul_lora(X, W, None, A, B, 2.0, out = out) is not None
outs[on] = out
assert outs[True].shape == (7, I) and torch.equal(outs[True], outs[False])
@pytest.mark.parametrize("device", DEVICES)
def test_fp32_base_under_autocast_keeps_the_base_then_lora_order(device, monkeypatch):
# addmm_ is not autocast: an fp32 base weight with bf16 activations takes the old order.
_skip_without_native_bf16(device)
half = torch.bfloat16
g = torch.Generator(device = "cpu").manual_seed(0)
X, W = torch.randn(7, H, generator = g), torch.randn(I, H, generator = g)
A, B = torch.randn(4, H, generator = g), torch.randn(I, 4, generator = g)
X, W, A, B = X.to(device, half), W.to(device), A.to(device), B.to(device)
model, block = _model(device, torch.float32)
X0 = torch.randn(2, 5, H, generator = g).to(device, half)
outs, runs = {}, {}
for on in (True, False):
_set(monkeypatch, on)
with torch.autocast(device, dtype = half):
outs[on] = kernel_utils.matmul_lora(X, W, None, A, B, 2.0)
runs[on] = _run(PATHS["o"][0](block), model, X0)
assert torch.equal(outs[True], outs[False])
(out_on, dX_on, g_on), (out_off, dX_off, g_off) = runs[True], runs[False]
assert torch.equal(out_on[0], out_off[0]) and torch.equal(dX_on, dX_off)
assert g_on.keys() == g_off.keys() and all(torch.equal(g_on[k], g_off[k]) for k in g_on)
@pytest.mark.skipif(not has_real_cuda(), reason = "needs bitsandbytes 4-bit on a real GPU")
def test_fp16_nf4_state_under_bf16_autocast_keeps_the_base_then_lora_order(monkeypatch):
# fast_dequantize returns the state's fp16, so the in-place base accumulate would mix dtypes.
bnb = pytest.importorskip("bitsandbytes")
_skip_without_native_bf16("cuda")
lin = bnb.nn.Linear4bit(H, I, bias = False, compute_dtype = torch.float16, quant_type = "nf4")
lin.weight = bnb.nn.Params4bit(
torch.randn(I, H, dtype = torch.float16), requires_grad = False, quant_type = "nf4"
)
lin = lin.cuda()
W, Wq = lin.weight, lin.weight.quant_state
assert Wq.dtype == torch.float16
g = torch.Generator(device = "cpu").manual_seed(0)
X = torch.randn(7, H, generator = g).cuda().bfloat16()
A, B = torch.randn(4, H, generator = g).cuda(), torch.randn(I, 4, generator = g).cuda()
outs = {}
for on in (True, False):
_set(monkeypatch, on)
with torch.autocast("cuda", dtype = torch.bfloat16):
outs[on] = kernel_utils.matmul_lora(X, W, Wq, A, B, 2.0)
assert torch.equal(outs[True], outs[False])
def test_small_update_survives_the_single_rounding(monkeypatch):
# bf16 base output 1.0 plus a rank-r term of 1/128 rounds once to 1.0078125; the old order rounds the
# base output first and the separate add then loses everything below half a bf16 step.
x = torch.tensor([[1.0, 1.0]], dtype = torch.bfloat16)
w = torch.tensor([[1.0, 1 / 256]], dtype = torch.bfloat16)
a = torch.tensor([[1.0, 0.0]], dtype = torch.bfloat16)
b = torch.tensor([[1 / 512]], dtype = torch.bfloat16)
exact = (x.double() @ w.double().t() + (x.double() @ a.double().t()) @ b.double().t()).to(
torch.bfloat16
)
_set(monkeypatch, True)
got = kernel_utils.matmul_lora(x, w, None, a, b, 1.0)
assert torch.equal(got, exact) and got.item() == 1.0078125
_set(monkeypatch, False)
assert kernel_utils.matmul_lora(x, w, None, a, b, 1.0).item() == 1.0
def test_a_caller_buffer_sharing_an_operand_keeps_main_behavior(monkeypatch):
# Writing the rank-r term first would overwrite a base operand the GEMM has not read yet.
_set(monkeypatch, True)
g = torch.Generator(device = "cpu").manual_seed(0)
X, W = torch.randn(H, H, generator = g), torch.randn(H, H, generator = g)
A, B = torch.randn(4, H, generator = g), torch.randn(H, 4, generator = g)
assert kernel_utils._shares_storage(W, X, W)
assert not kernel_utils._shares_storage(torch.empty(H, H), X, W, A, B, None)
# The branch taken is checked, not the values: matmul with out= overlapping an input is undefined on
# main too (on CPU a 64 x 64 result differs from call to call).
betas = []
real = kernel_utils.addmm_
monkeypatch.setattr(
kernel_utils, "addmm_", lambda *a, **k: betas.append(k.get("beta", 1)) or real(*a, **k)
)
out = W.clone()
kernel_utils.matmul_lora(X, out, None, A, B, 2.0, out = out)
assert betas == [1], "a buffer sharing W must take the base-then-LoRA order"
betas.clear()
kernel_utils.matmul_lora(X, W, None, A, B, 2.0, out = torch.empty(H, H))
assert betas == [0], "a separate buffer takes the fused order"
def test_shared_lora_a_grads_have_the_parameter_layout(monkeypatch):
# The shared backward hands back d_A whose .t() is contiguous like the [r, K] parameter, so autograd
# does not copy it into a new layout.
_set(monkeypatch, True)
X = torch.randn(10, H)
As = (torch.randn(4, H).t(), torch.randn(4, H).t())
Bs = (torch.randn(H, 4).t(), torch.randn(I, 4).t())
dYs = (torch.randn(10, H), torch.randn(10, I))
d_As, _, _ = fast_lora._shared_lora_backward(X, dYs, As, Bs, 2.0, None)
for dA in d_As:
assert dA.t().is_contiguous(), dA.stride()
def test_shared_lora_backward_can_be_recorded_for_a_second_derivative(monkeypatch):
# create_graph runs the backward with grad on: no in-place write through sibling views of one buffer.
_set(monkeypatch, True)
X = torch.randn(10, H, dtype = torch.float64)
As = (torch.randn(4, H, dtype = torch.float64).t(), torch.randn(3, H, dtype = torch.float64).t())
Bs = (torch.randn(H, 4, dtype = torch.float64).t(), torch.randn(I, 3, dtype = torch.float64).t())
dYs = tuple(torch.randn(10, n, dtype = torch.float64, requires_grad = True) for n in (H, I))
with torch.enable_grad():
d_As, d_Bs, dX = fast_lora._shared_lora_backward(X, dYs, As, Bs, 2.0, None)
total = sum(t.square().sum() for t in (*d_As, *d_Bs, dX))
got = torch.autograd.grad(total, dYs)
def ref(*dYs):
out = []
for dY, A, B in zip(dYs, As, Bs):
y_dB = dY @ B.t()
out += [2.0 * X.t() @ y_dB, 2.0 * (A.t() @ X.t()) @ dY, 2.0 * y_dB @ A.t()]
dX = out[2] + out[5]
return sum(t.square().sum() for t in (out[0], out[3], out[1], out[4], dX))
want = torch.autograd.grad(ref(*dYs), dYs)
for g, w in zip(got, want):
torch.testing.assert_close(g, w)
def test_shared_lora_a_grads_are_views_of_one_buffer(monkeypatch):
# Each d_A is a row block of one [sum r, K] buffer: no per-projection copy into the parameter layout.
_set(monkeypatch, True)
X = torch.randn(10, H)
As = (torch.randn(4, H).t(), torch.randn(3, H).t())
Bs = (torch.randn(H, 4).t(), torch.randn(I, 3).t())
dYs = (torch.randn(10, H), torch.randn(10, I))
d_As, d_Bs, dX = fast_lora._shared_lora_backward(X, dYs, As, Bs, 2.0, None)
ptrs = {dA.untyped_storage().data_ptr() for dA in d_As}
assert len(ptrs) == 1, "d_A blocks were copied out of the shared buffer"
X64 = X.double()
for dA, dB_, A, B, dY in zip(d_As, d_Bs, As, Bs, dYs):
y_dB = dY.double() @ B.double().t()
torch.testing.assert_close(dA.double(), 2.0 * X64.t() @ y_dB, rtol = 1e-4, atol = 1e-4)
torch.testing.assert_close(
dB_.double(), 2.0 * (A.double().t() @ X64.t()) @ dY.double(), rtol = 1e-4, atol = 1e-4
)
def _small_update(dtype = torch.bfloat16):
# Base output 1 + 1/256 plus a rank-r term 1/512: the fused order rounds once to 1.0078125, the
# base-then-LoRA order rounds the base to 1.0 first and then loses the update.
x = torch.tensor([[1.0, 1.0]], dtype = dtype)
w = torch.tensor([[1.0, 1 / 256]], dtype = dtype)
a = torch.tensor([[1.0, 0.0]], dtype = dtype)
b = torch.tensor([[1 / 512]], dtype = dtype)
return x, w, a, b
def test_compiled_caller_buffer_keeps_main_behavior(monkeypatch):
# Traced, a caller's buffer cannot be checked for aliasing, so it keeps the base-then-LoRA order.
_set(monkeypatch, True)
torch._dynamo.reset()
x, w, a, b = _small_update()
def f(x, w, a, b):
out = torch.empty(1, 1, dtype = x.dtype)
return kernel_utils.matmul_lora(x, w, None, a, b, 1.0, out = out)
got = torch.compile(f, backend = "aot_eager", fullgraph = True)(x, w, a, b)
assert got.item() == 1.0
# A buffer that aliases W: the old order reads W before the output is written.
g = torch.Generator(device = "cpu").manual_seed(1)
X, W = torch.randn(4, 4, generator = g), torch.randn(4, 4, generator = g)
A, B = torch.randn(4, 4, generator = g), torch.randn(4, 4, generator = g)
ref = X @ W.t() + 2.0 * (X @ A.t()) @ B.t()
def g_(X, W, A, B):
out = W.clone()
return kernel_utils.matmul_lora(X, out, None, A, B, 2.0, out = out)
torch._dynamo.reset()
got = torch.compile(g_, backend = "aot_eager", fullgraph = True)(X, W, A, B)
torch.testing.assert_close(got, ref, rtol = 1e-5, atol = 1e-5)
def test_fake_tracing_with_a_caller_buffer(monkeypatch):
# make_fx fake tensors have no data pointer: the buffer is treated as shared, not inspected.
from torch.fx.experimental.proxy_tensor import make_fx
_set(monkeypatch, True)
X, W, A, B = torch.randn(3, H), torch.randn(H, H), torch.randn(4, H), torch.randn(H, 4)
def f(X, W, A, B, out):
return kernel_utils.matmul_lora(X, W, None, A, B, 2.0, out = out)
gm = make_fx(f, tracing_mode = "fake")(X, W, A, B, torch.empty(3, H))
out = torch.empty(3, H)
torch.testing.assert_close(
gm(X, W, A, B, out), X @ W.t() + 2.0 * (X @ A.t()) @ B.t(), rtol = 1e-5, atol = 1e-5
)
@pytest.mark.parametrize("fused", [True, False])
def test_a_precomputed_xa_sharing_the_caller_buffer_is_not_reused(fused, monkeypatch):
# The base GEMM writes the buffer before the rank-r term reads XA: XA is recomputed instead.
_set(monkeypatch, True)
g = torch.Generator(device = "cpu").manual_seed(3)
X, W = torch.randn(3, 4, generator = g), torch.randn(2, 4, generator = g)
A, B = torch.randn(2, 4, generator = g), torch.randn(2, 2, generator = g)
ref = X @ W.t() + 2.0 * (X @ A.t()) @ B.t()
XA = X @ A.t()
got = kernel_utils.matmul_lora(X, W, None, A, B, 2.0, XA = XA, out = XA, fused = fused)
torch.testing.assert_close(got, ref, rtol = 1e-5, atol = 1e-5)
def test_sparse_base_weight_keeps_the_base_then_lora_order(monkeypatch):
# addmm_ cannot accumulate a dense X @ sparse W.t() in place; main's matmul handles it.
_set(monkeypatch, True)
X, A, B = torch.ones(1, 2), torch.ones(1, 2), torch.ones(2, 1)
W = torch.eye(2).to_sparse_coo()
assert not kernel_utils._lora_fused_ok(W, None, torch.float32)
torch.testing.assert_close(
kernel_utils.matmul_lora(X, W, None, A, B, 1.0), torch.full((1, 2), 3.0)
)
@pytest.mark.parametrize("dropout", [0.0, 0.1])
def test_multi_token_inference_follows_the_training_order(dropout, monkeypatch):
# fast_linear_forward with q_len > 1 takes the order the fused training kernels use: a configured
# lora_dropout (even in eval) keeps base-then-LoRA, a plain adapter takes the fused order.
_skip_without_native_bf16("cpu")
from peft import LoraConfig, get_peft_model
_set(monkeypatch, True)
lin = torch.nn.Sequential(torch.nn.Linear(2, 1, bias = False))
model = get_peft_model(
lin, LoraConfig(r = 1, lora_alpha = 1, lora_dropout = dropout, target_modules = ["0"])
)
model = model.to(torch.bfloat16).eval()
proj = model.base_model.model[0]
x, w, a, b = _small_update()
with torch.no_grad():
proj.base_layer.weight.copy_(w)
proj.lora_A["default"].weight.copy_(a)
proj.lora_B["default"].weight.copy_(b)
X = x.view(1, 1, 2).expand(1, 3, 2).contiguous()
got = kernel_utils.fast_linear_forward(proj, X)
assert got[0, 0, 0].item() == (1.0 if dropout else 1.0078125)
_DYNAMIC_RANK_CASE = """
import sys
import torch
import importlib.util
import unsloth # noqa: F401
if importlib.util.find_spec("triton") is None:
# The CPU path never loads the kernels: define them against the stub as the GPU path does, then drop it.
from unsloth_zoo.stubs import triton_stub
meta_path, modules = list(sys.meta_path), set(sys.modules)
triton_stub.inject_into_sys_modules()
from unsloth.kernels import fast_lora
sys.meta_path[:] = meta_path
for key in [k for k in sys.modules if k.split(".")[0] == "triton" and k not in modules]:
del sys.modules[key]
else:
from unsloth.kernels import fast_lora
torch.manual_seed(0)
weights = []
for n, r in zip((16, 12, 12), (2, 3, 4)):
W = torch.randn(n, 16) * 0.1
A, B = torch.randn(r, 16) * 0.1, torch.randn(n, r) * 0.1
if sys.argv[1] == "param":
A, B = torch.nn.Parameter(A), torch.nn.Parameter(B)
else:
A, B = A.requires_grad_(), B.requires_grad_()
weights.append((W, A, B))
(QW, QA, QB), (KW, KA, KB), (VW, VA, VB) = weights
def f(X, QA, QB, KA, KB, VA, VB):
q, k, v = fast_lora.LoRA_QKV.apply(
X, QW, None, QA, QB, 2.0, KW, None, KA, KB, 2.0, VW, None, VA, VB, 2.0, False
)
return q.sum() + k.sum() + v.sum()
g = torch.compile(f, backend = "aot_eager", fullgraph = True, dynamic = True)
for seq in (3, 5, 7):
X = torch.randn(2, seq, 16, requires_grad = True)
g(X, QA, QB, KA, KB, VA, VB).backward()
assert X.grad.shape == X.shape
print("OK")
"""
@pytest.mark.parametrize("param", [False, True])
def test_dynamic_compile_with_a_rank_equal_to_the_first_length(param):
# torch 2.14: dynamic-shape plain-tensor LoRA weights once recompiled to a width-0 dQ; they keep the
# base-then-LoRA order when traced. Fresh interpreter: depends on Dynamo state.
import os
import subprocess
import sys
from pathlib import Path
root = Path(__file__).resolve().parents[1]
env = dict(
os.environ,
PYTHONPATH = os.pathsep.join([str(root), os.environ.get("PYTHONPATH", "")]),
UNSLOTH_DISABLE_CPU_FALLBACK = "0",
)
p = subprocess.run(
[sys.executable, "-c", _DYNAMIC_RANK_CASE, "param" if param else "plain"],
cwd = root,
env = env,
capture_output = True,
text = True,
timeout = 900,
)
assert p.returncode == 0 and "OK" in p.stdout, (p.stdout + p.stderr)[-3000:]
@pytest.mark.parametrize("param", [False, True])
def test_traced_order_depends_on_parameter_lora_weights(param, monkeypatch):
_skip_without_native_bf16("cpu")
_set(monkeypatch, True)
torch._dynamo.reset()
x, w, a, b = _small_update()
if param:
a, b = torch.nn.Parameter(a), torch.nn.Parameter(b)
else:
a, b = a.requires_grad_(), b.requires_grad_()
def f(x, a, b):
return fast_lora.LoRA_W.apply(x.view(1, 1, 2), w, None, a, b, 1.0)
got = torch.compile(f, backend = "aot_eager", fullgraph = True)(x, a, b)
assert got.item() == (1.0078125 if param else 1.0)
class _FakeF8(torch.Tensor):
pass
@pytest.mark.parametrize("backend", ["packed", "int4", "float8"])
def test_a_precomputed_xa_sharing_the_buffer_is_not_reused_by_any_dispatch(backend, monkeypatch):
# Packed, INT4 and Float8Tensor bases write the caller's buffer too before the rank-r term reads XA.
_set(monkeypatch, True)
g = torch.Generator(device = "cpu").manual_seed(3)
X, W = torch.randn(3, 4, generator = g), torch.randn(2, 4, generator = g)
A, B = torch.randn(2, 4, generator = g), torch.randn(2, 2, generator = g)
ref = X @ W.t() + 2.0 * (X @ A.t()) @ B.t()
W_quant = None
if backend == "packed":
W_quant = type(
"Packed",
(),
{
"_unsloth_packed_weight_state": True,
"dtype": torch.float32,
"matmul": lambda self, X, W, out = None: torch.matmul(X, W.t(), out = out),
},
)()
monkeypatch.setattr(kernel_utils, "_is_packed_state", lambda q: q is W_quant)
elif backend == "int4":
Int4 = type("Int4QuantState", (), {})
monkeypatch.setattr(kernel_utils, "Int4QuantState", Int4)
monkeypatch.setattr(
kernel_utils, "_int4_matmul", lambda X, W, q, out = None: torch.matmul(X, W.t(), out = out)
)
W_quant = Int4()
else:
monkeypatch.setattr(kernel_utils, "Float8Tensor", _FakeF8)
W = W.as_subclass(_FakeF8)
W.block_size = (1, W.shape[1]) # not rowwise-after-transpose: the forward GEMM
XA = X @ A.t()
got = kernel_utils.matmul_lora(X, W, W_quant, A, B, 2.0, XA = XA, out = XA)
torch.testing.assert_close(got.as_subclass(torch.Tensor), ref, rtol = 1e-5, atol = 1e-5)
@pytest.mark.parametrize("on", [True, False])
def test_a_3d_precomputed_xa_is_flattened_with_x(on, monkeypatch):
_set(monkeypatch, on)
g = torch.Generator(device = "cpu").manual_seed(5)
X, W = torch.randn(2, 3, H, generator = g), torch.randn(I, H, generator = g)
A, B = torch.randn(4, H, generator = g), torch.randn(I, 4, generator = g)
ref = kernel_utils.matmul_lora(X, W, None, A, B, 2.0)
got = kernel_utils.matmul_lora(X, W, None, A, B, 2.0, XA = X @ A.t())
assert got.shape == ref.shape
torch.testing.assert_close(got, ref, rtol = 1e-5, atol = 1e-5)
def test_sparse_base_weight_keeps_the_base_then_lora_order_under_export(monkeypatch):
# torch.export traces with is_compiling() True: the layout check must still run there.
_set(monkeypatch, True)
if not hasattr(torch, "export"):
pytest.skip("torch.export is not available")
class M(torch.nn.Module):
def __init__(self):
super().__init__()
self.W = torch.eye(2).to_sparse_coo()
self.A, self.B = torch.ones(1, 2), torch.ones(2, 1)
def forward(self, x):
return kernel_utils.matmul_lora(x, self.W, None, self.A, self.B, 1.0)
x = torch.ones(1, 2)
ep = torch.export.export(M(), (x,))
torch.testing.assert_close(ep.module()(x), torch.full((1, 2), 3.0))
@pytest.fixture
def one_rank_mesh(tmp_path):
# One-rank gloo group on a file store (no TCP / libuv rendezvous, no environment variables); only a
# group this fixture created is destroyed.
if not isinstance(kernel_utils._LORA_DTENSOR, type):
pytest.skip("torch.distributed.tensor is not available")
import torch.distributed as dist
if not dist.is_available() or not dist.is_gloo_available():
pytest.skip("gloo is not available")
from torch.distributed.device_mesh import init_device_mesh
created = not dist.is_initialized()
if created:
store = dist.FileStore(str(tmp_path / "store"), 1)
dist.init_process_group("gloo", store = store, rank = 0, world_size = 1)
try:
yield init_device_mesh("cpu", (1,))
finally:
if created:
dist.destroy_process_group()
def test_dtensor_base_weight_keeps_the_base_then_lora_order(one_rank_mesh, monkeypatch):
# A DTensor carries placements the plain accumulator would drop: keep main's order.
from torch.distributed.tensor import Replicate, distribute_tensor
_set(monkeypatch, True)
W = distribute_tensor(torch.randn(4, 3), one_rank_mesh, [Replicate()])
assert not kernel_utils._lora_fused_ok(W, None, torch.float32)
@pytest.mark.parametrize("placement", ["replicate", "shard0", "shard1"])
def test_dtensor_activation_keeps_the_base_then_lora_order(placement, one_rank_mesh, monkeypatch):
# A DTensor X with plain W / A / B (implicit replication) keeps its value and placement.
from torch.distributed.tensor import Replicate, Shard, distribute_tensor
from torch.distributed.tensor.experimental import implicit_replication
g = torch.Generator(device = "cpu").manual_seed(7)
X, W = torch.randn(4, 6, generator = g), torch.randn(5, 6, generator = g)
A, B = torch.randn(2, 6, generator = g), torch.randn(5, 2, generator = g)
pl = {"replicate": Replicate(), "shard0": Shard(0), "shard1": Shard(1)}[placement]
Xd = distribute_tensor(X, one_rank_mesh, [pl])
out = {}
for on in (False, True):
_set(monkeypatch, on)
with implicit_replication():
try:
out[on] = kernel_utils.matmul_lora(Xd, W, None, A, B, 2.0)
except NotImplementedError as e:
if on:
raise
pytest.skip(f"main's order needs DTensor ops this torch lacks: {e}"[:200])
assert out[True].placements == out[False].placements
torch.testing.assert_close(out[True].full_tensor(), out[False].full_tensor())
def _dtensor_lora_inputs(mesh, which, placement):
from torch.distributed.tensor import Replicate, Shard, distribute_tensor
g = torch.Generator().manual_seed(0)
X, W = torch.randn(2, 4, 4, generator = g), torch.randn(4, 4, generator = g)
A, B = torch.randn(2, 4, generator = g), torch.randn(4, 2, generator = g)
pl = {"replicate": Replicate(), "shard0": Shard(0), "shard1": Shard(1)}[placement]
# Mixed plain / DTensor LoRA weights fail on main as well (in-place gradient writes), so the adapters
# are either both plain or both DTensor.
X = distribute_tensor(X, mesh, [pl])
if which == "XAB":
A, B = (distribute_tensor(t, mesh, [Replicate()]) for t in (A, B))
return X.requires_grad_(), W, A.requires_grad_(), B.requires_grad_()
def _apply_lora_function(name, X, W, A, B):
from unsloth.kernels.swiglu import _DWf_DW_dfg_traced
if name == "W":
return (fast_lora.LoRA_W.apply(X, W, None, A, B, 1.0),)
if name == "QKV":
return fast_lora.LoRA_QKV.apply(
X, W, None, A, B, 1.0, W, None, A, B, 1.0, W, None, A, B, 1.0
)
silu_mul = lambda e, g: torch.nn.functional.silu(e) * g
return (
fast_lora.LoRA_MLP.apply(
X,
W,
None,
A,
B,
1.0,
W,
None,
A,
B,
1.0,
W,
None,
A,
B,
1.0,
silu_mul,
_DWf_DW_dfg_traced,
False,
),
)
def _run_dtensor_lora(name, which, placement, mesh, monkeypatch, on):
# Main's order is the reference: where this torch's DTensor cannot run it (older releases lack the
# sharding rules for views of a sharded dim / in-place addmm), there is nothing to compare against.
from torch.distributed.tensor.experimental import implicit_replication
_set(monkeypatch, on)
X, W, A, B = _dtensor_lora_inputs(mesh, which, placement)
try:
with implicit_replication():
out = _apply_lora_function(name, X, W, A, B)
sum(o.sum() for o in out).backward()
except Exception as e: # noqa: BLE001
if on:
raise
pytest.skip(f"main's order needs DTensor ops this torch lacks: {e}"[:200])
return out, X.grad
@pytest.mark.parametrize("name", ["W", "QKV", "MLP"])
@pytest.mark.parametrize("placement", ["replicate", "shard0", "shard1"])
def test_dtensor_operands_keep_the_base_then_lora_order(
name, placement, one_rank_mesh, monkeypatch
):
# matmul_lora falls back for a DTensor X / A / B, so the decision the backward reads must agree: the
# shared and fused backward paths would otherwise redistribute the input gradient.
_, ref = _run_dtensor_lora(name, "XAB", placement, one_rank_mesh, monkeypatch, False)
out, got = _run_dtensor_lora(name, "XAB", placement, one_rank_mesh, monkeypatch, True)
assert out[0].grad_fn.fused is False
assert got.placements == ref.placements
torch.testing.assert_close(got.full_tensor(), ref.full_tensor())
@pytest.mark.parametrize("name", ["W", "QKV", "MLP"])
def test_a_dtensor_activation_records_the_base_then_lora_order(name, one_rank_mesh, monkeypatch):
# With plain adapters the backward cannot run under DTensor on main either, so only the recorded
# forward decision is checked here.
from torch.distributed.tensor.experimental import implicit_replication
X, W, A, B = _dtensor_lora_inputs(one_rank_mesh, "X", "replicate")
for on in (False, True):
_set(monkeypatch, on)
try:
with implicit_replication():
out = _apply_lora_function(name, X, W, A, B)
except Exception as e: # noqa: BLE001
if on:
raise
pytest.skip(f"main's order needs DTensor ops this torch lacks: {e}"[:200])
assert out[0].grad_fn.fused is False
def test_a_wrapper_subclass_base_keeps_the_base_then_lora_order(monkeypatch):
# TorchAO's NF4Tensor reports bf16 over packed storage and implements mm but not addmm_: with no quant
# state it must not qualify for the in-place accumulate.
nf4 = pytest.importorskip("torchao.quantization.quantize_.workflows.nf4.nf4_tensor")
_set(monkeypatch, True)
g = torch.Generator().manual_seed(0)
W = nf4.to_nf4(torch.randn(128, 128, generator = g).to(torch.bfloat16))
X = torch.randn(8, 128, generator = g).to(torch.bfloat16)
A = (torch.randn(4, 128, generator = g) * 0.1).to(torch.bfloat16)
B = (torch.randn(128, 4, generator = g) * 0.1).to(torch.bfloat16)
assert not kernel_utils._lora_fused_ok(W, None, torch.bfloat16)
got = kernel_utils.matmul_lora(X, W, None, A, B, 0.5)
_set(monkeypatch, False)
torch.testing.assert_close(got, kernel_utils.matmul_lora(X, W, None, A, B, 0.5), rtol = 0, atol = 0)
@pytest.mark.parametrize("layout", ["csr", "csc", "bsr"])
def test_compressed_sparse_lora_a_takes_one_product_per_projection(layout):
# A compressed sparse A cannot be concatenated for the shared X @ [A_1; A_2].t(): one product each.
if layout == "bsr" and not torch.backends.mkl.is_available():
# CPU dense @ block-sparse needs MKL (the ROCm wheels ship without it); main's product fails the same way.
pytest.skip("CPU dense @ block-sparse needs MKL")
g = torch.Generator().manual_seed(0)
X = torch.randn(4, 6, generator = g)
dense = [torch.randn(3, 6, generator = g) for _ in range(2)]
to = {
"csr": lambda t: t.to_sparse_csr(),
"csc": lambda t: t.to_sparse_csc(),
"bsr": lambda t: t.to_sparse_bsr((3, 3)),
}[layout]
views = fast_lora._shared_XA(X, [to(t) for t in dense], torch.float32)
for v, t in zip(views, dense):
v = v.to_dense() if v.layout != torch.strided else v
torch.testing.assert_close(v, X @ t.t(), rtol = 1e-5, atol = 1e-5)
def test_a_caller_out_sharing_the_quant_state_is_written_after_dequantizing(monkeypatch):
# The caller's buffer may share storage with the quant state (absmax): the weight is dequantized from
# it before the rank-r term is written into the buffer.
_set(monkeypatch, True)
class State:
pass
def dequant(
W,
W_quant,
use_global_buffer = True,
dtype = None,
):
return (W * W_quant.absmax.sum()).to(dtype)
monkeypatch.setattr(kernel_utils, "fast_dequantize", dequant)
monkeypatch.setattr(kernel_utils, "_quant_state_dtype", lambda q: torch.float32)
g = torch.Generator().manual_seed(0)
X = torch.randn(2, 8, generator = g)
W = torch.randn(4, 8, generator = g)
A, B = torch.randn(2, 8, generator = g), torch.randn(4, 2, generator = g)
def run(shared):
state = State()
buf = torch.empty(2, 4)
state.absmax = buf.view(-1)[:3] if shared else torch.empty(3)
state.absmax.copy_(torch.tensor([0.5, 0.25, 0.25]))
return kernel_utils.matmul_lora(X, W, state, A, B, 0.5, out = buf).clone()
torch.testing.assert_close(run(True), run(False), rtol = 0, atol = 0)
class _TaggedTensor(torch.Tensor):
# A plain storage-backed subclass, as a user library might wrap activations.
pass
def _tag(t):
return t.as_subclass(_TaggedTensor)
def _plain(t):
return t.as_subclass(torch.Tensor)
def _same(got, want, tagged):
# The subclass survives every in-place accumulate, and the arithmetic is the plain tensors' bit for bit.
assert (type(got) is _TaggedTensor) == tagged, type(got)
assert got.dtype == want.dtype
torch.testing.assert_close(_plain(got), _plain(want), rtol = 0, atol = 0)
def _tag_one(which, X, W, A, B):
return tuple(_tag(t) if n == which else t for n, t in zip("XWAB", (X, W, A, B)))
def _subclass_inputs():
g = torch.Generator().manual_seed(0)
X, W = torch.randn(16, 256, generator = g), torch.randn(32, 256, generator = g)
A, B = torch.randn(4, 256, generator = g), torch.randn(32, 4, generator = g)
return X, W, A, B
@pytest.mark.parametrize("on", [True, False])
@pytest.mark.parametrize("which", ["X", "W", "A", "B", "XA"])
def test_a_subclass_operand_keeps_its_type_through_matmul_lora(which, on, monkeypatch):
X, W, A, B = _subclass_inputs()
_set(monkeypatch, on)
xa = X @ A.t()
want = kernel_utils.matmul_lora(X, W, None, A, B, 0.5, XA = xa if which == "XA" else None)
XA = _tag(xa) if which == "XA" else None
X, W, A, B = _tag_one(which, X, W, A, B)
got = kernel_utils.matmul_lora(X, W, None, A, B, 0.5, XA = XA)
_same(got, want, True)
@pytest.mark.parametrize("on", [True, False])
@pytest.mark.parametrize("which", ["X", "A", "out"])
def test_a_caller_buffer_keeps_its_storage_and_the_subclass(which, on, monkeypatch):
X, W, A, B = _subclass_inputs()
_set(monkeypatch, on)
want = kernel_utils.matmul_lora(X, W, None, A, B, 0.5, out = torch.empty(16, 32))
buf = torch.empty(16, 32)
buf = _tag(buf) if which == "out" else buf
X, W, A, B = _tag_one(which, X, W, A, B)
got = kernel_utils.matmul_lora(X, W, None, A, B, 0.5, out = buf)
assert got.data_ptr() == buf.data_ptr()
_same(got, want, True)
class _NoReceiverTensor(torch.Tensor):
# A subclass that refuses to be the receiver of an in-place GEMM, though it may be an operand.
def addmm_(self, *args, **kwargs):
raise RuntimeError("not an addmm_ receiver")
@pytest.mark.parametrize("on", [True, False])
@pytest.mark.parametrize("caller_out", [False, True])
def test_the_base_gemm_accumulates_on_the_buffer_not_the_adapters_result(
caller_out, on, monkeypatch
):
X, W, A, B = _subclass_inputs()
_set(monkeypatch, on)
kw = (lambda: dict(out = torch.empty(16, 32))) if caller_out else dict
want = kernel_utils.matmul_lora(X, W, None, A, B, 0.5, **kw())
got = kernel_utils.matmul_lora(X, W, None, A.as_subclass(_NoReceiverTensor), B, 0.5, **kw())
torch.testing.assert_close(_plain(got), _plain(want), rtol = 0, atol = 0)
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
@pytest.mark.parametrize("which", ["X", "W", "B", "XA", "plain"])
def test_autocast_with_a_precomputed_xa_keeps_fp32_operands_finite(which, dtype, monkeypatch):
# XA is computed outside autocast. The in-place fused order is not autocast, so fp32 operands stay fp32
# and finite; a functional addmm would autocast and give 10240 in bf16, inf in fp16.
X, A = torch.ones(1, 1), torch.ones(1, 1)
W, B = torch.full((1, 1), -60000.0), torch.full((1, 1), 35000.0)
XA = X @ A.t()
if which == "XA":
XA = _tag(XA)
X, W, A, B = _tag_one(which, X, W, A, B)
_set(monkeypatch, True)
with torch.autocast("cpu", dtype = dtype):
got = kernel_utils.matmul_lora(X, W, None, A, B, 2.0, XA = XA)
assert got.dtype == torch.float32 and float(got) == 10000.0
assert (type(got) is _TaggedTensor) == (which != "plain")
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
@pytest.mark.parametrize("which", ["X", "W", "B", "XA"])
def test_autocast_with_a_half_xa_over_an_fp32_base_autocasts_the_base(which, dtype, monkeypatch):
# Half activations and adapters over an fp32 base weight take the base-then-LoRA order under autocast.
X, A, B = (torch.ones(*s, dtype = dtype) for s in ((2, 4), (1, 4), (3, 1)))
W = torch.ones(3, 4)
XA = X @ A.t()
if which == "XA":
XA = _tag(XA)
X, W, A, B = _tag_one(which, X, W, A, B)
_set(monkeypatch, True)
with torch.autocast("cpu", dtype = dtype):
got = kernel_utils.matmul_lora(X, W, None, A, B, 1.0, XA = XA)
assert got.dtype == dtype and (_plain(got) == 8).all()
assert type(got) is _TaggedTensor
@pytest.mark.parametrize("bias", [False, True])
@pytest.mark.parametrize("which", ["X_lora", "A", "B"])
def test_the_dropout_and_bias_route_keeps_a_subclass(which, bias, monkeypatch):
g = torch.Generator().manual_seed(0)
X, W = torch.randn(2, 8, 64, generator = g), torch.randn(32, 64, generator = g)
A, B = torch.randn(4, 64, generator = g), torch.randn(32, 4, generator = g)
X_lora = X * 0.5
b = torch.randn(32, generator = g) if bias else None
want = fast_lora._matmul_lora_ext(X, X_lora, W, None, A, B, 0.5, b)
X_lora, A, B = (
_tag(t) if n == which else t for n, t in zip(("X_lora", "A", "B"), (X_lora, A, B))
)
got = fast_lora._matmul_lora_ext(X, X_lora, W, None, A, B, 0.5, b)
assert got.shape == want.shape
_same(got, want, True)
@pytest.mark.parametrize("bsz", [1, 3])
@pytest.mark.parametrize("which", ["A", "B"])
def test_decode_keeps_a_subclass_adapter(which, bsz, monkeypatch):
g = torch.Generator().manual_seed(0)
X, W = torch.randn(bsz, 1, 64, generator = g), torch.randn(32, 64, generator = g)
A, B = torch.randn(4, 64, generator = g), torch.randn(32, 4, generator = g)
for name in (
"_has_non_lora_adapter",
"_has_multiple_active_adapters",
"_has_active_dora_adapter",
"_has_active_lora_bias",
"_has_packed_qweight",
):
monkeypatch.setattr(kernel_utils, name, lambda proj: False)
def run(A, B):
monkeypatch.setattr(
kernel_utils, "get_lora_parameters_bias", lambda proj: (W, None, A, B, 0.5, None)
)
return kernel_utils.fast_linear_forward(None, X)
want = run(A, B)
got = run(_tag(A), B) if which == "A" else run(A, _tag(B))
assert got.shape == want.shape
_same(got, want, True)
@pytest.mark.parametrize("bsz", [1, 3])
@pytest.mark.parametrize("fp8", [False, True])
def test_decode_packed_base_keeps_its_matmul(bsz, fp8):
_, block = _model("cpu", torch.float32)
proj = block.o_proj
calls = []
class Packed:
_unsloth_packed_weight_state = True
_unsloth_fp8_dgrad = fp8
def matmul(
self,
X,
W,
out = None,
):
calls.append(X.shape)
return torch.matmul(X, W.t(), out = out)
proj.base_layer.weight.quant_state = Packed()
X = torch.randn(bsz, 1, H)
with torch.inference_mode():
expected = proj(X)
actual = kernel_utils.fast_linear_forward(proj, X)
torch.testing.assert_close(actual, expected)
assert calls == [X.shape]
@pytest.mark.parametrize("bsz", [1, 3])
@pytest.mark.parametrize("tagged", [False, True])
def test_decode_returns_the_strided_out_with_its_strides(tagged, bsz, monkeypatch):
# The MLP writes down_proj into temp_up[..., :hd]: the caller's strided out comes back as is.
g = torch.Generator().manual_seed(0)
X, W = torch.randn(bsz, 1, 64, generator = g), torch.randn(32, 64, generator = g)
A, B = torch.randn(4, 64, generator = g), torch.randn(32, 4, generator = g)
for name in (
"_has_non_lora_adapter",
"_has_multiple_active_adapters",
"_has_active_dora_adapter",
"_has_active_lora_bias",
"_has_packed_qweight",
):
monkeypatch.setattr(kernel_utils, name, lambda proj: False)
B_in = _tag(B) if tagged else B
monkeypatch.setattr(
kernel_utils, "get_lora_parameters_bias", lambda proj: (W, None, A, B_in, 0.5, None)
)
out = torch.empty(bsz, 1, 40)[:, :, :32]
got = kernel_utils.fast_linear_forward(None, X, out = out)
assert got.data_ptr() == out.data_ptr() and got.stride() == out.stride()
assert (type(got) is _TaggedTensor) == tagged
want = X @ W.t() + 0.5 * (X @ A.t()) @ B.t()
torch.testing.assert_close(_plain(got), want, rtol = 1e-5, atol = 1e-5)
@pytest.mark.parametrize("which", ["A", "B"])
def test_the_traced_no_grad_accumulate_keeps_a_subclass_result(which, monkeypatch):
# The opaque op returns nothing, so a subclass operand takes the in-place method and keeps its type.
monkeypatch.setattr(torch.compiler, "is_compiling", lambda: True)
out, A, B = torch.zeros(3, 4), torch.randn(3, 2), torch.randn(2, 4)
want = torch.zeros(3, 4).addmm_(A, B, alpha = 0.5)
A, B = (_tag(A), B) if which == "A" else (A, _tag(B))
with torch.no_grad():
got = kernel_utils.addmm_(out, A, B, alpha = 0.5)
assert got.data_ptr() == out.data_ptr()
_same(got, want, True)
def test_the_traced_no_grad_accumulate_keeps_the_op_for_plain_tensors(monkeypatch):
if not kernel_utils._HAS_ADDMM_OP:
pytest.skip("torch.library.custom_op is not available")
monkeypatch.setattr(torch.compiler, "is_compiling", lambda: True)
calls = []
real = torch.ops.unsloth.addmm_
def op(*args):
calls.append(len(args))
return real(*args)
monkeypatch.setattr(torch.ops.unsloth, "addmm_", op)
with torch.no_grad():
kernel_utils.addmm_(torch.zeros(3, 4), torch.randn(3, 2), torch.randn(2, 4), alpha = 0.5)
assert calls == [5]