Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Keep tensor subclass types through the fused LoRA accumulates (#16249)

Daniel Han4b9a0ebcca23
diff --git a/tests/test_lora_first_addmm_op.py b/tests/test_lora_first_addmm_op.py
index 0b707b17f4a..ae93197eb18 100644
--- a/tests/test_lora_first_addmm_op.py
+++ b/tests/test_lora_first_addmm_op.py
@@ -343,9 +343,25 @@ print("HAS_OP", KU._HAS_LORA_FIRST_OP)
"""
+def _skip_without_triton():
+ # A fresh interpreter imports unsloth.kernels without this process's test stubs, so it needs triton.
+ import importlib.util
+
+ try:
+ spec = importlib.util.find_spec("triton")
+ except ValueError:
+ # A stub module in sys.modules with no spec.
+ spec = None
+ # unsloth_zoo's stand-in, while its finder is installed, is not a real triton either.
+ found = spec is not None and spec.origin != "triton_stub"
+ if not found:
+ pytest.skip("triton is not installed")
+
+
def test_a_partly_registered_op_is_not_used():
# Registration fails after the schema exists (autograd here): the op must stay disabled rather than be
# enabled because its name resolves. A fresh interpreter, since this process registered it already.
+ _skip_without_triton()
import os
import subprocess
import sys
@@ -389,6 +405,7 @@ print("REREGISTERED", len(calls), "HAS_OP", KU._HAS_LORA_FIRST_OP)
def test_a_reexecuted_module_reuses_the_completed_registration():
# Re-registering over a working op could leave it half replaced if a later step failed; a re-executed
# module must reuse the ready op instead of defining it again.
+ _skip_without_triton()
import os
import subprocess
import sys
diff --git a/tests/test_lora_fused_passes.py b/tests/test_lora_fused_passes.py
index 0607a656576..9587753f6b3 100644
--- a/tests/test_lora_fused_passes.py
+++ b/tests/test_lora_fused_passes.py
@@ -911,3 +911,189 @@ def test_a_caller_out_sharing_the_quant_state_is_written_after_dequantizing(monk
return kernel_utils.matmul_lora(X, W, state, A, B, 0.5, out = buf).clone()
torch.testing.assert_close(run(True), run(False), rtol = 0, atol = 0)
+
+
+class _TaggedTensor(torch.Tensor):
+ # A plain storage-backed subclass, as a user library might wrap activations.
+ pass
+
+
+def _tag(t):
+ return t.as_subclass(_TaggedTensor)
+
+
+def _plain(t):
+ return t.as_subclass(torch.Tensor)
+
+
+def _same(got, want, tagged):
+ # The subclass survives every in-place accumulate, and the arithmetic is the plain tensors' bit for bit.
+ assert (type(got) is _TaggedTensor) == tagged, type(got)
+ assert got.dtype == want.dtype
+ torch.testing.assert_close(_plain(got), _plain(want), rtol = 0, atol = 0)
+
+
+def _tag_one(which, X, W, A, B):
+ return tuple(_tag(t) if n == which else t for n, t in zip("XWAB", (X, W, A, B)))
+
+
+def _subclass_inputs():
+ g = torch.Generator().manual_seed(0)
+ X, W = torch.randn(16, 256, generator = g), torch.randn(32, 256, generator = g)
+ A, B = torch.randn(4, 256, generator = g), torch.randn(32, 4, generator = g)
+ return X, W, A, B
+
+
[email protected]("on", [True, False])
[email protected]("which", ["X", "W", "A", "B", "XA"])
+def test_a_subclass_operand_keeps_its_type_through_matmul_lora(which, on, monkeypatch):
+ X, W, A, B = _subclass_inputs()
+ _set(monkeypatch, on)
+ xa = X @ A.t()
+ want = kernel_utils.matmul_lora(X, W, None, A, B, 0.5, XA = xa if which == "XA" else None)
+ XA = _tag(xa) if which == "XA" else None
+ X, W, A, B = _tag_one(which, X, W, A, B)
+ got = kernel_utils.matmul_lora(X, W, None, A, B, 0.5, XA = XA)
+ _same(got, want, True)
+
+
[email protected]("on", [True, False])
[email protected]("which", ["X", "A", "out"])
+def test_a_caller_buffer_keeps_its_storage_and_the_subclass(which, on, monkeypatch):
+ X, W, A, B = _subclass_inputs()
+ _set(monkeypatch, on)
+ want = kernel_utils.matmul_lora(X, W, None, A, B, 0.5, out = torch.empty(16, 32))
+ buf = torch.empty(16, 32)
+ buf = _tag(buf) if which == "out" else buf
+ X, W, A, B = _tag_one(which, X, W, A, B)
+ got = kernel_utils.matmul_lora(X, W, None, A, B, 0.5, out = buf)
+ assert got.data_ptr() == buf.data_ptr()
+ _same(got, want, True)
+
+
+class _NoReceiverTensor(torch.Tensor):
+ # A subclass that refuses to be the receiver of an in-place GEMM, though it may be an operand.
+ def addmm_(self, *args, **kwargs):
+ raise RuntimeError("not an addmm_ receiver")
+
+
[email protected]("on", [True, False])
[email protected]("caller_out", [False, True])
+def test_the_base_gemm_accumulates_on_the_buffer_not_the_adapters_result(
+ caller_out, on, monkeypatch
+):
+ X, W, A, B = _subclass_inputs()
+ _set(monkeypatch, on)
+ kw = (lambda: dict(out = torch.empty(16, 32))) if caller_out else dict
+ want = kernel_utils.matmul_lora(X, W, None, A, B, 0.5, **kw())
+ got = kernel_utils.matmul_lora(X, W, None, A.as_subclass(_NoReceiverTensor), B, 0.5, **kw())
+ torch.testing.assert_close(_plain(got), _plain(want), rtol = 0, atol = 0)
+
+
[email protected]("dtype", [torch.bfloat16, torch.float16])
[email protected]("which", ["X", "W", "B", "XA", "plain"])
+def test_autocast_with_a_precomputed_xa_keeps_fp32_operands_finite(which, dtype, monkeypatch):
+ # XA is computed outside autocast. The in-place fused order is not autocast, so fp32 operands stay fp32
+ # and finite; a functional addmm would autocast and give 10240 in bf16, inf in fp16.
+ X, A = torch.ones(1, 1), torch.ones(1, 1)
+ W, B = torch.full((1, 1), -60000.0), torch.full((1, 1), 35000.0)
+ XA = X @ A.t()
+ if which == "XA":
+ XA = _tag(XA)
+ X, W, A, B = _tag_one(which, X, W, A, B)
+ _set(monkeypatch, True)
+ with torch.autocast("cpu", dtype = dtype):
+ got = kernel_utils.matmul_lora(X, W, None, A, B, 2.0, XA = XA)
+ assert got.dtype == torch.float32 and float(got) == 10000.0
+ assert (type(got) is _TaggedTensor) == (which != "plain")
+
+
[email protected]("dtype", [torch.bfloat16, torch.float16])
[email protected]("which", ["X", "W", "B", "XA"])
+def test_autocast_with_a_half_xa_over_an_fp32_base_autocasts_the_base(which, dtype, monkeypatch):
+ # Half activations and adapters over an fp32 base weight take the base-then-LoRA order under autocast.
+ X, A, B = (torch.ones(*s, dtype = dtype) for s in ((2, 4), (1, 4), (3, 1)))
+ W = torch.ones(3, 4)
+ XA = X @ A.t()
+ if which == "XA":
+ XA = _tag(XA)
+ X, W, A, B = _tag_one(which, X, W, A, B)
+ _set(monkeypatch, True)
+ with torch.autocast("cpu", dtype = dtype):
+ got = kernel_utils.matmul_lora(X, W, None, A, B, 1.0, XA = XA)
+ assert got.dtype == dtype and (_plain(got) == 8).all()
+ assert type(got) is _TaggedTensor
+
+
[email protected]("bias", [False, True])
[email protected]("which", ["X_lora", "A", "B"])
+def test_the_dropout_and_bias_route_keeps_a_subclass(which, bias, monkeypatch):
+ g = torch.Generator().manual_seed(0)
+ X, W = torch.randn(2, 8, 64, generator = g), torch.randn(32, 64, generator = g)
+ A, B = torch.randn(4, 64, generator = g), torch.randn(32, 4, generator = g)
+ X_lora = X * 0.5
+ b = torch.randn(32, generator = g) if bias else None
+ want = fast_lora._matmul_lora_ext(X, X_lora, W, None, A, B, 0.5, b)
+ X_lora, A, B = (
+ _tag(t) if n == which else t for n, t in zip(("X_lora", "A", "B"), (X_lora, A, B))
+ )
+ got = fast_lora._matmul_lora_ext(X, X_lora, W, None, A, B, 0.5, b)
+ assert got.shape == want.shape
+ _same(got, want, True)
+
+
[email protected]("bsz", [1, 3])
[email protected]("which", ["A", "B"])
+def test_decode_keeps_a_subclass_adapter(which, bsz, monkeypatch):
+ g = torch.Generator().manual_seed(0)
+ X, W = torch.randn(bsz, 1, 64, generator = g), torch.randn(32, 64, generator = g)
+ A, B = torch.randn(4, 64, generator = g), torch.randn(32, 4, generator = g)
+ for name in (
+ "_has_non_lora_adapter",
+ "_has_multiple_active_adapters",
+ "_has_active_dora_adapter",
+ "_has_active_lora_bias",
+ "_has_packed_qweight",
+ ):
+ monkeypatch.setattr(kernel_utils, name, lambda proj: False)
+
+ def run(A, B):
+ monkeypatch.setattr(
+ kernel_utils, "get_lora_parameters_bias", lambda proj: (W, None, A, B, 0.5, None)
+ )
+ return kernel_utils.fast_linear_forward(None, X)
+
+ want = run(A, B)
+ got = run(_tag(A), B) if which == "A" else run(A, _tag(B))
+ assert got.shape == want.shape
+ _same(got, want, True)
+
+
[email protected]("which", ["A", "B"])
+def test_the_traced_no_grad_accumulate_keeps_a_subclass_result(which, monkeypatch):
+ # The opaque op returns nothing, so a subclass operand takes the in-place method and keeps its type.
+ monkeypatch.setattr(torch.compiler, "is_compiling", lambda: True)
+ out, A, B = torch.zeros(3, 4), torch.randn(3, 2), torch.randn(2, 4)
+ want = torch.zeros(3, 4).addmm_(A, B, alpha = 0.5)
+ A, B = (_tag(A), B) if which == "A" else (A, _tag(B))
+ with torch.no_grad():
+ got = kernel_utils.addmm_(out, A, B, alpha = 0.5)
+ assert got.data_ptr() == out.data_ptr()
+ _same(got, want, True)
+
+
+def test_the_traced_no_grad_accumulate_keeps_the_op_for_plain_tensors(monkeypatch):
+ if not kernel_utils._HAS_ADDMM_OP:
+ pytest.skip("torch.library.custom_op is not available")
+ monkeypatch.setattr(torch.compiler, "is_compiling", lambda: True)
+ calls = []
+ real = torch.ops.unsloth.addmm_
+
+ def op(*args):
+ calls.append(len(args))
+ return real(*args)
+
+ monkeypatch.setattr(torch.ops.unsloth, "addmm_", op)
+ with torch.no_grad():
+ kernel_utils.addmm_(torch.zeros(3, 4), torch.randn(3, 2), torch.randn(2, 4), alpha = 0.5)
+ assert calls == [5]
diff --git a/unsloth/kernels/fast_lora.py b/unsloth/kernels/fast_lora.py
index a528bf8ae52..758b5a94ed1 100644
--- a/unsloth/kernels/fast_lora.py
+++ b/unsloth/kernels/fast_lora.py
@@ -314,12 +314,13 @@ def _matmul_lora_ext(X, X_lora, W, W_quant, A, B, S, bias):
XA = torch.matmul(X_lora.reshape(-1, X_lora.shape[-1]), A.to(dtype).t())
out2d = out.view(-1, out.shape[-1])
if bias is None:
- addmm_(out2d, XA, B.to(dtype).t(), alpha = S)
+ out2d = addmm_(out2d, XA, B.to(dtype).t(), alpha = S)
else:
# PEFT's rounding order: (lora_B(x) + bias) * scaling, then + base. No add(alpha = S): Inductor
# (torch 2.13) miscompiles it traced inside these Functions.
- out2d.add_(torch.addmm(bias.to(dtype), XA, B.to(dtype).t()).mul_(S))
- return out
+ out2d = out2d.add_(torch.addmm(bias.to(dtype), XA, B.to(dtype).t()).mul_(S))
+ # A subclass operand comes back from the in-place update as its own type; keep it.
+ return out if type(out2d) is type(out) else out2d.view(out.shape)
def _add_lora_dX(dX, y_dB, A, S, mask, p):
diff --git a/unsloth/kernels/utils.py b/unsloth/kernels/utils.py
index 75a34542104..cc094ced4bf 100644
--- a/unsloth/kernels/utils.py
+++ b/unsloth/kernels/utils.py
@@ -1848,13 +1848,13 @@ def fast_linear_forward(
# Without out=, mv / mm follow a different autocast dtype; the in-place update does not.
if temp_lora.dtype != dtype:
temp_lora = temp_lora.to(dtype)
- out.addmv_(B, temp_lora, alpha = lora_S)
+ out = out.addmv_(B, temp_lora, alpha = lora_S)
else:
out = out.view(bsz, out_dim)
temp_lora = torch_mm(X.view(bsz, in_dim), A.t(), out = temp_lora)
if temp_lora.dtype != dtype:
temp_lora = temp_lora.to(dtype)
- out.addmm_(temp_lora, B.t(), alpha = lora_S)
+ out = out.addmm_(temp_lora, B.t(), alpha = lora_S)
out = out.view(bsz, 1, out_dim)
if bias is not None:
@@ -1875,9 +1875,13 @@ def _lora_first_addmm(X, Wt, XA, Bt, alpha):
out = torch.addmm(XA.new_zeros(1), XA, Bt, beta = 0, alpha = alpha)
return torch.addmm(out, X, Wt)
out = torch.empty((X.shape[0], Wt.shape[1]), dtype = X.dtype, device = X.device)
- out.addmm_(XA, Bt, alpha = alpha, beta = 0)
- out.addmm_(X, Wt)
- return out
+ # Both updates mutate the plain buffer; a subclass operand's returned type is kept for the result.
+ return _accumulated(out.addmm_(XA, Bt, alpha = alpha, beta = 0), out.addmm_(X, Wt))
+
+
+def _accumulated(lora_out, base_out):
+ # The more derived of the two returned types, else the base GEMM's.
+ return lora_out if isinstance(lora_out, type(base_out)) else base_out
# Traced, x.addmm_ copies x first; this op stays in place. No derivative: grad off only.
@@ -1941,6 +1945,20 @@ if _HAS_ADDMM_OP:
_HAS_LORA_FIRST_OP = False
+# Tracing infrastructure (make_fx, export) wraps plain tensors; those are not user subclasses.
+_LORA_PLAIN_TYPES = (torch.Tensor, torch.nn.Parameter)
+try:
+ from torch._subclasses.fake_tensor import FakeTensor as _FakeTensor
+ from torch._subclasses.functional_tensor import FunctionalTensor as _FunctionalTensor
+ _LORA_PLAIN_TYPES += (_FakeTensor, _FunctionalTensor)
+except Exception:
+ pass
+
+
+def _lora_subclass(*tensors):
+ return any(t is not None and type(t) not in _LORA_PLAIN_TYPES for t in tensors)
+
+
def addmm_(
out,
A,
@@ -1953,6 +1971,8 @@ def addmm_(
and torch.compiler.is_compiling()
and not torch.is_grad_enabled()
and type(alpha) in (int, float)
+ # The op returns nothing, so a subclass operand takes the method below, which returns its type.
+ and not _lora_subclass(out, A, B)
):
torch.ops.unsloth.addmm_(out, A, B, alpha, beta)
return out
@@ -2096,8 +2116,7 @@ def matmul_lora(
# Eager only: a traced caller buffer cannot be checked for aliasing (_shares_storage). Dequantize
# first: the caller's buffer may share storage with the quant state (absmax, code, ...).
W = fast_dequantize(W, W_quant, use_global_buffer = use_global_buffer, dtype = dtype)
- addmm_(out, XA, B, alpha = s, beta = 0)
- out.addmm_(X, W.t())
+ out = _accumulated(addmm_(out, XA, B, alpha = s, beta = 0), out.addmm_(X, W.t()))
else:
W = fast_dequantize(W, W_quant, use_global_buffer = use_global_buffer, dtype = dtype)
if not _is_compiling():
@@ -2125,6 +2144,6 @@ def matmul_lora(
A, B = A.t(), B.t()
if XA is None:
XA = torch_matmul(X, A.to(dtype))
- addmm_(out, XA, B.to(dtype), alpha = s)
+ out = addmm_(out, XA, B.to(dtype), alpha = s)
return out.view(batch, seq_len, -1) if reshape else out