Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Keep tensor subclass types through the fused LoRA accumulates (#16249)
Daniel Han4b9a0ebcca23
diff --git a/tests/test_lora_first_addmm_op.py b/tests/test_lora_first_addmm_op.pyindex 0b707b17f4a..ae93197eb18 100644--- a/tests/test_lora_first_addmm_op.py+++ b/tests/test_lora_first_addmm_op.py@@ -343,9 +343,25 @@ print("HAS_OP", KU._HAS_LORA_FIRST_OP)"""+def _skip_without_triton():+ # A fresh interpreter imports unsloth.kernels without this process's test stubs, so it needs triton.+ import importlib.util++ try:+ spec = importlib.util.find_spec("triton")+ except ValueError:+ # A stub module in sys.modules with no spec.+ spec = None+ # unsloth_zoo's stand-in, while its finder is installed, is not a real triton either.+ found = spec is not None and spec.origin != "triton_stub"+ if not found:+ pytest.skip("triton is not installed")++def test_a_partly_registered_op_is_not_used():# Registration fails after the schema exists (autograd here): the op must stay disabled rather than be# enabled because its name resolves. A fresh interpreter, since this process registered it already.+ _skip_without_triton()import osimport subprocessimport sys@@ -389,6 +405,7 @@ print("REREGISTERED", len(calls), "HAS_OP", KU._HAS_LORA_FIRST_OP)def test_a_reexecuted_module_reuses_the_completed_registration():# Re-registering over a working op could leave it half replaced if a later step failed; a re-executed# module must reuse the ready op instead of defining it again.+ _skip_without_triton()import osimport subprocessimport sysdiff --git a/tests/test_lora_fused_passes.py b/tests/test_lora_fused_passes.pyindex 0607a656576..9587753f6b3 100644--- a/tests/test_lora_fused_passes.py+++ b/tests/test_lora_fused_passes.py@@ -911,3 +911,189 @@ def test_a_caller_out_sharing_the_quant_state_is_written_after_dequantizing(monkreturn kernel_utils.matmul_lora(X, W, state, A, B, 0.5, out = buf).clone()torch.testing.assert_close(run(True), run(False), rtol = 0, atol = 0)+++class _TaggedTensor(torch.Tensor):+ # A plain storage-backed subclass, as a user library might wrap activations.+ pass+++def _tag(t):+ return t.as_subclass(_TaggedTensor)+++def _plain(t):+ return t.as_subclass(torch.Tensor)+++def _same(got, want, tagged):+ # The subclass survives every in-place accumulate, and the arithmetic is the plain tensors' bit for bit.+ assert (type(got) is _TaggedTensor) == tagged, type(got)+ assert got.dtype == want.dtype+ torch.testing.assert_close(_plain(got), _plain(want), rtol = 0, atol = 0)+++def _tag_one(which, X, W, A, B):+ return tuple(_tag(t) if n == which else t for n, t in zip("XWAB", (X, W, A, B)))+++def _subclass_inputs():+ g = torch.Generator().manual_seed(0)+ X, W = torch.randn(16, 256, generator = g), torch.randn(32, 256, generator = g)+ A, B = torch.randn(4, 256, generator = g), torch.randn(32, 4, generator = g)+ return X, W, A, B++[email protected]("on", [True, False])[email protected]("which", ["X", "W", "A", "B", "XA"])+def test_a_subclass_operand_keeps_its_type_through_matmul_lora(which, on, monkeypatch):+ X, W, A, B = _subclass_inputs()+ _set(monkeypatch, on)+ xa = X @ A.t()+ want = kernel_utils.matmul_lora(X, W, None, A, B, 0.5, XA = xa if which == "XA" else None)+ XA = _tag(xa) if which == "XA" else None+ X, W, A, B = _tag_one(which, X, W, A, B)+ got = kernel_utils.matmul_lora(X, W, None, A, B, 0.5, XA = XA)+ _same(got, want, True)++[email protected]("on", [True, False])[email protected]("which", ["X", "A", "out"])+def test_a_caller_buffer_keeps_its_storage_and_the_subclass(which, on, monkeypatch):+ X, W, A, B = _subclass_inputs()+ _set(monkeypatch, on)+ want = kernel_utils.matmul_lora(X, W, None, A, B, 0.5, out = torch.empty(16, 32))+ buf = torch.empty(16, 32)+ buf = _tag(buf) if which == "out" else buf+ X, W, A, B = _tag_one(which, X, W, A, B)+ got = kernel_utils.matmul_lora(X, W, None, A, B, 0.5, out = buf)+ assert got.data_ptr() == buf.data_ptr()+ _same(got, want, True)+++class _NoReceiverTensor(torch.Tensor):+ # A subclass that refuses to be the receiver of an in-place GEMM, though it may be an operand.+ def addmm_(self, *args, **kwargs):+ raise RuntimeError("not an addmm_ receiver")++[email protected]("on", [True, False])[email protected]("caller_out", [False, True])+def test_the_base_gemm_accumulates_on_the_buffer_not_the_adapters_result(+ caller_out, on, monkeypatch+):+ X, W, A, B = _subclass_inputs()+ _set(monkeypatch, on)+ kw = (lambda: dict(out = torch.empty(16, 32))) if caller_out else dict+ want = kernel_utils.matmul_lora(X, W, None, A, B, 0.5, **kw())+ got = kernel_utils.matmul_lora(X, W, None, A.as_subclass(_NoReceiverTensor), B, 0.5, **kw())+ torch.testing.assert_close(_plain(got), _plain(want), rtol = 0, atol = 0)++[email protected]("dtype", [torch.bfloat16, torch.float16])[email protected]("which", ["X", "W", "B", "XA", "plain"])+def test_autocast_with_a_precomputed_xa_keeps_fp32_operands_finite(which, dtype, monkeypatch):+ # XA is computed outside autocast. The in-place fused order is not autocast, so fp32 operands stay fp32+ # and finite; a functional addmm would autocast and give 10240 in bf16, inf in fp16.+ X, A = torch.ones(1, 1), torch.ones(1, 1)+ W, B = torch.full((1, 1), -60000.0), torch.full((1, 1), 35000.0)+ XA = X @ A.t()+ if which == "XA":+ XA = _tag(XA)+ X, W, A, B = _tag_one(which, X, W, A, B)+ _set(monkeypatch, True)+ with torch.autocast("cpu", dtype = dtype):+ got = kernel_utils.matmul_lora(X, W, None, A, B, 2.0, XA = XA)+ assert got.dtype == torch.float32 and float(got) == 10000.0+ assert (type(got) is _TaggedTensor) == (which != "plain")++[email protected]("dtype", [torch.bfloat16, torch.float16])[email protected]("which", ["X", "W", "B", "XA"])+def test_autocast_with_a_half_xa_over_an_fp32_base_autocasts_the_base(which, dtype, monkeypatch):+ # Half activations and adapters over an fp32 base weight take the base-then-LoRA order under autocast.+ X, A, B = (torch.ones(*s, dtype = dtype) for s in ((2, 4), (1, 4), (3, 1)))+ W = torch.ones(3, 4)+ XA = X @ A.t()+ if which == "XA":+ XA = _tag(XA)+ X, W, A, B = _tag_one(which, X, W, A, B)+ _set(monkeypatch, True)+ with torch.autocast("cpu", dtype = dtype):+ got = kernel_utils.matmul_lora(X, W, None, A, B, 1.0, XA = XA)+ assert got.dtype == dtype and (_plain(got) == 8).all()+ assert type(got) is _TaggedTensor++[email protected]("bias", [False, True])[email protected]("which", ["X_lora", "A", "B"])+def test_the_dropout_and_bias_route_keeps_a_subclass(which, bias, monkeypatch):+ g = torch.Generator().manual_seed(0)+ X, W = torch.randn(2, 8, 64, generator = g), torch.randn(32, 64, generator = g)+ A, B = torch.randn(4, 64, generator = g), torch.randn(32, 4, generator = g)+ X_lora = X * 0.5+ b = torch.randn(32, generator = g) if bias else None+ want = fast_lora._matmul_lora_ext(X, X_lora, W, None, A, B, 0.5, b)+ X_lora, A, B = (+ _tag(t) if n == which else t for n, t in zip(("X_lora", "A", "B"), (X_lora, A, B))+ )+ got = fast_lora._matmul_lora_ext(X, X_lora, W, None, A, B, 0.5, b)+ assert got.shape == want.shape+ _same(got, want, True)++[email protected]("bsz", [1, 3])[email protected]("which", ["A", "B"])+def test_decode_keeps_a_subclass_adapter(which, bsz, monkeypatch):+ g = torch.Generator().manual_seed(0)+ X, W = torch.randn(bsz, 1, 64, generator = g), torch.randn(32, 64, generator = g)+ A, B = torch.randn(4, 64, generator = g), torch.randn(32, 4, generator = g)+ for name in (+ "_has_non_lora_adapter",+ "_has_multiple_active_adapters",+ "_has_active_dora_adapter",+ "_has_active_lora_bias",+ "_has_packed_qweight",+ ):+ monkeypatch.setattr(kernel_utils, name, lambda proj: False)++ def run(A, B):+ monkeypatch.setattr(+ kernel_utils, "get_lora_parameters_bias", lambda proj: (W, None, A, B, 0.5, None)+ )+ return kernel_utils.fast_linear_forward(None, X)++ want = run(A, B)+ got = run(_tag(A), B) if which == "A" else run(A, _tag(B))+ assert got.shape == want.shape+ _same(got, want, True)++[email protected]("which", ["A", "B"])+def test_the_traced_no_grad_accumulate_keeps_a_subclass_result(which, monkeypatch):+ # The opaque op returns nothing, so a subclass operand takes the in-place method and keeps its type.+ monkeypatch.setattr(torch.compiler, "is_compiling", lambda: True)+ out, A, B = torch.zeros(3, 4), torch.randn(3, 2), torch.randn(2, 4)+ want = torch.zeros(3, 4).addmm_(A, B, alpha = 0.5)+ A, B = (_tag(A), B) if which == "A" else (A, _tag(B))+ with torch.no_grad():+ got = kernel_utils.addmm_(out, A, B, alpha = 0.5)+ assert got.data_ptr() == out.data_ptr()+ _same(got, want, True)+++def test_the_traced_no_grad_accumulate_keeps_the_op_for_plain_tensors(monkeypatch):+ if not kernel_utils._HAS_ADDMM_OP:+ pytest.skip("torch.library.custom_op is not available")+ monkeypatch.setattr(torch.compiler, "is_compiling", lambda: True)+ calls = []+ real = torch.ops.unsloth.addmm_++ def op(*args):+ calls.append(len(args))+ return real(*args)++ monkeypatch.setattr(torch.ops.unsloth, "addmm_", op)+ with torch.no_grad():+ kernel_utils.addmm_(torch.zeros(3, 4), torch.randn(3, 2), torch.randn(2, 4), alpha = 0.5)+ assert calls == [5]diff --git a/unsloth/kernels/fast_lora.py b/unsloth/kernels/fast_lora.pyindex a528bf8ae52..758b5a94ed1 100644--- a/unsloth/kernels/fast_lora.py+++ b/unsloth/kernels/fast_lora.py@@ -314,12 +314,13 @@ def _matmul_lora_ext(X, X_lora, W, W_quant, A, B, S, bias):XA = torch.matmul(X_lora.reshape(-1, X_lora.shape[-1]), A.to(dtype).t())out2d = out.view(-1, out.shape[-1])if bias is None:- addmm_(out2d, XA, B.to(dtype).t(), alpha = S)+ out2d = addmm_(out2d, XA, B.to(dtype).t(), alpha = S)else:# PEFT's rounding order: (lora_B(x) + bias) * scaling, then + base. No add(alpha = S): Inductor# (torch 2.13) miscompiles it traced inside these Functions.- out2d.add_(torch.addmm(bias.to(dtype), XA, B.to(dtype).t()).mul_(S))- return out+ out2d = out2d.add_(torch.addmm(bias.to(dtype), XA, B.to(dtype).t()).mul_(S))+ # A subclass operand comes back from the in-place update as its own type; keep it.+ return out if type(out2d) is type(out) else out2d.view(out.shape)def _add_lora_dX(dX, y_dB, A, S, mask, p):diff --git a/unsloth/kernels/utils.py b/unsloth/kernels/utils.pyindex 75a34542104..cc094ced4bf 100644--- a/unsloth/kernels/utils.py+++ b/unsloth/kernels/utils.py@@ -1848,13 +1848,13 @@ def fast_linear_forward(# Without out=, mv / mm follow a different autocast dtype; the in-place update does not.if temp_lora.dtype != dtype:temp_lora = temp_lora.to(dtype)- out.addmv_(B, temp_lora, alpha = lora_S)+ out = out.addmv_(B, temp_lora, alpha = lora_S)else:out = out.view(bsz, out_dim)temp_lora = torch_mm(X.view(bsz, in_dim), A.t(), out = temp_lora)if temp_lora.dtype != dtype:temp_lora = temp_lora.to(dtype)- out.addmm_(temp_lora, B.t(), alpha = lora_S)+ out = out.addmm_(temp_lora, B.t(), alpha = lora_S)out = out.view(bsz, 1, out_dim)if bias is not None:@@ -1875,9 +1875,13 @@ def _lora_first_addmm(X, Wt, XA, Bt, alpha):out = torch.addmm(XA.new_zeros(1), XA, Bt, beta = 0, alpha = alpha)return torch.addmm(out, X, Wt)out = torch.empty((X.shape[0], Wt.shape[1]), dtype = X.dtype, device = X.device)- out.addmm_(XA, Bt, alpha = alpha, beta = 0)- out.addmm_(X, Wt)- return out+ # Both updates mutate the plain buffer; a subclass operand's returned type is kept for the result.+ return _accumulated(out.addmm_(XA, Bt, alpha = alpha, beta = 0), out.addmm_(X, Wt))+++def _accumulated(lora_out, base_out):+ # The more derived of the two returned types, else the base GEMM's.+ return lora_out if isinstance(lora_out, type(base_out)) else base_out# Traced, x.addmm_ copies x first; this op stays in place. No derivative: grad off only.@@ -1941,6 +1945,20 @@ if _HAS_ADDMM_OP:_HAS_LORA_FIRST_OP = False+# Tracing infrastructure (make_fx, export) wraps plain tensors; those are not user subclasses.+_LORA_PLAIN_TYPES = (torch.Tensor, torch.nn.Parameter)+try:+ from torch._subclasses.fake_tensor import FakeTensor as _FakeTensor+ from torch._subclasses.functional_tensor import FunctionalTensor as _FunctionalTensor+ _LORA_PLAIN_TYPES += (_FakeTensor, _FunctionalTensor)+except Exception:+ pass+++def _lora_subclass(*tensors):+ return any(t is not None and type(t) not in _LORA_PLAIN_TYPES for t in tensors)++def addmm_(out,A,@@ -1953,6 +1971,8 @@ def addmm_(and torch.compiler.is_compiling()and not torch.is_grad_enabled()and type(alpha) in (int, float)+ # The op returns nothing, so a subclass operand takes the method below, which returns its type.+ and not _lora_subclass(out, A, B)):torch.ops.unsloth.addmm_(out, A, B, alpha, beta)return out@@ -2096,8 +2116,7 @@ def matmul_lora(# Eager only: a traced caller buffer cannot be checked for aliasing (_shares_storage). Dequantize# first: the caller's buffer may share storage with the quant state (absmax, code, ...).W = fast_dequantize(W, W_quant, use_global_buffer = use_global_buffer, dtype = dtype)- addmm_(out, XA, B, alpha = s, beta = 0)- out.addmm_(X, W.t())+ out = _accumulated(addmm_(out, XA, B, alpha = s, beta = 0), out.addmm_(X, W.t()))else:W = fast_dequantize(W, W_quant, use_global_buffer = use_global_buffer, dtype = dtype)if not _is_compiling():@@ -2125,6 +2144,6 @@ def matmul_lora(A, B = A.t(), B.t()if XA is None:XA = torch_matmul(X, A.to(dtype))- addmm_(out, XA, B.to(dtype), alpha = s)+ out = addmm_(out, XA, B.to(dtype), alpha = s)return out.view(batch, seq_len, -1) if reshape else out