Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

LoRA decode: return the caller's out tensor so batched decode keeps one mm per projection (#16588)

Daniel Han562d80223392
diff --git a/tests/test_lora_decode_out_strides.py b/tests/test_lora_decode_out_strides.py
new file mode 100644
index 00000000000..f92209aa165
--- /dev/null
+++ b/tests/test_lora_decode_out_strides.py
@@ -0,0 +1,61 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
+
+"""LoRA decode returns the caller's strided `out` as is, so torch.matmul on it still folds to one mm."""
+
+import pytest
+import torch
+
+if not torch.cuda.is_available():
+ pytest.skip(
+ reason = "real bnb 4-bit / CUDA decode path: needs a CUDA GPU", allow_module_level = True
+ )
+peft = pytest.importorskip("peft")
+bnb = pytest.importorskip("bitsandbytes")
+
+import unsloth # noqa: F401 (must precede transformers / peft users)
+from unsloth.kernels.utils import fast_linear_forward
+
+DTYPE = torch.bfloat16 if torch.cuda.get_device_capability()[0] >= 8 else torch.float16
+N, K = 512, 256
+
+
+class _Tiny(torch.nn.Module):
+ def __init__(self, load_in_4bit):
+ super().__init__()
+ if load_in_4bit:
+ self.q = bnb.nn.Linear4bit(K, N, bias = False, compute_dtype = DTYPE, quant_type = "nf4")
+ w = torch.randn(N, K, generator = torch.Generator().manual_seed(0)) * 0.02
+ self.q.weight = bnb.nn.Params4bit(w.to(DTYPE), requires_grad = False, quant_type = "nf4")
+ else:
+ self.q = torch.nn.Linear(K, N, bias = False, dtype = DTYPE)
+
+
+def _proj(load_in_4bit):
+ model = _Tiny(load_in_4bit).cuda()
+ model = peft.get_peft_model(
+ model, peft.LoraConfig(r = 8, lora_alpha = 16, target_modules = ["q"], lora_dropout = 0.0)
+ )
+ with torch.no_grad():
+ for name, p in model.named_parameters():
+ if "lora_B" in name:
+ p.copy_(torch.randn_like(p) * 0.05)
+ proj = model.base_model.model.q
+ proj.eval()
+ return proj
+
+
[email protected]("load_in_4bit", [False, True])
[email protected]("bsz", [1, 2, 8, 33])
+def test_lora_decode_returns_out_with_its_strides(load_in_4bit, bsz):
+ proj = _proj(load_in_4bit)
+ X = torch.randn(bsz, 1, K, device = "cuda", dtype = DTYPE)
+ buffer = torch.empty(bsz, 1, N + 64, device = "cuda", dtype = DTYPE)
+ out = buffer[:, :, :N]
+ with torch.inference_mode():
+ y = fast_linear_forward(proj, X, out = out)
+ ref = proj(X)
+ assert y.data_ptr() == out.data_ptr() and y.shape == out.shape and y.stride() == out.stride()
+ # torch.matmul's fold precondition for a 3-D input.
+ assert y.stride(0) == y.stride(1) * y.shape[1]
+ torch.testing.assert_close(y.float(), ref.float(), rtol = 2e-2, atol = 2e-2)
diff --git a/tests/test_lora_fused_passes.py b/tests/test_lora_fused_passes.py
index c6f14c3c77b..51d1eab410b 100644
--- a/tests/test_lora_fused_passes.py
+++ b/tests/test_lora_fused_passes.py
@@ -1123,6 +1123,33 @@ def test_decode_packed_base_keeps_its_matmul(bsz, fp8):
assert calls == [X.shape]
[email protected]("bsz", [1, 3])
[email protected]("tagged", [False, True])
+def test_decode_returns_the_strided_out_with_its_strides(tagged, bsz, monkeypatch):
+ # The MLP writes down_proj into temp_up[..., :hd]: the caller's strided out comes back as is.
+ g = torch.Generator().manual_seed(0)
+ X, W = torch.randn(bsz, 1, 64, generator = g), torch.randn(32, 64, generator = g)
+ A, B = torch.randn(4, 64, generator = g), torch.randn(32, 4, generator = g)
+ for name in (
+ "_has_non_lora_adapter",
+ "_has_multiple_active_adapters",
+ "_has_active_dora_adapter",
+ "_has_active_lora_bias",
+ "_has_packed_qweight",
+ ):
+ monkeypatch.setattr(kernel_utils, name, lambda proj: False)
+ B_in = _tag(B) if tagged else B
+ monkeypatch.setattr(
+ kernel_utils, "get_lora_parameters_bias", lambda proj: (W, None, A, B_in, 0.5, None)
+ )
+ out = torch.empty(bsz, 1, 40)[:, :, :32]
+ got = kernel_utils.fast_linear_forward(None, X, out = out)
+ assert got.data_ptr() == out.data_ptr() and got.stride() == out.stride()
+ assert (type(got) is _TaggedTensor) == tagged
+ want = X @ W.t() + 0.5 * (X @ A.t()) @ B.t()
+ torch.testing.assert_close(_plain(got), want, rtol = 1e-5, atol = 1e-5)
+
+
@pytest.mark.parametrize("which", ["A", "B"])
def test_the_traced_no_grad_accumulate_keeps_a_subclass_result(which, monkeypatch):
# The opaque op returns nothing, so a subclass operand takes the in-place method and keeps its type.
diff --git a/unsloth/kernels/utils.py b/unsloth/kernels/utils.py
index 708d9d928b5..e3d1b4eba78 100644
--- a/unsloth/kernels/utils.py
+++ b/unsloth/kernels/utils.py
@@ -1847,20 +1847,22 @@ def fast_linear_forward(
else:
A, B = _fast_lora_weights(lora_A, lora_B, dtype)
+ # Update through a 1-D / 2-D view but return `out` itself: a re-view of a strided buffer
+ # (down_proj writes into temp_up[..., :hd]) breaks torch.matmul's fold to one mm downstream.
if bsz == 1:
- out = out.view(out_dim)
temp_lora = torch_mv(A, X.ravel(), out = temp_lora)
# Without out=, mv / mm follow a different autocast dtype; the in-place update does not.
if temp_lora.dtype != dtype:
temp_lora = temp_lora.to(dtype)
- out = out.addmv_(B, temp_lora, alpha = lora_S)
+ res = out.view(out_dim).addmv_(B, temp_lora, alpha = lora_S)
else:
- out = out.view(bsz, out_dim)
temp_lora = torch_mm(X.view(bsz, in_dim), A.t(), out = temp_lora)
if temp_lora.dtype != dtype:
temp_lora = temp_lora.to(dtype)
- out = out.addmm_(temp_lora, B.t(), alpha = lora_S)
- out = out.view(bsz, 1, out_dim)
+ res = out.view(bsz, out_dim).addmm_(temp_lora, B.t(), alpha = lora_S)
+ # A subclass operand makes the in-place method return its type (#16249).
+ if type(res) is not type(out):
+ out = out.as_subclass(type(res))
if bias is not None:
out += bias