Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
LoRA decode: return the caller's out tensor so batched decode keeps one mm per projection (#16588)
Daniel Han562d80223392
diff --git a/tests/test_lora_decode_out_strides.py b/tests/test_lora_decode_out_strides.pynew file mode 100644index 00000000000..f92209aa165--- /dev/null+++ b/tests/test_lora_decode_out_strides.py@@ -0,0 +1,61 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.++"""LoRA decode returns the caller's strided `out` as is, so torch.matmul on it still folds to one mm."""++import pytest+import torch++if not torch.cuda.is_available():+ pytest.skip(+ reason = "real bnb 4-bit / CUDA decode path: needs a CUDA GPU", allow_module_level = True+ )+peft = pytest.importorskip("peft")+bnb = pytest.importorskip("bitsandbytes")++import unsloth # noqa: F401 (must precede transformers / peft users)+from unsloth.kernels.utils import fast_linear_forward++DTYPE = torch.bfloat16 if torch.cuda.get_device_capability()[0] >= 8 else torch.float16+N, K = 512, 256+++class _Tiny(torch.nn.Module):+ def __init__(self, load_in_4bit):+ super().__init__()+ if load_in_4bit:+ self.q = bnb.nn.Linear4bit(K, N, bias = False, compute_dtype = DTYPE, quant_type = "nf4")+ w = torch.randn(N, K, generator = torch.Generator().manual_seed(0)) * 0.02+ self.q.weight = bnb.nn.Params4bit(w.to(DTYPE), requires_grad = False, quant_type = "nf4")+ else:+ self.q = torch.nn.Linear(K, N, bias = False, dtype = DTYPE)+++def _proj(load_in_4bit):+ model = _Tiny(load_in_4bit).cuda()+ model = peft.get_peft_model(+ model, peft.LoraConfig(r = 8, lora_alpha = 16, target_modules = ["q"], lora_dropout = 0.0)+ )+ with torch.no_grad():+ for name, p in model.named_parameters():+ if "lora_B" in name:+ p.copy_(torch.randn_like(p) * 0.05)+ proj = model.base_model.model.q+ proj.eval()+ return proj++[email protected]("load_in_4bit", [False, True])[email protected]("bsz", [1, 2, 8, 33])+def test_lora_decode_returns_out_with_its_strides(load_in_4bit, bsz):+ proj = _proj(load_in_4bit)+ X = torch.randn(bsz, 1, K, device = "cuda", dtype = DTYPE)+ buffer = torch.empty(bsz, 1, N + 64, device = "cuda", dtype = DTYPE)+ out = buffer[:, :, :N]+ with torch.inference_mode():+ y = fast_linear_forward(proj, X, out = out)+ ref = proj(X)+ assert y.data_ptr() == out.data_ptr() and y.shape == out.shape and y.stride() == out.stride()+ # torch.matmul's fold precondition for a 3-D input.+ assert y.stride(0) == y.stride(1) * y.shape[1]+ torch.testing.assert_close(y.float(), ref.float(), rtol = 2e-2, atol = 2e-2)diff --git a/tests/test_lora_fused_passes.py b/tests/test_lora_fused_passes.pyindex c6f14c3c77b..51d1eab410b 100644--- a/tests/test_lora_fused_passes.py+++ b/tests/test_lora_fused_passes.py@@ -1123,6 +1123,33 @@ def test_decode_packed_base_keeps_its_matmul(bsz, fp8):assert calls == [X.shape][email protected]("bsz", [1, 3])[email protected]("tagged", [False, True])+def test_decode_returns_the_strided_out_with_its_strides(tagged, bsz, monkeypatch):+ # The MLP writes down_proj into temp_up[..., :hd]: the caller's strided out comes back as is.+ g = torch.Generator().manual_seed(0)+ X, W = torch.randn(bsz, 1, 64, generator = g), torch.randn(32, 64, generator = g)+ A, B = torch.randn(4, 64, generator = g), torch.randn(32, 4, generator = g)+ for name in (+ "_has_non_lora_adapter",+ "_has_multiple_active_adapters",+ "_has_active_dora_adapter",+ "_has_active_lora_bias",+ "_has_packed_qweight",+ ):+ monkeypatch.setattr(kernel_utils, name, lambda proj: False)+ B_in = _tag(B) if tagged else B+ monkeypatch.setattr(+ kernel_utils, "get_lora_parameters_bias", lambda proj: (W, None, A, B_in, 0.5, None)+ )+ out = torch.empty(bsz, 1, 40)[:, :, :32]+ got = kernel_utils.fast_linear_forward(None, X, out = out)+ assert got.data_ptr() == out.data_ptr() and got.stride() == out.stride()+ assert (type(got) is _TaggedTensor) == tagged+ want = X @ W.t() + 0.5 * (X @ A.t()) @ B.t()+ torch.testing.assert_close(_plain(got), want, rtol = 1e-5, atol = 1e-5)++@pytest.mark.parametrize("which", ["A", "B"])def test_the_traced_no_grad_accumulate_keeps_a_subclass_result(which, monkeypatch):# The opaque op returns nothing, so a subclass operand takes the in-place method and keeps its type.diff --git a/unsloth/kernels/utils.py b/unsloth/kernels/utils.pyindex 708d9d928b5..e3d1b4eba78 100644--- a/unsloth/kernels/utils.py+++ b/unsloth/kernels/utils.py@@ -1847,20 +1847,22 @@ def fast_linear_forward(else:A, B = _fast_lora_weights(lora_A, lora_B, dtype)+ # Update through a 1-D / 2-D view but return `out` itself: a re-view of a strided buffer+ # (down_proj writes into temp_up[..., :hd]) breaks torch.matmul's fold to one mm downstream.if bsz == 1:- out = out.view(out_dim)temp_lora = torch_mv(A, X.ravel(), out = temp_lora)# Without out=, mv / mm follow a different autocast dtype; the in-place update does not.if temp_lora.dtype != dtype:temp_lora = temp_lora.to(dtype)- out = out.addmv_(B, temp_lora, alpha = lora_S)+ res = out.view(out_dim).addmv_(B, temp_lora, alpha = lora_S)else:- out = out.view(bsz, out_dim)temp_lora = torch_mm(X.view(bsz, in_dim), A.t(), out = temp_lora)if temp_lora.dtype != dtype:temp_lora = temp_lora.to(dtype)- out = out.addmm_(temp_lora, B.t(), alpha = lora_S)- out = out.view(bsz, 1, out_dim)+ res = out.view(bsz, out_dim).addmm_(temp_lora, B.t(), alpha = lora_S)+ # A subclass operand makes the in-place method return its type (#16249).+ if type(res) is not type(out):+ out = out.as_subclass(type(res))if bias is not None:out += bias