Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Multi-LoRA: keep the base output dtype on the PyTorch path under autocast (#16649)
Daniel Hanb13d8b0ebaba
diff --git a/tests/test_multi_lora_cpu_autocast_dtype.py b/tests/test_multi_lora_cpu_autocast_dtype.pynew file mode 100644index 00000000000..f9bc510839e--- /dev/null+++ b/tests/test_multi_lora_cpu_autocast_dtype.py@@ -0,0 +1,34 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++from __future__ import annotations++import pytest+import torch+from torch import nn+import unsloth # noqa: F401++from unsloth.kernels import multi_lora as ml++[email protected]("bias", [False, True])+def test_cpu_autocast_output_dtype_matches_the_base_layer(bias):+ torch.manual_seed(3)+ lin = nn.Linear(8, 4, bias = bias)+ wrapped = ml.MultiLoraLinear(lin, 2, r = 2)+ with torch.no_grad():+ for B in wrapped.lora_B:+ B.normal_()+ ids = torch.tensor([0, 1, -1])+ ml.set_multi_lora_batch(wrapped, ids)+ x = torch.randn(3, 8)+ with torch.autocast("cpu", dtype = torch.bfloat16):+ base_out = lin(x)+ out = wrapped(x)+ assert base_out.dtype == torch.bfloat16+ assert out.dtype == base_out.dtype+ A, B = wrapped.stacked()+ delta = ml.multi_lora_delta(x, A, B, wrapped.scales, ml.MultiLoraBatch.from_adapter_ids(ids, 2))+ torch.testing.assert_close(out.float(), base_out.float() + delta, rtol = 2e-2, atol = 2e-2)+ out.float().sum().backward()+ assert all(p.grad is not None and p.grad.dtype == torch.float32 for p in wrapped.lora_B)diff --git a/unsloth/kernels/multi_lora.py b/unsloth/kernels/multi_lora.pyindex 9572facd8dc..80c3803502f 100644--- a/unsloth/kernels/multi_lora.py+++ b/unsloth/kernels/multi_lora.py@@ -584,7 +584,9 @@ def multi_lora_linear(X, W, W_quant, bias, A, B, scales, batch: Optional[MultiLoY = MultiLoRA_W.apply(X, W, W_quant, A, B, scales, batch.sorted_idx, batch.seg_start)else:Wd = fast_dequantize(W, W_quant, dtype = X.dtype) if W_quant is not None else W.to(X.dtype)- Y = F.linear(X, Wd) + multi_lora_delta(X, A, B, scales, batch)+ Y = F.linear(X, Wd)+ # Autocast: base matmul is low precision but the delta promotes to X.dtype.+ Y = Y + multi_lora_delta(X, A, B, scales, batch).to(Y.dtype)if bias is not None:Y = Y + bias.to(Y.dtype)return Y