Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Multi-LoRA: keep the base output dtype on the PyTorch path under autocast (#16649)

Daniel Hanb13d8b0ebaba
diff --git a/tests/test_multi_lora_cpu_autocast_dtype.py b/tests/test_multi_lora_cpu_autocast_dtype.py
new file mode 100644
index 00000000000..f9bc510839e
--- /dev/null
+++ b/tests/test_multi_lora_cpu_autocast_dtype.py
@@ -0,0 +1,34 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+from __future__ import annotations
+
+import pytest
+import torch
+from torch import nn
+import unsloth # noqa: F401
+
+from unsloth.kernels import multi_lora as ml
+
+
[email protected]("bias", [False, True])
+def test_cpu_autocast_output_dtype_matches_the_base_layer(bias):
+ torch.manual_seed(3)
+ lin = nn.Linear(8, 4, bias = bias)
+ wrapped = ml.MultiLoraLinear(lin, 2, r = 2)
+ with torch.no_grad():
+ for B in wrapped.lora_B:
+ B.normal_()
+ ids = torch.tensor([0, 1, -1])
+ ml.set_multi_lora_batch(wrapped, ids)
+ x = torch.randn(3, 8)
+ with torch.autocast("cpu", dtype = torch.bfloat16):
+ base_out = lin(x)
+ out = wrapped(x)
+ assert base_out.dtype == torch.bfloat16
+ assert out.dtype == base_out.dtype
+ A, B = wrapped.stacked()
+ delta = ml.multi_lora_delta(x, A, B, wrapped.scales, ml.MultiLoraBatch.from_adapter_ids(ids, 2))
+ torch.testing.assert_close(out.float(), base_out.float() + delta, rtol = 2e-2, atol = 2e-2)
+ out.float().sum().backward()
+ assert all(p.grad is not None and p.grad.dtype == torch.float32 for p in wrapped.lora_B)
diff --git a/unsloth/kernels/multi_lora.py b/unsloth/kernels/multi_lora.py
index 9572facd8dc..80c3803502f 100644
--- a/unsloth/kernels/multi_lora.py
+++ b/unsloth/kernels/multi_lora.py
@@ -584,7 +584,9 @@ def multi_lora_linear(X, W, W_quant, bias, A, B, scales, batch: Optional[MultiLo
Y = MultiLoRA_W.apply(X, W, W_quant, A, B, scales, batch.sorted_idx, batch.seg_start)
else:
Wd = fast_dequantize(W, W_quant, dtype = X.dtype) if W_quant is not None else W.to(X.dtype)
- Y = F.linear(X, Wd) + multi_lora_delta(X, A, B, scales, batch)
+ Y = F.linear(X, Wd)
+ # Autocast: base matmul is low precision but the delta promotes to X.dtype.
+ Y = Y + multi_lora_delta(X, A, B, scales, batch).to(Y.dtype)
if bias is not None:
Y = Y + bias.to(Y.dtype)
return Y