Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Multi-LoRA: refuse 8-bit bitsandbytes base layers (#16648)

Daniel Hanf9c20c072317
diff --git a/tests/test_multi_lora_int8_base.py b/tests/test_multi_lora_int8_base.py
new file mode 100644
index 00000000000..483c0915223
--- /dev/null
+++ b/tests/test_multi_lora_int8_base.py
@@ -0,0 +1,50 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+from __future__ import annotations
+
+import pytest
+import torch
+from torch import nn
+import unsloth # noqa: F401
+
+from unsloth.kernels import multi_lora as ml
+
+
+def _model(proj: nn.Module) -> nn.Module:
+ model = nn.Module()
+ model.layer = nn.Module()
+ model.layer.proj = proj
+ model.layer.dense = nn.Linear(16, 12, bias = False)
+ return model
+
+
+def _assert_refused_untouched(model: nn.Module, proj: nn.Module) -> None:
+ with pytest.raises(NotImplementedError, match = r"layer\.proj.*4-bit or in 16-bit"):
+ ml.inject_multi_lora(model, ["proj", "dense"], num_adapters = 2, r = 2)
+ assert model.layer.proj is proj
+ assert isinstance(model.layer.dense, nn.Linear)
+ assert model.layer.dense.weight.requires_grad
+
+
+def test_bitsandbytes_linear8bit_base_is_refused():
+ bnb = pytest.importorskip("bitsandbytes")
+ proj = bnb.nn.Linear8bitLt(16, 12, bias = False, has_fp16_weights = False)
+ _assert_refused_untouched(_model(proj), proj)
+ with pytest.raises(NotImplementedError, match = "Linear8bitLt"):
+ ml.MultiLoraLinear(proj, 2, r = 2)
+
+
+def test_a_weight_carrying_int8_scales_is_refused():
+ proj = nn.Linear(16, 12, bias = False)
+ proj.weight.SCB = torch.ones(12)
+ _assert_refused_untouched(_model(proj), proj)
+
+
+def test_a_dense_base_is_still_wrapped():
+ model = _model(nn.Linear(16, 12, bias = False))
+ assert ml.inject_multi_lora(model, ["proj", "dense"], num_adapters = 2, r = 2) == [
+ "layer.proj",
+ "layer.dense",
+ ]
+ assert isinstance(model.layer.proj, ml.MultiLoraLinear)
diff --git a/unsloth/kernels/multi_lora.py b/unsloth/kernels/multi_lora.py
index 7365ff6bd94..9572facd8dc 100644
--- a/unsloth/kernels/multi_lora.py
+++ b/unsloth/kernels/multi_lora.py
@@ -590,6 +590,20 @@ def multi_lora_linear(X, W, W_quant, bias, A, B, scales, batch: Optional[MultiLo
return Y
+def _refuse_int8_base(name: str, base: nn.Module) -> None:
+ """An 8-bit bitsandbytes weight would be read as a dense int8 matrix with its scales dropped."""
+ W = getattr(base, "weight", None)
+ if (
+ any(cls.__name__ == "Linear8bitLt" for cls in type(base).__mro__)
+ or any(cls.__name__ == "Int8Params" for cls in type(W).__mro__)
+ or getattr(W, "SCB", None) is not None
+ ):
+ raise NotImplementedError(
+ f"Multi-LoRA does not support 8-bit bitsandbytes base layers ({name!r} is "
+ f"{type(base).__name__}). Load the base model in 4-bit or in 16-bit instead."
+ )
+
+
class MultiLoraLinear(nn.Module):
"""Frozen base linear with ``num_adapters`` LoRA adapters, separate (A, B) params each (PEFT init).
@@ -605,6 +619,7 @@ class MultiLoraLinear(nn.Module):
use_rslora: bool = False,
dtype: torch.dtype = torch.float32,
):
+ _refuse_int8_base(type(base).__name__, base)
super().__init__()
self.base_layer = base
for p in base.parameters():
@@ -687,6 +702,8 @@ def inject_multi_lora(
]
if not found:
raise ValueError(f"No linear layers named {targets} found.")
+ for name, module in found:
+ _refuse_int8_base(name, module)
_freeze_all_but_adapters(model)
for name, module in found:
parent_name, _, child = name.rpartition(".")