Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Multi-LoRA: per-adapter LoRA on fused MoE experts (#15742)

Daniel Hanbbfa0c0a5cea
diff --git a/tests/test_multi_lora_experts.py b/tests/test_multi_lora_experts.py
new file mode 100644
index 00000000000..82493859a4c
--- /dev/null
+++ b/tests/test_multi_lora_experts.py
@@ -0,0 +1,269 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""Batched expert LoRA on a fused-experts MoE: each adapter's rows match a lone PEFT run of that adapter."""
+
+from __future__ import annotations
+
+import copy
+
+import pytest
+import torch
+import unsloth # noqa: F401
+
+from real_accelerator import has_real_cuda
+from unsloth.kernels import multi_lora as ml
+
+transformers = pytest.importorskip("transformers")
+peft = pytest.importorskip("peft")
+Qwen3_5Moe = getattr(transformers, "Qwen3_5MoeForConditionalGeneration", None)
+pytestmark = pytest.mark.skipif(Qwen3_5Moe is None, reason = "transformers has no Qwen3.5 MoE")
+
+REPO = "trl-internal-testing/tiny-Qwen3_5MoeForConditionalGeneration-3.6"
+DENSE = ["q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj"]
+EXPERTS = ["mlp.experts.down_proj", "mlp.experts.gate_up_proj"]
+RANKS, ALPHAS = [2, 4, 3], [4.0, 8.0, 3.0]
+T = 6
+
+
+# Zoo's MoE forward launches Triton whenever CUDA is up, so on a GPU host everything runs on the GPU.
+DEVICE = "cuda" if has_real_cuda() else "cpu"
+TOL = 1e-4 if DEVICE == "cuda" else 1e-5
+
+
[email protected](scope = "module")
+def base():
+ from huggingface_hub import snapshot_download
+ path = snapshot_download(REPO)
+ return Qwen3_5Moe.from_pretrained(path, torch_dtype = torch.float32).eval().to(DEVICE)
+
+
+def _batched(base):
+ torch.manual_seed(0)
+ model = copy.deepcopy(base)
+ lm = model.model.language_model
+ ml.inject_multi_lora(lm, DENSE, 3, RANKS, ALPHAS)
+ wrapped = ml.inject_multi_lora_experts(lm, EXPERTS, 3, RANKS, ALPHAS)
+ with torch.no_grad():
+ # Non-zero B: with PEFT's zero init a wrong adapter would still give the base output.
+ for _, layer in ml._layers(model):
+ for B in layer.lora_B:
+ B.normal_(0, 0.5)
+ return model, wrapped
+
+
+def _lone(base, model, u):
+ """Adapter ``u`` exported, loaded into a plain PEFT model of the base, as a lone job would run it."""
+ cfg = peft.LoraConfig(
+ r = RANKS[u],
+ lora_alpha = ALPHAS[u],
+ target_modules = DENSE,
+ target_parameters = EXPERTS,
+ lora_dropout = 0.0,
+ )
+ lone = peft.get_peft_model(copy.deepcopy(base), cfg)
+ state = ml.adapter_state_dict(model, u)
+ result = peft.set_peft_model_state_dict(lone, state)
+ assert not result.unexpected_keys, result.unexpected_keys[:3]
+ assert not [k for k in result.missing_keys if "lora_" in k]
+ return lone
+
+
+def _ids():
+ return torch.randint(0, 100, (3, T), generator = torch.Generator().manual_seed(1)).to(DEVICE)
+
+
[email protected]("mode", ["stash", "fold"])
+def test_each_adapter_matches_its_lone_peft_run(base, monkeypatch, mode):
+ if mode == "fold":
+ monkeypatch.setattr(ml, "_zoo_stash_read_store", lambda experts: None)
+ model, wrapped = _batched(base)
+ assert len(wrapped) == 2 * len(model.model.language_model.layers)
+ ids = _ids()
+ ml.set_multi_lora_batch(model, torch.tensor([0, 1, 2], device = DEVICE), seq_len = T)
+ out = model(input_ids = ids, use_cache = False).logits
+ modes = {m.mode for m in model.modules() if isinstance(m, ml.MultiLoraExperts)}
+ assert modes == {mode}
+ for u in range(3):
+ lone = _lone(base, model, u)
+ ref = lone(input_ids = ids[u : u + 1], use_cache = False).logits[0]
+ moved = (ref - base(input_ids = ids[u : u + 1]).logits[0]).abs().max()
+ assert moved > 1e-3 # the adapter really changes the output
+ torch.testing.assert_close(out[u], ref, atol = TOL, rtol = 1e-4)
+
+
+def test_expert_grads_match_the_lone_run(base):
+ model, _ = _batched(base)
+ ids = _ids()
+ ml.set_multi_lora_batch(model, torch.tensor([0, 1, 2], device = DEVICE), seq_len = T)
+ u = 1
+ model(input_ids = ids, use_cache = False).logits[u].pow(2).mean().backward()
+ lone = _lone(base, model, u)
+ lone(input_ids = ids[u : u + 1], use_cache = False).logits[0].pow(2).mean().backward()
+ layout = ml._peft_moe_lora_b_layout()
+ for name, experts in model.named_modules():
+ if not isinstance(experts, ml.MultiLoraExperts):
+ continue
+ depth = len(experts.parameter_names)
+ for i, param in enumerate(experts.parameter_names):
+ holder = lone.base_model.model.get_submodule(name)
+ for _ in range(depth - 1 - i):
+ holder = holder.base_layer
+ mine = experts.adapters[param]
+ gA, gB = mine.lora_A[u].grad, mine.lora_B[u].grad
+ flat = ml.MultiLoraExpertParam.peft_tensors
+ packed = type("G", (), {"lora_A": {u: gA}, "lora_B": {u: gB}})
+ exp_A, exp_B = flat(packed, u, layout)
+ torch.testing.assert_close(
+ exp_A, holder.lora_A["default"].weight.grad, atol = 1e-6, rtol = 1e-4
+ )
+ torch.testing.assert_close(
+ exp_B, holder.lora_B["default"].weight.grad, atol = 1e-6, rtol = 1e-4
+ )
+ other = experts.adapters[param].lora_A[0].grad
+ assert other is None or other.abs().max() == 0
+ dense = [(n, m) for n, m in model.named_modules() if isinstance(m, ml.MultiLoraLinear)]
+ assert dense
+ for name, layer in dense:
+ holder = lone.base_model.model.get_submodule(name)
+ assert layer.lora_A[u].requires_grad and layer.lora_B[u].requires_grad
+ torch.testing.assert_close(
+ layer.lora_A[u].grad, holder.lora_A["default"].weight.grad, atol = 1e-6, rtol = 1e-4
+ )
+ torch.testing.assert_close(
+ layer.lora_B[u].grad, holder.lora_B["default"].weight.grad, atol = 1e-6, rtol = 1e-4
+ )
+
+
+def test_another_adapter_cannot_move_a_row(base):
+ model, _ = _batched(base)
+ ids = _ids()
+ ml.set_multi_lora_batch(model, torch.tensor([0, 1, 2], device = DEVICE), seq_len = T)
+ before = model(input_ids = ids, use_cache = False).logits.detach()
+ with torch.no_grad():
+ for _, layer in ml._layers(model):
+ layer.lora_A[0].mul_(3.0)
+ layer.lora_B[0].add_(1.0)
+ after = model(input_ids = ids, use_cache = False).logits.detach()
+ assert (after[0] - before[0]).abs().max() > 1e-3
+ torch.testing.assert_close(after[1:], before[1:], atol = 0, rtol = 0)
+
+
+def test_fold_mode_survives_non_reentrant_checkpointing(base, monkeypatch):
+ # A Zoo stash store that this forward never fills: the experts fall back to folding.
+ monkeypatch.setattr(ml, "_zoo_stash_read_store", lambda experts: {})
+ model, _ = _batched(base)
+ ids = _ids()
+ ml.set_multi_lora_batch(model, torch.tensor([0, 1, 2], device = DEVICE), seq_len = T)
+ experts = next(m for m in model.modules() if isinstance(m, ml.MultiLoraExperts))
+ block = next(m for m in model.modules() if getattr(m, "experts", None) is experts)
+ hidden = torch.randn(
+ 3 * T, model.config.text_config.hidden_size, device = DEVICE, requires_grad = True
+ )
+ router = block.gate
+ with torch.no_grad():
+ _, weights, indices = router(hidden)
+ out = torch.utils.checkpoint.checkpoint(experts, hidden, indices, weights, use_reentrant = False)
+ out.sum().backward()
+ assert experts.mode == "fold" and hidden.grad is not None
+
+
+def test_absent_adapters_stay_in_the_graph(base):
+ model, _ = _batched(base)
+ ids = _ids()
+ clean = None
+ with torch.no_grad():
+ ml.set_multi_lora_batch(model, torch.tensor([0, 1, 1], device = DEVICE), seq_len = T)
+ clean = model(input_ids = ids, use_cache = False).logits
+ # A diverged absent adapter must not reach the others' outputs.
+ for m in model.modules():
+ if isinstance(m, ml.MultiLoraExperts):
+ for adapter in m.adapters.values():
+ adapter.lora_B[2].fill_(float("nan"))
+ ml.set_multi_lora_batch(model, torch.tensor([0, 1, 1], device = DEVICE), seq_len = T)
+ logits = model(input_ids = ids, use_cache = False).logits
+ torch.testing.assert_close(logits, clean, atol = 0, rtol = 0)
+ logits.sum().backward()
+ for m in model.modules():
+ if isinstance(m, ml.MultiLoraExperts):
+ for adapter in m.adapters.values():
+ assert (
+ adapter.lora_A[2].grad is not None and adapter.lora_A[2].grad.abs().max() == 0
+ )
+ assert adapter.lora_A[1].grad is not None
+
+
[email protected](scope = "module")
+def input_major(base):
+ """The same experts stored input-major and flagged as Zoo's grouped-mm format (Qwen3-VL-MoE, GPT-OSS)."""
+ model = copy.deepcopy(base)
+ flagged = 0
+ for module in model.modules():
+ if (
+ hasattr(module, "gate_up_proj")
+ and hasattr(module, "down_proj")
+ and module.gate_up_proj.dim() == 3
+ ):
+ with torch.no_grad():
+ for name in ("gate_up_proj", "down_proj"):
+ param = getattr(module, name)
+ param.data = param.data.transpose(1, 2).contiguous()
+ module._unsloth_grouped_mm_format = True
+ flagged += 1
+ assert flagged
+ return model
+
+
[email protected]("mode", ["stash", "fold"])
+def test_input_major_grouped_mm_experts_match_their_lone_peft_run(
+ base, input_major, monkeypatch, mode
+):
+ if mode == "fold":
+ monkeypatch.setattr(ml, "_zoo_stash_read_store", lambda experts: None)
+ ids = _ids()
+ # Zoo's forward reads the flag: the relaid experts compute what the original ones do.
+ torch.testing.assert_close(
+ input_major(input_ids = ids).logits, base(input_ids = ids).logits, atol = TOL, rtol = 1e-4
+ )
+ model, _ = _batched(input_major)
+ ml.set_multi_lora_batch(model, torch.tensor([0, 1, 2], device = DEVICE), seq_len = T)
+ out = model(input_ids = ids, use_cache = False).logits
+ assert {m.mode for m in model.modules() if isinstance(m, ml.MultiLoraExperts)} == {mode}
+ for u in range(3):
+ ref = _lone(input_major, model, u)(input_ids = ids[u : u + 1], use_cache = False).logits[0]
+ torch.testing.assert_close(out[u], ref, atol = TOL, rtol = 1e-4)
+
+
+def test_fold_mode_under_autocast_matches_the_lone_run(base, monkeypatch):
+ monkeypatch.setattr(ml, "_zoo_stash_read_store", lambda experts: None)
+ model, _ = _batched(base)
+ ids = _ids()
+ ml.set_multi_lora_batch(model, torch.tensor([0, 1, 2], device = DEVICE), seq_len = T)
+ with torch.autocast(device_type = DEVICE, dtype = torch.bfloat16):
+ out = model(input_ids = ids, use_cache = False).logits
+ refs = [
+ _lone(base, model, u)(input_ids = ids[u : u + 1], use_cache = False).logits[0]
+ for u in range(3)
+ ]
+ for u in range(3):
+ # One bf16 ulp of the logits; the bf16 fold was off by about 0.1.
+ torch.testing.assert_close(out[u].float(), refs[u].float(), atol = 3e-2, rtol = 0)
+
+
+def test_experts_read_directly_by_their_parent_are_refused():
+ class Experts(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.gate_up_proj = torch.nn.Parameter(torch.zeros(2, 4, 8))
+ self.down_proj = torch.nn.Parameter(torch.zeros(2, 4, 4))
+
+ class FastBlock(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.experts = Experts()
+
+ def forward(self, x): # a grouped-GEMM fast path that never calls self.experts
+ return x @ self.experts.gate_up_proj[0]
+
+ with pytest.raises(ValueError, match = "reads experts.gate_up_proj itself"):
+ ml.inject_multi_lora_experts(FastBlock(), ["experts.gate_up_proj"], 2, 2, 2)
diff --git a/unsloth/kernels/multi_lora.py b/unsloth/kernels/multi_lora.py
index a8022097fa5..7543c379cee 100644
--- a/unsloth/kernels/multi_lora.py
+++ b/unsloth/kernels/multi_lora.py
@@ -8,8 +8,11 @@ CUDA/XPU, uses a PyTorch loop; LoRA dropout unsupported."""
from __future__ import annotations
import contextlib
+import functools
+import inspect
import math
import os
+import re
from dataclasses import dataclass
from typing import Iterable, Optional
@@ -35,9 +38,12 @@ except Exception: # noqa: BLE001
__all__ = [
"MultiLoraBatch",
"MultiLoraLinear",
+ "MultiLoraExperts",
+ "MultiLoraExpertParam",
"multi_lora_delta",
"multi_lora_linear",
"inject_multi_lora",
+ "inject_multi_lora_experts",
"set_multi_lora_batch",
"per_adapter_loss",
"adapter_token_counts",
@@ -654,8 +660,7 @@ def inject_multi_lora(
]
if not found:
raise ValueError(f"No linear layers named {targets} found.")
- for p in model.parameters():
- p.requires_grad_(False)
+ _freeze_all_but_adapters(model)
for name, module in found:
parent_name, _, child = name.rpartition(".")
parent = model.get_submodule(parent_name) if parent_name else model
@@ -663,8 +668,353 @@ def inject_multi_lora(
return [name for name, _ in found]
-def _layers(model: nn.Module) -> list[tuple[str, MultiLoraLinear]]:
- return [(n, m) for n, m in model.named_modules() if isinstance(m, MultiLoraLinear)]
+class MultiLoraExpertParam(nn.Module):
+ """LoRA on one fused expert stack: per adapter, A (E, r, in) and B (E, out, r), each expert as PEFT inits it."""
+
+ def __init__(
+ self,
+ num_experts: int,
+ in_features: int,
+ out_features: int,
+ num_adapters: int,
+ r: int | Iterable[int] = 16,
+ lora_alpha: float | Iterable[float] = 16,
+ use_rslora: bool = False,
+ device = None,
+ dtype: torch.dtype = torch.float32,
+ ):
+ super().__init__()
+ ranks = [int(r)] * num_adapters if isinstance(r, int) else [int(x) for x in r]
+ alphas = (
+ [float(lora_alpha)] * num_adapters
+ if isinstance(lora_alpha, (int, float))
+ else [float(x) for x in lora_alpha]
+ )
+ if len(ranks) != num_adapters or len(alphas) != num_adapters:
+ raise ValueError("Need one rank and one alpha per adapter.")
+ self.ranks = ranks
+ self.num_experts, self.in_features, self.out_features = (
+ num_experts,
+ in_features,
+ out_features,
+ )
+ kw = dict(dtype = dtype, device = device)
+ self.lora_A = nn.ParameterList(
+ [nn.Parameter(torch.zeros(num_experts, k, in_features, **kw)) for k in ranks]
+ )
+ self.lora_B = nn.ParameterList(
+ [nn.Parameter(torch.zeros(num_experts, out_features, k, **kw)) for k in ranks]
+ )
+ self.scales = [a / (math.sqrt(k) if use_rslora else k) for a, k in zip(alphas, ranks)]
+ self.reset_parameters()
+ self.batch: Optional[MultiLoraBatch] = None
+ self.seen: Optional[torch.Tensor] = None
+ self.trained: Optional[torch.Tensor] = None
+
+ @torch.no_grad()
+ def reset_parameters(self):
+ # PEFT inits A as one (E*r, in) matrix, so fan_in is `in` for every expert slice.
+ for A, B in zip(self.lora_A, self.lora_B):
+ for e in range(A.shape[0]):
+ nn.init.kaiming_uniform_(A[e], a = math.sqrt(5))
+ B.zero_()
+
+ @property
+ def num_adapters(self) -> int:
+ return len(self.lora_A)
+
+ def grouped_mm_factors(
+ self,
+ adapter: int,
+ reversed_io: bool = False,
+ ) -> tuple:
+ """Zoo's stash (first (E, in, r), second (E, r, out), scaling, E); ``reversed_io`` swaps the factors' roles
+ as Zoo's _reversed_lora_weights_for_grouped_mm does."""
+ A, B = self.lora_A[adapter], self.lora_B[adapter]
+ if reversed_io:
+ return B.contiguous(), A.contiguous(), self.scales[adapter], self.num_experts
+ return (
+ A.transpose(1, 2).contiguous(),
+ B.transpose(1, 2).contiguous(),
+ self.scales[adapter],
+ self.num_experts,
+ )
+
+ def folded(self, W: torch.Tensor, adapter: int, transposed: bool) -> torch.Tensor:
+ """W + delta in W's dtype, as PEFT's ParamWrapper folds it (one baddbmm, autocast off)."""
+ A, B = self.lora_A[adapter].to(W.dtype), self.lora_B[adapter].to(W.dtype)
+ lhs, rhs = (A.transpose(1, 2), B.transpose(1, 2)) if transposed else (B, A)
+ with torch.autocast(device_type = W.device.type, enabled = False):
+ return torch.baddbmm(W, lhs, rhs, alpha = self.scales[adapter])
+
+ def peft_tensors(self, adapter: int, b_layout: str) -> tuple[torch.Tensor, torch.Tensor]:
+ """PEFT ParamWrapper's packing: lora_A (E*r, in) expert-major, lora_B (out, E*r) in ``b_layout``."""
+ A, B = _full(self.lora_A[adapter]), _full(self.lora_B[adapter])
+ E, out, k = B.shape
+ flat_A = A.reshape(E * k, A.shape[-1])
+ if b_layout == "rank_major":
+ flat_B = B.permute(1, 2, 0).reshape(out, k * E)
+ else:
+ flat_B = B.permute(1, 0, 2).reshape(out, E * k)
+ return flat_A.detach().clone(), flat_B.detach().clone()
+
+
[email protected]_cache(maxsize = None)
+def _peft_expert_in_axis(is_transposed: bool) -> int:
+ """Which axis of an (E, d1, d2) stack the installed PEFT's ParamWrapper calls in_features (it moved in 0.19)."""
+ import peft
+
+ class Experts(nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.w = nn.Parameter(torch.zeros(2, 3, 5))
+ self.is_transposed = is_transposed
+
+ class Block(nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.experts = Experts()
+
+ model = peft.get_peft_model(Block(), peft.LoraConfig(r = 1, target_parameters = ["experts.w"]))
+ A = next(p for n, p in model.named_parameters() if "lora_A" in n)
+ return 1 if A.shape[-1] == 3 else 2
+
+
+def _expert_stack_shape(param) -> tuple[int, int, int]:
+ shape = getattr(param, "_original_shape", None) or tuple(param.shape)
+ if len(shape) != 3:
+ raise ValueError(f"Expected a fused (experts, d1, d2) parameter, got shape {tuple(shape)}.")
+ return tuple(int(x) for x in shape)
+
+
+def _stash_name(parameter_name: str) -> str:
+ return f"_unsloth_lora_{parameter_name}"
+
+
+def _zoo_stash_read_store(experts: nn.Module):
+ try:
+ from unsloth_zoo.temporary_patches.moe_utils import _MOE_LORA_STASH_READ_ATTR
+ except Exception: # noqa: BLE001 - without Unsloth Zoo's MoE forwards nothing reads a stash
+ return None
+ store = experts.__dict__.get(_MOE_LORA_STASH_READ_ATTR)
+ if store is None:
+ store = {}
+ setattr(experts, _MOE_LORA_STASH_READ_ATTR, store)
+ return store
+
+
+def _adapter_groups(batch: MultiLoraBatch) -> list:
+ """(adapter or None, token rows) per adapter present; one host sync per batch, shared by every MoE layer."""
+ groups = batch.__dict__.get("_expert_groups")
+ if groups is None:
+ seg = batch.seg_start.tolist()
+ groups = [
+ (u, batch.sorted_idx[seg[u] : seg[u + 1]].long())
+ for u in range(batch.num_adapters)
+ if seg[u + 1] > seg[u]
+ ]
+ if seg[-1] < batch.num_tokens:
+ groups.append((None, batch.sorted_idx[seg[-1] :].long()))
+ batch.__dict__["_expert_groups"] = groups
+ return groups
+
+
+class MultiLoraExperts(nn.Module):
+ """Frozen fused experts; each adapter's tokens run the experts' own forward with only its factors, as a lone
+ PEFT job does: through Zoo's stash when its MoE forward reads one, else with the delta folded in."""
+
+ def __init__(
+ self,
+ base: nn.Module,
+ parameter_names: Iterable[str],
+ num_adapters: int,
+ r: int | Iterable[int] = 16,
+ lora_alpha: float | Iterable[float] = 16,
+ use_rslora: bool = False,
+ ):
+ super().__init__()
+ self.base_layer = base
+ for p in base.parameters():
+ p.requires_grad_(False)
+ self.parameter_names = tuple(parameter_names)
+ is_transposed = bool(getattr(base, "is_transposed", False))
+ # Shapes and fold follow the installed PEFT; the stash follows Zoo's extractor (stored input axis).
+ peft_in_axis = _peft_expert_in_axis(is_transposed)
+ zoo_in_axis = (
+ 1 if is_transposed or getattr(base, "_unsloth_grouped_mm_format", False) else 2
+ )
+ self.transposed = peft_in_axis == 1
+ self.reversed_io = peft_in_axis != zoo_in_axis
+ self.adapters = nn.ModuleDict()
+ for name in self.parameter_names:
+ param = getattr(base, name)
+ E, d1, d2 = _expert_stack_shape(param)
+ out_features, in_features = (d2, d1) if self.transposed else (d1, d2)
+ device = param.device if param.device.type != "meta" else None
+ self.adapters[name] = MultiLoraExpertParam(
+ E, in_features, out_features, num_adapters, r, lora_alpha, use_rslora, device = device
+ )
+ self.mode: Optional[str] = None # "stash" or "fold", decided on the first adapter call
+
+ def __getattr__(self, name: str):
+ try:
+ return super().__getattr__(name)
+ except AttributeError:
+ return getattr(self._modules["base_layer"], name)
+
+ @property
+ def num_adapters(self) -> int:
+ return next(iter(self.adapters.values())).num_adapters
+
+ def _run_stash(self, adapter: int, args) -> Optional[torch.Tensor]:
+ base = self.base_layer
+ store = _zoo_stash_read_store(base)
+ if store is None:
+ return None
+ for name in self.parameter_names:
+ store.pop(name, None)
+ setattr(
+ base,
+ _stash_name(name),
+ self.adapters[name].grouped_mm_factors(adapter, self.reversed_io),
+ )
+ try:
+ out = base(*args)
+ finally:
+ for name in self.parameter_names:
+ if _stash_name(name) in base.__dict__:
+ delattr(base, _stash_name(name))
+ if all(store.get(name) for name in self.parameter_names):
+ return out
+ return None
+
+ def _run_fold(self, adapter: int, args) -> torch.Tensor:
+ base = self.base_layer
+ weights = {}
+ for name in self.parameter_names:
+ W = getattr(base, name)
+ if W.dtype not in (torch.float16, torch.bfloat16, torch.float32, torch.float64):
+ raise RuntimeError(
+ f"These experts' forward ignores the LoRA stash and {name} is quantized, so its delta cannot be folded."
+ )
+ weights[name] = self.adapters[name].folded(W, adapter, self.transposed)
+ return torch.func.functional_call(base, weights, args)
+
+ def _run(self, adapter: Optional[int], args) -> torch.Tensor:
+ if adapter is None:
+ return self.base_layer(*args)
+ if self.mode is None:
+ # Probe on one token without autograd, so a checkpointed forward and its recompute run the same ops.
+ with torch.no_grad():
+ probe = self._run_stash(adapter, tuple(a[:1] for a in args))
+ self.mode = "fold" if probe is None else "stash"
+ if self.mode == "fold":
+ return self._run_fold(adapter, args)
+ out = self._run_stash(adapter, args)
+ if out is None:
+ raise RuntimeError("The experts' forward stopped reading the LoRA stash mid-run.")
+ return out
+
+ def forward(self, hidden_states: torch.Tensor, *args):
+ batch = next(iter(self.adapters.values())).batch
+ if batch is None or hidden_states.shape[0] != batch.num_tokens:
+ raise ValueError(
+ "Call set_multi_lora_batch(model, ...) with this batch's adapter ids first."
+ )
+ groups = _adapter_groups(batch)
+ if len(groups) == 1:
+ out = self._run(groups[0][0], (hidden_states, *args))
+ else:
+ out = None
+ for u, rows in groups:
+ y = self._run(u, (hidden_states[rows], *(a[rows] for a in args)))
+ if out is None:
+ out = y.new_zeros((hidden_states.shape[0], *y.shape[1:]))
+ out = out.index_copy(0, rows, y)
+ present = {u for u, _ in groups}
+ absent = [u for u in range(self.num_adapters) if u not in present]
+ if absent and torch.is_grad_enabled():
+ # Default DDP needs every trainable parameter used; empty slices stay zero even for an inf / NaN adapter.
+ zero = sum(
+ A[u].reshape(-1)[:0].sum() + B[u].reshape(-1)[:0].sum()
+ for adapter in self.adapters.values()
+ for A, B in ((adapter.lora_A, adapter.lora_B),)
+ for u in absent
+ )
+ out = out + zero.to(out.dtype)
+ return out
+
+
+def inject_multi_lora_experts(
+ model: nn.Module,
+ target_parameters: str | Iterable[str],
+ num_adapters: int,
+ r: int | Iterable[int] = 16,
+ lora_alpha: float | Iterable[float] = 16,
+ use_rslora: bool = False,
+) -> list[str]:
+ """Wrap every fused experts module holding a ``target_parameters`` entry, matched on path suffix as PEFT does."""
+ targets = (
+ (target_parameters,) if isinstance(target_parameters, str) else tuple(target_parameters)
+ )
+ by_module: dict[str, list[str]] = {}
+ for name, module in model.named_modules():
+ if isinstance(module, MultiLoraExperts):
+ continue
+ for param, value in module._parameters.items():
+ if not isinstance(value, torch.Tensor):
+ continue
+ key = f"{name}.{param}"
+ if any(key == t or key.endswith("." + t) for t in targets):
+ by_module.setdefault(name, []).append(param)
+ if not by_module:
+ raise ValueError(f"No fused expert parameters named {targets} found.")
+ for name, params in by_module.items():
+ parent_name, _, child = name.rpartition(".")
+ parent = model.get_submodule(parent_name) if parent_name else model
+ bypass = _reads_experts_directly(parent, child, params)
+ if bypass:
+ raise ValueError(
+ f"{type(parent).__name__}.forward reads {child}.{bypass} itself instead of calling {child}(...), "
+ "so per-adapter expert LoRA would never run there."
+ )
+ _freeze_all_but_adapters(model)
+ for name, params in by_module.items():
+ parent_name, _, child = name.rpartition(".")
+ parent = model.get_submodule(parent_name) if parent_name else model
+ module = parent.get_submodule(child)
+ setattr(
+ parent, child, MultiLoraExperts(module, params, num_adapters, r, lora_alpha, use_rslora)
+ )
+ return [f"{name}.{param}" for name, params in by_module.items() for param in params]
+
+
+def _reads_experts_directly(parent: nn.Module, child: str, params: list) -> Optional[str]:
+ """The targeted parameter a parent's forward reads off its experts (e.g. a grouped-GEMM fast path), if any."""
+ try:
+ source = inspect.getsource(type(parent).forward)
+ except (OSError, TypeError):
+ return None
+ for param in params:
+ if re.search(rf"\bself\.{re.escape(child)}\.{re.escape(param)}\b", source):
+ return param
+ return None
+
+
+def _freeze_all_but_adapters(model: nn.Module) -> None:
+ # Dense and expert adapters are injected one after the other: neither call may freeze the other's.
+ keep = {id(p) for _, layer in _layers(model) for p in (*layer.lora_A, *layer.lora_B)}
+ for p in model.parameters():
+ if id(p) not in keep:
+ p.requires_grad_(False)
+
+
+def _layers(model: nn.Module) -> list[tuple[str, nn.Module]]:
+ return [
+ (n, m)
+ for n, m in model.named_modules()
+ if isinstance(m, (MultiLoraLinear, MultiLoraExpertParam))
+ ]
def set_multi_lora_batch(
@@ -861,6 +1211,28 @@ def adapter_state_dict(
"""Adapter ``adapter`` as a PEFT LoRA state dict (``<name>.lora_A.weight`` / ``lora_B.weight``)."""
out = {}
for name, layer in _layers(model):
- out[f"{prefix}{name}.lora_A.weight"] = _full(layer.lora_A[adapter]).detach().clone()
- out[f"{prefix}{name}.lora_B.weight"] = _full(layer.lora_B[adapter]).detach().clone()
+ if isinstance(layer, MultiLoraLinear):
+ out[f"{prefix}{name}.lora_A.weight"] = _full(layer.lora_A[adapter]).detach().clone()
+ out[f"{prefix}{name}.lora_B.weight"] = _full(layer.lora_B[adapter]).detach().clone()
+ b_layout = _peft_moe_lora_b_layout()
+ for name, experts in model.named_modules():
+ if not isinstance(experts, MultiLoraExperts):
+ continue
+ # PEFT nests one ParamWrapper per target parameter: the first listed is the innermost.
+ depth = len(experts.parameter_names)
+ for i, param in enumerate(experts.parameter_names):
+ key = f"{prefix}{name}" + ".base_layer" * (depth - 1 - i)
+ # Folded experts load through PEFT's own ParamWrapper delta, which is rank-major whatever Zoo uses.
+ layout = "rank_major" if experts.mode == "fold" else b_layout
+ A, B = experts.adapters[param].peft_tensors(adapter, layout)
+ out[f"{key}.lora_A.weight"] = A
+ out[f"{key}.lora_B.weight"] = B
return out
+
+
+def _peft_moe_lora_b_layout() -> str:
+ try:
+ from unsloth_zoo.temporary_patches.moe_utils import moe_lora_b_layout
+ return moe_lora_b_layout()
+ except Exception: # noqa: BLE001 - PEFT's own packing: weight_B.reshape(out, r, E)
+ return "rank_major"