Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Multi-LoRA: export expert LoRA in the layout vanilla PEFT loads (#16568)

Daniel Han02c8fa45b4fe
diff --git a/tests/test_multi_lora_expert_vanilla_export.py b/tests/test_multi_lora_expert_vanilla_export.py
new file mode 100644
index 00000000000..7a4e443e9eb
--- /dev/null
+++ b/tests/test_multi_lora_expert_vanilla_export.py
@@ -0,0 +1,210 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""A multi-LoRA expert export loads in vanilla PEFT with its trained delta, for experts Zoo re-lays out."""
+
+from __future__ import annotations
+
+import os
+import subprocess
+import sys
+import textwrap
+
+import pytest
+import torch
+import unsloth # noqa: F401
+
+from unsloth.kernels import multi_lora as ml
+
+transformers = pytest.importorskip("transformers")
+peft = pytest.importorskip("peft")
+pytestmark = pytest.mark.skipif(
+ not hasattr(transformers, "Qwen3VLMoeForConditionalGeneration"),
+ reason = "transformers has no Qwen3-VL-MoE",
+)
+
+EXPERTS = ["experts.gate_up_proj", "experts.down_proj"]
+R, ALPHA = 4, 8.0
+
+# Shared by this process and the vanilla child, so both build the same architecture.
+BUILD = textwrap.dedent(
+ """
+ import torch
+ import transformers
+
+ def build():
+ text = dict(vocab_size = 256, hidden_size = 64, intermediate_size = 48, moe_intermediate_size = 40,
+ num_hidden_layers = 1, num_attention_heads = 4, num_key_value_heads = 2, head_dim = 16,
+ num_experts = 4, num_experts_per_tok = 2, max_position_embeddings = 128, use_cache = False,
+ pad_token_id = 0)
+ rope = {"rope_type": "default", "mrope_section": [2, 3, 3]}
+ if int(transformers.__version__.split(".")[0]) >= 5:
+ text["rope_parameters"] = dict(rope, rope_theta = 10000.0)
+ else:
+ text.update(rope_scaling = rope, rope_theta = 10000.0)
+ vision = dict(depth = 1, hidden_size = 32, intermediate_size = 64, num_heads = 2, out_hidden_size = 64,
+ deepstack_visual_indexes = [0], num_position_embeddings = 16)
+ config = transformers.Qwen3VLMoeConfig(text_config = text, vision_config = vision,
+ tie_word_embeddings = False)
+ torch.manual_seed(0)
+ return transformers.Qwen3VLMoeForConditionalGeneration._from_config(config).float().eval()
+ """
+)
+
+VANILLA = BUILD + textwrap.dedent(
+ """
+ import sys
+ import peft
+ assert "unsloth" not in sys.modules and "unsloth_zoo" not in sys.modules
+ adapter_dir, out = sys.argv[1], sys.argv[2]
+ model = peft.PeftModel.from_pretrained(build(), adapter_dir)
+ deltas = {}
+ for name, module in model.named_modules():
+ if type(module).__name__ != "ParamWrapper":
+ continue
+ name = name.removeprefix("base_model.model.")
+ while name.endswith(".base_layer"):
+ name = name[: -len(".base_layer")]
+ deltas[f"{name}.{module.parameter_name}"] = module.get_delta_weight("default").detach().clone()
+ torch.save(deltas, out)
+ """
+)
+
+
+def _apply_zoo_patches():
+ from unsloth_zoo.temporary_patches.common import TEMPORARY_PATCHES
+ for patch in TEMPORARY_PATCHES:
+ try:
+ patch()
+ except Exception:
+ pass
+
+
+def _key(name, parameter_name):
+ name = name.removeprefix("base_model.model.")
+ while name.endswith(".base_layer"):
+ name = name[: -len(".base_layer")]
+ return f"{name}.{parameter_name}"
+
+
+def _export(tmp_path):
+ """A tiny Qwen3-VL-MoE with adapter 0 trained (non-zero B) and exported as Studio's batched trainer does."""
+ swap = pytest.importorskip("unsloth_zoo.temporary_patches.moe_lora_factor_swap")
+ _apply_zoo_patches()
+ scope = {}
+ exec(BUILD, scope)
+ model = scope["build"]()
+ lm = model.model.language_model
+ experts_cls = type(lm.layers[0].mlp.experts)
+ if not swap._factors_swapped(experts_cls):
+ pytest.skip(f"this Zoo does not re-lay out {experts_cls.__name__}")
+ ml.inject_multi_lora(lm, ["q_proj"], 2, R, ALPHA)
+ assert ml.inject_multi_lora_experts(lm, EXPERTS, 2, R, ALPHA)
+ with torch.no_grad():
+ # Non-zero B: PEFT's zero init would load any layout as a zero delta.
+ torch.manual_seed(1)
+ for _, layer in ml._layers(model):
+ for B in layer.lora_B:
+ B.normal_(0, 0.5)
+
+ from safetensors.torch import save_file
+
+ adapter_dir = tmp_path / "adapter"
+ adapter_dir.mkdir()
+ state = {k: v.detach().cpu().contiguous() for k, v in ml.adapter_state_dict(model, 0).items()}
+ save_file(state, str(adapter_dir / "adapter_model.safetensors"), metadata = {"format": "pt"})
+ peft.LoraConfig(
+ r = R,
+ lora_alpha = ALPHA,
+ lora_dropout = 0.0,
+ target_modules = ["q_proj"],
+ target_parameters = EXPERTS,
+ ).save_pretrained(str(adapter_dir))
+ return model, scope["build"], adapter_dir
+
+
+def _vanilla_load(tmp_path, adapter_dir):
+ script = tmp_path / "vanilla_load.py"
+ script.write_text(VANILLA)
+ out = tmp_path / "deltas.pt"
+ env = {k: v for k, v in os.environ.items() if not k.startswith("UNSLOTH")}
+ proc = subprocess.run(
+ [sys.executable, "-I", str(script), str(adapter_dir), str(out)],
+ capture_output = True,
+ text = True,
+ env = env,
+ timeout = 600,
+ )
+ return proc, out
+
+
+def test_multi_lora_expert_export_loads_in_vanilla_peft(tmp_path, monkeypatch):
+ monkeypatch.delenv("UNSLOTH_MOE_LORA_B_LAYOUT", raising = False)
+ model, build, adapter_dir = _export(tmp_path)
+ mine = {}
+ for name, experts in model.named_modules():
+ if isinstance(experts, ml.MultiLoraExperts):
+ for param in experts.parameter_names:
+ W = getattr(experts.base_layer, param)
+ delta = experts.adapters[param].folded(torch.zeros_like(W), 0, experts.transposed)
+ mine[f"{name}.{param.split('.')[-1]}"] = delta.detach()
+ assert len(mine) == len(EXPERTS)
+
+ proc, out = _vanilla_load(tmp_path, adapter_dir)
+ assert proc.returncode == 0, proc.stderr[-3000:]
+ vanilla = torch.load(out)
+ assert set(vanilla) == set(mine), (sorted(vanilla), sorted(mine))
+ for key, delta in mine.items():
+ # Qwen3-VL-MoE: Zoo stores [E, H, 2I] / [E, I, H] where vanilla stores [E, 2I, H] / [E, H, I].
+ assert vanilla[key].shape == delta.transpose(1, 2).shape, key
+ assert vanilla[key].abs().max() > 1e-3, key
+ torch.testing.assert_close(vanilla[key], delta.transpose(1, 2), atol = 1e-6, rtol = 1e-5)
+
+ # The same folder still reloads under Unsloth (Zoo maps the vanilla pair back) with the trained delta.
+ reloaded = peft.PeftModel.from_pretrained(build(), str(adapter_dir), torch_device = "cpu")
+ seen = 0
+ for name, module in reloaded.named_modules():
+ if type(module).__name__ == "ParamWrapper":
+ key = _key(name, module.parameter_name)
+ torch.testing.assert_close(
+ module.get_delta_weight("default"), mine[key], atol = 1e-6, rtol = 1e-5
+ )
+ seen += 1
+ assert seen == len(mine)
+
+
+def test_legacy_grouped_multi_lora_export_is_refused_by_vanilla_peft(tmp_path, monkeypatch):
+ """A legacy grouped lora_B saved 2D would load with experts mixed, so it is saved [out, E, R] and refused."""
+ swap = pytest.importorskip("unsloth_zoo.temporary_patches.moe_lora_factor_swap")
+ if not hasattr(swap, "_flatten_grouped_lora_B"):
+ pytest.skip("this Zoo saves a legacy grouped_by_expert lora_B 2D")
+ monkeypatch.setenv("UNSLOTH_MOE_LORA_B_LAYOUT", "grouped_by_expert")
+ model, build, adapter_dir = _export(tmp_path)
+ trained = {}
+ for name, experts in model.named_modules():
+ if isinstance(experts, ml.MultiLoraExperts):
+ for param in experts.parameter_names:
+ A, B = experts.adapters[param].peft_tensors(0, "grouped_by_expert")
+ trained[f"{name}.{param.split('.')[-1]}"] = (A, B)
+
+ from safetensors.torch import load_file
+
+ saved = load_file(str(adapter_dir / "adapter_model.safetensors"))
+ expert_B = [v for k, v in saved.items() if "experts" in k and ".lora_B." in k]
+ assert len(expert_B) == len(EXPERTS)
+ # A 3D lora_B is refused by shape on every PEFT, including 0.18 where a 2D one would load mixed.
+ assert all(B.dim() == 3 and B.shape[1] == 4 for B in expert_B), [
+ tuple(B.shape) for B in expert_B
+ ]
+ proc, _ = _vanilla_load(tmp_path, adapter_dir)
+ assert proc.returncode != 0 and "size mismatch" in proc.stderr, proc.stderr[-3000:]
+
+ reloaded = peft.PeftModel.from_pretrained(build(), str(adapter_dir), torch_device = "cpu")
+ seen = 0
+ for name, module in reloaded.named_modules():
+ if type(module).__name__ == "ParamWrapper":
+ A, B = trained[_key(name, module.parameter_name)]
+ torch.testing.assert_close(module.lora_A["default"].weight, A, atol = 0, rtol = 0)
+ torch.testing.assert_close(module.lora_B["default"].weight, B, atol = 0, rtol = 0)
+ seen += 1
+ assert seen == len(trained)
diff --git a/unsloth/kernels/multi_lora.py b/unsloth/kernels/multi_lora.py
index 373ab20c5d6..65ffae01afe 100644
--- a/unsloth/kernels/multi_lora.py
+++ b/unsloth/kernels/multi_lora.py
@@ -1653,11 +1653,32 @@ def adapter_state_dict(
# rank-major whatever Zoo uses; how they trained here (grouped or not) does not matter.
layout = "rank_major" if experts.peft_mode == "fold" else b_layout
A, B = experts.adapters[param].peft_tensors(adapter, layout)
+ num_experts = _full(experts.adapters[param].lora_B[adapter]).shape[0]
+ A, B = _vanilla_expert_factors(experts.base_layer, A, B, num_experts, layout)
out[f"{key}.lora_A.weight"] = A
out[f"{key}.lora_B.weight"] = B
return out
+def _vanilla_expert_factors(
+ base: nn.Module, A: torch.Tensor, B: torch.Tensor, num_experts: int, layout: str
+):
+ """Apply Zoo's moe_lora_factor_swap save rules (bypassed by this export) so vanilla PEFT loads it.
+
+ Only flagged classes, never a square pair; a legacy grouped lora_B is saved [out, E, R] as Zoo does."""
+ try:
+ from unsloth_zoo.temporary_patches import moe_lora_factor_swap as swap
+ except Exception: # noqa: BLE001 - Zoo without the converter: nothing is rewritten on any save
+ return A, B
+ if layout != "rank_major":
+ if hasattr(swap, "_flatten_grouped_lora_B") and B.shape[1] % num_experts == 0:
+ B = B.reshape(B.shape[0], num_experts, B.shape[1] // num_experts)
+ return A, B
+ if A.shape[1] == B.shape[0] or not swap._factors_swapped(type(base)):
+ return A, B
+ return swap.swap_expert_lora_factors(A, B, num_experts)
+
+
def _peft_moe_lora_b_layout() -> str:
try:
from unsloth_zoo.temporary_patches.moe_utils import moe_lora_b_layout