Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Fix tensor parallel adapter save after the MoE expert LoRA factor swap (#16654)

Daniel Hanf2fa80cc6376
diff --git a/tests/test_tp_save_expert_factor_swap_gloo.py b/tests/test_tp_save_expert_factor_swap_gloo.py
new file mode 100644
index 00000000000..e9be9073059
--- /dev/null
+++ b/tests/test_tp_save_expert_factor_swap_gloo.py
@@ -0,0 +1,170 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""save_tp_adapter on FSDP2 DTensor expert LoRA factors must match single-process save_pretrained."""
+
+import multiprocessing
+import os
+
+if multiprocessing.parent_process() is not None or getattr(
+ multiprocessing.current_process(), "_inheriting", False
+):
+ os.environ["CUDA_VISIBLE_DEVICES"] = "" # spawned gloo ranks: CPU only
+
+import sys # noqa: E402
+import tempfile # noqa: E402
+
+import pytest # noqa: E402
+import torch # noqa: E402
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from test_tp_native_gloo import _free_port # noqa: E402
+from test_tp_native_oracle import tpn # noqa: E402
+
+REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+MODES = ("noswap", "swap", "legacy")
+
+
+def _build(mode):
+ import peft
+ from torch import nn
+ from transformers.models.llama4.configuration_llama4 import Llama4TextConfig
+ from transformers.models.llama4.modeling_llama4 import Llama4TextExperts
+ import unsloth_zoo.temporary_patches.moe_lora_factor_swap as swap
+ from unsloth_zoo.temporary_patches.llama4_moe import patch_llama4_moe
+
+ patch_llama4_moe()
+ swap.patch_expert_lora_factor_swap()
+ Llama4TextExperts._unsloth_expert_lora_factors_swapped = mode != "noswap"
+ cfg = Llama4TextConfig(hidden_size = 16, intermediate_size = 12, num_local_experts = 4)
+
+ class Block(nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.experts = Llama4TextExperts(cfg)
+
+ torch.manual_seed(0)
+ model = Block()
+ for p in model.parameters():
+ nn.init.normal_(p, std = 0.02)
+ config = peft.LoraConfig(
+ r = 2, target_modules = [], target_parameters = ["experts.gate_up_proj", "experts.down_proj"]
+ )
+ model = peft.get_peft_model(model, config)
+ for name, p in model.named_parameters():
+ if "lora_B" in name:
+ nn.init.normal_(p, std = 0.02)
+ return model
+
+
+def _read(directory):
+ from safetensors.torch import load_file
+ return load_file(os.path.join(directory, "adapter_model.safetensors"))
+
+
+def _case(rank, world, tmp, mode):
+ try:
+ import importlib.util
+
+ # spawned ranks skip conftest's device spoof, which unsloth_zoo needs at import
+ spec = importlib.util.spec_from_file_location(
+ "_tp_conftest", os.path.join(REPO, "tests", "conftest.py")
+ )
+ spec.loader.exec_module(importlib.util.module_from_spec(spec))
+ import unsloth_zoo.temporary_patches.moe_lora_factor_swap # noqa: F401
+ from transformers.models.llama4.modeling_llama4 import Llama4TextExperts # noqa: F401
+ except Exception as e:
+ return {"skip": repr(e)}
+ import torch.distributed as dist
+ from torch.distributed.device_mesh import init_device_mesh
+ from torch.distributed.fsdp import fully_shard
+
+ if mode == "legacy":
+ os.environ["UNSLOTH_MOE_LORA_B_LAYOUT"] = "grouped_by_expert"
+ reference = os.path.join(tmp, f"{mode}_single")
+ if rank == 0:
+ _build(mode).save_pretrained(reference)
+ dist.barrier()
+ model = _build(mode)
+ fully_shard(model, mesh = init_device_mesh("cpu", (world,), mesh_dim_names = ("dp",)))
+ out = {
+ "dtensors": sum(
+ tpn._is_dtensor(v) for k, v in model.state_dict(keep_vars = True).items() if "lora_" in k
+ )
+ }
+ target = os.path.join(tmp, f"{mode}_tp")
+ try:
+ tpn.save_tp_adapter(model, target, save_embedding_layers = False)
+ except Exception as e:
+ out["error"] = f"{type(e).__name__}: {e}"
+ return out
+ if rank == 0:
+ out["single"] = _read(reference)
+ out["tp"] = _read(target)
+ return out
+
+
+def _worker(rank, world, port, tmp, mode):
+ os.environ.update(
+ MASTER_ADDR = "127.0.0.1",
+ MASTER_PORT = str(port),
+ RANK = str(rank),
+ WORLD_SIZE = str(world),
+ LOCAL_RANK = str(rank),
+ LOCAL_WORLD_SIZE = str(world),
+ )
+ import datetime
+
+ import torch.distributed as dist
+
+ torch.set_num_threads(2)
+ dist.init_process_group(
+ "gloo", rank = rank, world_size = world, timeout = datetime.timedelta(seconds = 120)
+ )
+ try:
+ torch.save(_case(rank, world, tmp, mode), os.path.join(tmp, f"result_{rank}.pt"))
+ finally:
+ dist.destroy_process_group()
+
+
+def _spawn(mode, world = 2):
+ ctx = multiprocessing.get_context("spawn")
+ port = _free_port()
+ with tempfile.TemporaryDirectory() as tmp:
+ procs = [
+ ctx.Process(target = _worker, args = (r, world, port, tmp, mode)) for r in range(world)
+ ]
+ for p in procs:
+ p.start()
+ for p in procs:
+ p.join(300)
+ assert [p.exitcode for p in procs] == [0] * world
+ results = [
+ torch.load(os.path.join(tmp, f"result_{r}.pt"), weights_only = False)
+ for r in range(world)
+ ]
+ if "skip" in results[0]:
+ pytest.skip(reason = f"unsloth_zoo without the expert factor swap: {results[0]['skip']}")
+ return results
+
+
[email protected]("mode", MODES)
+def test_tp_save_matches_single_process_after_expert_factor_swap(mode):
+ results = _spawn(mode)
+ for rank, res in enumerate(results):
+ assert res["dtensors"] > 0, (rank, res)
+ assert "error" not in res, (rank, res["error"])
+ single, tp = results[0]["single"], results[0]["tp"]
+ assert sorted(single) == sorted(tp)
+ for key in single:
+ assert single[key].shape == tp[key].shape, (key, single[key].shape, tp[key].shape)
+ torch.testing.assert_close(tp[key], single[key], rtol = 0, atol = 0, msg = key)
+ if mode == "swap":
+ # converted (vanilla PEFT) layout; down_proj is the outer wrapper
+ shapes = {k.split("model.experts.")[-1]: list(v.shape) for k, v in tp.items()}
+ assert shapes == {
+ "base_layer.lora_A.weight": [8, 24],
+ "base_layer.lora_B.weight": [16, 8],
+ "lora_A.weight": [8, 16],
+ "lora_B.weight": [12, 8],
+ }, shapes
diff --git a/unsloth/models/tp_native.py b/unsloth/models/tp_native.py
index 77ee7ba6b66..be3f69dd4b4 100644
--- a/unsloth/models/tp_native.py
+++ b/unsloth/models/tp_native.py
@@ -2241,8 +2241,12 @@ def tp_full_state_dict(model, adapters_only = False):
def _peft_state_dict(model, **kwargs):
"""What PEFT selects for save_pretrained, DTensors gathered; local tensors by reference (PEFT reruns
its selection inside save_pretrained and must see the same model)."""
+ import inspect
+
from peft import get_peft_model_state_dict
+ # Wrappers (zoo's expert factor swap) return new tensors the id match cannot find; save_pretrained converts.
+ get_peft_model_state_dict = inspect.unwrap(get_peft_model_state_dict)
raw = model.state_dict(keep_vars = True)
adapters = kwargs.get("selected_adapters") or list(model.peft_config)
known = {id(value) for value in raw.values()}