Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Fix tensor parallel adapter save after the MoE expert LoRA factor swap (#16654)
Daniel Hanf2fa80cc6376
diff --git a/tests/test_tp_save_expert_factor_swap_gloo.py b/tests/test_tp_save_expert_factor_swap_gloo.pynew file mode 100644index 00000000000..e9be9073059--- /dev/null+++ b/tests/test_tp_save_expert_factor_swap_gloo.py@@ -0,0 +1,170 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""save_tp_adapter on FSDP2 DTensor expert LoRA factors must match single-process save_pretrained."""++import multiprocessing+import os++if multiprocessing.parent_process() is not None or getattr(+ multiprocessing.current_process(), "_inheriting", False+):+ os.environ["CUDA_VISIBLE_DEVICES"] = "" # spawned gloo ranks: CPU only++import sys # noqa: E402+import tempfile # noqa: E402++import pytest # noqa: E402+import torch # noqa: E402++sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))+from test_tp_native_gloo import _free_port # noqa: E402+from test_tp_native_oracle import tpn # noqa: E402++REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))+MODES = ("noswap", "swap", "legacy")+++def _build(mode):+ import peft+ from torch import nn+ from transformers.models.llama4.configuration_llama4 import Llama4TextConfig+ from transformers.models.llama4.modeling_llama4 import Llama4TextExperts+ import unsloth_zoo.temporary_patches.moe_lora_factor_swap as swap+ from unsloth_zoo.temporary_patches.llama4_moe import patch_llama4_moe++ patch_llama4_moe()+ swap.patch_expert_lora_factor_swap()+ Llama4TextExperts._unsloth_expert_lora_factors_swapped = mode != "noswap"+ cfg = Llama4TextConfig(hidden_size = 16, intermediate_size = 12, num_local_experts = 4)++ class Block(nn.Module):+ def __init__(self):+ super().__init__()+ self.experts = Llama4TextExperts(cfg)++ torch.manual_seed(0)+ model = Block()+ for p in model.parameters():+ nn.init.normal_(p, std = 0.02)+ config = peft.LoraConfig(+ r = 2, target_modules = [], target_parameters = ["experts.gate_up_proj", "experts.down_proj"]+ )+ model = peft.get_peft_model(model, config)+ for name, p in model.named_parameters():+ if "lora_B" in name:+ nn.init.normal_(p, std = 0.02)+ return model+++def _read(directory):+ from safetensors.torch import load_file+ return load_file(os.path.join(directory, "adapter_model.safetensors"))+++def _case(rank, world, tmp, mode):+ try:+ import importlib.util++ # spawned ranks skip conftest's device spoof, which unsloth_zoo needs at import+ spec = importlib.util.spec_from_file_location(+ "_tp_conftest", os.path.join(REPO, "tests", "conftest.py")+ )+ spec.loader.exec_module(importlib.util.module_from_spec(spec))+ import unsloth_zoo.temporary_patches.moe_lora_factor_swap # noqa: F401+ from transformers.models.llama4.modeling_llama4 import Llama4TextExperts # noqa: F401+ except Exception as e:+ return {"skip": repr(e)}+ import torch.distributed as dist+ from torch.distributed.device_mesh import init_device_mesh+ from torch.distributed.fsdp import fully_shard++ if mode == "legacy":+ os.environ["UNSLOTH_MOE_LORA_B_LAYOUT"] = "grouped_by_expert"+ reference = os.path.join(tmp, f"{mode}_single")+ if rank == 0:+ _build(mode).save_pretrained(reference)+ dist.barrier()+ model = _build(mode)+ fully_shard(model, mesh = init_device_mesh("cpu", (world,), mesh_dim_names = ("dp",)))+ out = {+ "dtensors": sum(+ tpn._is_dtensor(v) for k, v in model.state_dict(keep_vars = True).items() if "lora_" in k+ )+ }+ target = os.path.join(tmp, f"{mode}_tp")+ try:+ tpn.save_tp_adapter(model, target, save_embedding_layers = False)+ except Exception as e:+ out["error"] = f"{type(e).__name__}: {e}"+ return out+ if rank == 0:+ out["single"] = _read(reference)+ out["tp"] = _read(target)+ return out+++def _worker(rank, world, port, tmp, mode):+ os.environ.update(+ MASTER_ADDR = "127.0.0.1",+ MASTER_PORT = str(port),+ RANK = str(rank),+ WORLD_SIZE = str(world),+ LOCAL_RANK = str(rank),+ LOCAL_WORLD_SIZE = str(world),+ )+ import datetime++ import torch.distributed as dist++ torch.set_num_threads(2)+ dist.init_process_group(+ "gloo", rank = rank, world_size = world, timeout = datetime.timedelta(seconds = 120)+ )+ try:+ torch.save(_case(rank, world, tmp, mode), os.path.join(tmp, f"result_{rank}.pt"))+ finally:+ dist.destroy_process_group()+++def _spawn(mode, world = 2):+ ctx = multiprocessing.get_context("spawn")+ port = _free_port()+ with tempfile.TemporaryDirectory() as tmp:+ procs = [+ ctx.Process(target = _worker, args = (r, world, port, tmp, mode)) for r in range(world)+ ]+ for p in procs:+ p.start()+ for p in procs:+ p.join(300)+ assert [p.exitcode for p in procs] == [0] * world+ results = [+ torch.load(os.path.join(tmp, f"result_{r}.pt"), weights_only = False)+ for r in range(world)+ ]+ if "skip" in results[0]:+ pytest.skip(reason = f"unsloth_zoo without the expert factor swap: {results[0]['skip']}")+ return results++[email protected]("mode", MODES)+def test_tp_save_matches_single_process_after_expert_factor_swap(mode):+ results = _spawn(mode)+ for rank, res in enumerate(results):+ assert res["dtensors"] > 0, (rank, res)+ assert "error" not in res, (rank, res["error"])+ single, tp = results[0]["single"], results[0]["tp"]+ assert sorted(single) == sorted(tp)+ for key in single:+ assert single[key].shape == tp[key].shape, (key, single[key].shape, tp[key].shape)+ torch.testing.assert_close(tp[key], single[key], rtol = 0, atol = 0, msg = key)+ if mode == "swap":+ # converted (vanilla PEFT) layout; down_proj is the outer wrapper+ shapes = {k.split("model.experts.")[-1]: list(v.shape) for k, v in tp.items()}+ assert shapes == {+ "base_layer.lora_A.weight": [8, 24],+ "base_layer.lora_B.weight": [16, 8],+ "lora_A.weight": [8, 16],+ "lora_B.weight": [12, 8],+ }, shapesdiff --git a/unsloth/models/tp_native.py b/unsloth/models/tp_native.pyindex 77ee7ba6b66..be3f69dd4b4 100644--- a/unsloth/models/tp_native.py+++ b/unsloth/models/tp_native.py@@ -2241,8 +2241,12 @@ def tp_full_state_dict(model, adapters_only = False):def _peft_state_dict(model, **kwargs):"""What PEFT selects for save_pretrained, DTensors gathered; local tensors by reference (PEFT rerunsits selection inside save_pretrained and must see the same model)."""+ import inspect+from peft import get_peft_model_state_dict+ # Wrappers (zoo's expert factor swap) return new tensors the id match cannot find; save_pretrained converts.+ get_peft_model_state_dict = inspect.unwrap(get_peft_model_state_dict)raw = model.state_dict(keep_vars = True)adapters = kwargs.get("selected_adapters") or list(model.peft_config)known = {id(value) for value in raw.values()}