Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Context parallel: gather the FSDP2 state dict in save_model on transformers 4.x (#14028)
Daniel Han88ac9b0ac9fa
diff --git a/tests/test_context_parallel_fsdp2_save_gloo.py b/tests/test_context_parallel_fsdp2_save_gloo.pynew file mode 100644index 00000000000..a0b6b3bd5dc--- /dev/null+++ b/tests/test_context_parallel_fsdp2_save_gloo.py@@ -0,0 +1,170 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""Trainer.save_model of a LoRA model under FSDP2 + context parallelism, two gloo ranks on CPU.++Unsloth installs a ParallelismConfig for the (dp_shard, cp) mesh. transformers < 5 dispatches+save_model on that config before FSDP, skipping the full state dict gather, so PEFT is handed DTensor+shards and fails. The patched save_model must write whole adapter tensors equal to the unsharded ones,+and leave the installed config in place afterwards.+"""++from __future__ import annotations++import os+import socket+import types++import pytest++torch = pytest.importorskip("torch")+dist = pytest.importorskip("torch.distributed")+pytest.importorskip("torch.distributed.fsdp")+pytest.importorskip("peft")+pytest.importorskip("accelerate")++if not dist.is_available() or not dist.is_gloo_available():+ pytest.skip("gloo backend unavailable", allow_module_level = True)+++def _free_port():+ with socket.socket() as s:+ s.bind(("127.0.0.1", 0))+ return s.getsockname()[1]+++def _accelerator_less_import():+ # Spawned ranks skip conftest, whose CUDA spoof lets `import unsloth` run on CPU-only runners.+ if torch.cuda.is_available():+ return+ import importlib.util+ import pathlib++ path = pathlib.Path(__file__).resolve().parent / "conftest.py"+ spec = importlib.util.spec_from_file_location("_unsloth_tests_conftest", path)+ spec.loader.exec_module(importlib.util.module_from_spec(spec))+++def _worker(rank, port, out_dir, user_config):+ _accelerator_less_import()+ os.environ.update(+ RANK = str(rank),+ LOCAL_RANK = str(rank),+ WORLD_SIZE = "2",+ MASTER_ADDR = "127.0.0.1",+ MASTER_PORT = str(port),+ )+ dist.init_process_group("gloo", rank = rank, world_size = 2)+ try:+ import json++ import transformers+ import trl+ from peft import LoraConfig, get_peft_model+ from torch.distributed.fsdp import fully_shard+ from transformers import LlamaConfig, LlamaForCausalLM++ from accelerate import Accelerator+ from accelerate.utils import DistributedType+ from unsloth import context_parallel as cp++ cp.DEVICE_TYPE_TORCH = "cpu"+ manager = cp.ContextParallelManager(2, "ulysses", fsdp2 = True)+ state = types.SimpleNamespace(+ device_mesh = manager.device_mesh,+ parallelism_config = None,+ fsdp_plugin = types.SimpleNamespace(state_dict_type = "FULL_STATE_DICT"),+ )+ if user_config:+ # A dp_shard only ParallelismConfig the user passed: still non-None for transformers 4.x.+ from accelerate.parallelism_config import ParallelismConfig+ state.parallelism_config = ParallelismConfig(dp_shard_size = 2)+ cp._install_fsdp2_parallelism_config(state, manager)+ installed = state.parallelism_config++ torch.manual_seed(0)+ cfg = LlamaConfig(+ vocab_size = 32,+ hidden_size = 16,+ intermediate_size = 32,+ num_attention_heads = 2,+ num_key_value_heads = 2,+ num_hidden_layers = 1,+ )+ lora = LoraConfig(r = 2, target_modules = ["q_proj", "v_proj"], init_lora_weights = False)+ model = get_peft_model(LlamaForCausalLM(cfg), lora)+ reference = {k: v.clone() for k, v in model.state_dict().items() if "lora_" in k}+ mesh = manager.device_mesh["dp_shard_cp"]+ for layer in model.base_model.model.model.layers:+ fully_shard(layer, mesh = mesh)+ fully_shard(model, mesh = mesh)++ class SFTTrainer(transformers.Trainer):+ pass++ trl.SFTTrainer = SFTTrainer+ cp.patch_sft_trainer()++ class Acc:+ parallelism_config = property(lambda self: self.state.parallelism_config)+ distributed_type = DistributedType.FSDP+ is_fsdp2 = True+ should_save_model = Accelerator.should_save_model+ get_state_dict = Accelerator.get_state_dict++ def unwrap_model(self, model, **kwargs):+ return model++ trainer = object.__new__(SFTTrainer)+ trainer.model = trainer.model_wrapped = model+ trainer.is_fsdp_enabled = True+ trainer.is_deepspeed_enabled = False+ trainer.processing_class = trainer.data_collator = None+ trainer.args = types.SimpleNamespace(+ output_dir = out_dir, should_save = rank == 0, push_to_hub = False, save_safetensors = True+ )+ trainer.accelerator = Acc()+ trainer.accelerator.state = state+ trainer._context_parallel_manager = manager++ result = {}+ try:+ trainer.save_model(out_dir)+ result["error"] = None+ except Exception as e:+ result["error"] = f"{type(e).__name__}: {e}"+ result["config_kept"] = state.parallelism_config is installed+ dist.barrier()+ if rank == 0 and result["error"] is None:+ from safetensors.torch import load_file++ saved = load_file(os.path.join(out_dir, "adapter_model.safetensors"))+ equal = 0+ for name, ref in reference.items():+ got = saved.get(name.replace(".default", ""))+ equal += int(got is not None and torch.equal(got, ref))+ result["lora_tensors"] = len(reference)+ result["saved_whole_and_equal"] = equal+ gathered = [None] * 2+ dist.all_gather_object(gathered, result)+ if rank == 0:+ with open(os.path.join(out_dir, "result.json"), "w") as f:+ json.dump(gathered, f)+ finally:+ dist.destroy_process_group()++[email protected]("user_config", [False, True])+def test_save_model_gathers_fsdp2_lora_under_context_parallel(tmp_path, user_config):+ import json++ import torch.multiprocessing as mp++ mp.spawn(_worker, args = (_free_port(), str(tmp_path), user_config), nprocs = 2, join = True)+ with open(tmp_path / "result.json") as f:+ results = json.load(f)+ for r in results:+ assert r["error"] is None, r["error"]+ assert r["config_kept"]+ assert results[0]["lora_tensors"] == 4+ assert results[0]["saved_whole_and_equal"] == 4diff --git a/unsloth/context_parallel.py b/unsloth/context_parallel.pyindex 8be15ea869b..7e7ddd36aa2 100644--- a/unsloth/context_parallel.py+++ b/unsloth/context_parallel.py@@ -540,6 +540,7 @@ def patch_sft_trainer() -> None:original_evaluate = _original("evaluate")original_predict = _original("predict")original_evaluation_loop = _original("evaluation_loop")+ original_save_model = _original("save_model")has_total_train_batch_size = hasattr(trainer_cls, "get_total_train_batch_size")if has_total_train_batch_size:original_get_total_train_batch_size = _original("get_total_train_batch_size")@@ -758,6 +759,20 @@ def patch_sft_trainer() -> None:_install_accelerator_state(self)return original_predict(self, test_dataset, *args, **kwargs)+ @functools.wraps(original_save_model)+ def patched_save_model(self, *args, **kwargs):+ state = getattr(getattr(self, "accelerator", None), "state", None)+ installed = _INSTALLED_PARALLELISM_CONFIG[1] if _INSTALLED_PARALLELISM_CONFIG else None+ if installed is None or getattr(state, "parallelism_config", None) is not installed:+ return original_save_model(self, *args, **kwargs)+ # transformers < 5 takes any non-None parallelism_config before FSDP and skips the FSDP2 state+ # dict gather, so save without one; even a user dp_shard config would take that branch.+ state.parallelism_config = None+ try:+ return original_save_model(self, *args, **kwargs)+ finally:+ state.parallelism_config = installed+@functools.wraps(original_evaluation_loop)def patched_evaluation_loop(self, *args, **kwargs):manager = getattr(self, "_context_parallel_manager", None)@@ -811,5 +826,6 @@ def patch_sft_trainer() -> None:trainer_cls.evaluate = patched_evaluatetrainer_cls.predict = patched_predicttrainer_cls.evaluation_loop = patched_evaluation_loop+ trainer_cls.save_model = patched_save_modeltrainer_cls.training_step = patched_training_steptrainer_cls.__unsloth_context_parallel__ = True