Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Context parallel: gather the FSDP2 state dict in save_model on transformers 4.x (#14028)

Daniel Han88ac9b0ac9fa
diff --git a/tests/test_context_parallel_fsdp2_save_gloo.py b/tests/test_context_parallel_fsdp2_save_gloo.py
new file mode 100644
index 00000000000..a0b6b3bd5dc
--- /dev/null
+++ b/tests/test_context_parallel_fsdp2_save_gloo.py
@@ -0,0 +1,170 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""Trainer.save_model of a LoRA model under FSDP2 + context parallelism, two gloo ranks on CPU.
+
+Unsloth installs a ParallelismConfig for the (dp_shard, cp) mesh. transformers < 5 dispatches
+save_model on that config before FSDP, skipping the full state dict gather, so PEFT is handed DTensor
+shards and fails. The patched save_model must write whole adapter tensors equal to the unsharded ones,
+and leave the installed config in place afterwards.
+"""
+
+from __future__ import annotations
+
+import os
+import socket
+import types
+
+import pytest
+
+torch = pytest.importorskip("torch")
+dist = pytest.importorskip("torch.distributed")
+pytest.importorskip("torch.distributed.fsdp")
+pytest.importorskip("peft")
+pytest.importorskip("accelerate")
+
+if not dist.is_available() or not dist.is_gloo_available():
+ pytest.skip("gloo backend unavailable", allow_module_level = True)
+
+
+def _free_port():
+ with socket.socket() as s:
+ s.bind(("127.0.0.1", 0))
+ return s.getsockname()[1]
+
+
+def _accelerator_less_import():
+ # Spawned ranks skip conftest, whose CUDA spoof lets `import unsloth` run on CPU-only runners.
+ if torch.cuda.is_available():
+ return
+ import importlib.util
+ import pathlib
+
+ path = pathlib.Path(__file__).resolve().parent / "conftest.py"
+ spec = importlib.util.spec_from_file_location("_unsloth_tests_conftest", path)
+ spec.loader.exec_module(importlib.util.module_from_spec(spec))
+
+
+def _worker(rank, port, out_dir, user_config):
+ _accelerator_less_import()
+ os.environ.update(
+ RANK = str(rank),
+ LOCAL_RANK = str(rank),
+ WORLD_SIZE = "2",
+ MASTER_ADDR = "127.0.0.1",
+ MASTER_PORT = str(port),
+ )
+ dist.init_process_group("gloo", rank = rank, world_size = 2)
+ try:
+ import json
+
+ import transformers
+ import trl
+ from peft import LoraConfig, get_peft_model
+ from torch.distributed.fsdp import fully_shard
+ from transformers import LlamaConfig, LlamaForCausalLM
+
+ from accelerate import Accelerator
+ from accelerate.utils import DistributedType
+ from unsloth import context_parallel as cp
+
+ cp.DEVICE_TYPE_TORCH = "cpu"
+ manager = cp.ContextParallelManager(2, "ulysses", fsdp2 = True)
+ state = types.SimpleNamespace(
+ device_mesh = manager.device_mesh,
+ parallelism_config = None,
+ fsdp_plugin = types.SimpleNamespace(state_dict_type = "FULL_STATE_DICT"),
+ )
+ if user_config:
+ # A dp_shard only ParallelismConfig the user passed: still non-None for transformers 4.x.
+ from accelerate.parallelism_config import ParallelismConfig
+ state.parallelism_config = ParallelismConfig(dp_shard_size = 2)
+ cp._install_fsdp2_parallelism_config(state, manager)
+ installed = state.parallelism_config
+
+ torch.manual_seed(0)
+ cfg = LlamaConfig(
+ vocab_size = 32,
+ hidden_size = 16,
+ intermediate_size = 32,
+ num_attention_heads = 2,
+ num_key_value_heads = 2,
+ num_hidden_layers = 1,
+ )
+ lora = LoraConfig(r = 2, target_modules = ["q_proj", "v_proj"], init_lora_weights = False)
+ model = get_peft_model(LlamaForCausalLM(cfg), lora)
+ reference = {k: v.clone() for k, v in model.state_dict().items() if "lora_" in k}
+ mesh = manager.device_mesh["dp_shard_cp"]
+ for layer in model.base_model.model.model.layers:
+ fully_shard(layer, mesh = mesh)
+ fully_shard(model, mesh = mesh)
+
+ class SFTTrainer(transformers.Trainer):
+ pass
+
+ trl.SFTTrainer = SFTTrainer
+ cp.patch_sft_trainer()
+
+ class Acc:
+ parallelism_config = property(lambda self: self.state.parallelism_config)
+ distributed_type = DistributedType.FSDP
+ is_fsdp2 = True
+ should_save_model = Accelerator.should_save_model
+ get_state_dict = Accelerator.get_state_dict
+
+ def unwrap_model(self, model, **kwargs):
+ return model
+
+ trainer = object.__new__(SFTTrainer)
+ trainer.model = trainer.model_wrapped = model
+ trainer.is_fsdp_enabled = True
+ trainer.is_deepspeed_enabled = False
+ trainer.processing_class = trainer.data_collator = None
+ trainer.args = types.SimpleNamespace(
+ output_dir = out_dir, should_save = rank == 0, push_to_hub = False, save_safetensors = True
+ )
+ trainer.accelerator = Acc()
+ trainer.accelerator.state = state
+ trainer._context_parallel_manager = manager
+
+ result = {}
+ try:
+ trainer.save_model(out_dir)
+ result["error"] = None
+ except Exception as e:
+ result["error"] = f"{type(e).__name__}: {e}"
+ result["config_kept"] = state.parallelism_config is installed
+ dist.barrier()
+ if rank == 0 and result["error"] is None:
+ from safetensors.torch import load_file
+
+ saved = load_file(os.path.join(out_dir, "adapter_model.safetensors"))
+ equal = 0
+ for name, ref in reference.items():
+ got = saved.get(name.replace(".default", ""))
+ equal += int(got is not None and torch.equal(got, ref))
+ result["lora_tensors"] = len(reference)
+ result["saved_whole_and_equal"] = equal
+ gathered = [None] * 2
+ dist.all_gather_object(gathered, result)
+ if rank == 0:
+ with open(os.path.join(out_dir, "result.json"), "w") as f:
+ json.dump(gathered, f)
+ finally:
+ dist.destroy_process_group()
+
+
[email protected]("user_config", [False, True])
+def test_save_model_gathers_fsdp2_lora_under_context_parallel(tmp_path, user_config):
+ import json
+
+ import torch.multiprocessing as mp
+
+ mp.spawn(_worker, args = (_free_port(), str(tmp_path), user_config), nprocs = 2, join = True)
+ with open(tmp_path / "result.json") as f:
+ results = json.load(f)
+ for r in results:
+ assert r["error"] is None, r["error"]
+ assert r["config_kept"]
+ assert results[0]["lora_tensors"] == 4
+ assert results[0]["saved_whole_and_equal"] == 4
diff --git a/unsloth/context_parallel.py b/unsloth/context_parallel.py
index 8be15ea869b..7e7ddd36aa2 100644
--- a/unsloth/context_parallel.py
+++ b/unsloth/context_parallel.py
@@ -540,6 +540,7 @@ def patch_sft_trainer() -> None:
original_evaluate = _original("evaluate")
original_predict = _original("predict")
original_evaluation_loop = _original("evaluation_loop")
+ original_save_model = _original("save_model")
has_total_train_batch_size = hasattr(trainer_cls, "get_total_train_batch_size")
if has_total_train_batch_size:
original_get_total_train_batch_size = _original("get_total_train_batch_size")
@@ -758,6 +759,20 @@ def patch_sft_trainer() -> None:
_install_accelerator_state(self)
return original_predict(self, test_dataset, *args, **kwargs)
+ @functools.wraps(original_save_model)
+ def patched_save_model(self, *args, **kwargs):
+ state = getattr(getattr(self, "accelerator", None), "state", None)
+ installed = _INSTALLED_PARALLELISM_CONFIG[1] if _INSTALLED_PARALLELISM_CONFIG else None
+ if installed is None or getattr(state, "parallelism_config", None) is not installed:
+ return original_save_model(self, *args, **kwargs)
+ # transformers < 5 takes any non-None parallelism_config before FSDP and skips the FSDP2 state
+ # dict gather, so save without one; even a user dp_shard config would take that branch.
+ state.parallelism_config = None
+ try:
+ return original_save_model(self, *args, **kwargs)
+ finally:
+ state.parallelism_config = installed
+
@functools.wraps(original_evaluation_loop)
def patched_evaluation_loop(self, *args, **kwargs):
manager = getattr(self, "_context_parallel_manager", None)
@@ -811,5 +826,6 @@ def patch_sft_trainer() -> None:
trainer_cls.evaluate = patched_evaluate
trainer_cls.predict = patched_predict
trainer_cls.evaluation_loop = patched_evaluation_loop
+ trainer_cls.save_model = patched_save_model
trainer_cls.training_step = patched_training_step
trainer_cls.__unsloth_context_parallel__ = True