Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
FSDP2 save: one writer for a tensor parallel x FSDP2 model (#16181)
Daniel Hanc62288858b3c
diff --git a/tests/test_fsdp2_tp_writer_rank_gloo.py b/tests/test_fsdp2_tp_writer_rank_gloo.pynew file mode 100644index 00000000000..56929ad3399--- /dev/null+++ b/tests/test_fsdp2_tp_writer_rank_gloo.py@@ -0,0 +1,99 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++import multiprocessing+import os++if multiprocessing.parent_process() is not None or getattr(+ multiprocessing.current_process(), "_inheriting", False+):+ os.environ["CUDA_VISIBLE_DEVICES"] = ""++import sys # noqa: E402++import pytest # noqa: E402+import torch # noqa: E402++sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))+from test_tp_native_oracle import LLAMA, build, tpn # noqa: E402+import test_tp_native_gloo as gloo # noqa: E402+++def _case_tp_fsdp2_writer(+ rank,+ world,+ tmp,+ flattened = False,+):+ import torch.distributed as dist+ from torch.distributed.device_mesh import init_device_mesh+ from test_fsdp2_native_units import _load_native++ native = _load_native()+ if not native._supports_ignored_params(native._fsdp_api()[2]):+ return {"unsupported": True}+ from unsloth.models import fsdp2_save++ mesh = init_device_mesh(+ "cpu",+ (2, 1, world // 2) if flattened else (2, world // 2),+ mesh_dim_names = ("dp", "cp", "tp") if flattened else ("dp", "tp"),+ )+ dp = mesh["dp", "cp"]._flatten("dp_cp") if flattened else mesh["dp"]+ model = build(LLAMA)+ tpn.apply_tensor_parallel(model, mesh["tp"])+ native.shard_model(model, native.FSDP2Config(param_dtype = torch.float32, mesh = dp))+ with fsdp2_save.fsdp2_gathered(model, trainable_only = True) as writer:+ gathered = bool(writer)+ state = native.native_full_state_dict(model, trainable_only = True)+ dist.barrier()+ return {+ "module": fsdp2_save.__file__,+ "writer_rank": fsdp2_save.mesh_writer_rank(model),+ "gathered": gathered,+ "state_keys": len(state),+ }+++gloo.CASES["tp_fsdp2_writer"] = _case_tp_fsdp2_writer+++def _spawn(world, flattened):+ import tempfile++ ctx = multiprocessing.get_context("spawn")+ port = gloo._free_port()+ with tempfile.TemporaryDirectory() as tmp:+ procs = [+ ctx.Process(+ target = _worker,+ args = (r, world, port, "tp_fsdp2_writer", tmp, {"flattened": flattened}),+ )+ for r in range(world)+ ]+ for p in procs:+ p.start()+ for p in procs:+ p.join(600)+ codes = [p.exitcode for p in procs]+ assert codes == [0] * world, f"rank exit codes {codes}"+ return [+ torch.load(os.path.join(tmp, f"result_{r}.pt"), weights_only = False)+ for r in range(world)+ ]+++def _worker(*args):+ gloo._worker(*args)++[email protected]("flattened", [False, True])+def test_tp_fsdp2_model_has_one_writer(flattened):+ results = _spawn(4, flattened)+ if results[0].get("unsupported"):+ pytest.skip(reason = "torch < 2.7: no fully_shard(ignored_params)")+ print("fsdp2_save:", results[0]["module"])+ assert [r["writer_rank"] for r in results] == [0, 0, 0, 0], results+ assert [r["gathered"] for r in results] == [True, False, False, False], results+ assert results[0]["state_keys"] > 0, results+ assert [r["state_keys"] for r in results[1:]] == [0, 0, 0], resultsdiff --git a/unsloth/models/fsdp2_save.py b/unsloth/models/fsdp2_save.pyindex 6232ad40158..c8032a27511 100644--- a/unsloth/models/fsdp2_save.py+++ b/unsloth/models/fsdp2_save.py@@ -90,6 +90,34 @@ def _unit_mesh(unit):return None+def _root_mesh(mesh):+ get = getattr(mesh, "_get_root_mesh", None)+ if get is not None:+ return get()+ from torch.distributed.device_mesh import _mesh_resources++ return _mesh_resources.get_root_mesh(mesh)+++def _composed_writer_rank(meshes):+ try:+ root = _root_mesh(meshes[0])+ if any(_root_mesh(m) is not root for m in meshes[1:]):+ return None+ axes = set()+ for m in meshes:+ # Flattened mesh names need not exist in the root.+ coordinates = torch.isin(root.mesh, m.mesh.reshape(-1)).nonzero()+ if coordinates.shape[0] != m.mesh.numel():+ return None+ axes.update((coordinates != coordinates[0]).any(dim = 0).nonzero().flatten().tolist())+ coordinate = root.get_coordinate()+ index = tuple(slice(None) if axis in axes else c for axis, c in enumerate(coordinate))+ return int(root.mesh[index].min())+ except Exception:+ return None++def mesh_writer_rank(model):"""Lowest global rank of the model's FSDP2 mesh: a sub-mesh without rank 0 never gathers there."""dist = _dist()@@ -99,7 +127,15 @@ def mesh_writer_rank(model):mesh = Noneif isinstance(model, torch.nn.Module):if DTensor is not None:- mesh = next((p.device_mesh for p in model.parameters() if isinstance(p, DTensor)), None)+ meshes = []+ for p in model.parameters():+ if isinstance(p, DTensor) and p.device_mesh not in meshes:+ meshes.append(p.device_mesh)+ if len(meshes) > 1:+ writer = _composed_writer_rank(meshes)+ if writer is not None:+ return writer+ mesh = meshes[0] if meshes else Noneif mesh is None:# Unsharded parameters (eval forward, no reshard) carry no mesh; the units still do.mesh = next(filter(None, map(_unit_mesh, fsdp2_units(model))), None)