Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

FSDP2 save: one writer for a tensor parallel x FSDP2 model (#16181)

Daniel Hanc62288858b3c
diff --git a/tests/test_fsdp2_tp_writer_rank_gloo.py b/tests/test_fsdp2_tp_writer_rank_gloo.py
new file mode 100644
index 00000000000..56929ad3399
--- /dev/null
+++ b/tests/test_fsdp2_tp_writer_rank_gloo.py
@@ -0,0 +1,99 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+import multiprocessing
+import os
+
+if multiprocessing.parent_process() is not None or getattr(
+ multiprocessing.current_process(), "_inheriting", False
+):
+ os.environ["CUDA_VISIBLE_DEVICES"] = ""
+
+import sys # noqa: E402
+
+import pytest # noqa: E402
+import torch # noqa: E402
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from test_tp_native_oracle import LLAMA, build, tpn # noqa: E402
+import test_tp_native_gloo as gloo # noqa: E402
+
+
+def _case_tp_fsdp2_writer(
+ rank,
+ world,
+ tmp,
+ flattened = False,
+):
+ import torch.distributed as dist
+ from torch.distributed.device_mesh import init_device_mesh
+ from test_fsdp2_native_units import _load_native
+
+ native = _load_native()
+ if not native._supports_ignored_params(native._fsdp_api()[2]):
+ return {"unsupported": True}
+ from unsloth.models import fsdp2_save
+
+ mesh = init_device_mesh(
+ "cpu",
+ (2, 1, world // 2) if flattened else (2, world // 2),
+ mesh_dim_names = ("dp", "cp", "tp") if flattened else ("dp", "tp"),
+ )
+ dp = mesh["dp", "cp"]._flatten("dp_cp") if flattened else mesh["dp"]
+ model = build(LLAMA)
+ tpn.apply_tensor_parallel(model, mesh["tp"])
+ native.shard_model(model, native.FSDP2Config(param_dtype = torch.float32, mesh = dp))
+ with fsdp2_save.fsdp2_gathered(model, trainable_only = True) as writer:
+ gathered = bool(writer)
+ state = native.native_full_state_dict(model, trainable_only = True)
+ dist.barrier()
+ return {
+ "module": fsdp2_save.__file__,
+ "writer_rank": fsdp2_save.mesh_writer_rank(model),
+ "gathered": gathered,
+ "state_keys": len(state),
+ }
+
+
+gloo.CASES["tp_fsdp2_writer"] = _case_tp_fsdp2_writer
+
+
+def _spawn(world, flattened):
+ import tempfile
+
+ ctx = multiprocessing.get_context("spawn")
+ port = gloo._free_port()
+ with tempfile.TemporaryDirectory() as tmp:
+ procs = [
+ ctx.Process(
+ target = _worker,
+ args = (r, world, port, "tp_fsdp2_writer", tmp, {"flattened": flattened}),
+ )
+ for r in range(world)
+ ]
+ for p in procs:
+ p.start()
+ for p in procs:
+ p.join(600)
+ codes = [p.exitcode for p in procs]
+ assert codes == [0] * world, f"rank exit codes {codes}"
+ return [
+ torch.load(os.path.join(tmp, f"result_{r}.pt"), weights_only = False)
+ for r in range(world)
+ ]
+
+
+def _worker(*args):
+ gloo._worker(*args)
+
+
[email protected]("flattened", [False, True])
+def test_tp_fsdp2_model_has_one_writer(flattened):
+ results = _spawn(4, flattened)
+ if results[0].get("unsupported"):
+ pytest.skip(reason = "torch < 2.7: no fully_shard(ignored_params)")
+ print("fsdp2_save:", results[0]["module"])
+ assert [r["writer_rank"] for r in results] == [0, 0, 0, 0], results
+ assert [r["gathered"] for r in results] == [True, False, False, False], results
+ assert results[0]["state_keys"] > 0, results
+ assert [r["state_keys"] for r in results[1:]] == [0, 0, 0], results
diff --git a/unsloth/models/fsdp2_save.py b/unsloth/models/fsdp2_save.py
index 6232ad40158..c8032a27511 100644
--- a/unsloth/models/fsdp2_save.py
+++ b/unsloth/models/fsdp2_save.py
@@ -90,6 +90,34 @@ def _unit_mesh(unit):
return None
+def _root_mesh(mesh):
+ get = getattr(mesh, "_get_root_mesh", None)
+ if get is not None:
+ return get()
+ from torch.distributed.device_mesh import _mesh_resources
+
+ return _mesh_resources.get_root_mesh(mesh)
+
+
+def _composed_writer_rank(meshes):
+ try:
+ root = _root_mesh(meshes[0])
+ if any(_root_mesh(m) is not root for m in meshes[1:]):
+ return None
+ axes = set()
+ for m in meshes:
+ # Flattened mesh names need not exist in the root.
+ coordinates = torch.isin(root.mesh, m.mesh.reshape(-1)).nonzero()
+ if coordinates.shape[0] != m.mesh.numel():
+ return None
+ axes.update((coordinates != coordinates[0]).any(dim = 0).nonzero().flatten().tolist())
+ coordinate = root.get_coordinate()
+ index = tuple(slice(None) if axis in axes else c for axis, c in enumerate(coordinate))
+ return int(root.mesh[index].min())
+ except Exception:
+ return None
+
+
def mesh_writer_rank(model):
"""Lowest global rank of the model's FSDP2 mesh: a sub-mesh without rank 0 never gathers there."""
dist = _dist()
@@ -99,7 +127,15 @@ def mesh_writer_rank(model):
mesh = None
if isinstance(model, torch.nn.Module):
if DTensor is not None:
- mesh = next((p.device_mesh for p in model.parameters() if isinstance(p, DTensor)), None)
+ meshes = []
+ for p in model.parameters():
+ if isinstance(p, DTensor) and p.device_mesh not in meshes:
+ meshes.append(p.device_mesh)
+ if len(meshes) > 1:
+ writer = _composed_writer_rank(meshes)
+ if writer is not None:
+ return writer
+ mesh = meshes[0] if meshes else None
if mesh is None:
# Unsharded parameters (eval forward, no reshard) carry no mesh; the units still do.
mesh = next(filter(None, map(_unit_mesh, fsdp2_units(model))), None)