Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

TP x FSDP2 save: one writer when only tp-mesh DTensors remain (#15257)

Shubhankar Tripathy984888be82cc
diff --git a/tests/test_fsdp2_tp_writer_rank_gloo.py b/tests/test_fsdp2_tp_writer_rank_gloo.py
index 56929ad3399..709f4179c35 100644
--- a/tests/test_fsdp2_tp_writer_rank_gloo.py
+++ b/tests/test_fsdp2_tp_writer_rank_gloo.py
@@ -24,6 +24,7 @@ def _case_tp_fsdp2_writer(
world,
tmp,
flattened = False,
+ layout = None,
):
import torch.distributed as dist
from torch.distributed.device_mesh import init_device_mesh
@@ -42,14 +43,30 @@ def _case_tp_fsdp2_writer(
dp = mesh["dp", "cp"]._flatten("dp_cp") if flattened else mesh["dp"]
model = build(LLAMA)
tpn.apply_tensor_parallel(model, mesh["tp"])
- native.shard_model(model, native.FSDP2Config(param_dtype = torch.float32, mesh = dp))
+ config = dict(param_dtype = torch.float32, mesh = dp)
+ if layout == "unsharded":
+ config.update(reshard_after_forward = False, reshard_last_block = False)
+ elif layout in ("ignored", "ignored_compiled"):
+ # Every trainable tp parameter left out of FSDP2 (#16180): only tp-mesh DTensors remain.
+ config["ignored_params"] = {p for p in model.parameters() if p.requires_grad}
+ native.shard_model(model, native.FSDP2Config(**config))
+ if layout == "unsharded":
+ # An eval forward leaves every unit gathered: the parameters are tp-only DTensors.
+ with torch.no_grad():
+ model(input_ids = torch.zeros(1, 4, dtype = torch.long), use_cache = False)
+ if layout == "ignored_compiled":
+ # The recorded dp mesh sits on the root, not on the OptimizedModule wrapper.
+ model = torch.compile(model, backend = "eager")
+ # Before any gather: native_full_state_dict and native_save_optimizer read the writer as is.
+ writer_rank = fsdp2_save.mesh_writer_rank(model)
+ state = native.native_full_state_dict(model, trainable_only = True)
with fsdp2_save.fsdp2_gathered(model, trainable_only = True) as writer:
gathered = bool(writer)
- state = native.native_full_state_dict(model, trainable_only = True)
dist.barrier()
return {
"module": fsdp2_save.__file__,
- "writer_rank": fsdp2_save.mesh_writer_rank(model),
+ "writer_rank": writer_rank,
+ "writer_rank_after": fsdp2_save.mesh_writer_rank(model),
"gathered": gathered,
"state_keys": len(state),
}
@@ -58,7 +75,11 @@ def _case_tp_fsdp2_writer(
gloo.CASES["tp_fsdp2_writer"] = _case_tp_fsdp2_writer
-def _spawn(world, flattened):
+def _spawn(
+ world,
+ flattened,
+ layout = None,
+):
import tempfile
ctx = multiprocessing.get_context("spawn")
@@ -67,7 +88,14 @@ def _spawn(world, flattened):
procs = [
ctx.Process(
target = _worker,
- args = (r, world, port, "tp_fsdp2_writer", tmp, {"flattened": flattened}),
+ args = (
+ r,
+ world,
+ port,
+ "tp_fsdp2_writer",
+ tmp,
+ {"flattened": flattened, "layout": layout},
+ ),
)
for r in range(world)
]
@@ -87,13 +115,30 @@ def _worker(*args):
gloo._worker(*args)
[email protected]("flattened", [False, True])
-def test_tp_fsdp2_model_has_one_writer(flattened):
- results = _spawn(4, flattened)
+ "flattened, layout",
+ [
+ (False, None),
+ (True, None),
+ (False, "unsharded"),
+ (False, "ignored"),
+ (False, "ignored_compiled"),
+ ],
+ ids = [
+ "sharded",
+ "flattened",
+ "unsharded_eval",
+ "all_trainable_ignored",
+ "all_trainable_ignored_compiled",
+ ],
+)
+def test_tp_fsdp2_model_has_one_writer(flattened, layout):
+ results = _spawn(4, flattened, layout)
if results[0].get("unsupported"):
pytest.skip(reason = "torch < 2.7: no fully_shard(ignored_params)")
print("fsdp2_save:", results[0]["module"])
assert [r["writer_rank"] for r in results] == [0, 0, 0, 0], results
+ assert [r["writer_rank_after"] for r in results] == [0, 0, 0, 0], results
assert [r["gathered"] for r in results] == [True, False, False, False], results
assert results[0]["state_keys"] > 0, results
assert [r["state_keys"] for r in results[1:]] == [0, 0, 0], results
diff --git a/tests/test_tp_native_gloo.py b/tests/test_tp_native_gloo.py
index ed116542d97..2c85c2967ce 100644
--- a/tests/test_tp_native_gloo.py
+++ b/tests/test_tp_native_gloo.py
@@ -665,6 +665,66 @@ def _case_tp_fsdp2_trainable_norms(rank, world, tmp):
}
+def _case_tp_fsdp2_writer(
+ rank,
+ world,
+ tmp,
+ head_only = False,
+ compiled = False,
+):
+ """dp 2 x tp 2: the collective saves name one writer for the model, not one per tp group."""
+ import importlib
+ from torch.distributed.device_mesh import init_device_mesh
+ from test_fsdp2_native_units import _load_native
+
+ native = _load_native()
+ save = importlib.import_module("unsloth.models.fsdp2_save")
+ mesh = init_device_mesh("cpu", (2, world // 2), mesh_dim_names = ("dp", "tp"))
+ model = build(*ARCHS["llama"], **({"target_modules": ["lm_head"]} if head_only else {}))
+ expected = {name: p.detach().clone() for name, p in model.named_parameters()}
+ tpn.apply_tensor_parallel(model, mesh["tp"])
+ if not native._supports_ignored_params(native._fsdp_api()[2]):
+ return {"unsupported": "torch 2.7+"}
+ native.shard_model(model, native.FSDP2Config(param_dtype = torch.float32, mesh = mesh["dp"]))
+ unwrapped_model = model
+ if compiled:
+ model = torch.compile(model, backend = "eager")
+ res = {"writer_rank": save.mesh_writer_rank(model)}
+ with save.fsdp2_gathered(model) as writer:
+ res["writer"] = writer
+ if writer:
+ for name, param in unwrapped_model.named_parameters():
+ torch.testing.assert_close(param, expected[name])
+ state = native.native_full_state_dict(model)
+ res["full_keys"] = len(state)
+ if rank == 0:
+ for name, param in expected.items():
+ torch.testing.assert_close(state[name], param)
+ return res
+
+
+def _case_tp_fsdp2_writer_pipeline_slice(rank, world, tmp):
+ import importlib
+ from torch.distributed.device_mesh import init_device_mesh
+ from test_fsdp2_native_units import _load_native
+
+ from torch.distributed.tensor.parallel import ColwiseParallel, parallelize_module
+
+ native = _load_native()
+ save = importlib.import_module("unsloth.models.fsdp2_save")
+ mesh = init_device_mesh("cpu", (2, 1, 2), mesh_dim_names = ("pp", "dp", "tp"))
+ # One pipeline stage per pp rank: a tp layer sharded by FSDP2 over the stage's dp slice.
+ model = torch.nn.Sequential(torch.nn.Linear(4, 4), torch.nn.Linear(4, 4))
+ model[0].weight.requires_grad_(False)
+ parallelize_module(model[0], mesh["tp"], ColwiseParallel())
+ parallelize_module(model[1], mesh["tp"], ColwiseParallel())
+ fully_shard = native._fsdp_api()[2]
+ if not native._supports_ignored_params(fully_shard):
+ return {"unsupported": "torch 2.7+"}
+ fully_shard(model, mesh = mesh["dp"], ignored_params = set(model[1].parameters()))
+ return {"writer_rank": save.mesh_writer_rank(model)}
+
+
class _Rows(torch.utils.data.Dataset):
def __init__(self, ids):
self.ids = ids
@@ -981,6 +1041,8 @@ CASES = {
"sharded_lora_init": _case_sharded_lora_init,
"tp_fsdp2": _case_tp_fsdp2,
"tp_fsdp2_trainable_norms": _case_tp_fsdp2_trainable_norms,
+ "tp_fsdp2_one_writer": _case_tp_fsdp2_writer,
+ "tp_fsdp2_writer_pipeline_slice": _case_tp_fsdp2_writer_pipeline_slice,
"trainer": _case_trainer,
"sampler": _case_sampler,
"sampler_resume": _case_sampler_resume,
@@ -1371,6 +1433,27 @@ def test_tp_x_fsdp2_refuses_trainable_head_norms():
assert res["error"] and "trainable per-head norms" in res["error"], res
[email protected]("compiled", [False, True])
[email protected]("head_only", [False, True])
+def test_tp2_x_fsdp2_dp2_saves_from_one_writer(head_only, compiled):
+ """A frozen tp weight's mesh is its own tp group: the writer must not follow it (two writers,
+ two concurrent writes of the same optimizer.pt / scheduler.pt and two Hub pushes)."""
+ skip_unsupported("llama")
+ results = _spawn("tp_fsdp2_one_writer", 4, head_only = head_only, compiled = compiled)
+ if "unsupported" in results[0]:
+ pytest.skip("torch < 2.7: tensor parallel x FSDP2 is refused")
+ assert [r["writer_rank"] for r in results] == [0, 0, 0, 0], results
+ assert [r["writer"] for r in results] == [True, False, False, False], results
+ assert [r["full_keys"] > 0 for r in results] == [True, False, False, False], results
+
+
+def test_tp_fsdp2_writer_keeps_pipeline_stages_separate():
+ results = _spawn("tp_fsdp2_writer_pipeline_slice", 4)
+ if "unsupported" in results[0]:
+ pytest.skip("torch < 2.7: no fully_shard(ignored_params)")
+ assert [r["writer_rank"] for r in results] == [0, 0, 2, 2], results
+
+
def test_trainer_splits_mixed_optimizer_groups_for_tp_only():
# modules_to_save lm_head: a plain trainable next to the LoRA DTensors (foreach / fused take one kind).
kw = {"modules_to_save": ["lm_head"]}
diff --git a/unsloth/models/fsdp2_save.py b/unsloth/models/fsdp2_save.py
index f83981edf35..73a170c8791 100644
--- a/unsloth/models/fsdp2_save.py
+++ b/unsloth/models/fsdp2_save.py
@@ -118,6 +118,14 @@ def _composed_writer_rank(meshes):
return None
+def _recorded_mesh(model):
+ """The mesh shard_model recorded on the root (under a torch.compile wrapper, on `_orig_mod`)."""
+ if not isinstance(model, torch.nn.Module):
+ return None
+ model = model._modules.get("_orig_mod", model)
+ return model.__dict__.get("_unsloth_fsdp2_mesh", None)
+
+
def mesh_writer_rank(model):
"""Lowest global rank of the model's FSDP2 mesh: a sub-mesh without rank 0 never gathers there."""
dist = _dist()
@@ -131,6 +139,15 @@ def mesh_writer_rank(model):
for p in model.parameters():
if isinstance(p, DTensor) and p.device_mesh not in meshes:
meshes.append(p.device_mesh)
+ if meshes:
+ # The dp mesh too: unsharded parameters (eval forward, no reshard) or every
+ # parameter in ignored_params leave only tp DTensors, one writer per tp group.
+ for dp_mesh in (
+ *map(_unit_mesh, fsdp2_units(model)),
+ _recorded_mesh(model),
+ ):
+ if dp_mesh is not None and dp_mesh not in meshes:
+ meshes.append(dp_mesh)
if len(meshes) > 1:
writer = _composed_writer_rank(meshes)
if writer is not None:
@@ -140,7 +157,7 @@ def mesh_writer_rank(model):
# Unsharded parameters (eval forward, no reshard) carry no mesh; the units still do.
mesh = next(filter(None, map(_unit_mesh, fsdp2_units(model))), None)
if mesh is None:
- mesh = model.__dict__.get("_unsloth_fsdp2_mesh", None)
+ mesh = _recorded_mesh(model)
try:
return int(mesh.mesh.min()) if mesh is not None else 0
except Exception: