Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

FSDP2 save: compose the writer across equal meshes, not only identical ones (#17158)

Daniel Han897c742079c4
diff --git a/tests/test_fsdp2_tp_writer_rank_gloo.py b/tests/test_fsdp2_tp_writer_rank_gloo.py
index 709f4179c35..a341bfa441a 100644
--- a/tests/test_fsdp2_tp_writer_rank_gloo.py
+++ b/tests/test_fsdp2_tp_writer_rank_gloo.py
@@ -25,6 +25,7 @@ def _case_tp_fsdp2_writer(
tmp,
flattened = False,
layout = None,
+ earlier_model = False,
):
import torch.distributed as dist
from torch.distributed.device_mesh import init_device_mesh
@@ -41,6 +42,13 @@ def _case_tp_fsdp2_writer(
mesh_dim_names = ("dp", "cp", "tp") if flattened else ("dp", "tp"),
)
dp = mesh["dp", "cp"]._flatten("dp_cp") if flattened else mesh["dp"]
+ if earlier_model:
+ # An earlier tp x FSDP2 model on an equal mesh in the same process: DTensor ops can hand the
+ # next model's tp parameters that earlier (equal, not identical) mesh.
+ earlier = init_device_mesh("cpu", (2, world // 2), mesh_dim_names = ("dp", "tp"))
+ first = build(LLAMA)
+ tpn.apply_tensor_parallel(first, earlier["tp"])
+ native.shard_model(first, native.FSDP2Config(param_dtype = torch.float32, mesh = earlier["dp"]))
model = build(LLAMA)
tpn.apply_tensor_parallel(model, mesh["tp"])
config = dict(param_dtype = torch.float32, mesh = dp)
@@ -79,6 +87,7 @@ def _spawn(
world,
flattened,
layout = None,
+ earlier_model = False,
):
import tempfile
@@ -94,7 +103,7 @@ def _spawn(
port,
"tp_fsdp2_writer",
tmp,
- {"flattened": flattened, "layout": layout},
+ {"flattened": flattened, "layout": layout, "earlier_model": earlier_model},
),
)
for r in range(world)
@@ -116,13 +125,15 @@ def _worker(*args):
@pytest.mark.parametrize(
- "flattened, layout",
+ "flattened, layout, earlier_model",
[
- (False, None),
- (True, None),
- (False, "unsharded"),
- (False, "ignored"),
- (False, "ignored_compiled"),
+ (False, None, False),
+ (True, None, False),
+ (False, "unsharded", False),
+ (False, "ignored", False),
+ (False, "ignored_compiled", False),
+ (False, "unsharded", True),
+ (False, "ignored", True),
],
ids = [
"sharded",
@@ -130,10 +141,12 @@ def _worker(*args):
"unsharded_eval",
"all_trainable_ignored",
"all_trainable_ignored_compiled",
+ "unsharded_eval_after_earlier_model",
+ "all_trainable_ignored_after_earlier_model",
],
)
-def test_tp_fsdp2_model_has_one_writer(flattened, layout):
- results = _spawn(4, flattened, layout)
+def test_tp_fsdp2_model_has_one_writer(flattened, layout, earlier_model):
+ results = _spawn(4, flattened, layout, earlier_model)
if results[0].get("unsupported"):
pytest.skip(reason = "torch < 2.7: no fully_shard(ignored_params)")
print("fsdp2_save:", results[0]["module"])
diff --git a/unsloth/models/fsdp2_save.py b/unsloth/models/fsdp2_save.py
index 73a170c8791..b5aeeaad82d 100644
--- a/unsloth/models/fsdp2_save.py
+++ b/unsloth/models/fsdp2_save.py
@@ -99,10 +99,25 @@ def _root_mesh(mesh):
return _mesh_resources.get_root_mesh(mesh)
+def _same_mesh(a, b):
+ """Same device type, dim names and rank layout. DTensor ops may hand back an equal mesh from an earlier
+ init_device_mesh of the same shape (torch caches specs by mesh equality), so identity is not enough."""
+ if a is b:
+ return True
+ try:
+ return (
+ a.device_type == b.device_type
+ and tuple(a.mesh_dim_names or ()) == tuple(b.mesh_dim_names or ())
+ and torch.equal(a.mesh, b.mesh)
+ )
+ except Exception:
+ return False
+
+
def _composed_writer_rank(meshes):
try:
root = _root_mesh(meshes[0])
- if any(_root_mesh(m) is not root for m in meshes[1:]):
+ if any(not _same_mesh(_root_mesh(m), root) for m in meshes[1:]):
return None
axes = set()
for m in meshes: