Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
FSDP2 save: compose the writer across equal meshes, not only identical ones (#17158)
Daniel Han897c742079c4
diff --git a/tests/test_fsdp2_tp_writer_rank_gloo.py b/tests/test_fsdp2_tp_writer_rank_gloo.pyindex 709f4179c35..a341bfa441a 100644--- a/tests/test_fsdp2_tp_writer_rank_gloo.py+++ b/tests/test_fsdp2_tp_writer_rank_gloo.py@@ -25,6 +25,7 @@ def _case_tp_fsdp2_writer(tmp,flattened = False,layout = None,+ earlier_model = False,):import torch.distributed as distfrom torch.distributed.device_mesh import init_device_mesh@@ -41,6 +42,13 @@ def _case_tp_fsdp2_writer(mesh_dim_names = ("dp", "cp", "tp") if flattened else ("dp", "tp"),)dp = mesh["dp", "cp"]._flatten("dp_cp") if flattened else mesh["dp"]+ if earlier_model:+ # An earlier tp x FSDP2 model on an equal mesh in the same process: DTensor ops can hand the+ # next model's tp parameters that earlier (equal, not identical) mesh.+ earlier = init_device_mesh("cpu", (2, world // 2), mesh_dim_names = ("dp", "tp"))+ first = build(LLAMA)+ tpn.apply_tensor_parallel(first, earlier["tp"])+ native.shard_model(first, native.FSDP2Config(param_dtype = torch.float32, mesh = earlier["dp"]))model = build(LLAMA)tpn.apply_tensor_parallel(model, mesh["tp"])config = dict(param_dtype = torch.float32, mesh = dp)@@ -79,6 +87,7 @@ def _spawn(world,flattened,layout = None,+ earlier_model = False,):import tempfile@@ -94,7 +103,7 @@ def _spawn(port,"tp_fsdp2_writer",tmp,- {"flattened": flattened, "layout": layout},+ {"flattened": flattened, "layout": layout, "earlier_model": earlier_model},),)for r in range(world)@@ -116,13 +125,15 @@ def _worker(*args):@pytest.mark.parametrize(- "flattened, layout",+ "flattened, layout, earlier_model",[- (False, None),- (True, None),- (False, "unsharded"),- (False, "ignored"),- (False, "ignored_compiled"),+ (False, None, False),+ (True, None, False),+ (False, "unsharded", False),+ (False, "ignored", False),+ (False, "ignored_compiled", False),+ (False, "unsharded", True),+ (False, "ignored", True),],ids = ["sharded",@@ -130,10 +141,12 @@ def _worker(*args):"unsharded_eval","all_trainable_ignored","all_trainable_ignored_compiled",+ "unsharded_eval_after_earlier_model",+ "all_trainable_ignored_after_earlier_model",],)-def test_tp_fsdp2_model_has_one_writer(flattened, layout):- results = _spawn(4, flattened, layout)+def test_tp_fsdp2_model_has_one_writer(flattened, layout, earlier_model):+ results = _spawn(4, flattened, layout, earlier_model)if results[0].get("unsupported"):pytest.skip(reason = "torch < 2.7: no fully_shard(ignored_params)")print("fsdp2_save:", results[0]["module"])diff --git a/unsloth/models/fsdp2_save.py b/unsloth/models/fsdp2_save.pyindex 73a170c8791..b5aeeaad82d 100644--- a/unsloth/models/fsdp2_save.py+++ b/unsloth/models/fsdp2_save.py@@ -99,10 +99,25 @@ def _root_mesh(mesh):return _mesh_resources.get_root_mesh(mesh)+def _same_mesh(a, b):+ """Same device type, dim names and rank layout. DTensor ops may hand back an equal mesh from an earlier+ init_device_mesh of the same shape (torch caches specs by mesh equality), so identity is not enough."""+ if a is b:+ return True+ try:+ return (+ a.device_type == b.device_type+ and tuple(a.mesh_dim_names or ()) == tuple(b.mesh_dim_names or ())+ and torch.equal(a.mesh, b.mesh)+ )+ except Exception:+ return False++def _composed_writer_rank(meshes):try:root = _root_mesh(meshes[0])- if any(_root_mesh(m) is not root for m in meshes[1:]):+ if any(not _same_mesh(_root_mesh(m), root) for m in meshes[1:]):return Noneaxes = set()for m in meshes: