Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
TP x FSDP2 save: one writer when only tp-mesh DTensors remain (#15257)
Shubhankar Tripathy984888be82cc
diff --git a/tests/test_fsdp2_tp_writer_rank_gloo.py b/tests/test_fsdp2_tp_writer_rank_gloo.pyindex 56929ad3399..709f4179c35 100644--- a/tests/test_fsdp2_tp_writer_rank_gloo.py+++ b/tests/test_fsdp2_tp_writer_rank_gloo.py@@ -24,6 +24,7 @@ def _case_tp_fsdp2_writer(world,tmp,flattened = False,+ layout = None,):import torch.distributed as distfrom torch.distributed.device_mesh import init_device_mesh@@ -42,14 +43,30 @@ def _case_tp_fsdp2_writer(dp = mesh["dp", "cp"]._flatten("dp_cp") if flattened else mesh["dp"]model = build(LLAMA)tpn.apply_tensor_parallel(model, mesh["tp"])- native.shard_model(model, native.FSDP2Config(param_dtype = torch.float32, mesh = dp))+ config = dict(param_dtype = torch.float32, mesh = dp)+ if layout == "unsharded":+ config.update(reshard_after_forward = False, reshard_last_block = False)+ elif layout in ("ignored", "ignored_compiled"):+ # Every trainable tp parameter left out of FSDP2 (#16180): only tp-mesh DTensors remain.+ config["ignored_params"] = {p for p in model.parameters() if p.requires_grad}+ native.shard_model(model, native.FSDP2Config(**config))+ if layout == "unsharded":+ # An eval forward leaves every unit gathered: the parameters are tp-only DTensors.+ with torch.no_grad():+ model(input_ids = torch.zeros(1, 4, dtype = torch.long), use_cache = False)+ if layout == "ignored_compiled":+ # The recorded dp mesh sits on the root, not on the OptimizedModule wrapper.+ model = torch.compile(model, backend = "eager")+ # Before any gather: native_full_state_dict and native_save_optimizer read the writer as is.+ writer_rank = fsdp2_save.mesh_writer_rank(model)+ state = native.native_full_state_dict(model, trainable_only = True)with fsdp2_save.fsdp2_gathered(model, trainable_only = True) as writer:gathered = bool(writer)- state = native.native_full_state_dict(model, trainable_only = True)dist.barrier()return {"module": fsdp2_save.__file__,- "writer_rank": fsdp2_save.mesh_writer_rank(model),+ "writer_rank": writer_rank,+ "writer_rank_after": fsdp2_save.mesh_writer_rank(model),"gathered": gathered,"state_keys": len(state),}@@ -58,7 +75,11 @@ def _case_tp_fsdp2_writer(gloo.CASES["tp_fsdp2_writer"] = _case_tp_fsdp2_writer-def _spawn(world, flattened):+def _spawn(+ world,+ flattened,+ layout = None,+):import tempfilectx = multiprocessing.get_context("spawn")@@ -67,7 +88,14 @@ def _spawn(world, flattened):procs = [ctx.Process(target = _worker,- args = (r, world, port, "tp_fsdp2_writer", tmp, {"flattened": flattened}),+ args = (+ r,+ world,+ port,+ "tp_fsdp2_writer",+ tmp,+ {"flattened": flattened, "layout": layout},+ ),)for r in range(world)]@@ -87,13 +115,30 @@ def _worker(*args):gloo._worker(*args)[email protected]("flattened", [False, True])-def test_tp_fsdp2_model_has_one_writer(flattened):- results = _spawn(4, flattened)+ "flattened, layout",+ [+ (False, None),+ (True, None),+ (False, "unsharded"),+ (False, "ignored"),+ (False, "ignored_compiled"),+ ],+ ids = [+ "sharded",+ "flattened",+ "unsharded_eval",+ "all_trainable_ignored",+ "all_trainable_ignored_compiled",+ ],+)+def test_tp_fsdp2_model_has_one_writer(flattened, layout):+ results = _spawn(4, flattened, layout)if results[0].get("unsupported"):pytest.skip(reason = "torch < 2.7: no fully_shard(ignored_params)")print("fsdp2_save:", results[0]["module"])assert [r["writer_rank"] for r in results] == [0, 0, 0, 0], results+ assert [r["writer_rank_after"] for r in results] == [0, 0, 0, 0], resultsassert [r["gathered"] for r in results] == [True, False, False, False], resultsassert results[0]["state_keys"] > 0, resultsassert [r["state_keys"] for r in results[1:]] == [0, 0, 0], resultsdiff --git a/tests/test_tp_native_gloo.py b/tests/test_tp_native_gloo.pyindex ed116542d97..2c85c2967ce 100644--- a/tests/test_tp_native_gloo.py+++ b/tests/test_tp_native_gloo.py@@ -665,6 +665,66 @@ def _case_tp_fsdp2_trainable_norms(rank, world, tmp):}+def _case_tp_fsdp2_writer(+ rank,+ world,+ tmp,+ head_only = False,+ compiled = False,+):+ """dp 2 x tp 2: the collective saves name one writer for the model, not one per tp group."""+ import importlib+ from torch.distributed.device_mesh import init_device_mesh+ from test_fsdp2_native_units import _load_native++ native = _load_native()+ save = importlib.import_module("unsloth.models.fsdp2_save")+ mesh = init_device_mesh("cpu", (2, world // 2), mesh_dim_names = ("dp", "tp"))+ model = build(*ARCHS["llama"], **({"target_modules": ["lm_head"]} if head_only else {}))+ expected = {name: p.detach().clone() for name, p in model.named_parameters()}+ tpn.apply_tensor_parallel(model, mesh["tp"])+ if not native._supports_ignored_params(native._fsdp_api()[2]):+ return {"unsupported": "torch 2.7+"}+ native.shard_model(model, native.FSDP2Config(param_dtype = torch.float32, mesh = mesh["dp"]))+ unwrapped_model = model+ if compiled:+ model = torch.compile(model, backend = "eager")+ res = {"writer_rank": save.mesh_writer_rank(model)}+ with save.fsdp2_gathered(model) as writer:+ res["writer"] = writer+ if writer:+ for name, param in unwrapped_model.named_parameters():+ torch.testing.assert_close(param, expected[name])+ state = native.native_full_state_dict(model)+ res["full_keys"] = len(state)+ if rank == 0:+ for name, param in expected.items():+ torch.testing.assert_close(state[name], param)+ return res+++def _case_tp_fsdp2_writer_pipeline_slice(rank, world, tmp):+ import importlib+ from torch.distributed.device_mesh import init_device_mesh+ from test_fsdp2_native_units import _load_native++ from torch.distributed.tensor.parallel import ColwiseParallel, parallelize_module++ native = _load_native()+ save = importlib.import_module("unsloth.models.fsdp2_save")+ mesh = init_device_mesh("cpu", (2, 1, 2), mesh_dim_names = ("pp", "dp", "tp"))+ # One pipeline stage per pp rank: a tp layer sharded by FSDP2 over the stage's dp slice.+ model = torch.nn.Sequential(torch.nn.Linear(4, 4), torch.nn.Linear(4, 4))+ model[0].weight.requires_grad_(False)+ parallelize_module(model[0], mesh["tp"], ColwiseParallel())+ parallelize_module(model[1], mesh["tp"], ColwiseParallel())+ fully_shard = native._fsdp_api()[2]+ if not native._supports_ignored_params(fully_shard):+ return {"unsupported": "torch 2.7+"}+ fully_shard(model, mesh = mesh["dp"], ignored_params = set(model[1].parameters()))+ return {"writer_rank": save.mesh_writer_rank(model)}++class _Rows(torch.utils.data.Dataset):def __init__(self, ids):self.ids = ids@@ -981,6 +1041,8 @@ CASES = {"sharded_lora_init": _case_sharded_lora_init,"tp_fsdp2": _case_tp_fsdp2,"tp_fsdp2_trainable_norms": _case_tp_fsdp2_trainable_norms,+ "tp_fsdp2_one_writer": _case_tp_fsdp2_writer,+ "tp_fsdp2_writer_pipeline_slice": _case_tp_fsdp2_writer_pipeline_slice,"trainer": _case_trainer,"sampler": _case_sampler,"sampler_resume": _case_sampler_resume,@@ -1371,6 +1433,27 @@ def test_tp_x_fsdp2_refuses_trainable_head_norms():assert res["error"] and "trainable per-head norms" in res["error"], res[email protected]("compiled", [False, True])[email protected]("head_only", [False, True])+def test_tp2_x_fsdp2_dp2_saves_from_one_writer(head_only, compiled):+ """A frozen tp weight's mesh is its own tp group: the writer must not follow it (two writers,+ two concurrent writes of the same optimizer.pt / scheduler.pt and two Hub pushes)."""+ skip_unsupported("llama")+ results = _spawn("tp_fsdp2_one_writer", 4, head_only = head_only, compiled = compiled)+ if "unsupported" in results[0]:+ pytest.skip("torch < 2.7: tensor parallel x FSDP2 is refused")+ assert [r["writer_rank"] for r in results] == [0, 0, 0, 0], results+ assert [r["writer"] for r in results] == [True, False, False, False], results+ assert [r["full_keys"] > 0 for r in results] == [True, False, False, False], results+++def test_tp_fsdp2_writer_keeps_pipeline_stages_separate():+ results = _spawn("tp_fsdp2_writer_pipeline_slice", 4)+ if "unsupported" in results[0]:+ pytest.skip("torch < 2.7: no fully_shard(ignored_params)")+ assert [r["writer_rank"] for r in results] == [0, 0, 2, 2], results++def test_trainer_splits_mixed_optimizer_groups_for_tp_only():# modules_to_save lm_head: a plain trainable next to the LoRA DTensors (foreach / fused take one kind).kw = {"modules_to_save": ["lm_head"]}diff --git a/unsloth/models/fsdp2_save.py b/unsloth/models/fsdp2_save.pyindex f83981edf35..73a170c8791 100644--- a/unsloth/models/fsdp2_save.py+++ b/unsloth/models/fsdp2_save.py@@ -118,6 +118,14 @@ def _composed_writer_rank(meshes):return None+def _recorded_mesh(model):+ """The mesh shard_model recorded on the root (under a torch.compile wrapper, on `_orig_mod`)."""+ if not isinstance(model, torch.nn.Module):+ return None+ model = model._modules.get("_orig_mod", model)+ return model.__dict__.get("_unsloth_fsdp2_mesh", None)++def mesh_writer_rank(model):"""Lowest global rank of the model's FSDP2 mesh: a sub-mesh without rank 0 never gathers there."""dist = _dist()@@ -131,6 +139,15 @@ def mesh_writer_rank(model):for p in model.parameters():if isinstance(p, DTensor) and p.device_mesh not in meshes:meshes.append(p.device_mesh)+ if meshes:+ # The dp mesh too: unsharded parameters (eval forward, no reshard) or every+ # parameter in ignored_params leave only tp DTensors, one writer per tp group.+ for dp_mesh in (+ *map(_unit_mesh, fsdp2_units(model)),+ _recorded_mesh(model),+ ):+ if dp_mesh is not None and dp_mesh not in meshes:+ meshes.append(dp_mesh)if len(meshes) > 1:writer = _composed_writer_rank(meshes)if writer is not None:@@ -140,7 +157,7 @@ def mesh_writer_rank(model):# Unsharded parameters (eval forward, no reshard) carry no mesh; the units still do.mesh = next(filter(None, map(_unit_mesh, fsdp2_units(model))), None)if mesh is None:- mesh = model.__dict__.get("_unsloth_fsdp2_mesh", None)+ mesh = _recorded_mesh(model)try:return int(mesh.mesh.min()) if mesh is not None else 0except Exception: