Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Tensor parallel x FSDP2: average trainable TP parameters left out of FSDP2 over the data parallel ranks (#15748)

Daniel Hanf27ae8add95d
diff --git a/tests/test_fsdp2_tp_ignored_params_dp_gloo.py b/tests/test_fsdp2_tp_ignored_params_dp_gloo.py
new file mode 100644
index 00000000000..683942e4934
--- /dev/null
+++ b/tests/test_fsdp2_tp_ignored_params_dp_gloo.py
@@ -0,0 +1,138 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""dp 2 x tp 2 gloo: a TP param in FSDP2 ignored_params is averaged over dp; replicas match a single process."""
+
+import multiprocessing
+import os
+
+if multiprocessing.parent_process() is not None or getattr(
+ multiprocessing.current_process(), "_inheriting", False
+):
+ os.environ["CUDA_VISIBLE_DEVICES"] = "" # spawned gloo ranks: CPU only, also on GPU hosts
+
+import sys # noqa: E402
+
+import pytest # noqa: E402
+import torch # noqa: E402
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from test_tp_native_oracle import LLAMA, _batch, build, tpn # noqa: E402
+import test_tp_native_gloo as gloo # noqa: E402
+
+_MAX_NORM = 0.05 # small enough that every step clips
+
+
+def _ignored_names(model):
+ return [
+ n
+ for n, p in model.named_parameters()
+ if p.requires_grad and "layers.0." in n and ("q_proj" in n or "down_proj" in n)
+ ]
+
+
+def _case_tp_fsdp2_ignored(rank, world, tmp):
+ import torch.distributed as dist
+ from torch.distributed.device_mesh import init_device_mesh
+ from torch.distributed.tensor import DTensor
+ from test_fsdp2_native_units import _load_native
+
+ native = _load_native()
+ if not native._supports_ignored_params(native._fsdp_api()[2]):
+ return {"unsupported": True}
+ mesh = init_device_mesh("cpu", (2, world // 2), mesh_dim_names = ("dp", "tp"))
+ dp = mesh["dp"].get_local_rank()
+ ref = build(LLAMA)
+ model = build(LLAMA)
+ ids = _batch(ref, bsz = 4)
+ mine = ids[2 * dp : 2 * dp + 2]
+ tpn.apply_tensor_parallel(model, mesh["tp"])
+ names = _ignored_names(model)
+ by_name = dict(model.named_parameters())
+ ignored = {by_name[n] for n in names}
+ native.shard_model(
+ model,
+ native.FSDP2Config(param_dtype = torch.float32, mesh = mesh["dp"], ignored_params = ignored),
+ )
+ by_name = dict(model.named_parameters())
+ kept_tp = [
+ n for n in names if isinstance(by_name[n], DTensor) and by_name[n].device_mesh.ndim == 1
+ ]
+ params = [p for p in model.parameters() if p.requires_grad]
+ ref_by_name = dict(ref.named_parameters())
+ ref_params = [p for p in ref.parameters() if p.requires_grad]
+ opt = torch.optim.SGD(params, lr = 1.0)
+ ref_opt = torch.optim.SGD(ref_params, lr = 1.0)
+ local, norms, ref_norms = [], [], []
+ for _ in range(3):
+ model(input_ids = mine, labels = mine, use_cache = False).loss.backward()
+ ref(input_ids = ids, labels = ids, use_cache = False).loss.backward()
+ norms.append(float(native.native_clip_grad_norm_(params, _MAX_NORM)))
+ ref_norms.append(float(torch.nn.utils.clip_grad_norm_(ref_params, _MAX_NORM)))
+ opt.step(), opt.zero_grad(set_to_none = True)
+ ref_opt.step(), ref_opt.zero_grad(set_to_none = True)
+ local.append({n: by_name[n].to_local().detach().clone() for n in kept_tp})
+ full = {n: tpn._full(by_name[n]).detach().clone() for n in kept_tp}
+ dist.barrier()
+ return {
+ "dp": dp,
+ "tp": mesh["tp"].get_local_rank(),
+ "names": names,
+ "kept_tp": kept_tp,
+ "local": local,
+ "full": full,
+ "ref": {n: ref_by_name[n].detach().clone() for n in kept_tp},
+ "norms": norms,
+ "ref_norms": ref_norms,
+ }
+
+
+gloo.CASES["tp_fsdp2_ignored"] = _case_tp_fsdp2_ignored
+
+
+def _spawn(world):
+ """test_tp_native_gloo._spawn with this module's case registered in every spawned rank."""
+ import tempfile
+
+ ctx = multiprocessing.get_context("spawn")
+ port = gloo._free_port()
+ with tempfile.TemporaryDirectory() as tmp:
+ procs = [
+ ctx.Process(target = _worker, args = (r, world, port, "tp_fsdp2_ignored", tmp, {}))
+ for r in range(world)
+ ]
+ for p in procs:
+ p.start()
+ for p in procs:
+ p.join(600)
+ codes = [p.exitcode for p in procs]
+ assert codes == [0] * world, f"rank exit codes {codes}"
+ return [
+ torch.load(os.path.join(tmp, f"result_{r}.pt"), weights_only = False)
+ for r in range(world)
+ ]
+
+
+def _worker(*args):
+ gloo._worker(*args)
+
+
+def test_tp_param_in_ignored_params_is_averaged_over_dp():
+ results = _spawn(4)
+ if results[0].get("unsupported"):
+ pytest.skip("torch < 2.7: no fully_shard(ignored_params)")
+ for res in results:
+ assert res["names"] and res["kept_tp"] == res["names"], (res["names"], res["kept_tp"])
+ for a in results:
+ for b in results:
+ if a["tp"] == b["tp"] and a["dp"] < b["dp"]:
+ for step, (la, lb) in enumerate(zip(a["local"], b["local"])):
+ for n in la:
+ assert torch.equal(la[n], lb[n]), f"dp replicas differ at step {step}: {n}"
+ for res in results:
+ assert res["norms"] == pytest.approx(res["ref_norms"], rel = 1e-5), (
+ res["norms"],
+ res["ref_norms"],
+ )
+ for n, ref in res["ref"].items():
+ torch.testing.assert_close(res["full"][n], ref, rtol = 1e-4, atol = 1e-6, msg = n)
diff --git a/unsloth/models/fsdp2_save.py b/unsloth/models/fsdp2_save.py
index 3a155253fdd..d2f45bca1db 100644
--- a/unsloth/models/fsdp2_save.py
+++ b/unsloth/models/fsdp2_save.py
@@ -1246,7 +1246,8 @@ def _sync_replicated_grads(
dtype = torch.float64 if p.dtype == torch.float64 else torch.float32
groups.setdefault((str(p.device), dtype), []).append(p)
for (device, dtype), group in groups.items():
- sizes = [p.numel() for p in group]
+ # A TP DTensor outside FSDP2: dp ranks hold the same tp slice, so average the local shard.
+ sizes = [_local_tensor(p).numel() for p in group]
total = sum(sizes)
key = tuple((id(p), n) for p, n in zip(group, sizes))
cached = buffers.get((device, dtype)) if buffers is not None else None
@@ -1256,7 +1257,7 @@ def _sync_replicated_grads(
flat = torch.empty(total + len(group), dtype = dtype, device = group[0].device)
if buffers is not None:
buffers[(device, dtype)] = (key, flat)
- grads = [p.grad for p in group]
+ grads = [_local_tensor(p.grad) for p in group]
local_all = all(g is not None for g in grads)
flags = flat[total:]
if local_all:
@@ -1279,14 +1280,63 @@ def _sync_replicated_grads(
offset = 0
for i, (p, g, n) in enumerate(zip(group, grads, sizes)):
if g is not None or used[i] > 0:
- value = flat[offset : offset + n].view(p.shape)
- if p.grad is None:
- p.grad = value.to(p.dtype, copy = True)
+ value = flat[offset : offset + n].view(_local_tensor(p).shape)
+ if g is None:
+ p.grad = _like_param(p, value.to(p.dtype, copy = True))
else:
- p.grad.copy_(value)
+ g.copy_(value)
offset += n
+def _local_tensor(t):
+ """Local shard view of a DTensor (writes reach it); anything else unchanged."""
+ local = getattr(t, "_local_tensor", None)
+ return local if local is not None and getattr(t, "device_mesh", None) is not None else t
+
+
+def _like_param(p, local):
+ if getattr(p, "device_mesh", None) is None or getattr(p, "_local_tensor", None) is None:
+ return local
+ return _dtensor_class().from_local(local, p.device_mesh, p.placements, run_check = False)
+
+
+def _fsdp_managed_param_ids(model):
+ """ids of FSDP2-managed params, or None when no FSDP2 state is readable (treat all DTensors as FSDP2's)."""
+ ids = set()
+ found = False
+ for unit in fsdp2_units(model):
+ try:
+ st = unit._get_fsdp_state()
+ except Exception:
+ continue
+ groups = getattr(st, "_fsdp_param_groups", None)
+ if groups is None:
+ group = getattr(st, "_fsdp_param_group", None)
+ groups = [group] if group is not None else []
+ for group in groups:
+ for fp in getattr(group, "fsdp_params", None) or ():
+ found = True
+ for attr in ("sharded_param", "_unsharded_param"):
+ t = getattr(fp, attr, None)
+ if t is not None:
+ ids.add(id(t))
+ return ids if found else None
+
+
+def _tp_shard_outside(param, mesh, dist):
+ """True when param's mesh meets the dp `mesh` only at this rank, so its local shard is replicated
+ over `mesh`; any other overlap would average different shards."""
+ if mesh is None:
+ return False
+ try:
+ mine = dist.get_rank()
+ own = set(param.device_mesh.mesh.flatten().tolist())
+ dp = set(mesh.mesh.flatten().tolist())
+ except Exception:
+ return False
+ return own & dp == {mine}
+
+
def _fsdp_grad_sync_deferred(model, cache):
"""True when every FSDP2 param group has set_requires_gradient_sync(False), i.e. a non-final
micro-batch. False when no group is found (sync every backward)."""
@@ -1355,9 +1405,16 @@ def reduce_unsharded_trainable_grads(model):
state["queued"] = True
torch.autograd.Variable._execution_engine.queue_callback(_sync)
+ managed = False
for name, param in model.named_parameters():
- if id(param) in seen or not param.requires_grad or isinstance(param, DTensor):
+ if id(param) in seen or not param.requires_grad:
continue
+ if isinstance(param, DTensor):
+ # FSDP2 does not reduce TP DTensors in ignored_params; average them here.
+ if managed is False:
+ managed = _fsdp_managed_param_ids(model)
+ if managed is None or id(param) in managed or not _tp_shard_outside(param, mesh, dist):
+ continue
seen.add(id(param))
if getattr(param, "_unsloth_grad_all_reduce", False):
continue