Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

FSDP2: replicated grad sync without host syncs, once per gradient accumulation cycle (#15135)

Daniel Han065955d6676c
diff --git a/tests/test_fsdp2_replicated_grad_sync_gloo.py b/tests/test_fsdp2_replicated_grad_sync_gloo.py
new file mode 100644
index 00000000000..ed81c787830
--- /dev/null
+++ b/tests/test_fsdp2_replicated_grad_sync_gloo.py
@@ -0,0 +1,432 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""The replicated-gradient all_reduce (`fsdp2_save._sync_replicated_grads`), on a gloo CPU world of 2:
+
+* gradients are bitwise equal to the previous implementation (fresh zeros buffer per sync, flags
+ always read back) over several steps, accumulated micro-steps, fp16 / fp32 / fp64 parameters and
+ rank-dependent None gradients;
+* the flat buffer is reused per (device, dtype) and rebuilt when the parameter set or a size
+ changes, and no gradient aliases it (the next sync overwrites it);
+* no device to host read (Tensor.tolist / item) nor host scalar item assignment when every local
+ gradient is present, exactly one read when some local gradient is None;
+* end to end through reduce_unsharded_trainable_grads on a fully_shard model, for every
+ UNSLOTH_FSDP2_GA_SYNC mode (skip / defer / each / auto), replicated and sharded gradients are
+ bitwise equal to the previous implementation."""
+
+import multiprocessing
+import os
+
+if multiprocessing.parent_process() is not None or getattr(
+ multiprocessing.current_process(), "_inheriting", False
+):
+ os.environ["CUDA_VISIBLE_DEVICES"] = "" # spawned gloo ranks: CPU only, also on GPU hosts
+
+import datetime # noqa: E402
+import importlib # noqa: E402
+import sys # noqa: E402
+import time # noqa: E402
+
+import pytest # noqa: E402
+import torch # noqa: E402
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from test_fsdp2_native_units import _load_native # noqa: E402
+from test_fsdp2_native_trainer_ckpt_gloo import _free_port # noqa: E402
+
+
+def _reference_sync(
+ params,
+ dist,
+ mesh = None,
+):
+ """The implementation before the buffer reuse / host-sync removal, verbatim."""
+ from unsloth.models.fsdp2_save import mesh_all_reduce
+
+ groups = {}
+ for p in params:
+ dtype = torch.float64 if p.dtype == torch.float64 else torch.float32
+ groups.setdefault((str(p.device), dtype), []).append(p)
+ for (device, dtype), group in groups.items():
+ sizes = [p.numel() for p in group]
+ total = sum(sizes)
+ flat = torch.zeros(total + len(group), dtype = dtype, device = group[0].device)
+ offset = 0
+ for i, (p, n) in enumerate(zip(group, sizes)):
+ if p.grad is not None:
+ flat[offset : offset + n].copy_(p.grad.detach().reshape(-1))
+ flat[total + i] = 1
+ offset += n
+ world = mesh_all_reduce(flat, mesh, dist)
+ flat[:total].div_(world)
+ used = flat[total:].tolist()
+ offset = 0
+ for i, (p, n) in enumerate(zip(group, sizes)):
+ if used[i] > 0:
+ value = flat[offset : offset + n].view(p.shape)
+ if p.grad is None:
+ p.grad = value.to(p.dtype).clone()
+ else:
+ p.grad.copy_(value)
+ offset += n
+
+
+def _save():
+ _load_native()
+ return importlib.import_module("unsloth.models.fsdp2_save")
+
+
+class _Reads:
+ """Counts Tensor.tolist / Tensor.item calls and item assignment of a Python scalar (the device to
+ host reads, and host scalar copies, a CUDA run would sync on)."""
+
+ def __init__(self):
+ self.n = 0
+
+ def __enter__(self):
+ self._saved = (torch.Tensor.tolist, torch.Tensor.item, torch.Tensor.__setitem__)
+ reads, (tolist0, item0, setitem0) = self, self._saved
+
+ def tolist(t, *a, **k):
+ reads.n += 1
+ return tolist0(t, *a, **k)
+
+ def item(t, *a, **k):
+ reads.n += 1
+ return item0(t, *a, **k)
+
+ def setitem(t, key, value):
+ if not isinstance(value, torch.Tensor):
+ reads.n += 1
+ return setitem0(t, key, value)
+
+ torch.Tensor.tolist, torch.Tensor.item, torch.Tensor.__setitem__ = tolist, item, setitem
+ return self
+
+ def __exit__(self, *exc):
+ torch.Tensor.tolist, torch.Tensor.item, torch.Tensor.__setitem__ = self._saved
+
+
+def _make_params(spec):
+ return [torch.nn.Parameter(torch.zeros(shape, dtype = dtype)) for shape, dtype in spec]
+
+
+def _grads(params):
+ return [None if p.grad is None else p.grad.clone() for p in params]
+
+
+def _same(a, b):
+ if len(a) != len(b):
+ return False
+ for x, y in zip(a, b):
+ if (x is None) != (y is None):
+ return False
+ if x is not None and (x.dtype != y.dtype or x.shape != y.shape or not torch.equal(x, y)):
+ return False
+ return True
+
+
+def _case_unit(rank):
+ import torch.distributed as dist
+
+ save = _save()
+ spec = [
+ ((3, 4), torch.float32),
+ ((5,), torch.float16),
+ ((2, 2), torch.float64),
+ ((7,), torch.float32),
+ ((4,), torch.bfloat16),
+ ((6,), torch.float16),
+ ]
+ old, new = _make_params(spec), _make_params(spec)
+ buffers = {}
+ gen = torch.Generator().manual_seed(100 + rank)
+ res = {"mismatch": [], "reads_all_present": [], "reads_some_none": [], "alias": 0}
+ # step pattern: which parameters get a local gradient this micro-step on this rank; rank 1
+ # leaves some None (a skipped branch), parameter 3 is never used on any rank in steps 0-1.
+ patterns = [
+ [0, 1, 2, 4, 5] if rank == 0 else [0, 1, 2],
+ [0, 1, 2, 4, 5],
+ [0, 1, 2, 3, 4, 5],
+ [0, 1, 2, 3, 4, 5],
+ [0, 2] if rank == 1 else [0, 1, 2, 3, 4, 5],
+ ]
+ for step, used in enumerate(patterns):
+ if step in (2, 4): # optimizer step boundary: zero_grad(set_to_none = True)
+ for p in old + new:
+ p.grad = None
+ for i in used:
+ g = (torch.randn(spec[i][0], generator = gen) * 1000).to(spec[i][1])
+ for p in (old[i], new[i]):
+ if p.grad is None:
+ p.grad = g.clone()
+ else:
+ p.grad.add_(g) # accumulated micro-step
+ _reference_sync(old, dist)
+ everyone_all = all(p.grad is not None for p in new)
+ with _Reads() as reads:
+ save._sync_replicated_grads(new, dist, None, buffers)
+ (res["reads_all_present"] if everyone_all else res["reads_some_none"]).append(reads.n)
+ if not _same(_grads(old), _grads(new)):
+ res["mismatch"].append(step)
+ for p in new:
+ if p.grad is None:
+ continue
+ for _, flat in buffers.values():
+ lo, hi = flat.data_ptr(), flat.data_ptr() + flat.numel() * flat.element_size()
+ if lo <= p.grad.data_ptr() < hi:
+ res["alias"] += 1
+ res["n_buffers"] = len(buffers)
+ ids_before = {k: id(v[1]) for k, v in buffers.items()}
+
+ # Buffer reused on an unchanged set, rebuilt when the set grows or a size changes.
+ for p in old + new:
+ p.grad = torch.ones_like(p)
+ save._sync_replicated_grads(new, dist, None, buffers)
+ res["reused"] = {str(k): id(v[1]) == ids_before[k] for k, v in buffers.items()}
+ extra = _make_params([((9,), torch.float32)])
+ old2, new2 = old + _make_params([((9,), torch.float32)]), new + extra
+ for p in old2 + new2:
+ p.grad = (
+ (torch.arange(p.numel(), dtype = torch.float32) * (rank + 1)).view(p.shape).to(p.dtype)
+ )
+ _reference_sync(old2, dist)
+ save._sync_replicated_grads(new2, dist, None, buffers)
+ res["grown_same"] = _same(_grads(old2), _grads(new2))
+ res["grown_rebuilt"] = (
+ id(buffers[("cpu", torch.float32)][1]) != ids_before[("cpu", torch.float32)]
+ )
+ res["fp64_reused"] = (
+ id(buffers[("cpu", torch.float64)][1]) == ids_before[("cpu", torch.float64)]
+ )
+ # A size change at the same position (same parameter count).
+ old3, new3 = _make_params([((4,), torch.float64)]), _make_params([((4,), torch.float64)])
+ for p in old3 + new3:
+ p.grad = torch.full((4,), float(rank + 2), dtype = torch.float64)
+ before64 = id(buffers[("cpu", torch.float64)][1])
+ _reference_sync(old3, dist)
+ save._sync_replicated_grads(new3, dist, None, buffers)
+ res["resized_same"] = _same(_grads(old3), _grads(new3))
+ res["resized_rebuilt"] = id(buffers[("cpu", torch.float64)][1]) != before64
+ # No buffers dict (direct callers): still correct.
+ old4, new4 = _make_params(spec), _make_params(spec)
+ for p in old4 + new4:
+ p.grad = torch.full(p.shape, float(rank + 1)).to(p.dtype)
+ if rank == 0:
+ old4[1].grad = new4[1].grad = None
+ _reference_sync(old4, dist)
+ save._sync_replicated_grads(new4, dist)
+ res["nobuf_same"] = _same(_grads(old4), _grads(new4))
+ return res
+
+
+class _Net(torch.nn.Module):
+ """Sharded fp32 body plus replicated (ignored) parameters of mixed dtype, one used only on some
+ micro-steps of some ranks."""
+
+ def __init__(self):
+ super().__init__()
+ torch.manual_seed(0)
+ self.body = torch.nn.Sequential(torch.nn.Linear(8, 16), torch.nn.Linear(16, 8))
+ self.norm = torch.nn.Parameter(torch.randn(8))
+ self.norm16 = torch.nn.Parameter(torch.randn(8).half())
+ self.branch = torch.nn.Parameter(torch.randn(8))
+
+ def forward(self, x, use_branch):
+ h = self.body(x) * self.norm + self.norm16.float()
+ if use_branch:
+ h = h * self.branch
+ return (h * h).mean()
+
+
+def _build(save):
+ from torch.distributed.device_mesh import init_device_mesh
+ from torch.distributed.fsdp import fully_shard
+
+ model = _Net()
+ mesh = init_device_mesh("cpu", (2,))
+ for layer in model.body:
+ fully_shard(layer, mesh = mesh)
+ fully_shard(model, mesh = mesh, ignored_params = {model.norm, model.norm16, model.branch})
+ names = save.reduce_unsharded_trainable_grads(model)
+ return model, names
+
+
+GA = 4
+
+
+def _train(save, native, rank, mode, impl):
+ """3 optimizer steps x GA micro-batches, non-final ones under fsdp2_native._no_grad_sync.
+ impl: "old" = the previous code (reference sync on every backward), "nodefer" = the new sync on
+ every backward, "new" = as shipped (the sync follows FSDP2's gradient sync flag), "oracle" = no
+ sync in backward, the reference sync once after the last micro-batch."""
+ import torch.distributed as dist
+
+ os.environ["UNSLOTH_FSDP2_GA_SYNC"] = mode
+ saved = (save._sync_replicated_grads, save._fsdp_grad_sync_deferred, save.mesh_all_reduce)
+ calls = []
+
+ def counting_all_reduce(*args, **kw):
+ calls[-1] += 1
+ return saved[2](*args, **kw)
+
+ save.mesh_all_reduce = counting_all_reduce
+ if impl == "old":
+ save._sync_replicated_grads = lambda params, d, mesh = None, buffers = None: _reference_sync(
+ params, d, mesh
+ )
+ if impl in ("old", "nodefer"):
+ save._fsdp_grad_sync_deferred = lambda *a: False
+ if impl == "oracle":
+ save._sync_replicated_grads = lambda *a, **k: None
+ try:
+ model, names = _build(save)
+ rep = [p for n, p in model.named_parameters() if n in set(names)]
+ opt = torch.optim.AdamW(model.parameters(), lr = 1e-2)
+ gen = torch.Generator().manual_seed(7 + rank)
+ out = []
+ for step in range(3):
+ for micro in range(GA):
+ x = torch.randn(4, 8, generator = gen)
+ # step 0: `branch` unused everywhere; later only some micro-batches of some ranks.
+ use_branch = (micro + rank + step) % 3 == 0 and step != 0
+ calls.append(0)
+ if micro == GA - 1:
+ model(x, use_branch).backward()
+ else:
+ with native._no_grad_sync(model):
+ model(x, use_branch).backward()
+ if impl == "oracle":
+ _reference_sync(rep, dist) # counted on the last micro-batch
+ grads = {}
+ for n, p in model.named_parameters():
+ g = p.grad
+ if g is not None and hasattr(g, "full_tensor"):
+ g = g.full_tensor()
+ grads[n] = None if g is None else g.detach().clone()
+ out.append(grads)
+ opt.step()
+ opt.zero_grad(set_to_none = True)
+ return names, out, calls
+ finally:
+ save._sync_replicated_grads, save._fsdp_grad_sync_deferred, save.mesh_all_reduce = saved
+
+
+def _diff(a_steps, b_steps):
+ bad, worst = [], 0.0
+ for step, (a, b) in enumerate(zip(a_steps, b_steps)):
+ for n in a:
+ x, y = a[n], b[n]
+ if (x is None) != (y is None):
+ bad.append((step, n, "none"))
+ elif x is not None and not torch.equal(x, y):
+ bad.append((step, n))
+ worst = max(worst, (x.double() - y.double()).abs().max().item())
+ return bad, worst
+
+
+def _case_modes(rank):
+ native = _load_native()
+ save = _save()
+ res = {}
+ for mode in ("skip", "defer", "each", "auto"):
+ runs = {
+ impl: _train(save, native, rank, mode, impl)
+ for impl in ("old", "nodefer", "new", "oracle")
+ }
+ names = runs["old"][0]
+ r = {"names": sorted(names), "branch_none_step0": runs["old"][1][0]["branch"] is None}
+ r["calls"] = {impl: v[2] for impl, v in runs.items()}
+ r["nodefer_vs_old"] = _diff(runs["old"][1], runs["nodefer"][1])[0]
+ r["new_vs_old"], r["new_vs_old_max"] = _diff(runs["old"][1], runs["new"][1])
+ r["new_vs_oracle"] = _diff(runs["oracle"][1], runs["new"][1])[0]
+ res[mode] = r
+ return res
+
+
+def _worker(rank, world, port, out, case):
+ os.environ.update(
+ MASTER_ADDR = "127.0.0.1",
+ MASTER_PORT = str(port),
+ RANK = str(rank),
+ WORLD_SIZE = str(world),
+ LOCAL_RANK = str(rank),
+ LOCAL_WORLD_SIZE = str(world),
+ UNSLOTH_FSDP2 = "1",
+ )
+ import torch.distributed as dist
+
+ dist.init_process_group(
+ "gloo", rank = rank, world_size = world, timeout = datetime.timedelta(seconds = 60)
+ )
+ try:
+ res = _case_unit(rank) if case == "unit" else _case_modes(rank)
+ out.put((rank, res))
+ except Exception as exc:
+ import traceback
+ out.put((rank, {"crash": f"{type(exc).__name__}: {exc}\n{traceback.format_exc()}"}))
+ finally:
+ try:
+ dist.destroy_process_group()
+ except Exception:
+ pass
+
+
+def _run(case, timeout = 240):
+ import torch.multiprocessing as mp
+
+ ctx = mp.get_context("spawn")
+ out = ctx.Queue()
+ procs = mp.start_processes(
+ _worker, args = (2, _free_port(), out, case), nprocs = 2, join = False, start_method = "spawn"
+ )
+ deadline = time.time() + timeout
+ while not procs.join(timeout = 5):
+ if time.time() > deadline:
+ for p in procs.processes:
+ p.kill()
+ pytest.fail(f"{case}: the ranks did not finish within {timeout} s (hung collective)")
+ results = {}
+ while not out.empty():
+ rank, r = out.get(timeout = 5)
+ results[rank] = r
+ assert sorted(results) == [0, 1], results
+ for rank, r in results.items():
+ print("REPL_SYNC_RESULT", case, rank, r)
+ assert "crash" not in r, r["crash"]
+ return results
+
+
+def test_replicated_sync_bitwise_buffer_reuse_and_no_host_read():
+ for rank, r in _run("unit").items():
+ assert r["mismatch"] == [], (rank, r)
+ assert r["alias"] == 0, (rank, r)
+ # Steps 2 and 3 (and 4 on rank 0) have every local gradient; the rest leave some None.
+ assert r["reads_all_present"] == [0] * (3 if rank == 0 else 2), (rank, r)
+ assert r["reads_some_none"] == [1] * (2 if rank == 0 else 3), (rank, r)
+ assert r["n_buffers"] == 2 and all(r["reused"].values()), (rank, r)
+ assert r["grown_same"] and r["grown_rebuilt"] and r["fp64_reused"], (rank, r)
+ assert r["resized_same"] and r["resized_rebuilt"], (rank, r)
+ assert r["nobuf_same"], (rank, r)
+
+
+def test_replicated_sync_follows_the_ga_sync_flag():
+ for rank, r in _run("modes", timeout = 400).items():
+ for mode, m in r.items():
+ assert m["names"] == ["branch", "norm", "norm16"], (rank, mode, m)
+ assert m["branch_none_step0"], (rank, mode, m)
+ # The new sync on every backward is bitwise the previous code (buffer / host-read change).
+ assert m["nodefer_vs_old"] == [], (rank, mode, m)
+ # Replicated all_reduces per micro-batch (GA 4, 3 steps): the previous code syncs every
+ # backward; shipped code only on the synced one, unless the mode syncs every micro-batch.
+ assert m["calls"]["old"] == [1] * (3 * GA), (rank, mode, m)
+ deferred = mode != "each" # auto without a fit layout = skip
+ want = ([0] * (GA - 1) + [1]) * 3 if deferred else [1] * (3 * GA)
+ assert m["calls"]["new"] == want, (rank, mode, m)
+ assert m["calls"]["oracle"] == ([0] * (GA - 1) + [1]) * 3, (rank, mode, m)
+ if deferred:
+ # Bitwise the accumulated local sum averaged once over the ranks.
+ assert m["new_vs_oracle"] == [], (rank, mode, m)
+ assert m["new_vs_old_max"] < 1e-2, (rank, mode, m)
+ else:
+ assert m["new_vs_old"] == [], (rank, mode, m)
diff --git a/unsloth/models/fsdp2_save.py b/unsloth/models/fsdp2_save.py
index f18cd05a7e0..617528d4089 100644
--- a/unsloth/models/fsdp2_save.py
+++ b/unsloth/models/fsdp2_save.py
@@ -796,14 +796,13 @@ def _sync_replicated_grads(
params,
dist,
mesh = None,
+ buffers = None,
):
- """One all_reduce per (device, reduce dtype) over the fixed parameter list, in its order, issued
- by every rank: a gradient this rank did not produce counts as zeros, and a per-parameter
- "has a gradient" flag rides along so a parameter no rank used keeps grad None (as with DDP).
- Sums in float32 (float64 stays float64), as FSDP2's reduce_dtype does: two fp16 gradients of
- 40000 overflow an fp16 sum although their mean is finite. Linear, so averaging the running
- accumulation after each micro-step equals averaging the accumulated sum once. Over the ranks of
- `mesh`, the model's own (mesh_all_reduce), not the whole world."""
+ """Average gradients over `mesh` (DDP-style), one all_reduce per (device, reduce dtype), entered
+ by every rank: a missing local gradient counts as zeros, and a "has a gradient" flag keeps grad
+ None for a parameter no rank used. Sums in float32 (fp16 sums overflow). The summed flags are
+ read (a host sync) only when some local gradient is None. `buffers` caches one flat buffer per
+ (device, dtype); gradients never alias it."""
groups = {}
for p in params:
dtype = torch.float64 if p.dtype == torch.float64 else torch.float32
@@ -811,35 +810,88 @@ def _sync_replicated_grads(
for (device, dtype), group in groups.items():
sizes = [p.numel() for p in group]
total = sum(sizes)
- flat = torch.zeros(total + len(group), dtype = dtype, device = group[0].device)
+ key = tuple((id(p), n) for p, n in zip(group, sizes))
+ cached = buffers.get((device, dtype)) if buffers is not None else None
+ if cached is not None and cached[0] == key:
+ flat = cached[1]
+ else:
+ flat = torch.empty(total + len(group), dtype = dtype, device = group[0].device)
+ if buffers is not None:
+ buffers[(device, dtype)] = (key, flat)
+ grads = [p.grad for p in group]
+ local_all = all(g is not None for g in grads)
+ flags = flat[total:]
+ if local_all:
+ flags.fill_(1)
+ else:
+ flags.zero_()
offset = 0
- for i, (p, n) in enumerate(zip(group, sizes)):
- if p.grad is not None:
- flat[offset : offset + n].copy_(p.grad.detach().reshape(-1))
- flat[total + i] = 1
+ for i, (g, n) in enumerate(zip(grads, sizes)):
+ if g is not None:
+ flat[offset : offset + n].copy_(g.detach().reshape(-1))
+ if not local_all:
+ # Not flags[i] = 1: that is a host sync on CUDA.
+ flags[i].fill_(1)
+ else:
+ flat[offset : offset + n].zero_()
offset += n
world = mesh_all_reduce(flat, mesh, dist)
flat[:total].div_(world)
- used = flat[total:].tolist() # one device sync, not one per parameter
+ used = None if local_all else flags.tolist()
offset = 0
- for i, (p, n) in enumerate(zip(group, sizes)):
- if used[i] > 0:
+ for i, (p, g, n) in enumerate(zip(group, grads, sizes)):
+ if g is not None or used[i] > 0:
value = flat[offset : offset + n].view(p.shape)
if p.grad is None:
- p.grad = value.to(p.dtype).clone()
+ p.grad = value.to(p.dtype, copy = True)
else:
p.grad.copy_(value)
offset += n
+def _fsdp_grad_sync_deferred(model, cache):
+ """True when every FSDP2 param group has set_requires_gradient_sync(False), i.e. a non-final
+ micro-batch. False when no group is found (sync every backward)."""
+ states = cache.get("fsdp_states")
+ if states is None:
+ states = []
+ try:
+ from torch.distributed.fsdp import FSDPModule
+ except Exception:
+ try:
+ from torch.distributed._composable.fsdp import FSDPModule
+ except Exception:
+ FSDPModule = None
+ if FSDPModule is not None:
+ for module in model.modules():
+ if isinstance(module, FSDPModule):
+ try:
+ states.append(module._get_fsdp_state())
+ except Exception:
+ pass
+ cache["fsdp_states"] = states
+ found = False
+ for st in states:
+ groups = getattr(st, "_fsdp_param_groups", None) # a list on newer torch (singular on 2.10)
+ if groups is None:
+ group = getattr(st, "_fsdp_param_group", None)
+ groups = [group] if group is not None else []
+ for group in groups:
+ reduce = getattr(group, "reduce_grads", None)
+ if reduce is None:
+ return False
+ if reduce:
+ return False
+ found = True
+ return found
+
+
def reduce_unsharded_trainable_grads(model):
- """Trainable parameters FSDP2 does not manage (e.g. in `ignored_modules`) get no gradient
- reduction, so every rank would step on its own gradient and the replicas drift apart. Average
- their gradients over the model's mesh ranks like DDP, once per backward, at its end, on every rank: a per-parameter
- hook would make a rank that did not use the parameter (a branch, a routed MoE expert) skip the
- collective the others enter, and hang. The sync is queued as an autograd engine callback from a
- hook on `model`'s forward outputs (and, as a fallback, on each of these parameters), so it runs
- after all gradients are accumulated and before the optimizer step / clip. Returns the names."""
+ """Average gradients of trainable params FSDP2 does not manage (e.g. `ignored_modules`) over the
+ mesh, else replicas drift. Runs once per backward as an autograd engine callback on every rank;
+ a per-parameter hook would hang when a rank skips the parameter (branch, routed MoE expert).
+ Skipped while FSDP2 grad sync is off (GA); the first synced backward averages the accumulated
+ sum. Returns the names."""
dist = _dist()
DTensor = _dtensor_class()
if dist is None or DTensor is None:
@@ -851,10 +903,14 @@ def reduce_unsharded_trainable_grads(model):
seen = set()
params = model.__dict__.setdefault("_unsloth_replicated_grad_params", [])
state = model.__dict__.setdefault("_unsloth_replicated_grad_state", {"queued": False})
+ buffers = state.setdefault("buffers", {})
def _sync():
state["queued"] = False
- _sync_replicated_grads(params, dist, mesh)
+ # Every rank enters the same branch: the GA flag is set alike on all ranks.
+ if _fsdp_grad_sync_deferred(model, state):
+ return
+ _sync_replicated_grads(params, dist, mesh, buffers)
def _queue(*_):
if not state["queued"]: