Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

FSDP2: sum fp16 replicated gradients over unsynced micro-batches in float32 (#16173)

Daniel Hanff3f063a3a80
diff --git a/tests/test_fsdp2_replicated_grad_sync_gloo.py b/tests/test_fsdp2_replicated_grad_sync_gloo.py
index ed81c787830..2142a3a3a47 100644
--- a/tests/test_fsdp2_replicated_grad_sync_gloo.py
+++ b/tests/test_fsdp2_replicated_grad_sync_gloo.py
@@ -1,18 +1,6 @@
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
-"""The replicated-gradient all_reduce (`fsdp2_save._sync_replicated_grads`), on a gloo CPU world of 2:
-
-* gradients are bitwise equal to the previous implementation (fresh zeros buffer per sync, flags
- always read back) over several steps, accumulated micro-steps, fp16 / fp32 / fp64 parameters and
- rank-dependent None gradients;
-* the flat buffer is reused per (device, dtype) and rebuilt when the parameter set or a size
- changes, and no gradient aliases it (the next sync overwrites it);
-* no device to host read (Tensor.tolist / item) nor host scalar item assignment when every local
- gradient is present, exactly one read when some local gradient is None;
-* end to end through reduce_unsharded_trainable_grads on a fully_shard model, for every
- UNSLOTH_FSDP2_GA_SYNC mode (skip / defer / each / auto), replicated and sharded gradients are
- bitwise equal to the previous implementation."""
import multiprocessing
import os
@@ -20,7 +8,7 @@ import os
if multiprocessing.parent_process() is not None or getattr(
multiprocessing.current_process(), "_inheriting", False
):
- os.environ["CUDA_VISIBLE_DEVICES"] = "" # spawned gloo ranks: CPU only, also on GPU hosts
+ os.environ["CUDA_VISIBLE_DEVICES"] = "" # Gloo ranks must stay on CPU even on GPU hosts.
import datetime # noqa: E402
import importlib # noqa: E402
@@ -40,7 +28,7 @@ def _reference_sync(
dist,
mesh = None,
):
- """The implementation before the buffer reuse / host-sync removal, verbatim."""
+ """Pre-buffer-reuse implementation, retained as an independent reference."""
from unsloth.models.fsdp2_save import mesh_all_reduce
groups = {}
@@ -77,8 +65,7 @@ def _save():
class _Reads:
- """Counts Tensor.tolist / Tensor.item calls and item assignment of a Python scalar (the device to
- host reads, and host scalar copies, a CUDA run would sync on)."""
+ """Count device-to-host reads and scalar assignments that synchronize CUDA."""
def __init__(self):
self.n = 0
@@ -142,8 +129,7 @@ def _case_unit(rank):
buffers = {}
gen = torch.Generator().manual_seed(100 + rank)
res = {"mismatch": [], "reads_all_present": [], "reads_some_none": [], "alias": 0}
- # step pattern: which parameters get a local gradient this micro-step on this rank; rank 1
- # leaves some None (a skipped branch), parameter 3 is never used on any rank in steps 0-1.
+
patterns = [
[0, 1, 2, 4, 5] if rank == 0 else [0, 1, 2],
[0, 1, 2, 4, 5],
@@ -152,7 +138,7 @@ def _case_unit(rank):
[0, 2] if rank == 1 else [0, 1, 2, 3, 4, 5],
]
for step, used in enumerate(patterns):
- if step in (2, 4): # optimizer step boundary: zero_grad(set_to_none = True)
+ if step in (2, 4):
for p in old + new:
p.grad = None
for i in used:
@@ -161,7 +147,7 @@ def _case_unit(rank):
if p.grad is None:
p.grad = g.clone()
else:
- p.grad.add_(g) # accumulated micro-step
+ p.grad.add_(g)
_reference_sync(old, dist)
everyone_all = all(p.grad is not None for p in new)
with _Reads() as reads:
@@ -179,7 +165,6 @@ def _case_unit(rank):
res["n_buffers"] = len(buffers)
ids_before = {k: id(v[1]) for k, v in buffers.items()}
- # Buffer reused on an unchanged set, rebuilt when the set grows or a size changes.
for p in old + new:
p.grad = torch.ones_like(p)
save._sync_replicated_grads(new, dist, None, buffers)
@@ -199,7 +184,7 @@ def _case_unit(rank):
res["fp64_reused"] = (
id(buffers[("cpu", torch.float64)][1]) == ids_before[("cpu", torch.float64)]
)
- # A size change at the same position (same parameter count).
+
old3, new3 = _make_params([((4,), torch.float64)]), _make_params([((4,), torch.float64)])
for p in old3 + new3:
p.grad = torch.full((4,), float(rank + 2), dtype = torch.float64)
@@ -208,7 +193,7 @@ def _case_unit(rank):
save._sync_replicated_grads(new3, dist, None, buffers)
res["resized_same"] = _same(_grads(old3), _grads(new3))
res["resized_rebuilt"] = id(buffers[("cpu", torch.float64)][1]) != before64
- # No buffers dict (direct callers): still correct.
+
old4, new4 = _make_params(spec), _make_params(spec)
for p in old4 + new4:
p.grad = torch.full(p.shape, float(rank + 1)).to(p.dtype)
@@ -221,9 +206,6 @@ def _case_unit(rank):
class _Net(torch.nn.Module):
- """Sharded fp32 body plus replicated (ignored) parameters of mixed dtype, one used only on some
- micro-steps of some ranks."""
-
def __init__(self):
super().__init__()
torch.manual_seed(0)
@@ -256,10 +238,6 @@ GA = 4
def _train(save, native, rank, mode, impl):
- """3 optimizer steps x GA micro-batches, non-final ones under fsdp2_native._no_grad_sync.
- impl: "old" = the previous code (reference sync on every backward), "nodefer" = the new sync on
- every backward, "new" = as shipped (the sync follows FSDP2's gradient sync flag), "oracle" = no
- sync in backward, the reference sync once after the last micro-batch."""
import torch.distributed as dist
os.environ["UNSLOTH_FSDP2_GA_SYNC"] = mode
@@ -272,10 +250,10 @@ def _train(save, native, rank, mode, impl):
save.mesh_all_reduce = counting_all_reduce
if impl == "old":
- save._sync_replicated_grads = lambda params, d, mesh = None, buffers = None: _reference_sync(
+ save._sync_replicated_grads = lambda params, d, mesh = None, *_: _reference_sync(
params, d, mesh
)
- if impl in ("old", "nodefer"):
+ if impl in ("old", "nodefer", "oracle"):
save._fsdp_grad_sync_deferred = lambda *a: False
if impl == "oracle":
save._sync_replicated_grads = lambda *a, **k: None
@@ -288,7 +266,7 @@ def _train(save, native, rank, mode, impl):
for step in range(3):
for micro in range(GA):
x = torch.randn(4, 8, generator = gen)
- # step 0: `branch` unused everywhere; later only some micro-batches of some ranks.
+
use_branch = (micro + rank + step) % 3 == 0 and step != 0
calls.append(0)
if micro == GA - 1:
@@ -297,7 +275,7 @@ def _train(save, native, rank, mode, impl):
with native._no_grad_sync(model):
model(x, use_branch).backward()
if impl == "oracle":
- _reference_sync(rep, dist) # counted on the last micro-batch
+ _reference_sync(rep, dist)
grads = {}
for n, p in model.named_parameters():
g = p.grad
@@ -312,13 +290,23 @@ def _train(save, native, rank, mode, impl):
save._sync_replicated_grads, save._fsdp_grad_sync_deferred, save.mesh_all_reduce = saved
-def _diff(a_steps, b_steps):
+def _diff(
+ a_steps,
+ b_steps,
+ skip = (),
+):
bad, worst = [], 0.0
for step, (a, b) in enumerate(zip(a_steps, b_steps)):
for n in a:
+ if n in skip:
+ continue
x, y = a[n], b[n]
if (x is None) != (y is None):
bad.append((step, n, "none"))
+ worst = float("inf")
+ elif x is not None and (not torch.isfinite(x).all() or not torch.isfinite(y).all()):
+ bad.append((step, n, "nonfinite"))
+ worst = float("inf")
elif x is not None and not torch.equal(x, y):
bad.append((step, n))
worst = max(worst, (x.double() - y.double()).abs().max().item())
@@ -339,11 +327,106 @@ def _case_modes(rank):
r["calls"] = {impl: v[2] for impl, v in runs.items()}
r["nodefer_vs_old"] = _diff(runs["old"][1], runs["nodefer"][1])[0]
r["new_vs_old"], r["new_vs_old_max"] = _diff(runs["old"][1], runs["new"][1])
- r["new_vs_oracle"] = _diff(runs["oracle"][1], runs["new"][1])[0]
+ # Wider fp16 sums change later updates, so only unaffected first-step gradients are bitwise.
+
+ r["new_vs_oracle"] = _diff(runs["oracle"][1][:1], runs["new"][1][:1], skip = ("norm16",))[0]
+ r["new_vs_oracle_max"] = _diff(runs["oracle"][1], runs["new"][1])[1]
res[mode] = r
return res
+class _Fp16Rep(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.body = torch.nn.Linear(4, 4)
+ self.rep = torch.nn.Parameter(torch.zeros(4, dtype = torch.float16))
+ self.fail_forward = False
+
+ def forward(
+ self,
+ x,
+ scale,
+ use_rep = True,
+ ):
+ loss = self.body(x).mean()
+ if self.fail_forward:
+ raise RuntimeError("injected forward failure")
+ return loss + (self.rep.float() * scale).sum() if use_rep else loss
+
+
+def _case_fp16_overflow(rank):
+ """Local 80000 overflows fp16 before averaging with the other rank to 60000."""
+ from torch.distributed.device_mesh import init_device_mesh
+ from torch.distributed.fsdp import fully_shard
+
+ native = _load_native()
+ save = _save()
+ os.environ["UNSLOTH_FSDP2_GA_SYNC"] = "defer"
+ torch.manual_seed(0)
+ model = _Fp16Rep()
+ mesh = init_device_mesh("cpu", (2,))
+ fully_shard(model.body, mesh = mesh)
+ fully_shard(model, mesh = mesh, ignored_params = {model.rep})
+ names = save.reduce_unsharded_trainable_grads(model)
+ scale = 40000.0 if rank == 0 else 20000.0
+ x = torch.randn(2, 4)
+ with native._no_grad_sync(model):
+ model(x, scale).backward()
+ model(x, scale).backward()
+ return {"names": names, "grad": model.rep.grad.float().tolist(), "file": save.__file__}
+
+
+def _case_fp16_edge(rank, mode):
+ from torch.distributed.device_mesh import init_device_mesh
+ from torch.distributed.fsdp import fully_shard
+
+ native = _load_native()
+ save = _save()
+ os.environ["UNSLOTH_FSDP2_GA_SYNC"] = "defer"
+ torch.manual_seed(0)
+ model = _Fp16Rep()
+ if mode == "wide":
+ model.rep.grad_dtype = torch.float32
+ mesh = init_device_mesh("cpu", (2,))
+ fully_shard(model.body, mesh = mesh)
+ fully_shard(model, mesh = mesh, ignored_params = {model.rep})
+ save.reduce_unsharded_trainable_grads(model)
+ owner = model
+ if mode == "outer_recovery":
+
+ class Outer(torch.nn.Module):
+ def __init__(self, inner):
+ super().__init__()
+ self.inner = inner
+
+ def forward(self, *args, **kwargs):
+ return self.inner(*args, **kwargs)
+
+ owner = Outer(model)
+ fully_shard(owner, mesh = mesh)
+ save.reduce_unsharded_trainable_grads(owner)
+ x = torch.randn(2, 4)
+ with native._no_grad_sync(owner):
+ owner(x, 4.0).backward()
+ if mode == "wide":
+ owner(x, 2.0, use_rep = False).backward()
+ else:
+ model.fail_forward = True
+ with pytest.raises(RuntimeError, match = "injected forward failure"):
+ owner(x, 9.0)
+ owner.reset_iter_state()
+ owner.zero_grad(set_to_none = True)
+ model.fail_forward = False
+ owner(x, 2.0).backward()
+ state = model.__dict__["_unsloth_replicated_grad_state"]
+ return {
+ "grad": model.rep.grad.tolist(),
+ "dtype": str(model.rep.grad.dtype),
+ "pending": bool(state.get("pending")),
+ "queued": state["queued"],
+ }
+
+
def _worker(rank, world, port, out, case):
os.environ.update(
MASTER_ADDR = "127.0.0.1",
@@ -360,7 +443,8 @@ def _worker(rank, world, port, out, case):
"gloo", rank = rank, world_size = world, timeout = datetime.timedelta(seconds = 60)
)
try:
- res = _case_unit(rank) if case == "unit" else _case_modes(rank)
+ cases = {"unit": _case_unit, "modes": _case_modes, "fp16_overflow": _case_fp16_overflow}
+ res = _case_fp16_edge(rank, case[5:]) if case.startswith("edge_") else cases[case](rank)
out.put((rank, res))
except Exception as exc:
import traceback
@@ -401,7 +485,7 @@ def test_replicated_sync_bitwise_buffer_reuse_and_no_host_read():
for rank, r in _run("unit").items():
assert r["mismatch"] == [], (rank, r)
assert r["alias"] == 0, (rank, r)
- # Steps 2 and 3 (and 4 on rank 0) have every local gradient; the rest leave some None.
+
assert r["reads_all_present"] == [0] * (3 if rank == 0 else 2), (rank, r)
assert r["reads_some_none"] == [1] * (2 if rank == 0 else 3), (rank, r)
assert r["n_buffers"] == 2 and all(r["reused"].values()), (rank, r)
@@ -415,18 +499,46 @@ def test_replicated_sync_follows_the_ga_sync_flag():
for mode, m in r.items():
assert m["names"] == ["branch", "norm", "norm16"], (rank, mode, m)
assert m["branch_none_step0"], (rank, mode, m)
- # The new sync on every backward is bitwise the previous code (buffer / host-read change).
+
assert m["nodefer_vs_old"] == [], (rank, mode, m)
- # Replicated all_reduces per micro-batch (GA 4, 3 steps): the previous code syncs every
- # backward; shipped code only on the synced one, unless the mode syncs every micro-batch.
+
assert m["calls"]["old"] == [1] * (3 * GA), (rank, mode, m)
- deferred = mode != "each" # auto without a fit layout = skip
+ deferred = mode != "each" # Auto without a fit layout defers synchronization.
want = ([0] * (GA - 1) + [1]) * 3 if deferred else [1] * (3 * GA)
assert m["calls"]["new"] == want, (rank, mode, m)
assert m["calls"]["oracle"] == ([0] * (GA - 1) + [1]) * 3, (rank, mode, m)
if deferred:
- # Bitwise the accumulated local sum averaged once over the ranks.
assert m["new_vs_oracle"] == [], (rank, mode, m)
+ assert m["new_vs_oracle_max"] < 1e-2, (rank, mode, m)
assert m["new_vs_old_max"] < 1e-2, (rank, mode, m)
else:
assert m["new_vs_old"] == [], (rank, mode, m)
+
+
+def test_fp16_replica_local_sum_over_micro_batches_does_not_overflow():
+ for rank, r in _run("fp16_overflow").items():
+ assert r["names"] == ["rep"], (rank, r)
+ assert r["grad"] == [60000.0] * 4, (rank, r)
+
+
[email protected]("mode", ["wide", "recovery", "outer_recovery"])
+def test_fp16_replica_gradient_dtype_and_recovery(mode):
+ FSDPModule = _save()._fsdp_module_class()
+ if mode == "wide" and not hasattr(torch.Tensor, "grad_dtype"):
+ pytest.skip("torch has no explicit gradient dtype")
+ if mode != "wide" and not hasattr(FSDPModule, "reset_iter_state"):
+ pytest.skip("torch has no iteration recovery API")
+ for rank, r in _run("edge_" + mode).items():
+ assert r["grad"] == [4.0 if mode == "wide" else 2.0] * 4, (rank, mode, r)
+ assert r["dtype"] == ("torch.float32" if mode == "wide" else "torch.float16"), r
+ assert not r["pending"] and not r["queued"], r
+
+
+ "left,right", [(1.0, None), (None, 1.0), (1.0, float("nan")), (float("inf"), float("inf"))]
+)
+def test_gradient_comparison_rejects_missing_and_nonfinite(left, right):
+ a = None if left is None else torch.tensor([left])
+ b = None if right is None else torch.tensor([right])
+ bad, worst = _diff([{"rep": a}], [{"rep": b}])
+ assert bad and worst == float("inf")
diff --git a/tests/test_pp_fsdp2_gloo.py b/tests/test_pp_fsdp2_gloo.py
index c9f857f2b55..d516a37a9fd 100644
--- a/tests/test_pp_fsdp2_gloo.py
+++ b/tests/test_pp_fsdp2_gloo.py
@@ -1,16 +1,15 @@
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
-"""Pipeline parallel composed with native FSDP2 data parallel and / or tensor parallel on a gloo CPU world
-(pp 2 x dp 2, pp 2 x tp 2, pp 2 x dp 2 x tp 2) vs the same model in one process on the global batch, fp32."""
+import functools
import multiprocessing
import os
if multiprocessing.parent_process() is not None or getattr(
multiprocessing.current_process(), "_inheriting", False
):
- os.environ["CUDA_VISIBLE_DEVICES"] = "" # spawned gloo ranks: CPU only, also on GPU hosts
+ os.environ["CUDA_VISIBLE_DEVICES"] = "" # Gloo ranks must stay on CPU even on GPU hosts.
import sys # noqa: E402
import tempfile # noqa: E402
@@ -25,23 +24,21 @@ from test_tp_native_oracle import tpn # noqa: E402
LLAMA = "hf-internal-testing/tiny-random-LlamaForCausalLM"
QWEN3 = "trl-internal-testing/tiny-Qwen3ForCausalLM"
GEMMA3 = "hf-internal-testing/tiny-random-Gemma3ForCausalLM"
-# Overrides so the head counts split over tp = 2 (as the TP suite).
+
ARCHS = {
"llama": (LLAMA, {}),
"qwen3": (QWEN3, {}),
"gemma3": (GEMMA3, {"num_attention_heads": 4, "num_key_value_heads": 2}),
}
+_LOCAL_CONFIG = False
N_LAYERS = 6
STEPS = 3
-# Step-0 grad norm tolerance vs the one-process reference. Gemma3's own fp32 grad norm sits 1.0e-4 from fp64 on one
-# process (Llama 7.8e-7), so any change of summation order (TP splits, CPU ISA) moves it by that much.
+# Gemma3 reduction-order noise needs a wider bound than Llama/Qwen3.
+
STEP0_NORM_REL = {"llama": 1e-5, "qwen3": 1e-5, "gemma3": 3e-4}
-# Step-0 grad tolerances: per element relative to the tensor's largest entry, and relative L2 over all
-# grads. Gemma3's fp32 reordering noise under tensor parallel depends on the CPU's kernels: worst
-# element / relative L2 2.8e-5 / 2.4e-5 with this host's AVX-512 MKL, 1.6e-4 / 1.4e-4 with MKL and
-# oneDNN held to AVX2, and 2e-4 per element on a CI runner, against 2.6e-6 for llama and 5.8e-6 for
-# qwen3 on the same AVX2 run (pp x dp gemma3 stays near 4e-7). llama and qwen3 keep the strict bounds
-# on the same tensor parallel code; a gemma3-specific fault (a wrong shard, a missed reduction) is O(1).
+# CPU ISA and TP summation order move near-zero Gemma3 gradients; keep other models strict.
+
+
GRAD_ATOL_REL = {"llama": 2e-5, "qwen3": 2e-5, "gemma3": 1e-3}
GRAD_L2_REL = {"llama": 5e-5, "qwen3": 5e-5, "gemma3": 5e-4}
@@ -62,7 +59,7 @@ def build(
from peft import LoraConfig, get_peft_model
from transformers import AutoConfig, AutoModelForCausalLM
- probe = AutoConfig.from_pretrained(repo)
+ probe = AutoConfig.from_pretrained(repo, local_files_only = _LOCAL_CONFIG)
values = probe.to_dict()
values["num_hidden_layers"] = N_LAYERS
values.pop("layer_types", None)
@@ -97,9 +94,7 @@ def global_batch(
g = torch.Generator().manual_seed(seed)
ids = torch.randint(0, vocab, (bsz, seq), generator = g)
labels = ids.clone()
- labels[
- 0, :3
- ] = -100 # uneven token counts per dp replica / micro-batch: the global count matters
+ labels[0, :3] = -100 # Uneven token counts exercise global normalization.
labels[-1, -2:] = -100
labels[1, -1] = -100
return ids, labels
@@ -143,7 +138,6 @@ def _trainable_param_groups(model, trainable = True):
def _count_reduces():
- """Wrap FSDP2's gradient reduce (one call per param group per synced backward)."""
import torch.distributed.fsdp._fully_shard._fsdp_param_group as pg
calls = [0]
@@ -176,8 +170,7 @@ def _case_compose(
fit_layout = None,
rows = 4,
):
- """apply_pipeline_parallel over mesh['pp'], then apply_tensor_parallel over mesh['tp'] and / or shard_model
- over mesh['dp']. Each dp replica trains on its rows of the global batch."""
+ """Split over pp before applying orthogonal tp/dp meshes; each dp rank gets its batch slice."""
import torch.distributed as dist
from safetensors.torch import load_file
from torch.distributed.device_mesh import init_device_mesh
@@ -197,30 +190,33 @@ def _case_compose(
model = build(repo, overrides)
pp = ppn.apply_pipeline_parallel(model, mesh["pp"], n_microbatches = n_mb, schedule = schedule)
refusals = {}
- if (
- world != dims["pp"]
- ): # a pp mesh smaller than the world: the other dimensions must be applied first
+ if world != dims["pp"]:
refusals["step_before_all_dims"] = _raised(lambda: pp.step(ids[mine], labels[mine]))
if "tp" in dims:
tpn.apply_tensor_parallel(model, mesh["tp"])
if "dp" in dims:
refusals["shard_model_pp_mesh"] = _raised(
lambda: native.shard_model(
- model, native.FSDP2Config(param_dtype = torch.float32, mesh = mesh["pp"])
+ model, native.FSDP2Config(device = "cpu", param_dtype = torch.float32, mesh = mesh["pp"])
)
)
refusals["shard_model_no_mesh"] = _raised(
- lambda: native.shard_model(model, native.FSDP2Config(param_dtype = torch.float32))
+ lambda: native.shard_model(
+ model, native.FSDP2Config(device = "cpu", param_dtype = torch.float32)
+ )
)
refusals["shard_model_inner_level"] = _raised(
lambda: native.shard_model(
- model.base_model, native.FSDP2Config(param_dtype = torch.float32, mesh = mesh["dp"])
+ model.base_model,
+ native.FSDP2Config(device = "cpu", param_dtype = torch.float32, mesh = mesh["dp"]),
)
)
refusals["shard_model_cpu_offload"] = _raised(
lambda: native.shard_model(
model,
- native.FSDP2Config(param_dtype = torch.float32, mesh = mesh["dp"], cpu_offload = True),
+ native.FSDP2Config(
+ device = "cpu", param_dtype = torch.float32, mesh = mesh["dp"], cpu_offload = True
+ ),
)
)
refusals["fsdp_after_refusals"] = sum(
@@ -228,7 +224,9 @@ def _case_compose(
)
native.shard_model(
model,
- native.FSDP2Config(param_dtype = torch.float32, mesh = mesh["dp"], fit_layout = fit_layout),
+ native.FSDP2Config(
+ device = "cpu", param_dtype = torch.float32, mesh = mesh["dp"], fit_layout = fit_layout
+ ),
)
from torch.distributed.fsdp import FSDPModule
@@ -267,7 +265,6 @@ def _case_compose(
"owned": {n for n, p in model.named_parameters() if p.requires_grad},
"reduces": reduces,
"groups": len(groups),
- # torchtitan's PP default: no unit reshards after forward (each micro-batch would all-gather again).
"reshard_after_forward": [
g.post_forward_mesh_info is not None
for g in _trainable_param_groups(model, trainable = False)
@@ -286,7 +283,6 @@ def _case_compose(
def _case_scaler(rank, world, tmp):
- """pp 2 x dp 2: an inf on one rank's (last stage, dp replica 1) gradient skips the step on every rank."""
from torch.distributed.device_mesh import init_device_mesh
native = _native()
@@ -295,15 +291,17 @@ def _case_scaler(rank, world, tmp):
ids, labels = global_batch(model, 8)
dp_rank = mesh["dp"].get_local_rank()
pp = ppn.apply_pipeline_parallel(model, mesh["pp"], n_microbatches = 2)
- native.shard_model(model, native.FSDP2Config(param_dtype = torch.float32, mesh = mesh["dp"]))
+ native.shard_model(
+ model, native.FSDP2Config(device = "cpu", param_dtype = torch.float32, mesh = mesh["dp"])
+ )
scaler = torch.amp.GradScaler("cpu", init_scale = 1024.0)
opt = torch.optim.SGD(pp.parameters(), lr = 1.0)
pp.step(
ids[4 * dp_rank : 4 * dp_rank + 4], labels[4 * dp_rank : 4 * dp_rank + 4], scaler = scaler
)
if rank == world - 1:
- # One rank (last stage, dp replica 1) finds an inf in its unscale (as a replicated gradient would);
- # set on its found_inf directly: an inf in a DTensor shard is already reduced over dp by GradScaler.
+ # Inject found_inf directly: DTensor shard infinities already propagate over dp.
+
unscale = scaler.unscale_
def unscale_with_inf(optimizer):
@@ -324,7 +322,6 @@ def _case_scaler(rank, world, tmp):
def _case_order_refusals(rank, world, tmp):
- """TP or FSDP2 before the pipeline split, and a non-composable pp mesh, are refused before any change."""
from torch.distributed.device_mesh import init_device_mesh
native = _native()
@@ -336,15 +333,19 @@ def _case_order_refusals(rank, world, tmp):
out["tp_first"] = _raised(lambda: ppn.apply_pipeline_parallel(model, mesh["pp"]))
out["tp_first_unchanged"] = not ppn.is_pipeline_parallel(model) and layers == N_LAYERS
model = build(LLAMA)
- native.shard_model(model, native.FSDP2Config(param_dtype = torch.float32, mesh = mesh["dp"]))
+ native.shard_model(
+ model, native.FSDP2Config(device = "cpu", param_dtype = torch.float32, mesh = mesh["dp"])
+ )
out["fsdp_first"] = _raised(lambda: ppn.apply_pipeline_parallel(model, mesh["pp"]))
out["fsdp_first_unchanged"] = not ppn.is_pipeline_parallel(model)
- # pp mesh over all 4 ranks = world: a plain v1 split; a 2-D mesh is refused.
+
model = build(LLAMA)
out["mesh_2d"] = _raised(lambda: ppn.apply_pipeline_parallel(model, mesh))
- # pp x dp: TP after shard_model is refused.
+
pp = ppn.apply_pipeline_parallel(model, mesh["pp"])
- native.shard_model(model, native.FSDP2Config(param_dtype = torch.float32, mesh = mesh["dp"]))
+ native.shard_model(
+ model, native.FSDP2Config(device = "cpu", param_dtype = torch.float32, mesh = mesh["dp"])
+ )
out["tp_after_fsdp"] = _raised(lambda: tpn.apply_tensor_parallel(model, mesh["dp"]))
out["shard_twice_is_noop"] = native.shard_model(model) is model
out["composed"] = pp.composed
@@ -352,7 +353,6 @@ def _case_order_refusals(rank, world, tmp):
def _case_pp_tp_mesh_refusal(rank, world, tmp):
- """pp 2 x tp 2: a tp mesh that is the pp mesh itself is refused before anything is sharded."""
from torch.distributed.device_mesh import init_device_mesh
mesh = init_device_mesh("cpu", (2, 2), mesh_dim_names = ("pp", "tp"))
@@ -362,11 +362,43 @@ def _case_pp_tp_mesh_refusal(rank, world, tmp):
return {"err": err, "tp": tpn.is_tensor_parallel(model)}
+def _case_fp16_replicas(rank, world, tmp):
+ from torch.distributed.device_mesh import init_device_mesh
+
+ native = _native()
+ mesh = init_device_mesh("cpu", (2, 2), mesh_dim_names = ("pp", "dp"))
+ model = build(LLAMA)
+ norm = model.get_base_model().model.norm.weight
+ norm.data = norm.data.half()
+ norm.requires_grad_(True)
+ ids, labels = global_batch(model, 8)
+ model(input_ids = ids, labels = labels, use_cache = False).loss.backward()
+ expected = norm.grad.detach().float().clone()
+ model.zero_grad(set_to_none = True)
+ pp = ppn.apply_pipeline_parallel(model, mesh["pp"], n_microbatches = 2)
+ native.shard_model(
+ model,
+ native.FSDP2Config(
+ device = "cpu", param_dtype = torch.float32, mesh = mesh["dp"], ignored_params = {norm}
+ ),
+ )
+ dp_rank = mesh["dp"].get_local_rank()
+ grads, pending = [], []
+ for _ in range(2):
+ pp.step(ids[4 * dp_rank : 4 * dp_rank + 4], labels[4 * dp_rank : 4 * dp_rank + 4])
+ if pp.is_last:
+ grads.append(None if norm.grad is None else norm.grad.detach().float().clone())
+ pending.append(bool(model.__dict__["_unsloth_replicated_grad_state"].get("pending")))
+ model.zero_grad(set_to_none = True)
+ return {"last": pp.is_last, "grads": grads, "expected": expected, "pending": pending}
+
+
CASES = {
"compose": _case_compose,
"scaler": _case_scaler,
"order": _case_order_refusals,
"pp_tp_mesh": _case_pp_tp_mesh_refusal,
+ "fp16_replicas": _case_fp16_replicas,
}
@@ -376,6 +408,8 @@ def _free_port():
def _worker(rank, world, port, case, tmp, kwargs):
+ global _LOCAL_CONFIG
+ _LOCAL_CONFIG = True
os.environ.update(
MASTER_ADDR = "127.0.0.1",
MASTER_PORT = str(port),
@@ -399,6 +433,13 @@ def _worker(rank, world, port, case, tmp, kwargs):
dist.destroy_process_group()
[email protected]_cache(maxsize = 1)
+def _prefetch_configs():
+ from transformers import AutoConfig
+ for repo, _ in ARCHS.values():
+ AutoConfig.from_pretrained(repo)
+
+
def _spawn(
case,
world = 4,
@@ -411,6 +452,7 @@ def _spawn(
native = _native()
if not native._supports_ignored_params(native._fsdp_api()[2]):
pytest.skip("native FSDP2 composition needs fully_shard(ignored_params) (torch 2.7+)")
+ _prefetch_configs()
ctx = multiprocessing.get_context("spawn")
port = _free_port()
with tempfile.TemporaryDirectory() as tmp:
@@ -451,8 +493,8 @@ def _check(
res["norms"],
ref["norms"],
)
- # Later norms follow 1-2 AdamW steps (lr 1e-2), whose first updates are ~lr * sign(grad): fp32 summation
- # order on near-zero grads moves them up to ~5e-4 (Gemma3; the pp-only split alone moves it 2.6e-4).
+ # AdamW updates amplify reduction-order noise in near-zero gradients.
+
assert res["norms"] == pytest.approx(ref["norms"], rel = 2e-3), (
rank,
res["norms"],
@@ -462,7 +504,7 @@ def _check(
num = den = 0.0
for name, grad in res["grads"].items():
want = ref["grads"][name]
- # Tensor parallel row-parallel sums reorder fp32 additions: tolerance relative to the tensor's scale.
+
atol = max(1e-6, grad_rel * float(want.abs().max()))
torch.testing.assert_close(
grad, want, rtol = 1e-4, atol = atol, msg = lambda m, name = name: f"{name}: {m}"
@@ -478,7 +520,7 @@ def _check(
), refusals
if "dp" in dims:
assert res["stage_root_fsdp"]
- # FSDP2 shards the trainable weights over dp; each group reduces once per step, not per micro-batch.
+
assert res["placements"] and all(
any("S(" in pl or "Shard" in pl for pl in pls) for pls in res["placements"].values()
), res["placements"]
@@ -509,7 +551,7 @@ def _check(
assert set(saved_pp) == set(saved_ref), sorted(set(saved_pp) ^ set(saved_ref))[:6]
for key, value in saved_ref.items():
assert saved_pp[key].shape == value.shape, key
- # 3 AdamW steps (lr 1e-2) carry fp32 summation order differences (as the pp-only suite).
+
torch.testing.assert_close(saved_pp[key], value, rtol = 1e-3, atol = 2e-3, msg = key)
@@ -526,7 +568,6 @@ def test_pp2_x_fsdp2_dp2_matches_one_process(arch):
def test_pp2_x_fsdp2_dp2_1f1b_sharded_layout_matches_one_process():
- # 1F1B, 4 micro-batches, the sharded layout (one FSDP2 unit per owned decoder layer, frozen weights sharded).
dims = {"pp": 2, "dp": 2}
_check(
_spawn("compose", arch = "llama", dims = dims, schedule = "1f1b", n_mb = 4, fit_layout = False),
@@ -548,7 +589,6 @@ def test_pp2_x_tp2_matches_one_process(arch):
@pytest.mark.parametrize("other", ["dp", "tp"])
def test_pp2_x_size_one_dp_or_tp_matches_one_process(other):
- # (pp 2, dp 1) / (pp 2, tp 1) on 2 ranks: FSDP2 / TP DTensors on a 1-rank mesh, the shape the 2-GPU proof runs.
dims = {"pp": 2, other: 1}
_check(_spawn("compose", world = 2, arch = "llama", dims = dims), dims)
@@ -565,6 +605,17 @@ def test_pp2_x_dp2_grad_scaler_skip_is_global():
assert res["scale"] == 512.0, res
+def test_pp2_x_fsdp2_dp2_consumes_fp16_replicated_gradients():
+ for res in _spawn("fp16_replicas"):
+ if not res["last"]:
+ continue
+ assert res["pending"] == [False, False], res
+ assert res["expected"].norm() > 0
+ for grad in res["grads"]:
+ assert grad is not None, res
+ torch.testing.assert_close(grad, res["expected"], rtol = 1e-2, atol = 1e-4)
+
+
def test_pp_composition_order_and_meshes_are_refused_before_any_change():
for res in _spawn("order"):
assert "must come before apply_tensor_parallel" in (res["tp_first"] or ""), res
diff --git a/unsloth/models/fsdp2_save.py b/unsloth/models/fsdp2_save.py
index 545f6713ee4..f83981edf35 100644
--- a/unsloth/models/fsdp2_save.py
+++ b/unsloth/models/fsdp2_save.py
@@ -1284,18 +1284,15 @@ def _sync_replicated_grads(
dist,
mesh = None,
buffers = None,
+ pending = None,
):
- """Average gradients over `mesh` (DDP-style), one all_reduce per (device, reduce dtype), entered
- by every rank: a missing local gradient counts as zeros, and a "has a gradient" flag keeps grad
- None for a parameter no rank used. Sums in float32 (fp16 sums overflow). The summed flags are
- read (a host sync) only when some local gradient is None. `buffers` caches one flat buffer per
- (device, dtype); gradients never alias it."""
+ """Average in float32; absent ranks contribute zero, globally unused grads stay None."""
groups = {}
for p in params:
dtype = torch.float64 if p.dtype == torch.float64 else torch.float32
groups.setdefault((str(p.device), dtype), []).append(p)
for (device, dtype), group in groups.items():
- # A TP DTensor outside FSDP2: dp ranks hold the same tp slice, so average the local shard.
+ # Across dp, an external TP DTensor represents matching local slices.
sizes = [_local_tensor(p).numel() for p in group]
total = sum(sizes)
key = tuple((id(p), n) for p, n in zip(group, sizes))
@@ -1307,18 +1304,24 @@ def _sync_replicated_grads(
if buffers is not None:
buffers[(device, dtype)] = (key, flat)
grads = [_local_tensor(p.grad) for p in group]
- local_all = all(g is not None for g in grads)
+ extra = [pending.get(id(p)) if pending else None for p in group]
+ local_all = all(g is not None or e is not None for g, e in zip(grads, extra))
flags = flat[total:]
if local_all:
flags.fill_(1)
else:
flags.zero_()
offset = 0
- for i, (g, n) in enumerate(zip(grads, sizes)):
- if g is not None:
- flat[offset : offset + n].copy_(g.detach().reshape(-1))
+ for i, (g, e, n) in enumerate(zip(grads, extra, sizes)):
+ if g is not None or e is not None:
+ if g is None:
+ flat[offset : offset + n].copy_(e.reshape(-1))
+ else:
+ flat[offset : offset + n].copy_(g.detach().reshape(-1))
+ if e is not None:
+ flat[offset : offset + n].add_(e.reshape(-1))
if not local_all:
- # Not flags[i] = 1: that is a host sync on CUDA.
+ # Python scalar assignment would synchronize CUDA.
flags[i].fill_(1)
else:
flat[offset : offset + n].zero_()
@@ -1327,8 +1330,8 @@ def _sync_replicated_grads(
flat[:total].div_(world)
used = None if local_all else flags.tolist()
offset = 0
- for i, (p, g, n) in enumerate(zip(group, grads, sizes)):
- if g is not None or used[i] > 0:
+ for i, (p, g, e, n) in enumerate(zip(group, grads, extra, sizes)):
+ if g is not None or e is not None or used[i] > 0:
value = flat[offset : offset + n].view(_local_tensor(p).shape)
if g is None:
p.grad = _like_param(p, value.to(p.dtype, copy = True))
@@ -1427,11 +1430,7 @@ def _fsdp_grad_sync_deferred(model, cache):
def reduce_unsharded_trainable_grads(model):
- """Average gradients of trainable params FSDP2 does not manage (e.g. `ignored_modules`) over the
- mesh, else replicas drift. Runs once per backward as an autograd engine callback on every rank;
- a per-parameter hook would hang when a rank skips the parameter (branch, routed MoE expert).
- Skipped while FSDP2 grad sync is off (GA); the first synced backward averages the accumulated
- sum. Returns the names."""
+ """Queue one fixed-order reduction per backward on every rank; per-param hooks can deadlock."""
dist = _dist()
DTensor = _dtensor_class()
if dist is None or DTensor is None:
@@ -1447,10 +1446,11 @@ def reduce_unsharded_trainable_grads(model):
def _sync():
state["queued"] = False
- # Every rank enters the same branch: the GA flag is set alike on all ranks.
+
if _fsdp_grad_sync_deferred(model, state):
+ _stash_fp16_grads(params, state)
return
- _sync_replicated_grads(params, dist, mesh, buffers)
+ _sync_replicated_grads(params, dist, mesh, buffers, state.pop("pending", None))
def _queue(*_):
if not state["queued"]:
@@ -1462,7 +1462,6 @@ def reduce_unsharded_trainable_grads(model):
if id(param) in seen or not param.requires_grad:
continue
if isinstance(param, DTensor):
- # FSDP2 does not reduce TP DTensors in ignored_params; average them here.
if managed is False:
managed = _fsdp_managed_param_ids(model)
if managed is None or id(param) in managed or not _tp_shard_outside(param, mesh, dist):
@@ -1478,7 +1477,7 @@ def reduce_unsharded_trainable_grads(model):
def _pre_forward(module, args):
if not _in_backward():
- state["queued"] = False # a backward that raised never ran its callback
+ state["queued"] = False # Failed backwards never ran their queued callback.
def _post_forward(module, args, output):
if not torch.is_grad_enabled():
@@ -1495,4 +1494,35 @@ def reduce_unsharded_trainable_grads(model):
model.register_forward_pre_hook(_pre_forward)
model.register_forward_hook(_post_forward)
model.__dict__["_unsloth_replicated_grad_hooked"] = True
+ _install_replicated_grad_reset(model)
return names
+
+
+def _stash_fp16_grads(params, state):
+ pending = state.setdefault("pending", {})
+ for p in params:
+ if p.dtype != torch.float16 or p.grad is None or p.grad.dtype != torch.float16:
+ continue
+ g = _local_tensor(p.grad).detach().float()
+ prev = pending.get(id(p))
+ pending[id(p)] = g if prev is None else prev.add_(g)
+ p.grad = None
+
+
+def _install_replicated_grad_reset(model):
+ reset = getattr(model, "reset_iter_state", None)
+ if reset is None or getattr(reset, "_unsloth_replicated_reset", False):
+ return
+
+ @functools.wraps(reset)
+ def reset_replicas(*args, **kwargs):
+ result = reset(*args, **kwargs)
+ for module in model.modules():
+ state = module.__dict__.get("_unsloth_replicated_grad_state")
+ if state is not None:
+ state.pop("pending", None)
+ state["queued"] = False
+ return result
+
+ reset_replicas._unsloth_replicated_reset = True
+ model.reset_iter_state = reset_replicas
diff --git a/unsloth/models/pp_native.py b/unsloth/models/pp_native.py
index 5bbf6453167..7e4d313b7a4 100644
--- a/unsloth/models/pp_native.py
+++ b/unsloth/models/pp_native.py
@@ -1444,11 +1444,14 @@ class PipelineParallel:
policy = MixedPrecisionPolicy(
param_dtype = None,
reduce_dtype = getattr(mp_policy, "reduce_dtype", None),
- cast_forward_inputs = False, # the received hidden states keep PPConfig.activation_dtype
+ cast_forward_inputs = False,
)
fully_shard(self.module, mesh = dp_mesh, mp_policy = policy, reshard_after_forward = False)
+ from .fsdp2_save import _install_replicated_grad_reset
+
+ _install_replicated_grad_reset(self.module)
self.dp_mesh = dp_mesh
- self._shapes.clear() # stages built before hold the unwrapped module
+ self._shapes.clear()
@property
def is_first(self):
@@ -1774,7 +1777,11 @@ class PipelineParallel:
state = self.model.__dict__.setdefault("_unsloth_replicated_grad_state", {"queued": False})
_sync_replicated_grads(
- params, dist, model_mesh(self.model) or self.dp_mesh, state.setdefault("buffers", {})
+ params,
+ dist,
+ model_mesh(self.model) or self.dp_mesh,
+ state.setdefault("buffers", {}),
+ state.pop("pending", None),
)
def _grad_copies(self, grad):