Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Native FSDP2 float16: skip the step when every rank overflows under fused AdamW (#14996)

Daniel Han96e1778afb66
diff --git a/tests/test_fsdp2_native_scaler_fused_gloo.py b/tests/test_fsdp2_native_scaler_fused_gloo.py
index 6326db0f46c..e512e57cb61 100644
--- a/tests/test_fsdp2_native_scaler_fused_gloo.py
+++ b/tests/test_fsdp2_native_scaler_fused_gloo.py
@@ -31,7 +31,7 @@ class _Handler:
return {"device": "cpu", "init_scale": 2.0**8}
-def _run(fused):
+def _run(fused, both = False):
native = _load_native()
rank = torch.distributed.get_rank()
old = torch.amp.GradScaler("cpu", init_scale = 2.0**8)
@@ -49,8 +49,8 @@ def _run(fused):
torch.manual_seed(100 + 10 * step + rank)
loss = model(torch.randn(4, 8)).pow(2).mean()
scaler.scale(loss).backward()
- if step == 1 and rank == 1:
- model.weight.grad[0, 0] = float("inf") # one rank overflows: both must skip
+ if step == 1 and (both or rank == 1):
+ model.weight.grad[0, 0] = float("inf") # one rank (or every rank) overflows: all skip
scaler.unscale_(opt)
torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
scaler.step(opt)
@@ -102,10 +102,11 @@ def _worker(rank, world, port, out):
try:
res = {}
for fused in (False, True):
- try:
- res[fused] = _run(fused)
- except Exception as e: # recorded, asserted in the parent
- res[fused] = repr(e)
+ for both in (False, True):
+ try:
+ res[fused, both] = _run(fused, both)
+ except Exception as e: # recorded, asserted in the parent
+ res[fused, both] = repr(e)
res["device"] = _device_kept()
try:
res["disabled"] = _disabled()
@@ -128,14 +129,20 @@ def test_swapped_scaler_keeps_the_device(ranks):
assert res["device"] == ("cpu", True), res["device"]
-def test_fused_adamw_steps_through_the_native_fp16_scaler(ranks):
[email protected]("both", [False, True], ids = ["one_rank_inf", "every_rank_inf"])
+def test_fused_adamw_steps_through_the_native_fp16_scaler(ranks, both):
+ # every_rank_inf: the all-reduced found_inf is the number of ranks that overflowed (2 here). The
+ # fused AdamW kernel skips only on found_inf == 1, so a 2 stepped on inf / NaN gradients and wrote
+ # NaN weights (Qwen2.5-VL on Kaggle T4x2: loss NaN from step 4 under FSDP2, finite under DDP).
for res in ranks:
for fused in (False, True):
- assert not isinstance(res[fused], str), f"fused={fused}: {res[fused]}"
- steps, scale, _ = res[fused]
+ out = res[fused, both]
+ assert not isinstance(out, str), f"fused={fused}: {out}"
+ steps, scale, params = out
assert steps == 2.0, (fused, steps) # 3 steps, the overflow one skipped on every rank
assert scale == 2.0**7, (fused, scale) # backed off once
- for a, b in zip(res[False][2], res[True][2]):
+ assert all(torch.isfinite(p).all() for p in params), fused
+ for a, b in zip(res[False, both][2], res[True, both][2]):
torch.testing.assert_close(a, b, rtol = 1e-5, atol = 1e-6)
diff --git a/unsloth/models/fsdp2_native.py b/unsloth/models/fsdp2_native.py
index 8fcb4ba60e5..5df2e14e847 100644
--- a/unsloth/models/fsdp2_native.py
+++ b/unsloth/models/fsdp2_native.py
@@ -2494,9 +2494,12 @@ def _scaler_for_native(trainer):
super().unscale_(optimizer)
if not self.is_enabled():
return
+ # The all-reduce SUMS found_inf: it is the number of ranks that overflowed. Fused
+ # optimizers skip the step only on found_inf == 1 (and subtract it from their step
+ # counts), so 2+ ranks overflowing stepped on inf / NaN gradients. Clamp to 0 / 1.
state = self._per_optimizer_states[id(optimizer)]
state["found_inf_per_device"] = {
- d: t.reshape(()) for d, t in state["found_inf_per_device"].items()
+ d: t.clamp(max = 1.0).reshape(()) for d, t in state["found_inf_per_device"].items()
}
kwargs = {}