Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Native FSDP2 float16: skip the step when every rank overflows under fused AdamW (#14996)
Daniel Han96e1778afb66
diff --git a/tests/test_fsdp2_native_scaler_fused_gloo.py b/tests/test_fsdp2_native_scaler_fused_gloo.pyindex 6326db0f46c..e512e57cb61 100644--- a/tests/test_fsdp2_native_scaler_fused_gloo.py+++ b/tests/test_fsdp2_native_scaler_fused_gloo.py@@ -31,7 +31,7 @@ class _Handler:return {"device": "cpu", "init_scale": 2.0**8}-def _run(fused):+def _run(fused, both = False):native = _load_native()rank = torch.distributed.get_rank()old = torch.amp.GradScaler("cpu", init_scale = 2.0**8)@@ -49,8 +49,8 @@ def _run(fused):torch.manual_seed(100 + 10 * step + rank)loss = model(torch.randn(4, 8)).pow(2).mean()scaler.scale(loss).backward()- if step == 1 and rank == 1:- model.weight.grad[0, 0] = float("inf") # one rank overflows: both must skip+ if step == 1 and (both or rank == 1):+ model.weight.grad[0, 0] = float("inf") # one rank (or every rank) overflows: all skipscaler.unscale_(opt)torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)scaler.step(opt)@@ -102,10 +102,11 @@ def _worker(rank, world, port, out):try:res = {}for fused in (False, True):- try:- res[fused] = _run(fused)- except Exception as e: # recorded, asserted in the parent- res[fused] = repr(e)+ for both in (False, True):+ try:+ res[fused, both] = _run(fused, both)+ except Exception as e: # recorded, asserted in the parent+ res[fused, both] = repr(e)res["device"] = _device_kept()try:res["disabled"] = _disabled()@@ -128,14 +129,20 @@ def test_swapped_scaler_keeps_the_device(ranks):assert res["device"] == ("cpu", True), res["device"]-def test_fused_adamw_steps_through_the_native_fp16_scaler(ranks):[email protected]("both", [False, True], ids = ["one_rank_inf", "every_rank_inf"])+def test_fused_adamw_steps_through_the_native_fp16_scaler(ranks, both):+ # every_rank_inf: the all-reduced found_inf is the number of ranks that overflowed (2 here). The+ # fused AdamW kernel skips only on found_inf == 1, so a 2 stepped on inf / NaN gradients and wrote+ # NaN weights (Qwen2.5-VL on Kaggle T4x2: loss NaN from step 4 under FSDP2, finite under DDP).for res in ranks:for fused in (False, True):- assert not isinstance(res[fused], str), f"fused={fused}: {res[fused]}"- steps, scale, _ = res[fused]+ out = res[fused, both]+ assert not isinstance(out, str), f"fused={fused}: {out}"+ steps, scale, params = outassert steps == 2.0, (fused, steps) # 3 steps, the overflow one skipped on every rankassert scale == 2.0**7, (fused, scale) # backed off once- for a, b in zip(res[False][2], res[True][2]):+ assert all(torch.isfinite(p).all() for p in params), fused+ for a, b in zip(res[False, both][2], res[True, both][2]):torch.testing.assert_close(a, b, rtol = 1e-5, atol = 1e-6)diff --git a/unsloth/models/fsdp2_native.py b/unsloth/models/fsdp2_native.pyindex 8fcb4ba60e5..5df2e14e847 100644--- a/unsloth/models/fsdp2_native.py+++ b/unsloth/models/fsdp2_native.py@@ -2494,9 +2494,12 @@ def _scaler_for_native(trainer):super().unscale_(optimizer)if not self.is_enabled():return+ # The all-reduce SUMS found_inf: it is the number of ranks that overflowed. Fused+ # optimizers skip the step only on found_inf == 1 (and subtract it from their step+ # counts), so 2+ ranks overflowing stepped on inf / NaN gradients. Clamp to 0 / 1.state = self._per_optimizer_states[id(optimizer)]state["found_inf_per_device"] = {- d: t.reshape(()) for d, t in state["found_inf_per_device"].items()+ d: t.clamp(max = 1.0).reshape(()) for d, t in state["found_inf_per_device"].items()}kwargs = {}