Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Native FSDP2 fp16 scaler: agree on overflow over ranks without an explicit unscale_, and give fused optimizers one 0 / 1 flag over all devices (#15416)

Daniel Han291a4c72bf4c
diff --git a/tests/test_fsdp2_native_scaler_fused_gloo.py b/tests/test_fsdp2_native_scaler_fused_gloo.py
index b8343c8ad1b..16eeabd3eed 100644
--- a/tests/test_fsdp2_native_scaler_fused_gloo.py
+++ b/tests/test_fsdp2_native_scaler_fused_gloo.py
@@ -31,7 +31,11 @@ class _Handler:
return {"device": "cpu", "init_scale": 2.0**8}
-def _run(fused, both = False):
+def _run(
+ fused,
+ both = False,
+ unscale = True,
+):
native = _load_native()
rank = torch.distributed.get_rank()
old = torch.amp.GradScaler("cpu", init_scale = 2.0**8)
@@ -51,8 +55,9 @@ def _run(fused, both = False):
scaler.scale(loss).backward()
if step == 1 and (both or rank == 1):
model.weight.grad[0, 0] = float("inf") # one rank (or every rank) overflows: all skip
- scaler.unscale_(opt)
- torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
+ if unscale:
+ scaler.unscale_(opt)
+ torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
scaler.step(opt)
scaler.update()
opt.zero_grad(set_to_none = True)
@@ -107,6 +112,10 @@ def _worker(rank, world, port, out):
res[fused, both] = _run(fused, both)
except Exception as e: # recorded, asserted in the parent
res[fused, both] = repr(e)
+ try:
+ res[fused, both, "no_unscale"] = _run(fused, both, unscale = False)
+ except Exception as e:
+ res[fused, both, "no_unscale"] = repr(e)
res["device"] = _device_kept()
try:
res["disabled"] = _disabled()
@@ -147,6 +156,21 @@ def test_fused_adamw_steps_through_the_native_fp16_scaler(ranks, both):
torch.testing.assert_close(a, b, rtol = 1e-5, atol = 1e-6)
[email protected]("both", [False, True], ids = ["one_rank_inf", "every_rank_inf"])
+def test_overflow_skips_on_every_rank_without_explicit_unscale(ranks, both):
+ # Fused step() without unscale_; before the fix one_rank_inf left ranks at 3 vs 2 steps.
+ for res in ranks:
+ for fused in (False, True):
+ out = res[fused, both, "no_unscale"]
+ assert not isinstance(out, str), f"fused={fused}: {out}"
+ steps, scale, params = out
+ assert steps == 2.0, (fused, both, steps)
+ assert scale == 2.0**7, (fused, both, scale)
+ assert all(torch.isfinite(p).all() for p in params), fused
+ for a, b in zip(res[False, both, "no_unscale"][2], res[True, both, "no_unscale"][2]):
+ torch.testing.assert_close(a, b, rtol = 1e-5, atol = 1e-6)
+
+
def test_disabled_scaler_still_steps(ranks):
for res in ranks:
assert res["disabled"] == 1.0, res["disabled"]
diff --git a/tests/test_fsdp2_native_scaler_mixed_device.py b/tests/test_fsdp2_native_scaler_mixed_device.py
new file mode 100644
index 00000000000..dc248927e90
--- /dev/null
+++ b/tests/test_fsdp2_native_scaler_mixed_device.py
@@ -0,0 +1,120 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""Fused AdamW over CPU + CUDA params: both overflowing summed found_inf to 2, so the step was not skipped."""
+
+import os
+import sys
+import types
+
+import pytest
+import torch
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from test_fsdp2_native_units import _load_native # noqa: E402
+from test_fsdp2_native_trainer_ckpt_gloo import _free_port # noqa: E402
+
+pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason = "needs a CUDA device")
+
+
+class _Handler:
+ def to_kwargs(self):
+ return {"device": "cuda", "init_scale": 2.0**8}
+
+
[email protected](scope = "module")
+def world_of_one():
+ # Once per module: ShardedGradScaler's default process_group argument is bound to the first WORLD.
+ import torch.distributed as dist
+
+ os.environ.update(MASTER_ADDR = "127.0.0.1", MASTER_PORT = str(_free_port()))
+ dist.init_process_group("cpu:gloo,cuda:nccl", rank = 0, world_size = 1)
+ torch.cuda.set_device(0)
+ try:
+ yield
+ finally:
+ dist.destroy_process_group()
+
+
[email protected]("explicit_unscale", [True, False], ids = ["clipping", "no_clipping"])
[email protected]("overflow", ["cpu", "cuda", "both"])
+def test_mixed_device_fused_adamw_skips_an_overflow_step(world_of_one, explicit_unscale, overflow):
+ native = _load_native()
+ old = torch.amp.GradScaler("cuda", init_scale = 2.0**8)
+ trainer = types.SimpleNamespace(
+ accelerator = types.SimpleNamespace(scaler = old, scaler_handler = _Handler())
+ )
+ assert native._scaler_for_native(trainer) is True
+ scaler = trainer.accelerator.scaler
+ torch.manual_seed(0)
+ on_cpu = torch.nn.Parameter(torch.randn(8, 8))
+ on_cuda = torch.nn.Parameter(torch.randn(8, 8, device = "cuda"))
+ opt = torch.optim.AdamW([on_cpu, on_cuda], lr = 1e-2, fused = True)
+ before = [on_cpu.detach().clone(), on_cuda.detach().clone()]
+ loss = on_cpu.sum().cuda() + on_cuda.sum()
+ scaler.scale(loss).backward()
+ if overflow in ("cpu", "both"):
+ on_cpu.grad[0, 0] = float("inf")
+ if overflow in ("cuda", "both"):
+ on_cuda.grad[0, 0] = float("inf")
+ if explicit_unscale:
+ scaler.unscale_(opt)
+ scaler.step(opt)
+ scaler.update()
+ for p, b in zip((on_cpu, on_cuda), before):
+ assert torch.equal(p.detach().cpu(), b.cpu()), (
+ overflow,
+ "the overflow step changed weights",
+ )
+ assert float(opt.state[p]["step"]) == 0.0, (overflow, float(opt.state[p]["step"]))
+ assert scaler.get_scale() == 2.0**7
+
+
+def _scaler():
+ native = _load_native()
+ old = torch.amp.GradScaler("cuda", init_scale = 2.0**8)
+ trainer = types.SimpleNamespace(
+ accelerator = types.SimpleNamespace(scaler = old, scaler_handler = _Handler())
+ )
+ assert native._scaler_for_native(trainer) is True
+ return trainer.accelerator.scaler
+
+
[email protected]("explicit_unscale", [True, False], ids = ["clipping", "no_clipping"])
[email protected]("overflow", ["none", "cpu", "cuda", "both"])
+def test_late_device_to_host_copies_keep_the_step_right(world_of_one, explicit_unscale, overflow):
+ scaler = _scaler()
+ torch.manual_seed(0)
+ on_cpu = torch.nn.Parameter(torch.randn(8, 8))
+ on_cuda = torch.nn.Parameter(torch.randn(8, 8, device = "cuda"))
+ opt = torch.optim.AdamW([on_cpu, on_cuda], lr = 1e-2, fused = True)
+ ref_cpu = torch.nn.Parameter(on_cpu.detach().clone())
+ ref_opt = torch.optim.AdamW([ref_cpu], lr = 1e-2)
+ before = [on_cpu.detach().clone(), on_cuda.detach().clone()]
+ scaler.scale(on_cpu.sum().cuda() + on_cuda.sum()).backward()
+ if overflow in ("cpu", "both"):
+ on_cpu.grad[0, 0] = float("inf")
+ if overflow in ("cuda", "both"):
+ on_cuda.grad[0, 0] = float("inf")
+ torch.cuda.synchronize()
+ torch.cuda._sleep(
+ 200_000_000
+ ) # keep the stream busy: any non_blocking copy to the host lands late
+ if explicit_unscale:
+ scaler.unscale_(opt)
+ scaler.step(opt)
+ scaler.update()
+ torch.cuda.synchronize()
+ if overflow == "none":
+ ref_cpu.grad = torch.ones_like(ref_cpu)
+ ref_opt.step()
+ torch.testing.assert_close(on_cpu.detach(), ref_cpu.detach())
+ assert scaler.get_scale() == 2.0**8
+ else:
+ for p, b in zip((on_cpu, on_cuda), before):
+ assert torch.equal(p.detach().cpu(), b.cpu()), (
+ overflow,
+ "the overflow step changed weights",
+ )
+ assert float(opt.state[p]["step"]) == 0.0, (overflow, float(opt.state[p]["step"]))
+ assert scaler.get_scale() == 2.0**7
diff --git a/tests/test_tp_native_scaler_fused_gloo.py b/tests/test_tp_native_scaler_fused_gloo.py
new file mode 100644
index 00000000000..e4cc3844f43
--- /dev/null
+++ b/tests/test_tp_native_scaler_fused_gloo.py
@@ -0,0 +1,98 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""TP fused AdamW without clipping: an inf in one rank's shard must skip the step on every rank."""
+
+import multiprocessing
+import os
+
+if multiprocessing.parent_process() is not None or getattr(
+ multiprocessing.current_process(), "_inheriting", False
+):
+ os.environ["CUDA_VISIBLE_DEVICES"] = "" # spawned gloo ranks: CPU only, also on GPU hosts
+
+import sys # noqa: E402
+import types # noqa: E402
+
+import pytest # noqa: E402
+import torch # noqa: E402
+from torch.distributed.tensor import Replicate, Shard # noqa: E402
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from test_fsdp2_native_trainer_ckpt_gloo import _free_port # noqa: E402
+
+
+def _run(fused, unscale):
+ from test_tp_native_oracle import QWEN3, _batch, build, tpn
+ from torch.distributed.device_mesh import init_device_mesh
+
+ rank = torch.distributed.get_rank()
+ old = torch.amp.GradScaler("cpu", init_scale = 2.0**8)
+ handler = types.SimpleNamespace(to_kwargs = lambda: {"device": "cpu", "init_scale": 2.0**8})
+ trainer = types.SimpleNamespace(
+ accelerator = types.SimpleNamespace(scaler = old, scaler_handler = handler)
+ )
+ tpn._tp_scaler(trainer, 1)
+ scaler = trainer.accelerator.scaler
+ assert scaler is not old
+ model = build(QWEN3)
+ mesh = init_device_mesh("cpu", (2,), mesh_dim_names = ("tp",))
+ tpn.apply_tensor_parallel(model, mesh["tp"])
+ model.train()
+ params = [p for p in model.parameters() if p.requires_grad]
+ # LoRA A is replicated and comes first: DTensor's own found_inf reduction reads only the first
+ # gradient's placements, so it does not reduce over tp and the overflow stays on one rank.
+ assert tuple(params[0].placements) == (Replicate(),), params[0].placements
+ sharded = next(p for p in params if any(isinstance(x, Shard) for x in p.placements))
+ opt = torch.optim.AdamW(params, lr = 1e-2, fused = fused, foreach = None if fused else False)
+ ids = _batch(model)
+ for step in range(3):
+ out = model(input_ids = ids, labels = ids, use_cache = False)
+ scaler.scale(out.loss).backward()
+ if step == 1 and rank == 1:
+ sharded.grad.to_local().view(-1)[0] = float("inf") # only rank 1's shard overflows
+ if unscale:
+ scaler.unscale_(opt)
+ scaler.step(opt)
+ scaler.update()
+ opt.zero_grad(set_to_none = True)
+ steps = float(opt.state[sharded]["step"])
+ return steps, float(scaler.get_scale()), sharded.full_tensor().detach().clone()
+
+
+def _worker(rank, world, port, out):
+ os.environ.update(MASTER_ADDR = "127.0.0.1", MASTER_PORT = str(port))
+ torch.distributed.init_process_group("gloo", rank = rank, world_size = world)
+ try:
+ res = {}
+ for fused in (False, True):
+ for unscale in (False, True):
+ try:
+ res[fused, unscale] = _run(fused, unscale)
+ except Exception as e:
+ res[fused, unscale] = repr(e)
+ torch.save(res, os.path.join(out, f"rank{rank}.pt"))
+ finally:
+ torch.distributed.destroy_process_group()
+
+
[email protected](scope = "module")
+def ranks(tmp_path_factory):
+ pytest.importorskip("peft")
+ out = str(tmp_path_factory.mktemp("tp_scaler"))
+ torch.multiprocessing.spawn(_worker, args = (2, _free_port(), out), nprocs = 2, join = True)
+ return [torch.load(os.path.join(out, f"rank{r}.pt")) for r in range(2)]
+
+
[email protected]("unscale", [False, True], ids = ["no_clipping", "clipping"])
[email protected]("fused", [False, True], ids = ["foreach", "fused"])
+def test_tp_overflow_on_one_shard_skips_the_step_on_every_rank(ranks, fused, unscale):
+ # Before: fused + no_clipping left rank 0 at 3 steps and scale 2**8 while rank 1 skipped (2, 2**7).
+ outs = [res[fused, unscale] for res in ranks]
+ for out in outs:
+ assert not isinstance(out, str), out
+ steps, scale, weight = out
+ assert steps == 2.0, (fused, unscale, steps)
+ assert scale == 2.0**7, (fused, unscale, scale)
+ assert torch.isfinite(weight).all()
+ torch.testing.assert_close(outs[0][2], outs[1][2], rtol = 0, atol = 0)
diff --git a/unsloth/models/fsdp2_native.py b/unsloth/models/fsdp2_native.py
index 2f5e8dd7c7a..a5ad76d4a90 100644
--- a/unsloth/models/fsdp2_native.py
+++ b/unsloth/models/fsdp2_native.py
@@ -3337,6 +3337,15 @@ def install_native_fsdp2_hooks():
return a and t
+def _has_cpu_tensors(optimizer, grads = False):
+ for group in optimizer.param_groups:
+ for p in group["params"]:
+ t = p.grad if grads else p
+ if t is not None and t.device.type == "cpu":
+ return True
+ return False
+
+
def _scaler_for_native(trainer):
"""float16: the GradScaler's inf check must agree over ranks, or one rank skips a step the
others take. ShardedGradScaler all-reduces it."""
@@ -3345,6 +3354,7 @@ def _scaler_for_native(trainer):
if scaler is None:
return False
try:
+ import torch.distributed as dist
from torch.distributed.fsdp.sharded_grad_scaler import ShardedGradScaler
except ImportError:
return False
@@ -3358,12 +3368,80 @@ def _scaler_for_native(trainer):
super().unscale_(optimizer)
if not self.is_enabled():
return
- # The all-reduce SUMS found_inf: it is the number of ranks that overflowed. Fused
- # optimizers skip the step only on found_inf == 1 (and subtract it from their step
- # counts), so 2+ ranks overflowing stepped on inf / NaN gradients. Clamp to 0 / 1.
+ self._one_flag(optimizer)
+
+ def _check_inf_per_device(self, optimizer):
+ # Fused step() without unscale_ (no clipping) lands here; ShardedGradScaler does not
+ # all-reduce here, so ranks diverged on overflow skips. Agree over ranks.
+ super()._check_inf_per_device(optimizer)
state = self._per_optimizer_states[id(optimizer)]
+ works, copies = [], []
+ for found_inf in state["found_inf_per_device"].values():
+ if self._device != "cpu" and found_inf.device.type == "cpu":
+ on_device = found_inf.to(self._device)
+ copies.append((found_inf, on_device))
+ found_inf = on_device
+ works.append(dist.all_reduce(found_inf, async_op = True, group = self.process_group))
+ for work in works:
+ work.wait()
+ for found_inf, on_device in copies:
+ found_inf.copy_(on_device)
+ self._one_flag(optimizer)
+ return state["found_inf_per_device"]
+
+ def _unscale_grads_(
+ self,
+ optimizer,
+ inv_scale,
+ found_inf,
+ allow_fp16 = True,
+ ):
+ # CPU grads get non_blocking copies of CUDA inv_scale / found_inf, read before they land.
+ if inv_scale.device.type != "cpu" and _has_cpu_tensors(optimizer, grads = True):
+ inv_scale, found_inf = inv_scale.cpu(), found_inf.cpu()
+ return super()._unscale_grads_(optimizer, inv_scale, found_inf, allow_fp16)
+
+ def step(self, optimizer, *args, **kwargs):
+ # Same race for a fused optimizer's grad_scale / found_inf on CPU parameters: use host copies.
+ if not (
+ self.is_enabled()
+ and self._device != "cpu"
+ and getattr(optimizer, "_step_supports_amp_scaling", False)
+ and _has_cpu_tensors(optimizer)
+ ):
+ return super().step(optimizer, *args, **kwargs)
+ had = "step" in optimizer.__dict__
+ previous = optimizer.__dict__.get("step")
+ real = optimizer.step
+
+ def step_with_host_flags(*a, **k):
+ for name in ("grad_scale", "found_inf"):
+ t = getattr(optimizer, name, None)
+ if isinstance(t, torch.Tensor) and t.device.type != "cpu":
+ setattr(optimizer, name, t.cpu())
+ return real(*a, **k)
+
+ optimizer.step = step_with_host_flags
+ try:
+ return super().step(optimizer, *args, **kwargs)
+ finally:
+ if had:
+ optimizer.step = previous
+ else:
+ del optimizer.step
+
+ def _one_flag(self, optimizer):
+ # found_inf is summed over ranks and devices, but fused kernels skip only on == 1.
+ # One 0 / 1 flag on the first device, zeros elsewhere (consumers sum them).
+ state = self._per_optimizer_states[id(optimizer)]
+ found = state["found_inf_per_device"]
+ if not found:
+ return
+ first = next(iter(found.values()))
+ total = sum(t.reshape(()).to(first.device) for t in found.values()).clamp(max = 1.0)
state["found_inf_per_device"] = {
- d: t.clamp(max = 1.0).reshape(()) for d, t in state["found_inf_per_device"].items()
+ d: total if i == 0 else torch.zeros((), dtype = t.dtype, device = t.device)
+ for i, (d, t) in enumerate(found.items())
}
kwargs = {}