Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Native FSDP2 fp16 scaler: agree on overflow over ranks without an explicit unscale_, and give fused optimizers one 0 / 1 flag over all devices (#15416)
Daniel Han291a4c72bf4c
diff --git a/tests/test_fsdp2_native_scaler_fused_gloo.py b/tests/test_fsdp2_native_scaler_fused_gloo.pyindex b8343c8ad1b..16eeabd3eed 100644--- a/tests/test_fsdp2_native_scaler_fused_gloo.py+++ b/tests/test_fsdp2_native_scaler_fused_gloo.py@@ -31,7 +31,11 @@ class _Handler:return {"device": "cpu", "init_scale": 2.0**8}-def _run(fused, both = False):+def _run(+ fused,+ both = False,+ unscale = True,+):native = _load_native()rank = torch.distributed.get_rank()old = torch.amp.GradScaler("cpu", init_scale = 2.0**8)@@ -51,8 +55,9 @@ def _run(fused, both = False):scaler.scale(loss).backward()if step == 1 and (both or rank == 1):model.weight.grad[0, 0] = float("inf") # one rank (or every rank) overflows: all skip- scaler.unscale_(opt)- torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)+ if unscale:+ scaler.unscale_(opt)+ torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)scaler.step(opt)scaler.update()opt.zero_grad(set_to_none = True)@@ -107,6 +112,10 @@ def _worker(rank, world, port, out):res[fused, both] = _run(fused, both)except Exception as e: # recorded, asserted in the parentres[fused, both] = repr(e)+ try:+ res[fused, both, "no_unscale"] = _run(fused, both, unscale = False)+ except Exception as e:+ res[fused, both, "no_unscale"] = repr(e)res["device"] = _device_kept()try:res["disabled"] = _disabled()@@ -147,6 +156,21 @@ def test_fused_adamw_steps_through_the_native_fp16_scaler(ranks, both):torch.testing.assert_close(a, b, rtol = 1e-5, atol = 1e-6)[email protected]("both", [False, True], ids = ["one_rank_inf", "every_rank_inf"])+def test_overflow_skips_on_every_rank_without_explicit_unscale(ranks, both):+ # Fused step() without unscale_; before the fix one_rank_inf left ranks at 3 vs 2 steps.+ for res in ranks:+ for fused in (False, True):+ out = res[fused, both, "no_unscale"]+ assert not isinstance(out, str), f"fused={fused}: {out}"+ steps, scale, params = out+ assert steps == 2.0, (fused, both, steps)+ assert scale == 2.0**7, (fused, both, scale)+ assert all(torch.isfinite(p).all() for p in params), fused+ for a, b in zip(res[False, both, "no_unscale"][2], res[True, both, "no_unscale"][2]):+ torch.testing.assert_close(a, b, rtol = 1e-5, atol = 1e-6)++def test_disabled_scaler_still_steps(ranks):for res in ranks:assert res["disabled"] == 1.0, res["disabled"]diff --git a/tests/test_fsdp2_native_scaler_mixed_device.py b/tests/test_fsdp2_native_scaler_mixed_device.pynew file mode 100644index 00000000000..dc248927e90--- /dev/null+++ b/tests/test_fsdp2_native_scaler_mixed_device.py@@ -0,0 +1,120 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""Fused AdamW over CPU + CUDA params: both overflowing summed found_inf to 2, so the step was not skipped."""++import os+import sys+import types++import pytest+import torch++sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))+from test_fsdp2_native_units import _load_native # noqa: E402+from test_fsdp2_native_trainer_ckpt_gloo import _free_port # noqa: E402++pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason = "needs a CUDA device")+++class _Handler:+ def to_kwargs(self):+ return {"device": "cuda", "init_scale": 2.0**8}++[email protected](scope = "module")+def world_of_one():+ # Once per module: ShardedGradScaler's default process_group argument is bound to the first WORLD.+ import torch.distributed as dist++ os.environ.update(MASTER_ADDR = "127.0.0.1", MASTER_PORT = str(_free_port()))+ dist.init_process_group("cpu:gloo,cuda:nccl", rank = 0, world_size = 1)+ torch.cuda.set_device(0)+ try:+ yield+ finally:+ dist.destroy_process_group()++[email protected]("explicit_unscale", [True, False], ids = ["clipping", "no_clipping"])[email protected]("overflow", ["cpu", "cuda", "both"])+def test_mixed_device_fused_adamw_skips_an_overflow_step(world_of_one, explicit_unscale, overflow):+ native = _load_native()+ old = torch.amp.GradScaler("cuda", init_scale = 2.0**8)+ trainer = types.SimpleNamespace(+ accelerator = types.SimpleNamespace(scaler = old, scaler_handler = _Handler())+ )+ assert native._scaler_for_native(trainer) is True+ scaler = trainer.accelerator.scaler+ torch.manual_seed(0)+ on_cpu = torch.nn.Parameter(torch.randn(8, 8))+ on_cuda = torch.nn.Parameter(torch.randn(8, 8, device = "cuda"))+ opt = torch.optim.AdamW([on_cpu, on_cuda], lr = 1e-2, fused = True)+ before = [on_cpu.detach().clone(), on_cuda.detach().clone()]+ loss = on_cpu.sum().cuda() + on_cuda.sum()+ scaler.scale(loss).backward()+ if overflow in ("cpu", "both"):+ on_cpu.grad[0, 0] = float("inf")+ if overflow in ("cuda", "both"):+ on_cuda.grad[0, 0] = float("inf")+ if explicit_unscale:+ scaler.unscale_(opt)+ scaler.step(opt)+ scaler.update()+ for p, b in zip((on_cpu, on_cuda), before):+ assert torch.equal(p.detach().cpu(), b.cpu()), (+ overflow,+ "the overflow step changed weights",+ )+ assert float(opt.state[p]["step"]) == 0.0, (overflow, float(opt.state[p]["step"]))+ assert scaler.get_scale() == 2.0**7+++def _scaler():+ native = _load_native()+ old = torch.amp.GradScaler("cuda", init_scale = 2.0**8)+ trainer = types.SimpleNamespace(+ accelerator = types.SimpleNamespace(scaler = old, scaler_handler = _Handler())+ )+ assert native._scaler_for_native(trainer) is True+ return trainer.accelerator.scaler++[email protected]("explicit_unscale", [True, False], ids = ["clipping", "no_clipping"])[email protected]("overflow", ["none", "cpu", "cuda", "both"])+def test_late_device_to_host_copies_keep_the_step_right(world_of_one, explicit_unscale, overflow):+ scaler = _scaler()+ torch.manual_seed(0)+ on_cpu = torch.nn.Parameter(torch.randn(8, 8))+ on_cuda = torch.nn.Parameter(torch.randn(8, 8, device = "cuda"))+ opt = torch.optim.AdamW([on_cpu, on_cuda], lr = 1e-2, fused = True)+ ref_cpu = torch.nn.Parameter(on_cpu.detach().clone())+ ref_opt = torch.optim.AdamW([ref_cpu], lr = 1e-2)+ before = [on_cpu.detach().clone(), on_cuda.detach().clone()]+ scaler.scale(on_cpu.sum().cuda() + on_cuda.sum()).backward()+ if overflow in ("cpu", "both"):+ on_cpu.grad[0, 0] = float("inf")+ if overflow in ("cuda", "both"):+ on_cuda.grad[0, 0] = float("inf")+ torch.cuda.synchronize()+ torch.cuda._sleep(+ 200_000_000+ ) # keep the stream busy: any non_blocking copy to the host lands late+ if explicit_unscale:+ scaler.unscale_(opt)+ scaler.step(opt)+ scaler.update()+ torch.cuda.synchronize()+ if overflow == "none":+ ref_cpu.grad = torch.ones_like(ref_cpu)+ ref_opt.step()+ torch.testing.assert_close(on_cpu.detach(), ref_cpu.detach())+ assert scaler.get_scale() == 2.0**8+ else:+ for p, b in zip((on_cpu, on_cuda), before):+ assert torch.equal(p.detach().cpu(), b.cpu()), (+ overflow,+ "the overflow step changed weights",+ )+ assert float(opt.state[p]["step"]) == 0.0, (overflow, float(opt.state[p]["step"]))+ assert scaler.get_scale() == 2.0**7diff --git a/tests/test_tp_native_scaler_fused_gloo.py b/tests/test_tp_native_scaler_fused_gloo.pynew file mode 100644index 00000000000..e4cc3844f43--- /dev/null+++ b/tests/test_tp_native_scaler_fused_gloo.py@@ -0,0 +1,98 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""TP fused AdamW without clipping: an inf in one rank's shard must skip the step on every rank."""++import multiprocessing+import os++if multiprocessing.parent_process() is not None or getattr(+ multiprocessing.current_process(), "_inheriting", False+):+ os.environ["CUDA_VISIBLE_DEVICES"] = "" # spawned gloo ranks: CPU only, also on GPU hosts++import sys # noqa: E402+import types # noqa: E402++import pytest # noqa: E402+import torch # noqa: E402+from torch.distributed.tensor import Replicate, Shard # noqa: E402++sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))+from test_fsdp2_native_trainer_ckpt_gloo import _free_port # noqa: E402+++def _run(fused, unscale):+ from test_tp_native_oracle import QWEN3, _batch, build, tpn+ from torch.distributed.device_mesh import init_device_mesh++ rank = torch.distributed.get_rank()+ old = torch.amp.GradScaler("cpu", init_scale = 2.0**8)+ handler = types.SimpleNamespace(to_kwargs = lambda: {"device": "cpu", "init_scale": 2.0**8})+ trainer = types.SimpleNamespace(+ accelerator = types.SimpleNamespace(scaler = old, scaler_handler = handler)+ )+ tpn._tp_scaler(trainer, 1)+ scaler = trainer.accelerator.scaler+ assert scaler is not old+ model = build(QWEN3)+ mesh = init_device_mesh("cpu", (2,), mesh_dim_names = ("tp",))+ tpn.apply_tensor_parallel(model, mesh["tp"])+ model.train()+ params = [p for p in model.parameters() if p.requires_grad]+ # LoRA A is replicated and comes first: DTensor's own found_inf reduction reads only the first+ # gradient's placements, so it does not reduce over tp and the overflow stays on one rank.+ assert tuple(params[0].placements) == (Replicate(),), params[0].placements+ sharded = next(p for p in params if any(isinstance(x, Shard) for x in p.placements))+ opt = torch.optim.AdamW(params, lr = 1e-2, fused = fused, foreach = None if fused else False)+ ids = _batch(model)+ for step in range(3):+ out = model(input_ids = ids, labels = ids, use_cache = False)+ scaler.scale(out.loss).backward()+ if step == 1 and rank == 1:+ sharded.grad.to_local().view(-1)[0] = float("inf") # only rank 1's shard overflows+ if unscale:+ scaler.unscale_(opt)+ scaler.step(opt)+ scaler.update()+ opt.zero_grad(set_to_none = True)+ steps = float(opt.state[sharded]["step"])+ return steps, float(scaler.get_scale()), sharded.full_tensor().detach().clone()+++def _worker(rank, world, port, out):+ os.environ.update(MASTER_ADDR = "127.0.0.1", MASTER_PORT = str(port))+ torch.distributed.init_process_group("gloo", rank = rank, world_size = world)+ try:+ res = {}+ for fused in (False, True):+ for unscale in (False, True):+ try:+ res[fused, unscale] = _run(fused, unscale)+ except Exception as e:+ res[fused, unscale] = repr(e)+ torch.save(res, os.path.join(out, f"rank{rank}.pt"))+ finally:+ torch.distributed.destroy_process_group()++[email protected](scope = "module")+def ranks(tmp_path_factory):+ pytest.importorskip("peft")+ out = str(tmp_path_factory.mktemp("tp_scaler"))+ torch.multiprocessing.spawn(_worker, args = (2, _free_port(), out), nprocs = 2, join = True)+ return [torch.load(os.path.join(out, f"rank{r}.pt")) for r in range(2)]++[email protected]("unscale", [False, True], ids = ["no_clipping", "clipping"])[email protected]("fused", [False, True], ids = ["foreach", "fused"])+def test_tp_overflow_on_one_shard_skips_the_step_on_every_rank(ranks, fused, unscale):+ # Before: fused + no_clipping left rank 0 at 3 steps and scale 2**8 while rank 1 skipped (2, 2**7).+ outs = [res[fused, unscale] for res in ranks]+ for out in outs:+ assert not isinstance(out, str), out+ steps, scale, weight = out+ assert steps == 2.0, (fused, unscale, steps)+ assert scale == 2.0**7, (fused, unscale, scale)+ assert torch.isfinite(weight).all()+ torch.testing.assert_close(outs[0][2], outs[1][2], rtol = 0, atol = 0)diff --git a/unsloth/models/fsdp2_native.py b/unsloth/models/fsdp2_native.pyindex 2f5e8dd7c7a..a5ad76d4a90 100644--- a/unsloth/models/fsdp2_native.py+++ b/unsloth/models/fsdp2_native.py@@ -3337,6 +3337,15 @@ def install_native_fsdp2_hooks():return a and t+def _has_cpu_tensors(optimizer, grads = False):+ for group in optimizer.param_groups:+ for p in group["params"]:+ t = p.grad if grads else p+ if t is not None and t.device.type == "cpu":+ return True+ return False++def _scaler_for_native(trainer):"""float16: the GradScaler's inf check must agree over ranks, or one rank skips a step theothers take. ShardedGradScaler all-reduces it."""@@ -3345,6 +3354,7 @@ def _scaler_for_native(trainer):if scaler is None:return Falsetry:+ import torch.distributed as distfrom torch.distributed.fsdp.sharded_grad_scaler import ShardedGradScalerexcept ImportError:return False@@ -3358,12 +3368,80 @@ def _scaler_for_native(trainer):super().unscale_(optimizer)if not self.is_enabled():return- # The all-reduce SUMS found_inf: it is the number of ranks that overflowed. Fused- # optimizers skip the step only on found_inf == 1 (and subtract it from their step- # counts), so 2+ ranks overflowing stepped on inf / NaN gradients. Clamp to 0 / 1.+ self._one_flag(optimizer)++ def _check_inf_per_device(self, optimizer):+ # Fused step() without unscale_ (no clipping) lands here; ShardedGradScaler does not+ # all-reduce here, so ranks diverged on overflow skips. Agree over ranks.+ super()._check_inf_per_device(optimizer)state = self._per_optimizer_states[id(optimizer)]+ works, copies = [], []+ for found_inf in state["found_inf_per_device"].values():+ if self._device != "cpu" and found_inf.device.type == "cpu":+ on_device = found_inf.to(self._device)+ copies.append((found_inf, on_device))+ found_inf = on_device+ works.append(dist.all_reduce(found_inf, async_op = True, group = self.process_group))+ for work in works:+ work.wait()+ for found_inf, on_device in copies:+ found_inf.copy_(on_device)+ self._one_flag(optimizer)+ return state["found_inf_per_device"]++ def _unscale_grads_(+ self,+ optimizer,+ inv_scale,+ found_inf,+ allow_fp16 = True,+ ):+ # CPU grads get non_blocking copies of CUDA inv_scale / found_inf, read before they land.+ if inv_scale.device.type != "cpu" and _has_cpu_tensors(optimizer, grads = True):+ inv_scale, found_inf = inv_scale.cpu(), found_inf.cpu()+ return super()._unscale_grads_(optimizer, inv_scale, found_inf, allow_fp16)++ def step(self, optimizer, *args, **kwargs):+ # Same race for a fused optimizer's grad_scale / found_inf on CPU parameters: use host copies.+ if not (+ self.is_enabled()+ and self._device != "cpu"+ and getattr(optimizer, "_step_supports_amp_scaling", False)+ and _has_cpu_tensors(optimizer)+ ):+ return super().step(optimizer, *args, **kwargs)+ had = "step" in optimizer.__dict__+ previous = optimizer.__dict__.get("step")+ real = optimizer.step++ def step_with_host_flags(*a, **k):+ for name in ("grad_scale", "found_inf"):+ t = getattr(optimizer, name, None)+ if isinstance(t, torch.Tensor) and t.device.type != "cpu":+ setattr(optimizer, name, t.cpu())+ return real(*a, **k)++ optimizer.step = step_with_host_flags+ try:+ return super().step(optimizer, *args, **kwargs)+ finally:+ if had:+ optimizer.step = previous+ else:+ del optimizer.step++ def _one_flag(self, optimizer):+ # found_inf is summed over ranks and devices, but fused kernels skip only on == 1.+ # One 0 / 1 flag on the first device, zeros elsewhere (consumers sum them).+ state = self._per_optimizer_states[id(optimizer)]+ found = state["found_inf_per_device"]+ if not found:+ return+ first = next(iter(found.values()))+ total = sum(t.reshape(()).to(first.device) for t in found.values()).clamp(max = 1.0)state["found_inf_per_device"] = {- d: t.clamp(max = 1.0).reshape(()) for d, t in state["found_inf_per_device"].items()+ d: total if i == 0 else torch.zeros((), dtype = t.dtype, device = t.device)+ for i, (d, t) in enumerate(found.items())}kwargs = {}