Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
FSDP2: sum fp16 replicated gradients over unsynced micro-batches in float32 (#16173)
Daniel Hanff3f063a3a80
diff --git a/tests/test_fsdp2_replicated_grad_sync_gloo.py b/tests/test_fsdp2_replicated_grad_sync_gloo.pyindex ed81c787830..2142a3a3a47 100644--- a/tests/test_fsdp2_replicated_grad_sync_gloo.py+++ b/tests/test_fsdp2_replicated_grad_sync_gloo.py@@ -1,18 +1,6 @@# SPDX-License-Identifier: AGPL-3.0-only# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.-"""The replicated-gradient all_reduce (`fsdp2_save._sync_replicated_grads`), on a gloo CPU world of 2:--* gradients are bitwise equal to the previous implementation (fresh zeros buffer per sync, flags- always read back) over several steps, accumulated micro-steps, fp16 / fp32 / fp64 parameters and- rank-dependent None gradients;-* the flat buffer is reused per (device, dtype) and rebuilt when the parameter set or a size- changes, and no gradient aliases it (the next sync overwrites it);-* no device to host read (Tensor.tolist / item) nor host scalar item assignment when every local- gradient is present, exactly one read when some local gradient is None;-* end to end through reduce_unsharded_trainable_grads on a fully_shard model, for every- UNSLOTH_FSDP2_GA_SYNC mode (skip / defer / each / auto), replicated and sharded gradients are- bitwise equal to the previous implementation."""import multiprocessingimport os@@ -20,7 +8,7 @@ import osif multiprocessing.parent_process() is not None or getattr(multiprocessing.current_process(), "_inheriting", False):- os.environ["CUDA_VISIBLE_DEVICES"] = "" # spawned gloo ranks: CPU only, also on GPU hosts+ os.environ["CUDA_VISIBLE_DEVICES"] = "" # Gloo ranks must stay on CPU even on GPU hosts.import datetime # noqa: E402import importlib # noqa: E402@@ -40,7 +28,7 @@ def _reference_sync(dist,mesh = None,):- """The implementation before the buffer reuse / host-sync removal, verbatim."""+ """Pre-buffer-reuse implementation, retained as an independent reference."""from unsloth.models.fsdp2_save import mesh_all_reducegroups = {}@@ -77,8 +65,7 @@ def _save():class _Reads:- """Counts Tensor.tolist / Tensor.item calls and item assignment of a Python scalar (the device to- host reads, and host scalar copies, a CUDA run would sync on)."""+ """Count device-to-host reads and scalar assignments that synchronize CUDA."""def __init__(self):self.n = 0@@ -142,8 +129,7 @@ def _case_unit(rank):buffers = {}gen = torch.Generator().manual_seed(100 + rank)res = {"mismatch": [], "reads_all_present": [], "reads_some_none": [], "alias": 0}- # step pattern: which parameters get a local gradient this micro-step on this rank; rank 1- # leaves some None (a skipped branch), parameter 3 is never used on any rank in steps 0-1.+patterns = [[0, 1, 2, 4, 5] if rank == 0 else [0, 1, 2],[0, 1, 2, 4, 5],@@ -152,7 +138,7 @@ def _case_unit(rank):[0, 2] if rank == 1 else [0, 1, 2, 3, 4, 5],]for step, used in enumerate(patterns):- if step in (2, 4): # optimizer step boundary: zero_grad(set_to_none = True)+ if step in (2, 4):for p in old + new:p.grad = Nonefor i in used:@@ -161,7 +147,7 @@ def _case_unit(rank):if p.grad is None:p.grad = g.clone()else:- p.grad.add_(g) # accumulated micro-step+ p.grad.add_(g)_reference_sync(old, dist)everyone_all = all(p.grad is not None for p in new)with _Reads() as reads:@@ -179,7 +165,6 @@ def _case_unit(rank):res["n_buffers"] = len(buffers)ids_before = {k: id(v[1]) for k, v in buffers.items()}- # Buffer reused on an unchanged set, rebuilt when the set grows or a size changes.for p in old + new:p.grad = torch.ones_like(p)save._sync_replicated_grads(new, dist, None, buffers)@@ -199,7 +184,7 @@ def _case_unit(rank):res["fp64_reused"] = (id(buffers[("cpu", torch.float64)][1]) == ids_before[("cpu", torch.float64)])- # A size change at the same position (same parameter count).+old3, new3 = _make_params([((4,), torch.float64)]), _make_params([((4,), torch.float64)])for p in old3 + new3:p.grad = torch.full((4,), float(rank + 2), dtype = torch.float64)@@ -208,7 +193,7 @@ def _case_unit(rank):save._sync_replicated_grads(new3, dist, None, buffers)res["resized_same"] = _same(_grads(old3), _grads(new3))res["resized_rebuilt"] = id(buffers[("cpu", torch.float64)][1]) != before64- # No buffers dict (direct callers): still correct.+old4, new4 = _make_params(spec), _make_params(spec)for p in old4 + new4:p.grad = torch.full(p.shape, float(rank + 1)).to(p.dtype)@@ -221,9 +206,6 @@ def _case_unit(rank):class _Net(torch.nn.Module):- """Sharded fp32 body plus replicated (ignored) parameters of mixed dtype, one used only on some- micro-steps of some ranks."""-def __init__(self):super().__init__()torch.manual_seed(0)@@ -256,10 +238,6 @@ GA = 4def _train(save, native, rank, mode, impl):- """3 optimizer steps x GA micro-batches, non-final ones under fsdp2_native._no_grad_sync.- impl: "old" = the previous code (reference sync on every backward), "nodefer" = the new sync on- every backward, "new" = as shipped (the sync follows FSDP2's gradient sync flag), "oracle" = no- sync in backward, the reference sync once after the last micro-batch."""import torch.distributed as distos.environ["UNSLOTH_FSDP2_GA_SYNC"] = mode@@ -272,10 +250,10 @@ def _train(save, native, rank, mode, impl):save.mesh_all_reduce = counting_all_reduceif impl == "old":- save._sync_replicated_grads = lambda params, d, mesh = None, buffers = None: _reference_sync(+ save._sync_replicated_grads = lambda params, d, mesh = None, *_: _reference_sync(params, d, mesh)- if impl in ("old", "nodefer"):+ if impl in ("old", "nodefer", "oracle"):save._fsdp_grad_sync_deferred = lambda *a: Falseif impl == "oracle":save._sync_replicated_grads = lambda *a, **k: None@@ -288,7 +266,7 @@ def _train(save, native, rank, mode, impl):for step in range(3):for micro in range(GA):x = torch.randn(4, 8, generator = gen)- # step 0: `branch` unused everywhere; later only some micro-batches of some ranks.+use_branch = (micro + rank + step) % 3 == 0 and step != 0calls.append(0)if micro == GA - 1:@@ -297,7 +275,7 @@ def _train(save, native, rank, mode, impl):with native._no_grad_sync(model):model(x, use_branch).backward()if impl == "oracle":- _reference_sync(rep, dist) # counted on the last micro-batch+ _reference_sync(rep, dist)grads = {}for n, p in model.named_parameters():g = p.grad@@ -312,13 +290,23 @@ def _train(save, native, rank, mode, impl):save._sync_replicated_grads, save._fsdp_grad_sync_deferred, save.mesh_all_reduce = saved-def _diff(a_steps, b_steps):+def _diff(+ a_steps,+ b_steps,+ skip = (),+):bad, worst = [], 0.0for step, (a, b) in enumerate(zip(a_steps, b_steps)):for n in a:+ if n in skip:+ continuex, y = a[n], b[n]if (x is None) != (y is None):bad.append((step, n, "none"))+ worst = float("inf")+ elif x is not None and (not torch.isfinite(x).all() or not torch.isfinite(y).all()):+ bad.append((step, n, "nonfinite"))+ worst = float("inf")elif x is not None and not torch.equal(x, y):bad.append((step, n))worst = max(worst, (x.double() - y.double()).abs().max().item())@@ -339,11 +327,106 @@ def _case_modes(rank):r["calls"] = {impl: v[2] for impl, v in runs.items()}r["nodefer_vs_old"] = _diff(runs["old"][1], runs["nodefer"][1])[0]r["new_vs_old"], r["new_vs_old_max"] = _diff(runs["old"][1], runs["new"][1])- r["new_vs_oracle"] = _diff(runs["oracle"][1], runs["new"][1])[0]+ # Wider fp16 sums change later updates, so only unaffected first-step gradients are bitwise.++ r["new_vs_oracle"] = _diff(runs["oracle"][1][:1], runs["new"][1][:1], skip = ("norm16",))[0]+ r["new_vs_oracle_max"] = _diff(runs["oracle"][1], runs["new"][1])[1]res[mode] = rreturn res+class _Fp16Rep(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.body = torch.nn.Linear(4, 4)+ self.rep = torch.nn.Parameter(torch.zeros(4, dtype = torch.float16))+ self.fail_forward = False++ def forward(+ self,+ x,+ scale,+ use_rep = True,+ ):+ loss = self.body(x).mean()+ if self.fail_forward:+ raise RuntimeError("injected forward failure")+ return loss + (self.rep.float() * scale).sum() if use_rep else loss+++def _case_fp16_overflow(rank):+ """Local 80000 overflows fp16 before averaging with the other rank to 60000."""+ from torch.distributed.device_mesh import init_device_mesh+ from torch.distributed.fsdp import fully_shard++ native = _load_native()+ save = _save()+ os.environ["UNSLOTH_FSDP2_GA_SYNC"] = "defer"+ torch.manual_seed(0)+ model = _Fp16Rep()+ mesh = init_device_mesh("cpu", (2,))+ fully_shard(model.body, mesh = mesh)+ fully_shard(model, mesh = mesh, ignored_params = {model.rep})+ names = save.reduce_unsharded_trainable_grads(model)+ scale = 40000.0 if rank == 0 else 20000.0+ x = torch.randn(2, 4)+ with native._no_grad_sync(model):+ model(x, scale).backward()+ model(x, scale).backward()+ return {"names": names, "grad": model.rep.grad.float().tolist(), "file": save.__file__}+++def _case_fp16_edge(rank, mode):+ from torch.distributed.device_mesh import init_device_mesh+ from torch.distributed.fsdp import fully_shard++ native = _load_native()+ save = _save()+ os.environ["UNSLOTH_FSDP2_GA_SYNC"] = "defer"+ torch.manual_seed(0)+ model = _Fp16Rep()+ if mode == "wide":+ model.rep.grad_dtype = torch.float32+ mesh = init_device_mesh("cpu", (2,))+ fully_shard(model.body, mesh = mesh)+ fully_shard(model, mesh = mesh, ignored_params = {model.rep})+ save.reduce_unsharded_trainable_grads(model)+ owner = model+ if mode == "outer_recovery":++ class Outer(torch.nn.Module):+ def __init__(self, inner):+ super().__init__()+ self.inner = inner++ def forward(self, *args, **kwargs):+ return self.inner(*args, **kwargs)++ owner = Outer(model)+ fully_shard(owner, mesh = mesh)+ save.reduce_unsharded_trainable_grads(owner)+ x = torch.randn(2, 4)+ with native._no_grad_sync(owner):+ owner(x, 4.0).backward()+ if mode == "wide":+ owner(x, 2.0, use_rep = False).backward()+ else:+ model.fail_forward = True+ with pytest.raises(RuntimeError, match = "injected forward failure"):+ owner(x, 9.0)+ owner.reset_iter_state()+ owner.zero_grad(set_to_none = True)+ model.fail_forward = False+ owner(x, 2.0).backward()+ state = model.__dict__["_unsloth_replicated_grad_state"]+ return {+ "grad": model.rep.grad.tolist(),+ "dtype": str(model.rep.grad.dtype),+ "pending": bool(state.get("pending")),+ "queued": state["queued"],+ }++def _worker(rank, world, port, out, case):os.environ.update(MASTER_ADDR = "127.0.0.1",@@ -360,7 +443,8 @@ def _worker(rank, world, port, out, case):"gloo", rank = rank, world_size = world, timeout = datetime.timedelta(seconds = 60))try:- res = _case_unit(rank) if case == "unit" else _case_modes(rank)+ cases = {"unit": _case_unit, "modes": _case_modes, "fp16_overflow": _case_fp16_overflow}+ res = _case_fp16_edge(rank, case[5:]) if case.startswith("edge_") else cases[case](rank)out.put((rank, res))except Exception as exc:import traceback@@ -401,7 +485,7 @@ def test_replicated_sync_bitwise_buffer_reuse_and_no_host_read():for rank, r in _run("unit").items():assert r["mismatch"] == [], (rank, r)assert r["alias"] == 0, (rank, r)- # Steps 2 and 3 (and 4 on rank 0) have every local gradient; the rest leave some None.+assert r["reads_all_present"] == [0] * (3 if rank == 0 else 2), (rank, r)assert r["reads_some_none"] == [1] * (2 if rank == 0 else 3), (rank, r)assert r["n_buffers"] == 2 and all(r["reused"].values()), (rank, r)@@ -415,18 +499,46 @@ def test_replicated_sync_follows_the_ga_sync_flag():for mode, m in r.items():assert m["names"] == ["branch", "norm", "norm16"], (rank, mode, m)assert m["branch_none_step0"], (rank, mode, m)- # The new sync on every backward is bitwise the previous code (buffer / host-read change).+assert m["nodefer_vs_old"] == [], (rank, mode, m)- # Replicated all_reduces per micro-batch (GA 4, 3 steps): the previous code syncs every- # backward; shipped code only on the synced one, unless the mode syncs every micro-batch.+assert m["calls"]["old"] == [1] * (3 * GA), (rank, mode, m)- deferred = mode != "each" # auto without a fit layout = skip+ deferred = mode != "each" # Auto without a fit layout defers synchronization.want = ([0] * (GA - 1) + [1]) * 3 if deferred else [1] * (3 * GA)assert m["calls"]["new"] == want, (rank, mode, m)assert m["calls"]["oracle"] == ([0] * (GA - 1) + [1]) * 3, (rank, mode, m)if deferred:- # Bitwise the accumulated local sum averaged once over the ranks.assert m["new_vs_oracle"] == [], (rank, mode, m)+ assert m["new_vs_oracle_max"] < 1e-2, (rank, mode, m)assert m["new_vs_old_max"] < 1e-2, (rank, mode, m)else:assert m["new_vs_old"] == [], (rank, mode, m)+++def test_fp16_replica_local_sum_over_micro_batches_does_not_overflow():+ for rank, r in _run("fp16_overflow").items():+ assert r["names"] == ["rep"], (rank, r)+ assert r["grad"] == [60000.0] * 4, (rank, r)++[email protected]("mode", ["wide", "recovery", "outer_recovery"])+def test_fp16_replica_gradient_dtype_and_recovery(mode):+ FSDPModule = _save()._fsdp_module_class()+ if mode == "wide" and not hasattr(torch.Tensor, "grad_dtype"):+ pytest.skip("torch has no explicit gradient dtype")+ if mode != "wide" and not hasattr(FSDPModule, "reset_iter_state"):+ pytest.skip("torch has no iteration recovery API")+ for rank, r in _run("edge_" + mode).items():+ assert r["grad"] == [4.0 if mode == "wide" else 2.0] * 4, (rank, mode, r)+ assert r["dtype"] == ("torch.float32" if mode == "wide" else "torch.float16"), r+ assert not r["pending"] and not r["queued"], r+++ "left,right", [(1.0, None), (None, 1.0), (1.0, float("nan")), (float("inf"), float("inf"))]+)+def test_gradient_comparison_rejects_missing_and_nonfinite(left, right):+ a = None if left is None else torch.tensor([left])+ b = None if right is None else torch.tensor([right])+ bad, worst = _diff([{"rep": a}], [{"rep": b}])+ assert bad and worst == float("inf")diff --git a/tests/test_pp_fsdp2_gloo.py b/tests/test_pp_fsdp2_gloo.pyindex c9f857f2b55..d516a37a9fd 100644--- a/tests/test_pp_fsdp2_gloo.py+++ b/tests/test_pp_fsdp2_gloo.py@@ -1,16 +1,15 @@# SPDX-License-Identifier: AGPL-3.0-only# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.-"""Pipeline parallel composed with native FSDP2 data parallel and / or tensor parallel on a gloo CPU world-(pp 2 x dp 2, pp 2 x tp 2, pp 2 x dp 2 x tp 2) vs the same model in one process on the global batch, fp32."""+import functoolsimport multiprocessingimport osif multiprocessing.parent_process() is not None or getattr(multiprocessing.current_process(), "_inheriting", False):- os.environ["CUDA_VISIBLE_DEVICES"] = "" # spawned gloo ranks: CPU only, also on GPU hosts+ os.environ["CUDA_VISIBLE_DEVICES"] = "" # Gloo ranks must stay on CPU even on GPU hosts.import sys # noqa: E402import tempfile # noqa: E402@@ -25,23 +24,21 @@ from test_tp_native_oracle import tpn # noqa: E402LLAMA = "hf-internal-testing/tiny-random-LlamaForCausalLM"QWEN3 = "trl-internal-testing/tiny-Qwen3ForCausalLM"GEMMA3 = "hf-internal-testing/tiny-random-Gemma3ForCausalLM"-# Overrides so the head counts split over tp = 2 (as the TP suite).+ARCHS = {"llama": (LLAMA, {}),"qwen3": (QWEN3, {}),"gemma3": (GEMMA3, {"num_attention_heads": 4, "num_key_value_heads": 2}),}+_LOCAL_CONFIG = FalseN_LAYERS = 6STEPS = 3-# Step-0 grad norm tolerance vs the one-process reference. Gemma3's own fp32 grad norm sits 1.0e-4 from fp64 on one-# process (Llama 7.8e-7), so any change of summation order (TP splits, CPU ISA) moves it by that much.+# Gemma3 reduction-order noise needs a wider bound than Llama/Qwen3.+STEP0_NORM_REL = {"llama": 1e-5, "qwen3": 1e-5, "gemma3": 3e-4}-# Step-0 grad tolerances: per element relative to the tensor's largest entry, and relative L2 over all-# grads. Gemma3's fp32 reordering noise under tensor parallel depends on the CPU's kernels: worst-# element / relative L2 2.8e-5 / 2.4e-5 with this host's AVX-512 MKL, 1.6e-4 / 1.4e-4 with MKL and-# oneDNN held to AVX2, and 2e-4 per element on a CI runner, against 2.6e-6 for llama and 5.8e-6 for-# qwen3 on the same AVX2 run (pp x dp gemma3 stays near 4e-7). llama and qwen3 keep the strict bounds-# on the same tensor parallel code; a gemma3-specific fault (a wrong shard, a missed reduction) is O(1).+# CPU ISA and TP summation order move near-zero Gemma3 gradients; keep other models strict.++GRAD_ATOL_REL = {"llama": 2e-5, "qwen3": 2e-5, "gemma3": 1e-3}GRAD_L2_REL = {"llama": 5e-5, "qwen3": 5e-5, "gemma3": 5e-4}@@ -62,7 +59,7 @@ def build(from peft import LoraConfig, get_peft_modelfrom transformers import AutoConfig, AutoModelForCausalLM- probe = AutoConfig.from_pretrained(repo)+ probe = AutoConfig.from_pretrained(repo, local_files_only = _LOCAL_CONFIG)values = probe.to_dict()values["num_hidden_layers"] = N_LAYERSvalues.pop("layer_types", None)@@ -97,9 +94,7 @@ def global_batch(g = torch.Generator().manual_seed(seed)ids = torch.randint(0, vocab, (bsz, seq), generator = g)labels = ids.clone()- labels[- 0, :3- ] = -100 # uneven token counts per dp replica / micro-batch: the global count matters+ labels[0, :3] = -100 # Uneven token counts exercise global normalization.labels[-1, -2:] = -100labels[1, -1] = -100return ids, labels@@ -143,7 +138,6 @@ def _trainable_param_groups(model, trainable = True):def _count_reduces():- """Wrap FSDP2's gradient reduce (one call per param group per synced backward)."""import torch.distributed.fsdp._fully_shard._fsdp_param_group as pgcalls = [0]@@ -176,8 +170,7 @@ def _case_compose(fit_layout = None,rows = 4,):- """apply_pipeline_parallel over mesh['pp'], then apply_tensor_parallel over mesh['tp'] and / or shard_model- over mesh['dp']. Each dp replica trains on its rows of the global batch."""+ """Split over pp before applying orthogonal tp/dp meshes; each dp rank gets its batch slice."""import torch.distributed as distfrom safetensors.torch import load_filefrom torch.distributed.device_mesh import init_device_mesh@@ -197,30 +190,33 @@ def _case_compose(model = build(repo, overrides)pp = ppn.apply_pipeline_parallel(model, mesh["pp"], n_microbatches = n_mb, schedule = schedule)refusals = {}- if (- world != dims["pp"]- ): # a pp mesh smaller than the world: the other dimensions must be applied first+ if world != dims["pp"]:refusals["step_before_all_dims"] = _raised(lambda: pp.step(ids[mine], labels[mine]))if "tp" in dims:tpn.apply_tensor_parallel(model, mesh["tp"])if "dp" in dims:refusals["shard_model_pp_mesh"] = _raised(lambda: native.shard_model(- model, native.FSDP2Config(param_dtype = torch.float32, mesh = mesh["pp"])+ model, native.FSDP2Config(device = "cpu", param_dtype = torch.float32, mesh = mesh["pp"])))refusals["shard_model_no_mesh"] = _raised(- lambda: native.shard_model(model, native.FSDP2Config(param_dtype = torch.float32))+ lambda: native.shard_model(+ model, native.FSDP2Config(device = "cpu", param_dtype = torch.float32)+ ))refusals["shard_model_inner_level"] = _raised(lambda: native.shard_model(- model.base_model, native.FSDP2Config(param_dtype = torch.float32, mesh = mesh["dp"])+ model.base_model,+ native.FSDP2Config(device = "cpu", param_dtype = torch.float32, mesh = mesh["dp"]),))refusals["shard_model_cpu_offload"] = _raised(lambda: native.shard_model(model,- native.FSDP2Config(param_dtype = torch.float32, mesh = mesh["dp"], cpu_offload = True),+ native.FSDP2Config(+ device = "cpu", param_dtype = torch.float32, mesh = mesh["dp"], cpu_offload = True+ ),))refusals["fsdp_after_refusals"] = sum(@@ -228,7 +224,9 @@ def _case_compose()native.shard_model(model,- native.FSDP2Config(param_dtype = torch.float32, mesh = mesh["dp"], fit_layout = fit_layout),+ native.FSDP2Config(+ device = "cpu", param_dtype = torch.float32, mesh = mesh["dp"], fit_layout = fit_layout+ ),)from torch.distributed.fsdp import FSDPModule@@ -267,7 +265,6 @@ def _case_compose("owned": {n for n, p in model.named_parameters() if p.requires_grad},"reduces": reduces,"groups": len(groups),- # torchtitan's PP default: no unit reshards after forward (each micro-batch would all-gather again)."reshard_after_forward": [g.post_forward_mesh_info is not Nonefor g in _trainable_param_groups(model, trainable = False)@@ -286,7 +283,6 @@ def _case_compose(def _case_scaler(rank, world, tmp):- """pp 2 x dp 2: an inf on one rank's (last stage, dp replica 1) gradient skips the step on every rank."""from torch.distributed.device_mesh import init_device_meshnative = _native()@@ -295,15 +291,17 @@ def _case_scaler(rank, world, tmp):ids, labels = global_batch(model, 8)dp_rank = mesh["dp"].get_local_rank()pp = ppn.apply_pipeline_parallel(model, mesh["pp"], n_microbatches = 2)- native.shard_model(model, native.FSDP2Config(param_dtype = torch.float32, mesh = mesh["dp"]))+ native.shard_model(+ model, native.FSDP2Config(device = "cpu", param_dtype = torch.float32, mesh = mesh["dp"])+ )scaler = torch.amp.GradScaler("cpu", init_scale = 1024.0)opt = torch.optim.SGD(pp.parameters(), lr = 1.0)pp.step(ids[4 * dp_rank : 4 * dp_rank + 4], labels[4 * dp_rank : 4 * dp_rank + 4], scaler = scaler)if rank == world - 1:- # One rank (last stage, dp replica 1) finds an inf in its unscale (as a replicated gradient would);- # set on its found_inf directly: an inf in a DTensor shard is already reduced over dp by GradScaler.+ # Inject found_inf directly: DTensor shard infinities already propagate over dp.+unscale = scaler.unscale_def unscale_with_inf(optimizer):@@ -324,7 +322,6 @@ def _case_scaler(rank, world, tmp):def _case_order_refusals(rank, world, tmp):- """TP or FSDP2 before the pipeline split, and a non-composable pp mesh, are refused before any change."""from torch.distributed.device_mesh import init_device_meshnative = _native()@@ -336,15 +333,19 @@ def _case_order_refusals(rank, world, tmp):out["tp_first"] = _raised(lambda: ppn.apply_pipeline_parallel(model, mesh["pp"]))out["tp_first_unchanged"] = not ppn.is_pipeline_parallel(model) and layers == N_LAYERSmodel = build(LLAMA)- native.shard_model(model, native.FSDP2Config(param_dtype = torch.float32, mesh = mesh["dp"]))+ native.shard_model(+ model, native.FSDP2Config(device = "cpu", param_dtype = torch.float32, mesh = mesh["dp"])+ )out["fsdp_first"] = _raised(lambda: ppn.apply_pipeline_parallel(model, mesh["pp"]))out["fsdp_first_unchanged"] = not ppn.is_pipeline_parallel(model)- # pp mesh over all 4 ranks = world: a plain v1 split; a 2-D mesh is refused.+model = build(LLAMA)out["mesh_2d"] = _raised(lambda: ppn.apply_pipeline_parallel(model, mesh))- # pp x dp: TP after shard_model is refused.+pp = ppn.apply_pipeline_parallel(model, mesh["pp"])- native.shard_model(model, native.FSDP2Config(param_dtype = torch.float32, mesh = mesh["dp"]))+ native.shard_model(+ model, native.FSDP2Config(device = "cpu", param_dtype = torch.float32, mesh = mesh["dp"])+ )out["tp_after_fsdp"] = _raised(lambda: tpn.apply_tensor_parallel(model, mesh["dp"]))out["shard_twice_is_noop"] = native.shard_model(model) is modelout["composed"] = pp.composed@@ -352,7 +353,6 @@ def _case_order_refusals(rank, world, tmp):def _case_pp_tp_mesh_refusal(rank, world, tmp):- """pp 2 x tp 2: a tp mesh that is the pp mesh itself is refused before anything is sharded."""from torch.distributed.device_mesh import init_device_meshmesh = init_device_mesh("cpu", (2, 2), mesh_dim_names = ("pp", "tp"))@@ -362,11 +362,43 @@ def _case_pp_tp_mesh_refusal(rank, world, tmp):return {"err": err, "tp": tpn.is_tensor_parallel(model)}+def _case_fp16_replicas(rank, world, tmp):+ from torch.distributed.device_mesh import init_device_mesh++ native = _native()+ mesh = init_device_mesh("cpu", (2, 2), mesh_dim_names = ("pp", "dp"))+ model = build(LLAMA)+ norm = model.get_base_model().model.norm.weight+ norm.data = norm.data.half()+ norm.requires_grad_(True)+ ids, labels = global_batch(model, 8)+ model(input_ids = ids, labels = labels, use_cache = False).loss.backward()+ expected = norm.grad.detach().float().clone()+ model.zero_grad(set_to_none = True)+ pp = ppn.apply_pipeline_parallel(model, mesh["pp"], n_microbatches = 2)+ native.shard_model(+ model,+ native.FSDP2Config(+ device = "cpu", param_dtype = torch.float32, mesh = mesh["dp"], ignored_params = {norm}+ ),+ )+ dp_rank = mesh["dp"].get_local_rank()+ grads, pending = [], []+ for _ in range(2):+ pp.step(ids[4 * dp_rank : 4 * dp_rank + 4], labels[4 * dp_rank : 4 * dp_rank + 4])+ if pp.is_last:+ grads.append(None if norm.grad is None else norm.grad.detach().float().clone())+ pending.append(bool(model.__dict__["_unsloth_replicated_grad_state"].get("pending")))+ model.zero_grad(set_to_none = True)+ return {"last": pp.is_last, "grads": grads, "expected": expected, "pending": pending}++CASES = {"compose": _case_compose,"scaler": _case_scaler,"order": _case_order_refusals,"pp_tp_mesh": _case_pp_tp_mesh_refusal,+ "fp16_replicas": _case_fp16_replicas,}@@ -376,6 +408,8 @@ def _free_port():def _worker(rank, world, port, case, tmp, kwargs):+ global _LOCAL_CONFIG+ _LOCAL_CONFIG = Trueos.environ.update(MASTER_ADDR = "127.0.0.1",MASTER_PORT = str(port),@@ -399,6 +433,13 @@ def _worker(rank, world, port, case, tmp, kwargs):dist.destroy_process_group()[email protected]_cache(maxsize = 1)+def _prefetch_configs():+ from transformers import AutoConfig+ for repo, _ in ARCHS.values():+ AutoConfig.from_pretrained(repo)++def _spawn(case,world = 4,@@ -411,6 +452,7 @@ def _spawn(native = _native()if not native._supports_ignored_params(native._fsdp_api()[2]):pytest.skip("native FSDP2 composition needs fully_shard(ignored_params) (torch 2.7+)")+ _prefetch_configs()ctx = multiprocessing.get_context("spawn")port = _free_port()with tempfile.TemporaryDirectory() as tmp:@@ -451,8 +493,8 @@ def _check(res["norms"],ref["norms"],)- # Later norms follow 1-2 AdamW steps (lr 1e-2), whose first updates are ~lr * sign(grad): fp32 summation- # order on near-zero grads moves them up to ~5e-4 (Gemma3; the pp-only split alone moves it 2.6e-4).+ # AdamW updates amplify reduction-order noise in near-zero gradients.+assert res["norms"] == pytest.approx(ref["norms"], rel = 2e-3), (rank,res["norms"],@@ -462,7 +504,7 @@ def _check(num = den = 0.0for name, grad in res["grads"].items():want = ref["grads"][name]- # Tensor parallel row-parallel sums reorder fp32 additions: tolerance relative to the tensor's scale.+atol = max(1e-6, grad_rel * float(want.abs().max()))torch.testing.assert_close(grad, want, rtol = 1e-4, atol = atol, msg = lambda m, name = name: f"{name}: {m}"@@ -478,7 +520,7 @@ def _check(), refusalsif "dp" in dims:assert res["stage_root_fsdp"]- # FSDP2 shards the trainable weights over dp; each group reduces once per step, not per micro-batch.+assert res["placements"] and all(any("S(" in pl or "Shard" in pl for pl in pls) for pls in res["placements"].values()), res["placements"]@@ -509,7 +551,7 @@ def _check(assert set(saved_pp) == set(saved_ref), sorted(set(saved_pp) ^ set(saved_ref))[:6]for key, value in saved_ref.items():assert saved_pp[key].shape == value.shape, key- # 3 AdamW steps (lr 1e-2) carry fp32 summation order differences (as the pp-only suite).+torch.testing.assert_close(saved_pp[key], value, rtol = 1e-3, atol = 2e-3, msg = key)@@ -526,7 +568,6 @@ def test_pp2_x_fsdp2_dp2_matches_one_process(arch):def test_pp2_x_fsdp2_dp2_1f1b_sharded_layout_matches_one_process():- # 1F1B, 4 micro-batches, the sharded layout (one FSDP2 unit per owned decoder layer, frozen weights sharded).dims = {"pp": 2, "dp": 2}_check(_spawn("compose", arch = "llama", dims = dims, schedule = "1f1b", n_mb = 4, fit_layout = False),@@ -548,7 +589,6 @@ def test_pp2_x_tp2_matches_one_process(arch):@pytest.mark.parametrize("other", ["dp", "tp"])def test_pp2_x_size_one_dp_or_tp_matches_one_process(other):- # (pp 2, dp 1) / (pp 2, tp 1) on 2 ranks: FSDP2 / TP DTensors on a 1-rank mesh, the shape the 2-GPU proof runs.dims = {"pp": 2, other: 1}_check(_spawn("compose", world = 2, arch = "llama", dims = dims), dims)@@ -565,6 +605,17 @@ def test_pp2_x_dp2_grad_scaler_skip_is_global():assert res["scale"] == 512.0, res+def test_pp2_x_fsdp2_dp2_consumes_fp16_replicated_gradients():+ for res in _spawn("fp16_replicas"):+ if not res["last"]:+ continue+ assert res["pending"] == [False, False], res+ assert res["expected"].norm() > 0+ for grad in res["grads"]:+ assert grad is not None, res+ torch.testing.assert_close(grad, res["expected"], rtol = 1e-2, atol = 1e-4)++def test_pp_composition_order_and_meshes_are_refused_before_any_change():for res in _spawn("order"):assert "must come before apply_tensor_parallel" in (res["tp_first"] or ""), resdiff --git a/unsloth/models/fsdp2_save.py b/unsloth/models/fsdp2_save.pyindex 545f6713ee4..f83981edf35 100644--- a/unsloth/models/fsdp2_save.py+++ b/unsloth/models/fsdp2_save.py@@ -1284,18 +1284,15 @@ def _sync_replicated_grads(dist,mesh = None,buffers = None,+ pending = None,):- """Average gradients over `mesh` (DDP-style), one all_reduce per (device, reduce dtype), entered- by every rank: a missing local gradient counts as zeros, and a "has a gradient" flag keeps grad- None for a parameter no rank used. Sums in float32 (fp16 sums overflow). The summed flags are- read (a host sync) only when some local gradient is None. `buffers` caches one flat buffer per- (device, dtype); gradients never alias it."""+ """Average in float32; absent ranks contribute zero, globally unused grads stay None."""groups = {}for p in params:dtype = torch.float64 if p.dtype == torch.float64 else torch.float32groups.setdefault((str(p.device), dtype), []).append(p)for (device, dtype), group in groups.items():- # A TP DTensor outside FSDP2: dp ranks hold the same tp slice, so average the local shard.+ # Across dp, an external TP DTensor represents matching local slices.sizes = [_local_tensor(p).numel() for p in group]total = sum(sizes)key = tuple((id(p), n) for p, n in zip(group, sizes))@@ -1307,18 +1304,24 @@ def _sync_replicated_grads(if buffers is not None:buffers[(device, dtype)] = (key, flat)grads = [_local_tensor(p.grad) for p in group]- local_all = all(g is not None for g in grads)+ extra = [pending.get(id(p)) if pending else None for p in group]+ local_all = all(g is not None or e is not None for g, e in zip(grads, extra))flags = flat[total:]if local_all:flags.fill_(1)else:flags.zero_()offset = 0- for i, (g, n) in enumerate(zip(grads, sizes)):- if g is not None:- flat[offset : offset + n].copy_(g.detach().reshape(-1))+ for i, (g, e, n) in enumerate(zip(grads, extra, sizes)):+ if g is not None or e is not None:+ if g is None:+ flat[offset : offset + n].copy_(e.reshape(-1))+ else:+ flat[offset : offset + n].copy_(g.detach().reshape(-1))+ if e is not None:+ flat[offset : offset + n].add_(e.reshape(-1))if not local_all:- # Not flags[i] = 1: that is a host sync on CUDA.+ # Python scalar assignment would synchronize CUDA.flags[i].fill_(1)else:flat[offset : offset + n].zero_()@@ -1327,8 +1330,8 @@ def _sync_replicated_grads(flat[:total].div_(world)used = None if local_all else flags.tolist()offset = 0- for i, (p, g, n) in enumerate(zip(group, grads, sizes)):- if g is not None or used[i] > 0:+ for i, (p, g, e, n) in enumerate(zip(group, grads, extra, sizes)):+ if g is not None or e is not None or used[i] > 0:value = flat[offset : offset + n].view(_local_tensor(p).shape)if g is None:p.grad = _like_param(p, value.to(p.dtype, copy = True))@@ -1427,11 +1430,7 @@ def _fsdp_grad_sync_deferred(model, cache):def reduce_unsharded_trainable_grads(model):- """Average gradients of trainable params FSDP2 does not manage (e.g. `ignored_modules`) over the- mesh, else replicas drift. Runs once per backward as an autograd engine callback on every rank;- a per-parameter hook would hang when a rank skips the parameter (branch, routed MoE expert).- Skipped while FSDP2 grad sync is off (GA); the first synced backward averages the accumulated- sum. Returns the names."""+ """Queue one fixed-order reduction per backward on every rank; per-param hooks can deadlock."""dist = _dist()DTensor = _dtensor_class()if dist is None or DTensor is None:@@ -1447,10 +1446,11 @@ def reduce_unsharded_trainable_grads(model):def _sync():state["queued"] = False- # Every rank enters the same branch: the GA flag is set alike on all ranks.+if _fsdp_grad_sync_deferred(model, state):+ _stash_fp16_grads(params, state)return- _sync_replicated_grads(params, dist, mesh, buffers)+ _sync_replicated_grads(params, dist, mesh, buffers, state.pop("pending", None))def _queue(*_):if not state["queued"]:@@ -1462,7 +1462,6 @@ def reduce_unsharded_trainable_grads(model):if id(param) in seen or not param.requires_grad:continueif isinstance(param, DTensor):- # FSDP2 does not reduce TP DTensors in ignored_params; average them here.if managed is False:managed = _fsdp_managed_param_ids(model)if managed is None or id(param) in managed or not _tp_shard_outside(param, mesh, dist):@@ -1478,7 +1477,7 @@ def reduce_unsharded_trainable_grads(model):def _pre_forward(module, args):if not _in_backward():- state["queued"] = False # a backward that raised never ran its callback+ state["queued"] = False # Failed backwards never ran their queued callback.def _post_forward(module, args, output):if not torch.is_grad_enabled():@@ -1495,4 +1494,35 @@ def reduce_unsharded_trainable_grads(model):model.register_forward_pre_hook(_pre_forward)model.register_forward_hook(_post_forward)model.__dict__["_unsloth_replicated_grad_hooked"] = True+ _install_replicated_grad_reset(model)return names+++def _stash_fp16_grads(params, state):+ pending = state.setdefault("pending", {})+ for p in params:+ if p.dtype != torch.float16 or p.grad is None or p.grad.dtype != torch.float16:+ continue+ g = _local_tensor(p.grad).detach().float()+ prev = pending.get(id(p))+ pending[id(p)] = g if prev is None else prev.add_(g)+ p.grad = None+++def _install_replicated_grad_reset(model):+ reset = getattr(model, "reset_iter_state", None)+ if reset is None or getattr(reset, "_unsloth_replicated_reset", False):+ return++ @functools.wraps(reset)+ def reset_replicas(*args, **kwargs):+ result = reset(*args, **kwargs)+ for module in model.modules():+ state = module.__dict__.get("_unsloth_replicated_grad_state")+ if state is not None:+ state.pop("pending", None)+ state["queued"] = False+ return result++ reset_replicas._unsloth_replicated_reset = True+ model.reset_iter_state = reset_replicasdiff --git a/unsloth/models/pp_native.py b/unsloth/models/pp_native.pyindex 5bbf6453167..7e4d313b7a4 100644--- a/unsloth/models/pp_native.py+++ b/unsloth/models/pp_native.py@@ -1444,11 +1444,14 @@ class PipelineParallel:policy = MixedPrecisionPolicy(param_dtype = None,reduce_dtype = getattr(mp_policy, "reduce_dtype", None),- cast_forward_inputs = False, # the received hidden states keep PPConfig.activation_dtype+ cast_forward_inputs = False,)fully_shard(self.module, mesh = dp_mesh, mp_policy = policy, reshard_after_forward = False)+ from .fsdp2_save import _install_replicated_grad_reset++ _install_replicated_grad_reset(self.module)self.dp_mesh = dp_mesh- self._shapes.clear() # stages built before hold the unwrapped module+ self._shapes.clear()@propertydef is_first(self):@@ -1774,7 +1777,11 @@ class PipelineParallel:state = self.model.__dict__.setdefault("_unsloth_replicated_grad_state", {"queued": False})_sync_replicated_grads(- params, dist, model_mesh(self.model) or self.dp_mesh, state.setdefault("buffers", {})+ params,+ dist,+ model_mesh(self.model) or self.dp_mesh,+ state.setdefault("buffers", {}),+ state.pop("pending", None),)def _grad_copies(self, grad):