Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

FSDP2 native: fewer collectives under gradient accumulation and checkpoint recompute (#15023)

Daniel Han96f60dab1fdc
diff --git a/tests/test_fsdp2_native_ga_sync_gloo.py b/tests/test_fsdp2_native_ga_sync_gloo.py
new file mode 100644
index 00000000000..5ab8747b450
--- /dev/null
+++ b/tests/test_fsdp2_native_ga_sync_gloo.py
@@ -0,0 +1,155 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""Native FSDP2 gradient accumulation modes (UNSLOTH_FSDP2_GA_SYNC), on a gloo CPU world of 2.
+
+Four micro-batches, the first three under the no_sync context the Trainer enters:
+* "skip" (default): no reduce-scatter on the non-final micro-batches (one per unit per step);
+* "defer": the same gradients bitwise, and fewer all-gathers (none in the fit layout) after the first;
+* "each": a reduce-scatter on every micro-batch, the same gradients up to fp32 summation order;
+* "auto" (default): "defer" for the fit layout, "skip" for the sharded layout.
+For the sharded layout (full fine-tuning) and the fit layout (LoRA, root unit)."""
+
+import multiprocessing
+import os
+
+if multiprocessing.parent_process() is not None or getattr(
+ multiprocessing.current_process(), "_inheriting", False
+):
+ os.environ["CUDA_VISIBLE_DEVICES"] = "" # spawned gloo ranks: CPU only, also on GPU hosts
+
+import importlib # noqa: E402
+import sys # noqa: E402
+
+import pytest # noqa: E402
+import torch # noqa: E402
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from test_fsdp2_native_layout_agree_gloo import REPO_ID, _init, _spawn # noqa: E402
+from test_fsdp2_native_units import _build, _load_native # noqa: E402
+
+MICRO = 4
+
+
+def _run(native, perf, counts, layout, mode):
+ from torch.distributed.tensor import DTensor
+
+ os.environ["UNSLOTH_FSDP2_GA_SYNC"] = mode # read on every no_sync entry
+ rank = int(os.environ["RANK"])
+ torch.manual_seed(0)
+ model = _build(REPO_ID, "AutoModelForCausalLM")
+ if layout == "lora":
+ from peft import LoraConfig, get_peft_model
+
+ model = get_peft_model(model, LoraConfig(r = 4, lora_alpha = 8, target_modules = "all-linear"))
+ g = torch.Generator().manual_seed(1)
+ with torch.no_grad():
+ for n, p in model.named_parameters():
+ if "lora_B" in n:
+ p.normal_(0, 0.02, generator = g)
+ cfg = native.FSDP2Config(param_dtype = torch.float32, fit_layout = True)
+ else:
+ cfg = native.FSDP2Config(param_dtype = torch.float32, fit_layout = False)
+ native.shard_model(model, cfg)
+ from torch.distributed.fsdp import FSDPModule
+
+ units = sum(isinstance(m, FSDPModule) for m in model.modules()) # blocks + root
+ per_micro = []
+ for i in range(MICRO):
+ before = dict(counts)
+ ids = torch.randint(
+ 3, 1000, (2, 16), generator = torch.Generator().manual_seed(100 * rank + i)
+ )
+ ctx = native._no_grad_sync(model) if i < MICRO - 1 else torch.enable_grad()
+ with ctx:
+ model(input_ids = ids, labels = ids, use_cache = False).loss.backward()
+ per_micro.append({k: counts[k] - before[k] for k in counts})
+ grads = {
+ n: (p.grad.full_tensor() if isinstance(p.grad, DTensor) else p.grad).clone()
+ for n, p in model.named_parameters()
+ if p.requires_grad and p.grad is not None
+ }
+ return {
+ "units": units,
+ "per_micro": per_micro,
+ "grads": grads,
+ "fits": model._unsloth_fsdp2_layout["fits"],
+ }
+
+
+def _worker(rank, world, port, out, path):
+ dist = _init(rank, world, port)
+ try:
+ native = _load_native()
+ perf = importlib.import_module("unsloth.models.fsdp2_perf")
+ fpg = importlib.import_module("torch.distributed.fsdp._fully_shard._fsdp_param_group")
+ counts = {"ag": 0, "rs": 0}
+ orig_ag, orig_rs = fpg.foreach_all_gather, fpg.foreach_reduce
+
+ def ag(*a, **k):
+ counts["ag"] += 1
+ return orig_ag(*a, **k)
+
+ def rs(*a, **k):
+ counts["rs"] += 1
+ return orig_rs(*a, **k)
+
+ fpg.foreach_all_gather, fpg.foreach_reduce = ag, rs
+ res = {
+ (layout, mode): _run(native, perf, counts, layout, mode)
+ for layout in ("full", "lora")
+ for mode in ("skip", "defer", "each", "auto")
+ }
+ if rank == 0: # a file: a large queued payload blocks the child's exit (join deadlock)
+ torch.save(res, path)
+ out.put("ok")
+ except Exception as exc:
+ import traceback
+ out.put({"error": f"{type(exc).__name__}: {exc}\n{traceback.format_exc()}"})
+ finally:
+ dist.destroy_process_group()
+
+
+def test_ga_sync_modes(tmp_path):
+ pytest.importorskip("transformers")
+ pytest.importorskip("peft")
+ path = str(tmp_path / "ga_sync.pt")
+ (payload,) = _spawn(_worker, path)
+ assert payload == "ok", payload["error"]
+ allres = torch.load(path)
+ for layout in ("full", "lora"):
+ res = {m: allres[(layout, m)] for m in ("skip", "defer", "each", "auto")}
+ for m, r in res.items():
+ print("GA_SYNC", layout, m, r["fits"], r["units"], r["per_micro"])
+ _check(layout, res)
+
+
+def _check(layout, res):
+ units = res["skip"]["units"]
+ assert units >= 1
+ if layout == "full":
+ assert units > 2 and res["skip"]["fits"] is False, res["skip"]["units"]
+ skip, defer, each = (res[m]["per_micro"] for m in ("skip", "defer", "each"))
+ units = skip[-1]["rs"] # units holding trainable parameters (one reduce-scatter each)
+ assert units >= (2 if layout == "full" else 1), skip
+ # skip: reduce only on the final micro-batch; every micro-batch gathers again.
+ assert [m["rs"] for m in skip] == [0] * (MICRO - 1) + [units], skip
+ assert all(m["ag"] >= units for m in skip), skip
+ # defer: same reductions; after the first micro-batch no unit is gathered for forward (kept from
+ # the previous backward). Units resharded after forward (sharded layout) still gather for backward.
+ assert [m["rs"] for m in defer] == [0] * (MICRO - 1) + [units], defer
+ assert defer[0]["ag"] == skip[0]["ag"], (defer, skip)
+ assert all(d["ag"] < s["ag"] for d, s in zip(defer[1:], skip[1:])), (defer, skip)
+ if layout == "lora": # fit layout: kept gathered after forward too, so no gather in between
+ assert all(m["ag"] == 0 for m in defer[1:-1]), defer
+ # each: one reduce per unit per micro-batch.
+ assert [m["rs"] for m in each] == [units] * MICRO, each
+ # auto (the default): defer for the fit layout, skip for the sharded one.
+ assert res["auto"]["per_micro"] == (defer if layout == "lora" else skip), res["auto"][
+ "per_micro"
+ ]
+ g_skip, g_defer, g_each = (res[m]["grads"] for m in ("skip", "defer", "each"))
+ assert g_skip and g_skip.keys() == g_defer.keys() == g_each.keys()
+ for n in g_skip:
+ assert torch.equal(g_skip[n], g_defer[n]), n
+ torch.testing.assert_close(g_each[n], g_skip[n], rtol = 1e-5, atol = 1e-7, msg = n)
diff --git a/tests/test_fsdp2_recompute_prefetch_gloo.py b/tests/test_fsdp2_recompute_prefetch_gloo.py
new file mode 100644
index 00000000000..eade8a92681
--- /dev/null
+++ b/tests/test_fsdp2_recompute_prefetch_gloo.py
@@ -0,0 +1,139 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""Native FSDP2 fit layout + gradient checkpointing, on a gloo CPU world of 2: the backward recompute must
+not fire the explicit forward prefetch (it re-gathered block i+1 right after its backward resharded it).
+Same gradients bitwise; fewer all-gathers under reentrant checkpointing (Unsloth's), unchanged counts under
+non-reentrant (FSDP already skips its pre-forward there); the
+UNSLOTH_FSDP2_RECOMPUTE_PREFETCH=1 kill switch restores torch's behaviour (the counter can fail)."""
+
+import multiprocessing
+import os
+
+if multiprocessing.parent_process() is not None or getattr(
+ multiprocessing.current_process(), "_inheriting", False
+):
+ os.environ["CUDA_VISIBLE_DEVICES"] = "" # spawned gloo ranks: CPU only, also on GPU hosts
+
+import importlib # noqa: E402
+import sys # noqa: E402
+
+import pytest # noqa: E402
+import torch # noqa: E402
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from test_fsdp2_native_layout_agree_gloo import REPO_ID, _init, _spawn # noqa: E402
+from test_fsdp2_native_units import _build, _load_native # noqa: E402
+
+STEPS = 2
+
+
+def _run(native, counts, reentrant, kill_switch):
+ from torch.distributed.tensor import DTensor
+
+ if kill_switch:
+ os.environ["UNSLOTH_FSDP2_RECOMPUTE_PREFETCH"] = "1"
+ else:
+ os.environ.pop("UNSLOTH_FSDP2_RECOMPUTE_PREFETCH", None)
+ rank = int(os.environ["RANK"])
+ torch.manual_seed(0)
+ model = _build(REPO_ID, "AutoModelForCausalLM") # full fine-tuning, 2 blocks
+ model.config.use_cache = False
+ model.gradient_checkpointing_enable(gradient_checkpointing_kwargs = {"use_reentrant": reentrant})
+ model.train()
+ if reentrant: # reentrant checkpointing only recomputes when its input requires grad
+ model.enable_input_require_grads()
+ native.shard_model(model, native.FSDP2Config(param_dtype = torch.float32, fit_layout = True))
+ layout = model._unsloth_fsdp2_layout
+ opt = torch.optim.SGD(model.parameters(), lr = 0.1)
+ per_step, grads = [], None
+ for step in range(STEPS):
+ before = counts["ag"]
+ ids = torch.randint(
+ 3, 1000, (2, 16), generator = torch.Generator().manual_seed(10 * rank + step)
+ )
+ model(input_ids = ids, labels = ids, use_cache = False).loss.backward()
+ per_step.append(counts["ag"] - before)
+ if step == 0:
+ grads = {
+ n: (p.grad.full_tensor() if isinstance(p.grad, DTensor) else p.grad).clone()
+ for n, p in model.named_parameters()
+ if p.grad is not None
+ }
+ opt.step()
+ opt.zero_grad(set_to_none = True)
+ params = {
+ n: (p.full_tensor() if isinstance(p, DTensor) else p).detach().clone()
+ for n, p in model.named_parameters()
+ }
+ return {
+ "per_step": per_step,
+ "grads": grads,
+ "params": params,
+ "fits": layout["fits"],
+ "prefetch": layout["prefetch"],
+ }
+
+
+def _worker(rank, world, port, out, path):
+ dist = _init(rank, world, port)
+ try:
+ native = _load_native()
+ importlib.import_module("unsloth.models.fsdp2_perf")
+ fpg = importlib.import_module("torch.distributed.fsdp._fully_shard._fsdp_param_group")
+ counts = {"ag": 0}
+ orig_ag = fpg.foreach_all_gather
+
+ def ag(*a, **k):
+ counts["ag"] += 1
+ return orig_ag(*a, **k)
+
+ fpg.foreach_all_gather = ag
+ res = {
+ (reentrant, kill): _run(native, counts, reentrant, kill)
+ for reentrant in (False, True)
+ for kill in (True, False)
+ }
+ if rank == 0: # a file: a large queued payload blocks the child's exit (join deadlock)
+ torch.save(res, path)
+ out.put("ok")
+ except Exception as exc:
+ import traceback
+ out.put({"error": f"{type(exc).__name__}: {exc}\n{traceback.format_exc()}"})
+ finally:
+ dist.destroy_process_group()
+
+
+def test_recompute_does_not_refire_forward_prefetch(tmp_path):
+ pytest.importorskip("transformers")
+ path = str(tmp_path / "recompute_prefetch.pt")
+ (payload,) = _spawn(_worker, path)
+ assert payload == "ok", payload["error"]
+ res = torch.load(path)
+ for reentrant in (False, True):
+ base, head = res[(reentrant, True)], res[(reentrant, False)]
+ print(
+ "RECOMPUTE_PREFETCH",
+ "reentrant" if reentrant else "nonreentrant",
+ "base",
+ base["per_step"],
+ "head",
+ head["per_step"],
+ )
+ assert base["fits"] and base["prefetch"], base
+ if reentrant:
+ # Reentrant checkpointing runs the forward under no_grad (no FSDP pre-backward hook), so the
+ # recompute's pre-forward runs in full and prefetches the block after it. The kill switch
+ # reproduces that waste (the counter can fail); the default removes it.
+ assert all(h < b for h, b in zip(head["per_step"], base["per_step"])), (
+ base["per_step"],
+ head["per_step"],
+ )
+ else:
+ # Non-reentrant: FSDP's pre-forward already returns early in PRE_BACKWARD; nothing to remove.
+ assert head["per_step"] == base["per_step"], (base["per_step"], head["per_step"])
+ assert base["grads"].keys() == head["grads"].keys() and base["grads"]
+ for n in base["grads"]:
+ assert torch.equal(base["grads"][n], head["grads"][n]), n
+ for n in base["params"]:
+ assert torch.equal(base["params"][n], head["params"][n]), n
diff --git a/unsloth/models/fsdp2_native.py b/unsloth/models/fsdp2_native.py
index 9f65570c38b..7470baf1a41 100644
--- a/unsloth/models/fsdp2_native.py
+++ b/unsloth/models/fsdp2_native.py
@@ -1702,13 +1702,44 @@ def native_fsdp2_unsharded(model):
_reshard_after_forward_restore(unit, saved[id(unit)])
+def _ga_mode(model = None):
+ """What non-final gradient accumulation micro-batches (Trainer / accelerate no_sync) do,
+ `UNSLOTH_FSDP2_GA_SYNC`:
+ "skip" = no reduce-scatter; units resharded after each backward and gathered again next micro-batch;
+ "defer" = no reduce-scatter and units kept unsharded until the last micro-batch (torchtitan
+ fsdp_defer_gradient_reduction): no re-all-gather, gradients bitwise equal to "skip", the gathered
+ weights stay resident (sharded layout: + one compute-dtype copy of the trainable weights);
+ "each" = reduce-scatter every micro-batch (torchtitan's default): no unsharded fp32 gradient
+ accumulators held across micro-batches, the leanest and the slowest;
+ "auto" (default) = "defer" for the fit layout (its units already stay gathered from forward to
+ backward, so keeping them gathered across micro-batches adds no peak memory), else "skip"."""
+ mode = os.environ.get("UNSLOTH_FSDP2_GA_SYNC", "auto").strip().lower() or "auto"
+ if mode in ("skip", "defer", "each"):
+ return mode
+ layout = getattr(model, "_unsloth_fsdp2_layout", None) if model is not None else None
+ return "defer" if isinstance(layout, dict) and layout.get("fits") else "skip"
+
+
@contextlib.contextmanager
def _no_grad_sync(model):
+ mode = _ga_mode(model)
+ if mode == "each":
+ yield
+ return
+ defer = mode == "defer" and hasattr(model, "set_reshard_after_backward")
model.set_requires_gradient_sync(False)
+ if defer:
+ model.set_reshard_after_backward(False)
+ if hasattr(model, "set_is_last_backward"):
+ model.set_is_last_backward(False)
try:
yield
finally:
model.set_requires_gradient_sync(True)
+ if defer:
+ model.set_reshard_after_backward(True)
+ if hasattr(model, "set_is_last_backward"):
+ model.set_is_last_backward(True)
def _native_models(accelerator):
diff --git a/unsloth/models/fsdp2_perf.py b/unsloth/models/fsdp2_perf.py
index 5fbbb2f6bc2..5122b78a404 100644
--- a/unsloth/models/fsdp2_perf.py
+++ b/unsloth/models/fsdp2_perf.py
@@ -225,11 +225,50 @@ def apply_fsdp2_prefetch(
behind = units[max(0, i - backward_depth) : i][::-1]
if ahead:
unit.set_modules_to_forward_prefetch(ahead)
+ _skip_forward_prefetch_in_backward(unit, ahead)
if behind:
unit.set_modules_to_backward_prefetch(behind)
return len(units)
+def _in_backward():
+ task = getattr(torch._C, "_current_graph_task_id", None)
+ return task is not None and task() != -1
+
+
+def _skip_forward_prefetch_in_backward(unit, ahead):
+ """Gradient checkpointing re-runs a block's forward inside backward, which fires its pre-forward
+ hook and with it the explicit forward prefetch of block i+1: block i+1 has already run its backward
+ and been resharded, so it is gathered again for nothing (one wasted all-gather per block per
+ micro-batch). Clear the forward prefetch while autograd is running a backward and restore it for
+ the next real forward. Public FSDPModule API only; `UNSLOTH_FSDP2_RECOMPUTE_PREFETCH=1` keeps torch's
+ behaviour."""
+ if os.environ.get("UNSLOTH_FSDP2_RECOMPUTE_PREFETCH", "0").strip().lower() in (
+ "1",
+ "true",
+ "yes",
+ ):
+ return
+ if getattr(unit, "_unsloth_recompute_prefetch_hooked", False):
+ unit._unsloth_forward_prefetch = list(ahead)
+ return
+ unit._unsloth_forward_prefetch = list(ahead)
+ state = {"cleared": False}
+
+ def _pre_forward(module, args):
+ if _in_backward():
+ if not state["cleared"]:
+ module.set_modules_to_forward_prefetch([])
+ state["cleared"] = True
+ elif state["cleared"]:
+ module.set_modules_to_forward_prefetch(module._unsloth_forward_prefetch)
+ state["cleared"] = False
+
+ # prepend: runs before FSDP's own pre-forward hook, which issues the prefetch.
+ unit.register_forward_pre_hook(_pre_forward, prepend = True)
+ unit._unsloth_recompute_prefetch_hooked = True
+
+
def _env_int(name, default):
try:
return max(1, int(os.environ.get(name, default)))