Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
FSDP2 native: fewer collectives under gradient accumulation and checkpoint recompute (#15023)
Daniel Han96f60dab1fdc
diff --git a/tests/test_fsdp2_native_ga_sync_gloo.py b/tests/test_fsdp2_native_ga_sync_gloo.pynew file mode 100644index 00000000000..5ab8747b450--- /dev/null+++ b/tests/test_fsdp2_native_ga_sync_gloo.py@@ -0,0 +1,155 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""Native FSDP2 gradient accumulation modes (UNSLOTH_FSDP2_GA_SYNC), on a gloo CPU world of 2.++Four micro-batches, the first three under the no_sync context the Trainer enters:+* "skip" (default): no reduce-scatter on the non-final micro-batches (one per unit per step);+* "defer": the same gradients bitwise, and fewer all-gathers (none in the fit layout) after the first;+* "each": a reduce-scatter on every micro-batch, the same gradients up to fp32 summation order;+* "auto" (default): "defer" for the fit layout, "skip" for the sharded layout.+For the sharded layout (full fine-tuning) and the fit layout (LoRA, root unit)."""++import multiprocessing+import os++if multiprocessing.parent_process() is not None or getattr(+ multiprocessing.current_process(), "_inheriting", False+):+ os.environ["CUDA_VISIBLE_DEVICES"] = "" # spawned gloo ranks: CPU only, also on GPU hosts++import importlib # noqa: E402+import sys # noqa: E402++import pytest # noqa: E402+import torch # noqa: E402++sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))+from test_fsdp2_native_layout_agree_gloo import REPO_ID, _init, _spawn # noqa: E402+from test_fsdp2_native_units import _build, _load_native # noqa: E402++MICRO = 4+++def _run(native, perf, counts, layout, mode):+ from torch.distributed.tensor import DTensor++ os.environ["UNSLOTH_FSDP2_GA_SYNC"] = mode # read on every no_sync entry+ rank = int(os.environ["RANK"])+ torch.manual_seed(0)+ model = _build(REPO_ID, "AutoModelForCausalLM")+ if layout == "lora":+ from peft import LoraConfig, get_peft_model++ model = get_peft_model(model, LoraConfig(r = 4, lora_alpha = 8, target_modules = "all-linear"))+ g = torch.Generator().manual_seed(1)+ with torch.no_grad():+ for n, p in model.named_parameters():+ if "lora_B" in n:+ p.normal_(0, 0.02, generator = g)+ cfg = native.FSDP2Config(param_dtype = torch.float32, fit_layout = True)+ else:+ cfg = native.FSDP2Config(param_dtype = torch.float32, fit_layout = False)+ native.shard_model(model, cfg)+ from torch.distributed.fsdp import FSDPModule++ units = sum(isinstance(m, FSDPModule) for m in model.modules()) # blocks + root+ per_micro = []+ for i in range(MICRO):+ before = dict(counts)+ ids = torch.randint(+ 3, 1000, (2, 16), generator = torch.Generator().manual_seed(100 * rank + i)+ )+ ctx = native._no_grad_sync(model) if i < MICRO - 1 else torch.enable_grad()+ with ctx:+ model(input_ids = ids, labels = ids, use_cache = False).loss.backward()+ per_micro.append({k: counts[k] - before[k] for k in counts})+ grads = {+ n: (p.grad.full_tensor() if isinstance(p.grad, DTensor) else p.grad).clone()+ for n, p in model.named_parameters()+ if p.requires_grad and p.grad is not None+ }+ return {+ "units": units,+ "per_micro": per_micro,+ "grads": grads,+ "fits": model._unsloth_fsdp2_layout["fits"],+ }+++def _worker(rank, world, port, out, path):+ dist = _init(rank, world, port)+ try:+ native = _load_native()+ perf = importlib.import_module("unsloth.models.fsdp2_perf")+ fpg = importlib.import_module("torch.distributed.fsdp._fully_shard._fsdp_param_group")+ counts = {"ag": 0, "rs": 0}+ orig_ag, orig_rs = fpg.foreach_all_gather, fpg.foreach_reduce++ def ag(*a, **k):+ counts["ag"] += 1+ return orig_ag(*a, **k)++ def rs(*a, **k):+ counts["rs"] += 1+ return orig_rs(*a, **k)++ fpg.foreach_all_gather, fpg.foreach_reduce = ag, rs+ res = {+ (layout, mode): _run(native, perf, counts, layout, mode)+ for layout in ("full", "lora")+ for mode in ("skip", "defer", "each", "auto")+ }+ if rank == 0: # a file: a large queued payload blocks the child's exit (join deadlock)+ torch.save(res, path)+ out.put("ok")+ except Exception as exc:+ import traceback+ out.put({"error": f"{type(exc).__name__}: {exc}\n{traceback.format_exc()}"})+ finally:+ dist.destroy_process_group()+++def test_ga_sync_modes(tmp_path):+ pytest.importorskip("transformers")+ pytest.importorskip("peft")+ path = str(tmp_path / "ga_sync.pt")+ (payload,) = _spawn(_worker, path)+ assert payload == "ok", payload["error"]+ allres = torch.load(path)+ for layout in ("full", "lora"):+ res = {m: allres[(layout, m)] for m in ("skip", "defer", "each", "auto")}+ for m, r in res.items():+ print("GA_SYNC", layout, m, r["fits"], r["units"], r["per_micro"])+ _check(layout, res)+++def _check(layout, res):+ units = res["skip"]["units"]+ assert units >= 1+ if layout == "full":+ assert units > 2 and res["skip"]["fits"] is False, res["skip"]["units"]+ skip, defer, each = (res[m]["per_micro"] for m in ("skip", "defer", "each"))+ units = skip[-1]["rs"] # units holding trainable parameters (one reduce-scatter each)+ assert units >= (2 if layout == "full" else 1), skip+ # skip: reduce only on the final micro-batch; every micro-batch gathers again.+ assert [m["rs"] for m in skip] == [0] * (MICRO - 1) + [units], skip+ assert all(m["ag"] >= units for m in skip), skip+ # defer: same reductions; after the first micro-batch no unit is gathered for forward (kept from+ # the previous backward). Units resharded after forward (sharded layout) still gather for backward.+ assert [m["rs"] for m in defer] == [0] * (MICRO - 1) + [units], defer+ assert defer[0]["ag"] == skip[0]["ag"], (defer, skip)+ assert all(d["ag"] < s["ag"] for d, s in zip(defer[1:], skip[1:])), (defer, skip)+ if layout == "lora": # fit layout: kept gathered after forward too, so no gather in between+ assert all(m["ag"] == 0 for m in defer[1:-1]), defer+ # each: one reduce per unit per micro-batch.+ assert [m["rs"] for m in each] == [units] * MICRO, each+ # auto (the default): defer for the fit layout, skip for the sharded one.+ assert res["auto"]["per_micro"] == (defer if layout == "lora" else skip), res["auto"][+ "per_micro"+ ]+ g_skip, g_defer, g_each = (res[m]["grads"] for m in ("skip", "defer", "each"))+ assert g_skip and g_skip.keys() == g_defer.keys() == g_each.keys()+ for n in g_skip:+ assert torch.equal(g_skip[n], g_defer[n]), n+ torch.testing.assert_close(g_each[n], g_skip[n], rtol = 1e-5, atol = 1e-7, msg = n)diff --git a/tests/test_fsdp2_recompute_prefetch_gloo.py b/tests/test_fsdp2_recompute_prefetch_gloo.pynew file mode 100644index 00000000000..eade8a92681--- /dev/null+++ b/tests/test_fsdp2_recompute_prefetch_gloo.py@@ -0,0 +1,139 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""Native FSDP2 fit layout + gradient checkpointing, on a gloo CPU world of 2: the backward recompute must+not fire the explicit forward prefetch (it re-gathered block i+1 right after its backward resharded it).+Same gradients bitwise; fewer all-gathers under reentrant checkpointing (Unsloth's), unchanged counts under+non-reentrant (FSDP already skips its pre-forward there); the+UNSLOTH_FSDP2_RECOMPUTE_PREFETCH=1 kill switch restores torch's behaviour (the counter can fail)."""++import multiprocessing+import os++if multiprocessing.parent_process() is not None or getattr(+ multiprocessing.current_process(), "_inheriting", False+):+ os.environ["CUDA_VISIBLE_DEVICES"] = "" # spawned gloo ranks: CPU only, also on GPU hosts++import importlib # noqa: E402+import sys # noqa: E402++import pytest # noqa: E402+import torch # noqa: E402++sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))+from test_fsdp2_native_layout_agree_gloo import REPO_ID, _init, _spawn # noqa: E402+from test_fsdp2_native_units import _build, _load_native # noqa: E402++STEPS = 2+++def _run(native, counts, reentrant, kill_switch):+ from torch.distributed.tensor import DTensor++ if kill_switch:+ os.environ["UNSLOTH_FSDP2_RECOMPUTE_PREFETCH"] = "1"+ else:+ os.environ.pop("UNSLOTH_FSDP2_RECOMPUTE_PREFETCH", None)+ rank = int(os.environ["RANK"])+ torch.manual_seed(0)+ model = _build(REPO_ID, "AutoModelForCausalLM") # full fine-tuning, 2 blocks+ model.config.use_cache = False+ model.gradient_checkpointing_enable(gradient_checkpointing_kwargs = {"use_reentrant": reentrant})+ model.train()+ if reentrant: # reentrant checkpointing only recomputes when its input requires grad+ model.enable_input_require_grads()+ native.shard_model(model, native.FSDP2Config(param_dtype = torch.float32, fit_layout = True))+ layout = model._unsloth_fsdp2_layout+ opt = torch.optim.SGD(model.parameters(), lr = 0.1)+ per_step, grads = [], None+ for step in range(STEPS):+ before = counts["ag"]+ ids = torch.randint(+ 3, 1000, (2, 16), generator = torch.Generator().manual_seed(10 * rank + step)+ )+ model(input_ids = ids, labels = ids, use_cache = False).loss.backward()+ per_step.append(counts["ag"] - before)+ if step == 0:+ grads = {+ n: (p.grad.full_tensor() if isinstance(p.grad, DTensor) else p.grad).clone()+ for n, p in model.named_parameters()+ if p.grad is not None+ }+ opt.step()+ opt.zero_grad(set_to_none = True)+ params = {+ n: (p.full_tensor() if isinstance(p, DTensor) else p).detach().clone()+ for n, p in model.named_parameters()+ }+ return {+ "per_step": per_step,+ "grads": grads,+ "params": params,+ "fits": layout["fits"],+ "prefetch": layout["prefetch"],+ }+++def _worker(rank, world, port, out, path):+ dist = _init(rank, world, port)+ try:+ native = _load_native()+ importlib.import_module("unsloth.models.fsdp2_perf")+ fpg = importlib.import_module("torch.distributed.fsdp._fully_shard._fsdp_param_group")+ counts = {"ag": 0}+ orig_ag = fpg.foreach_all_gather++ def ag(*a, **k):+ counts["ag"] += 1+ return orig_ag(*a, **k)++ fpg.foreach_all_gather = ag+ res = {+ (reentrant, kill): _run(native, counts, reentrant, kill)+ for reentrant in (False, True)+ for kill in (True, False)+ }+ if rank == 0: # a file: a large queued payload blocks the child's exit (join deadlock)+ torch.save(res, path)+ out.put("ok")+ except Exception as exc:+ import traceback+ out.put({"error": f"{type(exc).__name__}: {exc}\n{traceback.format_exc()}"})+ finally:+ dist.destroy_process_group()+++def test_recompute_does_not_refire_forward_prefetch(tmp_path):+ pytest.importorskip("transformers")+ path = str(tmp_path / "recompute_prefetch.pt")+ (payload,) = _spawn(_worker, path)+ assert payload == "ok", payload["error"]+ res = torch.load(path)+ for reentrant in (False, True):+ base, head = res[(reentrant, True)], res[(reentrant, False)]+ print(+ "RECOMPUTE_PREFETCH",+ "reentrant" if reentrant else "nonreentrant",+ "base",+ base["per_step"],+ "head",+ head["per_step"],+ )+ assert base["fits"] and base["prefetch"], base+ if reentrant:+ # Reentrant checkpointing runs the forward under no_grad (no FSDP pre-backward hook), so the+ # recompute's pre-forward runs in full and prefetches the block after it. The kill switch+ # reproduces that waste (the counter can fail); the default removes it.+ assert all(h < b for h, b in zip(head["per_step"], base["per_step"])), (+ base["per_step"],+ head["per_step"],+ )+ else:+ # Non-reentrant: FSDP's pre-forward already returns early in PRE_BACKWARD; nothing to remove.+ assert head["per_step"] == base["per_step"], (base["per_step"], head["per_step"])+ assert base["grads"].keys() == head["grads"].keys() and base["grads"]+ for n in base["grads"]:+ assert torch.equal(base["grads"][n], head["grads"][n]), n+ for n in base["params"]:+ assert torch.equal(base["params"][n], head["params"][n]), ndiff --git a/unsloth/models/fsdp2_native.py b/unsloth/models/fsdp2_native.pyindex 9f65570c38b..7470baf1a41 100644--- a/unsloth/models/fsdp2_native.py+++ b/unsloth/models/fsdp2_native.py@@ -1702,13 +1702,44 @@ def native_fsdp2_unsharded(model):_reshard_after_forward_restore(unit, saved[id(unit)])+def _ga_mode(model = None):+ """What non-final gradient accumulation micro-batches (Trainer / accelerate no_sync) do,+ `UNSLOTH_FSDP2_GA_SYNC`:+ "skip" = no reduce-scatter; units resharded after each backward and gathered again next micro-batch;+ "defer" = no reduce-scatter and units kept unsharded until the last micro-batch (torchtitan+ fsdp_defer_gradient_reduction): no re-all-gather, gradients bitwise equal to "skip", the gathered+ weights stay resident (sharded layout: + one compute-dtype copy of the trainable weights);+ "each" = reduce-scatter every micro-batch (torchtitan's default): no unsharded fp32 gradient+ accumulators held across micro-batches, the leanest and the slowest;+ "auto" (default) = "defer" for the fit layout (its units already stay gathered from forward to+ backward, so keeping them gathered across micro-batches adds no peak memory), else "skip"."""+ mode = os.environ.get("UNSLOTH_FSDP2_GA_SYNC", "auto").strip().lower() or "auto"+ if mode in ("skip", "defer", "each"):+ return mode+ layout = getattr(model, "_unsloth_fsdp2_layout", None) if model is not None else None+ return "defer" if isinstance(layout, dict) and layout.get("fits") else "skip"++@contextlib.contextmanagerdef _no_grad_sync(model):+ mode = _ga_mode(model)+ if mode == "each":+ yield+ return+ defer = mode == "defer" and hasattr(model, "set_reshard_after_backward")model.set_requires_gradient_sync(False)+ if defer:+ model.set_reshard_after_backward(False)+ if hasattr(model, "set_is_last_backward"):+ model.set_is_last_backward(False)try:yieldfinally:model.set_requires_gradient_sync(True)+ if defer:+ model.set_reshard_after_backward(True)+ if hasattr(model, "set_is_last_backward"):+ model.set_is_last_backward(True)def _native_models(accelerator):diff --git a/unsloth/models/fsdp2_perf.py b/unsloth/models/fsdp2_perf.pyindex 5fbbb2f6bc2..5122b78a404 100644--- a/unsloth/models/fsdp2_perf.py+++ b/unsloth/models/fsdp2_perf.py@@ -225,11 +225,50 @@ def apply_fsdp2_prefetch(behind = units[max(0, i - backward_depth) : i][::-1]if ahead:unit.set_modules_to_forward_prefetch(ahead)+ _skip_forward_prefetch_in_backward(unit, ahead)if behind:unit.set_modules_to_backward_prefetch(behind)return len(units)+def _in_backward():+ task = getattr(torch._C, "_current_graph_task_id", None)+ return task is not None and task() != -1+++def _skip_forward_prefetch_in_backward(unit, ahead):+ """Gradient checkpointing re-runs a block's forward inside backward, which fires its pre-forward+ hook and with it the explicit forward prefetch of block i+1: block i+1 has already run its backward+ and been resharded, so it is gathered again for nothing (one wasted all-gather per block per+ micro-batch). Clear the forward prefetch while autograd is running a backward and restore it for+ the next real forward. Public FSDPModule API only; `UNSLOTH_FSDP2_RECOMPUTE_PREFETCH=1` keeps torch's+ behaviour."""+ if os.environ.get("UNSLOTH_FSDP2_RECOMPUTE_PREFETCH", "0").strip().lower() in (+ "1",+ "true",+ "yes",+ ):+ return+ if getattr(unit, "_unsloth_recompute_prefetch_hooked", False):+ unit._unsloth_forward_prefetch = list(ahead)+ return+ unit._unsloth_forward_prefetch = list(ahead)+ state = {"cleared": False}++ def _pre_forward(module, args):+ if _in_backward():+ if not state["cleared"]:+ module.set_modules_to_forward_prefetch([])+ state["cleared"] = True+ elif state["cleared"]:+ module.set_modules_to_forward_prefetch(module._unsloth_forward_prefetch)+ state["cleared"] = False++ # prepend: runs before FSDP's own pre-forward hook, which issues the prefetch.+ unit.register_forward_pre_hook(_pre_forward, prepend = True)+ unit._unsloth_recompute_prefetch_hooked = True++def _env_int(name, default):try:return max(1, int(os.environ.get(name, default)))