Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Native FSDP2: restore the exact reshard-after-forward setting after generate (#14762)

Daniel Han5c1e04ef89c1
diff --git a/tests/test_fsdp2_native_unsharded_restore.py b/tests/test_fsdp2_native_unsharded_restore.py
new file mode 100644
index 00000000000..689f79ad181
--- /dev/null
+++ b/tests/test_fsdp2_native_unsharded_restore.py
@@ -0,0 +1,83 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""native_fsdp2_unsharded restores each unit's exact reshard-after-forward setting: int values
+(the setter takes only bool) and torch 2.6 / 2.7 (no setter). One-rank gloo group on the CPU."""
+
+import os
+import sys
+
+import pytest
+import torch
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from test_fsdp2_native_units import _load_native # noqa: E402
+
+
+def sharded(tmp_path, monkeypatch):
+ import torch.distributed as dist
+ from torch.distributed.device_mesh import init_device_mesh
+ from torch.distributed.fsdp import fully_shard
+
+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "")
+ native = _load_native()
+ dist.init_process_group(
+ "gloo", init_method = f"file://{tmp_path / 'store'}", rank = 0, world_size = 1
+ )
+ try:
+ mesh = init_device_mesh("cpu", (1,))
+ model = torch.nn.Sequential(torch.nn.Linear(4, 4), torch.nn.Linear(4, 2))
+ for block in model:
+ fully_shard(block, mesh = mesh, reshard_after_forward = True)
+ block._unsloth_reshard_after_forward = True
+ fully_shard(model, mesh = mesh)
+ model._unsloth_reshard_after_forward = False
+ setattr(model, native._NATIVE_FLAG, True)
+ model(torch.randn(1, 4)) # lazy init
+ yield native, model
+ finally:
+ dist.destroy_process_group()
+
+
+def _infos(model):
+ return [m._get_fsdp_state()._fsdp_param_group.post_forward_mesh_info for m in model]
+
+
+def _gathered(module):
+ from torch.distributed.tensor import DTensor
+ return not isinstance(module.weight, DTensor)
+
+
+def _generate_like(native, model):
+ before = _infos(model)
+ with native.native_fsdp2_unsharded(model):
+ model(torch.randn(1, 4))
+ assert _gathered(model[0]) and _gathered(model[1])
+ assert not _gathered(model[0]) and not _gathered(model[1])
+ after = _infos(model)
+ assert all(a is b for a, b in zip(after, before)), (before, after)
+ assert all(info is not None for info in before)
+
+
+def test_restores_the_exact_setting(sharded):
+ native, model = sharded
+ _generate_like(native, model)
+
+
+def test_int_reshard_after_forward_is_restored(sharded):
+ native, model = sharded
+ for block in model:
+ block._unsloth_reshard_after_forward = (
+ 2 # what shard_model records for FSDP2Config(... = 2)
+ )
+ _generate_like(native, model)
+
+
+def test_torch_without_set_reshard_after_forward(sharded, monkeypatch):
+ from torch.distributed.fsdp import FSDPModule
+
+ native, model = sharded
+ if hasattr(FSDPModule, "set_reshard_after_forward"):
+ monkeypatch.delattr(FSDPModule, "set_reshard_after_forward") # torch 2.6 / 2.7
+ _generate_like(native, model)
diff --git a/unsloth/models/fsdp2_native.py b/unsloth/models/fsdp2_native.py
index 9eda3e1af7a..c36d250e002 100644
--- a/unsloth/models/fsdp2_native.py
+++ b/unsloth/models/fsdp2_native.py
@@ -1781,29 +1781,28 @@ def native_full_state_dict(
def _reshard_after_forward_off(unit):
- """Keep `unit` gathered after its forward. FSDPModule.set_reshard_after_forward exists from torch
- 2.8; on 2.6 / 2.7 set the param group's post-forward mesh the same way (None is what
- reshard_after_forward = False resolves to). Returns what _reshard_after_forward_restore needs."""
- if hasattr(unit, "set_reshard_after_forward"):
- unit.set_reshard_after_forward(False, recurse = False)
- return None
- group = getattr(unit._get_fsdp_state(), "_fsdp_param_group", None)
- if group is None:
- return None
- previous = group.post_forward_mesh_info
- group.post_forward_mesh_info = None
- return (group, previous)
+ """Keep `unit` gathered after forward: post-forward mesh None, as set_reshard_after_forward(False)
+ writes. By hand since that setter is missing on torch 2.6 / 2.7 and refuses int values on restore."""
+ state = unit._get_fsdp_state()
+ groups = getattr(state, "_fsdp_param_groups", None)
+ if groups is None: # torch < 2.13: at most one param group
+ group = getattr(state, "_fsdp_param_group", None)
+ groups = [] if group is None else [group]
+ previous = [(group, group.post_forward_mesh_info) for group in groups]
+ auto = getattr(state, "_auto_reshard_after_forward", None)
+ for group in groups:
+ group.post_forward_mesh_info = None
+ if auto is not None:
+ state._auto_reshard_after_forward = False
+ return state, auto, previous
def _reshard_after_forward_restore(unit, saved):
- if hasattr(unit, "set_reshard_after_forward"):
- unit.set_reshard_after_forward(
- getattr(unit, "_unsloth_reshard_after_forward", True), recurse = False
- )
- return
- if saved is not None:
- group, previous = saved
- group.post_forward_mesh_info = previous
+ state, auto, previous = saved
+ for group, info in previous:
+ group.post_forward_mesh_info = info
+ if auto is not None:
+ state._auto_reshard_after_forward = auto
@contextlib.contextmanager
@@ -1836,10 +1835,17 @@ def native_fsdp2_unsharded(model):
yield model
finally:
root.__dict__["_unsloth_unsharded_depth"] = 0
+ errors = []
+ for unit in units:
+ try:
+ unit.reshard()
+ except Exception as exc:
+ errors.append(exc)
for unit in units:
- unit.reshard()
if id(unit) in saved:
_reshard_after_forward_restore(unit, saved[id(unit)])
+ if errors:
+ raise errors[0]
def _ga_mode(model = None):