Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Native FSDP2: keep accelerate FSDP2 for custom Accelerator loops, support unsharded generation on torch 2.6 / 2.7 (#14882)

Daniel Han482058cfb36e
diff --git a/tests/test_fsdp2_native_optim_reentry_gloo.py b/tests/test_fsdp2_native_optim_reentry_gloo.py
index 426e7303505..e765cb20441 100644
--- a/tests/test_fsdp2_native_optim_reentry_gloo.py
+++ b/tests/test_fsdp2_native_optim_reentry_gloo.py
@@ -117,9 +117,13 @@ def _case_optimizer_cls(native, tmp):
def _case_reentry(native):
+ model = native.shard_model(_tiny_llama(), native.FSDP2Config(param_dtype = torch.float32))
+ return _case_reentry_on(native, model)
+
+
+def _case_reentry_on(native, model):
from torch.distributed.tensor import DTensor
- model = native.shard_model(_tiny_llama(), native.FSDP2Config(param_dtype = torch.float32))
if getattr(model, "generation_config", None) is not None:
model.generation_config.pad_token_id = None
@@ -143,6 +147,30 @@ def _case_reentry(native):
return res
+def _case_reentry_no_setter(native):
+ """torch 2.6 / 2.7: FSDPModule has no set_reshard_after_forward (added in 2.8)."""
+ FSDPModule = native._fsdp_module_class()
+ removed = {}
+ for cls in FSDPModule.__mro__:
+ if "set_reshard_after_forward" in cls.__dict__:
+ removed[cls] = cls.__dict__["set_reshard_after_forward"]
+ delattr(cls, "set_reshard_after_forward")
+ try:
+ model = native.shard_model(_tiny_llama(), native.FSDP2Config(param_dtype = torch.float32))
+ units = [m for m in model.modules() if isinstance(m, FSDPModule)]
+ meshes = [u._get_fsdp_state()._fsdp_param_group for u in units]
+ before = [g.post_forward_mesh_info if g is not None else None for g in meshes]
+ res = {"has_setter": hasattr(units[0], "set_reshard_after_forward")}
+ res.update(_case_reentry_on(native, model))
+ after = [g.post_forward_mesh_info if g is not None else None for g in meshes]
+ res["mesh_restored"] = all(a is b for a, b in zip(before, after))
+ res["units"] = len(units)
+ return res
+ finally:
+ for cls, fn in removed.items():
+ setattr(cls, "set_reshard_after_forward", fn)
+
+
def _worker_case(case, native, rank, tmp):
if case == "stateful":
return _case_stateful(native, tmp)
@@ -150,6 +178,8 @@ def _worker_case(case, native, rank, tmp):
return _case_optimizer_cls(native, tmp)
if case == "reentry":
return _case_reentry(native)
+ if case == "reentry_no_setter":
+ return _case_reentry_no_setter(native)
raise KeyError(case)
@@ -246,6 +276,18 @@ def test_bnb_optimizer_cls_and_kwargs_is_refused(tmp_path):
assert not v["sharded"], (rank, key, v)
+def test_unsharded_context_without_reshard_setter(tmp_path):
+ """torch 2.6 / 2.7 FSDPModule has no set_reshard_after_forward: the gather / reshard still runs
+ and every unit's post-forward mesh comes back unchanged."""
+ pytest.importorskip("transformers")
+ for rank, r in _run("reentry_no_setter", tmp_path).items():
+ assert r["has_setter"] is False and r["units"] > 1, (rank, r)
+ assert r["before"] > 0 and r["outer"] == 0, (rank, r)
+ assert r["after_generate"] == 0 and r["after_inner_raise"] == 0, (rank, r)
+ assert r["after_outer"] == r["before"] and r["depth"] == 0, (rank, r)
+ assert r["mesh_restored"], (rank, r)
+
+
def test_unsharded_context_is_reentrant(tmp_path):
pytest.importorskip("transformers")
for rank, r in _run("reentry", tmp_path).items():
diff --git a/tests/test_fsdp2_native_raw_accelerator.py b/tests/test_fsdp2_native_raw_accelerator.py
new file mode 100644
index 00000000000..2205da0d791
--- /dev/null
+++ b/tests/test_fsdp2_native_raw_accelerator.py
@@ -0,0 +1,157 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""UNSLOTH_FSDP2=1 shards natively inside the Trainer only. A custom loop's own Accelerator
+(accelerator.prepare(model, optimizer) on an unsharded model) keeps accelerate's FSDP2, as before
+native FSDP2, instead of a plain DDP wrap: fsdp2_launch.install_raw_accelerator_fsdp2_fallback
+exports the accelerate FSDP2 launch env for it. A Trainer's accelerator, an explicit
+UNSLOTH_FSDP2_BACKEND, an explicit plugin, a CPU run, an already initialized AcceleratorState and an
+already natively sharded model are left alone. CPU only: accelerate is replaced by a recorder."""
+
+import os
+import sys
+import types
+
+import pytest
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from test_fsdp2_native_units import _load_native # noqa: E402
+
+_ENV = (
+ "ACCELERATE_USE_FSDP",
+ "FSDP_VERSION",
+ "FSDP_AUTO_WRAP_POLICY",
+ "FSDP_STATE_DICT_TYPE",
+ "FSDP_RESHARD_AFTER_FORWARD",
+ "ACCELERATE_USE_CPU",
+ "ACCELERATE_USE_DEEPSPEED",
+ "UNSLOTH_FSDP2_BACKEND",
+ "UNSLOTH_FSDP2_RESHARD_AFTER_FORWARD",
+)
+
+
+def launch(monkeypatch):
+ for key in _ENV:
+ # setenv first so teardown also drops what the fallback exports
+ monkeypatch.setenv(key, "")
+ monkeypatch.delenv(key)
+ for key, value in dict(UNSLOTH_FSDP2 = "1", WORLD_SIZE = "2", RANK = "0").items():
+ monkeypatch.setenv(key, value)
+ native = _load_native()
+ monkeypatch.setattr(native, "_SHARDED_ANY", False)
+ seen = []
+
+ class Accelerator:
+ def __init__(self, *args, **kwargs):
+ seen.append(os.environ.get("ACCELERATE_USE_FSDP"))
+
+ class AcceleratorState:
+ _shared_state = {}
+
+ accelerate = types.ModuleType("accelerate")
+ accelerate.Accelerator = Accelerator
+ state = types.ModuleType("accelerate.state")
+ state.AcceleratorState = AcceleratorState
+ accelerate.state = state
+ monkeypatch.setitem(sys.modules, "accelerate", accelerate)
+ monkeypatch.setitem(sys.modules, "accelerate.state", state)
+ module = sys.modules["unsloth.fsdp2_launch"]
+ monkeypatch.setattr(module, "_accelerate_fsdp2_support", lambda: True)
+ assert module.install_raw_accelerator_fsdp2_fallback() is True
+ return types.SimpleNamespace(
+ module = module, native = native, Accelerator = Accelerator, State = AcceleratorState, seen = seen
+ )
+
+
+def test_custom_loop_accelerator_gets_accelerate_fsdp2(launch):
+ assert launch.module.unsloth_fsdp2_native_requested()
+ launch.Accelerator()
+ assert launch.seen == ["true"]
+ assert os.environ["FSDP_VERSION"] == "2"
+ assert os.environ["FSDP_AUTO_WRAP_POLICY"] == "TRANSFORMER_BASED_WRAP"
+ assert os.environ["FSDP_STATE_DICT_TYPE"] == "FULL_STATE_DICT"
+ # From here on the process is the accelerate FSDP2 backend, also for a later Trainer.
+ assert not launch.module.unsloth_fsdp2_native_requested()
+
+
+def test_trainer_accelerator_stays_native(launch):
+ class Trainer:
+ args = object()
+
+ def create_accelerator_and_postprocess(self):
+ self.accelerator = launch.Accelerator()
+
+ Trainer().create_accelerator_and_postprocess()
+ assert launch.seen == [None]
+ assert launch.module.unsloth_fsdp2_native_requested()
+
+
+ "setup",
+ [
+ "backend_native",
+ "backend_accelerate",
+ "plugin",
+ "tp_plugin",
+ "parallelism_config",
+ "cpu",
+ "state",
+ "sharded",
+ "single",
+ ],
+)
+def test_left_alone(launch, monkeypatch, setup):
+ kwargs = {}
+ if setup == "backend_native":
+ monkeypatch.setenv("UNSLOTH_FSDP2_BACKEND", "native")
+ elif setup == "backend_accelerate":
+ monkeypatch.setenv("UNSLOTH_FSDP2_BACKEND", "accelerate")
+ elif setup == "plugin":
+ kwargs["fsdp_plugin"] = object()
+ elif setup == "tp_plugin":
+ kwargs["torch_tp_plugin"] = object()
+ elif setup == "parallelism_config":
+ kwargs["parallelism_config"] = object()
+ elif setup == "cpu":
+ kwargs["cpu"] = True
+ elif setup == "state":
+ monkeypatch.setattr(launch.State, "_shared_state", {"distributed_type": "MULTI_GPU"})
+ elif setup == "sharded":
+ monkeypatch.setattr(launch.native, "_SHARDED_ANY", True)
+ elif setup == "single":
+ monkeypatch.setenv("WORLD_SIZE", "1")
+ launch.Accelerator(**kwargs)
+ assert launch.seen == [None]
+
+
+def test_install_is_idempotent(launch):
+ patched = launch.Accelerator.__init__
+ assert launch.module.install_raw_accelerator_fsdp2_fallback() is True
+ assert launch.Accelerator.__init__ is patched
+
+
+def test_positional_options_count_as_explicit(launch, monkeypatch):
+ # Accelerator(True, False, None, 1, True) selects the CPU positionally.
+ seen = []
+
+ class Accelerator:
+ def __init__(
+ self,
+ device_placement = None,
+ split_batches = None,
+ mixed_precision = None,
+ gradient_accumulation_steps = 1,
+ cpu = False,
+ deepspeed_plugin = None,
+ fsdp_plugin = None,
+ ):
+ seen.append(os.environ.get("ACCELERATE_USE_FSDP"))
+
+ monkeypatch.setattr(sys.modules["accelerate"], "Accelerator", Accelerator)
+ assert launch.module.install_raw_accelerator_fsdp2_fallback() is True
+ Accelerator(True, False, None, 1, True)
+ Accelerator(None, None, None, 1, False, None, object())
+ assert seen == [None, None]
+ Accelerator(True)
+ assert seen == [None, None, "true"]
diff --git a/unsloth/__init__.py b/unsloth/__init__.py
index 14aa836b937..a1577b2fc92 100644
--- a/unsloth/__init__.py
+++ b/unsloth/__init__.py
@@ -1693,6 +1693,17 @@ else:
except Exception:
pass
+ # UNSLOTH_FSDP2=1 shards natively inside the Trainer; a custom loop's own Accelerator keeps
+ # accelerate's FSDP2 (fsdp2_launch.install_raw_accelerator_fsdp2_fallback). Env reads only.
+ try:
+ from .fsdp2_launch import (
+ install_raw_accelerator_fsdp2_fallback as _install_raw_accelerator_fallback,
+ )
+ _install_raw_accelerator_fallback()
+ del _install_raw_accelerator_fallback
+ except Exception:
+ pass
+
def get_gpu_memory_stats():
"""Return CUDA/ROCm/XPU/NPU device stats, peak memory, and total memory in GiB."""
try:
diff --git a/unsloth/fsdp2_launch.py b/unsloth/fsdp2_launch.py
index 5d2b39e8d41..b558f3d3d8d 100644
--- a/unsloth/fsdp2_launch.py
+++ b/unsloth/fsdp2_launch.py
@@ -15,6 +15,7 @@ __all__ = [
"NATIVE_BACKEND_TABLE",
"native_backend_conflicts",
"refuse_native_backend_conflicts",
+ "install_raw_accelerator_fsdp2_fallback",
]
_TRUTHY = ("1", "true", "yes", "on")
@@ -325,6 +326,11 @@ def apply_unsloth_fsdp2_launch_env():
"(ACCELERATE_USE_FSDP and FSDP_VERSION unset)."
)
return False
+ return _export_accelerate_fsdp2_env()
+
+
+def _export_accelerate_fsdp2_env():
+ """What `accelerate launch` exports for an FSDP2 config (explicit FSDP_* values win)."""
if _env_flag("ACCELERATE_USE_DEEPSPEED"):
raise RuntimeError(
"Unsloth: UNSLOTH_FSDP2=1 was set on a DeepSpeed launch. Unset one of them."
@@ -352,3 +358,102 @@ def apply_unsloth_fsdp2_launch_env():
)
os.environ.setdefault("FSDP_STATE_DICT_TYPE", "FULL_STATE_DICT")
return True
+
+
+_RAW_ACCELERATOR_PATCHED = "_unsloth_fsdp2_raw_accelerator_fallback"
+
+
+def _built_by_trainer(frame, max_depth = 64):
+ """An Accelerator built inside Trainer.create_accelerator_and_postprocess (native FSDP2 shards the
+ Trainer's model itself), told apart from one a custom loop builds."""
+ while frame is not None and max_depth > 0:
+ if frame.f_code.co_name == "create_accelerator_and_postprocess":
+ owner = frame.f_locals.get("self", None)
+ if owner is not None and hasattr(owner, "args"):
+ return True
+ frame = frame.f_back
+ max_depth -= 1
+ return False
+
+
+def _raw_accelerator_wants_fsdp2(kwargs):
+ """UNSLOTH_FSDP2=1 picked native FSDP2 by default (no UNSLOTH_FSDP2_BACKEND), and this Accelerator
+ is the first one, built by a custom loop that chose no distributed plugin itself."""
+ if _env_value("UNSLOTH_FSDP2_BACKEND") != "" or not unsloth_fsdp2_native_requested():
+ return False
+ if any(
+ kwargs.get(name) is not None
+ for name in (
+ "fsdp_plugin",
+ "deepspeed_plugin",
+ "deepspeed_plugins",
+ "megatron_lm_plugin",
+ "torch_tp_plugin",
+ "parallelism_config",
+ )
+ ):
+ return False
+ if kwargs.get("cpu") or _env_flag("ACCELERATE_USE_CPU"):
+ return False # accelerate has no FSDP on the CPU
+ try:
+ from accelerate.state import AcceleratorState
+ if getattr(AcceleratorState, "_shared_state", {}):
+ return False # an earlier Accelerator already fixed the launch for this process
+ except Exception:
+ return False
+ native = sys.modules.get("unsloth.models.fsdp2_native", None)
+ if native is not None and getattr(native, "_SHARDED_ANY", False):
+ return False # shard_model already sharded a model natively
+ return True
+
+
+def install_raw_accelerator_fsdp2_fallback():
+ """UNSLOTH_FSDP2=1 shards natively inside the Trainer only. A custom loop that builds its own
+ Accelerator and calls accelerator.prepare(model, optimizer) on an unsharded model would get a
+ plain DDP wrap (no parameter or optimizer sharding). The first such Accelerator instead gets the
+ accelerate FSDP2 launch env that UNSLOTH_FSDP2=1 exported before native FSDP2 existed (the same as
+ UNSLOTH_FSDP2_BACKEND=accelerate). An explicit UNSLOTH_FSDP2_BACKEND is left alone. Idempotent."""
+ if not unsloth_fsdp2_requested() or _env_world_size() <= 1:
+ return False
+ try:
+ from accelerate import Accelerator
+ except Exception:
+ return False
+ if getattr(Accelerator, _RAW_ACCELERATOR_PATCHED, False):
+ return True
+ import functools
+ import inspect
+
+ original_init = Accelerator.__init__
+
+ try:
+ signature = inspect.signature(original_init)
+ except (TypeError, ValueError):
+ signature = None
+
+ def _named(self, args, kwargs):
+ # Options passed positionally count as explicit too.
+ if signature is None or not args:
+ return kwargs
+ try:
+ return dict(signature.bind_partial(self, *args, **kwargs).arguments)
+ except TypeError:
+ return kwargs
+
+ @functools.wraps(original_init)
+ def __init__(self, *args, **kwargs):
+ if _raw_accelerator_wants_fsdp2(_named(self, args, kwargs)) and not _built_by_trainer(
+ sys._getframe(1)
+ ):
+ _export_accelerate_fsdp2_env()
+ if str(os.environ.get("RANK", "0")).strip() in ("", "0"):
+ print(
+ "Unsloth: UNSLOTH_FSDP2=1 with an Accelerator built outside a Trainer: accelerate's "
+ "FSDP2 shards this loop (as UNSLOTH_FSDP2_BACKEND=accelerate). Set "
+ "UNSLOTH_FSDP2_BACKEND explicitly to choose the backend."
+ )
+ original_init(self, *args, **kwargs)
+
+ Accelerator.__init__ = __init__
+ setattr(Accelerator, _RAW_ACCELERATOR_PATCHED, True)
+ return True
diff --git a/unsloth/models/fsdp2_native.py b/unsloth/models/fsdp2_native.py
index ea42916aec3..36c0099af9c 100644
--- a/unsloth/models/fsdp2_native.py
+++ b/unsloth/models/fsdp2_native.py
@@ -48,6 +48,8 @@ __all__ = [
]
_PATCHED = "_unsloth_fsdp2_native_patched"
+# Set once shard_model sharded a model in this process.
+_SHARDED_ANY = False
_NATIVE_FLAG = "_unsloth_fsdp2_native"
# Leaf module types that are never a shard unit on their own, even when repeated in a ModuleList.
_LEAF_TYPES = (
@@ -1470,6 +1472,8 @@ def shard_model(model, config: Optional[FSDP2Config] = None):
from .fsdp2_save import install_dcp_bnb_state_dict
install_dcp_bnb_state_dict()
install_native_fsdp2_hooks()
+ global _SHARDED_ANY
+ _SHARDED_ANY = True # fsdp2_launch then leaves a custom loop's Accelerator alone
classes = sorted({type(u).__name__.removeprefix("FSDP") for u in wrapped})
note = ""
if qlora is not None:
@@ -1621,6 +1625,32 @@ def native_full_state_dict(
return get_model_state_dict(model, options = options)
+def _reshard_after_forward_off(unit):
+ """Keep `unit` gathered after its forward. FSDPModule.set_reshard_after_forward exists from torch
+ 2.8; on 2.6 / 2.7 set the param group's post-forward mesh the same way (None is what
+ reshard_after_forward = False resolves to). Returns what _reshard_after_forward_restore needs."""
+ if hasattr(unit, "set_reshard_after_forward"):
+ unit.set_reshard_after_forward(False, recurse = False)
+ return None
+ group = getattr(unit._get_fsdp_state(), "_fsdp_param_group", None)
+ if group is None:
+ return None
+ previous = group.post_forward_mesh_info
+ group.post_forward_mesh_info = None
+ return (group, previous)
+
+
+def _reshard_after_forward_restore(unit, saved):
+ if hasattr(unit, "set_reshard_after_forward"):
+ unit.set_reshard_after_forward(
+ getattr(unit, "_unsloth_reshard_after_forward", True), recurse = False
+ )
+ return
+ if saved is not None:
+ group, previous = saved
+ group.post_forward_mesh_info = previous
+
+
@contextlib.contextmanager
def native_fsdp2_unsharded(model):
"""Collective (every rank must enter): gather every unit's whole weights and keep them gathered,
@@ -1642,9 +1672,10 @@ def native_fsdp2_unsharded(model):
root.__dict__["_unsloth_unsharded_depth"] -= 1
return
units = [m for m in model.modules() if isinstance(m, FSDPModule)]
+ saved = {}
try:
for unit in units:
- unit.set_reshard_after_forward(False, recurse = False)
+ saved[id(unit)] = _reshard_after_forward_off(unit)
for unit in units:
unit.unshard()
yield model
@@ -1652,9 +1683,8 @@ def native_fsdp2_unsharded(model):
root.__dict__["_unsloth_unsharded_depth"] = 0
for unit in units:
unit.reshard()
- unit.set_reshard_after_forward(
- getattr(unit, "_unsloth_reshard_after_forward", True), recurse = False
- )
+ if id(unit) in saved:
+ _reshard_after_forward_restore(unit, saved[id(unit)])
@contextlib.contextmanager