Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Native FSDP2: keep accelerate FSDP2 for custom Accelerator loops, support unsharded generation on torch 2.6 / 2.7 (#14882)
Daniel Han482058cfb36e
diff --git a/tests/test_fsdp2_native_optim_reentry_gloo.py b/tests/test_fsdp2_native_optim_reentry_gloo.pyindex 426e7303505..e765cb20441 100644--- a/tests/test_fsdp2_native_optim_reentry_gloo.py+++ b/tests/test_fsdp2_native_optim_reentry_gloo.py@@ -117,9 +117,13 @@ def _case_optimizer_cls(native, tmp):def _case_reentry(native):+ model = native.shard_model(_tiny_llama(), native.FSDP2Config(param_dtype = torch.float32))+ return _case_reentry_on(native, model)+++def _case_reentry_on(native, model):from torch.distributed.tensor import DTensor- model = native.shard_model(_tiny_llama(), native.FSDP2Config(param_dtype = torch.float32))if getattr(model, "generation_config", None) is not None:model.generation_config.pad_token_id = None@@ -143,6 +147,30 @@ def _case_reentry(native):return res+def _case_reentry_no_setter(native):+ """torch 2.6 / 2.7: FSDPModule has no set_reshard_after_forward (added in 2.8)."""+ FSDPModule = native._fsdp_module_class()+ removed = {}+ for cls in FSDPModule.__mro__:+ if "set_reshard_after_forward" in cls.__dict__:+ removed[cls] = cls.__dict__["set_reshard_after_forward"]+ delattr(cls, "set_reshard_after_forward")+ try:+ model = native.shard_model(_tiny_llama(), native.FSDP2Config(param_dtype = torch.float32))+ units = [m for m in model.modules() if isinstance(m, FSDPModule)]+ meshes = [u._get_fsdp_state()._fsdp_param_group for u in units]+ before = [g.post_forward_mesh_info if g is not None else None for g in meshes]+ res = {"has_setter": hasattr(units[0], "set_reshard_after_forward")}+ res.update(_case_reentry_on(native, model))+ after = [g.post_forward_mesh_info if g is not None else None for g in meshes]+ res["mesh_restored"] = all(a is b for a, b in zip(before, after))+ res["units"] = len(units)+ return res+ finally:+ for cls, fn in removed.items():+ setattr(cls, "set_reshard_after_forward", fn)++def _worker_case(case, native, rank, tmp):if case == "stateful":return _case_stateful(native, tmp)@@ -150,6 +178,8 @@ def _worker_case(case, native, rank, tmp):return _case_optimizer_cls(native, tmp)if case == "reentry":return _case_reentry(native)+ if case == "reentry_no_setter":+ return _case_reentry_no_setter(native)raise KeyError(case)@@ -246,6 +276,18 @@ def test_bnb_optimizer_cls_and_kwargs_is_refused(tmp_path):assert not v["sharded"], (rank, key, v)+def test_unsharded_context_without_reshard_setter(tmp_path):+ """torch 2.6 / 2.7 FSDPModule has no set_reshard_after_forward: the gather / reshard still runs+ and every unit's post-forward mesh comes back unchanged."""+ pytest.importorskip("transformers")+ for rank, r in _run("reentry_no_setter", tmp_path).items():+ assert r["has_setter"] is False and r["units"] > 1, (rank, r)+ assert r["before"] > 0 and r["outer"] == 0, (rank, r)+ assert r["after_generate"] == 0 and r["after_inner_raise"] == 0, (rank, r)+ assert r["after_outer"] == r["before"] and r["depth"] == 0, (rank, r)+ assert r["mesh_restored"], (rank, r)++def test_unsharded_context_is_reentrant(tmp_path):pytest.importorskip("transformers")for rank, r in _run("reentry", tmp_path).items():diff --git a/tests/test_fsdp2_native_raw_accelerator.py b/tests/test_fsdp2_native_raw_accelerator.pynew file mode 100644index 00000000000..2205da0d791--- /dev/null+++ b/tests/test_fsdp2_native_raw_accelerator.py@@ -0,0 +1,157 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""UNSLOTH_FSDP2=1 shards natively inside the Trainer only. A custom loop's own Accelerator+(accelerator.prepare(model, optimizer) on an unsharded model) keeps accelerate's FSDP2, as before+native FSDP2, instead of a plain DDP wrap: fsdp2_launch.install_raw_accelerator_fsdp2_fallback+exports the accelerate FSDP2 launch env for it. A Trainer's accelerator, an explicit+UNSLOTH_FSDP2_BACKEND, an explicit plugin, a CPU run, an already initialized AcceleratorState and an+already natively sharded model are left alone. CPU only: accelerate is replaced by a recorder."""++import os+import sys+import types++import pytest++sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))+from test_fsdp2_native_units import _load_native # noqa: E402++_ENV = (+ "ACCELERATE_USE_FSDP",+ "FSDP_VERSION",+ "FSDP_AUTO_WRAP_POLICY",+ "FSDP_STATE_DICT_TYPE",+ "FSDP_RESHARD_AFTER_FORWARD",+ "ACCELERATE_USE_CPU",+ "ACCELERATE_USE_DEEPSPEED",+ "UNSLOTH_FSDP2_BACKEND",+ "UNSLOTH_FSDP2_RESHARD_AFTER_FORWARD",+)+++def launch(monkeypatch):+ for key in _ENV:+ # setenv first so teardown also drops what the fallback exports+ monkeypatch.setenv(key, "")+ monkeypatch.delenv(key)+ for key, value in dict(UNSLOTH_FSDP2 = "1", WORLD_SIZE = "2", RANK = "0").items():+ monkeypatch.setenv(key, value)+ native = _load_native()+ monkeypatch.setattr(native, "_SHARDED_ANY", False)+ seen = []++ class Accelerator:+ def __init__(self, *args, **kwargs):+ seen.append(os.environ.get("ACCELERATE_USE_FSDP"))++ class AcceleratorState:+ _shared_state = {}++ accelerate = types.ModuleType("accelerate")+ accelerate.Accelerator = Accelerator+ state = types.ModuleType("accelerate.state")+ state.AcceleratorState = AcceleratorState+ accelerate.state = state+ monkeypatch.setitem(sys.modules, "accelerate", accelerate)+ monkeypatch.setitem(sys.modules, "accelerate.state", state)+ module = sys.modules["unsloth.fsdp2_launch"]+ monkeypatch.setattr(module, "_accelerate_fsdp2_support", lambda: True)+ assert module.install_raw_accelerator_fsdp2_fallback() is True+ return types.SimpleNamespace(+ module = module, native = native, Accelerator = Accelerator, State = AcceleratorState, seen = seen+ )+++def test_custom_loop_accelerator_gets_accelerate_fsdp2(launch):+ assert launch.module.unsloth_fsdp2_native_requested()+ launch.Accelerator()+ assert launch.seen == ["true"]+ assert os.environ["FSDP_VERSION"] == "2"+ assert os.environ["FSDP_AUTO_WRAP_POLICY"] == "TRANSFORMER_BASED_WRAP"+ assert os.environ["FSDP_STATE_DICT_TYPE"] == "FULL_STATE_DICT"+ # From here on the process is the accelerate FSDP2 backend, also for a later Trainer.+ assert not launch.module.unsloth_fsdp2_native_requested()+++def test_trainer_accelerator_stays_native(launch):+ class Trainer:+ args = object()++ def create_accelerator_and_postprocess(self):+ self.accelerator = launch.Accelerator()++ Trainer().create_accelerator_and_postprocess()+ assert launch.seen == [None]+ assert launch.module.unsloth_fsdp2_native_requested()+++ "setup",+ [+ "backend_native",+ "backend_accelerate",+ "plugin",+ "tp_plugin",+ "parallelism_config",+ "cpu",+ "state",+ "sharded",+ "single",+ ],+)+def test_left_alone(launch, monkeypatch, setup):+ kwargs = {}+ if setup == "backend_native":+ monkeypatch.setenv("UNSLOTH_FSDP2_BACKEND", "native")+ elif setup == "backend_accelerate":+ monkeypatch.setenv("UNSLOTH_FSDP2_BACKEND", "accelerate")+ elif setup == "plugin":+ kwargs["fsdp_plugin"] = object()+ elif setup == "tp_plugin":+ kwargs["torch_tp_plugin"] = object()+ elif setup == "parallelism_config":+ kwargs["parallelism_config"] = object()+ elif setup == "cpu":+ kwargs["cpu"] = True+ elif setup == "state":+ monkeypatch.setattr(launch.State, "_shared_state", {"distributed_type": "MULTI_GPU"})+ elif setup == "sharded":+ monkeypatch.setattr(launch.native, "_SHARDED_ANY", True)+ elif setup == "single":+ monkeypatch.setenv("WORLD_SIZE", "1")+ launch.Accelerator(**kwargs)+ assert launch.seen == [None]+++def test_install_is_idempotent(launch):+ patched = launch.Accelerator.__init__+ assert launch.module.install_raw_accelerator_fsdp2_fallback() is True+ assert launch.Accelerator.__init__ is patched+++def test_positional_options_count_as_explicit(launch, monkeypatch):+ # Accelerator(True, False, None, 1, True) selects the CPU positionally.+ seen = []++ class Accelerator:+ def __init__(+ self,+ device_placement = None,+ split_batches = None,+ mixed_precision = None,+ gradient_accumulation_steps = 1,+ cpu = False,+ deepspeed_plugin = None,+ fsdp_plugin = None,+ ):+ seen.append(os.environ.get("ACCELERATE_USE_FSDP"))++ monkeypatch.setattr(sys.modules["accelerate"], "Accelerator", Accelerator)+ assert launch.module.install_raw_accelerator_fsdp2_fallback() is True+ Accelerator(True, False, None, 1, True)+ Accelerator(None, None, None, 1, False, None, object())+ assert seen == [None, None]+ Accelerator(True)+ assert seen == [None, None, "true"]diff --git a/unsloth/__init__.py b/unsloth/__init__.pyindex 14aa836b937..a1577b2fc92 100644--- a/unsloth/__init__.py+++ b/unsloth/__init__.py@@ -1693,6 +1693,17 @@ else:except Exception:pass+ # UNSLOTH_FSDP2=1 shards natively inside the Trainer; a custom loop's own Accelerator keeps+ # accelerate's FSDP2 (fsdp2_launch.install_raw_accelerator_fsdp2_fallback). Env reads only.+ try:+ from .fsdp2_launch import (+ install_raw_accelerator_fsdp2_fallback as _install_raw_accelerator_fallback,+ )+ _install_raw_accelerator_fallback()+ del _install_raw_accelerator_fallback+ except Exception:+ pass+def get_gpu_memory_stats():"""Return CUDA/ROCm/XPU/NPU device stats, peak memory, and total memory in GiB."""try:diff --git a/unsloth/fsdp2_launch.py b/unsloth/fsdp2_launch.pyindex 5d2b39e8d41..b558f3d3d8d 100644--- a/unsloth/fsdp2_launch.py+++ b/unsloth/fsdp2_launch.py@@ -15,6 +15,7 @@ __all__ = ["NATIVE_BACKEND_TABLE","native_backend_conflicts","refuse_native_backend_conflicts",+ "install_raw_accelerator_fsdp2_fallback",]_TRUTHY = ("1", "true", "yes", "on")@@ -325,6 +326,11 @@ def apply_unsloth_fsdp2_launch_env():"(ACCELERATE_USE_FSDP and FSDP_VERSION unset).")return False+ return _export_accelerate_fsdp2_env()+++def _export_accelerate_fsdp2_env():+ """What `accelerate launch` exports for an FSDP2 config (explicit FSDP_* values win)."""if _env_flag("ACCELERATE_USE_DEEPSPEED"):raise RuntimeError("Unsloth: UNSLOTH_FSDP2=1 was set on a DeepSpeed launch. Unset one of them."@@ -352,3 +358,102 @@ def apply_unsloth_fsdp2_launch_env():)os.environ.setdefault("FSDP_STATE_DICT_TYPE", "FULL_STATE_DICT")return True+++_RAW_ACCELERATOR_PATCHED = "_unsloth_fsdp2_raw_accelerator_fallback"+++def _built_by_trainer(frame, max_depth = 64):+ """An Accelerator built inside Trainer.create_accelerator_and_postprocess (native FSDP2 shards the+ Trainer's model itself), told apart from one a custom loop builds."""+ while frame is not None and max_depth > 0:+ if frame.f_code.co_name == "create_accelerator_and_postprocess":+ owner = frame.f_locals.get("self", None)+ if owner is not None and hasattr(owner, "args"):+ return True+ frame = frame.f_back+ max_depth -= 1+ return False+++def _raw_accelerator_wants_fsdp2(kwargs):+ """UNSLOTH_FSDP2=1 picked native FSDP2 by default (no UNSLOTH_FSDP2_BACKEND), and this Accelerator+ is the first one, built by a custom loop that chose no distributed plugin itself."""+ if _env_value("UNSLOTH_FSDP2_BACKEND") != "" or not unsloth_fsdp2_native_requested():+ return False+ if any(+ kwargs.get(name) is not None+ for name in (+ "fsdp_plugin",+ "deepspeed_plugin",+ "deepspeed_plugins",+ "megatron_lm_plugin",+ "torch_tp_plugin",+ "parallelism_config",+ )+ ):+ return False+ if kwargs.get("cpu") or _env_flag("ACCELERATE_USE_CPU"):+ return False # accelerate has no FSDP on the CPU+ try:+ from accelerate.state import AcceleratorState+ if getattr(AcceleratorState, "_shared_state", {}):+ return False # an earlier Accelerator already fixed the launch for this process+ except Exception:+ return False+ native = sys.modules.get("unsloth.models.fsdp2_native", None)+ if native is not None and getattr(native, "_SHARDED_ANY", False):+ return False # shard_model already sharded a model natively+ return True+++def install_raw_accelerator_fsdp2_fallback():+ """UNSLOTH_FSDP2=1 shards natively inside the Trainer only. A custom loop that builds its own+ Accelerator and calls accelerator.prepare(model, optimizer) on an unsharded model would get a+ plain DDP wrap (no parameter or optimizer sharding). The first such Accelerator instead gets the+ accelerate FSDP2 launch env that UNSLOTH_FSDP2=1 exported before native FSDP2 existed (the same as+ UNSLOTH_FSDP2_BACKEND=accelerate). An explicit UNSLOTH_FSDP2_BACKEND is left alone. Idempotent."""+ if not unsloth_fsdp2_requested() or _env_world_size() <= 1:+ return False+ try:+ from accelerate import Accelerator+ except Exception:+ return False+ if getattr(Accelerator, _RAW_ACCELERATOR_PATCHED, False):+ return True+ import functools+ import inspect++ original_init = Accelerator.__init__++ try:+ signature = inspect.signature(original_init)+ except (TypeError, ValueError):+ signature = None++ def _named(self, args, kwargs):+ # Options passed positionally count as explicit too.+ if signature is None or not args:+ return kwargs+ try:+ return dict(signature.bind_partial(self, *args, **kwargs).arguments)+ except TypeError:+ return kwargs++ @functools.wraps(original_init)+ def __init__(self, *args, **kwargs):+ if _raw_accelerator_wants_fsdp2(_named(self, args, kwargs)) and not _built_by_trainer(+ sys._getframe(1)+ ):+ _export_accelerate_fsdp2_env()+ if str(os.environ.get("RANK", "0")).strip() in ("", "0"):+ print(+ "Unsloth: UNSLOTH_FSDP2=1 with an Accelerator built outside a Trainer: accelerate's "+ "FSDP2 shards this loop (as UNSLOTH_FSDP2_BACKEND=accelerate). Set "+ "UNSLOTH_FSDP2_BACKEND explicitly to choose the backend."+ )+ original_init(self, *args, **kwargs)++ Accelerator.__init__ = __init__+ setattr(Accelerator, _RAW_ACCELERATOR_PATCHED, True)+ return Truediff --git a/unsloth/models/fsdp2_native.py b/unsloth/models/fsdp2_native.pyindex ea42916aec3..36c0099af9c 100644--- a/unsloth/models/fsdp2_native.py+++ b/unsloth/models/fsdp2_native.py@@ -48,6 +48,8 @@ __all__ = []_PATCHED = "_unsloth_fsdp2_native_patched"+# Set once shard_model sharded a model in this process.+_SHARDED_ANY = False_NATIVE_FLAG = "_unsloth_fsdp2_native"# Leaf module types that are never a shard unit on their own, even when repeated in a ModuleList._LEAF_TYPES = (@@ -1470,6 +1472,8 @@ def shard_model(model, config: Optional[FSDP2Config] = None):from .fsdp2_save import install_dcp_bnb_state_dictinstall_dcp_bnb_state_dict()install_native_fsdp2_hooks()+ global _SHARDED_ANY+ _SHARDED_ANY = True # fsdp2_launch then leaves a custom loop's Accelerator aloneclasses = sorted({type(u).__name__.removeprefix("FSDP") for u in wrapped})note = ""if qlora is not None:@@ -1621,6 +1625,32 @@ def native_full_state_dict(return get_model_state_dict(model, options = options)+def _reshard_after_forward_off(unit):+ """Keep `unit` gathered after its forward. FSDPModule.set_reshard_after_forward exists from torch+ 2.8; on 2.6 / 2.7 set the param group's post-forward mesh the same way (None is what+ reshard_after_forward = False resolves to). Returns what _reshard_after_forward_restore needs."""+ if hasattr(unit, "set_reshard_after_forward"):+ unit.set_reshard_after_forward(False, recurse = False)+ return None+ group = getattr(unit._get_fsdp_state(), "_fsdp_param_group", None)+ if group is None:+ return None+ previous = group.post_forward_mesh_info+ group.post_forward_mesh_info = None+ return (group, previous)+++def _reshard_after_forward_restore(unit, saved):+ if hasattr(unit, "set_reshard_after_forward"):+ unit.set_reshard_after_forward(+ getattr(unit, "_unsloth_reshard_after_forward", True), recurse = False+ )+ return+ if saved is not None:+ group, previous = saved+ group.post_forward_mesh_info = previous++@contextlib.contextmanagerdef native_fsdp2_unsharded(model):"""Collective (every rank must enter): gather every unit's whole weights and keep them gathered,@@ -1642,9 +1672,10 @@ def native_fsdp2_unsharded(model):root.__dict__["_unsloth_unsharded_depth"] -= 1returnunits = [m for m in model.modules() if isinstance(m, FSDPModule)]+ saved = {}try:for unit in units:- unit.set_reshard_after_forward(False, recurse = False)+ saved[id(unit)] = _reshard_after_forward_off(unit)for unit in units:unit.unshard()yield model@@ -1652,9 +1683,8 @@ def native_fsdp2_unsharded(model):root.__dict__["_unsloth_unsharded_depth"] = 0for unit in units:unit.reshard()- unit.set_reshard_after_forward(- getattr(unit, "_unsloth_reshard_after_forward", True), recurse = False- )+ if id(unit) in saved:+ _reshard_after_forward_restore(unit, saved[id(unit)])@contextlib.contextmanager