Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

FSDP2: first-class LoRA and full finetuning, keep fused LoRA kernels per block (#13865)

Daniel Hanee2ce8ccf8e3
diff --git a/tests/test_fsdp2_config.py b/tests/test_fsdp2_config.py
new file mode 100644
index 00000000000..7543bdd840a
--- /dev/null
+++ b/tests/test_fsdp2_config.py
@@ -0,0 +1,295 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""What Unsloth fills into accelerate's FSDP2 config (unsloth/models/fsdp2.py). No GPU."""
+
+from __future__ import annotations
+
+import contextlib
+import types
+
+import pytest
+
+torch = pytest.importorskip("torch")
+pytest.importorskip("accelerate")
+fsdp = pytest.importorskip("torch.distributed.fsdp")
+MixedPrecisionPolicy = getattr(fsdp, "MixedPrecisionPolicy", None)
+if MixedPrecisionPolicy is None:
+ pytest.skip("torch without FSDP2", allow_module_level = True)
+
+from unsloth.models import fsdp2 # noqa: E402
+
+
[email protected](autouse = True)
+def _clean_env(monkeypatch):
+ for key in ("UNSLOTH_FSDP2_ACCELERATE_DEFAULTS", "UNSLOTH_FSDP2_FUSED_LORA"):
+ monkeypatch.delenv(key, raising = False)
+
+
+def _accelerator(
+ policy,
+ mixed_precision = "fp16",
+ version = 2,
+ auto_wrap_policy = object(),
+ auto = True,
+ seen = True,
+):
+ plugin = types.SimpleNamespace(
+ fsdp_version = version,
+ mixed_precision_policy = policy,
+ auto_wrap_policy = auto_wrap_policy,
+ reshard_after_forward = True,
+ )
+ if auto and seen:
+ setattr(plugin, fsdp2._AUTO_POLICY, policy)
+ if seen:
+ setattr(plugin, fsdp2._POLICY_SEEN, True)
+ return types.SimpleNamespace(
+ state = types.SimpleNamespace(fsdp_plugin = plugin), mixed_precision = mixed_precision
+ )
+
+
[email protected]("dtype,name", [(torch.float16, "fp16"), (torch.bfloat16, "bf16")])
+def test_accelerates_auto_policy_reduces_in_fp32(dtype, name):
+ acc = _accelerator(
+ MixedPrecisionPolicy(param_dtype = dtype, reduce_dtype = dtype, output_dtype = dtype), name
+ )
+ policy = fsdp2.fill_fsdp2_mixed_precision(acc)
+ # float16 gathers the stored dtype and leaves the casting to autocast, like the unsharded run.
+ assert policy.param_dtype == (None if dtype == torch.float16 else dtype)
+ assert policy.reduce_dtype == torch.float32
+ assert policy.output_dtype is None
+ assert acc.state.fsdp_plugin.mixed_precision_policy is policy
+
+
+def test_a_policy_the_user_wrote_is_left_alone():
+ user = MixedPrecisionPolicy(param_dtype = torch.float16, reduce_dtype = torch.float16)
+ acc = _accelerator(user, "bf16", auto = False)
+ assert fsdp2.fill_fsdp2_mixed_precision(acc) is user
+ user = MixedPrecisionPolicy(param_dtype = None, reduce_dtype = torch.float32)
+ acc = _accelerator(user, "fp16", auto = False)
+ assert fsdp2.fill_fsdp2_mixed_precision(acc) is user
+ # Same dtypes as accelerate's automatic policy, but written by the user.
+ user = MixedPrecisionPolicy(
+ param_dtype = torch.float16, reduce_dtype = torch.float16, output_dtype = torch.float16
+ )
+ acc = _accelerator(user, "fp16", auto = False)
+ assert fsdp2.fill_fsdp2_mixed_precision(acc) is user
+
+
+def test_an_accelerator_built_before_unsloth_import_is_not_guessed(monkeypatch):
+ """Without provenance, an fp16 policy shaped like accelerate's automatic one is refused, not rewritten."""
+ shaped = dict(param_dtype = torch.float16, reduce_dtype = torch.float16, output_dtype = torch.float16)
+ with pytest.raises(RuntimeError, match = "before `import unsloth`"):
+ fsdp2.fill_fsdp2_mixed_precision(
+ _accelerator(MixedPrecisionPolicy(**shaped), "fp16", seen = False)
+ )
+ bf16 = MixedPrecisionPolicy(
+ param_dtype = torch.bfloat16, reduce_dtype = torch.bfloat16, output_dtype = torch.bfloat16
+ )
+ assert fsdp2.fill_fsdp2_mixed_precision(_accelerator(bf16, "bf16", seen = False)) is bf16
+ other = MixedPrecisionPolicy(param_dtype = None, reduce_dtype = torch.float32)
+ assert fsdp2.fill_fsdp2_mixed_precision(_accelerator(other, "fp16", seen = False)) is other
+ monkeypatch.setenv("UNSLOTH_FSDP2_ACCELERATE_DEFAULTS", "1")
+ kept = MixedPrecisionPolicy(**shaped)
+ assert fsdp2.fill_fsdp2_mixed_precision(_accelerator(kept, "fp16", seen = False)) is kept
+
+
+def test_provenance_tags_only_the_policy_accelerate_builds():
+ from accelerate.utils import FullyShardedDataParallelPlugin
+
+ assert fsdp2.install_fsdp2_policy_provenance()
+ plugin = types.SimpleNamespace(fsdp_version = 2, mixed_precision_policy = None)
+ FullyShardedDataParallelPlugin.set_mixed_precision(plugin, "fp16")
+ assert getattr(plugin, fsdp2._AUTO_POLICY) is plugin.mixed_precision_policy
+ user = MixedPrecisionPolicy(
+ param_dtype = torch.float16, reduce_dtype = torch.float16, output_dtype = torch.float16
+ )
+ plugin = types.SimpleNamespace(fsdp_version = 2, mixed_precision_policy = user)
+ FullyShardedDataParallelPlugin.set_mixed_precision(plugin, "fp16")
+ assert plugin.mixed_precision_policy is user
+ assert not hasattr(plugin, fsdp2._AUTO_POLICY)
+ assert getattr(plugin, fsdp2._POLICY_SEEN) is True
+
+
+def test_no_mixed_precision_and_the_opt_out(monkeypatch):
+ auto = MixedPrecisionPolicy(
+ param_dtype = torch.float16, reduce_dtype = torch.float16, output_dtype = torch.float16
+ )
+ assert fsdp2.fill_fsdp2_mixed_precision(_accelerator(None, "no")) is None
+ monkeypatch.setenv("UNSLOTH_FSDP2_ACCELERATE_DEFAULTS", "1")
+ assert fsdp2.fill_fsdp2_mixed_precision(_accelerator(auto, "fp16")) is auto
+
+
+def test_trainer_setup_keeps_fused_lora_only_for_a_per_block_fsdp2_wrap(monkeypatch):
+ installed = []
+ monkeypatch.setattr(fsdp2, "install_fsdp2_prepare_hook", lambda: installed.append("prepare"))
+ monkeypatch.setattr(fsdp2, "install_fsdp2_no_sync", lambda: installed.append("no_sync"))
+
+ def trainer(**kw):
+ return types.SimpleNamespace(accelerator = _accelerator(None, **kw))
+
+ assert fsdp2.fsdp2_trainer_setup(trainer(version = 1)) == (False, None)
+ assert installed == []
+ assert fsdp2.fsdp2_trainer_setup(trainer()) == (True, None)
+ assert installed == ["prepare", "no_sync"]
+ keep, reason = fsdp2.fsdp2_trainer_setup(trainer(auto_wrap_policy = None))
+ assert keep is False and "root" in reason
+ monkeypatch.setenv("UNSLOTH_FSDP2_FUSED_LORA", "0")
+ keep, reason = fsdp2.fsdp2_trainer_setup(trainer())
+ assert keep is False and "UNSLOTH_FSDP2_FUSED_LORA" in reason
+
+
+def test_old_accelerate_no_sync_gets_fsdp2_gradient_sync(monkeypatch):
+ """accelerate < 1.11 only looked for `model.no_sync`, which FSDP2 modules lack."""
+ from accelerate import Accelerator
+
+ @contextlib.contextmanager
+ def old_no_sync(self, model):
+ context = getattr(model, "no_sync", contextlib.nullcontext)
+ with context():
+ yield
+
+ monkeypatch.setattr(Accelerator, "no_sync", old_no_sync)
+ assert fsdp2.install_fsdp2_no_sync() is True
+ assert Accelerator.no_sync is not old_no_sync
+ assert fsdp2.install_fsdp2_no_sync() is True # idempotent
+
+ calls = []
+ model = types.SimpleNamespace(set_requires_gradient_sync = calls.append)
+ fake = types.SimpleNamespace(is_fsdp2 = True)
+ with Accelerator.no_sync(fake, model):
+ assert calls == [False]
+ assert calls == [False, True]
+ fake.is_fsdp2 = False
+ with Accelerator.no_sync(fake, model):
+ pass
+ assert calls == [False, True]
+
+
+def test_current_accelerate_no_sync_is_not_touched(monkeypatch):
+ from accelerate import Accelerator
+ import inspect
+
+ original = Accelerator.no_sync
+ if "set_requires_gradient_sync" not in inspect.getsource(original):
+ pytest.skip("accelerate < 1.11 in this environment")
+ assert fsdp2.install_fsdp2_no_sync() is True
+ assert Accelerator.no_sync is original
+
+
+def test_the_trainer_init_wrapper_routes_fsdp_through_the_setup():
+ import inspect
+
+ from unsloth.models import _utils
+
+ source = inspect.getsource(_utils.patch_gradient_accumulation_fix)
+ assert "fsdp2_trainer_setup" in source and "_decline_fused_lora_for_fsdp" in source
+
+
+def test_bf16_full_finetuning_gets_one_trainable_dtype_per_unit():
+ """Unsloth's bfloat16 full finetuning: no autocast, bf16 linears, fp32 norms (FSDP2 refuses mixed units)."""
+ model = torch.nn.Sequential(torch.nn.Linear(4, 4).to(torch.bfloat16), torch.nn.LayerNorm(4))
+ frozen = torch.nn.Linear(4, 4).to(torch.bfloat16).requires_grad_(False)
+ model.append(frozen)
+ acc = _accelerator(None, "no")
+ assert fsdp2.unify_trainable_dtypes(acc, model) is True
+ assert {p.dtype for p in model.parameters() if p.requires_grad} == {torch.float32}
+ assert frozen.weight.dtype == torch.bfloat16
+ policy = acc.state.fsdp_plugin.mixed_precision_policy
+ assert policy.param_dtype == torch.bfloat16 and policy.reduce_dtype == torch.float32
+
+
[email protected]("explicit", [False, True])
+def test_fp16_full_finetuning_gets_one_trainable_dtype_per_unit(explicit):
+ model = torch.nn.Sequential(torch.nn.Linear(4, 4).to(torch.float16), torch.nn.LayerNorm(4))
+ user = (
+ MixedPrecisionPolicy(param_dtype = torch.float16, reduce_dtype = torch.float32)
+ if explicit
+ else None
+ )
+ acc = _accelerator(user, "no", auto = False)
+ assert fsdp2.unify_trainable_dtypes(acc, model) is True
+ assert {p.dtype for p in model.parameters()} == {torch.float32}
+ policy = acc.state.fsdp_plugin.mixed_precision_policy
+ if explicit:
+ assert policy is user
+ assert policy.param_dtype == torch.float16 and policy.reduce_dtype == torch.float32
+
+
+def test_uniform_or_autocast_runs_are_not_upcast():
+ model = torch.nn.Sequential(torch.nn.Linear(4, 4).to(torch.bfloat16))
+ assert fsdp2.unify_trainable_dtypes(_accelerator(None, "no"), model) is False
+ assert model[0].weight.dtype == torch.bfloat16
+ uniform = torch.nn.Sequential(torch.nn.Linear(4, 4).to(torch.bfloat16))
+ acc = _accelerator(None, "bf16")
+ policy = acc.state.fsdp_plugin.mixed_precision_policy
+ # Under autocast a uniform unit is left to accelerate's own upcast.
+ assert fsdp2.unify_trainable_dtypes(acc, uniform) is False
+ assert uniform[0].weight.dtype == torch.bfloat16
+ assert acc.state.fsdp_plugin.mixed_precision_policy is policy
+
+
+def test_mixed_precision_masters_are_unified_when_accelerate_does_not():
+ """accelerate 1.13's upcast is `param = param.to(torch.float32)`, which leaves the model alone."""
+ mixed = torch.nn.Sequential(
+ torch.nn.Linear(4, 4).to(torch.bfloat16),
+ torch.nn.LayerNorm(4),
+ torch.nn.Linear(4, 4).to(torch.bfloat16).requires_grad_(False),
+ )
+ acc = _accelerator(None, "bf16")
+ policy = acc.state.fsdp_plugin.mixed_precision_policy
+ assert fsdp2.unify_trainable_dtypes(acc, mixed) is True
+ assert {p.dtype for p in mixed.parameters() if p.requires_grad} == {torch.float32}
+ assert mixed[2].weight.dtype == torch.bfloat16
+ assert acc.state.fsdp_plugin.mixed_precision_policy is policy
+
+
+def test_install_fsdp2_hooks_patches_accelerate_without_a_trainer():
+ import accelerate.accelerator as accelerator_module
+ from accelerate import Accelerator
+ from accelerate.utils import FullyShardedDataParallelPlugin
+
+ fsdp2.install_fsdp2_hooks()
+ assert getattr(accelerator_module.fsdp2_prepare_model, fsdp2._PATCHED, False)
+ assert getattr(FullyShardedDataParallelPlugin.set_mixed_precision, fsdp2._PATCHED, False)
+ assert hasattr(Accelerator, "no_sync")
+ import inspect
+
+ import accelerate.utils.fsdp_utils as fsdp_utils
+
+ get = fsdp_utils._get_model_state_dict
+ assert getattr(get, fsdp2._PATCHED, False) or "full_state_dict" in inspect.getsource(get)
+
+
+def test_an_explicit_policy_still_gets_float32_master_weights():
+ """Accelerator mixed_precision="no" with a launcher policy: the masters still need one dtype."""
+ model = torch.nn.Sequential(torch.nn.Linear(4, 4).to(torch.bfloat16), torch.nn.LayerNorm(4))
+ user = MixedPrecisionPolicy(param_dtype = torch.bfloat16, reduce_dtype = torch.bfloat16)
+ acc = _accelerator(user, "no", auto = False)
+ assert fsdp2.unify_trainable_dtypes(acc, model) is True
+ assert {p.dtype for p in model.parameters()} == {torch.float32}
+ assert acc.state.fsdp_plugin.mixed_precision_policy is user
+
+
+def _lora_like():
+ model = torch.nn.Sequential(torch.nn.Linear(4, 4).to(torch.bfloat16), torch.nn.Linear(4, 4))
+ model[0].requires_grad_(False)
+ return model
+
+
+def test_frozen_weights_are_upcast_only_where_torch_checks_them():
+ policy = MixedPrecisionPolicy(param_dtype = torch.bfloat16, reduce_dtype = torch.float32)
+ model = _lora_like()
+ assert (
+ fsdp2.unify_frozen_dtypes(_accelerator(policy, "bf16"), model, frozen_checked = False)
+ is False
+ )
+ assert model[0].weight.dtype == torch.bfloat16
+ assert (
+ fsdp2.unify_frozen_dtypes(_accelerator(policy, "bf16"), model, frozen_checked = True) is True
+ )
+ assert {p.dtype for p in model.parameters()} == {torch.float32}
+ assert not model[0].weight.requires_grad
+ with pytest.raises(RuntimeError, match = "torch >= 2.8"):
+ fsdp2.unify_frozen_dtypes(_accelerator(None, "no"), _lora_like(), frozen_checked = True)
diff --git a/tests/test_fsdp2_detection.py b/tests/test_fsdp2_detection.py
new file mode 100644
index 00000000000..ef6057c8f50
--- /dev/null
+++ b/tests/test_fsdp2_detection.py
@@ -0,0 +1,307 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""FSDP2 detection in every launch mode: `accelerate launch` (ACCELERATE_USE_FSDP + FSDP_VERSION),
+FSDP_VERSION by hand, UNSLOTH_FSDP2=1 under plain torchrun, and a built Accelerator. No GPU."""
+
+from __future__ import annotations
+
+import ast
+import importlib.util
+import os
+import socket
+from pathlib import Path
+
+import pytest
+
+_ROOT = Path(__file__).resolve().parent.parent
+_LAUNCH = _ROOT / "unsloth" / "fsdp2_launch.py"
+
+_ENV = (
+ "UNSLOTH_FSDP2",
+ "UNSLOTH_FSDP2_RESHARD_AFTER_FORWARD",
+ "UNSLOTH_FSDP2_FUSED_LORA",
+ "UNSLOTH_FORCE_FUSED_LORA",
+ "ACCELERATE_USE_FSDP",
+ "ACCELERATE_USE_DEEPSPEED",
+ "FSDP_VERSION",
+ "FSDP_AUTO_WRAP_POLICY",
+ "FSDP_RESHARD_AFTER_FORWARD",
+ "FSDP_STATE_DICT_TYPE",
+ "WORLD_SIZE",
+)
+
+
+def _load_launch():
+ # By path: the module needs neither torch nor the unsloth package.
+ spec = importlib.util.spec_from_file_location("_unsloth_fsdp2_launch", _LAUNCH)
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+ return module
+
+
[email protected](autouse = True)
+def _clean_env(monkeypatch):
+ for key in _ENV:
+ monkeypatch.delenv(key, raising = False)
+
+
+def launch():
+ return _load_launch()
+
+
+def test_no_launcher_env_is_no_fsdp(launch):
+ assert launch.fsdp_version_from_env() == 0
+ assert launch.apply_unsloth_fsdp2_launch_env() is False
+ assert "ACCELERATE_USE_FSDP" not in os.environ
+
+
+ "use_fsdp,version,expected",
+ [
+ ("true", "2", 2),
+ ("true", "1", 1),
+ ("true", "", 1), # accelerate builds FSDP1 when no version is named
+ ("", "2", 2), # torchrun users export FSDP_VERSION by hand
+ ("", "0", 0),
+ ("false", "", 0),
+ ],
+)
+def test_accelerate_launch_env(monkeypatch, launch, use_fsdp, version, expected):
+ monkeypatch.setenv("ACCELERATE_USE_FSDP", use_fsdp)
+ monkeypatch.setenv("FSDP_VERSION", version)
+ assert launch.fsdp_version_from_env() == expected
+
+
+def test_unsloth_fsdp2_needs_more_than_one_process(monkeypatch, launch):
+ monkeypatch.setenv("UNSLOTH_FSDP2", "1")
+ assert launch.fsdp_version_from_env() == 0
+ assert launch.apply_unsloth_fsdp2_launch_env() is False
+ monkeypatch.setenv("WORLD_SIZE", "1")
+ assert launch.apply_unsloth_fsdp2_launch_env() is False
+ assert "FSDP_VERSION" not in os.environ
+
+
+def test_unsloth_fsdp2_exports_the_accelerate_fsdp2_launch(monkeypatch, launch):
+ monkeypatch.setenv("UNSLOTH_FSDP2", "1")
+ monkeypatch.setenv("WORLD_SIZE", "2")
+ assert launch.fsdp_version_from_env() == 2
+ assert launch.apply_unsloth_fsdp2_launch_env() is True
+ assert os.environ["ACCELERATE_USE_FSDP"] == "true"
+ assert os.environ["FSDP_VERSION"] == "2"
+ assert os.environ["FSDP_AUTO_WRAP_POLICY"] == "TRANSFORMER_BASED_WRAP"
+ assert os.environ["FSDP_STATE_DICT_TYPE"] == "FULL_STATE_DICT"
+ assert "FSDP_RESHARD_AFTER_FORWARD" not in os.environ
+ # Idempotent.
+ assert launch.apply_unsloth_fsdp2_launch_env() is True
+
+
+def test_explicit_fsdp_settings_win(monkeypatch, launch):
+ monkeypatch.setenv("UNSLOTH_FSDP2", "true")
+ monkeypatch.setenv("WORLD_SIZE", "4")
+ monkeypatch.setenv("FSDP_AUTO_WRAP_POLICY", "SIZE_BASED_WRAP")
+ monkeypatch.setenv("FSDP_STATE_DICT_TYPE", "SHARDED_STATE_DICT")
+ launch.apply_unsloth_fsdp2_launch_env()
+ assert os.environ["FSDP_AUTO_WRAP_POLICY"] == "SIZE_BASED_WRAP"
+ assert os.environ["FSDP_STATE_DICT_TYPE"] == "SHARDED_STATE_DICT"
+
+
[email protected]("value,expected", [("0", "false"), ("false", "false"), ("1", "true")])
+def test_reshard_after_forward_switch(monkeypatch, launch, value, expected):
+ monkeypatch.setenv("UNSLOTH_FSDP2", "1")
+ monkeypatch.setenv("WORLD_SIZE", "2")
+ monkeypatch.setenv("UNSLOTH_FSDP2_RESHARD_AFTER_FORWARD", value)
+ launch.apply_unsloth_fsdp2_launch_env()
+ assert os.environ["FSDP_RESHARD_AFTER_FORWARD"] == expected
+
+
+def _fake_accelerate(monkeypatch, launch, tmp_path, source):
+ (tmp_path / "utils").mkdir()
+ (tmp_path / "utils" / "fsdp_utils.py").write_text(source)
+ spec = importlib.util.spec_from_file_location("accelerate", tmp_path / "__init__.py")
+ spec.submodule_search_locations = [str(tmp_path)]
+ monkeypatch.setattr(launch.importlib.util, "find_spec", lambda name: spec)
+
+
+ "source,ok",
+ [
+ ("def fsdp_prepare_model(): pass\n", False), # accelerate < 1.6: no FSDP2
+ ("def fsdp2_prepare_model(): pass\n", False), # 1.6: no FSDP2 state dict options
+ ("def fsdp2_prepare_model(): pass\ndef _prepare_sd_options(): pass\n", True),
+ ],
+)
+def test_unsloth_fsdp2_needs_an_accelerate_with_fsdp2(monkeypatch, launch, tmp_path, source, ok):
+ _fake_accelerate(monkeypatch, launch, tmp_path, source)
+ monkeypatch.setenv("UNSLOTH_FSDP2", "1")
+ monkeypatch.setenv("WORLD_SIZE", "2")
+ if ok:
+ assert launch.apply_unsloth_fsdp2_launch_env() is True
+ else:
+ with pytest.raises(RuntimeError, match = "accelerate >= 1.7"):
+ launch.apply_unsloth_fsdp2_launch_env()
+ assert "FSDP_VERSION" not in os.environ
+
+
+def test_the_installed_accelerate_is_read_without_importing_it(launch):
+ pytest.importorskip("accelerate")
+ assert launch._accelerate_fsdp2_support() is True
+
+
+def test_conflicting_launches_are_refused(monkeypatch, launch):
+ monkeypatch.setenv("UNSLOTH_FSDP2", "1")
+ monkeypatch.setenv("WORLD_SIZE", "2")
+ monkeypatch.setenv("FSDP_VERSION", "1")
+ with pytest.raises(ValueError, match = "FSDP_VERSION=1"):
+ launch.apply_unsloth_fsdp2_launch_env()
+ monkeypatch.delenv("FSDP_VERSION")
+ monkeypatch.setenv("ACCELERATE_USE_DEEPSPEED", "true")
+ with pytest.raises(RuntimeError, match = "DeepSpeed"):
+ launch.apply_unsloth_fsdp2_launch_env()
+
+
+def test_the_env_becomes_accelerates_fsdp2_plugin(monkeypatch, launch):
+ """What accelerate builds from the exported env: FSDP2, one unit per `_no_split_modules` block."""
+ accelerate_utils = pytest.importorskip("accelerate.utils")
+ from torch.distributed.fsdp.wrap import transformer_auto_wrap_policy
+
+ monkeypatch.setenv("UNSLOTH_FSDP2", "1")
+ monkeypatch.setenv("WORLD_SIZE", "2")
+ monkeypatch.setenv("UNSLOTH_FSDP2_RESHARD_AFTER_FORWARD", "0")
+ launch.apply_unsloth_fsdp2_launch_env()
+ plugin = accelerate_utils.FullyShardedDataParallelPlugin()
+ assert plugin.fsdp_version == 2
+ assert plugin.auto_wrap_policy is transformer_auto_wrap_policy
+ assert plugin.reshard_after_forward is False
+ assert "FULL_STATE_DICT" in str(plugin.state_dict_type)
+
+
+def test_unsloth_init_applies_the_env_before_transformers():
+ """The call must sit in unsloth/__init__.py ahead of any import that pulls in transformers."""
+ source = (_ROOT / "unsloth" / "__init__.py").read_text(encoding = "utf-8")
+ tree = ast.parse(source)
+ call_line = None
+ first_heavy = None
+ for node in tree.body:
+ text = ast.unparse(node)
+ if call_line is None and "_apply_fsdp2_env()" in text:
+ call_line = node.lineno
+ if first_heavy is None and isinstance(node, (ast.Import, ast.ImportFrom)):
+ if any(x in text for x in ("transformers", "accelerate", "unsloth_zoo", ".models")):
+ first_heavy = node.lineno
+ assert call_line is not None
+ assert first_heavy is None or call_line < first_heavy
+
+
+def test_loader_utils_versions(monkeypatch):
+ loader_utils = pytest.importorskip("unsloth.models.loader_utils")
+ accelerate_state = pytest.importorskip("accelerate.state")
+ shared = accelerate_state.AcceleratorState._shared_state
+ monkeypatch.delitem(shared, "fsdp_plugin", raising = False)
+ monkeypatch.delitem(shared, "distributed_type", raising = False)
+
+ assert loader_utils.fsdp_version_requested() == 0
+ assert loader_utils.fsdp2_keeps_fused_lora() is False
+
+ monkeypatch.setenv("ACCELERATE_USE_FSDP", "true")
+ monkeypatch.setenv("FSDP_VERSION", "1")
+ assert loader_utils.fsdp_will_wrap() is True
+ assert loader_utils.fsdp2_keeps_fused_lora() is False
+
+ monkeypatch.setenv("FSDP_VERSION", "2")
+ assert loader_utils.fsdp2_requested() is True
+ assert loader_utils.fsdp2_keeps_fused_lora() is True
+ monkeypatch.setenv("UNSLOTH_FSDP2_FUSED_LORA", "0")
+ assert loader_utils.fsdp2_keeps_fused_lora() is False
+ monkeypatch.delenv("UNSLOTH_FSDP2_FUSED_LORA")
+
+ # UNSLOTH_FSDP2=1 under torchrun, before any env was exported.
+ monkeypatch.delenv("ACCELERATE_USE_FSDP")
+ monkeypatch.delenv("FSDP_VERSION")
+ monkeypatch.setenv("UNSLOTH_FSDP2", "1")
+ monkeypatch.setenv("WORLD_SIZE", "2")
+ assert loader_utils.fsdp2_requested() is True
+
+
+def test_a_built_accelerator_is_authoritative(monkeypatch):
+ """An FSDP1 plugin wins over a stale FSDP_VERSION=2 in the env, and vice versa."""
+ import types
+
+ loader_utils = pytest.importorskip("unsloth.models.loader_utils")
+ accelerate_state = pytest.importorskip("accelerate.state")
+ shared = accelerate_state.AcceleratorState._shared_state
+ monkeypatch.setenv("FSDP_VERSION", "2")
+ monkeypatch.setitem(shared, "fsdp_plugin", types.SimpleNamespace(fsdp_version = 1))
+ assert loader_utils.fsdp_version_requested() == 1
+ assert loader_utils.fsdp2_keeps_fused_lora() is False
+ monkeypatch.delenv("FSDP_VERSION")
+ monkeypatch.setitem(shared, "fsdp_plugin", types.SimpleNamespace(fsdp_version = 2))
+ assert loader_utils.fsdp2_requested() is True
+
+
+def _free_port():
+ with socket.socket() as s:
+ s.bind(("127.0.0.1", 0))
+ return s.getsockname()[1]
+
+
+def _rank_worker(rank, port, out_dir):
+ import json
+
+ import torch.distributed as dist
+
+ os.environ.update(
+ RANK = str(rank),
+ LOCAL_RANK = str(rank),
+ WORLD_SIZE = "2",
+ MASTER_ADDR = "127.0.0.1",
+ MASTER_PORT = str(port),
+ UNSLOTH_FSDP2 = "1",
+ )
+ launch = _load_launch()
+ launch.apply_unsloth_fsdp2_launch_env()
+ dist.init_process_group("gloo", rank = rank, world_size = 2)
+ try:
+ mine = (
+ launch.fsdp_version_from_env(),
+ os.environ.get("ACCELERATE_USE_FSDP"),
+ os.environ.get("FSDP_AUTO_WRAP_POLICY"),
+ )
+ everyone = [None, None]
+ dist.all_gather_object(everyone, mine)
+ if rank == 0:
+ with open(os.path.join(out_dir, "ranks.json"), "w") as f:
+ json.dump(everyone, f)
+ finally:
+ dist.destroy_process_group()
+
+
+def test_every_rank_reads_the_same_fsdp2_launch(tmp_path):
+ import json
+
+ dist = pytest.importorskip("torch.distributed")
+ if not dist.is_available() or not dist.is_gloo_available():
+ pytest.skip("gloo backend unavailable")
+ import torch.multiprocessing as mp
+
+ mp.spawn(_rank_worker, args = (_free_port(), str(tmp_path)), nprocs = 2, join = True)
+ ranks = json.loads((tmp_path / "ranks.json").read_text())
+ assert ranks[0] == ranks[1] == [2, "true", "TRANSFORMER_BASED_WRAP"]
+
+
+def test_unsloth_init_installs_the_fsdp2_hooks_without_a_launcher_env():
+ """`Accelerator(fsdp_plugin = FullyShardedDataParallelPlugin(fsdp_version = 2))` in a raw loop
+ sets no env before unsloth is imported, so the install must not be gated on it."""
+ import ast
+
+ source = (_ROOT / "unsloth" / "__init__.py").read_text(encoding = "utf-8")
+ gated = [
+ node
+ for node in ast.walk(ast.parse(source))
+ if isinstance(node, ast.If)
+ and "fsdp_version_from_env" in ast.unparse(node.test)
+ and "_install_fsdp2_hooks" in ast.unparse(node)
+ ]
+ assert not gated, "install_fsdp2_hooks is gated on the launcher env"
+ assert "_install_fsdp2_hooks()" in source
diff --git a/tests/test_fsdp2_frozen_dtypes_gloo.py b/tests/test_fsdp2_frozen_dtypes_gloo.py
new file mode 100644
index 00000000000..1408e22d4f7
--- /dev/null
+++ b/tests/test_fsdp2_frozen_dtypes_gloo.py
@@ -0,0 +1,106 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""LoRA under FSDP2 puts float32 adapters next to frozen 16-bit weights in one unit. torch < 2.8
+asserts one dtype over every parameter of a unit, frozen ones included; `unify_frozen_dtypes`
+upcasts the frozen ones there. Two gloo ranks on CPU, real `fully_shard`, no accelerate needed.
+"""
+
+from __future__ import annotations
+
+import importlib.util
+import os
+import pathlib
+import socket
+import types
+
+import pytest
+
+torch = pytest.importorskip("torch")
+dist = pytest.importorskip("torch.distributed")
+fsdp = pytest.importorskip("torch.distributed.fsdp")
+if not hasattr(fsdp, "fully_shard") or not hasattr(fsdp, "MixedPrecisionPolicy"):
+ pytest.skip("torch without the public FSDP2 API", allow_module_level = True)
+if not dist.is_available() or not dist.is_gloo_available():
+ pytest.skip("gloo backend unavailable", allow_module_level = True)
+
+_FSDP2_PY = pathlib.Path(__file__).resolve().parents[1] / "unsloth" / "models" / "fsdp2.py"
+
+
+def _fsdp2_module():
+ # By path, so this runs on torch versions the rest of unsloth's imports do not cover.
+ spec = importlib.util.spec_from_file_location("_unsloth_fsdp2_under_test", _FSDP2_PY)
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+ return module
+
+
+def _free_port():
+ with socket.socket() as s:
+ s.bind(("127.0.0.1", 0))
+ return s.getsockname()[1]
+
+
+class _Block(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.base = torch.nn.Linear(16, 16, bias = False).to(torch.bfloat16).requires_grad_(False)
+ self.lora_A = torch.nn.Linear(16, 4, bias = False)
+ self.lora_B = torch.nn.Linear(4, 16, bias = False)
+
+ def forward(self, x):
+ return self.base(x) + self.lora_B(self.lora_A(x.to(self.lora_A.weight.dtype))).to(x.dtype)
+
+
+def _worker(rank, port, unify, out):
+ os.environ.update(MASTER_ADDR = "127.0.0.1", MASTER_PORT = str(port))
+ dist.init_process_group("gloo", rank = rank, world_size = 2)
+ try:
+ fsdp2 = _fsdp2_module()
+ torch.manual_seed(0)
+ model = torch.nn.Sequential(_Block(), _Block())
+ policy = fsdp.MixedPrecisionPolicy(param_dtype = torch.bfloat16, reduce_dtype = torch.float32)
+ plugin = types.SimpleNamespace(mixed_precision_policy = policy)
+ acc = types.SimpleNamespace(
+ state = types.SimpleNamespace(fsdp_plugin = plugin), mixed_precision = "bf16"
+ )
+ upcast = fsdp2.unify_frozen_dtypes(acc, model) if unify else False
+ for block in model:
+ fsdp.fully_shard(block, mp_policy = policy)
+ fsdp.fully_shard(model, mp_policy = policy)
+ try:
+ model(torch.randn(2, 16, dtype = torch.bfloat16)).float().sum().backward()
+ result = "ok"
+ except AssertionError as e:
+ result = str(e)
+ if rank == 0:
+ with open(out, "w") as f:
+ f.write(f"{upcast}|{fsdp2._fsdp2_checks_frozen_dtypes()}|{result}")
+ finally:
+ dist.destroy_process_group()
+
+
+def _run(unify, tmp_path):
+ import torch.multiprocessing as mp
+
+ out = str(tmp_path / f"unify_{unify}.txt")
+ mp.spawn(_worker, args = (_free_port(), unify, out), nprocs = 2, join = True)
+ upcast, frozen_checked, result = open(out).read().split("|", 2)
+ return upcast == "True", frozen_checked == "True", result
+
+
+def test_frozen_16bit_weights_next_to_float32_adapters_shard_and_train(tmp_path):
+ upcast, frozen_checked, result = _run(True, tmp_path)
+ assert result == "ok", result
+ # Only torch < 2.8 needs (and gets) the upcast; newer torch keeps the frozen weights 16-bit.
+ assert upcast is frozen_checked
+ old_torch = tuple(int(x) for x in torch.__version__.split(".")[:2]) < (2, 8)
+ assert frozen_checked is old_torch
+
+
+def test_without_the_upcast_old_torch_refuses_the_unit(tmp_path):
+ _, frozen_checked, result = _run(False, tmp_path)
+ if frozen_checked:
+ assert "uniform original parameter dtype" in result
+ else:
+ assert result == "ok", result
diff --git a/tests/test_fsdp2_peft_checkpoint_gloo.py b/tests/test_fsdp2_peft_checkpoint_gloo.py
new file mode 100644
index 00000000000..b4613d0919a
--- /dev/null
+++ b/tests/test_fsdp2_peft_checkpoint_gloo.py
@@ -0,0 +1,176 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""FSDP2 + PEFT FULL_STATE_DICT checkpoints (what Trainer writes for resuming), two gloo ranks on CPU.
+
+accelerate < 1.15 builds the adapter-only full state dict with `get_peft_model_state_dict`, i.e.
+each rank's DTensor shard, and rank 0 writes only its own; `install_fsdp2_peft_full_state_dict`
+backports the 1.15 gather. The saved adapter must hold whole tensors equal to the unsharded ones,
+and loading it back must restore every rank's shard.
+"""
+
+from __future__ import annotations
+
+import os
+import socket
+import types
+
+import pytest
+
+torch = pytest.importorskip("torch")
+dist = pytest.importorskip("torch.distributed")
+pytest.importorskip("torch.distributed.fsdp")
+pytest.importorskip("peft")
+pytest.importorskip("accelerate")
+
+if not dist.is_available() or not dist.is_gloo_available():
+ pytest.skip("gloo backend unavailable", allow_module_level = True)
+
+
+def _free_port():
+ with socket.socket() as s:
+ s.bind(("127.0.0.1", 0))
+ return s.getsockname()[1]
+
+
+def _build():
+ from peft import LoraConfig, get_peft_model
+ from transformers import LlamaConfig, LlamaForCausalLM
+
+ cfg = LlamaConfig(
+ hidden_size = 64,
+ intermediate_size = 128,
+ num_hidden_layers = 2,
+ num_attention_heads = 4,
+ num_key_value_heads = 2,
+ vocab_size = 96,
+ tie_word_embeddings = False,
+ )
+ torch.manual_seed(0)
+ lora = LoraConfig(
+ r = 4, lora_alpha = 8, target_modules = ["q_proj", "down_proj"], init_lora_weights = False
+ )
+ return get_peft_model(LlamaForCausalLM(cfg), lora)
+
+
+def _accelerator_less_import():
+ # Spawned ranks skip conftest, whose CUDA spoof lets `import unsloth` run on CPU-only runners.
+ if torch.cuda.is_available():
+ return
+ import importlib.util
+ import pathlib
+
+ path = pathlib.Path(__file__).resolve().parent / "conftest.py"
+ spec = importlib.util.spec_from_file_location("_unsloth_tests_conftest", path)
+ spec.loader.exec_module(importlib.util.module_from_spec(spec))
+
+
+def _worker(rank, port, out_dir, backport):
+ _accelerator_less_import()
+ os.environ.update(
+ RANK = str(rank),
+ LOCAL_RANK = str(rank),
+ WORLD_SIZE = "2",
+ MASTER_ADDR = "127.0.0.1",
+ MASTER_PORT = str(port),
+ )
+ dist.init_process_group("gloo", rank = rank, world_size = 2)
+ try:
+ import json
+
+ from torch.distributed.device_mesh import init_device_mesh
+ from torch.distributed.tensor import DTensor
+
+ from accelerate import PartialState
+ from accelerate.utils import FullyShardedDataParallelPlugin
+ from accelerate.utils import fsdp_utils
+ from unsloth.models import fsdp2
+
+ PartialState(cpu = True) # accelerate's logger refuses to run without it
+ if backport:
+ assert fsdp2.install_fsdp2_peft_full_state_dict()
+ else:
+ # `import unsloth` installed it already; undo that to reproduce the unpatched accelerate.
+ for name in ("_get_model_state_dict", "_set_model_state_dict"):
+ fn = getattr(fsdp_utils, name)
+ setattr(fsdp_utils, name, getattr(fn, "__wrapped__", fn))
+ result = {"accelerate": __import__("accelerate").__version__}
+ mesh = init_device_mesh("cpu", (2,), mesh_dim_names = ("dp_shard",))
+ plugin = FullyShardedDataParallelPlugin(
+ fsdp_version = 2,
+ auto_wrap_policy = "TRANSFORMER_BASED_WRAP",
+ transformer_cls_names_to_wrap = ["LlamaDecoderLayer"],
+ state_dict_type = "FULL_STATE_DICT",
+ )
+ accelerator = types.SimpleNamespace(
+ state = types.SimpleNamespace(fsdp_plugin = plugin),
+ torch_device_mesh = mesh,
+ parallelism_config = types.SimpleNamespace(fsdp_dim_names = ("dp_shard",)),
+ mixed_precision = "no",
+ device = torch.device("cpu"),
+ num_processes = 2,
+ process_index = rank,
+ is_main_process = rank == 0,
+ is_fsdp2 = True,
+ wait_for_everyone = dist.barrier,
+ )
+ model = _build()
+ reference = {k: v.clone() for k, v in model.state_dict().items() if "lora_" in k}
+ model = fsdp_utils.fsdp2_prepare_model(accelerator, model)
+ fsdp_utils.save_fsdp_model(plugin, accelerator, model, out_dir, adapter_only = True)
+ dist.barrier()
+
+ if rank == 0:
+ saved = torch.load(os.path.join(out_dir, "pytorch_model_fsdp.bin"), weights_only = False)
+ # peft's keys drop the adapter name, get_model_state_dict's keep it.
+ saved = {k.replace(".default", ""): v for k, v in saved.items() if "lora_" in k}
+ whole = 0
+ for name, ref in reference.items():
+ got = saved.get(name.replace(".default", ""))
+ if isinstance(got, DTensor):
+ got = got.to_local()
+ if (
+ got is not None
+ and tuple(got.shape) == tuple(ref.shape)
+ and torch.equal(got.cpu(), ref)
+ ):
+ whole += 1
+ result["lora_tensors"] = len(reference)
+ result["saved_whole_and_equal"] = whole
+
+ with torch.no_grad():
+ for name, p in model.named_parameters():
+ if "lora_" in name:
+ p.zero_()
+ fsdp_utils.load_fsdp_model(plugin, accelerator, model, out_dir, adapter_only = True)
+ restored = 0
+ for name, p in model.named_parameters():
+ if "lora_" not in name:
+ continue
+ full = p.full_tensor() if isinstance(p, DTensor) else p
+ restored += int(torch.equal(full.detach().cpu(), reference[name]))
+ result["restored"] = restored
+ if rank == 0:
+ with open(os.path.join(out_dir, "result.json"), "w") as f:
+ json.dump(result, f)
+ finally:
+ dist.destroy_process_group()
+
+
+def _run(tmp_path, backport):
+ import json
+
+ import torch.multiprocessing as mp
+
+ mp.spawn(_worker, args = (_free_port(), str(tmp_path), backport), nprocs = 2, join = True)
+ with open(tmp_path / "result.json") as f:
+ return json.load(f)
+
+
+def test_full_state_dict_adapter_checkpoints_hold_every_rank_shard(tmp_path):
+ r = _run(tmp_path, backport = True)
+ assert r["lora_tensors"] == 8
+ assert (
+ r["saved_whole_and_equal"] == r["lora_tensors"]
+ ), "the checkpoint holds only rank 0's shards"
+ assert r["restored"] == r["lora_tensors"]
diff --git a/tests/test_fsdp2_wrap_gloo.py b/tests/test_fsdp2_wrap_gloo.py
new file mode 100644
index 00000000000..3c0564d68c9
--- /dev/null
+++ b/tests/test_fsdp2_wrap_gloo.py
@@ -0,0 +1,312 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""FSDP2 wrap through accelerate's real `fsdp2_prepare_model`, two gloo ranks on CPU.
+
+What Unsloth adds after accelerate shards the model (unsloth/models/fsdp2.py):
+- a layer that reads `.weight` directly (the fused LoRA kernels) sees whole, unsharded weights
+ inside a per-block unit's forward, so the fused path is kept there;
+- a finer wrap (a LoRA submodule as its own unit) would hand that layer a DTensor shard, so the
+ fused path is declined on every rank;
+- lm_head read directly by the CausalLM forward (Unsloth's fused CE) is gathered first;
+- the last decoder block stays gathered after forward;
+- losses and gradients match an unsharded copy of the model.
+"""
+
+from __future__ import annotations
+
+import os
+import socket
+import types
+
+import pytest
+
+torch = pytest.importorskip("torch")
+dist = pytest.importorskip("torch.distributed")
+pytest.importorskip("torch.distributed.fsdp")
+pytest.importorskip("peft")
+pytest.importorskip("accelerate")
+
+if not dist.is_available() or not dist.is_gloo_available():
+ pytest.skip("gloo backend unavailable", allow_module_level = True)
+
+N_LAYERS = 3
+
+
+def _free_port():
+ with socket.socket() as s:
+ s.bind(("127.0.0.1", 0))
+ return s.getsockname()[1]
+
+
+def _spy_mlp(self, X):
+ """Stands in for a fused LoRA MLP: reads every weight directly, never calls the projections."""
+ log = self._unsloth_spy_log
+
+ def proj(p, x):
+ base = p.base_layer.weight
+ A = p.lora_A["default"].weight
+ B = p.lora_B["default"].weight
+ from torch.distributed.tensor import DTensor
+
+ for w, full in ((base, (p.out_features, p.in_features)), (A, None), (B, None)):
+ log.append((isinstance(w, DTensor), tuple(w.shape), full))
+ return x @ base.t() + (x @ A.t()) @ B.t() * p.scaling["default"]
+
+ gate = proj(self.gate_proj, X)
+ up = proj(self.up_proj, X)
+ return proj(self.down_proj, torch.nn.functional.silu(gate) * up)
+
+
+def _spy_causal_forward(self, input_ids, labels):
+ """Unsloth's CausalLM forward reads lm_head.weight for the fused CE instead of calling it."""
+ hidden = self.model(input_ids = input_ids, use_cache = False).last_hidden_state
+ logits = hidden @ self.lm_head.weight.t()
+ return torch.nn.functional.cross_entropy(
+ logits[:, :-1].reshape(-1, logits.shape[-1]).float(), labels[:, 1:].reshape(-1)
+ )
+
+
+def _build(seed):
+ from peft import LoraConfig, get_peft_model
+ from transformers import LlamaConfig, LlamaForCausalLM
+
+ cfg = LlamaConfig(
+ hidden_size = 64,
+ intermediate_size = 128,
+ num_hidden_layers = N_LAYERS,
+ num_attention_heads = 4,
+ num_key_value_heads = 2,
+ vocab_size = 96,
+ tie_word_embeddings = False,
+ )
+ torch.manual_seed(seed)
+ base = LlamaForCausalLM(cfg)
+ lora = LoraConfig(
+ r = 4,
+ lora_alpha = 8,
+ target_modules = [
+ "q_proj",
+ "k_proj",
+ "v_proj",
+ "o_proj",
+ "gate_proj",
+ "up_proj",
+ "down_proj",
+ ],
+ lora_dropout = 0.0,
+ init_lora_weights = False,
+ )
+ model = get_peft_model(base, lora)
+ log = []
+ for layer in model.base_model.model.model.layers:
+ layer.mlp._unsloth_spy_log = log
+ layer.mlp.forward = types.MethodType(_spy_mlp, layer.mlp)
+ causal = model.base_model.model
+ causal.forward = types.MethodType(_spy_causal_forward, causal)
+ return model, log
+
+
+def _fake_accelerator(rank, mesh, plugin):
+ return types.SimpleNamespace(
+ state = types.SimpleNamespace(fsdp_plugin = plugin),
+ torch_device_mesh = mesh,
+ parallelism_config = types.SimpleNamespace(fsdp_dim_names = ("dp_shard",)),
+ mixed_precision = "no",
+ device = torch.device("cpu"),
+ is_main_process = rank == 0,
+ )
+
+
+def _accelerator_less_import():
+ # Spawned ranks skip conftest, whose CUDA spoof lets `import unsloth` run on CPU-only runners.
+ if torch.cuda.is_available():
+ return
+ import importlib.util
+ import pathlib
+
+ path = pathlib.Path(__file__).resolve().parent / "conftest.py"
+ spec = importlib.util.spec_from_file_location("_unsloth_tests_conftest", path)
+ spec.loader.exec_module(importlib.util.module_from_spec(spec))
+
+
+def _worker(rank, port, case, out_dir):
+ _accelerator_less_import()
+ os.environ.update(
+ RANK = str(rank),
+ LOCAL_RANK = str(rank),
+ WORLD_SIZE = "2",
+ MASTER_ADDR = "127.0.0.1",
+ MASTER_PORT = str(port),
+ )
+ for key in (
+ "UNSLOTH_FSDP2",
+ "FSDP_VERSION",
+ "ACCELERATE_USE_FSDP",
+ "UNSLOTH_FORCE_FUSED_LORA",
+ "UNSLOTH_FSDP2_FUSED_LORA",
+ ):
+ os.environ.pop(key, None)
+ if case == "opt_out":
+ os.environ["UNSLOTH_FSDP2_FUSED_LORA"] = "0"
+ import json
+
+ dist.init_process_group("gloo", rank = rank, world_size = 2)
+ try:
+ from torch.distributed.device_mesh import init_device_mesh
+ from torch.distributed.tensor import DTensor
+
+ import accelerate.accelerator as accelerator_module
+ from accelerate.utils import FullyShardedDataParallelPlugin
+ from unsloth.models import fsdp2, llama
+
+ llama._FUSED_LORA_MLPS = llama._FUSED_LORA_MLPS + (_spy_mlp,)
+ mesh = init_device_mesh("cpu", (2,), mesh_dim_names = ("dp_shard",))
+ wrap_names = ["LlamaDecoderLayer"] + (["Linear"] if case == "finer" else [])
+ plugin = FullyShardedDataParallelPlugin(
+ fsdp_version = 2,
+ auto_wrap_policy = "TRANSFORMER_BASED_WRAP",
+ transformer_cls_names_to_wrap = wrap_names,
+ reshard_after_forward = True,
+ activation_checkpointing = case == "checkpointed",
+ )
+ result = {}
+ result["agree_mixed"] = fsdp2._all_ranks_agree(rank == 0, torch.device("cpu"))
+
+ model, log = _build(seed = 0)
+ reference, _ = _build(seed = 0)
+ assert fsdp2.install_fsdp2_prepare_hook()
+ accelerator = _fake_accelerator(rank, mesh, plugin)
+ if case == "checkpointed":
+ # What Accelerator._prepare_fsdp2 does before sharding when the plugin asks for it.
+ from accelerate.utils.fsdp_utils import fsdp2_apply_ac
+ model = fsdp2_apply_ac(accelerator, model)
+ if case == "causal_unit":
+ # What accelerate <= 1.13 builds for transformer_cls_names_to_wrap = ["LlamaForCausalLM"]
+ # (1.14+ refuses that wrap).
+ from torch.distributed.fsdp import fully_shard
+
+ fully_shard(model.base_model.model, mesh = mesh)
+ fully_shard(model, mesh = mesh)
+ fsdp2.after_fsdp2_wrap(accelerator, model)
+ else:
+ model = accelerator_module.fsdp2_prepare_model(accelerator, model)
+ result["fused_kept"] = int(getattr(model, "_unsloth_fsdp2_fused_lora", -1))
+ result["blocks"] = int(getattr(model, "_unsloth_fsdp2_blocks", -1))
+ result["direct_read_units"] = int(getattr(model, "_unsloth_fsdp2_direct_read_units", -1))
+ layers = model.base_model.model.model.layers
+ result["spy_left"] = sum("forward" in layer.mlp.__dict__ for layer in layers)
+ if case == "causal_unit":
+ if rank == 0:
+ with open(os.path.join(out_dir, f"{case}.json"), "w") as f:
+ json.dump(result, f)
+ return
+
+ torch.manual_seed(1)
+ ids = torch.randint(0, 96, (4, 12))
+ local = ids[rank * 2 : rank * 2 + 2]
+ loss = model(input_ids = local, labels = local)
+ result["last_block_gathered"] = not isinstance(
+ layers[-1].mlp.gate_proj.base_layer.weight, DTensor
+ )
+ result["first_block_gathered"] = not isinstance(
+ layers[0].mlp.gate_proj.base_layer.weight, DTensor
+ )
+ loss.backward()
+ result["spy_calls"] = len(log)
+ result["spy_saw_shard"] = sum(1 for is_dt, _, _ in log if is_dt)
+ result["spy_full_shapes"] = all(full is None or shape == full for _, shape, full in log)
+
+ ref_loss = reference(input_ids = ids, labels = ids)
+ ref_loss.backward()
+ local_loss = torch.tensor([loss.item()])
+ dist.all_reduce(local_loss)
+ result["loss_diff"] = abs(local_loss.item() / 2 - ref_loss.item())
+ ref_params = dict(reference.named_parameters())
+ worst = 0.0
+ for name, p in model.named_parameters():
+ if p.grad is None:
+ continue
+ g = p.grad.full_tensor() if isinstance(p.grad, DTensor) else p.grad
+ # FSDP averages over the 2 ranks; each rank's loss is the mean over its half.
+ r = ref_params[name.replace("._checkpoint_wrapped_module", "")].grad
+ worst = max(worst, (g - r).abs().max().item() / (r.abs().max().item() + 1e-12))
+ result["grad_rel_err"] = worst
+ if rank == 0:
+ with open(os.path.join(out_dir, f"{case}.json"), "w") as f:
+ json.dump(result, f)
+ finally:
+ dist.destroy_process_group()
+
+
+def _run(case, tmp_path):
+ import json
+
+ import torch.multiprocessing as mp
+
+ mp.spawn(_worker, args = (_free_port(), case, str(tmp_path)), nprocs = 2, join = True)
+ with open(tmp_path / f"{case}.json") as f:
+ return json.load(f)
+
+
+def test_per_block_wrap_keeps_fused_layers_on_whole_weights(tmp_path):
+ r = _run("per_block", tmp_path)
+ assert r["agree_mixed"] is False, "ranks must agree on the strictest answer"
+ assert r["blocks"] == N_LAYERS
+ assert r["fused_kept"] == N_LAYERS
+ assert r["spy_left"] == N_LAYERS
+ # 3 MLPs x 3 projections x (base, A, B), forward only.
+ assert r["spy_calls"] == N_LAYERS * 9
+ assert r["spy_saw_shard"] == 0, "a direct .weight read saw a DTensor shard inside a block"
+ assert r["spy_full_shapes"]
+ import accelerate.utils.fsdp_utils as fsdp_utils
+
+ if hasattr(fsdp_utils, "_find_final_norm"):
+ # accelerate >= 1.14 gives [final norm, lm_head] their own unit, which nothing would gather.
+ assert (
+ r["direct_read_units"] >= 1
+ ), "lm_head's unit is not gathered before the CausalLM forward"
+ else:
+ # Older accelerate leaves lm_head in the root unit, which its own pre-forward gathers.
+ assert r["direct_read_units"] == 0
+ assert r["last_block_gathered"] is True
+ assert r["first_block_gathered"] is False
+ assert r["loss_diff"] < 1e-5
+ assert r["grad_rel_err"] < 1e-4
+
+
+def test_a_finer_wrap_declines_the_fused_layers(tmp_path):
+ r = _run("finer", tmp_path)
+ assert r["fused_kept"] == 0
+ assert r["spy_left"] == 0, "the fused forwards must be removed on every rank"
+ assert r["spy_calls"] == 0
+ assert r["loss_diff"] < 1e-5
+ assert r["grad_rel_err"] < 1e-4
+
+
+def test_a_unit_above_the_blocks_declines_the_fused_layers(tmp_path):
+ """peft's LoraModel calls the CausalLM's .forward directly, so a CausalLM unit's gather can be skipped."""
+ r = _run("causal_unit", tmp_path)
+ assert r["blocks"] == 0
+ assert r["fused_kept"] == 0
+ assert r["spy_left"] == 0
+
+
+def test_the_fused_lora_opt_out_is_honoured_after_wrapping(tmp_path):
+ """A raw Accelerator loop patches the model before any FSDP plugin exists."""
+ r = _run("opt_out", tmp_path)
+ assert r["fused_kept"] == 0
+ assert r["spy_left"] == 0
+ assert r["spy_calls"] == 0
+ assert r["loss_diff"] < 1e-5
+
+
+def test_activation_checkpointed_blocks_keep_the_last_block_gathered(tmp_path):
+ r = _run("checkpointed", tmp_path)
+ assert r["blocks"] == N_LAYERS
+ assert r["fused_kept"] == N_LAYERS
+ assert r["last_block_gathered"] is True
+ assert r["first_block_gathered"] is False
+ assert r["spy_saw_shard"] == 0
+ assert r["loss_diff"] < 1e-5
+ assert r["grad_rel_err"] < 1e-4
diff --git a/unsloth/__init__.py b/unsloth/__init__.py
index 28b61594eb8..3d4cedda2a9 100644
--- a/unsloth/__init__.py
+++ b/unsloth/__init__.py
@@ -18,6 +18,13 @@ os.environ["UNSLOTH_IS_PRESENT"] = "1"
# an explicit override, including "0".
os.environ.setdefault("TORCH_ROCM_AOTRITON_ENABLE_EXPERIMENTAL", "1")
+# UNSLOTH_FSDP2=1 under plain torchrun: export accelerate's FSDP2 launch env before transformers or
+# accelerate read it. A conflicting launch (DeepSpeed, FSDP_VERSION=1) raises here, on every rank.
+from .fsdp2_launch import apply_unsloth_fsdp2_launch_env as _apply_fsdp2_env
+
+_apply_fsdp2_env()
+del _apply_fsdp2_env
+
# Before transformers, which reads sentencepiece availability during its own import. On Windows
# the extension is never imported at all: a code integrity policy can refuse it by reputation,
# and any probe to find out whether this machine will is itself the refusal the user sees. See
@@ -1633,6 +1640,15 @@ else:
from ._gpu_init import *
from ._gpu_init import __version__
+ # Raw `Accelerator.prepare()` loops never reach the Trainer init that also installs these, and
+ # `Accelerator(fsdp_plugin = ...)` leaves no launcher env to gate on. Inert unless FSDP2 wraps.
+ try:
+ from .models.fsdp2 import install_fsdp2_hooks as _install_fsdp2_hooks
+ _install_fsdp2_hooks()
+ del _install_fsdp2_hooks
+ except Exception:
+ pass
+
def get_gpu_memory_stats():
"""Return CUDA/ROCm/XPU/NPU device stats, peak memory, and total memory in GiB."""
try:
diff --git a/unsloth/fsdp2_launch.py b/unsloth/fsdp2_launch.py
new file mode 100644
index 00000000000..4265d167b27
--- /dev/null
+++ b/unsloth/fsdp2_launch.py
@@ -0,0 +1,98 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""FSDP launch detection from the launcher env only (no torch): identical on every rank."""
+
+import importlib.util
+import os
+
+__all__ = [
+ "unsloth_fsdp2_requested",
+ "fsdp_version_from_env",
+ "apply_unsloth_fsdp2_launch_env",
+]
+
+_TRUTHY = ("1", "true", "yes", "on")
+_FALSY = ("0", "false", "no", "off")
+
+
+def _env_flag(name):
+ return str(os.environ.get(name, "")).strip().lower() in _TRUTHY
+
+
+def _env_world_size():
+ try:
+ return int(os.environ.get("WORLD_SIZE", "1"))
+ except ValueError:
+ return 1
+
+
+def unsloth_fsdp2_requested():
+ return _env_flag("UNSLOTH_FSDP2")
+
+
+def fsdp_version_from_env():
+ """0 for no FSDP, else 1 or 2. accelerate builds FSDP1 when ACCELERATE_USE_FSDP names no
+ version; FSDP_VERSION="0" means "not FSDP"."""
+ if unsloth_fsdp2_requested() and _env_world_size() > 1:
+ return 2
+ version = str(os.environ.get("FSDP_VERSION", "")).strip()
+ use_fsdp = _env_flag("ACCELERATE_USE_FSDP")
+ if version in ("", "0"):
+ return 1 if use_fsdp else 0
+ try:
+ return int(version)
+ except ValueError:
+ return 1 if use_fsdp else 0
+
+
+def _accelerate_fsdp2_support():
+ """None when accelerate is not found, else whether it has FSDP2 (1.6) and FSDP2 state dict
+ options (1.7; 1.6 saves FULL_STATE_DICT checkpoints as each rank's DTensor shard). Read from
+ the source: importing accelerate here would import torch."""
+ try:
+ spec = importlib.util.find_spec("accelerate")
+ except (ImportError, ValueError):
+ return None
+ for location in getattr(spec, "submodule_search_locations", None) or ():
+ try:
+ with open(os.path.join(location, "utils", "fsdp_utils.py"), encoding = "utf-8") as f:
+ source = f.read()
+ except OSError:
+ continue
+ return "def fsdp2_prepare_model" in source and "def _prepare_sd_options" in source
+ return None
+
+
+def apply_unsloth_fsdp2_launch_env():
+ """UNSLOTH_FSDP2=1 with WORLD_SIZE > 1: export what `accelerate launch` exports for an FSDP2
+ config (explicit FSDP_* values win). Must run before transformers / accelerate read the env."""
+ if not unsloth_fsdp2_requested() or _env_world_size() <= 1:
+ return False
+ if _env_flag("ACCELERATE_USE_DEEPSPEED"):
+ raise RuntimeError(
+ "Unsloth: UNSLOTH_FSDP2=1 was set on a DeepSpeed launch. Unset one of them."
+ )
+ version = str(os.environ.get("FSDP_VERSION", "")).strip()
+ if version not in ("", "2"):
+ raise ValueError(
+ f"Unsloth: UNSLOTH_FSDP2=1 asks for FSDP2, but FSDP_VERSION={version}. Unset one of them."
+ )
+ if _accelerate_fsdp2_support() is False:
+ # Older accelerate ignores FSDP_VERSION and builds FSDP1 from these variables.
+ raise RuntimeError(
+ "Unsloth: UNSLOTH_FSDP2=1 needs accelerate >= 1.7, which this accelerate is older than. "
+ "Upgrade it with `pip install -U accelerate`."
+ )
+ os.environ["ACCELERATE_USE_FSDP"] = "true"
+ os.environ["FSDP_VERSION"] = "2"
+ # accelerate's default is the root only.
+ os.environ.setdefault("FSDP_AUTO_WRAP_POLICY", "TRANSFORMER_BASED_WRAP")
+ reshard = os.environ.get("UNSLOTH_FSDP2_RESHARD_AFTER_FORWARD", None)
+ if reshard is not None and reshard.strip() != "":
+ os.environ.setdefault(
+ "FSDP_RESHARD_AFTER_FORWARD",
+ "false" if reshard.strip().lower() in _FALSY else "true",
+ )
+ os.environ.setdefault("FSDP_STATE_DICT_TYPE", "FULL_STATE_DICT")
+ return True
diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py
index 93656d1695c..6c9e9f3040d 100644
--- a/unsloth/models/_utils.py
+++ b/unsloth/models/_utils.py
@@ -5535,6 +5535,11 @@ def patch_gradient_accumulation_fix(Trainer):
apply_accepts_loss_kwargs_fix(model)
except Exception:
pass
+ try:
+ from .fsdp2 import install_fsdp2_policy_provenance
+ install_fsdp2_policy_provenance()
+ except Exception:
+ pass
_original_trainer_init(self, *args, **kwargs)
try:
accelerator = getattr(self, "accelerator", None)
@@ -5552,8 +5557,13 @@ def patch_gradient_accumulation_fix(Trainer):
except Exception:
pass
if getattr(self, "is_fsdp_enabled", False):
- from .llama import _decline_fused_lora_for_fsdp
- _decline_fused_lora_for_fsdp(getattr(self, "model", None))
+ # FSDP2 keeps the fused LoRA kernels when each decoder block is its own unit (checked
+ # again once accelerate has wrapped); FSDP1 and a root-only wrap decline them here.
+ from .fsdp2 import fsdp2_trainer_setup
+ keep_fused, reason = fsdp2_trainer_setup(self)
+ if not keep_fused:
+ from .llama import _decline_fused_lora_for_fsdp
+ _decline_fused_lora_for_fsdp(getattr(self, "model", None), reason)
_replan_auto_offload_safely(self)
_block_swap_for_trainer_backend(self)
_enable_ddp_batched_grad_copy(self)
diff --git a/unsloth/models/fsdp2.py b/unsloth/models/fsdp2.py
new file mode 100644
index 00000000000..ab91d92ee17
--- /dev/null
+++ b/unsloth/models/fsdp2.py
@@ -0,0 +1,557 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""What Unsloth adds around accelerate's `fsdp2_prepare_model` (which already wraps per block,
+keeps the [norm, lm_head] tail gathered, groups tied embeddings). `UNSLOTH_FSDP2_ACCELERATE_DEFAULTS=1`
+skips the mixed precision fill and the last-block setting.
+"""
+
+import contextlib
+import dataclasses
+import functools
+import inspect
+import os
+
+import torch
+
+__all__ = [
+ "fsdp2_trainer_setup",
+ "install_fsdp2_hooks",
+ "install_fsdp2_policy_provenance",
+ "install_fsdp2_prepare_hook",
+ "install_fsdp2_no_sync",
+ "install_fsdp2_peft_full_state_dict",
+]
+
+_PATCHED = "_unsloth_fsdp2_patched"
+_FALSY = ("0", "false", "no", "off")
+_TRUTHY = ("1", "true", "yes", "on")
+# Unsloth's backbone forwards call fast_rms_layernorm on these instead of the module.
+_DIRECT_READ_NORMS = ("norm", "final_layernorm")
+
+
+def _fsdp_module_class():
+ try:
+ from torch.distributed.fsdp import FSDPModule
+ return FSDPModule
+ except ImportError:
+ pass
+ try:
+ from torch.distributed._composable.fsdp import FSDPModule
+ return FSDPModule
+ except ImportError:
+ return None
+
+
+def _rank0():
+ try:
+ import torch.distributed as dist
+ return not (dist.is_available() and dist.is_initialized()) or dist.get_rank() == 0
+ except Exception:
+ return True
+
+
+def _print_rank0(message):
+ if _rank0():
+ print(message)
+
+
+def _keep_accelerate_defaults():
+ return os.environ.get("UNSLOTH_FSDP2_ACCELERATE_DEFAULTS", "0").strip().lower() in _TRUTHY
+
+
+def _compute_dtype(accelerator):
+ return {"fp16": torch.float16, "bf16": torch.bfloat16}.get(
+ str(getattr(accelerator, "mixed_precision", "no"))
+ )
+
+
+_AUTO_POLICY = "_unsloth_auto_mixed_precision_policy"
+_POLICY_SEEN = "_unsloth_mixed_precision_policy_seen"
+
+
+def install_fsdp2_policy_provenance():
+ """Tag the policy accelerate builds from `mixed_precision` itself, so a user-written policy with
+ the same dtypes is never mistaken for it. Must run before the Accelerator is created."""
+ try:
+ from accelerate.utils import FullyShardedDataParallelPlugin
+ except Exception:
+ return False
+ original = getattr(FullyShardedDataParallelPlugin, "set_mixed_precision", None)
+ if original is None:
+ return False
+ if getattr(original, _PATCHED, False):
+ return True
+
+ @functools.wraps(original)
+ def set_mixed_precision(self, mixed_precision, *args, **kwargs):
+ before = getattr(self, "mixed_precision_policy", None)
+ result = original(self, mixed_precision, *args, **kwargs)
+ override = kwargs.get("override", args[1] if len(args) > 1 else False)
+ after = getattr(self, "mixed_precision_policy", None)
+ setattr(self, _POLICY_SEEN, True)
+ if before is None and not override and after is not None:
+ setattr(self, _AUTO_POLICY, after)
+ return result
+
+ setattr(set_mixed_precision, _PATCHED, True)
+ FullyShardedDataParallelPlugin.set_mixed_precision = set_mixed_precision
+ return True
+
+
+def fill_fsdp2_mixed_precision(accelerator):
+ """Only rewrites accelerate's automatic policy (every dtype = compute dtype): it casts the loss
+ to float16, where the 65536 loss scale overflows and the scaler skips the first steps."""
+ plugin = getattr(getattr(accelerator, "state", None), "fsdp_plugin", None)
+ policy = getattr(plugin, "mixed_precision_policy", None)
+ if policy is None or _keep_accelerate_defaults():
+ return policy
+ automatic = getattr(plugin, _AUTO_POLICY, None) is policy
+ if not automatic and getattr(plugin, _POLICY_SEEN, False):
+ return policy
+ try:
+ from torch.distributed.fsdp import MixedPrecisionPolicy
+ except ImportError:
+ return policy
+ compute = _compute_dtype(accelerator)
+ if compute is None or not isinstance(policy, MixedPrecisionPolicy):
+ return policy
+ if (
+ policy.param_dtype != compute
+ or policy.reduce_dtype != compute
+ or getattr(policy, "output_dtype", None) not in (None, compute)
+ ):
+ return policy
+ if not automatic:
+ # The Accelerator was built before Unsloth was imported, so whether accelerate or the caller
+ # wrote this policy is unknown. Only float16 overflows with it.
+ if compute != torch.float16:
+ return policy
+ raise RuntimeError(
+ "Unsloth: this FSDP2 Accelerator was created before `import unsloth`, so Unsloth cannot "
+ "tell whether its float16 mixed precision policy (float16 gathers, reductions and "
+ "outputs, which overflow at the GradScaler's initial loss scale) was chosen by "
+ "accelerate or by you. Import unsloth before creating the Accelerator, or set "
+ "UNSLOTH_FSDP2_ACCELERATE_DEFAULTS=1 to keep the policy as it is."
+ )
+ changes = {"reduce_dtype": torch.float32}
+ if compute == torch.float16:
+ # float16 gathers move the embedding / norm / lm_head backward into float16, where scaled
+ # gradients overflow; gather the float32 masters and let autocast cast like one GPU does.
+ changes["param_dtype"] = None
+ if hasattr(policy, "output_dtype"):
+ changes["output_dtype"] = None
+ plugin.mixed_precision_policy = dataclasses.replace(policy, **changes)
+ return plugin.mixed_precision_policy
+
+
+def unify_trainable_dtypes(accelerator, model):
+ """16-bit full finetuning runs without autocast with float32 norms, which accelerate does not
+ upcast and FSDP2 refuses ("FSDP expects uniform original parameter dtype"). A policy the caller
+ set is kept; only the master weights are upcast."""
+ trainable = [p for p in model.parameters() if p.requires_grad and p.is_floating_point()]
+ dtypes = {p.dtype for p in trainable}
+ if str(getattr(accelerator, "mixed_precision", "no")) != "no":
+ # accelerate upcasts these itself, except 1.13 (`param = param.to(float32)` drops the result).
+ if len(dtypes) <= 1:
+ return False
+ for p in trainable:
+ if p.dtype != torch.float32:
+ p.data = p.data.to(torch.float32)
+ return True
+ half = torch.bfloat16 if torch.bfloat16 in dtypes else torch.float16
+ if len(dtypes) <= 1 or half not in dtypes:
+ return False
+ try:
+ from torch.distributed.fsdp import MixedPrecisionPolicy
+ except ImportError:
+ return False
+ plugin = getattr(getattr(accelerator, "state", None), "fsdp_plugin", None)
+ if plugin is None:
+ return False
+ for p in trainable:
+ if p.dtype != torch.float32:
+ p.data = p.data.to(torch.float32)
+ if getattr(plugin, "mixed_precision_policy", None) is None:
+ plugin.mixed_precision_policy = MixedPrecisionPolicy(
+ param_dtype = half, reduce_dtype = torch.float32
+ )
+ how = f"gathering them in {str(half).replace('torch.', '')} for compute"
+ else:
+ how = "gathering them as the given mixed_precision_policy says"
+ _print_rank0(
+ "Unsloth: FSDP2 needs one dtype per unit, and the trainable weights mix "
+ f"{sorted(str(d).replace('torch.', '') for d in dtypes)}. Keeping float32 master weights and {how}."
+ )
+ return True
+
+
+def _fsdp2_checks_frozen_dtypes():
+ """torch < 2.8 asserts one original dtype over every parameter of a unit, frozen ones included
+ (`FSDPParamGroup._init_mp_dtypes`); 2.8 checks only the trainable ones."""
+ try:
+ from torch.distributed.fsdp._fully_shard._fsdp_param_group import FSDPParamGroup
+ return "requires_grad" not in inspect.getsource(FSDPParamGroup._init_mp_dtypes)
+ except Exception:
+ pass
+ try:
+ major, minor = (int(x) for x in torch.__version__.split(".")[:2])
+ return (major, minor) < (2, 8)
+ except Exception:
+ return False
+
+
+def unify_frozen_dtypes(
+ accelerator,
+ model,
+ frozen_checked = None,
+):
+ """LoRA on torch < 2.8: float32 adapters next to frozen 16-bit weights in one unit. Upcast the
+ frozen weights too (what accelerate <= 1.12 did with `model.to(float32)`); the gather policy
+ casts them back down for compute."""
+ if frozen_checked is None:
+ frozen_checked = _fsdp2_checks_frozen_dtypes()
+ if not frozen_checked:
+ return False
+ dtypes = {p.dtype for p in model.parameters() if p.is_floating_point()}
+ if len(dtypes) <= 1:
+ return False
+ plugin = getattr(getattr(accelerator, "state", None), "fsdp_plugin", None)
+ if plugin is None:
+ return False
+ if (
+ getattr(plugin, "mixed_precision_policy", None) is None
+ and str(getattr(accelerator, "mixed_precision", "no")) == "no"
+ ):
+ raise RuntimeError(
+ f"Unsloth: FSDP2 on torch {torch.__version__} needs every parameter of a unit to share "
+ f"one dtype, but this model mixes {sorted(str(d) for d in dtypes)} and no mixed "
+ "precision is set to compute in 16-bit from float32 weights. Enable bf16 / fp16 mixed "
+ "precision, or use torch >= 2.8."
+ )
+ n = 0
+ for p in model.parameters():
+ # Quantized parameter subclasses keep their storage dtype.
+ if type(p) is torch.nn.Parameter and p.is_floating_point() and p.dtype != torch.float32:
+ p.data = p.data.to(torch.float32)
+ n += 1
+ if n:
+ _print_rank0(
+ f"Unsloth: FSDP2 on torch {torch.__version__} needs one dtype per unit, frozen weights "
+ f"included, so {n} 16-bit parameters are kept in float32 and gathered in the compute dtype."
+ )
+ return n > 0
+
+
+def _parents(model):
+ parents = {}
+ for parent in model.modules():
+ for child in parent.children():
+ parents.setdefault(id(child), parent)
+ return parents
+
+
+def _fsdp_owner(module, parents, FSDPModule):
+ """The FSDP unit whose pre-forward gathers `module`'s own parameters."""
+ while module is not None:
+ if isinstance(module, FSDPModule):
+ return module
+ module = parents.get(id(module))
+ return None
+
+
+def _decoder_blocks(model):
+ names = getattr(model, "_no_split_modules", None) or ()
+ names = set(names)
+ if not names:
+ return []
+ # fully_shard swaps a module's class for an FSDP<Name> subclass, so match along the MRO.
+ return [m for m in model.modules() if any(c.__name__ in names for c in type(m).__mro__)]
+
+
+def _all_ranks_agree(value, device):
+ """True only when every rank says True, so every rank takes the same branch."""
+ try:
+ import torch.distributed as dist
+
+ if not (dist.is_available() and dist.is_initialized()):
+ return value
+ flag = torch.tensor([1 if value else 0], dtype = torch.int32, device = device)
+ dist.all_reduce(flag, op = dist.ReduceOp.MIN)
+ return bool(flag.item())
+ except Exception:
+ return value
+
+
+def _in_decoder_block(module, block_ids, parents):
+ while module is not None:
+ parent = parents.get(id(module))
+ if id(module) in block_ids or isinstance(parent, torch.nn.ModuleList):
+ return True
+ module = parent
+ return False
+
+
+def _fused_lora_wrap_problem(
+ model,
+ fused,
+ parents,
+ FSDPModule,
+ blocks = (),
+):
+ """None when every fused layer's weights are gathered by a per-block unit, else why not."""
+ block_ids = {id(b) for b in blocks}
+ for module in fused:
+ owner = _fsdp_owner(module, parents, FSDPModule)
+ if owner is None or owner is model:
+ return "FSDP2 wraps only the root module, not each decoder block"
+ if not _in_decoder_block(owner, block_ids, parents):
+ # Blocks are always entered through __call__; a unit above them need not be (peft's
+ # LoraModel calls the CausalLM's .forward), so its pre-forward gather can be skipped.
+ return "FSDP2 wraps a module above the decoder blocks, not each block"
+ for sub in module.modules():
+ if sub is not module and isinstance(sub, FSDPModule):
+ return "FSDP2 made a submodule of a fused LoRA layer its own unit"
+ return None
+
+
+def _gate_fused_lora(
+ accelerator,
+ model,
+ parents,
+ FSDPModule,
+ blocks = None,
+):
+ from .llama import _decline_fused_lora_for_fsdp, _fused_lora_modules
+
+ fused = _fused_lora_modules(model)
+ if not fused or os.environ.get("UNSLOTH_FORCE_FUSED_LORA", "0") == "1":
+ model._unsloth_fsdp2_fused_lora = len(fused)
+ return len(fused)
+ if os.environ.get("UNSLOTH_FSDP2_FUSED_LORA", "1").strip().lower() in _FALSY:
+ # A raw loop patches the model before any FSDP plugin exists, so load time could not see it.
+ problem = "UNSLOTH_FSDP2_FUSED_LORA=0"
+ else:
+ if blocks is None:
+ blocks = _decoder_blocks(model)
+ problem = _fused_lora_wrap_problem(model, fused, parents, FSDPModule, blocks)
+ device = getattr(accelerator, "device", None) or torch.device("cpu")
+ if not _all_ranks_agree(problem is None, device):
+ _decline_fused_lora_for_fsdp(model, problem or "another rank's FSDP2 wrap")
+ model._unsloth_fsdp2_fused_lora = 0
+ return 0
+ model._unsloth_fsdp2_fused_lora = len(fused)
+ _print_rank0(
+ f"Unsloth: FSDP2 shards each decoder block as its own unit, so {len(fused)} fused LoRA "
+ "layers stay on (their weights are gathered before the block's forward)."
+ )
+ return len(fused)
+
+
+def _gather_units_read_directly(model, blocks, parents, FSDPModule):
+ """Unsloth reads lm_head / final norm weights directly (fused CE, fast RMSNorm), so FSDP's
+ pre-forward never gathers their unit (accelerate >= 1.14 splits it off)."""
+ if not hasattr(FSDPModule, "unshard"):
+ return 0
+ targets = []
+ get_output = getattr(model, "get_output_embeddings", None)
+ output = get_output() if callable(get_output) else None
+ if isinstance(output, torch.nn.Module):
+ targets.append(output)
+ for name, module in model.named_modules():
+ if name.rsplit(".", 1)[-1] in _DIRECT_READ_NORMS:
+ parent = parents.get(id(module))
+ if parent is not None and isinstance(
+ getattr(parent, "layers", None), torch.nn.ModuleList
+ ):
+ targets.append(module)
+ block_ids = {id(b) for b in blocks}
+ units = []
+ for target in targets:
+ owner = _fsdp_owner(target, parents, FSDPModule)
+ if owner is None or owner is model or id(owner) in block_ids:
+ continue
+ if all(owner is not u for u in units):
+ units.append(owner)
+ if not units:
+ return 0
+
+ # On the root: peft's LoraModel calls the CausalLM's .forward directly, skipping its hooks.
+ def _unsloth_gather_direct_reads(module, args):
+ for unit in units:
+ unit.unshard()
+
+ model.register_forward_pre_hook(_unsloth_gather_direct_reads)
+ return len(units)
+
+
+def after_fsdp2_wrap(accelerator, model):
+ FSDPModule = _fsdp_module_class()
+ if FSDPModule is None or not isinstance(model, FSDPModule):
+ return
+ if getattr(model, "_unsloth_fsdp2_prepared", False):
+ return
+ model._unsloth_fsdp2_prepared = True
+ parents = _parents(model)
+ blocks = [b for b in _decoder_blocks(model) if isinstance(b, FSDPModule)]
+ plugin = getattr(getattr(accelerator, "state", None), "fsdp_plugin", None)
+ if (
+ blocks
+ and not _keep_accelerate_defaults()
+ and getattr(plugin, "reshard_after_forward", True) is True
+ and hasattr(blocks[-1], "set_reshard_after_forward")
+ ):
+ # Backward starts with the last block, so resharding it after forward only re-gathers it.
+ blocks[-1].set_reshard_after_forward(False, recurse = False)
+ model._unsloth_fsdp2_blocks = len(blocks)
+ model._unsloth_fsdp2_direct_read_units = _gather_units_read_directly(
+ model, blocks, parents, FSDPModule
+ )
+ _gate_fused_lora(accelerator, model, parents, FSDPModule, blocks)
+
+
+def install_fsdp2_prepare_hook():
+ """Wrap accelerate's `fsdp2_prepare_model` (looked up by `Accelerator._prepare_fsdp2`)."""
+ try:
+ import accelerate.accelerator as accelerator_module
+ except Exception:
+ return False
+ original = getattr(accelerator_module, "fsdp2_prepare_model", None)
+ if original is None:
+ return False
+ if getattr(original, _PATCHED, False):
+ return True
+
+ @functools.wraps(original)
+ def fsdp2_prepare_model(accelerator, model, *args, **kwargs):
+ fill_fsdp2_mixed_precision(accelerator)
+ unify_trainable_dtypes(accelerator, model)
+ unify_frozen_dtypes(accelerator, model)
+ model = original(accelerator, model, *args, **kwargs)
+ after_fsdp2_wrap(accelerator, model)
+ return model
+
+ setattr(fsdp2_prepare_model, _PATCHED, True)
+ accelerator_module.fsdp2_prepare_model = fsdp2_prepare_model
+ return True
+
+
+def install_fsdp2_no_sync():
+ """accelerate < 1.11 looks for `model.no_sync`, which FSDP2 modules lack."""
+ try:
+ from accelerate import Accelerator
+ except Exception:
+ return False
+ original = Accelerator.no_sync
+ if getattr(original, _PATCHED, False):
+ return True
+ try:
+ source = inspect.getsource(original)
+ except (OSError, TypeError):
+ return False
+ if "set_requires_gradient_sync" in source:
+ return True
+
+ @contextlib.contextmanager
+ def no_sync(self, model):
+ if getattr(self, "is_fsdp2", False) and hasattr(model, "set_requires_gradient_sync"):
+ model.set_requires_gradient_sync(False)
+ try:
+ yield
+ finally:
+ model.set_requires_gradient_sync(True)
+ else:
+ with original(self, model):
+ yield
+
+ no_sync.__doc__ = original.__doc__
+ setattr(no_sync, _PATCHED, True)
+ Accelerator.no_sync = no_sync
+ return True
+
+
+def install_fsdp2_peft_full_state_dict():
+ """accelerate < 1.15 builds an FSDP2 PEFT model's FULL_STATE_DICT checkpoint (what Trainer writes for
+ resuming) from `get_peft_model_state_dict`, i.e. each rank's DTensor shard, and rank 0 saves only
+ its own. Backport of the 1.15 fix: gather the trainable weights with `get_model_state_dict`."""
+ try:
+ import accelerate.utils.fsdp_utils as fsdp_utils
+ except Exception:
+ return False
+ original_get = getattr(fsdp_utils, "_get_model_state_dict", None)
+ original_set = getattr(fsdp_utils, "_set_model_state_dict", None)
+ is_peft_model = getattr(fsdp_utils, "is_peft_model", None)
+ if original_get is None or original_set is None or is_peft_model is None:
+ return False
+ if getattr(original_get, _PATCHED, False):
+ return True
+ try:
+ source = inspect.getsource(original_get)
+ parameters = inspect.signature(original_get).parameters
+ except (OSError, TypeError, ValueError):
+ return False
+ if "full_state_dict" in source or "sd_options" not in parameters:
+ return True
+
+ def _full_peft(model, adapter_only, sd_options):
+ return (
+ adapter_only
+ and sd_options is not None
+ and getattr(sd_options, "full_state_dict", False)
+ and is_peft_model(model)
+ )
+
+ @functools.wraps(original_get)
+ def _get_model_state_dict(
+ model,
+ adapter_only = False,
+ sd_options = None,
+ ):
+ if _full_peft(model, adapter_only, sd_options):
+ from torch.distributed.checkpoint.state_dict import get_model_state_dict
+ options = dataclasses.replace(sd_options, ignore_frozen_params = True)
+ return get_model_state_dict(model, options = options)
+ return original_get(model, adapter_only = adapter_only, sd_options = sd_options)
+
+ @functools.wraps(original_set)
+ def _set_model_state_dict(
+ model,
+ state_dict,
+ adapter_only = False,
+ sd_options = None,
+ ):
+ if _full_peft(model, adapter_only, sd_options):
+ from torch.distributed.checkpoint.state_dict import set_model_state_dict
+ options = dataclasses.replace(sd_options, strict = False)
+ return set_model_state_dict(model, state_dict, options = options)
+ return original_set(model, state_dict, adapter_only = adapter_only, sd_options = sd_options)
+
+ setattr(_get_model_state_dict, _PATCHED, True)
+ setattr(_set_model_state_dict, _PATCHED, True)
+ fsdp_utils._get_model_state_dict = _get_model_state_dict
+ fsdp_utils._set_model_state_dict = _set_model_state_dict
+ return True
+
+
+def install_fsdp2_hooks():
+ """Everything FSDP2 needs before an Accelerator exists; also for raw `Accelerator.prepare()` loops."""
+ install_fsdp2_policy_provenance()
+ install_fsdp2_prepare_hook()
+ install_fsdp2_no_sync()
+ install_fsdp2_peft_full_state_dict()
+
+
+def fsdp2_trainer_setup(trainer):
+ """Returns (keep_fused_lora, reason); False for FSDP1 and for a root-only FSDP2 wrap."""
+ accelerator = getattr(trainer, "accelerator", None)
+ plugin = getattr(getattr(accelerator, "state", None), "fsdp_plugin", None)
+ if plugin is None or getattr(plugin, "fsdp_version", 1) != 2:
+ return False, None
+ install_fsdp2_prepare_hook()
+ install_fsdp2_no_sync()
+ install_fsdp2_peft_full_state_dict()
+ if os.environ.get("UNSLOTH_FSDP2_FUSED_LORA", "1").strip().lower() in _FALSY:
+ return False, "UNSLOTH_FSDP2_FUSED_LORA=0"
+ if getattr(plugin, "auto_wrap_policy", None) is None:
+ return False, "FSDP2 wraps only the root module (auto_wrap_policy NO_WRAP)"
+ return True, None
diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py
index 7713d0abb81..6fc7c4c4ed5 100644
--- a/unsloth/models/llama.py
+++ b/unsloth/models/llama.py
@@ -66,6 +66,7 @@ from .loader_utils import (
_restore_dropped_fp8_scales,
_prepare_compressed_tensors_model,
fsdp_will_wrap,
+ fsdp2_keeps_fused_lora,
planner_class_mismatch_reason,
planner_model_class,
planner_config_overrides,
@@ -2721,7 +2722,10 @@ def _fused_lora_skip_reason(
if float32_base:
reasons.append("the base weights are float32")
if fsdp:
- reasons.append("FSDP shards the weights they read (UNSLOTH_FORCE_FUSED_LORA=1 overrides)")
+ reasons.append(
+ "FSDP1 hands them shard views of the weights they read (UNSLOTH_FORCE_FUSED_LORA=1 "
+ "overrides; FSDP2 with one unit per decoder block keeps them)"
+ )
if not reasons:
return ""
return (
@@ -2874,11 +2878,32 @@ def _has_packed_base(*projs):
_FUSED_LORA_MLPS = (apply_lora_mlp_swiglu, apply_lora_mlp_geglu_exact, apply_lora_mlp_geglu_approx)
-def _decline_fused_lora_for_fsdp(model) -> int:
+def _fused_lora_modules(model) -> list:
+ """The modules patch_peft_model gave a fused LoRA kernel (attention for QKV / O, MLPs)."""
+ if model is None:
+ return []
+ fused = []
+ for module in model.modules():
+ if (
+ getattr(module, "apply_qkv", None) is apply_lora_qkv
+ or getattr(module, "apply_o", None) is apply_lora_o
+ ):
+ fused.append(module)
+ continue
+ for name in ("forward", "_unsloth_forward"):
+ func = getattr(module.__dict__.get(name), "__func__", None)
+ if getattr(func, "func", func) in _FUSED_LORA_MLPS:
+ fused.append(module)
+ break
+ return fused
+
+
+def _decline_fused_lora_for_fsdp(model, reason = None) -> int:
"""Put peft's forwards back on layers patch_peft_model gave a fused LoRA kernel.
`TrainingArguments(fsdp=...)` only reaches the Accelerator inside `Trainer.__init__`, after
patch_peft_model ran with no launcher env to see, so the Trainer calls this once it knows.
+ Under FSDP2 the check after accelerate shards the model calls it too, with its `reason`.
"""
if model is None or os.environ.get("UNSLOTH_FORCE_FUSED_LORA", "0") == "1":
return 0
@@ -2900,8 +2925,10 @@ def _decline_fused_lora_for_fsdp(model) -> int:
module._unsloth_forward = module.__class__.forward
n += 1
if n:
+ if reason is None:
+ reason = "the Trainer enabled FSDP"
logger.warning_once(
- f"Unsloth: the Trainer enabled FSDP, so {n} fused LoRA projections were switched back "
+ f"Unsloth: {reason}, so {n} fused LoRA projections were switched back "
"to peft's forward, which FSDP can unshard. Set UNSLOTH_FORCE_FUSED_LORA=1 to keep them."
)
return n
@@ -4754,8 +4781,9 @@ class FastLlamaModel:
float32_base = (
_base_weight_dtype(model.model.model.layers[0].self_attn.q_proj) == torch.float32
)
- # Fused kernels read `.weight` directly, bypassing FSDP's unshard hook, so under FSDP they see shard views (#409).
- fused_lora_declined_for_fsdp = fsdp_will_wrap()
+ # Fused kernels read `.weight` directly, bypassing FSDP's unshard hook, so under FSDP1 they see shard
+ # views (#409). FSDP2 unshards each block before its forward; the wrap is re-checked once built.
+ fused_lora_declined_for_fsdp = fsdp_will_wrap() and not fsdp2_keeps_fused_lora()
if (
lora_dropout == 0
diff --git a/unsloth/models/loader_utils.py b/unsloth/models/loader_utils.py
index 8f40869a4e5..2e3dc2e21ee 100644
--- a/unsloth/models/loader_utils.py
+++ b/unsloth/models/loader_utils.py
@@ -10,6 +10,7 @@
# limitations under the License.
from ..device_type import DEVICE_TYPE_TORCH, clean_gpu_cache
+from ..fsdp2_launch import fsdp_version_from_env
import hashlib
import importlib
import os
@@ -130,6 +131,31 @@ def _infer_local_rank(rank):
_TRUTHY = ("1", "true", "yes", "on")
+def fsdp_version_requested():
+ """The FSDP version this process trains with: 0 for none, else 1 or 2.
+
+ A built Accelerator is authoritative; before one exists (model load, patch_peft_model) the
+ launcher env is the only signal (`fsdp_version_from_env`: `accelerate launch` with an FSDP
+ config, FSDP_VERSION by hand, or UNSLOTH_FSDP2=1 under torchrun). Every rank of a job sees the
+ same env and the same Accelerator config, so every rank gets the same answer.
+ `TrainingArguments(fsdp = ...)` alone is invisible until the Trainer builds its Accelerator;
+ the Trainer re-checks then (`_unsloth_trainer_init`).
+ """
+ try:
+ from accelerate.state import AcceleratorState
+
+ shared = AcceleratorState._shared_state
+ plugin = shared.get("fsdp_plugin", None)
+ if plugin is not None:
+ return int(getattr(plugin, "fsdp_version", 1) or 1)
+ distributed_type = shared.get("distributed_type", None)
+ if distributed_type is not None and "FSDP" in str(distributed_type).upper():
+ return fsdp_version_from_env() or 1
+ except Exception:
+ pass
+ return fsdp_version_from_env()
+
+
def fsdp_will_wrap():
"""True when this process will hand the model to FSDP (unsloth#409).
@@ -139,32 +165,32 @@ def fsdp_will_wrap():
"""
if os.environ.get("UNSLOTH_FORCE_FUSED_LORA", "0") == "1":
return False
- try:
- from accelerate.state import AcceleratorState
- distributed_type = AcceleratorState._shared_state.get("distributed_type", None)
- if distributed_type is not None and "FSDP" in str(distributed_type).upper():
- return True
- except Exception:
- pass
- if str(os.environ.get("ACCELERATE_USE_FSDP", "")).strip().lower() in _TRUTHY:
- return True
- # "0" is how an accelerate config says "not FSDP".
- if str(os.environ.get("FSDP_VERSION", "")).strip() not in ("", "0"):
- return True
- return False
+ return fsdp_version_requested() > 0
def fsdp2_requested():
- """True when this process runs under FSDP2: `accelerate launch` exports FSDP_VERSION=2, an
- Accelerator built already records its fsdp_plugin."""
- if os.environ.get("FSDP_VERSION", "").strip() == "2":
- return True
- try:
- from accelerate.state import AcceleratorState
- plugin = AcceleratorState._shared_state.get("fsdp_plugin", None)
- return getattr(plugin, "fsdp_version", None) == 2
- except Exception:
+ """True when this process runs under FSDP2: `accelerate launch` exports FSDP_VERSION=2,
+ UNSLOTH_FSDP2=1 under torchrun, or an Accelerator built already records its fsdp_plugin."""
+ return fsdp_version_requested() == 2
+
+
+def fsdp2_keeps_fused_lora():
+ """Load-time half of the fused LoRA gate under FSDP.
+
+ FSDP1 hands the fused kernels flat shard views (unsloth#409), so they are declined. FSDP2
+ unshards a block's parameters in its pre-forward hook and the kernels run inside the block's
+ forward, so with one FSDP unit per decoder block they read whole weights; the wrap accelerate
+ actually builds is checked once it exists (`unsloth.models.fsdp2`), which declines them for a
+ root-only or finer wrap. `UNSLOTH_FSDP2_FUSED_LORA=0` declines them under FSDP2 as well.
+ """
+ if str(os.environ.get("UNSLOTH_FSDP2_FUSED_LORA", "1")).strip().lower() in (
+ "0",
+ "false",
+ "no",
+ "off",
+ ):
return False
+ return fsdp2_requested()
def raise_if_fast_inference_under_fsdp2(fast_inference):