Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
FSDP2: first-class LoRA and full finetuning, keep fused LoRA kernels per block (#13865)
Daniel Hanee2ce8ccf8e3
diff --git a/tests/test_fsdp2_config.py b/tests/test_fsdp2_config.pynew file mode 100644index 00000000000..7543bdd840a--- /dev/null+++ b/tests/test_fsdp2_config.py@@ -0,0 +1,295 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""What Unsloth fills into accelerate's FSDP2 config (unsloth/models/fsdp2.py). No GPU."""++from __future__ import annotations++import contextlib+import types++import pytest++torch = pytest.importorskip("torch")+pytest.importorskip("accelerate")+fsdp = pytest.importorskip("torch.distributed.fsdp")+MixedPrecisionPolicy = getattr(fsdp, "MixedPrecisionPolicy", None)+if MixedPrecisionPolicy is None:+ pytest.skip("torch without FSDP2", allow_module_level = True)++from unsloth.models import fsdp2 # noqa: E402++[email protected](autouse = True)+def _clean_env(monkeypatch):+ for key in ("UNSLOTH_FSDP2_ACCELERATE_DEFAULTS", "UNSLOTH_FSDP2_FUSED_LORA"):+ monkeypatch.delenv(key, raising = False)+++def _accelerator(+ policy,+ mixed_precision = "fp16",+ version = 2,+ auto_wrap_policy = object(),+ auto = True,+ seen = True,+):+ plugin = types.SimpleNamespace(+ fsdp_version = version,+ mixed_precision_policy = policy,+ auto_wrap_policy = auto_wrap_policy,+ reshard_after_forward = True,+ )+ if auto and seen:+ setattr(plugin, fsdp2._AUTO_POLICY, policy)+ if seen:+ setattr(plugin, fsdp2._POLICY_SEEN, True)+ return types.SimpleNamespace(+ state = types.SimpleNamespace(fsdp_plugin = plugin), mixed_precision = mixed_precision+ )++[email protected]("dtype,name", [(torch.float16, "fp16"), (torch.bfloat16, "bf16")])+def test_accelerates_auto_policy_reduces_in_fp32(dtype, name):+ acc = _accelerator(+ MixedPrecisionPolicy(param_dtype = dtype, reduce_dtype = dtype, output_dtype = dtype), name+ )+ policy = fsdp2.fill_fsdp2_mixed_precision(acc)+ # float16 gathers the stored dtype and leaves the casting to autocast, like the unsharded run.+ assert policy.param_dtype == (None if dtype == torch.float16 else dtype)+ assert policy.reduce_dtype == torch.float32+ assert policy.output_dtype is None+ assert acc.state.fsdp_plugin.mixed_precision_policy is policy+++def test_a_policy_the_user_wrote_is_left_alone():+ user = MixedPrecisionPolicy(param_dtype = torch.float16, reduce_dtype = torch.float16)+ acc = _accelerator(user, "bf16", auto = False)+ assert fsdp2.fill_fsdp2_mixed_precision(acc) is user+ user = MixedPrecisionPolicy(param_dtype = None, reduce_dtype = torch.float32)+ acc = _accelerator(user, "fp16", auto = False)+ assert fsdp2.fill_fsdp2_mixed_precision(acc) is user+ # Same dtypes as accelerate's automatic policy, but written by the user.+ user = MixedPrecisionPolicy(+ param_dtype = torch.float16, reduce_dtype = torch.float16, output_dtype = torch.float16+ )+ acc = _accelerator(user, "fp16", auto = False)+ assert fsdp2.fill_fsdp2_mixed_precision(acc) is user+++def test_an_accelerator_built_before_unsloth_import_is_not_guessed(monkeypatch):+ """Without provenance, an fp16 policy shaped like accelerate's automatic one is refused, not rewritten."""+ shaped = dict(param_dtype = torch.float16, reduce_dtype = torch.float16, output_dtype = torch.float16)+ with pytest.raises(RuntimeError, match = "before `import unsloth`"):+ fsdp2.fill_fsdp2_mixed_precision(+ _accelerator(MixedPrecisionPolicy(**shaped), "fp16", seen = False)+ )+ bf16 = MixedPrecisionPolicy(+ param_dtype = torch.bfloat16, reduce_dtype = torch.bfloat16, output_dtype = torch.bfloat16+ )+ assert fsdp2.fill_fsdp2_mixed_precision(_accelerator(bf16, "bf16", seen = False)) is bf16+ other = MixedPrecisionPolicy(param_dtype = None, reduce_dtype = torch.float32)+ assert fsdp2.fill_fsdp2_mixed_precision(_accelerator(other, "fp16", seen = False)) is other+ monkeypatch.setenv("UNSLOTH_FSDP2_ACCELERATE_DEFAULTS", "1")+ kept = MixedPrecisionPolicy(**shaped)+ assert fsdp2.fill_fsdp2_mixed_precision(_accelerator(kept, "fp16", seen = False)) is kept+++def test_provenance_tags_only_the_policy_accelerate_builds():+ from accelerate.utils import FullyShardedDataParallelPlugin++ assert fsdp2.install_fsdp2_policy_provenance()+ plugin = types.SimpleNamespace(fsdp_version = 2, mixed_precision_policy = None)+ FullyShardedDataParallelPlugin.set_mixed_precision(plugin, "fp16")+ assert getattr(plugin, fsdp2._AUTO_POLICY) is plugin.mixed_precision_policy+ user = MixedPrecisionPolicy(+ param_dtype = torch.float16, reduce_dtype = torch.float16, output_dtype = torch.float16+ )+ plugin = types.SimpleNamespace(fsdp_version = 2, mixed_precision_policy = user)+ FullyShardedDataParallelPlugin.set_mixed_precision(plugin, "fp16")+ assert plugin.mixed_precision_policy is user+ assert not hasattr(plugin, fsdp2._AUTO_POLICY)+ assert getattr(plugin, fsdp2._POLICY_SEEN) is True+++def test_no_mixed_precision_and_the_opt_out(monkeypatch):+ auto = MixedPrecisionPolicy(+ param_dtype = torch.float16, reduce_dtype = torch.float16, output_dtype = torch.float16+ )+ assert fsdp2.fill_fsdp2_mixed_precision(_accelerator(None, "no")) is None+ monkeypatch.setenv("UNSLOTH_FSDP2_ACCELERATE_DEFAULTS", "1")+ assert fsdp2.fill_fsdp2_mixed_precision(_accelerator(auto, "fp16")) is auto+++def test_trainer_setup_keeps_fused_lora_only_for_a_per_block_fsdp2_wrap(monkeypatch):+ installed = []+ monkeypatch.setattr(fsdp2, "install_fsdp2_prepare_hook", lambda: installed.append("prepare"))+ monkeypatch.setattr(fsdp2, "install_fsdp2_no_sync", lambda: installed.append("no_sync"))++ def trainer(**kw):+ return types.SimpleNamespace(accelerator = _accelerator(None, **kw))++ assert fsdp2.fsdp2_trainer_setup(trainer(version = 1)) == (False, None)+ assert installed == []+ assert fsdp2.fsdp2_trainer_setup(trainer()) == (True, None)+ assert installed == ["prepare", "no_sync"]+ keep, reason = fsdp2.fsdp2_trainer_setup(trainer(auto_wrap_policy = None))+ assert keep is False and "root" in reason+ monkeypatch.setenv("UNSLOTH_FSDP2_FUSED_LORA", "0")+ keep, reason = fsdp2.fsdp2_trainer_setup(trainer())+ assert keep is False and "UNSLOTH_FSDP2_FUSED_LORA" in reason+++def test_old_accelerate_no_sync_gets_fsdp2_gradient_sync(monkeypatch):+ """accelerate < 1.11 only looked for `model.no_sync`, which FSDP2 modules lack."""+ from accelerate import Accelerator++ @contextlib.contextmanager+ def old_no_sync(self, model):+ context = getattr(model, "no_sync", contextlib.nullcontext)+ with context():+ yield++ monkeypatch.setattr(Accelerator, "no_sync", old_no_sync)+ assert fsdp2.install_fsdp2_no_sync() is True+ assert Accelerator.no_sync is not old_no_sync+ assert fsdp2.install_fsdp2_no_sync() is True # idempotent++ calls = []+ model = types.SimpleNamespace(set_requires_gradient_sync = calls.append)+ fake = types.SimpleNamespace(is_fsdp2 = True)+ with Accelerator.no_sync(fake, model):+ assert calls == [False]+ assert calls == [False, True]+ fake.is_fsdp2 = False+ with Accelerator.no_sync(fake, model):+ pass+ assert calls == [False, True]+++def test_current_accelerate_no_sync_is_not_touched(monkeypatch):+ from accelerate import Accelerator+ import inspect++ original = Accelerator.no_sync+ if "set_requires_gradient_sync" not in inspect.getsource(original):+ pytest.skip("accelerate < 1.11 in this environment")+ assert fsdp2.install_fsdp2_no_sync() is True+ assert Accelerator.no_sync is original+++def test_the_trainer_init_wrapper_routes_fsdp_through_the_setup():+ import inspect++ from unsloth.models import _utils++ source = inspect.getsource(_utils.patch_gradient_accumulation_fix)+ assert "fsdp2_trainer_setup" in source and "_decline_fused_lora_for_fsdp" in source+++def test_bf16_full_finetuning_gets_one_trainable_dtype_per_unit():+ """Unsloth's bfloat16 full finetuning: no autocast, bf16 linears, fp32 norms (FSDP2 refuses mixed units)."""+ model = torch.nn.Sequential(torch.nn.Linear(4, 4).to(torch.bfloat16), torch.nn.LayerNorm(4))+ frozen = torch.nn.Linear(4, 4).to(torch.bfloat16).requires_grad_(False)+ model.append(frozen)+ acc = _accelerator(None, "no")+ assert fsdp2.unify_trainable_dtypes(acc, model) is True+ assert {p.dtype for p in model.parameters() if p.requires_grad} == {torch.float32}+ assert frozen.weight.dtype == torch.bfloat16+ policy = acc.state.fsdp_plugin.mixed_precision_policy+ assert policy.param_dtype == torch.bfloat16 and policy.reduce_dtype == torch.float32++[email protected]("explicit", [False, True])+def test_fp16_full_finetuning_gets_one_trainable_dtype_per_unit(explicit):+ model = torch.nn.Sequential(torch.nn.Linear(4, 4).to(torch.float16), torch.nn.LayerNorm(4))+ user = (+ MixedPrecisionPolicy(param_dtype = torch.float16, reduce_dtype = torch.float32)+ if explicit+ else None+ )+ acc = _accelerator(user, "no", auto = False)+ assert fsdp2.unify_trainable_dtypes(acc, model) is True+ assert {p.dtype for p in model.parameters()} == {torch.float32}+ policy = acc.state.fsdp_plugin.mixed_precision_policy+ if explicit:+ assert policy is user+ assert policy.param_dtype == torch.float16 and policy.reduce_dtype == torch.float32+++def test_uniform_or_autocast_runs_are_not_upcast():+ model = torch.nn.Sequential(torch.nn.Linear(4, 4).to(torch.bfloat16))+ assert fsdp2.unify_trainable_dtypes(_accelerator(None, "no"), model) is False+ assert model[0].weight.dtype == torch.bfloat16+ uniform = torch.nn.Sequential(torch.nn.Linear(4, 4).to(torch.bfloat16))+ acc = _accelerator(None, "bf16")+ policy = acc.state.fsdp_plugin.mixed_precision_policy+ # Under autocast a uniform unit is left to accelerate's own upcast.+ assert fsdp2.unify_trainable_dtypes(acc, uniform) is False+ assert uniform[0].weight.dtype == torch.bfloat16+ assert acc.state.fsdp_plugin.mixed_precision_policy is policy+++def test_mixed_precision_masters_are_unified_when_accelerate_does_not():+ """accelerate 1.13's upcast is `param = param.to(torch.float32)`, which leaves the model alone."""+ mixed = torch.nn.Sequential(+ torch.nn.Linear(4, 4).to(torch.bfloat16),+ torch.nn.LayerNorm(4),+ torch.nn.Linear(4, 4).to(torch.bfloat16).requires_grad_(False),+ )+ acc = _accelerator(None, "bf16")+ policy = acc.state.fsdp_plugin.mixed_precision_policy+ assert fsdp2.unify_trainable_dtypes(acc, mixed) is True+ assert {p.dtype for p in mixed.parameters() if p.requires_grad} == {torch.float32}+ assert mixed[2].weight.dtype == torch.bfloat16+ assert acc.state.fsdp_plugin.mixed_precision_policy is policy+++def test_install_fsdp2_hooks_patches_accelerate_without_a_trainer():+ import accelerate.accelerator as accelerator_module+ from accelerate import Accelerator+ from accelerate.utils import FullyShardedDataParallelPlugin++ fsdp2.install_fsdp2_hooks()+ assert getattr(accelerator_module.fsdp2_prepare_model, fsdp2._PATCHED, False)+ assert getattr(FullyShardedDataParallelPlugin.set_mixed_precision, fsdp2._PATCHED, False)+ assert hasattr(Accelerator, "no_sync")+ import inspect++ import accelerate.utils.fsdp_utils as fsdp_utils++ get = fsdp_utils._get_model_state_dict+ assert getattr(get, fsdp2._PATCHED, False) or "full_state_dict" in inspect.getsource(get)+++def test_an_explicit_policy_still_gets_float32_master_weights():+ """Accelerator mixed_precision="no" with a launcher policy: the masters still need one dtype."""+ model = torch.nn.Sequential(torch.nn.Linear(4, 4).to(torch.bfloat16), torch.nn.LayerNorm(4))+ user = MixedPrecisionPolicy(param_dtype = torch.bfloat16, reduce_dtype = torch.bfloat16)+ acc = _accelerator(user, "no", auto = False)+ assert fsdp2.unify_trainable_dtypes(acc, model) is True+ assert {p.dtype for p in model.parameters()} == {torch.float32}+ assert acc.state.fsdp_plugin.mixed_precision_policy is user+++def _lora_like():+ model = torch.nn.Sequential(torch.nn.Linear(4, 4).to(torch.bfloat16), torch.nn.Linear(4, 4))+ model[0].requires_grad_(False)+ return model+++def test_frozen_weights_are_upcast_only_where_torch_checks_them():+ policy = MixedPrecisionPolicy(param_dtype = torch.bfloat16, reduce_dtype = torch.float32)+ model = _lora_like()+ assert (+ fsdp2.unify_frozen_dtypes(_accelerator(policy, "bf16"), model, frozen_checked = False)+ is False+ )+ assert model[0].weight.dtype == torch.bfloat16+ assert (+ fsdp2.unify_frozen_dtypes(_accelerator(policy, "bf16"), model, frozen_checked = True) is True+ )+ assert {p.dtype for p in model.parameters()} == {torch.float32}+ assert not model[0].weight.requires_grad+ with pytest.raises(RuntimeError, match = "torch >= 2.8"):+ fsdp2.unify_frozen_dtypes(_accelerator(None, "no"), _lora_like(), frozen_checked = True)diff --git a/tests/test_fsdp2_detection.py b/tests/test_fsdp2_detection.pynew file mode 100644index 00000000000..ef6057c8f50--- /dev/null+++ b/tests/test_fsdp2_detection.py@@ -0,0 +1,307 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""FSDP2 detection in every launch mode: `accelerate launch` (ACCELERATE_USE_FSDP + FSDP_VERSION),+FSDP_VERSION by hand, UNSLOTH_FSDP2=1 under plain torchrun, and a built Accelerator. No GPU."""++from __future__ import annotations++import ast+import importlib.util+import os+import socket+from pathlib import Path++import pytest++_ROOT = Path(__file__).resolve().parent.parent+_LAUNCH = _ROOT / "unsloth" / "fsdp2_launch.py"++_ENV = (+ "UNSLOTH_FSDP2",+ "UNSLOTH_FSDP2_RESHARD_AFTER_FORWARD",+ "UNSLOTH_FSDP2_FUSED_LORA",+ "UNSLOTH_FORCE_FUSED_LORA",+ "ACCELERATE_USE_FSDP",+ "ACCELERATE_USE_DEEPSPEED",+ "FSDP_VERSION",+ "FSDP_AUTO_WRAP_POLICY",+ "FSDP_RESHARD_AFTER_FORWARD",+ "FSDP_STATE_DICT_TYPE",+ "WORLD_SIZE",+)+++def _load_launch():+ # By path: the module needs neither torch nor the unsloth package.+ spec = importlib.util.spec_from_file_location("_unsloth_fsdp2_launch", _LAUNCH)+ module = importlib.util.module_from_spec(spec)+ spec.loader.exec_module(module)+ return module++[email protected](autouse = True)+def _clean_env(monkeypatch):+ for key in _ENV:+ monkeypatch.delenv(key, raising = False)+++def launch():+ return _load_launch()+++def test_no_launcher_env_is_no_fsdp(launch):+ assert launch.fsdp_version_from_env() == 0+ assert launch.apply_unsloth_fsdp2_launch_env() is False+ assert "ACCELERATE_USE_FSDP" not in os.environ+++ "use_fsdp,version,expected",+ [+ ("true", "2", 2),+ ("true", "1", 1),+ ("true", "", 1), # accelerate builds FSDP1 when no version is named+ ("", "2", 2), # torchrun users export FSDP_VERSION by hand+ ("", "0", 0),+ ("false", "", 0),+ ],+)+def test_accelerate_launch_env(monkeypatch, launch, use_fsdp, version, expected):+ monkeypatch.setenv("ACCELERATE_USE_FSDP", use_fsdp)+ monkeypatch.setenv("FSDP_VERSION", version)+ assert launch.fsdp_version_from_env() == expected+++def test_unsloth_fsdp2_needs_more_than_one_process(monkeypatch, launch):+ monkeypatch.setenv("UNSLOTH_FSDP2", "1")+ assert launch.fsdp_version_from_env() == 0+ assert launch.apply_unsloth_fsdp2_launch_env() is False+ monkeypatch.setenv("WORLD_SIZE", "1")+ assert launch.apply_unsloth_fsdp2_launch_env() is False+ assert "FSDP_VERSION" not in os.environ+++def test_unsloth_fsdp2_exports_the_accelerate_fsdp2_launch(monkeypatch, launch):+ monkeypatch.setenv("UNSLOTH_FSDP2", "1")+ monkeypatch.setenv("WORLD_SIZE", "2")+ assert launch.fsdp_version_from_env() == 2+ assert launch.apply_unsloth_fsdp2_launch_env() is True+ assert os.environ["ACCELERATE_USE_FSDP"] == "true"+ assert os.environ["FSDP_VERSION"] == "2"+ assert os.environ["FSDP_AUTO_WRAP_POLICY"] == "TRANSFORMER_BASED_WRAP"+ assert os.environ["FSDP_STATE_DICT_TYPE"] == "FULL_STATE_DICT"+ assert "FSDP_RESHARD_AFTER_FORWARD" not in os.environ+ # Idempotent.+ assert launch.apply_unsloth_fsdp2_launch_env() is True+++def test_explicit_fsdp_settings_win(monkeypatch, launch):+ monkeypatch.setenv("UNSLOTH_FSDP2", "true")+ monkeypatch.setenv("WORLD_SIZE", "4")+ monkeypatch.setenv("FSDP_AUTO_WRAP_POLICY", "SIZE_BASED_WRAP")+ monkeypatch.setenv("FSDP_STATE_DICT_TYPE", "SHARDED_STATE_DICT")+ launch.apply_unsloth_fsdp2_launch_env()+ assert os.environ["FSDP_AUTO_WRAP_POLICY"] == "SIZE_BASED_WRAP"+ assert os.environ["FSDP_STATE_DICT_TYPE"] == "SHARDED_STATE_DICT"++[email protected]("value,expected", [("0", "false"), ("false", "false"), ("1", "true")])+def test_reshard_after_forward_switch(monkeypatch, launch, value, expected):+ monkeypatch.setenv("UNSLOTH_FSDP2", "1")+ monkeypatch.setenv("WORLD_SIZE", "2")+ monkeypatch.setenv("UNSLOTH_FSDP2_RESHARD_AFTER_FORWARD", value)+ launch.apply_unsloth_fsdp2_launch_env()+ assert os.environ["FSDP_RESHARD_AFTER_FORWARD"] == expected+++def _fake_accelerate(monkeypatch, launch, tmp_path, source):+ (tmp_path / "utils").mkdir()+ (tmp_path / "utils" / "fsdp_utils.py").write_text(source)+ spec = importlib.util.spec_from_file_location("accelerate", tmp_path / "__init__.py")+ spec.submodule_search_locations = [str(tmp_path)]+ monkeypatch.setattr(launch.importlib.util, "find_spec", lambda name: spec)+++ "source,ok",+ [+ ("def fsdp_prepare_model(): pass\n", False), # accelerate < 1.6: no FSDP2+ ("def fsdp2_prepare_model(): pass\n", False), # 1.6: no FSDP2 state dict options+ ("def fsdp2_prepare_model(): pass\ndef _prepare_sd_options(): pass\n", True),+ ],+)+def test_unsloth_fsdp2_needs_an_accelerate_with_fsdp2(monkeypatch, launch, tmp_path, source, ok):+ _fake_accelerate(monkeypatch, launch, tmp_path, source)+ monkeypatch.setenv("UNSLOTH_FSDP2", "1")+ monkeypatch.setenv("WORLD_SIZE", "2")+ if ok:+ assert launch.apply_unsloth_fsdp2_launch_env() is True+ else:+ with pytest.raises(RuntimeError, match = "accelerate >= 1.7"):+ launch.apply_unsloth_fsdp2_launch_env()+ assert "FSDP_VERSION" not in os.environ+++def test_the_installed_accelerate_is_read_without_importing_it(launch):+ pytest.importorskip("accelerate")+ assert launch._accelerate_fsdp2_support() is True+++def test_conflicting_launches_are_refused(monkeypatch, launch):+ monkeypatch.setenv("UNSLOTH_FSDP2", "1")+ monkeypatch.setenv("WORLD_SIZE", "2")+ monkeypatch.setenv("FSDP_VERSION", "1")+ with pytest.raises(ValueError, match = "FSDP_VERSION=1"):+ launch.apply_unsloth_fsdp2_launch_env()+ monkeypatch.delenv("FSDP_VERSION")+ monkeypatch.setenv("ACCELERATE_USE_DEEPSPEED", "true")+ with pytest.raises(RuntimeError, match = "DeepSpeed"):+ launch.apply_unsloth_fsdp2_launch_env()+++def test_the_env_becomes_accelerates_fsdp2_plugin(monkeypatch, launch):+ """What accelerate builds from the exported env: FSDP2, one unit per `_no_split_modules` block."""+ accelerate_utils = pytest.importorskip("accelerate.utils")+ from torch.distributed.fsdp.wrap import transformer_auto_wrap_policy++ monkeypatch.setenv("UNSLOTH_FSDP2", "1")+ monkeypatch.setenv("WORLD_SIZE", "2")+ monkeypatch.setenv("UNSLOTH_FSDP2_RESHARD_AFTER_FORWARD", "0")+ launch.apply_unsloth_fsdp2_launch_env()+ plugin = accelerate_utils.FullyShardedDataParallelPlugin()+ assert plugin.fsdp_version == 2+ assert plugin.auto_wrap_policy is transformer_auto_wrap_policy+ assert plugin.reshard_after_forward is False+ assert "FULL_STATE_DICT" in str(plugin.state_dict_type)+++def test_unsloth_init_applies_the_env_before_transformers():+ """The call must sit in unsloth/__init__.py ahead of any import that pulls in transformers."""+ source = (_ROOT / "unsloth" / "__init__.py").read_text(encoding = "utf-8")+ tree = ast.parse(source)+ call_line = None+ first_heavy = None+ for node in tree.body:+ text = ast.unparse(node)+ if call_line is None and "_apply_fsdp2_env()" in text:+ call_line = node.lineno+ if first_heavy is None and isinstance(node, (ast.Import, ast.ImportFrom)):+ if any(x in text for x in ("transformers", "accelerate", "unsloth_zoo", ".models")):+ first_heavy = node.lineno+ assert call_line is not None+ assert first_heavy is None or call_line < first_heavy+++def test_loader_utils_versions(monkeypatch):+ loader_utils = pytest.importorskip("unsloth.models.loader_utils")+ accelerate_state = pytest.importorskip("accelerate.state")+ shared = accelerate_state.AcceleratorState._shared_state+ monkeypatch.delitem(shared, "fsdp_plugin", raising = False)+ monkeypatch.delitem(shared, "distributed_type", raising = False)++ assert loader_utils.fsdp_version_requested() == 0+ assert loader_utils.fsdp2_keeps_fused_lora() is False++ monkeypatch.setenv("ACCELERATE_USE_FSDP", "true")+ monkeypatch.setenv("FSDP_VERSION", "1")+ assert loader_utils.fsdp_will_wrap() is True+ assert loader_utils.fsdp2_keeps_fused_lora() is False++ monkeypatch.setenv("FSDP_VERSION", "2")+ assert loader_utils.fsdp2_requested() is True+ assert loader_utils.fsdp2_keeps_fused_lora() is True+ monkeypatch.setenv("UNSLOTH_FSDP2_FUSED_LORA", "0")+ assert loader_utils.fsdp2_keeps_fused_lora() is False+ monkeypatch.delenv("UNSLOTH_FSDP2_FUSED_LORA")++ # UNSLOTH_FSDP2=1 under torchrun, before any env was exported.+ monkeypatch.delenv("ACCELERATE_USE_FSDP")+ monkeypatch.delenv("FSDP_VERSION")+ monkeypatch.setenv("UNSLOTH_FSDP2", "1")+ monkeypatch.setenv("WORLD_SIZE", "2")+ assert loader_utils.fsdp2_requested() is True+++def test_a_built_accelerator_is_authoritative(monkeypatch):+ """An FSDP1 plugin wins over a stale FSDP_VERSION=2 in the env, and vice versa."""+ import types++ loader_utils = pytest.importorskip("unsloth.models.loader_utils")+ accelerate_state = pytest.importorskip("accelerate.state")+ shared = accelerate_state.AcceleratorState._shared_state+ monkeypatch.setenv("FSDP_VERSION", "2")+ monkeypatch.setitem(shared, "fsdp_plugin", types.SimpleNamespace(fsdp_version = 1))+ assert loader_utils.fsdp_version_requested() == 1+ assert loader_utils.fsdp2_keeps_fused_lora() is False+ monkeypatch.delenv("FSDP_VERSION")+ monkeypatch.setitem(shared, "fsdp_plugin", types.SimpleNamespace(fsdp_version = 2))+ assert loader_utils.fsdp2_requested() is True+++def _free_port():+ with socket.socket() as s:+ s.bind(("127.0.0.1", 0))+ return s.getsockname()[1]+++def _rank_worker(rank, port, out_dir):+ import json++ import torch.distributed as dist++ os.environ.update(+ RANK = str(rank),+ LOCAL_RANK = str(rank),+ WORLD_SIZE = "2",+ MASTER_ADDR = "127.0.0.1",+ MASTER_PORT = str(port),+ UNSLOTH_FSDP2 = "1",+ )+ launch = _load_launch()+ launch.apply_unsloth_fsdp2_launch_env()+ dist.init_process_group("gloo", rank = rank, world_size = 2)+ try:+ mine = (+ launch.fsdp_version_from_env(),+ os.environ.get("ACCELERATE_USE_FSDP"),+ os.environ.get("FSDP_AUTO_WRAP_POLICY"),+ )+ everyone = [None, None]+ dist.all_gather_object(everyone, mine)+ if rank == 0:+ with open(os.path.join(out_dir, "ranks.json"), "w") as f:+ json.dump(everyone, f)+ finally:+ dist.destroy_process_group()+++def test_every_rank_reads_the_same_fsdp2_launch(tmp_path):+ import json++ dist = pytest.importorskip("torch.distributed")+ if not dist.is_available() or not dist.is_gloo_available():+ pytest.skip("gloo backend unavailable")+ import torch.multiprocessing as mp++ mp.spawn(_rank_worker, args = (_free_port(), str(tmp_path)), nprocs = 2, join = True)+ ranks = json.loads((tmp_path / "ranks.json").read_text())+ assert ranks[0] == ranks[1] == [2, "true", "TRANSFORMER_BASED_WRAP"]+++def test_unsloth_init_installs_the_fsdp2_hooks_without_a_launcher_env():+ """`Accelerator(fsdp_plugin = FullyShardedDataParallelPlugin(fsdp_version = 2))` in a raw loop+ sets no env before unsloth is imported, so the install must not be gated on it."""+ import ast++ source = (_ROOT / "unsloth" / "__init__.py").read_text(encoding = "utf-8")+ gated = [+ node+ for node in ast.walk(ast.parse(source))+ if isinstance(node, ast.If)+ and "fsdp_version_from_env" in ast.unparse(node.test)+ and "_install_fsdp2_hooks" in ast.unparse(node)+ ]+ assert not gated, "install_fsdp2_hooks is gated on the launcher env"+ assert "_install_fsdp2_hooks()" in sourcediff --git a/tests/test_fsdp2_frozen_dtypes_gloo.py b/tests/test_fsdp2_frozen_dtypes_gloo.pynew file mode 100644index 00000000000..1408e22d4f7--- /dev/null+++ b/tests/test_fsdp2_frozen_dtypes_gloo.py@@ -0,0 +1,106 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""LoRA under FSDP2 puts float32 adapters next to frozen 16-bit weights in one unit. torch < 2.8+asserts one dtype over every parameter of a unit, frozen ones included; `unify_frozen_dtypes`+upcasts the frozen ones there. Two gloo ranks on CPU, real `fully_shard`, no accelerate needed.+"""++from __future__ import annotations++import importlib.util+import os+import pathlib+import socket+import types++import pytest++torch = pytest.importorskip("torch")+dist = pytest.importorskip("torch.distributed")+fsdp = pytest.importorskip("torch.distributed.fsdp")+if not hasattr(fsdp, "fully_shard") or not hasattr(fsdp, "MixedPrecisionPolicy"):+ pytest.skip("torch without the public FSDP2 API", allow_module_level = True)+if not dist.is_available() or not dist.is_gloo_available():+ pytest.skip("gloo backend unavailable", allow_module_level = True)++_FSDP2_PY = pathlib.Path(__file__).resolve().parents[1] / "unsloth" / "models" / "fsdp2.py"+++def _fsdp2_module():+ # By path, so this runs on torch versions the rest of unsloth's imports do not cover.+ spec = importlib.util.spec_from_file_location("_unsloth_fsdp2_under_test", _FSDP2_PY)+ module = importlib.util.module_from_spec(spec)+ spec.loader.exec_module(module)+ return module+++def _free_port():+ with socket.socket() as s:+ s.bind(("127.0.0.1", 0))+ return s.getsockname()[1]+++class _Block(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.base = torch.nn.Linear(16, 16, bias = False).to(torch.bfloat16).requires_grad_(False)+ self.lora_A = torch.nn.Linear(16, 4, bias = False)+ self.lora_B = torch.nn.Linear(4, 16, bias = False)++ def forward(self, x):+ return self.base(x) + self.lora_B(self.lora_A(x.to(self.lora_A.weight.dtype))).to(x.dtype)+++def _worker(rank, port, unify, out):+ os.environ.update(MASTER_ADDR = "127.0.0.1", MASTER_PORT = str(port))+ dist.init_process_group("gloo", rank = rank, world_size = 2)+ try:+ fsdp2 = _fsdp2_module()+ torch.manual_seed(0)+ model = torch.nn.Sequential(_Block(), _Block())+ policy = fsdp.MixedPrecisionPolicy(param_dtype = torch.bfloat16, reduce_dtype = torch.float32)+ plugin = types.SimpleNamespace(mixed_precision_policy = policy)+ acc = types.SimpleNamespace(+ state = types.SimpleNamespace(fsdp_plugin = plugin), mixed_precision = "bf16"+ )+ upcast = fsdp2.unify_frozen_dtypes(acc, model) if unify else False+ for block in model:+ fsdp.fully_shard(block, mp_policy = policy)+ fsdp.fully_shard(model, mp_policy = policy)+ try:+ model(torch.randn(2, 16, dtype = torch.bfloat16)).float().sum().backward()+ result = "ok"+ except AssertionError as e:+ result = str(e)+ if rank == 0:+ with open(out, "w") as f:+ f.write(f"{upcast}|{fsdp2._fsdp2_checks_frozen_dtypes()}|{result}")+ finally:+ dist.destroy_process_group()+++def _run(unify, tmp_path):+ import torch.multiprocessing as mp++ out = str(tmp_path / f"unify_{unify}.txt")+ mp.spawn(_worker, args = (_free_port(), unify, out), nprocs = 2, join = True)+ upcast, frozen_checked, result = open(out).read().split("|", 2)+ return upcast == "True", frozen_checked == "True", result+++def test_frozen_16bit_weights_next_to_float32_adapters_shard_and_train(tmp_path):+ upcast, frozen_checked, result = _run(True, tmp_path)+ assert result == "ok", result+ # Only torch < 2.8 needs (and gets) the upcast; newer torch keeps the frozen weights 16-bit.+ assert upcast is frozen_checked+ old_torch = tuple(int(x) for x in torch.__version__.split(".")[:2]) < (2, 8)+ assert frozen_checked is old_torch+++def test_without_the_upcast_old_torch_refuses_the_unit(tmp_path):+ _, frozen_checked, result = _run(False, tmp_path)+ if frozen_checked:+ assert "uniform original parameter dtype" in result+ else:+ assert result == "ok", resultdiff --git a/tests/test_fsdp2_peft_checkpoint_gloo.py b/tests/test_fsdp2_peft_checkpoint_gloo.pynew file mode 100644index 00000000000..b4613d0919a--- /dev/null+++ b/tests/test_fsdp2_peft_checkpoint_gloo.py@@ -0,0 +1,176 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""FSDP2 + PEFT FULL_STATE_DICT checkpoints (what Trainer writes for resuming), two gloo ranks on CPU.++accelerate < 1.15 builds the adapter-only full state dict with `get_peft_model_state_dict`, i.e.+each rank's DTensor shard, and rank 0 writes only its own; `install_fsdp2_peft_full_state_dict`+backports the 1.15 gather. The saved adapter must hold whole tensors equal to the unsharded ones,+and loading it back must restore every rank's shard.+"""++from __future__ import annotations++import os+import socket+import types++import pytest++torch = pytest.importorskip("torch")+dist = pytest.importorskip("torch.distributed")+pytest.importorskip("torch.distributed.fsdp")+pytest.importorskip("peft")+pytest.importorskip("accelerate")++if not dist.is_available() or not dist.is_gloo_available():+ pytest.skip("gloo backend unavailable", allow_module_level = True)+++def _free_port():+ with socket.socket() as s:+ s.bind(("127.0.0.1", 0))+ return s.getsockname()[1]+++def _build():+ from peft import LoraConfig, get_peft_model+ from transformers import LlamaConfig, LlamaForCausalLM++ cfg = LlamaConfig(+ hidden_size = 64,+ intermediate_size = 128,+ num_hidden_layers = 2,+ num_attention_heads = 4,+ num_key_value_heads = 2,+ vocab_size = 96,+ tie_word_embeddings = False,+ )+ torch.manual_seed(0)+ lora = LoraConfig(+ r = 4, lora_alpha = 8, target_modules = ["q_proj", "down_proj"], init_lora_weights = False+ )+ return get_peft_model(LlamaForCausalLM(cfg), lora)+++def _accelerator_less_import():+ # Spawned ranks skip conftest, whose CUDA spoof lets `import unsloth` run on CPU-only runners.+ if torch.cuda.is_available():+ return+ import importlib.util+ import pathlib++ path = pathlib.Path(__file__).resolve().parent / "conftest.py"+ spec = importlib.util.spec_from_file_location("_unsloth_tests_conftest", path)+ spec.loader.exec_module(importlib.util.module_from_spec(spec))+++def _worker(rank, port, out_dir, backport):+ _accelerator_less_import()+ os.environ.update(+ RANK = str(rank),+ LOCAL_RANK = str(rank),+ WORLD_SIZE = "2",+ MASTER_ADDR = "127.0.0.1",+ MASTER_PORT = str(port),+ )+ dist.init_process_group("gloo", rank = rank, world_size = 2)+ try:+ import json++ from torch.distributed.device_mesh import init_device_mesh+ from torch.distributed.tensor import DTensor++ from accelerate import PartialState+ from accelerate.utils import FullyShardedDataParallelPlugin+ from accelerate.utils import fsdp_utils+ from unsloth.models import fsdp2++ PartialState(cpu = True) # accelerate's logger refuses to run without it+ if backport:+ assert fsdp2.install_fsdp2_peft_full_state_dict()+ else:+ # `import unsloth` installed it already; undo that to reproduce the unpatched accelerate.+ for name in ("_get_model_state_dict", "_set_model_state_dict"):+ fn = getattr(fsdp_utils, name)+ setattr(fsdp_utils, name, getattr(fn, "__wrapped__", fn))+ result = {"accelerate": __import__("accelerate").__version__}+ mesh = init_device_mesh("cpu", (2,), mesh_dim_names = ("dp_shard",))+ plugin = FullyShardedDataParallelPlugin(+ fsdp_version = 2,+ auto_wrap_policy = "TRANSFORMER_BASED_WRAP",+ transformer_cls_names_to_wrap = ["LlamaDecoderLayer"],+ state_dict_type = "FULL_STATE_DICT",+ )+ accelerator = types.SimpleNamespace(+ state = types.SimpleNamespace(fsdp_plugin = plugin),+ torch_device_mesh = mesh,+ parallelism_config = types.SimpleNamespace(fsdp_dim_names = ("dp_shard",)),+ mixed_precision = "no",+ device = torch.device("cpu"),+ num_processes = 2,+ process_index = rank,+ is_main_process = rank == 0,+ is_fsdp2 = True,+ wait_for_everyone = dist.barrier,+ )+ model = _build()+ reference = {k: v.clone() for k, v in model.state_dict().items() if "lora_" in k}+ model = fsdp_utils.fsdp2_prepare_model(accelerator, model)+ fsdp_utils.save_fsdp_model(plugin, accelerator, model, out_dir, adapter_only = True)+ dist.barrier()++ if rank == 0:+ saved = torch.load(os.path.join(out_dir, "pytorch_model_fsdp.bin"), weights_only = False)+ # peft's keys drop the adapter name, get_model_state_dict's keep it.+ saved = {k.replace(".default", ""): v for k, v in saved.items() if "lora_" in k}+ whole = 0+ for name, ref in reference.items():+ got = saved.get(name.replace(".default", ""))+ if isinstance(got, DTensor):+ got = got.to_local()+ if (+ got is not None+ and tuple(got.shape) == tuple(ref.shape)+ and torch.equal(got.cpu(), ref)+ ):+ whole += 1+ result["lora_tensors"] = len(reference)+ result["saved_whole_and_equal"] = whole++ with torch.no_grad():+ for name, p in model.named_parameters():+ if "lora_" in name:+ p.zero_()+ fsdp_utils.load_fsdp_model(plugin, accelerator, model, out_dir, adapter_only = True)+ restored = 0+ for name, p in model.named_parameters():+ if "lora_" not in name:+ continue+ full = p.full_tensor() if isinstance(p, DTensor) else p+ restored += int(torch.equal(full.detach().cpu(), reference[name]))+ result["restored"] = restored+ if rank == 0:+ with open(os.path.join(out_dir, "result.json"), "w") as f:+ json.dump(result, f)+ finally:+ dist.destroy_process_group()+++def _run(tmp_path, backport):+ import json++ import torch.multiprocessing as mp++ mp.spawn(_worker, args = (_free_port(), str(tmp_path), backport), nprocs = 2, join = True)+ with open(tmp_path / "result.json") as f:+ return json.load(f)+++def test_full_state_dict_adapter_checkpoints_hold_every_rank_shard(tmp_path):+ r = _run(tmp_path, backport = True)+ assert r["lora_tensors"] == 8+ assert (+ r["saved_whole_and_equal"] == r["lora_tensors"]+ ), "the checkpoint holds only rank 0's shards"+ assert r["restored"] == r["lora_tensors"]diff --git a/tests/test_fsdp2_wrap_gloo.py b/tests/test_fsdp2_wrap_gloo.pynew file mode 100644index 00000000000..3c0564d68c9--- /dev/null+++ b/tests/test_fsdp2_wrap_gloo.py@@ -0,0 +1,312 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""FSDP2 wrap through accelerate's real `fsdp2_prepare_model`, two gloo ranks on CPU.++What Unsloth adds after accelerate shards the model (unsloth/models/fsdp2.py):+- a layer that reads `.weight` directly (the fused LoRA kernels) sees whole, unsharded weights+ inside a per-block unit's forward, so the fused path is kept there;+- a finer wrap (a LoRA submodule as its own unit) would hand that layer a DTensor shard, so the+ fused path is declined on every rank;+- lm_head read directly by the CausalLM forward (Unsloth's fused CE) is gathered first;+- the last decoder block stays gathered after forward;+- losses and gradients match an unsharded copy of the model.+"""++from __future__ import annotations++import os+import socket+import types++import pytest++torch = pytest.importorskip("torch")+dist = pytest.importorskip("torch.distributed")+pytest.importorskip("torch.distributed.fsdp")+pytest.importorskip("peft")+pytest.importorskip("accelerate")++if not dist.is_available() or not dist.is_gloo_available():+ pytest.skip("gloo backend unavailable", allow_module_level = True)++N_LAYERS = 3+++def _free_port():+ with socket.socket() as s:+ s.bind(("127.0.0.1", 0))+ return s.getsockname()[1]+++def _spy_mlp(self, X):+ """Stands in for a fused LoRA MLP: reads every weight directly, never calls the projections."""+ log = self._unsloth_spy_log++ def proj(p, x):+ base = p.base_layer.weight+ A = p.lora_A["default"].weight+ B = p.lora_B["default"].weight+ from torch.distributed.tensor import DTensor++ for w, full in ((base, (p.out_features, p.in_features)), (A, None), (B, None)):+ log.append((isinstance(w, DTensor), tuple(w.shape), full))+ return x @ base.t() + (x @ A.t()) @ B.t() * p.scaling["default"]++ gate = proj(self.gate_proj, X)+ up = proj(self.up_proj, X)+ return proj(self.down_proj, torch.nn.functional.silu(gate) * up)+++def _spy_causal_forward(self, input_ids, labels):+ """Unsloth's CausalLM forward reads lm_head.weight for the fused CE instead of calling it."""+ hidden = self.model(input_ids = input_ids, use_cache = False).last_hidden_state+ logits = hidden @ self.lm_head.weight.t()+ return torch.nn.functional.cross_entropy(+ logits[:, :-1].reshape(-1, logits.shape[-1]).float(), labels[:, 1:].reshape(-1)+ )+++def _build(seed):+ from peft import LoraConfig, get_peft_model+ from transformers import LlamaConfig, LlamaForCausalLM++ cfg = LlamaConfig(+ hidden_size = 64,+ intermediate_size = 128,+ num_hidden_layers = N_LAYERS,+ num_attention_heads = 4,+ num_key_value_heads = 2,+ vocab_size = 96,+ tie_word_embeddings = False,+ )+ torch.manual_seed(seed)+ base = LlamaForCausalLM(cfg)+ lora = LoraConfig(+ r = 4,+ lora_alpha = 8,+ target_modules = [+ "q_proj",+ "k_proj",+ "v_proj",+ "o_proj",+ "gate_proj",+ "up_proj",+ "down_proj",+ ],+ lora_dropout = 0.0,+ init_lora_weights = False,+ )+ model = get_peft_model(base, lora)+ log = []+ for layer in model.base_model.model.model.layers:+ layer.mlp._unsloth_spy_log = log+ layer.mlp.forward = types.MethodType(_spy_mlp, layer.mlp)+ causal = model.base_model.model+ causal.forward = types.MethodType(_spy_causal_forward, causal)+ return model, log+++def _fake_accelerator(rank, mesh, plugin):+ return types.SimpleNamespace(+ state = types.SimpleNamespace(fsdp_plugin = plugin),+ torch_device_mesh = mesh,+ parallelism_config = types.SimpleNamespace(fsdp_dim_names = ("dp_shard",)),+ mixed_precision = "no",+ device = torch.device("cpu"),+ is_main_process = rank == 0,+ )+++def _accelerator_less_import():+ # Spawned ranks skip conftest, whose CUDA spoof lets `import unsloth` run on CPU-only runners.+ if torch.cuda.is_available():+ return+ import importlib.util+ import pathlib++ path = pathlib.Path(__file__).resolve().parent / "conftest.py"+ spec = importlib.util.spec_from_file_location("_unsloth_tests_conftest", path)+ spec.loader.exec_module(importlib.util.module_from_spec(spec))+++def _worker(rank, port, case, out_dir):+ _accelerator_less_import()+ os.environ.update(+ RANK = str(rank),+ LOCAL_RANK = str(rank),+ WORLD_SIZE = "2",+ MASTER_ADDR = "127.0.0.1",+ MASTER_PORT = str(port),+ )+ for key in (+ "UNSLOTH_FSDP2",+ "FSDP_VERSION",+ "ACCELERATE_USE_FSDP",+ "UNSLOTH_FORCE_FUSED_LORA",+ "UNSLOTH_FSDP2_FUSED_LORA",+ ):+ os.environ.pop(key, None)+ if case == "opt_out":+ os.environ["UNSLOTH_FSDP2_FUSED_LORA"] = "0"+ import json++ dist.init_process_group("gloo", rank = rank, world_size = 2)+ try:+ from torch.distributed.device_mesh import init_device_mesh+ from torch.distributed.tensor import DTensor++ import accelerate.accelerator as accelerator_module+ from accelerate.utils import FullyShardedDataParallelPlugin+ from unsloth.models import fsdp2, llama++ llama._FUSED_LORA_MLPS = llama._FUSED_LORA_MLPS + (_spy_mlp,)+ mesh = init_device_mesh("cpu", (2,), mesh_dim_names = ("dp_shard",))+ wrap_names = ["LlamaDecoderLayer"] + (["Linear"] if case == "finer" else [])+ plugin = FullyShardedDataParallelPlugin(+ fsdp_version = 2,+ auto_wrap_policy = "TRANSFORMER_BASED_WRAP",+ transformer_cls_names_to_wrap = wrap_names,+ reshard_after_forward = True,+ activation_checkpointing = case == "checkpointed",+ )+ result = {}+ result["agree_mixed"] = fsdp2._all_ranks_agree(rank == 0, torch.device("cpu"))++ model, log = _build(seed = 0)+ reference, _ = _build(seed = 0)+ assert fsdp2.install_fsdp2_prepare_hook()+ accelerator = _fake_accelerator(rank, mesh, plugin)+ if case == "checkpointed":+ # What Accelerator._prepare_fsdp2 does before sharding when the plugin asks for it.+ from accelerate.utils.fsdp_utils import fsdp2_apply_ac+ model = fsdp2_apply_ac(accelerator, model)+ if case == "causal_unit":+ # What accelerate <= 1.13 builds for transformer_cls_names_to_wrap = ["LlamaForCausalLM"]+ # (1.14+ refuses that wrap).+ from torch.distributed.fsdp import fully_shard++ fully_shard(model.base_model.model, mesh = mesh)+ fully_shard(model, mesh = mesh)+ fsdp2.after_fsdp2_wrap(accelerator, model)+ else:+ model = accelerator_module.fsdp2_prepare_model(accelerator, model)+ result["fused_kept"] = int(getattr(model, "_unsloth_fsdp2_fused_lora", -1))+ result["blocks"] = int(getattr(model, "_unsloth_fsdp2_blocks", -1))+ result["direct_read_units"] = int(getattr(model, "_unsloth_fsdp2_direct_read_units", -1))+ layers = model.base_model.model.model.layers+ result["spy_left"] = sum("forward" in layer.mlp.__dict__ for layer in layers)+ if case == "causal_unit":+ if rank == 0:+ with open(os.path.join(out_dir, f"{case}.json"), "w") as f:+ json.dump(result, f)+ return++ torch.manual_seed(1)+ ids = torch.randint(0, 96, (4, 12))+ local = ids[rank * 2 : rank * 2 + 2]+ loss = model(input_ids = local, labels = local)+ result["last_block_gathered"] = not isinstance(+ layers[-1].mlp.gate_proj.base_layer.weight, DTensor+ )+ result["first_block_gathered"] = not isinstance(+ layers[0].mlp.gate_proj.base_layer.weight, DTensor+ )+ loss.backward()+ result["spy_calls"] = len(log)+ result["spy_saw_shard"] = sum(1 for is_dt, _, _ in log if is_dt)+ result["spy_full_shapes"] = all(full is None or shape == full for _, shape, full in log)++ ref_loss = reference(input_ids = ids, labels = ids)+ ref_loss.backward()+ local_loss = torch.tensor([loss.item()])+ dist.all_reduce(local_loss)+ result["loss_diff"] = abs(local_loss.item() / 2 - ref_loss.item())+ ref_params = dict(reference.named_parameters())+ worst = 0.0+ for name, p in model.named_parameters():+ if p.grad is None:+ continue+ g = p.grad.full_tensor() if isinstance(p.grad, DTensor) else p.grad+ # FSDP averages over the 2 ranks; each rank's loss is the mean over its half.+ r = ref_params[name.replace("._checkpoint_wrapped_module", "")].grad+ worst = max(worst, (g - r).abs().max().item() / (r.abs().max().item() + 1e-12))+ result["grad_rel_err"] = worst+ if rank == 0:+ with open(os.path.join(out_dir, f"{case}.json"), "w") as f:+ json.dump(result, f)+ finally:+ dist.destroy_process_group()+++def _run(case, tmp_path):+ import json++ import torch.multiprocessing as mp++ mp.spawn(_worker, args = (_free_port(), case, str(tmp_path)), nprocs = 2, join = True)+ with open(tmp_path / f"{case}.json") as f:+ return json.load(f)+++def test_per_block_wrap_keeps_fused_layers_on_whole_weights(tmp_path):+ r = _run("per_block", tmp_path)+ assert r["agree_mixed"] is False, "ranks must agree on the strictest answer"+ assert r["blocks"] == N_LAYERS+ assert r["fused_kept"] == N_LAYERS+ assert r["spy_left"] == N_LAYERS+ # 3 MLPs x 3 projections x (base, A, B), forward only.+ assert r["spy_calls"] == N_LAYERS * 9+ assert r["spy_saw_shard"] == 0, "a direct .weight read saw a DTensor shard inside a block"+ assert r["spy_full_shapes"]+ import accelerate.utils.fsdp_utils as fsdp_utils++ if hasattr(fsdp_utils, "_find_final_norm"):+ # accelerate >= 1.14 gives [final norm, lm_head] their own unit, which nothing would gather.+ assert (+ r["direct_read_units"] >= 1+ ), "lm_head's unit is not gathered before the CausalLM forward"+ else:+ # Older accelerate leaves lm_head in the root unit, which its own pre-forward gathers.+ assert r["direct_read_units"] == 0+ assert r["last_block_gathered"] is True+ assert r["first_block_gathered"] is False+ assert r["loss_diff"] < 1e-5+ assert r["grad_rel_err"] < 1e-4+++def test_a_finer_wrap_declines_the_fused_layers(tmp_path):+ r = _run("finer", tmp_path)+ assert r["fused_kept"] == 0+ assert r["spy_left"] == 0, "the fused forwards must be removed on every rank"+ assert r["spy_calls"] == 0+ assert r["loss_diff"] < 1e-5+ assert r["grad_rel_err"] < 1e-4+++def test_a_unit_above_the_blocks_declines_the_fused_layers(tmp_path):+ """peft's LoraModel calls the CausalLM's .forward directly, so a CausalLM unit's gather can be skipped."""+ r = _run("causal_unit", tmp_path)+ assert r["blocks"] == 0+ assert r["fused_kept"] == 0+ assert r["spy_left"] == 0+++def test_the_fused_lora_opt_out_is_honoured_after_wrapping(tmp_path):+ """A raw Accelerator loop patches the model before any FSDP plugin exists."""+ r = _run("opt_out", tmp_path)+ assert r["fused_kept"] == 0+ assert r["spy_left"] == 0+ assert r["spy_calls"] == 0+ assert r["loss_diff"] < 1e-5+++def test_activation_checkpointed_blocks_keep_the_last_block_gathered(tmp_path):+ r = _run("checkpointed", tmp_path)+ assert r["blocks"] == N_LAYERS+ assert r["fused_kept"] == N_LAYERS+ assert r["last_block_gathered"] is True+ assert r["first_block_gathered"] is False+ assert r["spy_saw_shard"] == 0+ assert r["loss_diff"] < 1e-5+ assert r["grad_rel_err"] < 1e-4diff --git a/unsloth/__init__.py b/unsloth/__init__.pyindex 28b61594eb8..3d4cedda2a9 100644--- a/unsloth/__init__.py+++ b/unsloth/__init__.py@@ -18,6 +18,13 @@ os.environ["UNSLOTH_IS_PRESENT"] = "1"# an explicit override, including "0".os.environ.setdefault("TORCH_ROCM_AOTRITON_ENABLE_EXPERIMENTAL", "1")+# UNSLOTH_FSDP2=1 under plain torchrun: export accelerate's FSDP2 launch env before transformers or+# accelerate read it. A conflicting launch (DeepSpeed, FSDP_VERSION=1) raises here, on every rank.+from .fsdp2_launch import apply_unsloth_fsdp2_launch_env as _apply_fsdp2_env++_apply_fsdp2_env()+del _apply_fsdp2_env+# Before transformers, which reads sentencepiece availability during its own import. On Windows# the extension is never imported at all: a code integrity policy can refuse it by reputation,# and any probe to find out whether this machine will is itself the refusal the user sees. See@@ -1633,6 +1640,15 @@ else:from ._gpu_init import *from ._gpu_init import __version__+ # Raw `Accelerator.prepare()` loops never reach the Trainer init that also installs these, and+ # `Accelerator(fsdp_plugin = ...)` leaves no launcher env to gate on. Inert unless FSDP2 wraps.+ try:+ from .models.fsdp2 import install_fsdp2_hooks as _install_fsdp2_hooks+ _install_fsdp2_hooks()+ del _install_fsdp2_hooks+ except Exception:+ pass+def get_gpu_memory_stats():"""Return CUDA/ROCm/XPU/NPU device stats, peak memory, and total memory in GiB."""try:diff --git a/unsloth/fsdp2_launch.py b/unsloth/fsdp2_launch.pynew file mode 100644index 00000000000..4265d167b27--- /dev/null+++ b/unsloth/fsdp2_launch.py@@ -0,0 +1,98 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""FSDP launch detection from the launcher env only (no torch): identical on every rank."""++import importlib.util+import os++__all__ = [+ "unsloth_fsdp2_requested",+ "fsdp_version_from_env",+ "apply_unsloth_fsdp2_launch_env",+]++_TRUTHY = ("1", "true", "yes", "on")+_FALSY = ("0", "false", "no", "off")+++def _env_flag(name):+ return str(os.environ.get(name, "")).strip().lower() in _TRUTHY+++def _env_world_size():+ try:+ return int(os.environ.get("WORLD_SIZE", "1"))+ except ValueError:+ return 1+++def unsloth_fsdp2_requested():+ return _env_flag("UNSLOTH_FSDP2")+++def fsdp_version_from_env():+ """0 for no FSDP, else 1 or 2. accelerate builds FSDP1 when ACCELERATE_USE_FSDP names no+ version; FSDP_VERSION="0" means "not FSDP"."""+ if unsloth_fsdp2_requested() and _env_world_size() > 1:+ return 2+ version = str(os.environ.get("FSDP_VERSION", "")).strip()+ use_fsdp = _env_flag("ACCELERATE_USE_FSDP")+ if version in ("", "0"):+ return 1 if use_fsdp else 0+ try:+ return int(version)+ except ValueError:+ return 1 if use_fsdp else 0+++def _accelerate_fsdp2_support():+ """None when accelerate is not found, else whether it has FSDP2 (1.6) and FSDP2 state dict+ options (1.7; 1.6 saves FULL_STATE_DICT checkpoints as each rank's DTensor shard). Read from+ the source: importing accelerate here would import torch."""+ try:+ spec = importlib.util.find_spec("accelerate")+ except (ImportError, ValueError):+ return None+ for location in getattr(spec, "submodule_search_locations", None) or ():+ try:+ with open(os.path.join(location, "utils", "fsdp_utils.py"), encoding = "utf-8") as f:+ source = f.read()+ except OSError:+ continue+ return "def fsdp2_prepare_model" in source and "def _prepare_sd_options" in source+ return None+++def apply_unsloth_fsdp2_launch_env():+ """UNSLOTH_FSDP2=1 with WORLD_SIZE > 1: export what `accelerate launch` exports for an FSDP2+ config (explicit FSDP_* values win). Must run before transformers / accelerate read the env."""+ if not unsloth_fsdp2_requested() or _env_world_size() <= 1:+ return False+ if _env_flag("ACCELERATE_USE_DEEPSPEED"):+ raise RuntimeError(+ "Unsloth: UNSLOTH_FSDP2=1 was set on a DeepSpeed launch. Unset one of them."+ )+ version = str(os.environ.get("FSDP_VERSION", "")).strip()+ if version not in ("", "2"):+ raise ValueError(+ f"Unsloth: UNSLOTH_FSDP2=1 asks for FSDP2, but FSDP_VERSION={version}. Unset one of them."+ )+ if _accelerate_fsdp2_support() is False:+ # Older accelerate ignores FSDP_VERSION and builds FSDP1 from these variables.+ raise RuntimeError(+ "Unsloth: UNSLOTH_FSDP2=1 needs accelerate >= 1.7, which this accelerate is older than. "+ "Upgrade it with `pip install -U accelerate`."+ )+ os.environ["ACCELERATE_USE_FSDP"] = "true"+ os.environ["FSDP_VERSION"] = "2"+ # accelerate's default is the root only.+ os.environ.setdefault("FSDP_AUTO_WRAP_POLICY", "TRANSFORMER_BASED_WRAP")+ reshard = os.environ.get("UNSLOTH_FSDP2_RESHARD_AFTER_FORWARD", None)+ if reshard is not None and reshard.strip() != "":+ os.environ.setdefault(+ "FSDP_RESHARD_AFTER_FORWARD",+ "false" if reshard.strip().lower() in _FALSY else "true",+ )+ os.environ.setdefault("FSDP_STATE_DICT_TYPE", "FULL_STATE_DICT")+ return Truediff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.pyindex 93656d1695c..6c9e9f3040d 100644--- a/unsloth/models/_utils.py+++ b/unsloth/models/_utils.py@@ -5535,6 +5535,11 @@ def patch_gradient_accumulation_fix(Trainer):apply_accepts_loss_kwargs_fix(model)except Exception:pass+ try:+ from .fsdp2 import install_fsdp2_policy_provenance+ install_fsdp2_policy_provenance()+ except Exception:+ pass_original_trainer_init(self, *args, **kwargs)try:accelerator = getattr(self, "accelerator", None)@@ -5552,8 +5557,13 @@ def patch_gradient_accumulation_fix(Trainer):except Exception:passif getattr(self, "is_fsdp_enabled", False):- from .llama import _decline_fused_lora_for_fsdp- _decline_fused_lora_for_fsdp(getattr(self, "model", None))+ # FSDP2 keeps the fused LoRA kernels when each decoder block is its own unit (checked+ # again once accelerate has wrapped); FSDP1 and a root-only wrap decline them here.+ from .fsdp2 import fsdp2_trainer_setup+ keep_fused, reason = fsdp2_trainer_setup(self)+ if not keep_fused:+ from .llama import _decline_fused_lora_for_fsdp+ _decline_fused_lora_for_fsdp(getattr(self, "model", None), reason)_replan_auto_offload_safely(self)_block_swap_for_trainer_backend(self)_enable_ddp_batched_grad_copy(self)diff --git a/unsloth/models/fsdp2.py b/unsloth/models/fsdp2.pynew file mode 100644index 00000000000..ab91d92ee17--- /dev/null+++ b/unsloth/models/fsdp2.py@@ -0,0 +1,557 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""What Unsloth adds around accelerate's `fsdp2_prepare_model` (which already wraps per block,+keeps the [norm, lm_head] tail gathered, groups tied embeddings). `UNSLOTH_FSDP2_ACCELERATE_DEFAULTS=1`+skips the mixed precision fill and the last-block setting.+"""++import contextlib+import dataclasses+import functools+import inspect+import os++import torch++__all__ = [+ "fsdp2_trainer_setup",+ "install_fsdp2_hooks",+ "install_fsdp2_policy_provenance",+ "install_fsdp2_prepare_hook",+ "install_fsdp2_no_sync",+ "install_fsdp2_peft_full_state_dict",+]++_PATCHED = "_unsloth_fsdp2_patched"+_FALSY = ("0", "false", "no", "off")+_TRUTHY = ("1", "true", "yes", "on")+# Unsloth's backbone forwards call fast_rms_layernorm on these instead of the module.+_DIRECT_READ_NORMS = ("norm", "final_layernorm")+++def _fsdp_module_class():+ try:+ from torch.distributed.fsdp import FSDPModule+ return FSDPModule+ except ImportError:+ pass+ try:+ from torch.distributed._composable.fsdp import FSDPModule+ return FSDPModule+ except ImportError:+ return None+++def _rank0():+ try:+ import torch.distributed as dist+ return not (dist.is_available() and dist.is_initialized()) or dist.get_rank() == 0+ except Exception:+ return True+++def _print_rank0(message):+ if _rank0():+ print(message)+++def _keep_accelerate_defaults():+ return os.environ.get("UNSLOTH_FSDP2_ACCELERATE_DEFAULTS", "0").strip().lower() in _TRUTHY+++def _compute_dtype(accelerator):+ return {"fp16": torch.float16, "bf16": torch.bfloat16}.get(+ str(getattr(accelerator, "mixed_precision", "no"))+ )+++_AUTO_POLICY = "_unsloth_auto_mixed_precision_policy"+_POLICY_SEEN = "_unsloth_mixed_precision_policy_seen"+++def install_fsdp2_policy_provenance():+ """Tag the policy accelerate builds from `mixed_precision` itself, so a user-written policy with+ the same dtypes is never mistaken for it. Must run before the Accelerator is created."""+ try:+ from accelerate.utils import FullyShardedDataParallelPlugin+ except Exception:+ return False+ original = getattr(FullyShardedDataParallelPlugin, "set_mixed_precision", None)+ if original is None:+ return False+ if getattr(original, _PATCHED, False):+ return True++ @functools.wraps(original)+ def set_mixed_precision(self, mixed_precision, *args, **kwargs):+ before = getattr(self, "mixed_precision_policy", None)+ result = original(self, mixed_precision, *args, **kwargs)+ override = kwargs.get("override", args[1] if len(args) > 1 else False)+ after = getattr(self, "mixed_precision_policy", None)+ setattr(self, _POLICY_SEEN, True)+ if before is None and not override and after is not None:+ setattr(self, _AUTO_POLICY, after)+ return result++ setattr(set_mixed_precision, _PATCHED, True)+ FullyShardedDataParallelPlugin.set_mixed_precision = set_mixed_precision+ return True+++def fill_fsdp2_mixed_precision(accelerator):+ """Only rewrites accelerate's automatic policy (every dtype = compute dtype): it casts the loss+ to float16, where the 65536 loss scale overflows and the scaler skips the first steps."""+ plugin = getattr(getattr(accelerator, "state", None), "fsdp_plugin", None)+ policy = getattr(plugin, "mixed_precision_policy", None)+ if policy is None or _keep_accelerate_defaults():+ return policy+ automatic = getattr(plugin, _AUTO_POLICY, None) is policy+ if not automatic and getattr(plugin, _POLICY_SEEN, False):+ return policy+ try:+ from torch.distributed.fsdp import MixedPrecisionPolicy+ except ImportError:+ return policy+ compute = _compute_dtype(accelerator)+ if compute is None or not isinstance(policy, MixedPrecisionPolicy):+ return policy+ if (+ policy.param_dtype != compute+ or policy.reduce_dtype != compute+ or getattr(policy, "output_dtype", None) not in (None, compute)+ ):+ return policy+ if not automatic:+ # The Accelerator was built before Unsloth was imported, so whether accelerate or the caller+ # wrote this policy is unknown. Only float16 overflows with it.+ if compute != torch.float16:+ return policy+ raise RuntimeError(+ "Unsloth: this FSDP2 Accelerator was created before `import unsloth`, so Unsloth cannot "+ "tell whether its float16 mixed precision policy (float16 gathers, reductions and "+ "outputs, which overflow at the GradScaler's initial loss scale) was chosen by "+ "accelerate or by you. Import unsloth before creating the Accelerator, or set "+ "UNSLOTH_FSDP2_ACCELERATE_DEFAULTS=1 to keep the policy as it is."+ )+ changes = {"reduce_dtype": torch.float32}+ if compute == torch.float16:+ # float16 gathers move the embedding / norm / lm_head backward into float16, where scaled+ # gradients overflow; gather the float32 masters and let autocast cast like one GPU does.+ changes["param_dtype"] = None+ if hasattr(policy, "output_dtype"):+ changes["output_dtype"] = None+ plugin.mixed_precision_policy = dataclasses.replace(policy, **changes)+ return plugin.mixed_precision_policy+++def unify_trainable_dtypes(accelerator, model):+ """16-bit full finetuning runs without autocast with float32 norms, which accelerate does not+ upcast and FSDP2 refuses ("FSDP expects uniform original parameter dtype"). A policy the caller+ set is kept; only the master weights are upcast."""+ trainable = [p for p in model.parameters() if p.requires_grad and p.is_floating_point()]+ dtypes = {p.dtype for p in trainable}+ if str(getattr(accelerator, "mixed_precision", "no")) != "no":+ # accelerate upcasts these itself, except 1.13 (`param = param.to(float32)` drops the result).+ if len(dtypes) <= 1:+ return False+ for p in trainable:+ if p.dtype != torch.float32:+ p.data = p.data.to(torch.float32)+ return True+ half = torch.bfloat16 if torch.bfloat16 in dtypes else torch.float16+ if len(dtypes) <= 1 or half not in dtypes:+ return False+ try:+ from torch.distributed.fsdp import MixedPrecisionPolicy+ except ImportError:+ return False+ plugin = getattr(getattr(accelerator, "state", None), "fsdp_plugin", None)+ if plugin is None:+ return False+ for p in trainable:+ if p.dtype != torch.float32:+ p.data = p.data.to(torch.float32)+ if getattr(plugin, "mixed_precision_policy", None) is None:+ plugin.mixed_precision_policy = MixedPrecisionPolicy(+ param_dtype = half, reduce_dtype = torch.float32+ )+ how = f"gathering them in {str(half).replace('torch.', '')} for compute"+ else:+ how = "gathering them as the given mixed_precision_policy says"+ _print_rank0(+ "Unsloth: FSDP2 needs one dtype per unit, and the trainable weights mix "+ f"{sorted(str(d).replace('torch.', '') for d in dtypes)}. Keeping float32 master weights and {how}."+ )+ return True+++def _fsdp2_checks_frozen_dtypes():+ """torch < 2.8 asserts one original dtype over every parameter of a unit, frozen ones included+ (`FSDPParamGroup._init_mp_dtypes`); 2.8 checks only the trainable ones."""+ try:+ from torch.distributed.fsdp._fully_shard._fsdp_param_group import FSDPParamGroup+ return "requires_grad" not in inspect.getsource(FSDPParamGroup._init_mp_dtypes)+ except Exception:+ pass+ try:+ major, minor = (int(x) for x in torch.__version__.split(".")[:2])+ return (major, minor) < (2, 8)+ except Exception:+ return False+++def unify_frozen_dtypes(+ accelerator,+ model,+ frozen_checked = None,+):+ """LoRA on torch < 2.8: float32 adapters next to frozen 16-bit weights in one unit. Upcast the+ frozen weights too (what accelerate <= 1.12 did with `model.to(float32)`); the gather policy+ casts them back down for compute."""+ if frozen_checked is None:+ frozen_checked = _fsdp2_checks_frozen_dtypes()+ if not frozen_checked:+ return False+ dtypes = {p.dtype for p in model.parameters() if p.is_floating_point()}+ if len(dtypes) <= 1:+ return False+ plugin = getattr(getattr(accelerator, "state", None), "fsdp_plugin", None)+ if plugin is None:+ return False+ if (+ getattr(plugin, "mixed_precision_policy", None) is None+ and str(getattr(accelerator, "mixed_precision", "no")) == "no"+ ):+ raise RuntimeError(+ f"Unsloth: FSDP2 on torch {torch.__version__} needs every parameter of a unit to share "+ f"one dtype, but this model mixes {sorted(str(d) for d in dtypes)} and no mixed "+ "precision is set to compute in 16-bit from float32 weights. Enable bf16 / fp16 mixed "+ "precision, or use torch >= 2.8."+ )+ n = 0+ for p in model.parameters():+ # Quantized parameter subclasses keep their storage dtype.+ if type(p) is torch.nn.Parameter and p.is_floating_point() and p.dtype != torch.float32:+ p.data = p.data.to(torch.float32)+ n += 1+ if n:+ _print_rank0(+ f"Unsloth: FSDP2 on torch {torch.__version__} needs one dtype per unit, frozen weights "+ f"included, so {n} 16-bit parameters are kept in float32 and gathered in the compute dtype."+ )+ return n > 0+++def _parents(model):+ parents = {}+ for parent in model.modules():+ for child in parent.children():+ parents.setdefault(id(child), parent)+ return parents+++def _fsdp_owner(module, parents, FSDPModule):+ """The FSDP unit whose pre-forward gathers `module`'s own parameters."""+ while module is not None:+ if isinstance(module, FSDPModule):+ return module+ module = parents.get(id(module))+ return None+++def _decoder_blocks(model):+ names = getattr(model, "_no_split_modules", None) or ()+ names = set(names)+ if not names:+ return []+ # fully_shard swaps a module's class for an FSDP<Name> subclass, so match along the MRO.+ return [m for m in model.modules() if any(c.__name__ in names for c in type(m).__mro__)]+++def _all_ranks_agree(value, device):+ """True only when every rank says True, so every rank takes the same branch."""+ try:+ import torch.distributed as dist++ if not (dist.is_available() and dist.is_initialized()):+ return value+ flag = torch.tensor([1 if value else 0], dtype = torch.int32, device = device)+ dist.all_reduce(flag, op = dist.ReduceOp.MIN)+ return bool(flag.item())+ except Exception:+ return value+++def _in_decoder_block(module, block_ids, parents):+ while module is not None:+ parent = parents.get(id(module))+ if id(module) in block_ids or isinstance(parent, torch.nn.ModuleList):+ return True+ module = parent+ return False+++def _fused_lora_wrap_problem(+ model,+ fused,+ parents,+ FSDPModule,+ blocks = (),+):+ """None when every fused layer's weights are gathered by a per-block unit, else why not."""+ block_ids = {id(b) for b in blocks}+ for module in fused:+ owner = _fsdp_owner(module, parents, FSDPModule)+ if owner is None or owner is model:+ return "FSDP2 wraps only the root module, not each decoder block"+ if not _in_decoder_block(owner, block_ids, parents):+ # Blocks are always entered through __call__; a unit above them need not be (peft's+ # LoraModel calls the CausalLM's .forward), so its pre-forward gather can be skipped.+ return "FSDP2 wraps a module above the decoder blocks, not each block"+ for sub in module.modules():+ if sub is not module and isinstance(sub, FSDPModule):+ return "FSDP2 made a submodule of a fused LoRA layer its own unit"+ return None+++def _gate_fused_lora(+ accelerator,+ model,+ parents,+ FSDPModule,+ blocks = None,+):+ from .llama import _decline_fused_lora_for_fsdp, _fused_lora_modules++ fused = _fused_lora_modules(model)+ if not fused or os.environ.get("UNSLOTH_FORCE_FUSED_LORA", "0") == "1":+ model._unsloth_fsdp2_fused_lora = len(fused)+ return len(fused)+ if os.environ.get("UNSLOTH_FSDP2_FUSED_LORA", "1").strip().lower() in _FALSY:+ # A raw loop patches the model before any FSDP plugin exists, so load time could not see it.+ problem = "UNSLOTH_FSDP2_FUSED_LORA=0"+ else:+ if blocks is None:+ blocks = _decoder_blocks(model)+ problem = _fused_lora_wrap_problem(model, fused, parents, FSDPModule, blocks)+ device = getattr(accelerator, "device", None) or torch.device("cpu")+ if not _all_ranks_agree(problem is None, device):+ _decline_fused_lora_for_fsdp(model, problem or "another rank's FSDP2 wrap")+ model._unsloth_fsdp2_fused_lora = 0+ return 0+ model._unsloth_fsdp2_fused_lora = len(fused)+ _print_rank0(+ f"Unsloth: FSDP2 shards each decoder block as its own unit, so {len(fused)} fused LoRA "+ "layers stay on (their weights are gathered before the block's forward)."+ )+ return len(fused)+++def _gather_units_read_directly(model, blocks, parents, FSDPModule):+ """Unsloth reads lm_head / final norm weights directly (fused CE, fast RMSNorm), so FSDP's+ pre-forward never gathers their unit (accelerate >= 1.14 splits it off)."""+ if not hasattr(FSDPModule, "unshard"):+ return 0+ targets = []+ get_output = getattr(model, "get_output_embeddings", None)+ output = get_output() if callable(get_output) else None+ if isinstance(output, torch.nn.Module):+ targets.append(output)+ for name, module in model.named_modules():+ if name.rsplit(".", 1)[-1] in _DIRECT_READ_NORMS:+ parent = parents.get(id(module))+ if parent is not None and isinstance(+ getattr(parent, "layers", None), torch.nn.ModuleList+ ):+ targets.append(module)+ block_ids = {id(b) for b in blocks}+ units = []+ for target in targets:+ owner = _fsdp_owner(target, parents, FSDPModule)+ if owner is None or owner is model or id(owner) in block_ids:+ continue+ if all(owner is not u for u in units):+ units.append(owner)+ if not units:+ return 0++ # On the root: peft's LoraModel calls the CausalLM's .forward directly, skipping its hooks.+ def _unsloth_gather_direct_reads(module, args):+ for unit in units:+ unit.unshard()++ model.register_forward_pre_hook(_unsloth_gather_direct_reads)+ return len(units)+++def after_fsdp2_wrap(accelerator, model):+ FSDPModule = _fsdp_module_class()+ if FSDPModule is None or not isinstance(model, FSDPModule):+ return+ if getattr(model, "_unsloth_fsdp2_prepared", False):+ return+ model._unsloth_fsdp2_prepared = True+ parents = _parents(model)+ blocks = [b for b in _decoder_blocks(model) if isinstance(b, FSDPModule)]+ plugin = getattr(getattr(accelerator, "state", None), "fsdp_plugin", None)+ if (+ blocks+ and not _keep_accelerate_defaults()+ and getattr(plugin, "reshard_after_forward", True) is True+ and hasattr(blocks[-1], "set_reshard_after_forward")+ ):+ # Backward starts with the last block, so resharding it after forward only re-gathers it.+ blocks[-1].set_reshard_after_forward(False, recurse = False)+ model._unsloth_fsdp2_blocks = len(blocks)+ model._unsloth_fsdp2_direct_read_units = _gather_units_read_directly(+ model, blocks, parents, FSDPModule+ )+ _gate_fused_lora(accelerator, model, parents, FSDPModule, blocks)+++def install_fsdp2_prepare_hook():+ """Wrap accelerate's `fsdp2_prepare_model` (looked up by `Accelerator._prepare_fsdp2`)."""+ try:+ import accelerate.accelerator as accelerator_module+ except Exception:+ return False+ original = getattr(accelerator_module, "fsdp2_prepare_model", None)+ if original is None:+ return False+ if getattr(original, _PATCHED, False):+ return True++ @functools.wraps(original)+ def fsdp2_prepare_model(accelerator, model, *args, **kwargs):+ fill_fsdp2_mixed_precision(accelerator)+ unify_trainable_dtypes(accelerator, model)+ unify_frozen_dtypes(accelerator, model)+ model = original(accelerator, model, *args, **kwargs)+ after_fsdp2_wrap(accelerator, model)+ return model++ setattr(fsdp2_prepare_model, _PATCHED, True)+ accelerator_module.fsdp2_prepare_model = fsdp2_prepare_model+ return True+++def install_fsdp2_no_sync():+ """accelerate < 1.11 looks for `model.no_sync`, which FSDP2 modules lack."""+ try:+ from accelerate import Accelerator+ except Exception:+ return False+ original = Accelerator.no_sync+ if getattr(original, _PATCHED, False):+ return True+ try:+ source = inspect.getsource(original)+ except (OSError, TypeError):+ return False+ if "set_requires_gradient_sync" in source:+ return True++ @contextlib.contextmanager+ def no_sync(self, model):+ if getattr(self, "is_fsdp2", False) and hasattr(model, "set_requires_gradient_sync"):+ model.set_requires_gradient_sync(False)+ try:+ yield+ finally:+ model.set_requires_gradient_sync(True)+ else:+ with original(self, model):+ yield++ no_sync.__doc__ = original.__doc__+ setattr(no_sync, _PATCHED, True)+ Accelerator.no_sync = no_sync+ return True+++def install_fsdp2_peft_full_state_dict():+ """accelerate < 1.15 builds an FSDP2 PEFT model's FULL_STATE_DICT checkpoint (what Trainer writes for+ resuming) from `get_peft_model_state_dict`, i.e. each rank's DTensor shard, and rank 0 saves only+ its own. Backport of the 1.15 fix: gather the trainable weights with `get_model_state_dict`."""+ try:+ import accelerate.utils.fsdp_utils as fsdp_utils+ except Exception:+ return False+ original_get = getattr(fsdp_utils, "_get_model_state_dict", None)+ original_set = getattr(fsdp_utils, "_set_model_state_dict", None)+ is_peft_model = getattr(fsdp_utils, "is_peft_model", None)+ if original_get is None or original_set is None or is_peft_model is None:+ return False+ if getattr(original_get, _PATCHED, False):+ return True+ try:+ source = inspect.getsource(original_get)+ parameters = inspect.signature(original_get).parameters+ except (OSError, TypeError, ValueError):+ return False+ if "full_state_dict" in source or "sd_options" not in parameters:+ return True++ def _full_peft(model, adapter_only, sd_options):+ return (+ adapter_only+ and sd_options is not None+ and getattr(sd_options, "full_state_dict", False)+ and is_peft_model(model)+ )++ @functools.wraps(original_get)+ def _get_model_state_dict(+ model,+ adapter_only = False,+ sd_options = None,+ ):+ if _full_peft(model, adapter_only, sd_options):+ from torch.distributed.checkpoint.state_dict import get_model_state_dict+ options = dataclasses.replace(sd_options, ignore_frozen_params = True)+ return get_model_state_dict(model, options = options)+ return original_get(model, adapter_only = adapter_only, sd_options = sd_options)++ @functools.wraps(original_set)+ def _set_model_state_dict(+ model,+ state_dict,+ adapter_only = False,+ sd_options = None,+ ):+ if _full_peft(model, adapter_only, sd_options):+ from torch.distributed.checkpoint.state_dict import set_model_state_dict+ options = dataclasses.replace(sd_options, strict = False)+ return set_model_state_dict(model, state_dict, options = options)+ return original_set(model, state_dict, adapter_only = adapter_only, sd_options = sd_options)++ setattr(_get_model_state_dict, _PATCHED, True)+ setattr(_set_model_state_dict, _PATCHED, True)+ fsdp_utils._get_model_state_dict = _get_model_state_dict+ fsdp_utils._set_model_state_dict = _set_model_state_dict+ return True+++def install_fsdp2_hooks():+ """Everything FSDP2 needs before an Accelerator exists; also for raw `Accelerator.prepare()` loops."""+ install_fsdp2_policy_provenance()+ install_fsdp2_prepare_hook()+ install_fsdp2_no_sync()+ install_fsdp2_peft_full_state_dict()+++def fsdp2_trainer_setup(trainer):+ """Returns (keep_fused_lora, reason); False for FSDP1 and for a root-only FSDP2 wrap."""+ accelerator = getattr(trainer, "accelerator", None)+ plugin = getattr(getattr(accelerator, "state", None), "fsdp_plugin", None)+ if plugin is None or getattr(plugin, "fsdp_version", 1) != 2:+ return False, None+ install_fsdp2_prepare_hook()+ install_fsdp2_no_sync()+ install_fsdp2_peft_full_state_dict()+ if os.environ.get("UNSLOTH_FSDP2_FUSED_LORA", "1").strip().lower() in _FALSY:+ return False, "UNSLOTH_FSDP2_FUSED_LORA=0"+ if getattr(plugin, "auto_wrap_policy", None) is None:+ return False, "FSDP2 wraps only the root module (auto_wrap_policy NO_WRAP)"+ return True, Nonediff --git a/unsloth/models/llama.py b/unsloth/models/llama.pyindex 7713d0abb81..6fc7c4c4ed5 100644--- a/unsloth/models/llama.py+++ b/unsloth/models/llama.py@@ -66,6 +66,7 @@ from .loader_utils import (_restore_dropped_fp8_scales,_prepare_compressed_tensors_model,fsdp_will_wrap,+ fsdp2_keeps_fused_lora,planner_class_mismatch_reason,planner_model_class,planner_config_overrides,@@ -2721,7 +2722,10 @@ def _fused_lora_skip_reason(if float32_base:reasons.append("the base weights are float32")if fsdp:- reasons.append("FSDP shards the weights they read (UNSLOTH_FORCE_FUSED_LORA=1 overrides)")+ reasons.append(+ "FSDP1 hands them shard views of the weights they read (UNSLOTH_FORCE_FUSED_LORA=1 "+ "overrides; FSDP2 with one unit per decoder block keeps them)"+ )if not reasons:return ""return (@@ -2874,11 +2878,32 @@ def _has_packed_base(*projs):_FUSED_LORA_MLPS = (apply_lora_mlp_swiglu, apply_lora_mlp_geglu_exact, apply_lora_mlp_geglu_approx)-def _decline_fused_lora_for_fsdp(model) -> int:+def _fused_lora_modules(model) -> list:+ """The modules patch_peft_model gave a fused LoRA kernel (attention for QKV / O, MLPs)."""+ if model is None:+ return []+ fused = []+ for module in model.modules():+ if (+ getattr(module, "apply_qkv", None) is apply_lora_qkv+ or getattr(module, "apply_o", None) is apply_lora_o+ ):+ fused.append(module)+ continue+ for name in ("forward", "_unsloth_forward"):+ func = getattr(module.__dict__.get(name), "__func__", None)+ if getattr(func, "func", func) in _FUSED_LORA_MLPS:+ fused.append(module)+ break+ return fused+++def _decline_fused_lora_for_fsdp(model, reason = None) -> int:"""Put peft's forwards back on layers patch_peft_model gave a fused LoRA kernel.`TrainingArguments(fsdp=...)` only reaches the Accelerator inside `Trainer.__init__`, afterpatch_peft_model ran with no launcher env to see, so the Trainer calls this once it knows.+ Under FSDP2 the check after accelerate shards the model calls it too, with its `reason`."""if model is None or os.environ.get("UNSLOTH_FORCE_FUSED_LORA", "0") == "1":return 0@@ -2900,8 +2925,10 @@ def _decline_fused_lora_for_fsdp(model) -> int:module._unsloth_forward = module.__class__.forwardn += 1if n:+ if reason is None:+ reason = "the Trainer enabled FSDP"logger.warning_once(- f"Unsloth: the Trainer enabled FSDP, so {n} fused LoRA projections were switched back "+ f"Unsloth: {reason}, so {n} fused LoRA projections were switched back ""to peft's forward, which FSDP can unshard. Set UNSLOTH_FORCE_FUSED_LORA=1 to keep them.")return n@@ -4754,8 +4781,9 @@ class FastLlamaModel:float32_base = (_base_weight_dtype(model.model.model.layers[0].self_attn.q_proj) == torch.float32)- # Fused kernels read `.weight` directly, bypassing FSDP's unshard hook, so under FSDP they see shard views (#409).- fused_lora_declined_for_fsdp = fsdp_will_wrap()+ # Fused kernels read `.weight` directly, bypassing FSDP's unshard hook, so under FSDP1 they see shard+ # views (#409). FSDP2 unshards each block before its forward; the wrap is re-checked once built.+ fused_lora_declined_for_fsdp = fsdp_will_wrap() and not fsdp2_keeps_fused_lora()if (lora_dropout == 0diff --git a/unsloth/models/loader_utils.py b/unsloth/models/loader_utils.pyindex 8f40869a4e5..2e3dc2e21ee 100644--- a/unsloth/models/loader_utils.py+++ b/unsloth/models/loader_utils.py@@ -10,6 +10,7 @@# limitations under the License.from ..device_type import DEVICE_TYPE_TORCH, clean_gpu_cache+from ..fsdp2_launch import fsdp_version_from_envimport hashlibimport importlibimport os@@ -130,6 +131,31 @@ def _infer_local_rank(rank):_TRUTHY = ("1", "true", "yes", "on")+def fsdp_version_requested():+ """The FSDP version this process trains with: 0 for none, else 1 or 2.++ A built Accelerator is authoritative; before one exists (model load, patch_peft_model) the+ launcher env is the only signal (`fsdp_version_from_env`: `accelerate launch` with an FSDP+ config, FSDP_VERSION by hand, or UNSLOTH_FSDP2=1 under torchrun). Every rank of a job sees the+ same env and the same Accelerator config, so every rank gets the same answer.+ `TrainingArguments(fsdp = ...)` alone is invisible until the Trainer builds its Accelerator;+ the Trainer re-checks then (`_unsloth_trainer_init`).+ """+ try:+ from accelerate.state import AcceleratorState++ shared = AcceleratorState._shared_state+ plugin = shared.get("fsdp_plugin", None)+ if plugin is not None:+ return int(getattr(plugin, "fsdp_version", 1) or 1)+ distributed_type = shared.get("distributed_type", None)+ if distributed_type is not None and "FSDP" in str(distributed_type).upper():+ return fsdp_version_from_env() or 1+ except Exception:+ pass+ return fsdp_version_from_env()++def fsdp_will_wrap():"""True when this process will hand the model to FSDP (unsloth#409).@@ -139,32 +165,32 @@ def fsdp_will_wrap():"""if os.environ.get("UNSLOTH_FORCE_FUSED_LORA", "0") == "1":return False- try:- from accelerate.state import AcceleratorState- distributed_type = AcceleratorState._shared_state.get("distributed_type", None)- if distributed_type is not None and "FSDP" in str(distributed_type).upper():- return True- except Exception:- pass- if str(os.environ.get("ACCELERATE_USE_FSDP", "")).strip().lower() in _TRUTHY:- return True- # "0" is how an accelerate config says "not FSDP".- if str(os.environ.get("FSDP_VERSION", "")).strip() not in ("", "0"):- return True- return False+ return fsdp_version_requested() > 0def fsdp2_requested():- """True when this process runs under FSDP2: `accelerate launch` exports FSDP_VERSION=2, an- Accelerator built already records its fsdp_plugin."""- if os.environ.get("FSDP_VERSION", "").strip() == "2":- return True- try:- from accelerate.state import AcceleratorState- plugin = AcceleratorState._shared_state.get("fsdp_plugin", None)- return getattr(plugin, "fsdp_version", None) == 2- except Exception:+ """True when this process runs under FSDP2: `accelerate launch` exports FSDP_VERSION=2,+ UNSLOTH_FSDP2=1 under torchrun, or an Accelerator built already records its fsdp_plugin."""+ return fsdp_version_requested() == 2+++def fsdp2_keeps_fused_lora():+ """Load-time half of the fused LoRA gate under FSDP.++ FSDP1 hands the fused kernels flat shard views (unsloth#409), so they are declined. FSDP2+ unshards a block's parameters in its pre-forward hook and the kernels run inside the block's+ forward, so with one FSDP unit per decoder block they read whole weights; the wrap accelerate+ actually builds is checked once it exists (`unsloth.models.fsdp2`), which declines them for a+ root-only or finer wrap. `UNSLOTH_FSDP2_FUSED_LORA=0` declines them under FSDP2 as well.+ """+ if str(os.environ.get("UNSLOTH_FSDP2_FUSED_LORA", "1")).strip().lower() in (+ "0",+ "false",+ "no",+ "off",+ ):return False+ return fsdp2_requested()def raise_if_fast_inference_under_fsdp2(fast_inference):