Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
FSDP2: save_pretrained, merged saves and 4-bit checkpoints for sharded models (#13895)
Daniel Han81e51f6168dd
diff --git a/tests/test_fsdp2_save_args.py b/tests/test_fsdp2_save_args.pynew file mode 100644index 00000000000..18f5ba7e89e--- /dev/null+++ b/tests/test_fsdp2_save_args.py@@ -0,0 +1,98 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""Which FSDP2 saves gather (unsloth/save.py `_fsdp2_collective_save`). No GPU work."""++import types++import pytest++torch = pytest.importorskip("torch")+peft = pytest.importorskip("peft")++from unsloth import save as unsloth_save # noqa: E402+from unsloth.models import fsdp2_save # noqa: E402+++def _merged(+ self,+ save_directory,+ tokenizer = None,+ save_method = "merged_16bit",+ push_to_hub = False,+ token = None,+ is_main_process = True,+ state_dict = None,+):+ return ("wrote", state_dict)+++class _Model:+ def original_model_save_pretrained(+ self,+ save_directory,+ is_main_process = True,+ state_dict = None,+ **kwargs,+ ):+ return ("wrote", state_dict)+++def _forwarding(self, *args, **kwargs):+ return self.original_model_save_pretrained(*args, **kwargs)+++def no_gather(monkeypatch):+ monkeypatch.setattr(fsdp2_save, "fsdp2_units", lambda model: [object()])+ monkeypatch.setattr(fsdp2_save, "fsdp2_gather_active", lambda: False)++ def _refuse(*args, **kwargs):+ raise AssertionError("a caller's own state_dict must not start a collective gather")++ monkeypatch.setattr(fsdp2_save, "fsdp2_gathered", _refuse)+++def test_a_positional_state_dict_is_passed_through(no_gather):+ sd = {"w": torch.zeros(1)}+ merged = unsloth_save._fsdp2_collective_save(lambda *a: False)(_merged)+ # (save_directory, tokenizer, save_method, push_to_hub, token, is_main_process, state_dict)+ assert merged(_Model(), "out", None, "merged_16bit", False, None, True, sd) == ("wrote", sd)+ assert merged(_Model(), "out", state_dict = sd) == ("wrote", sd)+ forwarding = unsloth_save._fsdp2_collective_save(lambda *a: False)(_forwarding)+ # PreTrainedModel.save_pretrained(save_directory, is_main_process, state_dict, ...)+ assert forwarding(_Model(), "out", True, sd) == ("wrote", sd)+++def test_peft_saves_gather_the_frozen_tensors_peft_writes():+ from peft import LoraConfig, get_peft_model+ from transformers import LlamaConfig, LlamaForCausalLM++ cfg = LlamaConfig(+ hidden_size = 32,+ intermediate_size = 64,+ num_hidden_layers = 1,+ num_attention_heads = 2,+ num_key_value_heads = 1,+ vocab_size = 64,+ )+ model = get_peft_model(LlamaForCausalLM(cfg), LoraConfig(r = 2, target_modules = ["q_proj"]))+ model.add_adapter("other", LoraConfig(r = 2, target_modules = ["q_proj"]))+ include = unsloth_save._fsdp2_peft_saved_frozen(model, {})+ names = {n for n, p in model.named_parameters() if not p.requires_grad and include(n, p)}+ assert any(".other." in n for n in names), names+ assert any("embed_tokens" in n for n in names) and any("lm_head" in n for n in names), names+ assert not any("q_proj.base_layer" in n for n in names), names+ # A biased output head: PEFT saves its bias with the weight.+ head = model.get_output_embeddings()+ head.bias = torch.nn.Parameter(torch.zeros(head.out_features), requires_grad = False)+ include = unsloth_save._fsdp2_peft_saved_frozen(model, {})+ assert any(n.endswith("lm_head.bias") for n, p in model.named_parameters() if include(n, p))+ include = unsloth_save._fsdp2_peft_saved_frozen(model, {"save_embedding_layers": False})+ names = {n for n, p in model.named_parameters() if include(n, p)}+ assert not any("embed_tokens" in n for n in names), names+ model.original_model_save_pretrained = model.save_pretrained+ arguments = unsloth_save._fsdp2_save_arguments(+ _forwarding, model, ("out", True, None, False), {}+ )+ assert arguments["save_embedding_layers"] is Falsediff --git a/tests/test_fsdp2_save_gloo.py b/tests/test_fsdp2_save_gloo.pynew file mode 100644index 00000000000..abce71b0d51--- /dev/null+++ b/tests/test_fsdp2_save_gloo.py@@ -0,0 +1,519 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""Saving under FSDP2 (unsloth/models/fsdp2_save.py), two gloo ranks on CPU.++- `fsdp2_gathered`: rank 0 sees whole parameters (on CPU) equal to an unsharded copy, also after an eval+ forward left a unit unsharded; the other rank writes nothing; the sharded parameters come back and+ training continues; a nested call does not gather again.+- bitsandbytes 4-bit weights come back as Params4bit with their QuantState and the exact packed bytes.+- trainable parameters FSDP2 leaves unsharded (`ignored_params`) drift apart across ranks unless their+ gradients are averaged (`reduce_unsharded_trainable_grads`).+- torch.distributed.checkpoint cannot build a state dict of a bitsandbytes 4-bit model (the Trainer's+ FSDP2 checkpoint) until `install_dcp_bnb_state_dict`.+The modules are loaded from their files (importing unsloth needs a GPU).+"""++import importlib.util+import os+import socket+import sys+from pathlib import Path++import pytest+import torch++_ROOT = Path(__file__).resolve().parents[1]+++def _load(name):+ path = _ROOT / "unsloth" / "models" / f"{name}.py"+ spec = importlib.util.spec_from_file_location(f"_unsloth_{name}_under_test", path)+ module = importlib.util.module_from_spec(spec)+ spec.loader.exec_module(module)+ return module+++fsdp_mod = pytest.importorskip("torch.distributed.fsdp")+if not hasattr(fsdp_mod, "fully_shard"):+ pytest.skip("torch has no FSDP2 fully_shard", allow_module_level = True)+if (+ sys.platform == "win32"+ or not torch.distributed.is_available()+ or not torch.distributed.is_gloo_available()+):+ pytest.skip("needs torch.distributed with gloo", allow_module_level = True)++N_LAYERS = 3+++def _free_port():+ with socket.socket() as s:+ s.bind(("127.0.0.1", 0))+ return s.getsockname()[1]+++class _Block(torch.nn.Module):+ def __init__(self, seed):+ super().__init__()+ torch.manual_seed(seed)+ # Odd sizes so the shards are uneven.+ self.base = torch.nn.Linear(37, 37, bias = False).requires_grad_(False)+ self.lora_A = torch.nn.Linear(37, 4, bias = False)+ self.lora_B = torch.nn.Linear(4, 37, bias = False)+ torch.nn.init.normal_(self.lora_B.weight, std = 0.1)++ def forward(self, x):+ return x + torch.tanh(self.base(x) + self.lora_B(self.lora_A(x)))+++class _Model(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.layers = torch.nn.ModuleList([_Block(10 * i) for i in range(N_LAYERS)])+ torch.manual_seed(99)+ self.head = torch.nn.Linear(37, 5, bias = True)++ def forward(self, x):+ for layer in self.layers:+ x = layer(x)+ return self.head(x)+++def _shard(+ model,+ world_size,+ ignored = None,+):+ from torch.distributed.device_mesh import init_device_mesh+ from torch.distributed.fsdp import fully_shard++ mesh = init_device_mesh("cpu", (world_size,))+ kwargs = {"ignored_params": ignored} if ignored else {}+ for layer in model.layers:+ fully_shard(layer, mesh = mesh, **kwargs)+ fully_shard(model, mesh = mesh, **kwargs)+ # What Unsloth does to the last block: it stays unsharded after a forward.+ model.layers[-1].set_reshard_after_forward(False)+ return model+++def _is_dtensor(t):+ from torch.distributed.tensor import DTensor+ return isinstance(t, DTensor)+++def _case_gather(rank, world_size, m):+ model = _shard(_Model(), world_size)+ reference = _Model()+ x = torch.randn(6, 37, generator = torch.Generator().manual_seed(rank))+ with torch.no_grad():+ model(x)+ # The eval forward left the last block unsharded: a plain (non DTensor) parameter.+ unsharded_after_eval = sum(not _is_dtensor(p) for p in model.parameters())+ out = {"unsharded_after_eval": unsharded_after_eval}++ with m.fsdp2_gathered(model, trainable_only = True) as writer:+ out["trainable_writer"] = writer+ out["trainable_only"] = sorted(+ {+ ("plain" if not _is_dtensor(p) else "dtensor", p.requires_grad)+ for p in model.parameters()+ }+ )++ # A frozen parameter the save still writes (an inactive adapter, an embedding).+ with m.fsdp2_gathered(model, trainable_only = True, include = lambda name, p: True) as writer:+ out["included"] = sorted(+ {+ ("plain" if not _is_dtensor(p) else "dtensor", p.requires_grad)+ for p in model.parameters()+ }+ )++ with m.fsdp2_gathered(model) as writer:+ out["writer"] = writer+ with m.fsdp2_gathered(model) as nested:+ out["nested"] = nested+ if writer:+ ref = dict(reference.named_parameters())+ got = dict(model.named_parameters())+ out["names_match"] = sorted(ref) == sorted(got)+ out["plain_cpu"] = all(+ not _is_dtensor(p) and p.device.type == "cpu" for p in got.values()+ )+ out["exact"] = all(torch.equal(got[k].detach(), ref[k].detach()) for k in ref)+ out["state_dict_keys"] = sorted(model.state_dict()) == sorted(reference.state_dict())+ out["requires_grad_kept"] = all(+ got[k].requires_grad == ref[k].requires_grad for k in ref+ )+ else:+ out["all_dtensor_on_other_rank"] = all(_is_dtensor(p) for p in model.parameters())+ out["all_dtensor_after"] = all(_is_dtensor(p) for p in model.parameters())++ # Training continues on the restored shards and matches the unsharded model step for step.+ opt = torch.optim.SGD([p for p in model.parameters() if p.requires_grad], lr = 0.1)+ ref_opt = torch.optim.SGD([p for p in reference.parameters() if p.requires_grad], lr = 0.1)+ import torch.distributed as dist++ xs = [+ torch.randn(6, 37, generator = torch.Generator().manual_seed(100 + r))+ for r in range(world_size)+ ]+ model(xs[rank]).square().mean().backward()+ opt.step()+ sum(reference(xr).square().mean() for xr in xs).div(world_size).backward()+ ref_opt.step()+ diffs = []+ for (n, p), (_, q) in zip(model.named_parameters(), reference.named_parameters()):+ full = p.full_tensor() if _is_dtensor(p) else p+ diffs.append(float((full.detach() - q.detach()).abs().max()))+ out["step_max_diff"] = max(diffs)+ return out+++def _case_gather_4bit(rank, world_size, m):+ bnb = pytest.importorskip("bitsandbytes")+ q = _load("fsdp2_qlora")+ from torch.distributed.device_mesh import init_device_mesh+ from torch.distributed.fsdp import fully_shard++ def lin(seed):+ torch.manual_seed(seed)+ layer = bnb.nn.Linear4bit(+ 128,+ 96,+ bias = False,+ compute_dtype = torch.bfloat16,+ quant_type = "nf4",+ quant_storage = torch.bfloat16,+ )+ layer.weight = bnb.nn.Params4bit(+ torch.randn(96, 128) * 0.05,+ requires_grad = False,+ quant_type = "nf4",+ quant_storage = torch.bfloat16,+ blocksize = 64,+ module = layer,+ )+ return layer.to("cpu")++ model = torch.nn.Sequential(lin(0), lin(1))+ before = [+ (+ l.weight.data.view(torch.uint8).clone(),+ bnb.functional.dequantize_4bit(l.weight.data, l.weight.quant_state),+ )+ for l in model+ ]+ info = q.prepare_fsdp2_qlora(model)+ mesh = init_device_mesh("cpu", (world_size,))+ for layer in model:+ fully_shard(layer, mesh = mesh)+ fully_shard(model, mesh = mesh)+ out = {"patched": info["patched"]}+ with m.fsdp2_gathered(model) as writer:+ if writer:+ checks = []+ for layer, (raw, deq) in zip(model, before):+ w = layer.weight+ checks.append(+ dict(+ params4bit = type(w).__name__ == "Params4bit",+ quant_state = w.quant_state is not None,+ bytes_equal = torch.equal(w.data.view(torch.uint8), raw),+ dequant_equal = torch.equal(+ bnb.functional.dequantize_4bit(w.data, w.quant_state), deq+ ),+ module = w.module is layer,+ )+ )+ out["checks"] = checks+ # bitsandbytes' own state dict (what save_pretrained writes) has the QuantState entries.+ out["absmax_keys"] = sum(k.endswith("weight.absmax") for k in model.state_dict())+ out["dtensor_after"] = all(_is_dtensor(p) for p in model.parameters())+ return out+++def _case_replicated(rank, world_size, m, hooked):+ import torch.distributed as dist++ model = _Model()+ reference = _Model()+ head = [model.head.weight, model.head.bias]+ _shard(model, world_size, ignored = set(head))+ names = m.reduce_unsharded_trainable_grads(model) if hooked else []+ opt = torch.optim.SGD([p for p in model.parameters() if p.requires_grad], lr = 0.5)+ ref_opt = torch.optim.SGD([p for p in reference.parameters() if p.requires_grad], lr = 0.5)+ for step in range(3):+ xs = [+ torch.randn(6, 37, generator = torch.Generator().manual_seed(10 * step + r))+ for r in range(world_size)+ ]+ opt.zero_grad()+ model(xs[rank]).square().mean().backward()+ opt.step()+ ref_opt.zero_grad()+ sum(reference(xr).square().mean() for xr in xs).div(world_size).backward()+ ref_opt.step()+ w = model.head.weight.detach().clone()+ gathered = [torch.empty_like(w) for _ in range(world_size)]+ dist.all_gather(gathered, w)+ return dict(+ names = names,+ head_is_dtensor = _is_dtensor(model.head.weight),+ rank_spread = max(float((g - gathered[0]).abs().max()) for g in gathered),+ vs_reference = float((w - reference.head.weight.detach()).abs().max()),+ )+++def _case_dcp_bnb(rank, world_size, m):+ bnb = pytest.importorskip("bitsandbytes")+ q = _load("fsdp2_qlora")+ from torch.distributed.checkpoint.state_dict import StateDictOptions+ from torch.distributed.device_mesh import init_device_mesh+ from torch.distributed.fsdp import fully_shard+ import torch.distributed.checkpoint.state_dict as dcp++ torch.manual_seed(0)+ layer = bnb.nn.Linear4bit(+ 128,+ 64,+ bias = False,+ compute_dtype = torch.bfloat16,+ quant_type = "nf4",+ quant_storage = torch.bfloat16,+ )+ layer.weight = bnb.nn.Params4bit(+ torch.randn(64, 128) * 0.05,+ requires_grad = False,+ quant_type = "nf4",+ quant_storage = torch.bfloat16,+ blocksize = 64,+ module = layer,+ )+ model = torch.nn.Sequential(layer.to("cpu"), torch.nn.Linear(64, 8))+ q.prepare_fsdp2_qlora(model)+ fully_shard(model, mesh = init_device_mesh("cpu", (world_size,)))+ # What accelerate's save_fsdp_model / load_fsdp_model ask for on an adapter checkpoint.+ options = StateDictOptions(full_state_dict = True, cpu_offload = True, ignore_frozen_params = True)+ out = {}+ try:+ dcp.get_model_state_dict(model, options = options)+ out["before"] = "ok"+ except AttributeError as error:+ out["before"] = f"AttributeError: {error}"+ out["installed"] = m.install_dcp_bnb_state_dict()+ # What Trainer.save_model asks for through accelerate's get_state_dict: a full state dict, frozen+ # weights included. bitsandbytes' extra keys cannot be resolved on a sharded weight either way.+ try:+ full = dcp.get_model_state_dict(+ model, options = StateDictOptions(full_state_dict = True, cpu_offload = True)+ )+ out["full_sharded_keys"] = sorted(full)+ except AttributeError as error:+ out["full_sharded_keys"] = f"AttributeError: {error}"+ sd = dcp.get_model_state_dict(model, options = options)+ out["after_keys"] = sorted(sd)+ expected = {k: v.clone() for k, v in sd.items()}+ with torch.no_grad():+ model[1].weight.zero_()+ dcp.set_model_state_dict(+ model,+ sd if rank == 0 else {},+ options = StateDictOptions(full_state_dict = True, broadcast_from_rank0 = True, strict = False),+ )+ restored = model[1].weight.full_tensor()+ out["roundtrip"] = rank != 0 or torch.equal(restored, expected["1.weight"])+ # A full DCP state dict of an unsharded 4-bit model keeps what dequantizes it.+ plain = torch.nn.Sequential(+ bnb.nn.Linear4bit(128, 64, bias = False, compute_dtype = torch.bfloat16, quant_type = "nf4")+ )+ plain[0].weight = bnb.nn.Params4bit(+ torch.randn(64, 128) * 0.05,+ requires_grad = False,+ quant_type = "nf4",+ blocksize = 64,+ module = plain[0],+ )+ plain = plain.to("cpu")+ out["full_keys"] = sorted(+ dcp.get_model_state_dict(plain, options = StateDictOptions(cpu_offload = True))+ )+ # accelerate's adapter-only path drops it (and no longer fails).+ import accelerate.utils.fsdp_utils as fsdp_utils++ from peft import LoraConfig, get_peft_model++ class _Net(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.q = bnb.nn.Linear4bit(+ 128,+ 64,+ bias = False,+ compute_dtype = torch.bfloat16,+ quant_type = "nf4",+ quant_storage = torch.bfloat16,+ )+ self.q.weight = bnb.nn.Params4bit(+ torch.randn(64, 128) * 0.05,+ requires_grad = False,+ quant_type = "nf4",+ quant_storage = torch.bfloat16,+ blocksize = 64,+ module = self.q,+ )++ def forward(self, x):+ return self.q(x)++ peft_model = get_peft_model(_Net().to("cpu"), LoraConfig(r = 4, target_modules = ["q"]))+ q.prepare_fsdp2_qlora(peft_model)+ fully_shard(peft_model, mesh = init_device_mesh("cpu", (world_size,)))+ sd = fsdp_utils._get_model_state_dict(+ peft_model,+ adapter_only = True,+ sd_options = StateDictOptions(full_state_dict = True, cpu_offload = True),+ )+ out["adapter_keys"] = sorted(k.split("q.", 1)[-1] for k in sd)+ return out+++def _case_slow_writer(rank, world_size, m):+ """rank 0's write (a merged or quantized export, a Hub upload) outlasts the collective timeout."""+ import time++ import torch.distributed as dist++ model = _shard(_Model(), world_size)+ with m.fsdp2_gathered(model) as writer:+ if writer:+ time.sleep(10)+ dist.barrier()+ return {"ok": True}+++def _worker(rank, world_size, port, case, queue):+ try:+ os.environ.update(+ MASTER_ADDR = "127.0.0.1",+ MASTER_PORT = str(port),+ RANK = str(rank),+ WORLD_SIZE = str(world_size),+ )+ import torch.distributed as dist++ import datetime++ timeout = datetime.timedelta(seconds = 4) if case == "slow_writer" else None+ dist.init_process_group(+ "gloo", rank = rank, world_size = world_size, **({"timeout": timeout} if timeout else {})+ )+ m = _load("fsdp2_save")+ if case == "slow_writer":+ result = _case_slow_writer(rank, world_size, m)+ elif case == "gather":+ result = _case_gather(rank, world_size, m)+ elif case == "gather_4bit":+ result = _case_gather_4bit(rank, world_size, m)+ elif case.startswith("replicated"):+ result = _case_replicated(rank, world_size, m, hooked = case.endswith("hooked"))+ else:+ result = _case_dcp_bnb(rank, world_size, m)+ queue.put((rank, result))+ dist.destroy_process_group()+ except BaseException as error:+ import traceback+ queue.put((rank, {"error": f"{type(error).__name__}: {error}\n{traceback.format_exc()}"}))+++def _run(case, world_size = 2):+ import torch.multiprocessing as mp++ ctx = mp.get_context("spawn")+ queue = ctx.Queue()+ port = _free_port()+ procs = [+ ctx.Process(target = _worker, args = (r, world_size, port, case, queue))+ for r in range(world_size)+ ]+ for p in procs:+ p.start()+ results = {}+ for _ in procs:+ rank, result = queue.get(timeout = 240)+ results[rank] = result+ for p in procs:+ p.join(timeout = 60)+ for r in results.values():+ assert "error" not in r, r.get("error")+ return results+++def test_gather_gives_rank0_the_full_model_and_restores_shards():+ results = _run("gather")+ r0, r1 = results[0], results[1]+ assert r0["unsharded_after_eval"] > 0, r0+ assert r0["trainable_writer"] is True and r1["trainable_writer"] is False+ # trainable_only: adapters plain on rank 0, frozen weights stay sharded.+ assert r0["trainable_only"] == [("dtensor", False), ("plain", True)], r0+ assert r1["trainable_only"] == [("dtensor", False), ("dtensor", True)], r1+ assert r0["included"] == [("plain", False), ("plain", True)], r0+ assert r0["writer"] is True and r1["writer"] is False+ assert r0["nested"] is True and r1["nested"] is False+ assert r0["names_match"] and r0["plain_cpu"] and r0["exact"], r0+ assert r0["state_dict_keys"] and r0["requires_grad_kept"], r0+ assert r1["all_dtensor_on_other_rank"], r1+ for r in (r0, r1):+ assert r["all_dtensor_after"], r+ assert r["step_max_diff"] < 1e-6, r+++def test_gather_rebuilds_params4bit_with_exact_bytes():+ pytest.importorskip("bitsandbytes")+ results = _run("gather_4bit")+ r0 = results[0]+ assert r0["patched"], r0+ assert len(r0["checks"]) == 2 and all(all(c.values()) for c in r0["checks"]), r0["checks"]+ assert r0["absmax_keys"] == 2, r0+ assert all(r["dtensor_after"] for r in results.values()), results+++ not hasattr(torch.distributed.fsdp, "fully_shard")+ or "ignored_params"+ not in __import__("inspect").signature(torch.distributed.fsdp.fully_shard).parameters,+ reason = "fully_shard has no ignored_params",+)+def test_unsharded_trainable_params_need_their_grads_averaged():+ base = _run("replicated")+ assert not base[0]["head_is_dtensor"], base+ # Control: no gradient reduction for the ignored head, so the replicas drift apart.+ assert base[0]["rank_spread"] > 1e-3, base+ hooked = _run("replicated_hooked")+ assert hooked[0]["names"] == ["head.weight", "head.bias"], hooked+ assert hooked[0]["rank_spread"] == 0.0, hooked+ for r in hooked.values():+ assert r["vs_reference"] < 1e-6, r+++def test_dcp_state_dict_of_bitsandbytes_4bit():+ pytest.importorskip("bitsandbytes")+ results = _run("dcp_bnb")+ r = results[0]+ # A sharded 4-bit weight is a DTensor, which has no `absmax` attribute for `weight.absmax`.+ assert r["before"].startswith("AttributeError") and "absmax" in r["before"], r+ assert r["installed"], r+ assert r["after_keys"] == ["1.bias", "1.weight"], r+ assert r["full_sharded_keys"] == ["0.weight", "1.bias", "1.weight"], r["full_sharded_keys"]+ assert all(x["roundtrip"] for x in results.values()), results+ assert "0.weight.absmax" in r["full_keys"], r["full_keys"]+ assert r["adapter_keys"] == ["lora_A.default.weight", "lora_B.default.weight"], r[+ "adapter_keys"+ ]+++def test_a_slow_writer_does_not_time_out_the_other_ranks():+ results = _run("slow_writer")+ assert all(r["ok"] for r in results.values()), resultsdiff --git a/unsloth/models/fsdp2.py b/unsloth/models/fsdp2.pyindex f4a9cf7874e..d86c79a1b3e 100644--- a/unsloth/models/fsdp2.py+++ b/unsloth/models/fsdp2.py@@ -409,6 +409,14 @@ def after_fsdp2_wrap(accelerator, model):model, blocks, parents, FSDPModule)_gate_fused_lora(accelerator, model, parents, FSDPModule, blocks)+ from .fsdp2_save import reduce_unsharded_trainable_grads++ replicated = reduce_unsharded_trainable_grads(model)+ if replicated:+ _print_rank0(+ f"Unsloth: FSDP2 leaves {len(replicated)} trainable parameters unsharded (e.g. "+ f"{replicated[0]}); their gradients are averaged over ranks like DDP."+ )def install_fsdp2_prepare_hook():@@ -553,6 +561,9 @@ def install_fsdp2_hooks():install_fsdp2_prepare_hook()install_fsdp2_no_sync()install_fsdp2_peft_full_state_dict()+ from .fsdp2_save import install_dcp_bnb_state_dict++ install_dcp_bnb_state_dict()def fsdp2_trainer_setup(trainer):@@ -564,6 +575,9 @@ def fsdp2_trainer_setup(trainer):install_fsdp2_prepare_hook()install_fsdp2_no_sync()install_fsdp2_peft_full_state_dict()+ from .fsdp2_save import install_dcp_bnb_state_dict++ install_dcp_bnb_state_dict()if os.environ.get("UNSLOTH_FSDP2_FUSED_LORA", "1").strip().lower() in _FALSY:return False, "UNSLOTH_FSDP2_FUSED_LORA=0"if getattr(plugin, "auto_wrap_policy", None) is None:diff --git a/unsloth/models/fsdp2_save.py b/unsloth/models/fsdp2_save.pynew file mode 100644index 00000000000..6b7cfca52de--- /dev/null+++ b/unsloth/models/fsdp2_save.py@@ -0,0 +1,372 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""Saving an FSDP2-sharded model. Every rank must call the save: the shards are all-gathered one+parameter at a time, rank 0 swaps the full tensors (on CPU) into the modules, runs the ordinary+single-GPU save code, and puts the sharded parameters back. bitsandbytes 4-bit weights are gathered+as raw bytes and rebuilt as Params4bit with their QuantState, so a merge can still dequantize them.+"""++__all__ = [+ "fsdp2_units",+ "fsdp2_gathered",+ "fsdp2_gather_active",+ "install_dcp_bnb_state_dict",+ "reduce_unsharded_trainable_grads",+]++import contextlib+import datetime+import functools+import os++import torch++# Writer flag of the gather in progress: a nested save (save_pretrained_merged -> save_pretrained,+# or on a submodule) must neither gather again nor reshard what rank 0 swapped in.+_ACTIVE = []+++def _fsdp_module_class():+ try:+ from torch.distributed.fsdp import FSDPModule+ return FSDPModule+ except ImportError:+ pass+ try:+ from torch.distributed._composable.fsdp import FSDPModule+ return FSDPModule+ except ImportError:+ return None+++def _dtensor_class():+ try:+ from torch.distributed.tensor import DTensor+ return DTensor+ except ImportError:+ try:+ from torch.distributed._tensor import DTensor+ return DTensor+ except ImportError:+ return None+++def fsdp2_units(model):+ FSDPModule = _fsdp_module_class()+ if FSDPModule is None or not isinstance(model, torch.nn.Module):+ return []+ return [m for m in model.modules() if isinstance(m, FSDPModule)]+++def _dist():+ import torch.distributed as dist+ if dist.is_available() and dist.is_initialized():+ return dist+ return None+++_SAVE_COUNTER = [0]+++def _save_timeout():+ try:+ return datetime.timedelta(+ seconds = float(os.environ.get("UNSLOTH_FSDP2_SAVE_TIMEOUT", 7 * 86400))+ )+ except ValueError:+ return datetime.timedelta(days = 7)+++def _wait_for_writer(dist, writer):+ """The other ranks wait for rank 0's write, so no later collective overtakes it. A store key, not+ a barrier: a merged or quantized export (and a Hub upload) can outlast the process group's+ collective timeout, and NCCL ranks blocked in a barrier would abort before rank 0 arrives."""+ _SAVE_COUNTER[0] += 1+ key = f"unsloth_fsdp2_save_{_SAVE_COUNTER[0]}"+ try:+ store = dist.distributed_c10d._get_default_store()+ except Exception:+ store = None+ if store is None:+ dist.barrier()+ return+ if writer:+ store.set(key, "done")+ else:+ store.wait([key], _save_timeout())+++def _gather_full(param, DTensor):+ """All ranks: the full tensor of a sharded parameter. Packed 4-bit bytes travel as uint8, so no+ float copy kernel can rewrite NaN bit patterns inside them."""+ local = param.to_local()+ mesh_type = param.device_mesh.device_type+ if local.device.type == "cpu" and mesh_type != "cpu":+ # CPUOffloadPolicy keeps shards on the host; NCCL can only gather device tensors.+ device = torch.device(mesh_type, getattr(torch, mesh_type).current_device())+ local = local.to(device)+ if getattr(param, "quant_state", None) is None or param.dtype == torch.uint8:+ return DTensor.from_local(+ local, param.device_mesh, param.placements, shape = param.shape, stride = param.stride()+ ).full_tensor()+ elif getattr(param, "quant_state", None) is None or param.dtype == torch.uint8:+ return param.full_tensor()+ as_bytes = local.view(torch.uint8)+ shape = list(param.shape)+ shape[-1] = shape[-1] * param.element_size()+ stride = []+ acc = 1+ for size in reversed(shape):+ stride.insert(0, acc)+ acc *= size+ gathered = DTensor.from_local(+ as_bytes,+ param.device_mesh,+ param.placements,+ shape = torch.Size(shape),+ stride = tuple(stride),+ ).full_tensor()+ return gathered.view(param.dtype)+++def _rebuild(param, full, module):+ """A full parameter that looks like the one FSDP2 sharded (Params4bit keeps its QuantState)."""+ quant_state = getattr(param, "quant_state", None)+ if quant_state is None:+ return torch.nn.Parameter(full, requires_grad = param.requires_grad)+ from bitsandbytes.nn import Params4bit++ new = torch.Tensor._make_subclass(Params4bit, full, False)+ for key in (+ "quant_state",+ "blocksize",+ "compress_statistics",+ "quant_type",+ "quant_storage",+ "bnb_quantized",+ ):+ if hasattr(param, key):+ setattr(new, key, getattr(param, key))+ new.module = module+ return new+++def fsdp2_gathered(+ model,+ trainable_only = False,+ offload_to_cpu = True,+ include = None,+):+ """Yields None for a model FSDP2 does not shard (nothing happens). Otherwise every rank must+ enter; rank 0 sees whole parameters (and yields True), the other ranks yield False and should+ not write. `trainable_only` gathers just the trainable parameters (enough for an adapter save),+ plus those `include(name, param)` selects (frozen ones the save still writes).+ Packed 4-bit weights stay on their device: the merge dequantizes them there."""+ if _ACTIVE:+ yield _ACTIVE[-1]+ return+ units = fsdp2_units(model)+ DTensor = _dtensor_class()+ if not units or DTensor is None:+ yield None+ return+ dist = _dist()+ rank = dist.get_rank() if dist is not None else 0+ writer = rank == 0+ for unit in units:+ # An eval forward leaves the last block and the lm_head unit gathered; make every+ # parameter a DTensor again so all ranks gather the same list.+ if hasattr(unit, "reshard"):+ unit.reshard()+ owners = {}+ for module in model.modules():+ for name, param in module._parameters.items():+ if param is not None:+ owners.setdefault(id(param), []).append((module, name))+ replaced = {}+ seen = set()+ pushed = False+ try:+ for name, param in model.named_parameters():+ if id(param) in seen:+ continue+ seen.add(id(param))+ if not isinstance(param, DTensor):+ continue+ if (+ trainable_only+ and not param.requires_grad+ and not (include and include(name, param))+ ):+ continue+ full = _gather_full(param, DTensor)+ if not writer:+ del full+ continue+ module = owners[id(param)][0][0]+ if offload_to_cpu and getattr(param, "quant_state", None) is None:+ full = full.cpu()+ replaced[id(param)] = (param, _rebuild(param, full.detach(), module))+ if writer:+ for key, (old, new) in replaced.items():+ for module, name in owners[key]:+ module._parameters[name] = new+ _ACTIVE.append(writer)+ pushed = True+ yield writer+ finally:+ if pushed:+ _ACTIVE.pop()+ for key, (old, new) in replaced.items():+ for module, name in owners[key]:+ module._parameters[name] = old+ replaced.clear()+ if dist is not None:+ _wait_for_writer(dist, writer)+++def fsdp2_gather_active():+ return bool(_ACTIVE)+++_DCP_DEPTH = [0]+_ADAPTER_ONLY = [0]+_DCP_FUNCTIONS = (+ "get_model_state_dict",+ "set_model_state_dict",+ "get_state_dict",+ "set_state_dict",+ "get_optimizer_state_dict",+ "set_optimizer_state_dict",+)+++def _drops_frozen_weights(kwargs):+ """Only adapter-only state dicts (accelerate's `adapter_only` path, or an explicit+ `ignore_frozen_params`) may skip the QuantState: they never keep the frozen 4-bit weights. A full+ DCP checkpoint still needs it to dequantize them."""+ if _ADAPTER_ONLY[0]:+ return True+ return bool(getattr(kwargs.get("options", None), "ignore_frozen_params", False))+++def _has_sharded_4bit(args, kwargs):+ """A Linear4bit whose weight FSDP2 made a DTensor: DCP cannot resolve `weight.absmax` on it at all,+ so keeping bitsandbytes' entries only raises (Trainer.save_model -> accelerate get_state_dict)."""+ model = args[0] if args else kwargs.get("model", None)+ DTensor = _dtensor_class()+ if DTensor is None or not isinstance(model, torch.nn.Module):+ return False+ try:+ from bitsandbytes.nn import Linear4bit+ except Exception:+ return False+ return any(+ isinstance(m, Linear4bit) and isinstance(getattr(m, "weight", None), DTensor)+ for m in model.modules()+ )+++def _wrap_accelerate_adapter_only():+ try:+ import accelerate.utils.fsdp_utils as fsdp_utils+ except Exception:+ return+ for name in ("_get_model_state_dict", "_set_model_state_dict"):+ fn = getattr(fsdp_utils, name, None)+ if fn is None or getattr(fn, "_unsloth_adapter_only_flag", False):+ continue++ @functools.wraps(fn)+ def wrapper(+ *args,+ _fn = fn,+ **kwargs,+ ):+ if not kwargs.get("adapter_only", False):+ return _fn(*args, **kwargs)+ _ADAPTER_ONLY[0] += 1+ try:+ return _fn(*args, **kwargs)+ finally:+ _ADAPTER_ONLY[0] -= 1++ wrapper._unsloth_adapter_only_flag = True+ setattr(fsdp_utils, name, wrapper)+++def install_dcp_bnb_state_dict():+ """torch.distributed.checkpoint resolves every state dict key as an attribute path, and+ bitsandbytes' `weight.absmax` / `weight.quant_state.bitsandbytes__nf4` entries are not one+ (AttributeError in `_get_fqns`). That breaks every FSDP2 Trainer adapter checkpoint of a 4-bit+ model. Inside adapter-only DCP calls (`_drops_frozen_weights`) Linear4bit reports just its weight+ and bias; the QuantState is unchanged in memory. Other DCP calls keep bitsandbytes' entries."""+ try:+ import torch.distributed.checkpoint.state_dict as dcp_state_dict+ from bitsandbytes.nn import Linear4bit+ except Exception:+ return False+ _wrap_accelerate_adapter_only()+ original_save = Linear4bit._save_to_state_dict+ if getattr(original_save, "_unsloth_fsdp2_patched", False):+ return True++ @functools.wraps(original_save)+ def _save_to_state_dict(self, destination, prefix, keep_vars):+ if _DCP_DEPTH[0]:+ return torch.nn.Module._save_to_state_dict(self, destination, prefix, keep_vars)+ return original_save(self, destination, prefix, keep_vars)++ _save_to_state_dict._unsloth_fsdp2_patched = True+ Linear4bit._save_to_state_dict = _save_to_state_dict++ def _wrap(fn):+ @functools.wraps(fn)+ def wrapper(*args, **kwargs):+ if not (_drops_frozen_weights(kwargs) or _has_sharded_4bit(args, kwargs)):+ return fn(*args, **kwargs)+ _DCP_DEPTH[0] += 1+ try:+ return fn(*args, **kwargs)+ finally:+ _DCP_DEPTH[0] -= 1++ wrapper._unsloth_fsdp2_patched = True+ return wrapper++ for name in _DCP_FUNCTIONS:+ fn = getattr(dcp_state_dict, name, None)+ if fn is not None and not getattr(fn, "_unsloth_fsdp2_patched", False):+ setattr(dcp_state_dict, name, _wrap(fn))+ return True+++def reduce_unsharded_trainable_grads(model):+ """Trainable parameters FSDP2 does not manage (e.g. in `ignored_modules`) get no gradient+ reduction, so every rank would step on its own gradient and the replicas drift apart. Average+ their gradient over ranks as it is produced, like DDP. Returns the parameter names."""+ dist = _dist()+ DTensor = _dtensor_class()+ if dist is None or DTensor is None or dist.get_world_size() == 1:+ return []+ names = []+ seen = set()+ for name, param in model.named_parameters():+ if id(param) in seen or not param.requires_grad or isinstance(param, DTensor):+ continue+ seen.add(id(param))+ if getattr(param, "_unsloth_grad_all_reduce", False):+ continue+ world = dist.get_world_size()++ def _average(grad, world = world):+ grad = grad.clone()+ dist.all_reduce(grad)+ return grad.div_(world)++ param.register_hook(_average)+ param._unsloth_grad_all_reduce = True+ names.append(name)+ return namesdiff --git a/unsloth/save.py b/unsloth/save.pyindex f54f25eb52c..a1acad4caea 100644--- a/unsloth/save.py+++ b/unsloth/save.py@@ -6097,6 +6097,107 @@ def unsloth_generic_save(return+def _fsdp2_save_arguments(fn, self, args, kwargs):+ """The call's arguments by name, bound against `fn`, or against the wrapped original+ save_pretrained when `fn` only forwards *args. {} when neither binds."""+ import inspect++ targets = [(fn, (self,) + tuple(args))]+ original = getattr(self, "original_model_save_pretrained", None)+ if callable(original):+ targets.append((original, tuple(args)))+ for target, positional in targets:+ try:+ signature = inspect.signature(target)+ bound = signature.bind_partial(*positional, **kwargs)+ except (TypeError, ValueError):+ continue+ if any(p.kind is p.VAR_POSITIONAL for p in signature.parameters.values()):+ continue+ arguments = dict(bound.arguments)+ for p in signature.parameters.values():+ if p.kind is p.VAR_KEYWORD:+ arguments.update(arguments.pop(p.name, {}))+ return arguments+ return dict(kwargs)+++def _fsdp2_peft_saved_frozen(model, arguments):+ """Frozen parameters a PEFT adapter save still writes: every adapter's (inactive ones are frozen,+ `selected_adapters` defaults to all), `modules_to_save` copies, and the embeddings unless+ `save_embedding_layers = False`."""+ adapters = set(getattr(model, "peft_config", None) or {})+ embeddings = set()+ if arguments.get("save_embedding_layers", "auto") is not False:+ for getter in ("get_input_embeddings", "get_output_embeddings"):+ try:+ module = getattr(model, getter)()+ except Exception:+ module = None+ # PEFT copies every entry under the embedding module (a decoder bias too), not just weight.+ if module is not None:+ embeddings.update(id(p) for p in module.parameters())++ def include(name, param):+ parts = name.split(".")+ return (+ id(param) in embeddings+ or "modules_to_save" in parts+ or any(a in parts for a in adapters)+ )++ return include+++def _fsdp2_collective_save(trainable_only):+ """Under FSDP2 every rank calls the save: the shards are gathered, rank 0 writes, the others+ return None. `trainable_only(self, args, kwargs)` says whether the adapter alone is enough.+ A model FSDP2 does not shard, or a call that brings its own `state_dict`, is passed through."""++ def decorate(fn):+ @functools.wraps(fn)+ def wrapper(self, *args, **kwargs):+ from .models.fsdp2_save import fsdp2_units, fsdp2_gathered, fsdp2_gather_active++ if fsdp2_gather_active() or not fsdp2_units(self):+ return fn(self, *args, **kwargs)+ arguments = _fsdp2_save_arguments(fn, self, args, kwargs)+ if arguments.get("state_dict") is not None:+ return fn(self, *args, **kwargs)+ only = trainable_only(self, args, kwargs)+ include = _fsdp2_peft_saved_frozen(self, arguments) if only else None+ if _fsdp2_rank() == 0:+ print(+ "Unsloth: FSDP2 save: gathering the "+ + ("trainable" if only else "sharded")+ + " weights to rank 0, which writes. Every rank must make this call."+ )+ with fsdp2_gathered(self, trainable_only = only, include = include) as writer:+ if writer is False:+ return None+ return fn(self, *args, **kwargs)++ return wrapper++ return decorate+++def _fsdp2_rank():+ import torch.distributed as dist+ return dist.get_rank() if dist.is_available() and dist.is_initialized() else 0+++def _merged_save_method(args, kwargs):+ if "save_method" in kwargs:+ return kwargs["save_method"]+ # (save_directory, tokenizer, save_method, ...)+ return args[2] if len(args) > 2 else "merged_16bit"+++@_fsdp2_collective_save(+ lambda self, args, kwargs: isinstance(self, PeftModel)+ and _is_adapter_save_method(_merged_save_method(args, kwargs))+)def unsloth_generic_save_pretrained_merged(self,save_directory: Union[str, os.PathLike],@@ -7895,6 +7996,7 @@ def patch_saving_functions(model, vision = False):'''exec(push_to_hub_text, globals())+ @_fsdp2_collective_save(lambda self, args, kwargs: isinstance(self, PeftModel))def unsloth_model_save_pretrained(self, *args, **kwargs):"""`safe_serialization = None` means the safetensors default, not a pickle.