Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

FSDP2: save_pretrained, merged saves and 4-bit checkpoints for sharded models (#13895)

Daniel Han81e51f6168dd
diff --git a/tests/test_fsdp2_save_args.py b/tests/test_fsdp2_save_args.py
new file mode 100644
index 00000000000..18f5ba7e89e
--- /dev/null
+++ b/tests/test_fsdp2_save_args.py
@@ -0,0 +1,98 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""Which FSDP2 saves gather (unsloth/save.py `_fsdp2_collective_save`). No GPU work."""
+
+import types
+
+import pytest
+
+torch = pytest.importorskip("torch")
+peft = pytest.importorskip("peft")
+
+from unsloth import save as unsloth_save # noqa: E402
+from unsloth.models import fsdp2_save # noqa: E402
+
+
+def _merged(
+ self,
+ save_directory,
+ tokenizer = None,
+ save_method = "merged_16bit",
+ push_to_hub = False,
+ token = None,
+ is_main_process = True,
+ state_dict = None,
+):
+ return ("wrote", state_dict)
+
+
+class _Model:
+ def original_model_save_pretrained(
+ self,
+ save_directory,
+ is_main_process = True,
+ state_dict = None,
+ **kwargs,
+ ):
+ return ("wrote", state_dict)
+
+
+def _forwarding(self, *args, **kwargs):
+ return self.original_model_save_pretrained(*args, **kwargs)
+
+
+def no_gather(monkeypatch):
+ monkeypatch.setattr(fsdp2_save, "fsdp2_units", lambda model: [object()])
+ monkeypatch.setattr(fsdp2_save, "fsdp2_gather_active", lambda: False)
+
+ def _refuse(*args, **kwargs):
+ raise AssertionError("a caller's own state_dict must not start a collective gather")
+
+ monkeypatch.setattr(fsdp2_save, "fsdp2_gathered", _refuse)
+
+
+def test_a_positional_state_dict_is_passed_through(no_gather):
+ sd = {"w": torch.zeros(1)}
+ merged = unsloth_save._fsdp2_collective_save(lambda *a: False)(_merged)
+ # (save_directory, tokenizer, save_method, push_to_hub, token, is_main_process, state_dict)
+ assert merged(_Model(), "out", None, "merged_16bit", False, None, True, sd) == ("wrote", sd)
+ assert merged(_Model(), "out", state_dict = sd) == ("wrote", sd)
+ forwarding = unsloth_save._fsdp2_collective_save(lambda *a: False)(_forwarding)
+ # PreTrainedModel.save_pretrained(save_directory, is_main_process, state_dict, ...)
+ assert forwarding(_Model(), "out", True, sd) == ("wrote", sd)
+
+
+def test_peft_saves_gather_the_frozen_tensors_peft_writes():
+ from peft import LoraConfig, get_peft_model
+ from transformers import LlamaConfig, LlamaForCausalLM
+
+ cfg = LlamaConfig(
+ hidden_size = 32,
+ intermediate_size = 64,
+ num_hidden_layers = 1,
+ num_attention_heads = 2,
+ num_key_value_heads = 1,
+ vocab_size = 64,
+ )
+ model = get_peft_model(LlamaForCausalLM(cfg), LoraConfig(r = 2, target_modules = ["q_proj"]))
+ model.add_adapter("other", LoraConfig(r = 2, target_modules = ["q_proj"]))
+ include = unsloth_save._fsdp2_peft_saved_frozen(model, {})
+ names = {n for n, p in model.named_parameters() if not p.requires_grad and include(n, p)}
+ assert any(".other." in n for n in names), names
+ assert any("embed_tokens" in n for n in names) and any("lm_head" in n for n in names), names
+ assert not any("q_proj.base_layer" in n for n in names), names
+ # A biased output head: PEFT saves its bias with the weight.
+ head = model.get_output_embeddings()
+ head.bias = torch.nn.Parameter(torch.zeros(head.out_features), requires_grad = False)
+ include = unsloth_save._fsdp2_peft_saved_frozen(model, {})
+ assert any(n.endswith("lm_head.bias") for n, p in model.named_parameters() if include(n, p))
+ include = unsloth_save._fsdp2_peft_saved_frozen(model, {"save_embedding_layers": False})
+ names = {n for n, p in model.named_parameters() if include(n, p)}
+ assert not any("embed_tokens" in n for n in names), names
+ model.original_model_save_pretrained = model.save_pretrained
+ arguments = unsloth_save._fsdp2_save_arguments(
+ _forwarding, model, ("out", True, None, False), {}
+ )
+ assert arguments["save_embedding_layers"] is False
diff --git a/tests/test_fsdp2_save_gloo.py b/tests/test_fsdp2_save_gloo.py
new file mode 100644
index 00000000000..abce71b0d51
--- /dev/null
+++ b/tests/test_fsdp2_save_gloo.py
@@ -0,0 +1,519 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""Saving under FSDP2 (unsloth/models/fsdp2_save.py), two gloo ranks on CPU.
+
+- `fsdp2_gathered`: rank 0 sees whole parameters (on CPU) equal to an unsharded copy, also after an eval
+ forward left a unit unsharded; the other rank writes nothing; the sharded parameters come back and
+ training continues; a nested call does not gather again.
+- bitsandbytes 4-bit weights come back as Params4bit with their QuantState and the exact packed bytes.
+- trainable parameters FSDP2 leaves unsharded (`ignored_params`) drift apart across ranks unless their
+ gradients are averaged (`reduce_unsharded_trainable_grads`).
+- torch.distributed.checkpoint cannot build a state dict of a bitsandbytes 4-bit model (the Trainer's
+ FSDP2 checkpoint) until `install_dcp_bnb_state_dict`.
+The modules are loaded from their files (importing unsloth needs a GPU).
+"""
+
+import importlib.util
+import os
+import socket
+import sys
+from pathlib import Path
+
+import pytest
+import torch
+
+_ROOT = Path(__file__).resolve().parents[1]
+
+
+def _load(name):
+ path = _ROOT / "unsloth" / "models" / f"{name}.py"
+ spec = importlib.util.spec_from_file_location(f"_unsloth_{name}_under_test", path)
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+ return module
+
+
+fsdp_mod = pytest.importorskip("torch.distributed.fsdp")
+if not hasattr(fsdp_mod, "fully_shard"):
+ pytest.skip("torch has no FSDP2 fully_shard", allow_module_level = True)
+if (
+ sys.platform == "win32"
+ or not torch.distributed.is_available()
+ or not torch.distributed.is_gloo_available()
+):
+ pytest.skip("needs torch.distributed with gloo", allow_module_level = True)
+
+N_LAYERS = 3
+
+
+def _free_port():
+ with socket.socket() as s:
+ s.bind(("127.0.0.1", 0))
+ return s.getsockname()[1]
+
+
+class _Block(torch.nn.Module):
+ def __init__(self, seed):
+ super().__init__()
+ torch.manual_seed(seed)
+ # Odd sizes so the shards are uneven.
+ self.base = torch.nn.Linear(37, 37, bias = False).requires_grad_(False)
+ self.lora_A = torch.nn.Linear(37, 4, bias = False)
+ self.lora_B = torch.nn.Linear(4, 37, bias = False)
+ torch.nn.init.normal_(self.lora_B.weight, std = 0.1)
+
+ def forward(self, x):
+ return x + torch.tanh(self.base(x) + self.lora_B(self.lora_A(x)))
+
+
+class _Model(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.layers = torch.nn.ModuleList([_Block(10 * i) for i in range(N_LAYERS)])
+ torch.manual_seed(99)
+ self.head = torch.nn.Linear(37, 5, bias = True)
+
+ def forward(self, x):
+ for layer in self.layers:
+ x = layer(x)
+ return self.head(x)
+
+
+def _shard(
+ model,
+ world_size,
+ ignored = None,
+):
+ from torch.distributed.device_mesh import init_device_mesh
+ from torch.distributed.fsdp import fully_shard
+
+ mesh = init_device_mesh("cpu", (world_size,))
+ kwargs = {"ignored_params": ignored} if ignored else {}
+ for layer in model.layers:
+ fully_shard(layer, mesh = mesh, **kwargs)
+ fully_shard(model, mesh = mesh, **kwargs)
+ # What Unsloth does to the last block: it stays unsharded after a forward.
+ model.layers[-1].set_reshard_after_forward(False)
+ return model
+
+
+def _is_dtensor(t):
+ from torch.distributed.tensor import DTensor
+ return isinstance(t, DTensor)
+
+
+def _case_gather(rank, world_size, m):
+ model = _shard(_Model(), world_size)
+ reference = _Model()
+ x = torch.randn(6, 37, generator = torch.Generator().manual_seed(rank))
+ with torch.no_grad():
+ model(x)
+ # The eval forward left the last block unsharded: a plain (non DTensor) parameter.
+ unsharded_after_eval = sum(not _is_dtensor(p) for p in model.parameters())
+ out = {"unsharded_after_eval": unsharded_after_eval}
+
+ with m.fsdp2_gathered(model, trainable_only = True) as writer:
+ out["trainable_writer"] = writer
+ out["trainable_only"] = sorted(
+ {
+ ("plain" if not _is_dtensor(p) else "dtensor", p.requires_grad)
+ for p in model.parameters()
+ }
+ )
+
+ # A frozen parameter the save still writes (an inactive adapter, an embedding).
+ with m.fsdp2_gathered(model, trainable_only = True, include = lambda name, p: True) as writer:
+ out["included"] = sorted(
+ {
+ ("plain" if not _is_dtensor(p) else "dtensor", p.requires_grad)
+ for p in model.parameters()
+ }
+ )
+
+ with m.fsdp2_gathered(model) as writer:
+ out["writer"] = writer
+ with m.fsdp2_gathered(model) as nested:
+ out["nested"] = nested
+ if writer:
+ ref = dict(reference.named_parameters())
+ got = dict(model.named_parameters())
+ out["names_match"] = sorted(ref) == sorted(got)
+ out["plain_cpu"] = all(
+ not _is_dtensor(p) and p.device.type == "cpu" for p in got.values()
+ )
+ out["exact"] = all(torch.equal(got[k].detach(), ref[k].detach()) for k in ref)
+ out["state_dict_keys"] = sorted(model.state_dict()) == sorted(reference.state_dict())
+ out["requires_grad_kept"] = all(
+ got[k].requires_grad == ref[k].requires_grad for k in ref
+ )
+ else:
+ out["all_dtensor_on_other_rank"] = all(_is_dtensor(p) for p in model.parameters())
+ out["all_dtensor_after"] = all(_is_dtensor(p) for p in model.parameters())
+
+ # Training continues on the restored shards and matches the unsharded model step for step.
+ opt = torch.optim.SGD([p for p in model.parameters() if p.requires_grad], lr = 0.1)
+ ref_opt = torch.optim.SGD([p for p in reference.parameters() if p.requires_grad], lr = 0.1)
+ import torch.distributed as dist
+
+ xs = [
+ torch.randn(6, 37, generator = torch.Generator().manual_seed(100 + r))
+ for r in range(world_size)
+ ]
+ model(xs[rank]).square().mean().backward()
+ opt.step()
+ sum(reference(xr).square().mean() for xr in xs).div(world_size).backward()
+ ref_opt.step()
+ diffs = []
+ for (n, p), (_, q) in zip(model.named_parameters(), reference.named_parameters()):
+ full = p.full_tensor() if _is_dtensor(p) else p
+ diffs.append(float((full.detach() - q.detach()).abs().max()))
+ out["step_max_diff"] = max(diffs)
+ return out
+
+
+def _case_gather_4bit(rank, world_size, m):
+ bnb = pytest.importorskip("bitsandbytes")
+ q = _load("fsdp2_qlora")
+ from torch.distributed.device_mesh import init_device_mesh
+ from torch.distributed.fsdp import fully_shard
+
+ def lin(seed):
+ torch.manual_seed(seed)
+ layer = bnb.nn.Linear4bit(
+ 128,
+ 96,
+ bias = False,
+ compute_dtype = torch.bfloat16,
+ quant_type = "nf4",
+ quant_storage = torch.bfloat16,
+ )
+ layer.weight = bnb.nn.Params4bit(
+ torch.randn(96, 128) * 0.05,
+ requires_grad = False,
+ quant_type = "nf4",
+ quant_storage = torch.bfloat16,
+ blocksize = 64,
+ module = layer,
+ )
+ return layer.to("cpu")
+
+ model = torch.nn.Sequential(lin(0), lin(1))
+ before = [
+ (
+ l.weight.data.view(torch.uint8).clone(),
+ bnb.functional.dequantize_4bit(l.weight.data, l.weight.quant_state),
+ )
+ for l in model
+ ]
+ info = q.prepare_fsdp2_qlora(model)
+ mesh = init_device_mesh("cpu", (world_size,))
+ for layer in model:
+ fully_shard(layer, mesh = mesh)
+ fully_shard(model, mesh = mesh)
+ out = {"patched": info["patched"]}
+ with m.fsdp2_gathered(model) as writer:
+ if writer:
+ checks = []
+ for layer, (raw, deq) in zip(model, before):
+ w = layer.weight
+ checks.append(
+ dict(
+ params4bit = type(w).__name__ == "Params4bit",
+ quant_state = w.quant_state is not None,
+ bytes_equal = torch.equal(w.data.view(torch.uint8), raw),
+ dequant_equal = torch.equal(
+ bnb.functional.dequantize_4bit(w.data, w.quant_state), deq
+ ),
+ module = w.module is layer,
+ )
+ )
+ out["checks"] = checks
+ # bitsandbytes' own state dict (what save_pretrained writes) has the QuantState entries.
+ out["absmax_keys"] = sum(k.endswith("weight.absmax") for k in model.state_dict())
+ out["dtensor_after"] = all(_is_dtensor(p) for p in model.parameters())
+ return out
+
+
+def _case_replicated(rank, world_size, m, hooked):
+ import torch.distributed as dist
+
+ model = _Model()
+ reference = _Model()
+ head = [model.head.weight, model.head.bias]
+ _shard(model, world_size, ignored = set(head))
+ names = m.reduce_unsharded_trainable_grads(model) if hooked else []
+ opt = torch.optim.SGD([p for p in model.parameters() if p.requires_grad], lr = 0.5)
+ ref_opt = torch.optim.SGD([p for p in reference.parameters() if p.requires_grad], lr = 0.5)
+ for step in range(3):
+ xs = [
+ torch.randn(6, 37, generator = torch.Generator().manual_seed(10 * step + r))
+ for r in range(world_size)
+ ]
+ opt.zero_grad()
+ model(xs[rank]).square().mean().backward()
+ opt.step()
+ ref_opt.zero_grad()
+ sum(reference(xr).square().mean() for xr in xs).div(world_size).backward()
+ ref_opt.step()
+ w = model.head.weight.detach().clone()
+ gathered = [torch.empty_like(w) for _ in range(world_size)]
+ dist.all_gather(gathered, w)
+ return dict(
+ names = names,
+ head_is_dtensor = _is_dtensor(model.head.weight),
+ rank_spread = max(float((g - gathered[0]).abs().max()) for g in gathered),
+ vs_reference = float((w - reference.head.weight.detach()).abs().max()),
+ )
+
+
+def _case_dcp_bnb(rank, world_size, m):
+ bnb = pytest.importorskip("bitsandbytes")
+ q = _load("fsdp2_qlora")
+ from torch.distributed.checkpoint.state_dict import StateDictOptions
+ from torch.distributed.device_mesh import init_device_mesh
+ from torch.distributed.fsdp import fully_shard
+ import torch.distributed.checkpoint.state_dict as dcp
+
+ torch.manual_seed(0)
+ layer = bnb.nn.Linear4bit(
+ 128,
+ 64,
+ bias = False,
+ compute_dtype = torch.bfloat16,
+ quant_type = "nf4",
+ quant_storage = torch.bfloat16,
+ )
+ layer.weight = bnb.nn.Params4bit(
+ torch.randn(64, 128) * 0.05,
+ requires_grad = False,
+ quant_type = "nf4",
+ quant_storage = torch.bfloat16,
+ blocksize = 64,
+ module = layer,
+ )
+ model = torch.nn.Sequential(layer.to("cpu"), torch.nn.Linear(64, 8))
+ q.prepare_fsdp2_qlora(model)
+ fully_shard(model, mesh = init_device_mesh("cpu", (world_size,)))
+ # What accelerate's save_fsdp_model / load_fsdp_model ask for on an adapter checkpoint.
+ options = StateDictOptions(full_state_dict = True, cpu_offload = True, ignore_frozen_params = True)
+ out = {}
+ try:
+ dcp.get_model_state_dict(model, options = options)
+ out["before"] = "ok"
+ except AttributeError as error:
+ out["before"] = f"AttributeError: {error}"
+ out["installed"] = m.install_dcp_bnb_state_dict()
+ # What Trainer.save_model asks for through accelerate's get_state_dict: a full state dict, frozen
+ # weights included. bitsandbytes' extra keys cannot be resolved on a sharded weight either way.
+ try:
+ full = dcp.get_model_state_dict(
+ model, options = StateDictOptions(full_state_dict = True, cpu_offload = True)
+ )
+ out["full_sharded_keys"] = sorted(full)
+ except AttributeError as error:
+ out["full_sharded_keys"] = f"AttributeError: {error}"
+ sd = dcp.get_model_state_dict(model, options = options)
+ out["after_keys"] = sorted(sd)
+ expected = {k: v.clone() for k, v in sd.items()}
+ with torch.no_grad():
+ model[1].weight.zero_()
+ dcp.set_model_state_dict(
+ model,
+ sd if rank == 0 else {},
+ options = StateDictOptions(full_state_dict = True, broadcast_from_rank0 = True, strict = False),
+ )
+ restored = model[1].weight.full_tensor()
+ out["roundtrip"] = rank != 0 or torch.equal(restored, expected["1.weight"])
+ # A full DCP state dict of an unsharded 4-bit model keeps what dequantizes it.
+ plain = torch.nn.Sequential(
+ bnb.nn.Linear4bit(128, 64, bias = False, compute_dtype = torch.bfloat16, quant_type = "nf4")
+ )
+ plain[0].weight = bnb.nn.Params4bit(
+ torch.randn(64, 128) * 0.05,
+ requires_grad = False,
+ quant_type = "nf4",
+ blocksize = 64,
+ module = plain[0],
+ )
+ plain = plain.to("cpu")
+ out["full_keys"] = sorted(
+ dcp.get_model_state_dict(plain, options = StateDictOptions(cpu_offload = True))
+ )
+ # accelerate's adapter-only path drops it (and no longer fails).
+ import accelerate.utils.fsdp_utils as fsdp_utils
+
+ from peft import LoraConfig, get_peft_model
+
+ class _Net(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.q = bnb.nn.Linear4bit(
+ 128,
+ 64,
+ bias = False,
+ compute_dtype = torch.bfloat16,
+ quant_type = "nf4",
+ quant_storage = torch.bfloat16,
+ )
+ self.q.weight = bnb.nn.Params4bit(
+ torch.randn(64, 128) * 0.05,
+ requires_grad = False,
+ quant_type = "nf4",
+ quant_storage = torch.bfloat16,
+ blocksize = 64,
+ module = self.q,
+ )
+
+ def forward(self, x):
+ return self.q(x)
+
+ peft_model = get_peft_model(_Net().to("cpu"), LoraConfig(r = 4, target_modules = ["q"]))
+ q.prepare_fsdp2_qlora(peft_model)
+ fully_shard(peft_model, mesh = init_device_mesh("cpu", (world_size,)))
+ sd = fsdp_utils._get_model_state_dict(
+ peft_model,
+ adapter_only = True,
+ sd_options = StateDictOptions(full_state_dict = True, cpu_offload = True),
+ )
+ out["adapter_keys"] = sorted(k.split("q.", 1)[-1] for k in sd)
+ return out
+
+
+def _case_slow_writer(rank, world_size, m):
+ """rank 0's write (a merged or quantized export, a Hub upload) outlasts the collective timeout."""
+ import time
+
+ import torch.distributed as dist
+
+ model = _shard(_Model(), world_size)
+ with m.fsdp2_gathered(model) as writer:
+ if writer:
+ time.sleep(10)
+ dist.barrier()
+ return {"ok": True}
+
+
+def _worker(rank, world_size, port, case, queue):
+ try:
+ os.environ.update(
+ MASTER_ADDR = "127.0.0.1",
+ MASTER_PORT = str(port),
+ RANK = str(rank),
+ WORLD_SIZE = str(world_size),
+ )
+ import torch.distributed as dist
+
+ import datetime
+
+ timeout = datetime.timedelta(seconds = 4) if case == "slow_writer" else None
+ dist.init_process_group(
+ "gloo", rank = rank, world_size = world_size, **({"timeout": timeout} if timeout else {})
+ )
+ m = _load("fsdp2_save")
+ if case == "slow_writer":
+ result = _case_slow_writer(rank, world_size, m)
+ elif case == "gather":
+ result = _case_gather(rank, world_size, m)
+ elif case == "gather_4bit":
+ result = _case_gather_4bit(rank, world_size, m)
+ elif case.startswith("replicated"):
+ result = _case_replicated(rank, world_size, m, hooked = case.endswith("hooked"))
+ else:
+ result = _case_dcp_bnb(rank, world_size, m)
+ queue.put((rank, result))
+ dist.destroy_process_group()
+ except BaseException as error:
+ import traceback
+ queue.put((rank, {"error": f"{type(error).__name__}: {error}\n{traceback.format_exc()}"}))
+
+
+def _run(case, world_size = 2):
+ import torch.multiprocessing as mp
+
+ ctx = mp.get_context("spawn")
+ queue = ctx.Queue()
+ port = _free_port()
+ procs = [
+ ctx.Process(target = _worker, args = (r, world_size, port, case, queue))
+ for r in range(world_size)
+ ]
+ for p in procs:
+ p.start()
+ results = {}
+ for _ in procs:
+ rank, result = queue.get(timeout = 240)
+ results[rank] = result
+ for p in procs:
+ p.join(timeout = 60)
+ for r in results.values():
+ assert "error" not in r, r.get("error")
+ return results
+
+
+def test_gather_gives_rank0_the_full_model_and_restores_shards():
+ results = _run("gather")
+ r0, r1 = results[0], results[1]
+ assert r0["unsharded_after_eval"] > 0, r0
+ assert r0["trainable_writer"] is True and r1["trainable_writer"] is False
+ # trainable_only: adapters plain on rank 0, frozen weights stay sharded.
+ assert r0["trainable_only"] == [("dtensor", False), ("plain", True)], r0
+ assert r1["trainable_only"] == [("dtensor", False), ("dtensor", True)], r1
+ assert r0["included"] == [("plain", False), ("plain", True)], r0
+ assert r0["writer"] is True and r1["writer"] is False
+ assert r0["nested"] is True and r1["nested"] is False
+ assert r0["names_match"] and r0["plain_cpu"] and r0["exact"], r0
+ assert r0["state_dict_keys"] and r0["requires_grad_kept"], r0
+ assert r1["all_dtensor_on_other_rank"], r1
+ for r in (r0, r1):
+ assert r["all_dtensor_after"], r
+ assert r["step_max_diff"] < 1e-6, r
+
+
+def test_gather_rebuilds_params4bit_with_exact_bytes():
+ pytest.importorskip("bitsandbytes")
+ results = _run("gather_4bit")
+ r0 = results[0]
+ assert r0["patched"], r0
+ assert len(r0["checks"]) == 2 and all(all(c.values()) for c in r0["checks"]), r0["checks"]
+ assert r0["absmax_keys"] == 2, r0
+ assert all(r["dtensor_after"] for r in results.values()), results
+
+
+ not hasattr(torch.distributed.fsdp, "fully_shard")
+ or "ignored_params"
+ not in __import__("inspect").signature(torch.distributed.fsdp.fully_shard).parameters,
+ reason = "fully_shard has no ignored_params",
+)
+def test_unsharded_trainable_params_need_their_grads_averaged():
+ base = _run("replicated")
+ assert not base[0]["head_is_dtensor"], base
+ # Control: no gradient reduction for the ignored head, so the replicas drift apart.
+ assert base[0]["rank_spread"] > 1e-3, base
+ hooked = _run("replicated_hooked")
+ assert hooked[0]["names"] == ["head.weight", "head.bias"], hooked
+ assert hooked[0]["rank_spread"] == 0.0, hooked
+ for r in hooked.values():
+ assert r["vs_reference"] < 1e-6, r
+
+
+def test_dcp_state_dict_of_bitsandbytes_4bit():
+ pytest.importorskip("bitsandbytes")
+ results = _run("dcp_bnb")
+ r = results[0]
+ # A sharded 4-bit weight is a DTensor, which has no `absmax` attribute for `weight.absmax`.
+ assert r["before"].startswith("AttributeError") and "absmax" in r["before"], r
+ assert r["installed"], r
+ assert r["after_keys"] == ["1.bias", "1.weight"], r
+ assert r["full_sharded_keys"] == ["0.weight", "1.bias", "1.weight"], r["full_sharded_keys"]
+ assert all(x["roundtrip"] for x in results.values()), results
+ assert "0.weight.absmax" in r["full_keys"], r["full_keys"]
+ assert r["adapter_keys"] == ["lora_A.default.weight", "lora_B.default.weight"], r[
+ "adapter_keys"
+ ]
+
+
+def test_a_slow_writer_does_not_time_out_the_other_ranks():
+ results = _run("slow_writer")
+ assert all(r["ok"] for r in results.values()), results
diff --git a/unsloth/models/fsdp2.py b/unsloth/models/fsdp2.py
index f4a9cf7874e..d86c79a1b3e 100644
--- a/unsloth/models/fsdp2.py
+++ b/unsloth/models/fsdp2.py
@@ -409,6 +409,14 @@ def after_fsdp2_wrap(accelerator, model):
model, blocks, parents, FSDPModule
)
_gate_fused_lora(accelerator, model, parents, FSDPModule, blocks)
+ from .fsdp2_save import reduce_unsharded_trainable_grads
+
+ replicated = reduce_unsharded_trainable_grads(model)
+ if replicated:
+ _print_rank0(
+ f"Unsloth: FSDP2 leaves {len(replicated)} trainable parameters unsharded (e.g. "
+ f"{replicated[0]}); their gradients are averaged over ranks like DDP."
+ )
def install_fsdp2_prepare_hook():
@@ -553,6 +561,9 @@ def install_fsdp2_hooks():
install_fsdp2_prepare_hook()
install_fsdp2_no_sync()
install_fsdp2_peft_full_state_dict()
+ from .fsdp2_save import install_dcp_bnb_state_dict
+
+ install_dcp_bnb_state_dict()
def fsdp2_trainer_setup(trainer):
@@ -564,6 +575,9 @@ def fsdp2_trainer_setup(trainer):
install_fsdp2_prepare_hook()
install_fsdp2_no_sync()
install_fsdp2_peft_full_state_dict()
+ from .fsdp2_save import install_dcp_bnb_state_dict
+
+ install_dcp_bnb_state_dict()
if os.environ.get("UNSLOTH_FSDP2_FUSED_LORA", "1").strip().lower() in _FALSY:
return False, "UNSLOTH_FSDP2_FUSED_LORA=0"
if getattr(plugin, "auto_wrap_policy", None) is None:
diff --git a/unsloth/models/fsdp2_save.py b/unsloth/models/fsdp2_save.py
new file mode 100644
index 00000000000..6b7cfca52de
--- /dev/null
+++ b/unsloth/models/fsdp2_save.py
@@ -0,0 +1,372 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""Saving an FSDP2-sharded model. Every rank must call the save: the shards are all-gathered one
+parameter at a time, rank 0 swaps the full tensors (on CPU) into the modules, runs the ordinary
+single-GPU save code, and puts the sharded parameters back. bitsandbytes 4-bit weights are gathered
+as raw bytes and rebuilt as Params4bit with their QuantState, so a merge can still dequantize them.
+"""
+
+__all__ = [
+ "fsdp2_units",
+ "fsdp2_gathered",
+ "fsdp2_gather_active",
+ "install_dcp_bnb_state_dict",
+ "reduce_unsharded_trainable_grads",
+]
+
+import contextlib
+import datetime
+import functools
+import os
+
+import torch
+
+# Writer flag of the gather in progress: a nested save (save_pretrained_merged -> save_pretrained,
+# or on a submodule) must neither gather again nor reshard what rank 0 swapped in.
+_ACTIVE = []
+
+
+def _fsdp_module_class():
+ try:
+ from torch.distributed.fsdp import FSDPModule
+ return FSDPModule
+ except ImportError:
+ pass
+ try:
+ from torch.distributed._composable.fsdp import FSDPModule
+ return FSDPModule
+ except ImportError:
+ return None
+
+
+def _dtensor_class():
+ try:
+ from torch.distributed.tensor import DTensor
+ return DTensor
+ except ImportError:
+ try:
+ from torch.distributed._tensor import DTensor
+ return DTensor
+ except ImportError:
+ return None
+
+
+def fsdp2_units(model):
+ FSDPModule = _fsdp_module_class()
+ if FSDPModule is None or not isinstance(model, torch.nn.Module):
+ return []
+ return [m for m in model.modules() if isinstance(m, FSDPModule)]
+
+
+def _dist():
+ import torch.distributed as dist
+ if dist.is_available() and dist.is_initialized():
+ return dist
+ return None
+
+
+_SAVE_COUNTER = [0]
+
+
+def _save_timeout():
+ try:
+ return datetime.timedelta(
+ seconds = float(os.environ.get("UNSLOTH_FSDP2_SAVE_TIMEOUT", 7 * 86400))
+ )
+ except ValueError:
+ return datetime.timedelta(days = 7)
+
+
+def _wait_for_writer(dist, writer):
+ """The other ranks wait for rank 0's write, so no later collective overtakes it. A store key, not
+ a barrier: a merged or quantized export (and a Hub upload) can outlast the process group's
+ collective timeout, and NCCL ranks blocked in a barrier would abort before rank 0 arrives."""
+ _SAVE_COUNTER[0] += 1
+ key = f"unsloth_fsdp2_save_{_SAVE_COUNTER[0]}"
+ try:
+ store = dist.distributed_c10d._get_default_store()
+ except Exception:
+ store = None
+ if store is None:
+ dist.barrier()
+ return
+ if writer:
+ store.set(key, "done")
+ else:
+ store.wait([key], _save_timeout())
+
+
+def _gather_full(param, DTensor):
+ """All ranks: the full tensor of a sharded parameter. Packed 4-bit bytes travel as uint8, so no
+ float copy kernel can rewrite NaN bit patterns inside them."""
+ local = param.to_local()
+ mesh_type = param.device_mesh.device_type
+ if local.device.type == "cpu" and mesh_type != "cpu":
+ # CPUOffloadPolicy keeps shards on the host; NCCL can only gather device tensors.
+ device = torch.device(mesh_type, getattr(torch, mesh_type).current_device())
+ local = local.to(device)
+ if getattr(param, "quant_state", None) is None or param.dtype == torch.uint8:
+ return DTensor.from_local(
+ local, param.device_mesh, param.placements, shape = param.shape, stride = param.stride()
+ ).full_tensor()
+ elif getattr(param, "quant_state", None) is None or param.dtype == torch.uint8:
+ return param.full_tensor()
+ as_bytes = local.view(torch.uint8)
+ shape = list(param.shape)
+ shape[-1] = shape[-1] * param.element_size()
+ stride = []
+ acc = 1
+ for size in reversed(shape):
+ stride.insert(0, acc)
+ acc *= size
+ gathered = DTensor.from_local(
+ as_bytes,
+ param.device_mesh,
+ param.placements,
+ shape = torch.Size(shape),
+ stride = tuple(stride),
+ ).full_tensor()
+ return gathered.view(param.dtype)
+
+
+def _rebuild(param, full, module):
+ """A full parameter that looks like the one FSDP2 sharded (Params4bit keeps its QuantState)."""
+ quant_state = getattr(param, "quant_state", None)
+ if quant_state is None:
+ return torch.nn.Parameter(full, requires_grad = param.requires_grad)
+ from bitsandbytes.nn import Params4bit
+
+ new = torch.Tensor._make_subclass(Params4bit, full, False)
+ for key in (
+ "quant_state",
+ "blocksize",
+ "compress_statistics",
+ "quant_type",
+ "quant_storage",
+ "bnb_quantized",
+ ):
+ if hasattr(param, key):
+ setattr(new, key, getattr(param, key))
+ new.module = module
+ return new
+
+
+def fsdp2_gathered(
+ model,
+ trainable_only = False,
+ offload_to_cpu = True,
+ include = None,
+):
+ """Yields None for a model FSDP2 does not shard (nothing happens). Otherwise every rank must
+ enter; rank 0 sees whole parameters (and yields True), the other ranks yield False and should
+ not write. `trainable_only` gathers just the trainable parameters (enough for an adapter save),
+ plus those `include(name, param)` selects (frozen ones the save still writes).
+ Packed 4-bit weights stay on their device: the merge dequantizes them there."""
+ if _ACTIVE:
+ yield _ACTIVE[-1]
+ return
+ units = fsdp2_units(model)
+ DTensor = _dtensor_class()
+ if not units or DTensor is None:
+ yield None
+ return
+ dist = _dist()
+ rank = dist.get_rank() if dist is not None else 0
+ writer = rank == 0
+ for unit in units:
+ # An eval forward leaves the last block and the lm_head unit gathered; make every
+ # parameter a DTensor again so all ranks gather the same list.
+ if hasattr(unit, "reshard"):
+ unit.reshard()
+ owners = {}
+ for module in model.modules():
+ for name, param in module._parameters.items():
+ if param is not None:
+ owners.setdefault(id(param), []).append((module, name))
+ replaced = {}
+ seen = set()
+ pushed = False
+ try:
+ for name, param in model.named_parameters():
+ if id(param) in seen:
+ continue
+ seen.add(id(param))
+ if not isinstance(param, DTensor):
+ continue
+ if (
+ trainable_only
+ and not param.requires_grad
+ and not (include and include(name, param))
+ ):
+ continue
+ full = _gather_full(param, DTensor)
+ if not writer:
+ del full
+ continue
+ module = owners[id(param)][0][0]
+ if offload_to_cpu and getattr(param, "quant_state", None) is None:
+ full = full.cpu()
+ replaced[id(param)] = (param, _rebuild(param, full.detach(), module))
+ if writer:
+ for key, (old, new) in replaced.items():
+ for module, name in owners[key]:
+ module._parameters[name] = new
+ _ACTIVE.append(writer)
+ pushed = True
+ yield writer
+ finally:
+ if pushed:
+ _ACTIVE.pop()
+ for key, (old, new) in replaced.items():
+ for module, name in owners[key]:
+ module._parameters[name] = old
+ replaced.clear()
+ if dist is not None:
+ _wait_for_writer(dist, writer)
+
+
+def fsdp2_gather_active():
+ return bool(_ACTIVE)
+
+
+_DCP_DEPTH = [0]
+_ADAPTER_ONLY = [0]
+_DCP_FUNCTIONS = (
+ "get_model_state_dict",
+ "set_model_state_dict",
+ "get_state_dict",
+ "set_state_dict",
+ "get_optimizer_state_dict",
+ "set_optimizer_state_dict",
+)
+
+
+def _drops_frozen_weights(kwargs):
+ """Only adapter-only state dicts (accelerate's `adapter_only` path, or an explicit
+ `ignore_frozen_params`) may skip the QuantState: they never keep the frozen 4-bit weights. A full
+ DCP checkpoint still needs it to dequantize them."""
+ if _ADAPTER_ONLY[0]:
+ return True
+ return bool(getattr(kwargs.get("options", None), "ignore_frozen_params", False))
+
+
+def _has_sharded_4bit(args, kwargs):
+ """A Linear4bit whose weight FSDP2 made a DTensor: DCP cannot resolve `weight.absmax` on it at all,
+ so keeping bitsandbytes' entries only raises (Trainer.save_model -> accelerate get_state_dict)."""
+ model = args[0] if args else kwargs.get("model", None)
+ DTensor = _dtensor_class()
+ if DTensor is None or not isinstance(model, torch.nn.Module):
+ return False
+ try:
+ from bitsandbytes.nn import Linear4bit
+ except Exception:
+ return False
+ return any(
+ isinstance(m, Linear4bit) and isinstance(getattr(m, "weight", None), DTensor)
+ for m in model.modules()
+ )
+
+
+def _wrap_accelerate_adapter_only():
+ try:
+ import accelerate.utils.fsdp_utils as fsdp_utils
+ except Exception:
+ return
+ for name in ("_get_model_state_dict", "_set_model_state_dict"):
+ fn = getattr(fsdp_utils, name, None)
+ if fn is None or getattr(fn, "_unsloth_adapter_only_flag", False):
+ continue
+
+ @functools.wraps(fn)
+ def wrapper(
+ *args,
+ _fn = fn,
+ **kwargs,
+ ):
+ if not kwargs.get("adapter_only", False):
+ return _fn(*args, **kwargs)
+ _ADAPTER_ONLY[0] += 1
+ try:
+ return _fn(*args, **kwargs)
+ finally:
+ _ADAPTER_ONLY[0] -= 1
+
+ wrapper._unsloth_adapter_only_flag = True
+ setattr(fsdp_utils, name, wrapper)
+
+
+def install_dcp_bnb_state_dict():
+ """torch.distributed.checkpoint resolves every state dict key as an attribute path, and
+ bitsandbytes' `weight.absmax` / `weight.quant_state.bitsandbytes__nf4` entries are not one
+ (AttributeError in `_get_fqns`). That breaks every FSDP2 Trainer adapter checkpoint of a 4-bit
+ model. Inside adapter-only DCP calls (`_drops_frozen_weights`) Linear4bit reports just its weight
+ and bias; the QuantState is unchanged in memory. Other DCP calls keep bitsandbytes' entries."""
+ try:
+ import torch.distributed.checkpoint.state_dict as dcp_state_dict
+ from bitsandbytes.nn import Linear4bit
+ except Exception:
+ return False
+ _wrap_accelerate_adapter_only()
+ original_save = Linear4bit._save_to_state_dict
+ if getattr(original_save, "_unsloth_fsdp2_patched", False):
+ return True
+
+ @functools.wraps(original_save)
+ def _save_to_state_dict(self, destination, prefix, keep_vars):
+ if _DCP_DEPTH[0]:
+ return torch.nn.Module._save_to_state_dict(self, destination, prefix, keep_vars)
+ return original_save(self, destination, prefix, keep_vars)
+
+ _save_to_state_dict._unsloth_fsdp2_patched = True
+ Linear4bit._save_to_state_dict = _save_to_state_dict
+
+ def _wrap(fn):
+ @functools.wraps(fn)
+ def wrapper(*args, **kwargs):
+ if not (_drops_frozen_weights(kwargs) or _has_sharded_4bit(args, kwargs)):
+ return fn(*args, **kwargs)
+ _DCP_DEPTH[0] += 1
+ try:
+ return fn(*args, **kwargs)
+ finally:
+ _DCP_DEPTH[0] -= 1
+
+ wrapper._unsloth_fsdp2_patched = True
+ return wrapper
+
+ for name in _DCP_FUNCTIONS:
+ fn = getattr(dcp_state_dict, name, None)
+ if fn is not None and not getattr(fn, "_unsloth_fsdp2_patched", False):
+ setattr(dcp_state_dict, name, _wrap(fn))
+ return True
+
+
+def reduce_unsharded_trainable_grads(model):
+ """Trainable parameters FSDP2 does not manage (e.g. in `ignored_modules`) get no gradient
+ reduction, so every rank would step on its own gradient and the replicas drift apart. Average
+ their gradient over ranks as it is produced, like DDP. Returns the parameter names."""
+ dist = _dist()
+ DTensor = _dtensor_class()
+ if dist is None or DTensor is None or dist.get_world_size() == 1:
+ return []
+ names = []
+ seen = set()
+ for name, param in model.named_parameters():
+ if id(param) in seen or not param.requires_grad or isinstance(param, DTensor):
+ continue
+ seen.add(id(param))
+ if getattr(param, "_unsloth_grad_all_reduce", False):
+ continue
+ world = dist.get_world_size()
+
+ def _average(grad, world = world):
+ grad = grad.clone()
+ dist.all_reduce(grad)
+ return grad.div_(world)
+
+ param.register_hook(_average)
+ param._unsloth_grad_all_reduce = True
+ names.append(name)
+ return names
diff --git a/unsloth/save.py b/unsloth/save.py
index f54f25eb52c..a1acad4caea 100644
--- a/unsloth/save.py
+++ b/unsloth/save.py
@@ -6097,6 +6097,107 @@ def unsloth_generic_save(
return
+def _fsdp2_save_arguments(fn, self, args, kwargs):
+ """The call's arguments by name, bound against `fn`, or against the wrapped original
+ save_pretrained when `fn` only forwards *args. {} when neither binds."""
+ import inspect
+
+ targets = [(fn, (self,) + tuple(args))]
+ original = getattr(self, "original_model_save_pretrained", None)
+ if callable(original):
+ targets.append((original, tuple(args)))
+ for target, positional in targets:
+ try:
+ signature = inspect.signature(target)
+ bound = signature.bind_partial(*positional, **kwargs)
+ except (TypeError, ValueError):
+ continue
+ if any(p.kind is p.VAR_POSITIONAL for p in signature.parameters.values()):
+ continue
+ arguments = dict(bound.arguments)
+ for p in signature.parameters.values():
+ if p.kind is p.VAR_KEYWORD:
+ arguments.update(arguments.pop(p.name, {}))
+ return arguments
+ return dict(kwargs)
+
+
+def _fsdp2_peft_saved_frozen(model, arguments):
+ """Frozen parameters a PEFT adapter save still writes: every adapter's (inactive ones are frozen,
+ `selected_adapters` defaults to all), `modules_to_save` copies, and the embeddings unless
+ `save_embedding_layers = False`."""
+ adapters = set(getattr(model, "peft_config", None) or {})
+ embeddings = set()
+ if arguments.get("save_embedding_layers", "auto") is not False:
+ for getter in ("get_input_embeddings", "get_output_embeddings"):
+ try:
+ module = getattr(model, getter)()
+ except Exception:
+ module = None
+ # PEFT copies every entry under the embedding module (a decoder bias too), not just weight.
+ if module is not None:
+ embeddings.update(id(p) for p in module.parameters())
+
+ def include(name, param):
+ parts = name.split(".")
+ return (
+ id(param) in embeddings
+ or "modules_to_save" in parts
+ or any(a in parts for a in adapters)
+ )
+
+ return include
+
+
+def _fsdp2_collective_save(trainable_only):
+ """Under FSDP2 every rank calls the save: the shards are gathered, rank 0 writes, the others
+ return None. `trainable_only(self, args, kwargs)` says whether the adapter alone is enough.
+ A model FSDP2 does not shard, or a call that brings its own `state_dict`, is passed through."""
+
+ def decorate(fn):
+ @functools.wraps(fn)
+ def wrapper(self, *args, **kwargs):
+ from .models.fsdp2_save import fsdp2_units, fsdp2_gathered, fsdp2_gather_active
+
+ if fsdp2_gather_active() or not fsdp2_units(self):
+ return fn(self, *args, **kwargs)
+ arguments = _fsdp2_save_arguments(fn, self, args, kwargs)
+ if arguments.get("state_dict") is not None:
+ return fn(self, *args, **kwargs)
+ only = trainable_only(self, args, kwargs)
+ include = _fsdp2_peft_saved_frozen(self, arguments) if only else None
+ if _fsdp2_rank() == 0:
+ print(
+ "Unsloth: FSDP2 save: gathering the "
+ + ("trainable" if only else "sharded")
+ + " weights to rank 0, which writes. Every rank must make this call."
+ )
+ with fsdp2_gathered(self, trainable_only = only, include = include) as writer:
+ if writer is False:
+ return None
+ return fn(self, *args, **kwargs)
+
+ return wrapper
+
+ return decorate
+
+
+def _fsdp2_rank():
+ import torch.distributed as dist
+ return dist.get_rank() if dist.is_available() and dist.is_initialized() else 0
+
+
+def _merged_save_method(args, kwargs):
+ if "save_method" in kwargs:
+ return kwargs["save_method"]
+ # (save_directory, tokenizer, save_method, ...)
+ return args[2] if len(args) > 2 else "merged_16bit"
+
+
+@_fsdp2_collective_save(
+ lambda self, args, kwargs: isinstance(self, PeftModel)
+ and _is_adapter_save_method(_merged_save_method(args, kwargs))
+)
def unsloth_generic_save_pretrained_merged(
self,
save_directory: Union[str, os.PathLike],
@@ -7895,6 +7996,7 @@ def patch_saving_functions(model, vision = False):
'''
exec(push_to_hub_text, globals())
+ @_fsdp2_collective_save(lambda self, args, kwargs: isinstance(self, PeftModel))
def unsloth_model_save_pretrained(self, *args, **kwargs):
"""`safe_serialization = None` means the safetensors default, not a pickle.