Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Native FSDP2: sharded resume reads for weights and an opt-in sharded optimizer checkpoint (#15042)

Daniel Han75a288757f27
diff --git a/tests/test_fsdp2_native_dcp_resume_gloo.py b/tests/test_fsdp2_native_dcp_resume_gloo.py
new file mode 100644
index 00000000000..c3b8c961f02
--- /dev/null
+++ b/tests/test_fsdp2_native_dcp_resume_gloo.py
@@ -0,0 +1,922 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""Native FSDP2 sharded resume on a gloo CPU world of 2 (fsdp2_native.py loaded on its own, no GPU):
+
+* weights: a full-model safetensors checkpoint whose keys match the sharded state dict is read with
+ dcp.load + HuggingFaceStorageReader (each rank its own slices); the result is bitwise equal to the
+ full load, for untied and tied embeddings and a multi-file index;
+* fallbacks to the full load: a .bin checkpoint, renamed keys, a stray safetensors file, a dtype
+ that differs, a LoRA adapter checkpoint, no usable reader (torch < 2.9, 2.8's signature);
+* optimizer: the sharded directory (native_save_optimizer_sharded) restores the same state, bitwise,
+ as the full optimizer.pt;
+* Trainer: save_steps checkpoints and resume_from_checkpoint with UNSLOTH_FSDP2_DCP_OPTIM=1 and =0
+ end bitwise equal to the uninterrupted run, and a reused checkpoint directory keeps one format."""
+
+import datetime
+import os
+import sys
+
+import pytest
+import torch
+
+if __import__("multiprocessing").parent_process() is not None or getattr(
+ __import__("multiprocessing").current_process(), "_inheriting", False
+):
+ os.environ["CUDA_VISIBLE_DEVICES"] = ""
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from test_fsdp2_native_units import _load_native # noqa: E402
+
+REPO_ID = "hf-internal-testing/tiny-random-LlamaForCausalLM"
+
+
+def _free_port():
+ from tests._rendezvous_port import free_port
+ return free_port()
+
+
+def _hf_reader_available():
+ import inspect
+
+ import torch.distributed.checkpoint as dcp
+
+ reader = getattr(dcp, "HuggingFaceStorageReader", None)
+ return reader is not None and "thread_count" in inspect.signature(reader.__init__).parameters
+
+
+def _model(
+ seed,
+ tied = False,
+ lora = False,
+):
+ import transformers
+
+ config = transformers.AutoConfig.from_pretrained(REPO_ID)
+ config.pad_token_id = None # tiny repo's -1 fails save_pretrained's config check
+ config.tie_word_embeddings = tied
+ torch.manual_seed(0)
+ model = transformers.AutoModelForCausalLM.from_config(config, torch_dtype = torch.float32)
+ if getattr(model, "generation_config", None) is not None:
+ model.generation_config.pad_token_id = None
+ if tied:
+ model.tie_weights()
+ elif model.lm_head.weight is model.model.embed_tokens.weight:
+ # Some transformers versions tie this tiny config regardless of the flag.
+ model.lm_head.weight = torch.nn.Parameter(model.lm_head.weight.detach().clone())
+ assert (model.lm_head.weight is model.model.embed_tokens.weight) == tied
+ g = torch.Generator().manual_seed(seed)
+ if lora:
+ from peft import LoraConfig, get_peft_model
+ model = get_peft_model(model, LoraConfig(r = 4, lora_alpha = 8, target_modules = "all-linear"))
+ with torch.no_grad():
+ for n, p in model.named_parameters():
+ if "lora_B" in n:
+ p.normal_(0, 0.02, generator = g)
+ elif seed:
+ with torch.no_grad():
+ for p in model.parameters():
+ p.add_(torch.randn(p.shape, generator = g) * 0.01)
+ return model
+
+
+def _batch(step, rank):
+ g = torch.Generator().manual_seed(100 * step + rank)
+ ids = torch.randint(3, 1000, (2, 16), generator = g)
+ return {"input_ids": ids, "labels": ids.clone(), "use_cache": False}
+
+
+def _step(model, opt, step, rank):
+ loss = model(**_batch(step, rank)).loss
+ loss.backward()
+ opt.step()
+ opt.zero_grad(set_to_none = True)
+ return loss.detach().clone()
+
+
+def _full_model(model):
+ from torch.distributed.checkpoint.state_dict import StateDictOptions, get_model_state_dict
+ sd = get_model_state_dict(model, options = StateDictOptions(full_state_dict = True))
+ return {k: v.detach().clone() for k, v in sd.items()}
+
+
+def _full_optim(model, opt):
+ from torch.distributed.checkpoint.state_dict import StateDictOptions, get_optimizer_state_dict
+ return get_optimizer_state_dict(model, opt, options = StateDictOptions(full_state_dict = True))
+
+
+def _diff_models(a, b):
+ """Names whose tensors are not bitwise equal (or missing on one side)."""
+ out = sorted(set(a) ^ set(b))
+ out += [k for k in a if k in b and not torch.equal(a[k], b[k])]
+ return out
+
+
+def _diff_optims(a, b):
+ out = sorted(set(a["state"]) ^ set(b["state"]))
+ for k, v in a["state"].items():
+ w = b["state"].get(k, {})
+ for name, t in v.items():
+ u = w.get(name)
+ if isinstance(t, torch.Tensor):
+ if not isinstance(u, torch.Tensor) or not torch.equal(t, u):
+ out.append(f"{k}.{name}")
+ elif t != u:
+ out.append(f"{k}.{name}")
+
+ def _plain(groups):
+ # The full load hands Adam's betas back as a list, the sharded one as the tuple it was.
+ return [{k: list(v) if isinstance(v, tuple) else v for k, v in g.items()} for g in groups]
+
+ if _plain(a["param_groups"]) != _plain(b["param_groups"]):
+ out.append(("param_groups", a["param_groups"], b["param_groups"]))
+ return out
+
+
+def _init(
+ rank,
+ world,
+ port,
+ timeout = 300,
+):
+ os.environ.update(
+ MASTER_ADDR = "127.0.0.1",
+ MASTER_PORT = str(port),
+ RANK = str(rank),
+ WORLD_SIZE = str(world),
+ LOCAL_RANK = str(rank),
+ LOCAL_WORLD_SIZE = str(world),
+ UNSLOTH_FSDP2 = "1",
+ )
+ import torch.distributed as dist
+
+ dist.init_process_group(
+ "gloo", rank = rank, world_size = world, timeout = datetime.timedelta(seconds = timeout)
+ )
+ return dist
+
+
+def _spy(native):
+ """Records what each _native_load_hf_sharded call returned (None = the full load ran)."""
+ calls = []
+ original = native._native_load_hf_sharded
+
+ def spy(root, checkpoint):
+ n = original(root, checkpoint)
+ calls.append(n)
+ return n
+
+ native._native_load_hf_sharded = spy
+ return calls
+
+
+def _case_roundtrip(native, dist, rank, tmp, tied, shard_files):
+ cfg = native.FSDP2Config(param_dtype = torch.float32)
+ calls = _spy(native)
+
+ def _new(seed):
+ m = native.shard_model(_model(seed, tied = tied), cfg)
+ o = torch.optim.AdamW(m.parameters(), lr = 1e-2, foreach = False)
+ return m, o
+
+ model, opt = _new(1)
+ for step in (1, 2):
+ _step(model, opt, step, rank)
+ ckpt = os.path.join(tmp, "ckpt")
+ state = native.native_full_state_dict(model)
+ if rank == 0:
+ kwargs = {"max_shard_size": "200KB"} if shard_files else {}
+ model.save_pretrained(ckpt, state_dict = state, safe_serialization = True, **kwargs)
+ native.native_save_optimizer(
+ model, opt, os.path.join(ckpt, "optimizer.pt"), should_save = rank == 0
+ )
+ native.native_save_optimizer_sharded(model, opt, os.path.join(ckpt, "optimizer_dcp"))
+ dist.barrier()
+ want = [_step(model, opt, step, rank).item() for step in (3, 4)]
+
+ res = {"files": sorted(os.listdir(ckpt))}
+ arms = {}
+ for arm in ("old", "new"):
+ if arm == "old":
+ os.environ["UNSLOTH_FSDP2_DCP_RESUME"] = "0"
+ else:
+ os.environ.pop("UNSLOTH_FSDP2_DCP_RESUME", None)
+ m, o = _new(2) # different fresh values: the load must replace them
+ native.native_load_model_checkpoint(m, ckpt)
+ opt_src = "optimizer.pt" if arm == "old" else "optimizer_dcp"
+ native.native_load_optimizer(m, o, os.path.join(ckpt, opt_src))
+ arms[arm] = {"model": _full_model(m), "optim": _full_optim(m, o)}
+ arms[arm]["losses"] = [_step(m, o, step, rank).item() for step in (3, 4)]
+ arms[arm]["after"] = _full_model(m)
+ if tied:
+ arms[arm]["still_tied"] = m.lm_head.weight is m.model.embed_tokens.weight
+ os.environ.pop("UNSLOTH_FSDP2_DCP_RESUME", None)
+ res["calls"] = calls
+ res["model_diff"] = _diff_models(arms["old"]["model"], arms["new"]["model"])
+ res["optim_diff"] = _diff_optims(arms["old"]["optim"], arms["new"]["optim"])
+ res["after_diff"] = _diff_models(arms["old"]["after"], arms["new"]["after"])
+ res["want"] = want
+ res["losses"] = {arm: arms[arm]["losses"] for arm in arms}
+ res["n_tensors"] = len(arms["new"]["model"])
+ if tied:
+ res["still_tied"] = {arm: arms[arm]["still_tied"] for arm in arms}
+ return res
+
+
+def _case_fallbacks(native, dist, rank, tmp):
+ import shutil
+
+ from safetensors.torch import load_file, save_file
+
+ cfg = native.FSDP2Config(param_dtype = torch.float32)
+ calls = _spy(native)
+ src = native.shard_model(_model(1), cfg)
+ want = _full_model(src)
+ state = native.native_full_state_dict(src)
+ base = os.path.join(tmp, "base")
+ dirs = {k: os.path.join(tmp, k) for k in ("bin", "renamed", "stray", "bf16", "gate", "sig")}
+ if rank == 0:
+ src.save_pretrained(base, state_dict = state, safe_serialization = True)
+ saved = load_file(os.path.join(base, "model.safetensors"))
+ for k in ("bin", "renamed", "stray", "bf16", "gate", "sig"):
+ shutil.copytree(base, dirs[k])
+ # transformers 5 no longer writes .bin weights: an old-style checkpoint by hand.
+ os.remove(os.path.join(dirs["bin"], "model.safetensors"))
+ torch.save(saved, os.path.join(dirs["bin"], "pytorch_model.bin"))
+ save_file(
+ {f"base.{k}": v for k, v in saved.items()},
+ os.path.join(dirs["renamed"], "model.safetensors"),
+ metadata = {"format": "pt"},
+ )
+ save_file({"x": torch.zeros(1)}, os.path.join(dirs["stray"], "extra.safetensors"))
+ save_file(
+ {k: v.to(torch.bfloat16) for k, v in saved.items()},
+ os.path.join(dirs["bf16"], "model.safetensors"),
+ metadata = {"format": "pt"},
+ )
+ dist.barrier()
+ res = {}
+ for name in ("bin", "stray", "renamed", "bf16"):
+ dst = native.shard_model(_model(2), cfg)
+ before = len(calls)
+ try:
+ native.native_load_model_checkpoint(dst, dirs[name])
+ got = _full_model(dst)
+ res[name] = {"diff": _diff_models(want, got) if name != "bf16" else None}
+ except Exception as exc:
+ res[name] = {"error": f"{type(exc).__name__}: {str(exc)[:200]}"}
+ res[name]["calls"] = calls[before:]
+
+ # torch < 2.9 (no reader with thread_count): simulated at the gate.
+ original_gate = native._hf_reader_class
+ native._hf_reader_class = lambda: None
+ dst = native.shard_model(_model(2), cfg)
+ before = len(calls)
+ native.native_load_model_checkpoint(dst, dirs["gate"])
+ res["gate"] = {"calls": calls[before:], "diff": _diff_models(want, _full_model(dst))}
+ native._hf_reader_class = original_gate
+
+ # torch 2.8's public reader has another signature: the probe must refuse it.
+ import torch.distributed.checkpoint as dcp
+
+ class Reader28:
+ def __init__(
+ self,
+ path,
+ token = None,
+ ):
+ pass
+
+ held = getattr(dcp, "HuggingFaceStorageReader", None)
+ dcp.HuggingFaceStorageReader = Reader28
+ try:
+ res["sig_probe"] = native._hf_reader_class()
+ dst = native.shard_model(_model(2), cfg)
+ before = len(calls)
+ native.native_load_model_checkpoint(dst, dirs["sig"])
+ res["sig"] = {"calls": calls[before:], "diff": _diff_models(want, _full_model(dst))}
+ finally:
+ dcp.HuggingFaceStorageReader = held
+ res["sig_probe"] = res["sig_probe"] is not None
+
+ # A LoRA adapter checkpoint stays on the adapter load.
+ lsrc = native.shard_model(_model(1, lora = True), cfg)
+ lwant = {n: v for n, v in _full_model(lsrc).items() if "lora_" in n}
+ lstate = native.native_full_state_dict(lsrc, trainable_only = True)
+ lora_dir = os.path.join(tmp, "lora")
+ if rank == 0:
+ lsrc.save_pretrained(lora_dir, state_dict = lstate, safe_serialization = True)
+ dist.barrier()
+ ldst = native.shard_model(_model(2, lora = True), cfg)
+ before = len(calls)
+ native.native_load_model_checkpoint(ldst, lora_dir)
+ lgot = {n: v for n, v in _full_model(ldst).items() if "lora_" in n}
+ res["lora"] = {
+ "calls": calls[before:],
+ "diff": _diff_models(lwant, lgot),
+ "n": len(lwant),
+ "files": sorted(os.listdir(lora_dir)),
+ }
+ return res
+
+
+def _case_trainer(native, dist, rank, tmp):
+ from datasets import Dataset
+ from transformers import Trainer, TrainingArguments
+
+ cfg = native.FSDP2Config(param_dtype = torch.float32)
+ rows = [
+ {"input_ids": list(range(5 + i, 21 + i)), "labels": list(range(5 + i, 21 + i))}
+ for i in range(4)
+ ]
+ ds = Dataset.from_list(rows * 4)
+
+ def _args(out):
+ return TrainingArguments(
+ output_dir = out,
+ report_to = [],
+ use_cpu = True,
+ max_steps = 4,
+ save_steps = 2,
+ save_strategy = "steps",
+ learning_rate = 1e-2,
+ per_device_train_batch_size = 2,
+ lr_scheduler_type = "linear",
+ logging_steps = 1,
+ seed = 0,
+ data_seed = 0,
+ dataloader_num_workers = 0,
+ )
+
+ res = {}
+ for fmt in ("dcp", "full"):
+ out = os.path.join(tmp, f"out_{fmt}")
+ os.environ["UNSLOTH_FSDP2_DCP_OPTIM"] = "1" if fmt == "dcp" else "0"
+ model = native.shard_model(_model(1), cfg)
+ Trainer(model = model, args = _args(out), train_dataset = ds).train()
+ want = _full_model(model)
+ ck2 = os.path.join(out, "checkpoint-2")
+ files = sorted(os.listdir(ck2))
+ dist.barrier()
+ # Resume with the flag flipped: the checkpoint's own format is read either way, and the
+ # re-save of checkpoint-4 must replace the other format, not sit beside it.
+ os.environ["UNSLOTH_FSDP2_DCP_OPTIM"] = "0" if fmt == "dcp" else "1"
+ resumed = native.shard_model(_model(2), cfg)
+ Trainer(model = resumed, args = _args(out), train_dataset = ds).train(resume_from_checkpoint = ck2)
+ got = _full_model(resumed)
+ dist.barrier()
+ res[fmt] = {
+ "ck2_files": files,
+ "ck4_files": sorted(os.listdir(os.path.join(out, "checkpoint-4"))),
+ "diff": _diff_models(want, got),
+ "n": len(want),
+ }
+ os.environ.pop("UNSLOTH_FSDP2_DCP_OPTIM", None)
+ return res
+
+
+class _ReaderError(OSError):
+ pass
+
+
+def _case_diverge(native, dist, rank, tmp, part):
+ """Two ranks with different environment / torch / file views must still take one path."""
+ import shutil
+ import types
+
+ import torch.distributed.checkpoint as dcp
+ from transformers import Trainer
+
+ cfg = native.FSDP2Config(param_dtype = torch.float32)
+ res = {}
+ if part in ("env", "gate", "read"):
+ calls = _spy(native)
+ src = native.shard_model(_model(1), cfg)
+ want = _full_model(src)
+ state = native.native_full_state_dict(src)
+ ckpt = os.path.join(tmp, "ckpt")
+ if rank == 0:
+ src.save_pretrained(ckpt, state_dict = state, safe_serialization = True)
+ dist.barrier()
+
+ def _load(name):
+ dst = native.shard_model(_model(2), cfg)
+ before = len(calls)
+ native.native_load_model_checkpoint(dst, ckpt)
+ res[name] = {"calls": calls[before:], "diff": _diff_models(want, _full_model(dst))}
+
+ if part == "env":
+ # Rank 1 alone turns the sharded read off.
+ if rank == 1:
+ os.environ["UNSLOTH_FSDP2_DCP_RESUME"] = "0"
+ _load("env")
+ os.environ.pop("UNSLOTH_FSDP2_DCP_RESUME", None)
+ elif part == "gate":
+ # Rank 1 alone has no usable reader (another torch build).
+ if rank == 1:
+ native._hf_reader_class = lambda: None
+ _load("gate")
+ else:
+ # A read error, on rank 1 alone and on both: dcp.load raises a CheckpointException.
+ reader_cls = dcp.HuggingFaceStorageReader
+ held_read = reader_cls.read_data
+ for name, ranks in (("read_error_r1", (1,)), ("read_error_both", (0, 1))):
+
+ def _fail(self, *args, **kwargs):
+ raise _ReaderError("injected unreadable shard")
+
+ if rank in ranks:
+ reader_cls.read_data = _fail
+ _load(name)
+ reader_cls.read_data = held_read
+ return res
+
+ model = native.shard_model(_model(1), cfg)
+ opt = torch.optim.AdamW(model.parameters(), lr = 1e-2, foreach = False)
+ _step(model, opt, 1, rank)
+ sched = torch.optim.lr_scheduler.LambdaLR(opt, lambda s: 1.0)
+ args = types.SimpleNamespace(should_save = rank == 0)
+ if part == "save":
+ # Optimizer save through the Trainer hook with a different flag per rank.
+ trainer = types.SimpleNamespace(
+ model = model, optimizer = opt, lr_scheduler = sched, args = args, is_fsdp_enabled = False
+ )
+ save_dir = os.path.join(tmp, "opt_mixed_flag")
+ os.makedirs(save_dir, exist_ok = True) # save_model made it, in a Trainer checkpoint
+ os.environ["UNSLOTH_FSDP2_DCP_OPTIM"] = "1" if rank == 0 else "0"
+ Trainer._save_optimizer_and_scheduler(trainer, save_dir)
+ os.environ.pop("UNSLOTH_FSDP2_DCP_OPTIM", None)
+ dist.barrier()
+ res["save_mixed_flag_files"] = sorted(os.listdir(save_dir))
+ return res
+
+ # Node-local views: every rank passes its own directory. both = .pt + dcp + scheduler,
+ # pt = .pt + scheduler, dcp = dcp + scheduler.
+ want_opt = _full_optim(model, opt)
+ views = {k: os.path.join(tmp, f"view_{k}") for k in ("both", "pt", "dcp")}
+ native.native_save_optimizer_sharded(model, opt, os.path.join(views["both"], "optimizer_dcp"))
+ native.native_save_optimizer(
+ model, opt, os.path.join(views["both"], "optimizer.pt"), should_save = rank == 0
+ )
+ if rank == 0:
+ torch.save(sched.state_dict(), os.path.join(views["both"], "scheduler.pt"))
+ shutil.copytree(views["both"], views["pt"])
+ shutil.rmtree(os.path.join(views["pt"], "optimizer_dcp"))
+ shutil.copytree(views["both"], views["dcp"])
+ os.remove(os.path.join(views["dcp"], "optimizer.pt"))
+ dist.barrier()
+
+ def _resume(name, view):
+ m = native.shard_model(_model(2), cfg)
+ o = torch.optim.AdamW(m.parameters(), lr = 1e-2, foreach = False)
+ sc = torch.optim.lr_scheduler.LambdaLR(o, lambda s: 1.0)
+ t = types.SimpleNamespace(
+ model = m, optimizer = o, lr_scheduler = sc, args = args, is_fsdp_enabled = False
+ )
+ try:
+ Trainer._load_optimizer_and_scheduler(t, view)
+ res[name] = {"diff": _diff_optims(want_opt, _full_optim(m, o))}
+ except RuntimeError as exc:
+ res[name] = {"error": str(exc)[:200]}
+
+ # Rank 0 sees both formats, rank 1 only optimizer.pt: both read optimizer.pt.
+ _resume("load_both_vs_pt", views["both"] if rank == 0 else views["pt"])
+ # Rank 0 sees only the sharded dir, rank 1 only optimizer.pt: no common format, both refuse.
+ _resume("load_dcp_vs_pt", views["dcp"] if rank == 0 else views["pt"])
+ return res
+
+
+def _case_amsgrad(native, dist, rank, tmp):
+ """A replicated (ignored) parameter in an AMSGrad group, resumed into an optimizer built with
+ amsgrad = False: the restored hyperparameters ask for max_exp_avg_sq on the next step."""
+ cfg_holder = {}
+
+ def _new(seed, amsgrad):
+ model = _model(seed)
+ first = model.model.layers[0].input_layernorm.weight
+ cfg = native.FSDP2Config(param_dtype = torch.float32, ignored_params = {first})
+ model = native.shard_model(model, cfg)
+ rep = [model.model.layers[0].input_layernorm.weight]
+ rest = [p for p in model.parameters() if p is not rep[0]]
+ cfg_holder["rep_is_dtensor"] = type(rep[0]).__name__
+ opt = torch.optim.AdamW(
+ [{"params": rest}, {"params": rep, "amsgrad": amsgrad}], lr = 1e-2, foreach = False
+ )
+ return model, opt
+
+ model, opt = _new(1, True)
+ for step in (1, 2):
+ _step(model, opt, step, rank)
+ ckpt = os.path.join(tmp, "ckpt")
+ os.makedirs(ckpt, exist_ok = True)
+ native.native_save_optimizer(
+ model, opt, os.path.join(ckpt, "optimizer.pt"), should_save = rank == 0
+ )
+ native.native_save_optimizer_sharded(model, opt, os.path.join(ckpt, "optimizer_dcp"))
+ state = native.native_full_state_dict(model)
+ if rank == 0:
+ model.save_pretrained(ckpt, state_dict = state, safe_serialization = True)
+ dist.barrier()
+ res = {"rep_type": cfg_holder["rep_is_dtensor"]}
+ arms = {}
+ for arm, src in (("old", "optimizer.pt"), ("new", "optimizer_dcp")):
+ m, o = _new(2, False)
+ native.native_load_model_checkpoint(m, ckpt)
+ try:
+ native.native_load_optimizer(m, o, os.path.join(ckpt, src))
+ loss = _step(m, o, 3, rank).item()
+ arms[arm] = {"loss": loss, "after": _full_model(m), "optim": _full_optim(m, o)}
+ except Exception as exc:
+ res[f"{arm}_error"] = f"{type(exc).__name__}: {str(exc)[:200]}"
+ if "old" in arms and "new" in arms:
+ res["loss"] = [arms["old"]["loss"], arms["new"]["loss"]]
+ res["after_diff"] = _diff_models(arms["old"]["after"], arms["new"]["after"])
+ res["optim_diff"] = [
+ str(d)[:200] for d in _diff_optims(arms["old"]["optim"], arms["new"]["optim"])
+ ]
+ return res
+
+
+def _case_unstepped(native, dist, rank, tmp):
+ """One trainable parameter never got a gradient before the save (an unused expert or branch), so
+ the saved optimizer holds no state for it; a fresh optimizer's target does."""
+
+ def _new(seed):
+ model = native.shard_model(_model(seed), native.FSDP2Config(param_dtype = torch.float32))
+ return model, torch.optim.AdamW(model.parameters(), lr = 1e-2, foreach = False)
+
+ def _skip(model):
+ return model.model.layers[0].post_attention_layernorm.weight
+
+ model, opt = _new(1)
+ for step in (1, 2):
+ model(**_batch(step, rank)).loss.backward()
+ _skip(model).grad = None
+ opt.step()
+ opt.zero_grad(set_to_none = True)
+ ckpt = os.path.join(tmp, "ckpt")
+ os.makedirs(ckpt, exist_ok = True)
+ native.native_save_optimizer(
+ model, opt, os.path.join(ckpt, "optimizer.pt"), should_save = rank == 0
+ )
+ native.native_save_optimizer_sharded(model, opt, os.path.join(ckpt, "optimizer_dcp"))
+ state = native.native_full_state_dict(model)
+ if rank == 0:
+ model.save_pretrained(ckpt, state_dict = state, safe_serialization = True)
+ dist.barrier()
+ res = {"saved_has_state": _skip(model) in opt.state}
+ # The uninterrupted run's next step is what a resume has to reproduce.
+ arms = {
+ "ref": {
+ "has_state": False,
+ "loss": _step(model, opt, 3, rank).item(),
+ "after": _full_model(model),
+ "optim": _full_optim(model, opt),
+ }
+ }
+ for arm, src in (("old", "optimizer.pt"), ("new", "optimizer_dcp")):
+ m, o = _new(2)
+ native.native_load_model_checkpoint(m, ckpt)
+ try:
+ native.native_load_optimizer(m, o, os.path.join(ckpt, src))
+ arms[arm] = {
+ "has_state": _skip(m) in o.state and bool(o.state[_skip(m)]),
+ "loss": _step(m, o, 3, rank).item(),
+ "after": _full_model(m),
+ "optim": _full_optim(m, o),
+ }
+ except BaseException as exc: # CheckpointException is a BaseException
+ res[f"{arm}_error"] = f"{type(exc).__name__}: {str(exc)[:300]}"
+ if "new" in arms:
+ res["has_state"] = arms["new"]["has_state"]
+ res["loss"] = [arms["ref"]["loss"], arms["new"]["loss"]]
+ res["after_diff"] = _diff_models(arms["ref"]["after"], arms["new"]["after"])
+ res["optim_diff"] = [
+ str(d)[:200] for d in _diff_optims(arms["ref"]["optim"], arms["new"]["optim"])
+ ]
+ return res
+
+
+def _case_momentum(native, dist, rank, tmp):
+ """SGD saved with momentum, resumed into SGD built with momentum = 0, whose fresh optimizer state
+ is empty: the momentum buffers must still be read (the restored param groups ask for them)."""
+
+ def _new(seed, momentum):
+ model = native.shard_model(_model(seed), native.FSDP2Config(param_dtype = torch.float32))
+ return model, torch.optim.SGD(model.parameters(), lr = 1e-2, momentum = momentum, foreach = False)
+
+ model, opt = _new(1, 0.9)
+ for step in (1, 2):
+ _step(model, opt, step, rank)
+ ckpt = os.path.join(tmp, "ckpt")
+ os.makedirs(ckpt, exist_ok = True)
+ native.native_save_optimizer_sharded(model, opt, os.path.join(ckpt, "optimizer_dcp"))
+ state = native.native_full_state_dict(model)
+ if rank == 0:
+ model.save_pretrained(ckpt, state_dict = state, safe_serialization = True)
+ dist.barrier()
+ ref = {
+ "loss": _step(model, opt, 3, rank).item(),
+ "after": _full_model(model),
+ "optim": _full_optim(model, opt),
+ }
+ m, o = _new(2, 0.0)
+ native.native_load_model_checkpoint(m, ckpt)
+ res = {}
+ try:
+ native.native_load_optimizer(m, o, os.path.join(ckpt, "optimizer_dcp"))
+ res["buffers"] = sum("momentum_buffer" in st for st in o.state.values())
+ res["momentum"] = o.param_groups[0]["momentum"]
+ loss = _step(m, o, 3, rank).item()
+ res["loss"] = [ref["loss"], loss]
+ res["after_diff"] = _diff_models(ref["after"], _full_model(m))
+ res["optim_diff"] = [str(d)[:200] for d in _diff_optims(ref["optim"], _full_optim(m, o))]
+ except BaseException as exc: # CheckpointException is a BaseException
+ res["error"] = f"{type(exc).__name__}: {str(exc)[:300]}"
+ return res
+
+
+def _case_bnb_keys(native, dist, rank, tmp):
+ """A sharded checkpoint holding the local-shard bitsandbytes Adam keys (state1 / state2), resumed
+ into torch's AdamW: the keys are renamed as the optimizer.pt path renames them."""
+ import torch.distributed.checkpoint as dcp
+ from torch.distributed.checkpoint.state_dict import StateDictOptions, get_optimizer_state_dict
+
+ def _new(seed):
+ model = native.shard_model(_model(seed), native.FSDP2Config(param_dtype = torch.float32))
+ return model, torch.optim.AdamW(model.parameters(), lr = 1e-2, foreach = False)
+
+ model, opt = _new(1)
+ for step in (1, 2):
+ _step(model, opt, step, rank)
+ ckpt = os.path.join(tmp, "ckpt")
+ os.makedirs(ckpt, exist_ok = True)
+ state = get_optimizer_state_dict(model, opt, options = StateDictOptions(full_state_dict = False))
+ names = {
+ "exp_avg": "state1",
+ "exp_avg_sq": "state2",
+ } # a copy: the live entries stay as they are
+ state["state"] = {
+ fqn: {names.get(k, k): v for k, v in entry.items()} for fqn, entry in state["state"].items()
+ }
+ dcp.save(state, checkpoint_id = os.path.join(ckpt, "optimizer_dcp"))
+ weights = native.native_full_state_dict(model)
+ if rank == 0:
+ model.save_pretrained(ckpt, state_dict = weights, safe_serialization = True)
+ dist.barrier()
+ ref = {"loss": _step(model, opt, 3, rank).item(), "after": _full_model(model)}
+ m, o = _new(2)
+ native.native_load_model_checkpoint(m, ckpt)
+ res = {}
+ try:
+ native.native_load_optimizer(m, o, os.path.join(ckpt, "optimizer_dcp"))
+ res["keys"] = sorted({k for st in o.state.values() for k in st})
+ res["loss"] = [ref["loss"], _step(m, o, 3, rank).item()]
+ res["after_diff"] = _diff_models(ref["after"], _full_model(m))
+ except BaseException as exc: # CheckpointException is a BaseException
+ res["error"] = f"{type(exc).__name__}: {str(exc)[:300]}"
+ return res
+
+
+def _case_reshard(native, dist, rank, tmp, phase):
+ """Save with one world size, resume with another: the weights (sharded read) and the
+ optimizer_dcp directory both come back equal to the saver's full state."""
+ cfg = native.FSDP2Config(param_dtype = torch.float32)
+ calls = _spy(native)
+ ckpt = os.path.join(tmp, "ckpt")
+ ref = os.path.join(tmp, "ref.pt")
+ model = native.shard_model(_model(1 if phase == "save" else 2), cfg)
+ opt = torch.optim.AdamW(model.parameters(), lr = 1e-2, foreach = False)
+ if phase == "save":
+ for step in (1, 2):
+ _step(model, opt, step, rank)
+ state = native.native_full_state_dict(model)
+ if rank == 0:
+ model.save_pretrained(ckpt, state_dict = state, safe_serialization = True)
+ native.native_save_optimizer_sharded(model, opt, os.path.join(ckpt, "optimizer_dcp"))
+ full = {"model": _full_model(model), "optim": _full_optim(model, opt)}
+ if rank == 0:
+ torch.save(full, ref)
+ dist.barrier()
+ return {"world": dist.get_world_size()}
+ native.native_load_model_checkpoint(model, ckpt)
+ native.native_load_optimizer(model, opt, os.path.join(ckpt, "optimizer_dcp"))
+ want = torch.load(ref, weights_only = False)
+ res = {
+ "world": dist.get_world_size(),
+ "calls": calls,
+ "model_diff": _diff_models(want["model"], _full_model(model)),
+ "optim_diff": _diff_optims(want["optim"], _full_optim(model, opt)),
+ }
+ res["loss"] = _step(model, opt, 3, rank).item() # the restored state steps
+ return res
+
+
+_SHORT_TIMEOUT_CASES = ("amsgrad", "unstepped", "momentum", "bnb_keys")
+
+
+def _worker(rank, world, port, out, case, tmp):
+ # Divergence cases run on a short collective timeout: a rank waiting on a collective the
+ # other never joins fails within a minute instead of hanging the suite.
+ short = case in _SHORT_TIMEOUT_CASES or case.startswith("diverge_")
+ dist = _init(rank, world, port, timeout = 60 if short else 300)
+ try:
+ native = _load_native()
+ if case == "fallbacks":
+ res = _case_fallbacks(native, dist, rank, tmp)
+ elif case == "trainer":
+ res = _case_trainer(native, dist, rank, tmp)
+ elif case.startswith("diverge_"):
+ res = _case_diverge(native, dist, rank, tmp, case[len("diverge_") :])
+ elif case == "amsgrad":
+ res = _case_amsgrad(native, dist, rank, tmp)
+ elif case == "unstepped":
+ res = _case_unstepped(native, dist, rank, tmp)
+ elif case == "momentum":
+ res = _case_momentum(native, dist, rank, tmp)
+ elif case == "bnb_keys":
+ res = _case_bnb_keys(native, dist, rank, tmp)
+ elif case.startswith("reshard_"):
+ res = _case_reshard(native, dist, rank, tmp, case[len("reshard_") :])
+ else:
+ res = _case_roundtrip(
+ native, dist, rank, tmp, tied = case == "tied", shard_files = case == "index"
+ )
+ out.put((rank, res))
+ except BaseException as exc: # CheckpointException is a BaseException
+ import traceback
+ out.put((rank, {"crash": f"{type(exc).__name__}: {exc}\n{traceback.format_exc()}"}))
+ finally:
+ try:
+ dist.destroy_process_group()
+ except Exception:
+ pass
+
+
+def _run(
+ case,
+ tmp,
+ timeout = 900,
+ world = 2,
+):
+ import queue
+
+ import torch.multiprocessing as mp
+
+ ctx = mp.get_context("spawn")
+ out = ctx.Queue()
+ port = _free_port()
+ procs = [
+ ctx.Process(target = _worker, args = (r, world, port, out, case, tmp)) for r in range(world)
+ ]
+ for p in procs:
+ p.start()
+ results = {}
+ try:
+ for _ in procs:
+ rank, r = out.get(timeout = timeout)
+ results[rank] = r
+ except queue.Empty:
+ pass
+ finally:
+ for p in procs:
+ p.join(timeout = 60)
+ if p.is_alive():
+ p.kill()
+ assert sorted(results) == list(
+ range(world)
+ ), f"{case}: ranks did not report within {timeout} s: {results}"
+ for rank, r in results.items():
+ print("DCP_RESUME", case, rank, {k: v for k, v in r.items() if k != "crash"})
+ assert "crash" not in r, r["crash"]
+ return results
+
+
[email protected]("case", ["untied", "tied", "index"])
+def test_sharded_resume_matches_full_load(case, tmp_path):
+ pytest.importorskip("transformers")
+ if not _hf_reader_available():
+ pytest.skip("torch < 2.9: no HuggingFaceStorageReader(path, thread_count)")
+ for rank, r in _run(case, str(tmp_path)).items():
+ # The sharded read ran once (the "new" arm) and the full load once (the "old" arm).
+ assert r["calls"][0] is None and isinstance(r["calls"][1], int) and r["calls"][1] > 0, r
+ assert r["model_diff"] == [], r["model_diff"]
+ assert r["optim_diff"] == [], r["optim_diff"]
+ assert r["after_diff"] == [], r["after_diff"]
+ assert r["losses"]["old"] == r["want"] and r["losses"]["new"] == r["want"], r
+ if case == "tied":
+ assert r["still_tied"] == {"old": True, "new": True}, r
+ if case == "index":
+ assert "model.safetensors.index.json" in r["files"], r["files"]
+
+
+def test_sharded_resume_fallbacks(tmp_path):
+ pytest.importorskip("transformers")
+ pytest.importorskip("peft")
+ reader = _hf_reader_available()
+ for rank, r in _run("fallbacks", str(tmp_path)).items():
+ for name in ("bin", "stray", "bf16", "gate", "sig", "lora"):
+ assert r[name]["calls"] == [None], (name, r[name])
+ for name in ("bin", "stray", "gate", "sig", "lora"):
+ assert r[name].get("diff") == [], (name, r[name])
+ assert r["lora"]["n"] > 0 and "adapter_model.safetensors" in r["lora"]["files"], r["lora"]
+ assert "error" not in r["bf16"], r["bf16"]
+ # Renamed keys: the full load runs and reports the missing tensors, as it always did.
+ assert r["renamed"]["calls"] == [None], r["renamed"]
+ assert "found no saved tensor" in r["renamed"].get("error", ""), r["renamed"]
+ assert r["sig_probe"] is False
+ if not reader:
+ pytest.skip("torch < 2.9: every case is a fallback already")
+
+
+def test_trainer_resume_both_optimizer_formats(tmp_path):
+ pytest.importorskip("transformers")
+ pytest.importorskip("datasets")
+ for rank, r in _run("trainer", str(tmp_path)).items():
+ dcp, full = r["dcp"], r["full"]
+ assert "optimizer_dcp" in dcp["ck2_files"] and "optimizer.pt" not in dcp["ck2_files"], dcp
+ assert (
+ "optimizer.pt" in full["ck2_files"] and "optimizer_dcp" not in full["ck2_files"]
+ ), full
+ # checkpoint-4 re-saved with the flag flipped holds only the newer format.
+ assert "optimizer.pt" in dcp["ck4_files"] and "optimizer_dcp" not in dcp["ck4_files"], dcp
+ assert (
+ "optimizer_dcp" in full["ck4_files"] and "optimizer.pt" not in full["ck4_files"]
+ ), full
+ assert dcp["diff"] == [] and full["diff"] == [], r
+
+
[email protected]("part", ["env", "gate", "read", "save", "load"])
+def test_ranks_with_different_views_agree(part, tmp_path):
+ pytest.importorskip("transformers")
+ if part == "read" and not _hf_reader_available():
+ pytest.skip("torch < 2.9: no sharded read to fail")
+ for rank, r in _run(f"diverge_{part}", str(tmp_path), timeout = 600).items():
+ names = {"env": ["env"], "gate": ["gate"], "read": ["read_error_r1", "read_error_both"]}
+ for name in names.get(part, []):
+ # Every rank took the full load (a lone sharded rank would have hung or crashed).
+ assert r[name]["calls"] == [None], (name, r[name])
+ assert r[name]["diff"] == [], (name, r[name])
+ if part == "save":
+ # Mixed UNSLOTH_FSDP2_DCP_OPTIM: the vote picks the full file on every rank.
+ assert "optimizer.pt" in r["save_mixed_flag_files"], r
+ assert "optimizer_dcp" not in r["save_mixed_flag_files"], r
+ if part == "load":
+ assert r["load_both_vs_pt"].get("diff") == [], r["load_both_vs_pt"]
+ assert "different optimizer" in r["load_dcp_vs_pt"].get("error", ""), r
+
+
+def test_sharded_optimizer_restores_amsgrad_state(tmp_path):
+ pytest.importorskip("transformers")
+ for rank, r in _run("amsgrad", str(tmp_path), timeout = 600).items():
+ assert r["rep_type"] != "DTensor", r # the AMSGrad group's parameter is replicated
+ assert "old_error" not in r, r
+ assert "new_error" not in r, r
+ assert r["loss"][0] == r["loss"][1], r
+ assert r["after_diff"] == [] and r["optim_diff"] == [], r
+
+
[email protected]("save_world, load_world", [(2, 3), (3, 2)])
+def test_sharded_resume_on_another_world_size(save_world, load_world, tmp_path):
+ pytest.importorskip("transformers")
+ if not _hf_reader_available():
+ pytest.skip("torch < 2.9: no HuggingFaceStorageReader(path, thread_count)")
+ _run("reshard_save", str(tmp_path), world = save_world)
+ results = _run("reshard_load", str(tmp_path), world = load_world)
+ for rank, r in results.items():
+ assert r["world"] == load_world, r
+ assert (
+ r["calls"] and isinstance(r["calls"][0], int) and r["calls"][0] > 0
+ ), r # sharded read
+ assert r["model_diff"] == [], r["model_diff"]
+ assert r["optim_diff"] == [], r["optim_diff"]
+
+
+def test_sharded_optimizer_resumes_a_parameter_never_stepped(tmp_path):
+ pytest.importorskip("transformers")
+ for rank, r in _run("unstepped", str(tmp_path), timeout = 600).items():
+ assert r["saved_has_state"] is False, r
+ # The optimizer.pt path refuses this checkpoint in torch's strict set_optimizer_state_dict
+ # (a separate, older limitation); the sharded one restores it.
+ assert "new_error" not in r, r
+ assert r["has_state"] is False, r # still without state, as when saved
+ assert r["loss"][0] == r["loss"][1], r
+ assert r["after_diff"] == [] and r["optim_diff"] == [], r
+
+
+def test_sharded_optimizer_restores_state_the_fresh_optimizer_lacks(tmp_path):
+ pytest.importorskip("transformers")
+ for rank, r in _run("momentum", str(tmp_path), timeout = 600).items():
+ assert "error" not in r, r
+ assert r["momentum"] == 0.9 and r["buffers"] > 0, r
+ assert r["loss"][0] == r["loss"][1], r
+ assert r["after_diff"] == [] and r["optim_diff"] == [], r
+
+
+def test_sharded_optimizer_renames_bitsandbytes_adam_keys(tmp_path):
+ pytest.importorskip("transformers")
+ for rank, r in _run("bnb_keys", str(tmp_path), timeout = 600).items():
+ assert "error" not in r, r
+ assert "exp_avg" in r["keys"] and "state1" not in r["keys"], r
+ assert r["loss"][0] == r["loss"][1], r
+ assert r["after_diff"] == [], r
diff --git a/unsloth/models/fsdp2_native.py b/unsloth/models/fsdp2_native.py
index 3392e23f72a..85c661c7233 100644
--- a/unsloth/models/fsdp2_native.py
+++ b/unsloth/models/fsdp2_native.py
@@ -2041,10 +2041,13 @@ def native_save_optimizer(
def native_load_optimizer(model, optimizer, path_or_state):
- """Collective: every rank reads the full optimizer state (CPU) and keeps only its own shards."""
+ """Collective: every rank reads the full optimizer state (CPU) and keeps only its own shards. A
+ directory written by native_save_optimizer_sharded is read sharded instead (each rank its slice)."""
from torch.distributed.checkpoint.state_dict import StateDictOptions, set_optimizer_state_dict
state = path_or_state
+ if isinstance(state, (str, os.PathLike)) and _is_dcp_dir(state):
+ return native_load_optimizer_sharded(model, optimizer, state)
if not isinstance(state, dict):
state = torch.load(state, map_location = "cpu", weights_only = True)
from .fsdp2_bnb_optim import rename_optimizer_state_for
@@ -2060,6 +2063,334 @@ def native_load_optimizer(model, optimizer, path_or_state):
return state
+# Sharded (torch.distributed.checkpoint) resume. The optimizer directory is opt-in
+# (UNSLOTH_FSDP2_DCP_OPTIM=1): only a native FSDP2 run can read it back, while the full optimizer.pt
+# also loads into a plain (single GPU) Trainer resume, which would silently skip a directory it does
+# not know. Every resume reads both formats, whichever the checkpoint holds.
+_OPTIMIZER_DCP_DIR = "optimizer_dcp"
+
+
+def _env_on(name, default):
+ value = os.environ.get(name, "").strip().lower()
+ if not value:
+ return default
+ return value not in ("0", "false", "no", "off")
+
+
+def _is_dcp_dir(path):
+ return os.path.isdir(path) and os.path.isfile(os.path.join(path, ".metadata"))
+
+
+def _dcp_group(root):
+ """(process group for torch.distributed.checkpoint, usable): the world (None) when the model's mesh
+ spans it, the mesh's own group for a 1-D sub-mesh. A multi-dimensional sub-mesh has no single
+ group, so the sharded paths are not used for it."""
+ import torch.distributed as dist
+
+ if not (dist.is_available() and dist.is_initialized()):
+ return None, False
+ from torch.distributed.tensor import DTensor
+
+ mesh = next(
+ (p.device_mesh for p in root.parameters() if isinstance(p, DTensor)),
+ None,
+ )
+ if mesh is None or mesh.size() == dist.get_world_size():
+ return None, True
+ if mesh.ndim == 1:
+ return mesh.get_group(0), True
+ return None, False
+
+
+def _all_ranks_agree(flags, group):
+ """Collective: True only when a flag holds on every rank of `group` (environment, files and
+ checkpoint metadata are read per rank; every rank must take the same path). `flags` is one bool
+ or a list of them (one all_reduce for all)."""
+ import torch.distributed as dist
+
+ single = not isinstance(flags, (list, tuple))
+ values = [flags] if single else list(flags)
+ backend = str(dist.get_backend(group))
+ device = "cpu"
+ if "gloo" not in backend and "nccl" in backend:
+ device = torch.device("cuda", torch.cuda.current_device())
+ t = torch.tensor([1 if f else 0 for f in values], dtype = torch.int32, device = device)
+ dist.all_reduce(t, op = dist.ReduceOp.MIN, group = group)
+ out = [bool(v) for v in t.tolist()]
+ return out[0] if single else out
+
+
+def native_save_optimizer_sharded(model, optimizer, directory):
+ """Collective: every rank writes only its own optimizer shards to `directory` with
+ torch.distributed.checkpoint (get_optimizer_state_dict(full_state_dict = False) + dcp.save), so no
+ rank holds the full optimizer state. Read back with native_load_optimizer(model, opt, directory);
+ DCP reshards to another world size."""
+ import torch.distributed.checkpoint as dcp
+ from torch.distributed.checkpoint.state_dict import StateDictOptions, get_optimizer_state_dict
+
+ root = _native_root(model)
+ group, usable = _dcp_group(root)
+ if not usable:
+ raise RuntimeError(
+ "Unsloth: the sharded optimizer checkpoint needs a process group spanning the model's mesh."
+ )
+ state = get_optimizer_state_dict(
+ root,
+ _inner_optimizer(optimizer),
+ options = StateDictOptions(full_state_dict = False),
+ )
+ dcp.save(state, checkpoint_id = os.fspath(directory), process_group = group)
+ return state
+
+
+def native_load_optimizer_sharded(model, optimizer, directory):
+ """Collective: each rank reads only its own slices of a native_save_optimizer_sharded directory."""
+ import torch.distributed.checkpoint as dcp
+ from torch.distributed.checkpoint.state_dict import (
+ StateDictOptions,
+ get_optimizer_state_dict,
+ set_optimizer_state_dict,
+ )
+
+ root = _native_root(model)
+ optimizer = _inner_optimizer(optimizer)
+ group, usable = _dcp_group(root)
+ if not usable:
+ raise RuntimeError(
+ f"Unsloth: {directory} is a sharded optimizer checkpoint, which needs a process group "
+ "spanning the model's mesh."
+ )
+ options = StateDictOptions(full_state_dict = False)
+ # The target layout: this rank's shards (empty optimizer state is initialised first).
+ state = get_optimizer_state_dict(root, optimizer, options = options)
+ unstepped = _match_saved_optimizer_keys(state, directory, root)
+ dcp.load(state, checkpoint_id = os.fspath(directory), process_group = group)
+ if unstepped:
+ # Parameters the saved optimizer held but never stepped: listed in its (now loaded) param
+ # groups, without state. Anything else without state is not this optimizer's checkpoint.
+ saved = {f for g in state.get("param_groups", []) for f in (g.get("params") or [])}
+ unknown = [f for f in unstepped if f not in saved]
+ if unknown:
+ raise RuntimeError(
+ f"Unsloth: {directory} has no optimizer entry for {unknown[0]} "
+ f"({len(unknown)} parameters), so it was not saved from this model's optimizer."
+ )
+ # torch's strict check refuses a trainable parameter without state; these were never stepped.
+ options = StateDictOptions(full_state_dict = False, strict = False)
+ from .fsdp2_bnb_optim import rename_optimizer_state_for
+
+ # As the optimizer.pt path: torch Adam keys to the local-shard bitsandbytes keys, or back.
+ rename_optimizer_state_for(optimizer, state)
+ set_optimizer_state_dict(root, optimizer, state, options = options)
+ return state
+
+
+def _match_saved_optimizer_keys(
+ state,
+ directory,
+ root = None,
+):
+ """dcp.load fills exactly the keys of its target, and the target comes from this optimizer's own
+ (fresh) state, which get_optimizer_state_dict initialises for every trainable parameter. Match it
+ to the checkpoint in both directions:
+ - a parameter the saved run never stepped (no gradient yet: an unused expert or branch) has no
+ saved state, so its entries are dropped and it stays without state, as it was when saved;
+ - a per-parameter tensor the saved run kept and this one does not (AMSGrad's max_exp_avg_sq when
+ the saved group had amsgrad = True, or SGD's momentum_buffer resumed into momentum = 0, whose
+ fresh state is empty) is added, shaped from the checkpoint metadata: like the parameter's other
+ state, or the parameter (same sharding), when the global shape matches, else a plain tensor
+ read whole.
+ Returns the parameters left without state."""
+ from torch.distributed.checkpoint import FileSystemReader
+
+ metadata = FileSystemReader(os.fspath(directory)).read_metadata().state_dict_metadata
+ per_param = state.setdefault("state", {})
+ unstepped = []
+ for fqn in list(per_param):
+ entry = per_param[fqn]
+ for name in [n for n in entry if f"state.{fqn}.{n}" not in metadata]:
+ del entry[name]
+ if not entry:
+ del per_param[fqn]
+ unstepped.append(fqn)
+ params = dict(root.named_parameters()) if root is not None else {}
+ for key in metadata:
+ if key.startswith("state.") and "." in key[len("state.") :]:
+ fqn = key[len("state.") :].rsplit(".", 1)[0]
+ if fqn not in per_param and fqn in params:
+ per_param[fqn] = {}
+ for fqn, entry in per_param.items():
+ prefix = f"state.{fqn}."
+ like = next(
+ (v for v in entry.values() if isinstance(v, torch.Tensor) and v.dim() > 0),
+ params.get(fqn),
+ )
+ for key, md in metadata.items():
+ if not key.startswith(prefix):
+ continue
+ name = key[len(prefix) :]
+ size = getattr(md, "size", None)
+ dtype = getattr(getattr(md, "properties", None), "dtype", None)
+ if "." in name or name in entry or size is None or dtype is None:
+ continue
+ if like is not None and tuple(like.shape) == tuple(size):
+ entry[name] = torch.empty_like(like, dtype = dtype)
+ else:
+ entry[name] = torch.empty(tuple(size), dtype = dtype)
+ return unstepped
+
+
+def _hf_reader_class():
+ """torch >= 2.9 HuggingFaceStorageReader(path, thread_count) (2.8's public one has another
+ signature, 2.7's is private), probed by feature; None when absent or switched off
+ (UNSLOTH_FSDP2_DCP_RESUME=0)."""
+ if not _env_on("UNSLOTH_FSDP2_DCP_RESUME", True):
+ return None
+ try:
+ import inspect
+ import torch.distributed.checkpoint as dcp
+
+ reader = getattr(dcp, "HuggingFaceStorageReader", None)
+ if reader is None or "thread_count" not in inspect.signature(reader.__init__).parameters:
+ return None
+ except Exception:
+ return None
+ return reader
+
+
+def _hf_safetensors_files(checkpoint):
+ """The safetensors files the full-model load would read (same precedence as
+ _read_checkpoint_weights), or None when that load would read something else (an adapter, a .bin).
+ The HF reader lists every *.safetensors in the directory, so a stray one also means None."""
+ import json
+
+ files = None
+ for name in _WEIGHT_FILES:
+ if os.path.isfile(os.path.join(checkpoint, name)):
+ files = [name] if name == "model.safetensors" else None
+ break
+ else:
+ index = os.path.join(checkpoint, "model.safetensors.index.json")
+ if not os.path.isfile(index):
+ return None
+ with open(index, encoding = "utf-8") as f:
+ files = sorted(set(json.load(f)["weight_map"].values()))
+ if not files:
+ return None
+ present = sorted(
+ f
+ for f in os.listdir(checkpoint)
+ if f.endswith(".safetensors") and os.path.isfile(os.path.join(checkpoint, f))
+ )
+ return files if present == files else None
+
+
+def _clean_fqn(name):
+ # get_model_state_dict drops these wrapper segments from its keys.
+ skip = ("_orig_mod", "_checkpoint_wrapped_module", "_fsdp_wrapped_module")
+ return ".".join(part for part in name.split(".") if part not in skip)
+
+
+def _match_hf_state(root, state, metadata):
+ """`state` (get_model_state_dict, sharded) restricted to what the checkpoint holds, or None when
+ the checkpoint does not match the model one to one (different key names, as a transformers weight
+ conversion would write; a parameter it lacks; a shape or dtype that differs; a non-tensor entry).
+ Tied parameters are read once, under a name the checkpoint has; a buffer the checkpoint lacks
+ keeps its value (as the full load does)."""
+ groups = {}
+ for name, param in root.named_parameters(remove_duplicate = False):
+ groups.setdefault(id(param), []).append(_clean_fqn(name))
+ param_keys = set()
+ for names in groups.values():
+ keys = [k for k in dict.fromkeys(names) if k in state]
+ param_keys.update(keys)
+ if len(keys) < 2:
+ continue
+ keep = next((k for k in keys if k in metadata), None)
+ if keep is None:
+ return None
+ for k in keys:
+ if k != keep:
+ state.pop(k)
+ for key in list(state):
+ if key not in metadata:
+ if key in param_keys:
+ return None
+ state.pop(key)
+ continue
+ value = state[key]
+ md = metadata[key]
+ properties = getattr(md, "properties", None)
+ if (
+ not isinstance(value, torch.Tensor)
+ or getattr(md, "size", None) is None
+ or tuple(md.size) != tuple(value.shape)
+ or getattr(properties, "dtype", None) != value.dtype
+ ):
+ return None
+ return state if state else None
+
+
+def _native_load_hf_sharded(root, checkpoint):
+ """Collective: a full-model safetensors checkpoint whose keys, shapes and dtypes equal the model's
+ sharded state dict is read with dcp.load + HuggingFaceStorageReader, so each rank reads only its
+ own slices (no rank holds the full model on the host). Returns the number of tensors loaded, or
+ None, on every rank alike, when the full load must run instead. Not for PEFT or quantized models."""
+ # The mesh is the same on every rank of it, so this needs no vote (and has no group to vote on).
+ group, usable = _dcp_group(root)
+ if not usable:
+ return None
+ # Everything else is read per rank (environment, torch build, files, checkpoint metadata) and
+ # decided by one vote, so no rank can take the full load while another waits in dcp.load.
+ state = reader = None
+ try:
+ reader_cls = _hf_reader_class()
+ if (
+ reader_cls is not None
+ and not _is_peft(root)
+ and not any(_is_quantized_param(p) for p in root.parameters())
+ and _hf_safetensors_files(checkpoint) is not None
+ ):
+ from torch.distributed.checkpoint.state_dict import (
+ StateDictOptions,
+ get_model_state_dict,
+ )
+
+ reader = reader_cls(checkpoint, thread_count = max(1, min(4, os.cpu_count() or 1)))
+ metadata = reader.read_metadata().state_dict_metadata
+ state = get_model_state_dict(root, options = StateDictOptions(full_state_dict = False))
+ state = _match_hf_state(root, state, metadata)
+ except Exception:
+ state = None
+ if not _all_ranks_agree(state is not None, group):
+ return None
+ import torch.distributed.checkpoint as dcp
+ from torch.distributed.checkpoint.state_dict import StateDictOptions, set_model_state_dict
+
+ try:
+ from torch.distributed.checkpoint.api import CheckpointException
+ except Exception:
+ CheckpointException = Exception
+ error = None
+ try:
+ dcp.load(state, storage_reader = reader, process_group = group)
+ except (Exception, CheckpointException) as exc:
+ # A read error reaches every rank as a CheckpointException (a BaseException, so `except
+ # Exception` alone would let it abort the resume).
+ error = exc
+ # The fallback is voted on too: the full load rewrites every tensor this one may have touched.
+ if not _all_ranks_agree(error is None, group):
+ _print_rank0(
+ f"Unsloth: native FSDP2 sharded resume from {checkpoint} failed "
+ f"({type(error).__name__ if error is not None else 'on another rank'}); loading the "
+ "full checkpoint on every rank instead."
+ )
+ return None
+ n_loaded = len(state)
+ set_model_state_dict(root, state, options = StateDictOptions(full_state_dict = False, strict = False))
+ return n_loaded
+
+
_WEIGHT_FILES = (
"adapter_model.safetensors",
"adapter_model.bin",
@@ -2185,6 +2516,12 @@ def native_load_model_checkpoint(model, checkpoint):
from torch.distributed.checkpoint.state_dict import StateDictOptions, set_model_state_dict
root = _native_root(model)
+ n_loaded = _native_load_hf_sharded(root, checkpoint)
+ if n_loaded is not None:
+ _print_rank0(
+ f"Unsloth: native FSDP2 resumed {n_loaded} tensors from {checkpoint} (sharded read)."
+ )
+ return n_loaded
adapters = []
active = getattr(root, "active_adapters", None)
if isinstance(active, (list, tuple)):
@@ -2406,12 +2743,30 @@ def _install_trainer_hooks():
if not is_native_fsdp2(model) or getattr(self, "is_fsdp_enabled", False):
return original_save_optimizer(self, output_dir)
# Collective full optimizer state (the original saves rank 0's shards only), then the scheduler.
- native_save_optimizer(
- model,
- self.optimizer,
- os.path.join(output_dir, _OPTIMIZER_NAME),
- should_save = self.args.should_save,
- )
+ # UNSLOTH_FSDP2_DCP_OPTIM=1: each rank writes its own shards instead. A stale file of the
+ # other format in a reused directory is removed, so a resume cannot read it.
+ # The flag is read per rank: every rank writes the format they all asked for (a mixed
+ # choice would pair dcp.save's collectives with the full gather's), before any cleanup.
+ group, usable = _dcp_group(_native_root(model))
+ sharded = usable and _all_ranks_agree(_env_on("UNSLOTH_FSDP2_DCP_OPTIM", False), group)
+ stale = os.path.join(output_dir, _OPTIMIZER_NAME if sharded else _OPTIMIZER_DCP_DIR)
+ if self.args.should_save and os.path.lexists(stale):
+ if os.path.isdir(stale) and not os.path.islink(stale):
+ import shutil
+ shutil.rmtree(stale)
+ else:
+ os.remove(stale)
+ if sharded:
+ native_save_optimizer_sharded(
+ model, self.optimizer, os.path.join(output_dir, _OPTIMIZER_DCP_DIR)
+ )
+ else:
+ native_save_optimizer(
+ model,
+ self.optimizer,
+ os.path.join(output_dir, _OPTIMIZER_NAME),
+ should_save = self.args.should_save,
+ )
scheduler = getattr(self, "lr_scheduler", None)
if self.args.should_save and scheduler is not None:
torch.save(scheduler.state_dict(), os.path.join(output_dir, _SCHEDULER_NAME))
@@ -2425,11 +2780,38 @@ def _install_trainer_hooks():
checkpoint is None
or not is_native_fsdp2(model)
or getattr(self, "is_fsdp_enabled", False)
- or not os.path.isfile(os.path.join(checkpoint, _OPTIMIZER_NAME))
- or not os.path.isfile(os.path.join(checkpoint, _SCHEDULER_NAME))
):
return original_load_optimizer(self, checkpoint, *args, **kwargs)
- native_load_optimizer(model, self.optimizer, os.path.join(checkpoint, _OPTIMIZER_NAME))
+ local = [
+ _is_dcp_dir(os.path.join(checkpoint, _OPTIMIZER_DCP_DIR)),
+ os.path.isfile(os.path.join(checkpoint, _OPTIMIZER_NAME)),
+ os.path.isfile(os.path.join(checkpoint, _SCHEDULER_NAME)),
+ ]
+ group, usable = _dcp_group(_native_root(model))
+ if usable:
+ # What each rank sees on (possibly node-local) storage is voted on, so all ranks read
+ # the same format or skip together: [all see dcp, all see .pt, all see the scheduler,
+ # none sees an optimizer, none sees the scheduler].
+ sharded, full, scheduler, none_saved, no_scheduler = _all_ranks_agree(
+ local + [not (local[0] or local[1]), not local[2]], group
+ )
+ else:
+ # A multi-dimensional sub-mesh has no one group to vote on; it only reads optimizer.pt.
+ sharded, full, scheduler = False, local[1], local[2]
+ none_saved, no_scheduler = not full, not scheduler
+ if none_saved or no_scheduler:
+ return original_load_optimizer(self, checkpoint, *args, **kwargs)
+ if not scheduler or not (sharded or full):
+ raise RuntimeError(
+ f"Unsloth: native FSDP2 resume: the ranks see different optimizer / scheduler files "
+ f"in {checkpoint} (node-local storage?); every rank must read the same checkpoint."
+ )
+ # Whichever format the checkpoint holds (the sharded directory when both are present).
+ native_load_optimizer(
+ model,
+ self.optimizer,
+ os.path.join(checkpoint, _OPTIMIZER_DCP_DIR if sharded else _OPTIMIZER_NAME),
+ )
self.lr_scheduler.load_state_dict(
torch.load(os.path.join(checkpoint, _SCHEDULER_NAME), weights_only = True)
)