Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Native FSDP2 optimizer.pt resume: parameters the saved run never stepped (#15050)

Daniel Han925e1c062657
diff --git a/tests/test_fsdp2_native_optim_unstepped_gloo.py b/tests/test_fsdp2_native_optim_unstepped_gloo.py
new file mode 100644
index 00000000000..efaa0b7afc6
--- /dev/null
+++ b/tests/test_fsdp2_native_optim_unstepped_gloo.py
@@ -0,0 +1,288 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""Native FSDP2 optimizer resume on a gloo CPU world of 2 when a trainable parameter never got a
+gradient before the save: it stays stateless and the next step is bitwise equal to the
+uninterrupted run; a checkpoint not listing a trainable parameter is still refused."""
+
+import datetime
+import multiprocessing
+import os
+
+if multiprocessing.parent_process() is not None or getattr(
+ multiprocessing.current_process(), "_inheriting", False
+):
+ os.environ["CUDA_VISIBLE_DEVICES"] = "" # spawned gloo ranks: CPU only, also on GPU hosts
+
+import copy # noqa: E402
+import sys # noqa: E402
+
+import pytest # noqa: E402
+import torch # noqa: E402
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from test_fsdp2_native_units import _load_native # noqa: E402
+
+REPO_ID = "hf-internal-testing/tiny-random-LlamaForCausalLM"
+
+
+def _free_port():
+ from tests._rendezvous_port import free_port
+ return free_port()
+
+
+def _model(seed):
+ import transformers
+
+ config = transformers.AutoConfig.from_pretrained(REPO_ID)
+ config.pad_token_id = None
+ config.tie_word_embeddings = False
+ torch.manual_seed(0)
+ model = transformers.AutoModelForCausalLM.from_config(config, torch_dtype = torch.float32)
+ if model.lm_head.weight is model.model.embed_tokens.weight:
+ model.lm_head.weight = torch.nn.Parameter(model.lm_head.weight.detach().clone())
+ g = torch.Generator().manual_seed(seed)
+ with torch.no_grad():
+ for p in model.parameters():
+ p.add_(torch.randn(p.shape, generator = g) * 0.01)
+ return model
+
+
+def _batch(step, rank):
+ g = torch.Generator().manual_seed(100 * step + rank)
+ ids = torch.randint(3, 1000, (2, 16), generator = g)
+ return {"input_ids": ids, "labels": ids.clone(), "use_cache": False}
+
+
+def _step(
+ model,
+ opt,
+ step,
+ rank,
+ skip = None,
+):
+ loss = model(**_batch(step, rank)).loss
+ loss.backward()
+ if skip is not None:
+ skip.grad = None # never reached by the loss: no gradient, no optimizer state
+ opt.step()
+ opt.zero_grad(set_to_none = True)
+ return loss.detach().clone()
+
+
+def _full_model(model):
+ from torch.distributed.checkpoint.state_dict import StateDictOptions, get_model_state_dict
+ sd = get_model_state_dict(model, options = StateDictOptions(full_state_dict = True))
+ return {k: v.detach().clone() for k, v in sd.items()}
+
+
+def _full_optim(model, opt):
+ from torch.distributed.checkpoint.state_dict import StateDictOptions, get_optimizer_state_dict
+ return get_optimizer_state_dict(model, opt, options = StateDictOptions(full_state_dict = True))
+
+
+def _diff_models(a, b):
+ out = sorted(set(a) ^ set(b))
+ return out + [k for k in a if k in b and not torch.equal(a[k], b[k])]
+
+
+def _diff_optims(a, b):
+ out = sorted(set(a["state"]) ^ set(b["state"]))
+ for k, v in a["state"].items():
+ w = b["state"].get(k, {})
+ for name in sorted(set(v) | set(w)):
+ t, u = v.get(name), w.get(name)
+ if isinstance(t, torch.Tensor) or isinstance(u, torch.Tensor):
+ if not (
+ isinstance(t, torch.Tensor)
+ and isinstance(u, torch.Tensor)
+ and torch.equal(t, u)
+ ):
+ out.append(f"{k}.{name}")
+ elif t != u:
+ out.append(f"{k}.{name}")
+
+ def _groups(o):
+ # The full-state load hands betas back as a list (torch's state-dict flattening).
+ return [
+ {k: list(v) if isinstance(v, tuple) else v for k, v in g.items()}
+ for g in o["param_groups"]
+ ]
+
+ if _groups(a) != _groups(b):
+ out.append("param_groups")
+ return out
+
+
+_SKIP = "model.layers.0.post_attention_layernorm.weight"
+
+
+def _new(native, seed):
+ from torch.distributed.checkpoint.state_dict import StateDictOptions, set_model_state_dict
+ model = native.shard_model(_model(seed), native.FSDP2Config(param_dtype = torch.float32))
+ return (
+ model,
+ torch.optim.AdamW(model.parameters(), lr = 1e-2, foreach = False),
+ (
+ StateDictOptions,
+ set_model_state_dict,
+ ),
+ )
+
+
+def _skip(model):
+ return model.model.layers[0].post_attention_layernorm.weight
+
+
+def _has_state(opt, param):
+ return bool(opt.state.get(param))
+
+
+def _case_resume(native, dist, rank, tmp):
+ model, opt, (Options, set_model) = _new(native, 1)
+ for step in (1, 2):
+ _step(model, opt, step, rank, skip = _skip(model))
+ path = os.path.join(tmp, "optimizer.pt")
+ native.native_save_optimizer(model, opt, path, should_save = rank == 0)
+ saved = torch.load(path, map_location = "cpu", weights_only = True) # rank 0's full state
+ weights = _full_model(model)
+ res = {
+ "saved_has_state": _has_state(opt, _skip(model)),
+ "saved_in_file": _SKIP in saved["state"],
+ "saved_in_groups": any(_SKIP in g["params"] for g in saved["param_groups"]),
+ }
+ want = {
+ "loss": _step(model, opt, 3, rank).item(),
+ "after": _full_model(model),
+ "optim": _full_optim(model, opt),
+ }
+ for arm, src in (("file", path), ("state", copy.deepcopy(saved))):
+ m, o, _ = _new(native, 2)
+ set_model(m, weights, options = Options(full_state_dict = True))
+ try:
+ native.native_load_optimizer(m, o, src)
+ except BaseException as exc:
+ res[f"{arm}_error"] = f"{type(exc).__name__}: {str(exc)[:300]}"
+ continue
+ res[f"{arm}_has_state"] = _has_state(o, _skip(m))
+ res[f"{arm}_n_state"] = sum(1 for p in m.parameters() if _has_state(o, p))
+ loss = _step(m, o, 3, rank).item()
+ res[f"{arm}_loss"] = [want["loss"], loss]
+ res[f"{arm}_after_diff"] = _diff_models(want["after"], _full_model(m))
+ res[f"{arm}_optim_diff"] = [
+ str(d)[:200] for d in _diff_optims(want["optim"], _full_optim(m, o))
+ ]
+ res["n_trainable"] = sum(1 for p in model.parameters() if p.requires_grad)
+ return res
+
+
+def _case_refuse(native, dist, rank, tmp):
+ """A checkpoint missing the parameter entirely, or from another model, still raises."""
+ model, opt, _ = _new(native, 1)
+ for step in (1, 2):
+ _step(model, opt, step, rank, skip = _skip(model))
+ saved = _full_optim(model, opt) # full on every rank (no cpu_offload)
+ res = {}
+ dropped = copy.deepcopy(saved)
+ for g in dropped["param_groups"]:
+ g["params"] = [f for f in g["params"] if f != _SKIP]
+ other = copy.deepcopy(saved)
+ other["state"] = {f"other.{k}": v for k, v in other["state"].items()}
+ for g in other["param_groups"]:
+ g["params"] = [f"other.{f}" for f in g["params"]]
+ for arm, src in (("dropped", dropped), ("other", other)):
+ m, o, _ = _new(native, 2)
+ try:
+ native.native_load_optimizer(m, o, src)
+ res[f"{arm}_error"] = ""
+ except BaseException as exc:
+ res[f"{arm}_error"] = f"{type(exc).__name__}: {str(exc)[:300]}"
+ return res
+
+
+def _worker(rank, world, port, out, case, tmp):
+ os.environ.update(
+ MASTER_ADDR = "127.0.0.1",
+ MASTER_PORT = str(port),
+ RANK = str(rank),
+ WORLD_SIZE = str(world),
+ LOCAL_RANK = str(rank),
+ LOCAL_WORLD_SIZE = str(world),
+ UNSLOTH_FSDP2 = "1",
+ )
+ import torch.distributed as dist
+
+ dist.init_process_group(
+ "gloo", rank = rank, world_size = world, timeout = datetime.timedelta(seconds = 120)
+ )
+ try:
+ native = _load_native()
+ res = (_case_resume if case == "resume" else _case_refuse)(native, dist, rank, tmp)
+ out.put((rank, res))
+ except BaseException as exc:
+ import traceback
+ out.put((rank, {"crash": f"{type(exc).__name__}: {exc}\n{traceback.format_exc()}"}))
+ finally:
+ try:
+ dist.destroy_process_group()
+ except Exception:
+ pass
+
+
+def _run(
+ case,
+ tmp,
+ world = 2,
+ timeout = 600,
+):
+ import queue
+
+ import torch.multiprocessing as mp
+
+ ctx = mp.get_context("spawn")
+ out = ctx.Queue()
+ port = _free_port()
+ procs = [
+ ctx.Process(target = _worker, args = (r, world, port, out, case, tmp)) for r in range(world)
+ ]
+ for p in procs:
+ p.start()
+ results = {}
+ try:
+ for _ in procs:
+ rank, r = out.get(timeout = timeout)
+ results[rank] = r
+ except queue.Empty:
+ pass
+ finally:
+ for p in procs:
+ p.join(timeout = 60)
+ if p.is_alive():
+ p.kill()
+ assert sorted(results) == list(range(world)), f"{case}: ranks did not report: {results}"
+ for rank, r in results.items():
+ print("OPTIM_UNSTEPPED", case, rank, r)
+ assert "crash" not in r, r["crash"]
+ return results
+
+
+def test_resume_with_a_parameter_never_stepped(tmp_path):
+ pytest.importorskip("transformers")
+ for rank, r in _run("resume", str(tmp_path)).items():
+ # The setup: the saved optimizer lists the parameter but holds no state for it.
+ assert r["saved_has_state"] is False and r["saved_in_file"] is False, r
+ assert r["saved_in_groups"] is True, r
+ for arm in ("file", "state"):
+ assert f"{arm}_error" not in r, r
+ assert r[f"{arm}_has_state"] is False, r
+ assert r[f"{arm}_n_state"] == r["n_trainable"] - 1, r
+ assert r[f"{arm}_loss"][0] == r[f"{arm}_loss"][1], r
+ assert r[f"{arm}_after_diff"] == [], r
+ assert r[f"{arm}_optim_diff"] == [], r
+
+
+def test_resume_refuses_another_models_optimizer(tmp_path):
+ pytest.importorskip("transformers")
+ for rank, r in _run("refuse", str(tmp_path)).items():
+ assert r["dropped_error"], r
+ assert r["other_error"], r
diff --git a/unsloth/models/fsdp2_native.py b/unsloth/models/fsdp2_native.py
index 58f06cd5c21..11266a12593 100644
--- a/unsloth/models/fsdp2_native.py
+++ b/unsloth/models/fsdp2_native.py
@@ -2134,15 +2134,64 @@ def native_load_optimizer(model, optimizer, path_or_state):
# A checkpoint from torch's AdamW resumed into a local-shard bitsandbytes Adam, or back.
rename_optimizer_state_for(_inner_optimizer(optimizer), state)
- set_optimizer_state_dict(
- _native_root(model),
- _inner_optimizer(optimizer),
- state,
- options = StateDictOptions(full_state_dict = True),
- )
+ root = _native_root(model)
+ inner = _inner_optimizer(optimizer)
+ unstepped = _unstepped_optimizer_params(root, inner, state)
+ # torch refuses a trainable parameter without saved state, and strict = False would keep the
+ # fresh init state for it. Hiding these (requires_grad off) leaves them stateless, as saved.
+ try:
+ for param in unstepped:
+ param.requires_grad_(False)
+ set_optimizer_state_dict(
+ root,
+ inner,
+ state,
+ options = StateDictOptions(full_state_dict = True),
+ )
+ finally:
+ for param in unstepped:
+ param.requires_grad_(True)
+ for param in unstepped:
+ inner.state.pop(param, None)
return state
+def _unstepped_optimizer_params(root, optimizer, state):
+ """Trainable parameters listed in the saved param groups but with no saved state (never got a
+ gradient). Raises for one not listed at all: another model's optimizer."""
+ saved_state = state.get("state") or {}
+ if not isinstance(saved_state, dict) or all(isinstance(k, int) for k in saved_state):
+ return [] # not keyed by FQN: torch loads it as given
+ try:
+ from torch.distributed.checkpoint.state_dict import _get_fqns
+ except ImportError:
+ _get_fqns = None
+ fqns = {}
+ for name, param in root.named_parameters(remove_duplicate = False):
+ names = _get_fqns(root, name) if _get_fqns is not None else {name}
+ fqns.setdefault(param, set()).update(names)
+ listed = {f for g in state.get("param_groups") or [] for f in (g.get("params") or [])}
+ saved = set(saved_state)
+ unstepped, seen, unknown = [], set(), []
+ for group in optimizer.param_groups:
+ for param in group["params"]:
+ names = fqns.get(param)
+ if not param.requires_grad or not names or names & saved:
+ continue
+ if names & listed:
+ if id(param) not in seen:
+ seen.add(id(param))
+ unstepped.append(param)
+ else:
+ unknown.append(sorted(names)[0])
+ if unknown:
+ raise RuntimeError(
+ f"Unsloth: the optimizer checkpoint has no entry for {unknown[0]} ({len(unknown)} "
+ "parameters), so it was not saved from this model's optimizer."
+ )
+ return unstepped
+
+
# Sharded (torch.distributed.checkpoint) resume. The optimizer directory is opt-in
# (UNSLOTH_FSDP2_DCP_OPTIM=1): only a native FSDP2 run can read it back, while the full optimizer.pt
# also loads into a plain (single GPU) Trainer resume, which would silently skip a directory it does