Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Native FSDP2 optimizer.pt resume: parameters the saved run never stepped (#15050)
Daniel Han925e1c062657
diff --git a/tests/test_fsdp2_native_optim_unstepped_gloo.py b/tests/test_fsdp2_native_optim_unstepped_gloo.pynew file mode 100644index 00000000000..efaa0b7afc6--- /dev/null+++ b/tests/test_fsdp2_native_optim_unstepped_gloo.py@@ -0,0 +1,288 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""Native FSDP2 optimizer resume on a gloo CPU world of 2 when a trainable parameter never got a+gradient before the save: it stays stateless and the next step is bitwise equal to the+uninterrupted run; a checkpoint not listing a trainable parameter is still refused."""++import datetime+import multiprocessing+import os++if multiprocessing.parent_process() is not None or getattr(+ multiprocessing.current_process(), "_inheriting", False+):+ os.environ["CUDA_VISIBLE_DEVICES"] = "" # spawned gloo ranks: CPU only, also on GPU hosts++import copy # noqa: E402+import sys # noqa: E402++import pytest # noqa: E402+import torch # noqa: E402++sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))+from test_fsdp2_native_units import _load_native # noqa: E402++REPO_ID = "hf-internal-testing/tiny-random-LlamaForCausalLM"+++def _free_port():+ from tests._rendezvous_port import free_port+ return free_port()+++def _model(seed):+ import transformers++ config = transformers.AutoConfig.from_pretrained(REPO_ID)+ config.pad_token_id = None+ config.tie_word_embeddings = False+ torch.manual_seed(0)+ model = transformers.AutoModelForCausalLM.from_config(config, torch_dtype = torch.float32)+ if model.lm_head.weight is model.model.embed_tokens.weight:+ model.lm_head.weight = torch.nn.Parameter(model.lm_head.weight.detach().clone())+ g = torch.Generator().manual_seed(seed)+ with torch.no_grad():+ for p in model.parameters():+ p.add_(torch.randn(p.shape, generator = g) * 0.01)+ return model+++def _batch(step, rank):+ g = torch.Generator().manual_seed(100 * step + rank)+ ids = torch.randint(3, 1000, (2, 16), generator = g)+ return {"input_ids": ids, "labels": ids.clone(), "use_cache": False}+++def _step(+ model,+ opt,+ step,+ rank,+ skip = None,+):+ loss = model(**_batch(step, rank)).loss+ loss.backward()+ if skip is not None:+ skip.grad = None # never reached by the loss: no gradient, no optimizer state+ opt.step()+ opt.zero_grad(set_to_none = True)+ return loss.detach().clone()+++def _full_model(model):+ from torch.distributed.checkpoint.state_dict import StateDictOptions, get_model_state_dict+ sd = get_model_state_dict(model, options = StateDictOptions(full_state_dict = True))+ return {k: v.detach().clone() for k, v in sd.items()}+++def _full_optim(model, opt):+ from torch.distributed.checkpoint.state_dict import StateDictOptions, get_optimizer_state_dict+ return get_optimizer_state_dict(model, opt, options = StateDictOptions(full_state_dict = True))+++def _diff_models(a, b):+ out = sorted(set(a) ^ set(b))+ return out + [k for k in a if k in b and not torch.equal(a[k], b[k])]+++def _diff_optims(a, b):+ out = sorted(set(a["state"]) ^ set(b["state"]))+ for k, v in a["state"].items():+ w = b["state"].get(k, {})+ for name in sorted(set(v) | set(w)):+ t, u = v.get(name), w.get(name)+ if isinstance(t, torch.Tensor) or isinstance(u, torch.Tensor):+ if not (+ isinstance(t, torch.Tensor)+ and isinstance(u, torch.Tensor)+ and torch.equal(t, u)+ ):+ out.append(f"{k}.{name}")+ elif t != u:+ out.append(f"{k}.{name}")++ def _groups(o):+ # The full-state load hands betas back as a list (torch's state-dict flattening).+ return [+ {k: list(v) if isinstance(v, tuple) else v for k, v in g.items()}+ for g in o["param_groups"]+ ]++ if _groups(a) != _groups(b):+ out.append("param_groups")+ return out+++_SKIP = "model.layers.0.post_attention_layernorm.weight"+++def _new(native, seed):+ from torch.distributed.checkpoint.state_dict import StateDictOptions, set_model_state_dict+ model = native.shard_model(_model(seed), native.FSDP2Config(param_dtype = torch.float32))+ return (+ model,+ torch.optim.AdamW(model.parameters(), lr = 1e-2, foreach = False),+ (+ StateDictOptions,+ set_model_state_dict,+ ),+ )+++def _skip(model):+ return model.model.layers[0].post_attention_layernorm.weight+++def _has_state(opt, param):+ return bool(opt.state.get(param))+++def _case_resume(native, dist, rank, tmp):+ model, opt, (Options, set_model) = _new(native, 1)+ for step in (1, 2):+ _step(model, opt, step, rank, skip = _skip(model))+ path = os.path.join(tmp, "optimizer.pt")+ native.native_save_optimizer(model, opt, path, should_save = rank == 0)+ saved = torch.load(path, map_location = "cpu", weights_only = True) # rank 0's full state+ weights = _full_model(model)+ res = {+ "saved_has_state": _has_state(opt, _skip(model)),+ "saved_in_file": _SKIP in saved["state"],+ "saved_in_groups": any(_SKIP in g["params"] for g in saved["param_groups"]),+ }+ want = {+ "loss": _step(model, opt, 3, rank).item(),+ "after": _full_model(model),+ "optim": _full_optim(model, opt),+ }+ for arm, src in (("file", path), ("state", copy.deepcopy(saved))):+ m, o, _ = _new(native, 2)+ set_model(m, weights, options = Options(full_state_dict = True))+ try:+ native.native_load_optimizer(m, o, src)+ except BaseException as exc:+ res[f"{arm}_error"] = f"{type(exc).__name__}: {str(exc)[:300]}"+ continue+ res[f"{arm}_has_state"] = _has_state(o, _skip(m))+ res[f"{arm}_n_state"] = sum(1 for p in m.parameters() if _has_state(o, p))+ loss = _step(m, o, 3, rank).item()+ res[f"{arm}_loss"] = [want["loss"], loss]+ res[f"{arm}_after_diff"] = _diff_models(want["after"], _full_model(m))+ res[f"{arm}_optim_diff"] = [+ str(d)[:200] for d in _diff_optims(want["optim"], _full_optim(m, o))+ ]+ res["n_trainable"] = sum(1 for p in model.parameters() if p.requires_grad)+ return res+++def _case_refuse(native, dist, rank, tmp):+ """A checkpoint missing the parameter entirely, or from another model, still raises."""+ model, opt, _ = _new(native, 1)+ for step in (1, 2):+ _step(model, opt, step, rank, skip = _skip(model))+ saved = _full_optim(model, opt) # full on every rank (no cpu_offload)+ res = {}+ dropped = copy.deepcopy(saved)+ for g in dropped["param_groups"]:+ g["params"] = [f for f in g["params"] if f != _SKIP]+ other = copy.deepcopy(saved)+ other["state"] = {f"other.{k}": v for k, v in other["state"].items()}+ for g in other["param_groups"]:+ g["params"] = [f"other.{f}" for f in g["params"]]+ for arm, src in (("dropped", dropped), ("other", other)):+ m, o, _ = _new(native, 2)+ try:+ native.native_load_optimizer(m, o, src)+ res[f"{arm}_error"] = ""+ except BaseException as exc:+ res[f"{arm}_error"] = f"{type(exc).__name__}: {str(exc)[:300]}"+ return res+++def _worker(rank, world, port, out, case, tmp):+ os.environ.update(+ MASTER_ADDR = "127.0.0.1",+ MASTER_PORT = str(port),+ RANK = str(rank),+ WORLD_SIZE = str(world),+ LOCAL_RANK = str(rank),+ LOCAL_WORLD_SIZE = str(world),+ UNSLOTH_FSDP2 = "1",+ )+ import torch.distributed as dist++ dist.init_process_group(+ "gloo", rank = rank, world_size = world, timeout = datetime.timedelta(seconds = 120)+ )+ try:+ native = _load_native()+ res = (_case_resume if case == "resume" else _case_refuse)(native, dist, rank, tmp)+ out.put((rank, res))+ except BaseException as exc:+ import traceback+ out.put((rank, {"crash": f"{type(exc).__name__}: {exc}\n{traceback.format_exc()}"}))+ finally:+ try:+ dist.destroy_process_group()+ except Exception:+ pass+++def _run(+ case,+ tmp,+ world = 2,+ timeout = 600,+):+ import queue++ import torch.multiprocessing as mp++ ctx = mp.get_context("spawn")+ out = ctx.Queue()+ port = _free_port()+ procs = [+ ctx.Process(target = _worker, args = (r, world, port, out, case, tmp)) for r in range(world)+ ]+ for p in procs:+ p.start()+ results = {}+ try:+ for _ in procs:+ rank, r = out.get(timeout = timeout)+ results[rank] = r+ except queue.Empty:+ pass+ finally:+ for p in procs:+ p.join(timeout = 60)+ if p.is_alive():+ p.kill()+ assert sorted(results) == list(range(world)), f"{case}: ranks did not report: {results}"+ for rank, r in results.items():+ print("OPTIM_UNSTEPPED", case, rank, r)+ assert "crash" not in r, r["crash"]+ return results+++def test_resume_with_a_parameter_never_stepped(tmp_path):+ pytest.importorskip("transformers")+ for rank, r in _run("resume", str(tmp_path)).items():+ # The setup: the saved optimizer lists the parameter but holds no state for it.+ assert r["saved_has_state"] is False and r["saved_in_file"] is False, r+ assert r["saved_in_groups"] is True, r+ for arm in ("file", "state"):+ assert f"{arm}_error" not in r, r+ assert r[f"{arm}_has_state"] is False, r+ assert r[f"{arm}_n_state"] == r["n_trainable"] - 1, r+ assert r[f"{arm}_loss"][0] == r[f"{arm}_loss"][1], r+ assert r[f"{arm}_after_diff"] == [], r+ assert r[f"{arm}_optim_diff"] == [], r+++def test_resume_refuses_another_models_optimizer(tmp_path):+ pytest.importorskip("transformers")+ for rank, r in _run("refuse", str(tmp_path)).items():+ assert r["dropped_error"], r+ assert r["other_error"], rdiff --git a/unsloth/models/fsdp2_native.py b/unsloth/models/fsdp2_native.pyindex 58f06cd5c21..11266a12593 100644--- a/unsloth/models/fsdp2_native.py+++ b/unsloth/models/fsdp2_native.py@@ -2134,15 +2134,64 @@ def native_load_optimizer(model, optimizer, path_or_state):# A checkpoint from torch's AdamW resumed into a local-shard bitsandbytes Adam, or back.rename_optimizer_state_for(_inner_optimizer(optimizer), state)- set_optimizer_state_dict(- _native_root(model),- _inner_optimizer(optimizer),- state,- options = StateDictOptions(full_state_dict = True),- )+ root = _native_root(model)+ inner = _inner_optimizer(optimizer)+ unstepped = _unstepped_optimizer_params(root, inner, state)+ # torch refuses a trainable parameter without saved state, and strict = False would keep the+ # fresh init state for it. Hiding these (requires_grad off) leaves them stateless, as saved.+ try:+ for param in unstepped:+ param.requires_grad_(False)+ set_optimizer_state_dict(+ root,+ inner,+ state,+ options = StateDictOptions(full_state_dict = True),+ )+ finally:+ for param in unstepped:+ param.requires_grad_(True)+ for param in unstepped:+ inner.state.pop(param, None)return state+def _unstepped_optimizer_params(root, optimizer, state):+ """Trainable parameters listed in the saved param groups but with no saved state (never got a+ gradient). Raises for one not listed at all: another model's optimizer."""+ saved_state = state.get("state") or {}+ if not isinstance(saved_state, dict) or all(isinstance(k, int) for k in saved_state):+ return [] # not keyed by FQN: torch loads it as given+ try:+ from torch.distributed.checkpoint.state_dict import _get_fqns+ except ImportError:+ _get_fqns = None+ fqns = {}+ for name, param in root.named_parameters(remove_duplicate = False):+ names = _get_fqns(root, name) if _get_fqns is not None else {name}+ fqns.setdefault(param, set()).update(names)+ listed = {f for g in state.get("param_groups") or [] for f in (g.get("params") or [])}+ saved = set(saved_state)+ unstepped, seen, unknown = [], set(), []+ for group in optimizer.param_groups:+ for param in group["params"]:+ names = fqns.get(param)+ if not param.requires_grad or not names or names & saved:+ continue+ if names & listed:+ if id(param) not in seen:+ seen.add(id(param))+ unstepped.append(param)+ else:+ unknown.append(sorted(names)[0])+ if unknown:+ raise RuntimeError(+ f"Unsloth: the optimizer checkpoint has no entry for {unknown[0]} ({len(unknown)} "+ "parameters), so it was not saved from this model's optimizer."+ )+ return unstepped++# Sharded (torch.distributed.checkpoint) resume. The optimizer directory is opt-in# (UNSLOTH_FSDP2_DCP_OPTIM=1): only a native FSDP2 run can read it back, while the full optimizer.pt# also loads into a plain (single GPU) Trainer resume, which would silently skip a directory it does