Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Resume native FSDP2 optimizer checkpoints across torch AdamW and bnb 8-bit (#15092)
Daniel Hanf6d79cfb652d
diff --git a/tests/test_fsdp2_native_dcp_bnb_8bit_gloo.py b/tests/test_fsdp2_native_dcp_bnb_8bit_gloo.pynew file mode 100644index 00000000000..fa1641fd4e1--- /dev/null+++ b/tests/test_fsdp2_native_dcp_bnb_8bit_gloo.py@@ -0,0 +1,414 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""The sharded optimizer checkpoint (optimizer_dcp/, UNSLOTH_FSDP2_DCP_OPTIM=1) with the local-shard+bitsandbytes AdamW8bit under native FSDP2 (gloo CPU world, bitsandbytes' CPU kernels):++* save + resume at the same world size restores the 8-bit state (uint8 codes, absmax, qmap) bitwise+ and the next steps equal the uninterrupted run, bitwise; optimizer.pt does the same;+* a Trainer with optim = 'adamw_8bit' writes optimizer_dcp/ and resumes to the uninterrupted weights;+* a torch AdamW optimizer_dcp/ resumed into AdamW8bit (exp_avg -> state1) equals the optimizer.pt+ path of the same checkpoint, bitwise;+* save at one world size, resume at another: the restored (dequantized) state matches the saver's+ within the 8-bit quantization error, and steps."""++import os+import sys++import pytest+import torch++if __import__("multiprocessing").parent_process() is not None or getattr(+ __import__("multiprocessing").current_process(), "_inheriting", False+):+ os.environ["CUDA_VISIBLE_DEVICES"] = ""++sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))+from test_fsdp2_native_units import _load_native # noqa: E402+from test_fsdp2_native_dcp_resume_gloo import ( # noqa: E402+ _batch,+ _diff_models,+ _free_port,+ _full_model,+ _full_optim,+ _init,+ _model,+ _step,+)++_MIN_8BIT = 256 # the 16 x 16 attention projections and the MLP get 8-bit state too+++def _bnb(model):+ import bitsandbytes as bnb++ from unsloth.models.fsdp2_bnb_optim import localize_bnb_optimizer++ opt = bnb.optim.AdamW8bit(+ model.parameters(), lr = 1e-2, weight_decay = 0.01, min_8bit_size = _MIN_8BIT+ )+ assert localize_bnb_optimizer(opt)+ return opt+++def _torch(model):+ return torch.optim.AdamW(model.parameters(), lr = 1e-2, foreach = False)+++def _new(native, seed, make):+ model = native.shard_model(_model(seed), native.FSDP2Config(param_dtype = torch.float32))+ return model, make(model)+++def _raw(model, opt):+ """This rank's raw optimizer state per parameter (uint8 codes, absmax, qmap, step)."""+ out = {}+ for name, p in model.named_parameters():+ state = opt.state.get(p, {})+ out[name] = {k: v.detach().clone() if torch.is_tensor(v) else v for k, v in state.items()}+ return out+++def _diff_raw(a, b):+ out = sorted(set(a) ^ set(b))+ for n in a:+ if n not in b:+ continue+ if sorted(a[n]) != sorted(b[n]):+ out.append((n, sorted(a[n]), sorted(b[n])))+ continue+ for k, v in a[n].items():+ w = b[n][k]+ same = (+ torch.is_tensor(w) and v.dtype == w.dtype and torch.equal(v, w)+ if torch.is_tensor(v)+ else v == w+ )+ if not same:+ out.append((n, k))+ return out+++_PAIRS = (("state1", "exp_avg"), ("state2", "exp_avg_sq"))+++def _max_rel(a, b):+ """Largest relative difference of the Adam moments of two full (FQN-keyed) optimizer states,+ either optimizer's key names."""+ rel = 0.0+ for fqn, entry in a.items():+ other = b[fqn]+ for keys in _PAIRS:+ x = next(entry[k] for k in keys if k in entry).float()+ y = next(other[k] for k in keys if k in other).float()+ rel = max(rel, ((x - y).norm() / (x.norm() + 1e-30)).item())+ return rel+++def _uint8_names(model, opt):+ return sorted(+ n+ for n, p in model.named_parameters()+ if opt.state.get(p, {}).get("state1") is not None+ and opt.state[p]["state1"].dtype == torch.uint8+ )+++def _save(native, model, opt, ckpt, rank, dist):+ os.makedirs(ckpt, exist_ok = True)+ weights = native.native_full_state_dict(model)+ if rank == 0:+ model.save_pretrained(ckpt, state_dict = weights, safe_serialization = True)+ native.native_save_optimizer(+ model, opt, os.path.join(ckpt, "optimizer.pt"), should_save = rank == 0+ )+ native.native_save_optimizer_sharded(model, opt, os.path.join(ckpt, "optimizer_dcp"))+ dist.barrier()+++def _case_same_world(native, dist, rank, tmp):+ model, opt = _new(native, 1, _bnb)+ for step in (1, 2):+ _step(model, opt, step, rank)+ ckpt = os.path.join(tmp, "ckpt")+ _save(native, model, opt, ckpt, rank, dist)+ saved_raw = _raw(model, opt)+ res = {"uint8": _uint8_names(model, opt), "n_params": len(saved_raw)}+ want = [_step(model, opt, step, rank).item() for step in (3, 4)]+ want_after = _full_model(model)+ for arm, src in (("dcp", "optimizer_dcp"), ("pt", "optimizer.pt")):+ m, o = _new(native, 2, _bnb)+ native.native_load_model_checkpoint(m, ckpt)+ try:+ native.native_load_optimizer(m, o, os.path.join(ckpt, src))+ got = {"raw_diff": [str(d)[:200] for d in _diff_raw(saved_raw, _raw(m, o))]}+ got["losses"] = [_step(m, o, step, rank).item() for step in (3, 4)]+ got["want"] = want+ got["after_diff"] = _diff_models(want_after, _full_model(m))+ except BaseException as exc: # CheckpointException is a BaseException+ got = {"error": f"{type(exc).__name__}: {str(exc)[:400]}"}+ res[arm] = got+ return res+++def _case_cross(native, dist, rank, tmp, saver, loader):+ """One optimizer saved, the other resumed (torch AdamW <-> AdamW8bit): the sharded directory gives+ what optimizer.pt gives, bitwise."""+ model, opt = _new(native, 1, saver)+ for step in (1, 2):+ _step(model, opt, step, rank)+ ckpt = os.path.join(tmp, "ckpt")+ _save(native, model, opt, ckpt, rank, dist)+ saved = _full_optim(model, opt)["state"]+ arms = {}+ res = {}+ for arm, src in (("pt", "optimizer.pt"), ("dcp", "optimizer_dcp")):+ m, o = _new(native, 2, loader)+ native.native_load_model_checkpoint(m, ckpt)+ try:+ native.native_load_optimizer(m, o, os.path.join(ckpt, src))+ arms[arm] = {"raw": _raw(m, o), "uint8": _uint8_names(m, o)}+ arms[arm]["keys"] = sorted({k for st in o.state.values() for k in st})+ arms[arm]["group_keys"] = sorted(o.param_groups[0])+ arms[arm]["max_rel"] = _max_rel(saved, _full_optim(m, o)["state"])+ arms[arm]["losses"] = [_step(m, o, step, rank).item() for step in (3, 4)]+ arms[arm]["after"] = _full_model(m)+ except BaseException as exc: # CheckpointException is a BaseException+ res[f"{arm}_error"] = f"{type(exc).__name__}: {str(exc)[:400]}"+ if len(arms) == 2:+ res["uint8"] = arms["dcp"]["uint8"]+ res["keys"] = arms["dcp"]["keys"]+ res["max_rel"] = arms["dcp"]["max_rel"]+ res["group_keys"] = [arms["pt"]["group_keys"], arms["dcp"]["group_keys"]]+ res["steps"] = sorted(+ {float(v["step"]) for v in arms["dcp"]["raw"].values() if "step" in v}+ )+ res["raw_diff"] = [str(d)[:200] for d in _diff_raw(arms["pt"]["raw"], arms["dcp"]["raw"])]+ res["losses"] = [arms["pt"]["losses"], arms["dcp"]["losses"]]+ res["after_diff"] = _diff_models(arms["pt"]["after"], arms["dcp"]["after"])+ return res+++def _case_reshard(native, dist, rank, tmp, phase):+ """AdamW8bit optimizer_dcp/ saved at one world size and resumed at another. The 8-bit blocks follow+ each rank's shard, so the restored state is re-quantized on other blocks: compared dequantized."""+ ckpt = os.path.join(tmp, "ckpt")+ ref = os.path.join(tmp, "ref.pt")+ if phase == "save":+ model, opt = _new(native, 1, _bnb)+ for step in (1, 2):+ _step(model, opt, step, rank)+ _save(native, model, opt, ckpt, rank, dist)+ full = {"model": _full_model(model), "optim": _full_optim(model, opt)}+ if rank == 0:+ torch.save(full, ref)+ dist.barrier()+ return {"world": dist.get_world_size()}+ model, opt = _new(native, 2, _bnb)+ native.native_load_model_checkpoint(model, ckpt)+ res = {"world": dist.get_world_size()}+ try:+ native.native_load_optimizer(model, opt, os.path.join(ckpt, "optimizer_dcp"))+ except BaseException as exc: # CheckpointException is a BaseException+ res["error"] = f"{type(exc).__name__}: {str(exc)[:400]}"+ return res+ want = torch.load(ref, weights_only = False)+ got = _full_optim(model, opt)+ res["model_diff"] = _diff_models(want["model"], _full_model(model))+ res["missing"] = sorted(set(want["optim"]["state"]) ^ set(got["state"]))+ res["max_rel"] = _max_rel(want["optim"]["state"], got["state"])+ res["steps"] = sorted(+ {+ (entry.get("step"), got["state"][fqn].get("step"))+ for fqn, entry in want["optim"]["state"].items()+ }+ )+ res["loss"] = _step(model, opt, 3, rank).item() # the restored state steps+ return res+++def _case_trainer(native, dist, rank, tmp):+ from datasets import Dataset+ from transformers import Trainer, TrainingArguments++ from test_fsdp2_native_trainer_args_gloo import _install_wrapped_trainer_init++ _install_wrapped_trainer_init()+ cfg = native.FSDP2Config(param_dtype = torch.float32)+ rows = [+ {"input_ids": list(range(5 + i, 21 + i)), "labels": list(range(5 + i, 21 + i))}+ for i in range(4)+ ]+ ds = Dataset.from_list(rows * 4)++ def _args(out):+ return TrainingArguments(+ output_dir = out,+ report_to = [],+ use_cpu = True,+ optim = "adamw_8bit",+ max_steps = 4,+ save_steps = 2,+ save_strategy = "steps",+ learning_rate = 1e-2,+ per_device_train_batch_size = 2,+ lr_scheduler_type = "linear",+ logging_steps = 1,+ seed = 0,+ data_seed = 0,+ dataloader_num_workers = 0,+ )++ os.environ["UNSLOTH_FSDP2_DCP_OPTIM"] = "1"+ out = os.path.join(tmp, "out")+ try:+ model = native.shard_model(_model(1), cfg)+ trainer = Trainer(model = model, args = _args(out), train_dataset = ds)+ trainer.train()+ inner = native._inner_optimizer(trainer.optimizer)+ want = _full_model(model)+ ck2 = os.path.join(out, "checkpoint-2")+ files = sorted(os.listdir(ck2))+ dist.barrier()+ resumed = native.shard_model(_model(2), cfg)+ Trainer(model = resumed, args = _args(out), train_dataset = ds).train(resume_from_checkpoint = ck2)+ got = _full_model(resumed)+ dist.barrier()+ finally:+ os.environ.pop("UNSLOTH_FSDP2_DCP_OPTIM", None)+ return {+ "local_shards": getattr(type(inner), "_unsloth_fsdp2_local_shards", False),+ "uint8": _uint8_names(native._native_root(model), inner),+ "files": files,+ "diff": _diff_models(want, got),+ }+++def _worker(rank, world, port, out, case, tmp):+ dist = _init(rank, world, port, timeout = 120)+ try:+ native = _load_native()+ if case.startswith("reshard_"):+ res = _case_reshard(native, dist, rank, tmp, case[len("reshard_") :])+ elif case == "torch_to_bnb":+ res = _case_cross(native, dist, rank, tmp, _torch, _bnb)+ elif case == "bnb_to_torch":+ res = _case_cross(native, dist, rank, tmp, _bnb, _torch)+ else:+ res = {"same_world": _case_same_world, "trainer": _case_trainer}[case](+ native, dist, rank, tmp+ )+ out.put((rank, res))+ except BaseException as exc: # CheckpointException is a BaseException+ import traceback+ out.put((rank, {"crash": f"{type(exc).__name__}: {exc}\n{traceback.format_exc()}"}))+ finally:+ try:+ dist.destroy_process_group()+ except Exception:+ pass+++def _run(+ case,+ tmp,+ timeout = 600,+ world = 2,+):+ import queue++ import torch.multiprocessing as mp++ ctx = mp.get_context("spawn")+ out = ctx.Queue()+ port = _free_port()+ procs = [+ ctx.Process(target = _worker, args = (r, world, port, out, case, tmp)) for r in range(world)+ ]+ for p in procs:+ p.start()+ results = {}+ try:+ for _ in procs:+ rank, r = out.get(timeout = timeout)+ results[rank] = r+ except queue.Empty:+ pass+ finally:+ for p in procs:+ p.join(timeout = 60)+ if p.is_alive():+ p.kill()+ assert sorted(results) == list(+ range(world)+ ), f"{case}: ranks did not report within {timeout} s: {results}"+ for rank, r in results.items():+ print("DCP_BNB8", case, rank, {k: v for k, v in r.items() if k != "crash"})+ assert "crash" not in r, r["crash"]+ return results+++def _skip_without_cpu_8bit():+ bnb = pytest.importorskip("bitsandbytes")+ pytest.importorskip("transformers")+ probe = torch.nn.Parameter(torch.zeros(8))+ probe.grad = torch.ones(8)+ try:+ bnb.optim.AdamW8bit([probe], min_8bit_size = 0).step()+ except RuntimeError as exc: # bitsandbytes <= 0.48: 8-bit kernels are CUDA only+ pytest.skip(f"this bitsandbytes cannot step 8-bit state on the CPU ({str(exc)[:60]})")+++def test_sharded_bnb_8bit_resume_is_bitwise(tmp_path):+ _skip_without_cpu_8bit()+ for rank, r in _run("same_world", str(tmp_path)).items():+ assert len(r["uint8"]) > 2, r # embedding, lm_head and the projections hold 8-bit state+ for arm in ("dcp", "pt"):+ assert "error" not in r[arm], (arm, r[arm])+ assert r[arm]["raw_diff"] == [], (arm, r[arm]["raw_diff"])+ assert r[arm]["losses"] == r[arm]["want"], (arm, r[arm])+ assert r[arm]["after_diff"] == [], (arm, r[arm])++[email protected]("case", ["torch_to_bnb", "bnb_to_torch"])+def test_sharded_checkpoint_resumes_across_torch_adamw_and_bnb_8bit(case, tmp_path):+ _skip_without_cpu_8bit()+ for rank, r in _run(case, str(tmp_path)).items():+ # Each optimizer's own hyperparameter keys (amsgrad / foreach vs alpha / t_alpha) used to+ # make both loaders refuse the other optimizer's checkpoint.+ assert "pt_error" not in r and "dcp_error" not in r, r+ assert r["group_keys"][0] == r["group_keys"][1], r+ if case == "torch_to_bnb":+ assert len(r["uint8"]) > 2 and "state1" in r["keys"], r+ assert "amsgrad" not in r["group_keys"][1] and "alpha" in r["group_keys"][1], r+ else:+ assert "exp_avg" in r["keys"] and "state1" not in r["keys"], r+ assert "amsgrad" in r["group_keys"][1] and "alpha" not in r["group_keys"][1], r+ assert r["steps"] == [2.0], r+ # The moments carried over: exactly into torch's float32 state, within 8-bit error into bnb.+ assert r["max_rel"] == 0.0 if case == "bnb_to_torch" else r["max_rel"] < 2e-2, r+ assert r["raw_diff"] == [], r["raw_diff"]+ assert r["losses"][0] == r["losses"][1], r+ assert r["after_diff"] == [], r+++def test_trainer_adamw_8bit_resumes_from_sharded_optimizer(tmp_path):+ _skip_without_cpu_8bit()+ pytest.importorskip("datasets")+ for rank, r in _run("trainer", str(tmp_path), timeout = 900).items():+ assert r["local_shards"] and r["uint8"], r+ assert "optimizer_dcp" in r["files"] and "optimizer.pt" not in r["files"], r+ assert r["diff"] == [], r++[email protected]("save_world, load_world", [(2, 1), (1, 2)])+def test_sharded_bnb_8bit_resume_on_another_world_size(save_world, load_world, tmp_path):+ _skip_without_cpu_8bit()+ _run("reshard_save", str(tmp_path), world = save_world)+ for rank, r in _run("reshard_load", str(tmp_path), world = load_world).items():+ assert r["world"] == load_world and "error" not in r, r+ assert r["model_diff"] == [] and r["missing"] == [], r+ assert r["steps"] == [(2, 2)], r+ # Re-quantized on the new world's blocks: within 8-bit blockwise error, not bitwise.+ assert r["max_rel"] < 2e-2, r+ assert r["loss"] == r["loss"], rdiff --git a/unsloth/models/fsdp2_bnb_optim.py b/unsloth/models/fsdp2_bnb_optim.pyindex cea78485838..30400a19de8 100644--- a/unsloth/models/fsdp2_bnb_optim.py+++ b/unsloth/models/fsdp2_bnb_optim.py@@ -471,17 +471,44 @@ def rename_optimizer_state_for(optimizer, full_state):to_torch = isinstance(optimizer, (torch.optim.Adam, torch.optim.AdamW))if not (to_bnb or to_torch):return full_state+ renamed = Falsefor entry in states.values():if not isinstance(entry, dict):continuefor bnb_key, torch_key in _TORCH_ADAM_KEYS.items():if to_bnb and torch_key in entry and bnb_key not in entry:entry[bnb_key] = entry.pop(torch_key).float()+ renamed = Trueelif to_torch and bnb_key in entry and torch_key not in entry:entry[torch_key] = entry.pop(bnb_key)+ renamed = Truestep = entry.get("step")if to_bnb and torch.is_tensor(step):entry["step"] = int(step.item())elif to_torch and step is not None and not torch.is_tensor(step):entry["step"] = torch.tensor(float(step))+ if renamed:+ _align_param_group_keys(optimizer, full_state)return full_state+++def _align_param_group_keys(optimizer, full_state):+ """Saved param groups from the other optimizer carry its own hyperparameter keys (torch AdamW's+ amsgrad / foreach / ..., bitsandbytes' alpha / t_alpha / ...), which torch's state dict loaders+ refuse. Keep the keys both optimizers have (lr, betas, eps, weight_decay: the saved values), drop+ the saved-only ones and take this optimizer's own value for the rest. Only called for a+ checkpoint of the other optimizer (its state keys were renamed)."""+ saved_groups = full_state.get("param_groups")+ live_groups = getattr(optimizer, "param_groups", None)+ if not isinstance(saved_groups, list) or not live_groups:+ return+ if len(saved_groups) != len(live_groups):+ return # torch's loader reports the group mismatch itself+ for saved, live in zip(saved_groups, live_groups):+ if not isinstance(saved, dict):+ continue+ for key in [k for k in saved if k != "params" and k not in live]:+ del saved[key]+ for key, value in live.items():+ if key != "params" and key not in saved:+ saved[key] = valuediff --git a/unsloth/models/fsdp2_native.py b/unsloth/models/fsdp2_native.pyindex ae0c85fe953..58f06cd5c21 100644--- a/unsloth/models/fsdp2_native.py+++ b/unsloth/models/fsdp2_native.py@@ -2244,7 +2244,12 @@ def native_load_optimizer_sharded(model, optimizer, directory):# The target layout: this rank's shards (empty optimizer state is initialised first).state = get_optimizer_state_dict(root, optimizer, options = options)unstepped = _match_saved_optimizer_keys(state, directory, root)+ # Hyperparameter keys the checkpoint lacks (another optimizer class, or an older torch) keep this+ # optimizer's values; dcp.load would refuse them as missing.+ unsaved = _drop_unsaved_group_keys(state, directory)dcp.load(state, checkpoint_id = os.fspath(directory), process_group = group)+ for index, key, value in unsaved:+ state["param_groups"][index].setdefault(key, value)if unstepped:# Parameters the saved optimizer held but never stepped: listed in its (now loaded) param# groups, without state. Anything else without state is not this optimizer's checkpoint.@@ -2265,6 +2270,23 @@ def native_load_optimizer_sharded(model, optimizer, directory):return state+def _drop_unsaved_group_keys(state, directory):+ """Removes (and returns as (group index, key, value)) the param group entries of the dcp.load+ target that the checkpoint does not hold."""+ from torch.distributed.checkpoint import FileSystemReader++ metadata = FileSystemReader(os.fspath(directory)).read_metadata().state_dict_metadata+ dropped = []+ for index, group in enumerate(state.get("param_groups") or []):+ prefix = f"param_groups.{index}."+ if not isinstance(group, dict) or not any(k.startswith(prefix) for k in metadata):+ continue # not flattened per key: leave it to dcp.load+ for key in [k for k in group if k != "params"]:+ if prefix + key not in metadata:+ dropped.append((index, key, group.pop(key)))+ return dropped++def _match_saved_optimizer_keys(state,directory,@@ -2290,6 +2312,14 @@ def _match_saved_optimizer_keys(entry = per_param[fqn]for name in [n for n in entry if f"state.{fqn}.{n}" not in metadata]:del entry[name]+ for name, value in list(entry.items()):+ # Saved as a tensor where this optimizer keeps a plain value (torch AdamW's tensor step+ # resumed into bitsandbytes' int step): read the tensor, the rename converts it.+ md = metadata[f"state.{fqn}.{name}"]+ size = getattr(md, "size", None)+ dtype = getattr(getattr(md, "properties", None), "dtype", None)+ if not isinstance(value, torch.Tensor) and size is not None and dtype is not None:+ entry[name] = torch.empty(tuple(size), dtype = dtype)if not entry:del per_param[fqn]unstepped.append(fqn)