Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Keep paged bitsandbytes optimizer state paged on checkpoint resume (#13092)

Daniel Han0f14817ed0be
diff --git a/tests/test_bnb_paged_optimizer_resume.py b/tests/test_bnb_paged_optimizer_resume.py
new file mode 100644
index 00000000000..6207b664e79
--- /dev/null
+++ b/tests/test_bnb_paged_optimizer_resume.py
@@ -0,0 +1,140 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+"""Each case runs in a subprocess: the patch rebinds bitsandbytes' Optimizer8bit.load_state_dict."""
+
+import json
+import os
+import subprocess
+import sys
+import textwrap
+
+import pytest
+
+pytest.importorskip("bitsandbytes")
+torch = pytest.importorskip("torch")
+if not torch.cuda.is_available():
+ pytest.skip(reason = "paged bitsandbytes optimizers need a CUDA device", allow_module_level = True)
+
+_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+
+_RUNNER = textwrap.dedent(
+ """
+ import importlib.util, io, json, sys
+ import torch, bitsandbytes as bnb
+ mode, optim_name = sys.argv[1], sys.argv[2]
+ if mode == "fixed":
+ spec = importlib.util.spec_from_file_location("unsloth_import_fixes_under_test", sys.argv[3])
+ fixes = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(fixes)
+ fixes.patch_bitsandbytes_paged_optimizer_resume()
+ fixes.patch_bitsandbytes_paged_optimizer_resume() # idempotent
+ Optim = getattr(bnb.optim, optim_name)
+ torch.manual_seed(0)
+ shapes = [(2048, 1024), (64, 2)] # one paged (>= 1e5 elements), one too small to page
+ params = [torch.nn.Parameter(torch.randn(s, device = "cuda")) for s in shapes]
+ grads = [[torch.randn(s, device = "cuda") for s in shapes] for _ in range(4)]
+
+ def step(opt, ps, g):
+ for p, x in zip(ps, g):
+ p.grad = x.clone()
+ opt.step()
+
+ opt = Optim(params, lr = 1e-2)
+ step(opt, params, grads[0]); step(opt, params, grads[1])
+ buf = io.BytesIO(); torch.save(opt.state_dict(), buf); buf.seek(0)
+ saved = torch.load(buf, map_location = "cpu", weights_only = True) # Trainer resume path
+
+ resumed_params = [torch.nn.Parameter(p.detach().clone()) for p in params]
+ resumed = Optim(resumed_params, lr = 1e-2)
+ managed = bnb.functional.GlobalPageManager.get_instance().paged_tensors
+
+ def is_managed(t):
+ return any(t is m for m in managed)
+
+ def paged(o, p):
+ return {k: is_managed(o.state[p][k]) for k in ("state1", "state2") if k in o.state[p]}
+
+ def placed(o, p): # every tensor is managed or on the parameter's device, as in a fresh run
+ return all(is_managed(v) or v.device == p.device for v in o.state[p].values() if torch.is_tensor(v))
+
+ torch.cuda.synchronize(); before_load = torch.cuda.memory_allocated()
+ resumed.load_state_dict(saved)
+ torch.cuda.synchronize(); load_cuda_bytes = torch.cuda.memory_allocated() - before_load
+ after_load = paged(resumed, resumed_params[0])
+ placed_after_load = all(placed(resumed, p) for p in resumed_params)
+ for g in grads[2:]:
+ step(opt, params, g); step(resumed, resumed_params, g)
+
+ state_bytes = sum(v.numel() * v.element_size() for k, v in opt.state[params[0]].items() if k in ("state1", "state2"))
+ print(json.dumps({
+ "state_bytes": state_bytes,
+ "load_cuda_bytes": load_cuda_bytes,
+ "fresh_paged": paged(opt, params[0]),
+ "after_load_paged": after_load,
+ "placed_after_load": placed_after_load,
+ "resumed_paged": paged(resumed, resumed_params[0]),
+ "resumed_small_paged": paged(resumed, resumed_params[1]),
+ "max_abs_diff": max((a - b).abs().max().item() for a, b in zip(params, resumed_params)),
+ }))
+ """
+)
+
+
+def _run(mode, optim_name):
+ out = subprocess.run(
+ [
+ sys.executable,
+ "-c",
+ _RUNNER,
+ mode,
+ optim_name,
+ os.path.join(_ROOT, "unsloth", "import_fixes.py"),
+ ],
+ capture_output = True,
+ text = True,
+ timeout = 600,
+ )
+ assert out.returncode == 0, out.stderr[-4000:]
+ return json.loads(out.stdout.strip().splitlines()[-1])
+
+
+ "optim_name", ["PagedAdamW8bit", "PagedAdamW32bit", "PagedLion8bit", "PagedAdEMAMix8bit"]
+)
+def test_resumed_paged_optimizer_state_stays_paged(optim_name):
+ import bitsandbytes
+
+ if not hasattr(bitsandbytes.optim, optim_name):
+ pytest.skip(reason = f"bitsandbytes {bitsandbytes.__version__} has no {optim_name}")
+ fixed = _run("fixed", optim_name)
+ assert fixed["fresh_paged"] and all(fixed["fresh_paged"].values())
+ # Paged from the load on, so the first resumed forward and backward never hold it in the
+ # CUDA allocator (a fresh run has no state there yet), and never staged there whole either.
+ assert fixed["after_load_paged"] == fixed["resumed_paged"] == fixed["fresh_paged"]
+ assert fixed["placed_after_load"]
+ assert fixed["load_cuda_bytes"] < fixed["state_bytes"] // 4
+ assert not any(fixed["resumed_small_paged"].values())
+ # Resuming continues the same trajectory as never stopping.
+ assert fixed["max_abs_diff"] == 0.0
+
+
+def test_non_paged_optimizer_untouched():
+ base = _run("base", "AdamW8bit")
+ fixed = _run("fixed", "AdamW8bit")
+ assert base == fixed
+ assert fixed["placed_after_load"]
+ assert not any(fixed["resumed_paged"].values())
diff --git a/unsloth/_gpu_init.py b/unsloth/_gpu_init.py
index e965fc50c5f..33bdaad2980 100644
--- a/unsloth/_gpu_init.py
+++ b/unsloth/_gpu_init.py
@@ -718,6 +718,12 @@ elif DEVICE_TYPE == "xpu":
pass
+# After the bitsandbytes import above, never before: it only patches an already-imported bitsandbytes.
+from .import_fixes import patch_bitsandbytes_paged_optimizer_resume
+
+patch_bitsandbytes_paged_optimizer_resume()
+del patch_bitsandbytes_paged_optimizer_resume
+
from .models import *
from .models import __version__
from .save import *
diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.py
index 93a3ee219d7..171d2c29226 100644
--- a/unsloth/import_fixes.py
+++ b/unsloth/import_fixes.py
@@ -5105,6 +5105,75 @@ def patch_unsafe_trainer_rng_load():
logger.info("Unsloth: Hardened Trainer._load_rng_state rng loading (CVE-2026-1839).")
+def patch_bitsandbytes_paged_optimizer_resume():
+ """Keep paged bitsandbytes optimizer state paged after a checkpoint resume (#2168).
+ Optimizer8bit.load_state_dict moves every state1/state2 into plain CUDA memory, so a resumed
+ paged_* run holds that state in the CUDA allocator (or OOMs loading it), where a fresh run
+ keeps it in paged memory that can spill to CPU. Load it on the host instead and copy each
+ tensor into the buffer a fresh run would allocate (get_state_buffer, or AdEMAMix's doubled
+ state1); everything else still moves to the parameter's device as before."""
+ if "bitsandbytes" not in sys.modules:
+ return
+ try:
+ from bitsandbytes.optim.optimizer import Optimizer8bit
+ except Exception:
+ return
+ load_state_dict = getattr(Optimizer8bit, "load_state_dict", None)
+ if (
+ load_state_dict is None
+ or not hasattr(Optimizer8bit, "get_state_buffer")
+ or getattr(load_state_dict, "_unsloth_repage", False)
+ ):
+ return
+ try:
+ stages_on_host = "move_to_device" in inspect.signature(load_state_dict).parameters
+ except (TypeError, ValueError):
+ stages_on_host = False
+
+ import torch
+
+ def _paged_buffer(self, p, value):
+ if value.shape == p.shape:
+ buffer = self.get_state_buffer(p, dtype = value.dtype)
+ elif value.shape == (2, *p.shape) and hasattr(self, "_get_state_double_buffer"):
+ buffer = self._get_state_double_buffer(p, dtype = value.dtype)
+ else:
+ return None
+ return buffer if getattr(buffer, "is_paged", False) else None
+
+ @functools.wraps(load_state_dict)
+ def _unsloth_load_state_dict(self, state_dict, *args, **kwargs):
+ if not getattr(self, "is_paged", False):
+ return load_state_dict(self, state_dict, *args, **kwargs)
+ move_to_device = kwargs.pop("move_to_device", args[0] if args else True)
+ if stages_on_host:
+ result = load_state_dict(self, state_dict, False, *args[1:], **kwargs)
+ else:
+ result = load_state_dict(self, state_dict, *args, **kwargs)
+ non_castable = getattr(self, "non_castable_tensor_keys", ("state1", "state2"))
+ for group in self.param_groups:
+ for p in group["params"]:
+ state = self.state.get(p)
+ if not state:
+ continue
+ for key, value in state.items():
+ # No is_paged check: torch.save keeps that attribute, so a loaded host copy still claims it.
+ if key not in non_castable or not isinstance(value, torch.Tensor):
+ continue
+ buffer = None
+ if move_to_device and key in ("state1", "state2") and p.device.type != "cpu":
+ buffer = _paged_buffer(self, p, value)
+ if buffer is not None:
+ buffer.copy_(value)
+ state[key] = buffer
+ elif move_to_device and stages_on_host:
+ state[key] = value.to(p.device)
+ return result
+
+ _unsloth_load_state_dict._unsloth_repage = True
+ Optimizer8bit.load_state_dict = _unsloth_load_state_dict
+
+
_PT2_UNSAFE_LOAD_ENV = "UNSLOTH_ALLOW_UNSAFE_PT2_LOAD"
# torch.export's .pt2 readers: their weights_only=False loads unpickle archive bytes.
_PT2_LOADER_MODULES = frozenset(