Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Keep paged bitsandbytes optimizer state paged on checkpoint resume (#13092)
Daniel Han0f14817ed0be
diff --git a/tests/test_bnb_paged_optimizer_resume.py b/tests/test_bnb_paged_optimizer_resume.pynew file mode 100644index 00000000000..6207b664e79--- /dev/null+++ b/tests/test_bnb_paged_optimizer_resume.py@@ -0,0 +1,140 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.+#+# This program is free software: you can redistribute it and/or modify+# it under the terms of the GNU Affero General Public License as published+# by the Free Software Foundation, either version 3 of the License, or+# (at your option) any later version.+#+# This program is distributed in the hope that it will be useful,+# but WITHOUT ANY WARRANTY; without even the implied warranty of+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+# GNU Affero General Public License for more details.+#+# You should have received a copy of the GNU Affero General Public License+# along with this program. If not, see <https://www.gnu.org/licenses/>.++"""Each case runs in a subprocess: the patch rebinds bitsandbytes' Optimizer8bit.load_state_dict."""++import json+import os+import subprocess+import sys+import textwrap++import pytest++pytest.importorskip("bitsandbytes")+torch = pytest.importorskip("torch")+if not torch.cuda.is_available():+ pytest.skip(reason = "paged bitsandbytes optimizers need a CUDA device", allow_module_level = True)++_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))++_RUNNER = textwrap.dedent(+ """+ import importlib.util, io, json, sys+ import torch, bitsandbytes as bnb+ mode, optim_name = sys.argv[1], sys.argv[2]+ if mode == "fixed":+ spec = importlib.util.spec_from_file_location("unsloth_import_fixes_under_test", sys.argv[3])+ fixes = importlib.util.module_from_spec(spec)+ spec.loader.exec_module(fixes)+ fixes.patch_bitsandbytes_paged_optimizer_resume()+ fixes.patch_bitsandbytes_paged_optimizer_resume() # idempotent+ Optim = getattr(bnb.optim, optim_name)+ torch.manual_seed(0)+ shapes = [(2048, 1024), (64, 2)] # one paged (>= 1e5 elements), one too small to page+ params = [torch.nn.Parameter(torch.randn(s, device = "cuda")) for s in shapes]+ grads = [[torch.randn(s, device = "cuda") for s in shapes] for _ in range(4)]++ def step(opt, ps, g):+ for p, x in zip(ps, g):+ p.grad = x.clone()+ opt.step()++ opt = Optim(params, lr = 1e-2)+ step(opt, params, grads[0]); step(opt, params, grads[1])+ buf = io.BytesIO(); torch.save(opt.state_dict(), buf); buf.seek(0)+ saved = torch.load(buf, map_location = "cpu", weights_only = True) # Trainer resume path++ resumed_params = [torch.nn.Parameter(p.detach().clone()) for p in params]+ resumed = Optim(resumed_params, lr = 1e-2)+ managed = bnb.functional.GlobalPageManager.get_instance().paged_tensors++ def is_managed(t):+ return any(t is m for m in managed)++ def paged(o, p):+ return {k: is_managed(o.state[p][k]) for k in ("state1", "state2") if k in o.state[p]}++ def placed(o, p): # every tensor is managed or on the parameter's device, as in a fresh run+ return all(is_managed(v) or v.device == p.device for v in o.state[p].values() if torch.is_tensor(v))++ torch.cuda.synchronize(); before_load = torch.cuda.memory_allocated()+ resumed.load_state_dict(saved)+ torch.cuda.synchronize(); load_cuda_bytes = torch.cuda.memory_allocated() - before_load+ after_load = paged(resumed, resumed_params[0])+ placed_after_load = all(placed(resumed, p) for p in resumed_params)+ for g in grads[2:]:+ step(opt, params, g); step(resumed, resumed_params, g)++ state_bytes = sum(v.numel() * v.element_size() for k, v in opt.state[params[0]].items() if k in ("state1", "state2"))+ print(json.dumps({+ "state_bytes": state_bytes,+ "load_cuda_bytes": load_cuda_bytes,+ "fresh_paged": paged(opt, params[0]),+ "after_load_paged": after_load,+ "placed_after_load": placed_after_load,+ "resumed_paged": paged(resumed, resumed_params[0]),+ "resumed_small_paged": paged(resumed, resumed_params[1]),+ "max_abs_diff": max((a - b).abs().max().item() for a, b in zip(params, resumed_params)),+ }))+ """+)+++def _run(mode, optim_name):+ out = subprocess.run(+ [+ sys.executable,+ "-c",+ _RUNNER,+ mode,+ optim_name,+ os.path.join(_ROOT, "unsloth", "import_fixes.py"),+ ],+ capture_output = True,+ text = True,+ timeout = 600,+ )+ assert out.returncode == 0, out.stderr[-4000:]+ return json.loads(out.stdout.strip().splitlines()[-1])+++ "optim_name", ["PagedAdamW8bit", "PagedAdamW32bit", "PagedLion8bit", "PagedAdEMAMix8bit"]+)+def test_resumed_paged_optimizer_state_stays_paged(optim_name):+ import bitsandbytes++ if not hasattr(bitsandbytes.optim, optim_name):+ pytest.skip(reason = f"bitsandbytes {bitsandbytes.__version__} has no {optim_name}")+ fixed = _run("fixed", optim_name)+ assert fixed["fresh_paged"] and all(fixed["fresh_paged"].values())+ # Paged from the load on, so the first resumed forward and backward never hold it in the+ # CUDA allocator (a fresh run has no state there yet), and never staged there whole either.+ assert fixed["after_load_paged"] == fixed["resumed_paged"] == fixed["fresh_paged"]+ assert fixed["placed_after_load"]+ assert fixed["load_cuda_bytes"] < fixed["state_bytes"] // 4+ assert not any(fixed["resumed_small_paged"].values())+ # Resuming continues the same trajectory as never stopping.+ assert fixed["max_abs_diff"] == 0.0+++def test_non_paged_optimizer_untouched():+ base = _run("base", "AdamW8bit")+ fixed = _run("fixed", "AdamW8bit")+ assert base == fixed+ assert fixed["placed_after_load"]+ assert not any(fixed["resumed_paged"].values())diff --git a/unsloth/_gpu_init.py b/unsloth/_gpu_init.pyindex e965fc50c5f..33bdaad2980 100644--- a/unsloth/_gpu_init.py+++ b/unsloth/_gpu_init.py@@ -718,6 +718,12 @@ elif DEVICE_TYPE == "xpu":pass+# After the bitsandbytes import above, never before: it only patches an already-imported bitsandbytes.+from .import_fixes import patch_bitsandbytes_paged_optimizer_resume++patch_bitsandbytes_paged_optimizer_resume()+del patch_bitsandbytes_paged_optimizer_resume+from .models import *from .models import __version__from .save import *diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.pyindex 93a3ee219d7..171d2c29226 100644--- a/unsloth/import_fixes.py+++ b/unsloth/import_fixes.py@@ -5105,6 +5105,75 @@ def patch_unsafe_trainer_rng_load():logger.info("Unsloth: Hardened Trainer._load_rng_state rng loading (CVE-2026-1839).")+def patch_bitsandbytes_paged_optimizer_resume():+ """Keep paged bitsandbytes optimizer state paged after a checkpoint resume (#2168).+ Optimizer8bit.load_state_dict moves every state1/state2 into plain CUDA memory, so a resumed+ paged_* run holds that state in the CUDA allocator (or OOMs loading it), where a fresh run+ keeps it in paged memory that can spill to CPU. Load it on the host instead and copy each+ tensor into the buffer a fresh run would allocate (get_state_buffer, or AdEMAMix's doubled+ state1); everything else still moves to the parameter's device as before."""+ if "bitsandbytes" not in sys.modules:+ return+ try:+ from bitsandbytes.optim.optimizer import Optimizer8bit+ except Exception:+ return+ load_state_dict = getattr(Optimizer8bit, "load_state_dict", None)+ if (+ load_state_dict is None+ or not hasattr(Optimizer8bit, "get_state_buffer")+ or getattr(load_state_dict, "_unsloth_repage", False)+ ):+ return+ try:+ stages_on_host = "move_to_device" in inspect.signature(load_state_dict).parameters+ except (TypeError, ValueError):+ stages_on_host = False++ import torch++ def _paged_buffer(self, p, value):+ if value.shape == p.shape:+ buffer = self.get_state_buffer(p, dtype = value.dtype)+ elif value.shape == (2, *p.shape) and hasattr(self, "_get_state_double_buffer"):+ buffer = self._get_state_double_buffer(p, dtype = value.dtype)+ else:+ return None+ return buffer if getattr(buffer, "is_paged", False) else None++ @functools.wraps(load_state_dict)+ def _unsloth_load_state_dict(self, state_dict, *args, **kwargs):+ if not getattr(self, "is_paged", False):+ return load_state_dict(self, state_dict, *args, **kwargs)+ move_to_device = kwargs.pop("move_to_device", args[0] if args else True)+ if stages_on_host:+ result = load_state_dict(self, state_dict, False, *args[1:], **kwargs)+ else:+ result = load_state_dict(self, state_dict, *args, **kwargs)+ non_castable = getattr(self, "non_castable_tensor_keys", ("state1", "state2"))+ for group in self.param_groups:+ for p in group["params"]:+ state = self.state.get(p)+ if not state:+ continue+ for key, value in state.items():+ # No is_paged check: torch.save keeps that attribute, so a loaded host copy still claims it.+ if key not in non_castable or not isinstance(value, torch.Tensor):+ continue+ buffer = None+ if move_to_device and key in ("state1", "state2") and p.device.type != "cpu":+ buffer = _paged_buffer(self, p, value)+ if buffer is not None:+ buffer.copy_(value)+ state[key] = buffer+ elif move_to_device and stages_on_host:+ state[key] = value.to(p.device)+ return result++ _unsloth_load_state_dict._unsloth_repage = True+ Optimizer8bit.load_state_dict = _unsloth_load_state_dict++_PT2_UNSAFE_LOAD_ENV = "UNSLOTH_ALLOW_UNSAFE_PT2_LOAD"# torch.export's .pt2 readers: their weights_only=False loads unpickle archive bytes._PT2_LOADER_MODULES = frozenset(