Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Add opt-in optimizer state CPU offload (UNSLOTH_OFFLOAD_OPTIMIZER_STATE=1) (#13882)
Daniel Hane539ad5edac1
diff --git a/tests/test_optimizer_state_offload.py b/tests/test_optimizer_state_offload.pynew file mode 100644index 00000000000..d1388d2b5ee--- /dev/null+++ b/tests/test_optimizer_state_offload.py@@ -0,0 +1,547 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.+#+# This program is free software: you can redistribute it and/or modify+# it under the terms of the GNU Affero General Public License as published+# by the Free Software Foundation, either version 3 of the License, or+# (at your option) any later version.+#+# This program is distributed in the hope that it will be useful,+# but WITHOUT ANY WARRANTY; without even the implied warranty of+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+# GNU Affero General Public License for more details.+#+# You should have received a copy of the GNU Affero General Public License+# along with this program. If not, see <https://www.gnu.org/licenses/>.++"""UNSLOTH_OFFLOAD_OPTIMIZER_STATE: loaded from its file so the CPU cases need no GPU unsloth import."""++import importlib.util+import io+import os+import types++import pytest+import torch++_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))+_PATH = os.path.join(_ROOT, "unsloth", "optimizers", "state_offload.py")+++def _module():+ spec = importlib.util.spec_from_file_location("_unsloth_state_offload", _PATH)+ module = importlib.util.module_from_spec(spec)+ spec.loader.exec_module(module)+ return module+++def _fake_trainer(optimizer):+ trainer = types.SimpleNamespace(optimizer = optimizer, is_deepspeed_enabled = False)+ trainer.calls = 0++ def create_scheduler(num_training_steps, optimizer = None):+ trainer.calls += 1+ return "scheduler"++ trainer.create_scheduler = create_scheduler+ return trainer+++def test_env_off_is_a_no_op(monkeypatch):+ monkeypatch.delenv("UNSLOTH_OFFLOAD_OPTIMIZER_STATE", raising = False)+ module = _module()+ optimizer = torch.optim.AdamW([torch.nn.Parameter(torch.randn(4))])+ trainer = _fake_trainer(optimizer)+ original = trainer.create_scheduler+ assert module.install_optimizer_state_offload(trainer) is False+ assert trainer.create_scheduler is original+ monkeypatch.setenv("UNSLOTH_OFFLOAD_OPTIMIZER_STATE", "0")+ assert module.install_optimizer_state_offload(trainer) is False+ assert trainer.create_scheduler is original+++def test_env_on_wraps_create_scheduler(monkeypatch):+ monkeypatch.setenv("UNSLOTH_OFFLOAD_OPTIMIZER_STATE", "1")+ module = _module()+ optimizer = torch.optim.AdamW([torch.nn.Parameter(torch.randn(4))])+ trainer = _fake_trainer(optimizer)+ assert module.install_optimizer_state_offload(trainer) is True+ wrapped = trainer.create_scheduler+ assert module.install_optimizer_state_offload(trainer) is True+ assert trainer.create_scheduler is wrapped+ assert trainer.create_scheduler(10) == "scheduler"+ assert trainer.calls == 1+ # CPU parameters: nothing to offload, optimizer untouched.+ assert getattr(optimizer, "_unsloth_state_offloader", None) is None+ assert "load_state_dict" not in vars(optimizer)+++def test_deepspeed_is_skipped(monkeypatch):+ monkeypatch.setenv("UNSLOTH_OFFLOAD_OPTIMIZER_STATE", "1")+ module = _module()+ trainer = _fake_trainer(None)+ trainer.is_deepspeed_enabled = True+ original = trainer.create_scheduler+ assert module.install_optimizer_state_offload(trainer) is False+ assert trainer.create_scheduler is original+++def _train(+ optimizer_factory,+ offload,+ steps = 6,+ seed = 0,+ chunk_bytes = None,+):+ module = _module()+ torch.manual_seed(seed)+ model = torch.nn.Sequential(+ torch.nn.Linear(64, 8192), torch.nn.GELU(), torch.nn.Linear(8192, 64)+ ).cuda()+ optimizer = optimizer_factory(model.parameters())+ # Bound before offloading, as accelerate keeps it and swaps it back in on every fp16 step.+ step_fn = optimizer.step+ if offload:+ assert module.offload_optimizer_state(optimizer, chunk_bytes = chunk_bytes) is not None+ x = torch.randn(32, 64, device = "cuda", generator = None)+ losses = []+ for step in range(steps):+ loss = (model(x + step) ** 2).mean()+ loss.backward()+ torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)+ step_fn()+ optimizer.zero_grad(set_to_none = True)+ losses.append(loss.item())+ return model, optimizer, losses+++def _state_devices(optimizer):+ return {+ v.device.type+ for state in optimizer.state.values()+ for v in state.values()+ if isinstance(v, torch.Tensor) and v.numel() > 1+ }+++_FACTORIES = {+ "adamw": lambda p: torch.optim.AdamW(p, lr = 1e-3, weight_decay = 0.01),+ "adamw_foreach": lambda p: torch.optim.AdamW(p, lr = 1e-3, foreach = True),+ "adamw_fused": lambda p: torch.optim.AdamW(p, lr = 1e-3, fused = True),+}+++def _bnb_factory():+ bnb = pytest.importorskip("bitsandbytes")+ return lambda p: bnb.optim.AdamW8bit(p, lr = 1e-3)++[email protected](not torch.cuda.is_available(), reason = "needs CUDA")[email protected]("name", ["adamw", "adamw_foreach", "adamw_fused", "adamw_8bit"])[email protected]("chunk_bytes", [None, 1])+def test_offload_matches_on_device_steps_and_resumes(name, chunk_bytes):+ factory = _bnb_factory() if name == "adamw_8bit" else _FACTORIES[name]+ base_model, base_opt, base_losses = _train(factory, offload = False)+ head_model, head_opt, head_losses = _train(factory, offload = True, chunk_bytes = chunk_bytes)+ assert head_losses == base_losses+ for a, b in zip(base_model.parameters(), head_model.parameters()):+ assert torch.equal(a, b)+ assert _state_devices(base_opt) == {"cuda"}+ assert _state_devices(head_opt) == {"cpu"}+ assert head_opt._unsloth_state_offloader.steps == 6++ # Checkpoint written while offloaded, loaded the way HF Trainer does (map_location = "cpu").+ buffer = io.BytesIO()+ torch.save(head_opt.state_dict(), buffer)+ buffer.seek(0)+ saved = torch.load(buffer, map_location = "cpu", weights_only = False)+ resumed_model, resumed_opt, _ = _train(factory, offload = True, steps = 0, chunk_bytes = chunk_bytes)+ resumed_model.load_state_dict(head_model.state_dict())+ resumed_opt.load_state_dict(saved)+ assert _state_devices(resumed_opt) == {"cpu"}++ x = torch.randn(32, 64, device = "cuda")+ for model, optimizer in ((base_model, base_opt), (resumed_model, resumed_opt)):+ (model(x) ** 2).mean().backward()+ optimizer.step()+ optimizer.zero_grad(set_to_none = True)+ for a, b in zip(base_model.parameters(), resumed_model.parameters()):+ assert torch.equal(a, b)+ assert _state_devices(resumed_opt) == {"cpu"}++[email protected](not torch.cuda.is_available(), reason = "needs CUDA")+def test_chunked_step_bounds_device_state_and_keeps_grads():+ module = _module()+ params = [torch.nn.Parameter(torch.randn(1 << 20, device = "cuda")) for _ in range(4)]+ optimizer = torch.optim.AdamW(params, lr = 1e-3)+ offloader = module.offload_optimizer_state(optimizer, chunk_bytes = 9 << 20)+ for p in params:+ p.grad = torch.ones_like(p)+ optimizer.step()+ for p in params:+ p.grad = torch.ones_like(p)+ grads = [p.grad for p in params]+ torch.cuda.synchronize()+ base = torch.cuda.memory_allocated()+ torch.cuda.reset_peak_memory_stats()+ optimizer.step()+ peak = torch.cuda.max_memory_allocated() - base+ assert all(p.grad is g for p, g in zip(params, grads))+ # One parameter's state (2 x 4 MiB) on device at a time, not all four (32 MiB).+ assert peak < 16 << 20+ assert _state_devices(optimizer) == {"cpu"}+ assert offloader.steps == 2+++class _GroupCountingSGD(torch.optim.SGD):+ # Like schedulefree: group-level bookkeeping advances on every step call.+ def step(self, closure = None):+ for group in self.param_groups:+ group["k"] = group.get("k", 0) + 1+ return super().step(closure)+++def test_group_global_optimizer_steps_once_per_step():+ module = _module()+ params = [torch.nn.Parameter(torch.randn(16)) for _ in range(3)]+ optimizer = _GroupCountingSGD(params, lr = 0.1)+ offloader = module._StateOffloader(optimizer, chunk_bytes = 1)+ optimizer.register_step_pre_hook(offloader._pre_step)+ optimizer.register_step_post_hook(offloader._post_step)+ expected = [p.detach() - 0.1 for p in params]+ for p in params:+ p.grad = torch.ones_like(p)+ optimizer.step()+ assert optimizer.param_groups[0]["k"] == 1+ for p, e in zip(params, expected):+ assert torch.allclose(p, e)+++def test_chunk_safety_by_step_owner():+ module = _module()+ p = [torch.nn.Parameter(torch.randn(2))]+ assert module._chunk_safe(torch.optim.AdamW(p))+ assert not module._chunk_safe(_GroupCountingSGD(p, lr = 0.1))+++def test_unstepped_chunk_size_counts_amsgrad_moment():+ module = _module()+ n = 1000+ params = [torch.nn.Parameter(torch.randn(n)) for _ in range(2)]+ # 8 bytes per fp32 parameter would fit both in 20n; amsgrad holds 12n each.+ plain = module._StateOffloader(torch.optim.AdamW(params), chunk_bytes = 20 * n)+ assert len(plain._chunks(params)) == 1+ amsgrad = module._StateOffloader(torch.optim.AdamW(params, amsgrad = True), chunk_bytes = 20 * n)+ assert len(amsgrad._chunks(params)) == 2+++class _Projector:+ __slots__ = ("ortho_matrix", "rank")++ def __init__(self, ortho_matrix):+ self.ortho_matrix = ortho_matrix+ self.rank = 2+++def test_nested_state_tensors_are_sized():+ module = _module()+ param = torch.nn.Parameter(torch.randn(4))+ optimizer = torch.optim.AdamW([param])+ optimizer.state[param]["projector"] = _Projector(torch.zeros(1000))+ offloader = module._StateOffloader(optimizer, chunk_bytes = 1)+ assert [name for name, _ in module._tensor_attrs(optimizer.state[param]["projector"])] == [+ "ortho_matrix"+ ]+ other = torch.nn.Parameter(torch.randn(4))+ assert len(offloader._chunks([param, other])) == 2++[email protected](not torch.cuda.is_available(), reason = "needs CUDA")+def test_nested_state_object_round_trip():+ module = _module()+ param = torch.nn.Parameter(torch.randn(64, device = "cuda"))+ optimizer = torch.optim.AdamW([param], lr = 1e-3)+ param.grad = torch.ones_like(param)+ optimizer.step()+ ortho = torch.randn(64, 8, device = "cuda")+ projector = _Projector(ortho.clone())+ optimizer.state[param]["projector"] = projector+ offloader = module.offload_optimizer_state(optimizer)+ assert projector.ortho_matrix.device.type == "cpu"+ assert offloader.offloaded_bytes >= ortho.numel() * 4+ offloader.restore()+ assert projector.ortho_matrix.device.type == "cuda"+ assert torch.equal(projector.ortho_matrix, ortho)+ param.grad = torch.ones_like(param)+ optimizer.step()+ assert projector.ortho_matrix.device.type == "cpu"++[email protected](not torch.cuda.is_available(), reason = "needs CUDA")+def test_layerwise_optimizer_dict_is_offloaded():+ module = _module()+ LayerWiseDummyOptimizer = pytest.importorskip(+ "transformers.trainer_pt_utils"+ ).LayerWiseDummyOptimizer+ params = [torch.nn.Parameter(torch.randn(256, device = "cuda")) for _ in range(2)]+ optimizer_dict = {p: torch.optim.AdamW([p], lr = 1e-3) for p in params}+ dummy = LayerWiseDummyOptimizer(optimizer_dict = optimizer_dict, lr = 1e-3)+ offloaders = module.offload_optimizer_state(dummy)+ assert offloaders is not None and len(offloaders) == 2+ for p in params:+ p.grad = torch.ones_like(p)+ optimizer_dict[p].step()+ for inner in optimizer_dict.values():+ assert _state_devices(inner) == {"cpu"}+++def _hooked(module, optimizer, chunk_bytes):+ offloader = module._StateOffloader(optimizer, chunk_bytes = chunk_bytes)+ optimizer.register_step_pre_hook(offloader._pre_step)+ optimizer.register_step_post_hook(offloader._post_step)+ return offloader+++def test_chunked_step_synchronizes_chunk_copies(monkeypatch):+ module = _module()+ params = [torch.nn.Parameter(torch.randn(16)) for _ in range(3)]+ offloader = _hooked(module, torch.optim.AdamW(params), chunk_bytes = 1)+ offload = offloader.offload+ # CPU tensors are never copied, so stand in for the device a chunk copy was issued on.+ monkeypatch.setattr(+ offloader, "offload", lambda params = None: offload(params) | ({"dev"} if params else set())+ )+ synced = []+ monkeypatch.setattr(module, "_synchronize", synced.append)+ for p in params:+ p.grad = torch.ones_like(p)+ offloader.optimizer.step()+ assert synced == ["dev"]+++def test_unstepped_chunk_size_counts_centered_rmsprop_momentum():+ module = _module()+ n = 1000+ params = [torch.nn.Parameter(torch.randn(n)) for _ in range(2)]+ plain = module._StateOffloader(torch.optim.RMSprop(params, centered = True), chunk_bytes = 20 * n)+ assert len(plain._chunks(params)) == 1+ both = torch.optim.RMSprop(params, centered = True, momentum = 0.9)+ assert len(module._StateOffloader(both, chunk_bytes = 20 * n)._chunks(params)) == 2+++def test_single_chunk_restores_only_active_state(monkeypatch):+ module = _module()+ active, idle = torch.nn.Parameter(torch.randn(16)), torch.nn.Parameter(torch.randn(16))+ offloader = _hooked(module, torch.optim.AdamW([active, idle]), chunk_bytes = None)+ restored = []+ monkeypatch.setattr(offloader, "restore", lambda params = None: restored.append(params))+ active.grad = torch.ones_like(active)+ offloader.optimizer.step()+ assert restored == [[active]]+++def test_chunked_step_runs_external_hooks_once():+ module = _module()+ params = [torch.nn.Parameter(torch.randn(16)) for _ in range(3)]+ optimizer = torch.optim.AdamW(params, lr = 0.1)+ _hooked(module, optimizer, chunk_bytes = 1)+ calls = []+ optimizer.register_step_pre_hook(lambda *a: calls.append("pre"))+ optimizer.register_step_post_hook(lambda *a: calls.append("post"))+ reference = torch.optim.AdamW([torch.nn.Parameter(p.detach().clone()) for p in params], lr = 0.1)+ for p, q in zip(params, reference.param_groups[0]["params"]):+ p.grad = torch.ones_like(p)+ q.grad = torch.ones_like(q)+ optimizer.step()+ reference.step()+ assert calls == ["pre", "post"]+ for p, q in zip(params, reference.param_groups[0]["params"]):+ assert torch.equal(p, q)+++def test_list_state_tensors_are_sized():+ module = _module()+ params = [torch.nn.Parameter(torch.randn(4)) for _ in range(2)]+ optimizer = torch.optim.AdamW(params)+ optimizer.state[params[0]]["old_dirs"] = [torch.zeros(1000)]+ assert len(module._StateOffloader(optimizer, chunk_bytes = 100)._chunks(params)) == 2++[email protected](not torch.cuda.is_available(), reason = "needs CUDA")+def test_lbfgs_history_is_offloaded():+ module = _module()+ torch.manual_seed(0)+ target = torch.randn(64, device = "cuda")++ def run(offload):+ p = torch.nn.Parameter(torch.zeros(64, device = "cuda"))+ optimizer = torch.optim.LBFGS([p], lr = 0.5, history_size = 4, max_iter = 3)+ if offload:+ module.offload_optimizer_state(optimizer)++ def closure():+ optimizer.zero_grad()+ loss = ((p - target) ** 2).sum()+ loss.backward()+ return loss++ for _ in range(3):+ optimizer.step(closure)+ return p.detach(), optimizer.state[p]++ base, _ = run(False)+ head, state = run(True)+ assert torch.allclose(base, head)+ assert state["old_dirs"] and all(+ t.device.type == "cpu" for t in state["old_dirs"] + state["ro"]+ )+++def test_chunked_step_runs_after_later_pre_hooks():+ module = _module()+ params = [torch.nn.Parameter(torch.randn(16)) for _ in range(3)]+ reference = [torch.nn.Parameter(p.detach().clone()) for p in params]++ def halve_and_fill(optimizer, args, kwargs):+ # Clips existing gradients and creates the missing one, as a later library hook might.+ for group in optimizer.param_groups:+ group["lr"] = 0.05+ for p in group["params"]:+ p.grad = torch.full_like(p, 0.5) if p.grad is None else p.grad * 0.5++ optimizer = torch.optim.AdamW(params, lr = 0.1)+ offloader = _hooked(module, optimizer, chunk_bytes = 1)+ optimizer.register_step_pre_hook(halve_and_fill)+ plain = torch.optim.AdamW(reference, lr = 0.1)+ plain.register_step_pre_hook(halve_and_fill)+ for p, q in zip(params[:2], reference[:2]):+ p.grad = torch.ones_like(p)+ q.grad = torch.ones_like(q)+ optimizer.step()+ plain.step()+ for p, q in zip(params, reference):+ assert torch.equal(p, q)+ assert torch.equal(optimizer.state[p]["exp_avg"], plain.state[q]["exp_avg"])+ assert offloader.hidden is None and offloader.steps == 1+++def test_positional_none_closure_still_chunks():+ # accelerate calls optimizer.step(closure) with closure = None.+ module = _module()+ params = [torch.nn.Parameter(torch.randn(16)) for _ in range(3)]+ reference = torch.optim.AdamW([torch.nn.Parameter(p.detach().clone()) for p in params], lr = 0.1)+ optimizer = torch.optim.AdamW(params, lr = 0.1)+ chunked = []+ offloader = _hooked(module, optimizer, chunk_bytes = 1)+ restore = offloader.restore+ offloader.restore = lambda params = None: (chunked.append(params), restore(params))+ for p, q in zip(params, reference.param_groups[0]["params"]):+ p.grad = torch.ones_like(p)+ q.grad = torch.ones_like(q)+ assert optimizer.step(None) is None+ reference.step()+ assert chunked == [[p] for p in params]+ for p, q in zip(params, reference.param_groups[0]["params"]):+ assert torch.equal(p, q)+++def _big_params(n = 4, numel = 1 << 20):+ return [torch.nn.Parameter(torch.randn(numel, device = "cuda")) for _ in range(n)]++[email protected](not torch.cuda.is_available(), reason = "needs CUDA")[email protected]("fused", [False, True])+def test_load_state_dict_keeps_state_on_host(fused):+ module = _module()+ params = _big_params()+ source = torch.optim.AdamW(params, lr = 1e-3, fused = fused)+ for p in params:+ p.grad = torch.ones_like(p)+ source.step()+ saved = {+ "state": {+ k: {n: t.cpu().clone() for n, t in v.items()}+ for k, v in source.state_dict()["state"].items()+ },+ "param_groups": source.state_dict()["param_groups"],+ }+ clones = [torch.nn.Parameter(p.detach().clone()) for p in params]+ optimizer = torch.optim.AdamW(clones, lr = 1e-3, fused = fused)+ offloader = module.offload_optimizer_state(optimizer, chunk_bytes = 9 << 20)+ torch.cuda.synchronize()+ base = torch.cuda.memory_allocated()+ torch.cuda.reset_peak_memory_stats()+ optimizer.load_state_dict(saved)+ torch.cuda.synchronize()+ # All four parameters' moments would be 32 MiB on device.+ assert torch.cuda.max_memory_allocated() - base < 1 << 20+ assert _state_devices(optimizer) == {"cpu"}+ # Fused steps keep "step" on the parameter's device, the others on the host.+ step_device = offloader.offloaded.get((clones[0], "step"))+ assert (step_device.type if step_device is not None else "cpu") == ("cuda" if fused else "cpu")+ for p, q in zip(params, clones):+ p.grad = torch.ones_like(p)+ q.grad = torch.ones_like(q)+ source.step()+ optimizer.step()+ for p, q in zip(params, clones):+ assert torch.equal(p, q)+ assert _state_devices(optimizer) == {"cpu"}++[email protected](not torch.cuda.is_available(), reason = "needs CUDA")+def test_resumed_bitsandbytes_first_step_stays_chunked():+ bnb = pytest.importorskip("bitsandbytes")+ module = _module()+ params = _big_params()+ source = bnb.optim.AdamW8bit(params, lr = 1e-3)+ for p in params:+ p.grad = torch.ones_like(p)+ source.step()+ buffer = io.BytesIO()+ torch.save(source.state_dict(), buffer)+ buffer.seek(0)+ saved = torch.load(buffer, map_location = "cpu", weights_only = False)+ clones = [torch.nn.Parameter(p.detach().clone()) for p in params]+ optimizer = bnb.optim.AdamW8bit(clones, lr = 1e-3)+ module.offload_optimizer_state(optimizer, chunk_bytes = 3 << 20)+ optimizer.load_state_dict(saved)+ for p, q in zip(params, clones):+ p.grad = torch.ones_like(p)+ q.grad = torch.ones_like(q)+ torch.cuda.synchronize()+ base = torch.cuda.memory_allocated()+ torch.cuda.reset_peak_memory_stats()+ optimizer.step()+ peak = torch.cuda.max_memory_allocated() - base+ source.step()+ # One parameter's 8-bit moments (about 2 MiB) at a time, not all four (about 8 MiB).+ assert peak < 5 << 20+ for p, q in zip(params, clones):+ assert torch.equal(p, q)+ assert _state_devices(optimizer) == {"cpu"}+++def test_unstepped_chunk_size_counts_the_ademamix_double_moment():+ module = _module()+ n = 1000+ params = [torch.nn.Parameter(torch.randn(n)) for _ in range(2)]++ class AdEMAMix(torch.optim.AdamW):+ pass++ plain = module._StateOffloader(torch.optim.AdamW(params), chunk_bytes = 20 * n)+ assert len(plain._chunks(params)) == 1+ ademamix = module._StateOffloader(AdEMAMix(params), chunk_bytes = 20 * n)+ assert len(ademamix._chunks(params)) == 2+++def test_mps_synchronize_takes_no_device(monkeypatch):+ module = _module()+ calls = []+ monkeypatch.setattr(+ torch, "mps", types.SimpleNamespace(synchronize = lambda *a: calls.append(a)), raising = False+ )+ module._synchronize(torch.device("mps"))+ assert calls == [()]diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.pyindex b1ebe0ac311..93656d1695c 100644--- a/unsloth/models/_utils.py+++ b/unsloth/models/_utils.py@@ -5557,6 +5557,9 @@ def patch_gradient_accumulation_fix(Trainer):_replan_auto_offload_safely(self)_block_swap_for_trainer_backend(self)_enable_ddp_batched_grad_copy(self)+ if os.environ.get("UNSLOTH_OFFLOAD_OPTIMIZER_STATE", "0") != "0":+ from ..optimizers.state_offload import install_optimizer_state_offload+ install_optimizer_state_offload(self)_unsloth_trainer_init.__wrapped__ = _original_trainer_initTrainer.__init__ = _unsloth_trainer_initdiff --git a/unsloth/optimizers/state_offload.py b/unsloth/optimizers/state_offload.pynew file mode 100644index 00000000000..d3c04d458de--- /dev/null+++ b/unsloth/optimizers/state_offload.py@@ -0,0 +1,468 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""Optimizer-state-only CPU offload (UNSLOTH_OFFLOAD_OPTIMIZER_STATE=1).++State lives in pinned host memory between steps and visits the GPU only inside optimizer.step,+so peak memory is max(activations, optimizer state) instead of their sum. The step runs in chunks+of UNSLOTH_OFFLOAD_OPTIMIZER_STATE_CHUNK_MB (default 1024) of state, so the step itself never holds+all of it on device either (parameter-local optimizers only, see _CHUNK_SAFE_MODULES).+"""++import copy+import os++import torch++__all__ = [+ "optimizer_state_offload_enabled",+ "offload_optimizer_state",+ "install_optimizer_state_offload",+]++_ENV = "UNSLOTH_OFFLOAD_OPTIMIZER_STATE"++try:+ from torch.distributed.tensor import DTensor+except Exception:+ DTensor = None+++def _chunk_bytes():+ try:+ mb = float(os.environ.get(_ENV + "_CHUNK_MB", "1024"))+ except ValueError:+ mb = 1024.0+ return int(mb * 2**20) if mb > 0 else float("inf")+++def optimizer_state_offload_enabled():+ return os.environ.get(_ENV, "0").strip().lower() in ("1", "true", "yes", "on")+++def _innermost(optimizer):+ # accelerate's AcceleratedOptimizer is itself an Optimizer subclass wrapping .optimizer.+ while isinstance(getattr(optimizer, "optimizer", None), torch.optim.Optimizer):+ optimizer = optimizer.optimizer+ return optimizer+++# Classes whose step only touches parameters that have a gradient, so stepping one chunk at a time+# is the same update. Others (schedulefree advances group["k"] on every call) step unchunked.+_CHUNK_SAFE_MODULES = (+ "torch.optim.",+ "bitsandbytes.",+ "transformers.optimization",+ "unsloth.optimizers.",+)+++def _chunk_safe(optimizer):+ for cls in type(optimizer).__mro__:+ if "step" in vars(cls):+ return cls.__module__.startswith(_CHUNK_SAFE_MODULES)+ return False+++def _tensor_attrs(value):+ # Tensors held by state objects such as GaLoreProjector (__slots__, no __dict__).+ if isinstance(value, (torch.Tensor, dict, list, tuple, str, int, float, bool)) or value is None:+ return ()+ names = set(getattr(value, "__dict__", ()))+ for cls in type(value).__mro__:+ slots = vars(cls).get("__slots__", ())+ names.update((slots,) if isinstance(slots, str) else slots)+ return [+ (name, getattr(value, name))+ for name in sorted(names)+ if isinstance(getattr(value, name, None), torch.Tensor)+ ]+++def _held_tensors(value):+ if isinstance(value, torch.Tensor):+ return [value]+ # LBFGS keeps its history (old_dirs, old_stps, ro) as lists of tensors.+ if isinstance(value, (list, tuple)):+ return [v for v in value if isinstance(v, torch.Tensor)]+ return [t for _, t in _tensor_attrs(value)]+++def _source_device(local, param, key, device_of):+ if local.device.type != "cpu":+ return local.device+ return device_of(param, key, local) if device_of is not None else None+++def _synchronize(device):+ module = getattr(torch, device.type, None)+ sync = getattr(module, "synchronize", None)+ if sync is not None:+ # torch.mps.synchronize takes no device argument.+ sync() if device.type == "mps" else sync(device)+++class _StateOffloader:+ def __init__(+ self,+ optimizer,+ pin_memory = True,+ chunk_bytes = None,+ ):+ self.optimizer = optimizer+ self.pin_memory = pin_memory+ self.chunk_bytes = chunk_bytes if chunk_bytes is not None else _chunk_bytes()+ self.buffers = {}+ self.offloaded = {}+ self.offloaded_bytes = 0+ self.steps = 0+ self.inside = False+ self.hidden = None+ self.pending = set()++ def _host_buffer(self, param, key, src):+ buffer = self.buffers.get((param, key))+ if buffer is None or buffer.shape != src.shape or buffer.dtype != src.dtype:+ try:+ buffer = torch.empty(+ src.shape, dtype = src.dtype, device = "cpu", pin_memory = self.pin_memory+ )+ except RuntimeError:+ # No pinnable host memory (some WSL / container setups): pageable still works.+ self.pin_memory = False+ buffer = torch.empty(src.shape, dtype = src.dtype, device = "cpu")+ self.buffers[(param, key)] = buffer+ buffer.copy_(src, non_blocking = self.pin_memory)+ return buffer++ def _states(self, params):+ if params is None:+ return list(self.optimizer.state.items())+ state = self.optimizer.state+ return [(p, state[p]) for p in params if p in state]++ def offload(+ self,+ params = None,+ device_of = None,+ ):+ # device_of(param, key, tensor) names the device of state that is already on the host+ # (a checkpoint just loaded there), None to leave it host-only.+ devices = set()+ for param, state in self._states(params):+ for key, value in list(state.items()):+ if isinstance(value, (list, tuple)):+ items = list(value)+ for i, local in enumerate(items):+ if not isinstance(local, torch.Tensor):+ continue+ device = _source_device(local, param, (key, i), device_of)+ if device is not None:+ devices.add(local.device)+ items[i] = self._host_buffer(param, (key, i), local)+ self.offloaded[(param, (key, i))] = device+ if isinstance(value, list):+ value[:] = items+ else:+ state[key] = type(value)(items)+ continue+ for name, local in _tensor_attrs(value):+ device = _source_device(local, param, (key, name), device_of)+ if device is None:+ continue+ devices.add(local.device)+ setattr(value, name, self._host_buffer(param, (key, name), local))+ self.offloaded[(param, (key, name))] = device+ is_dtensor = DTensor is not None and isinstance(value, DTensor)+ local = value._local_tensor if is_dtensor else value+ if not isinstance(local, torch.Tensor):+ continue+ device = _source_device(local, param, key, device_of)+ if device is None:+ continue+ devices.add(local.device)+ host = self._host_buffer(param, key, local)+ self.offloaded[(param, key)] = device+ if is_dtensor:+ value = copy.copy(value)+ value._local_tensor = host+ state[key] = value+ else:+ state[key] = host+ return devices++ def finish(self, devices):+ # The host copies must be complete before anything reads them (torch.save, the next step).+ for device in devices:+ _synchronize(device)+ self.offloaded_bytes = sum(b.numel() * b.element_size() for b in self.buffers.values())++ def restore(self, params = None):+ for param, state in self._states(params):+ for key, value in list(state.items()):+ if isinstance(value, (list, tuple)):+ items = list(value)+ for i, local in enumerate(items):+ device = self.offloaded.get((param, (key, i)))+ if (+ device is not None+ and isinstance(local, torch.Tensor)+ and local.device != device+ ):+ items[i] = local.to(device, non_blocking = True)+ if isinstance(value, list):+ value[:] = items+ else:+ state[key] = type(value)(items)+ continue+ for name, local in _tensor_attrs(value):+ device = self.offloaded.get((param, (key, name)))+ if device is not None and local.device != device:+ setattr(value, name, local.to(device, non_blocking = True))+ device = self.offloaded.get((param, key))+ if device is None:+ continue+ is_dtensor = DTensor is not None and isinstance(value, DTensor)+ local = value._local_tensor if is_dtensor else value+ if not isinstance(local, torch.Tensor) or local.device == device:+ continue+ local = local.to(device, non_blocking = True)+ if is_dtensor:+ value = copy.copy(value)+ value._local_tensor = local+ state[key] = value+ else:+ state[key] = local++ def _chunks(self, params):+ # Optimizers update only parameters that have a gradient, so stepping one chunk at a time+ # (other gradients hidden) bounds the on-device state to chunk_bytes instead of all of it.+ chunks, chunk, size = [], [], 0+ state = self.optimizer.state+ groups = {p: g for g in self.optimizer.param_groups for p in g["params"]}+ for p in params:+ held = [t for v in state.get(p, {}).values() for t in _held_tensors(v)]+ if held:+ nbytes = sum(v.numel() * v.element_size() for v in held)+ else:+ # Not stepped yet: two moments, a third with amsgrad or centered RMSprop with+ # momentum (square_avg, momentum_buffer, grad_avg), at least fp32 each.+ group = groups.get(p, {})+ # bitsandbytes AdEMAMix keeps a double-sized state1 next to state2.+ third = (+ group.get("amsgrad")+ or (group.get("centered") and group.get("momentum", 0) > 0)+ or "ademamix" in type(self.optimizer).__name__.lower()+ )+ moments = 3 if third else 2+ nbytes = moments * max(p.element_size(), 4) * p.numel()+ if chunk and size + nbytes > self.chunk_bytes:+ chunks.append(chunk)+ chunk, size = [], 0+ chunk.append(p)+ size += nbytes+ if chunk:+ chunks.append(chunk)+ return chunks++ def _pre_step(self, optimizer, args, kwargs):+ if self.inside:+ return+ # torch passes the step's own args to hooks, self included.+ has_self = bool(args) and args[0] is optimizer+ rest = args[1:] if has_self else args+ closure = rest[0] if rest else kwargs.get("closure")+ # A closure recomputes the loss on every call, so it must run once over all parameters.+ if closure is not None or not _chunk_safe(optimizer):+ self.restore()+ return++ # Chunk-safe steps call a closure first, before any update, so the chunked updates run+ # after every step pre hook (registered later ones too), as a plain step would.+ def _chunked_updates():+ self._step_chunks(optimizer, rest[1:], dict(kwargs))+ return None++ if rest:+ rest = (_chunked_updates,) + tuple(rest[1:])+ else:+ kwargs = dict(kwargs, closure = _chunked_updates)+ return ((optimizer,) + tuple(rest) if has_self else tuple(rest)), kwargs++ def _step_chunks(self, optimizer, args, kwargs):+ kwargs.pop("closure", None)+ params = [+ p for group in optimizer.param_groups for p in group["params"] if p.grad is not None+ ]+ chunks = self._chunks(params)+ # Chunk-safe steps never touch parameters without a gradient, so their state stays on host.+ if len(chunks) <= 1:+ self.restore(params)+ return+ # Step each chunk through the unhooked class step, so registered step hooks run once for+ # the logical step, then hide every gradient so the caller's own step finds nothing left.+ step = type(optimizer).step+ if getattr(step, "hooked", False):+ step = getattr(step, "__wrapped__", step)+ grads = [p.grad for p in params]+ devices = set()+ self.inside = True+ try:+ for chunk in chunks:+ members = set(chunk)+ for p, g in zip(params, grads):+ p.grad = g if p in members else None+ self.restore(chunk)+ step(optimizer, *args, **kwargs)+ devices |= self.offload(chunk)+ except BaseException:+ for p, g in zip(params, grads):+ p.grad = g+ self.finish(devices | self.offload())+ raise+ finally:+ self.inside = False+ for p in params:+ p.grad = None+ self.hidden = (params, grads)+ # The chunk copies are still in flight; the post hook's offload finds nothing left to copy.+ self.pending = devices++ def _post_step(self, optimizer, args, kwargs):+ if self.inside:+ return+ if self.hidden is not None:+ for p, g in zip(*self.hidden):+ p.grad = g+ self.hidden = None+ self.steps += 1+ devices, self.pending = self.pending, set()+ self.finish(devices | self.offload())+++def offload_optimizer_state(+ optimizer,+ pin_memory = True,+ chunk_bytes = None,+):+ """Keep `optimizer`'s state on the host between steps. Idempotent; returns the offloader or None."""+ optimizer = _innermost(optimizer)+ if not isinstance(optimizer, torch.optim.Optimizer):+ return None+ # transformers' LayerWiseDummyOptimizer (galore/apollo *_layerwise): the real per-parameter+ # optimizers live in optimizer_dict and step from gradient hooks.+ layerwise = getattr(optimizer, "optimizer_dict", None)+ if isinstance(layerwise, dict):+ unique = {id(o): o for o in layerwise.values()}.values()+ offloaders = [offload_optimizer_state(o, pin_memory, chunk_bytes) for o in unique]+ offloaders = [o for o in offloaders if o is not None]+ return offloaders or None+ existing = getattr(optimizer, "_unsloth_state_offloader", None)+ if existing is not None:+ return existing+ # Paged bitsandbytes state already spills to host through unified memory.+ if getattr(optimizer, "is_paged", False):+ return None+ if not any(p.device.type != "cpu" for group in optimizer.param_groups for p in group["params"]):+ return None++ offloader = _StateOffloader(optimizer, pin_memory = pin_memory, chunk_bytes = chunk_bytes)+ # Hooks, not an optimizer.step override: accelerate swaps the instance's step attribute on+ # every fp16 step, and hooks run whichever bound step a caller kept.+ optimizer.register_step_pre_hook(offloader._pre_step)+ optimizer.register_step_post_hook(offloader._post_step)++ # Instance override, not register_load_state_dict_post_hook: bitsandbytes replaces+ # Optimizer.load_state_dict and does not run torch's post hooks. Loading moves every+ # state tensor to its parameter's device, so offload again right after.+ load_state_dict = optimizer.load_state_dict++ def _load_state_dict_and_offload(state_dict, *args, **kwargs):+ moved = {}+ original = vars(torch.optim.Optimizer).get("_process_value_according_to_param_policy")+ if isinstance(original, staticmethod):+ policy = original.__func__++ # torch casts every loaded state tensor onto its parameter's device; keep it on the+ # host instead so a resume never holds the whole state on device at once.+ def _host_policy(param, value, *policy_args, **policy_kwargs):+ if param.device.type in ("cpu", "meta"):+ return policy(param, value, *policy_args, **policy_kwargs)+ stand_in = torch.empty(0, dtype = param.dtype, device = "meta")+ out = policy(stand_in, value, *policy_args, **policy_kwargs)+ if out.device.type != "meta":+ return out+ out = value.to(dtype = out.dtype)+ moved[id(out)] = param.device+ return out++ torch.optim.Optimizer._process_value_according_to_param_policy = staticmethod(+ _host_policy+ )+ try:+ result = load_state_dict(state_dict, *args, **kwargs)+ finally:+ if isinstance(original, staticmethod):+ torch.optim.Optimizer._process_value_according_to_param_policy = original+ if moved:+ device_of = lambda param, key, local: moved.get(id(local))+ elif callable(getattr(optimizer, "to_gpu", None)) and hasattr(optimizer, "initialized"):+ # bitsandbytes loads state on the host and its first step moves all of it to the device+ # (to_gpu), defeating the chunks: do its first-step setup here, minus that move.+ if not optimizer.initialized:+ if callable(getattr(optimizer, "check_overrides", None)):+ optimizer.check_overrides()+ optimizer.initialized = True+ device_of = lambda param, key, local: (+ param.device+ if not isinstance(key, tuple)+ and param.device.type != "cpu"+ and not getattr(local, "is_paged", False)+ else None+ )+ else:+ device_of = None+ offloader.finish(offloader.offload(device_of = device_of))+ return result++ optimizer.load_state_dict = _load_state_dict_and_offload+ optimizer._unsloth_state_offloader = offloader+ offloader.finish(offloader.offload())+ return offloader+++_PRINTED = False+++def install_optimizer_state_offload(trainer):+ """Offload the trainer's optimizer state once its optimizer is final (inside create_scheduler)."""+ if not optimizer_state_offload_enabled():+ return False+ if getattr(trainer, "is_deepspeed_enabled", False):+ print(+ "Unsloth: UNSLOTH_OFFLOAD_OPTIMIZER_STATE is ignored under DeepSpeed (use its offload_optimizer)."+ )+ return False+ create_scheduler = trainer.create_scheduler+ if getattr(create_scheduler, "_unsloth_state_offload", False):+ return True++ def _create_scheduler_offload_state(*args, **kwargs):+ global _PRINTED+ scheduler = create_scheduler(*args, **kwargs)+ optimizer = kwargs.get("optimizer", args[1] if len(args) > 1 else None)+ optimizer = optimizer if optimizer is not None else getattr(trainer, "optimizer", None)+ if (+ optimizer is not None+ and offload_optimizer_state(optimizer) is not None+ and not _PRINTED+ ):+ _PRINTED = True+ print(+ "Unsloth: Optimizer state is offloaded to CPU between steps (UNSLOTH_OFFLOAD_OPTIMIZER_STATE=1)."+ )+ return scheduler++ _create_scheduler_offload_state._unsloth_state_offload = True+ trainer.create_scheduler = _create_scheduler_offload_state+ return True