Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Add opt-in optimizer state CPU offload (UNSLOTH_OFFLOAD_OPTIMIZER_STATE=1) (#13882)

Daniel Hane539ad5edac1
diff --git a/tests/test_optimizer_state_offload.py b/tests/test_optimizer_state_offload.py
new file mode 100644
index 00000000000..d1388d2b5ee
--- /dev/null
+++ b/tests/test_optimizer_state_offload.py
@@ -0,0 +1,547 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+"""UNSLOTH_OFFLOAD_OPTIMIZER_STATE: loaded from its file so the CPU cases need no GPU unsloth import."""
+
+import importlib.util
+import io
+import os
+import types
+
+import pytest
+import torch
+
+_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+_PATH = os.path.join(_ROOT, "unsloth", "optimizers", "state_offload.py")
+
+
+def _module():
+ spec = importlib.util.spec_from_file_location("_unsloth_state_offload", _PATH)
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+ return module
+
+
+def _fake_trainer(optimizer):
+ trainer = types.SimpleNamespace(optimizer = optimizer, is_deepspeed_enabled = False)
+ trainer.calls = 0
+
+ def create_scheduler(num_training_steps, optimizer = None):
+ trainer.calls += 1
+ return "scheduler"
+
+ trainer.create_scheduler = create_scheduler
+ return trainer
+
+
+def test_env_off_is_a_no_op(monkeypatch):
+ monkeypatch.delenv("UNSLOTH_OFFLOAD_OPTIMIZER_STATE", raising = False)
+ module = _module()
+ optimizer = torch.optim.AdamW([torch.nn.Parameter(torch.randn(4))])
+ trainer = _fake_trainer(optimizer)
+ original = trainer.create_scheduler
+ assert module.install_optimizer_state_offload(trainer) is False
+ assert trainer.create_scheduler is original
+ monkeypatch.setenv("UNSLOTH_OFFLOAD_OPTIMIZER_STATE", "0")
+ assert module.install_optimizer_state_offload(trainer) is False
+ assert trainer.create_scheduler is original
+
+
+def test_env_on_wraps_create_scheduler(monkeypatch):
+ monkeypatch.setenv("UNSLOTH_OFFLOAD_OPTIMIZER_STATE", "1")
+ module = _module()
+ optimizer = torch.optim.AdamW([torch.nn.Parameter(torch.randn(4))])
+ trainer = _fake_trainer(optimizer)
+ assert module.install_optimizer_state_offload(trainer) is True
+ wrapped = trainer.create_scheduler
+ assert module.install_optimizer_state_offload(trainer) is True
+ assert trainer.create_scheduler is wrapped
+ assert trainer.create_scheduler(10) == "scheduler"
+ assert trainer.calls == 1
+ # CPU parameters: nothing to offload, optimizer untouched.
+ assert getattr(optimizer, "_unsloth_state_offloader", None) is None
+ assert "load_state_dict" not in vars(optimizer)
+
+
+def test_deepspeed_is_skipped(monkeypatch):
+ monkeypatch.setenv("UNSLOTH_OFFLOAD_OPTIMIZER_STATE", "1")
+ module = _module()
+ trainer = _fake_trainer(None)
+ trainer.is_deepspeed_enabled = True
+ original = trainer.create_scheduler
+ assert module.install_optimizer_state_offload(trainer) is False
+ assert trainer.create_scheduler is original
+
+
+def _train(
+ optimizer_factory,
+ offload,
+ steps = 6,
+ seed = 0,
+ chunk_bytes = None,
+):
+ module = _module()
+ torch.manual_seed(seed)
+ model = torch.nn.Sequential(
+ torch.nn.Linear(64, 8192), torch.nn.GELU(), torch.nn.Linear(8192, 64)
+ ).cuda()
+ optimizer = optimizer_factory(model.parameters())
+ # Bound before offloading, as accelerate keeps it and swaps it back in on every fp16 step.
+ step_fn = optimizer.step
+ if offload:
+ assert module.offload_optimizer_state(optimizer, chunk_bytes = chunk_bytes) is not None
+ x = torch.randn(32, 64, device = "cuda", generator = None)
+ losses = []
+ for step in range(steps):
+ loss = (model(x + step) ** 2).mean()
+ loss.backward()
+ torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
+ step_fn()
+ optimizer.zero_grad(set_to_none = True)
+ losses.append(loss.item())
+ return model, optimizer, losses
+
+
+def _state_devices(optimizer):
+ return {
+ v.device.type
+ for state in optimizer.state.values()
+ for v in state.values()
+ if isinstance(v, torch.Tensor) and v.numel() > 1
+ }
+
+
+_FACTORIES = {
+ "adamw": lambda p: torch.optim.AdamW(p, lr = 1e-3, weight_decay = 0.01),
+ "adamw_foreach": lambda p: torch.optim.AdamW(p, lr = 1e-3, foreach = True),
+ "adamw_fused": lambda p: torch.optim.AdamW(p, lr = 1e-3, fused = True),
+}
+
+
+def _bnb_factory():
+ bnb = pytest.importorskip("bitsandbytes")
+ return lambda p: bnb.optim.AdamW8bit(p, lr = 1e-3)
+
+
[email protected](not torch.cuda.is_available(), reason = "needs CUDA")
[email protected]("name", ["adamw", "adamw_foreach", "adamw_fused", "adamw_8bit"])
[email protected]("chunk_bytes", [None, 1])
+def test_offload_matches_on_device_steps_and_resumes(name, chunk_bytes):
+ factory = _bnb_factory() if name == "adamw_8bit" else _FACTORIES[name]
+ base_model, base_opt, base_losses = _train(factory, offload = False)
+ head_model, head_opt, head_losses = _train(factory, offload = True, chunk_bytes = chunk_bytes)
+ assert head_losses == base_losses
+ for a, b in zip(base_model.parameters(), head_model.parameters()):
+ assert torch.equal(a, b)
+ assert _state_devices(base_opt) == {"cuda"}
+ assert _state_devices(head_opt) == {"cpu"}
+ assert head_opt._unsloth_state_offloader.steps == 6
+
+ # Checkpoint written while offloaded, loaded the way HF Trainer does (map_location = "cpu").
+ buffer = io.BytesIO()
+ torch.save(head_opt.state_dict(), buffer)
+ buffer.seek(0)
+ saved = torch.load(buffer, map_location = "cpu", weights_only = False)
+ resumed_model, resumed_opt, _ = _train(factory, offload = True, steps = 0, chunk_bytes = chunk_bytes)
+ resumed_model.load_state_dict(head_model.state_dict())
+ resumed_opt.load_state_dict(saved)
+ assert _state_devices(resumed_opt) == {"cpu"}
+
+ x = torch.randn(32, 64, device = "cuda")
+ for model, optimizer in ((base_model, base_opt), (resumed_model, resumed_opt)):
+ (model(x) ** 2).mean().backward()
+ optimizer.step()
+ optimizer.zero_grad(set_to_none = True)
+ for a, b in zip(base_model.parameters(), resumed_model.parameters()):
+ assert torch.equal(a, b)
+ assert _state_devices(resumed_opt) == {"cpu"}
+
+
[email protected](not torch.cuda.is_available(), reason = "needs CUDA")
+def test_chunked_step_bounds_device_state_and_keeps_grads():
+ module = _module()
+ params = [torch.nn.Parameter(torch.randn(1 << 20, device = "cuda")) for _ in range(4)]
+ optimizer = torch.optim.AdamW(params, lr = 1e-3)
+ offloader = module.offload_optimizer_state(optimizer, chunk_bytes = 9 << 20)
+ for p in params:
+ p.grad = torch.ones_like(p)
+ optimizer.step()
+ for p in params:
+ p.grad = torch.ones_like(p)
+ grads = [p.grad for p in params]
+ torch.cuda.synchronize()
+ base = torch.cuda.memory_allocated()
+ torch.cuda.reset_peak_memory_stats()
+ optimizer.step()
+ peak = torch.cuda.max_memory_allocated() - base
+ assert all(p.grad is g for p, g in zip(params, grads))
+ # One parameter's state (2 x 4 MiB) on device at a time, not all four (32 MiB).
+ assert peak < 16 << 20
+ assert _state_devices(optimizer) == {"cpu"}
+ assert offloader.steps == 2
+
+
+class _GroupCountingSGD(torch.optim.SGD):
+ # Like schedulefree: group-level bookkeeping advances on every step call.
+ def step(self, closure = None):
+ for group in self.param_groups:
+ group["k"] = group.get("k", 0) + 1
+ return super().step(closure)
+
+
+def test_group_global_optimizer_steps_once_per_step():
+ module = _module()
+ params = [torch.nn.Parameter(torch.randn(16)) for _ in range(3)]
+ optimizer = _GroupCountingSGD(params, lr = 0.1)
+ offloader = module._StateOffloader(optimizer, chunk_bytes = 1)
+ optimizer.register_step_pre_hook(offloader._pre_step)
+ optimizer.register_step_post_hook(offloader._post_step)
+ expected = [p.detach() - 0.1 for p in params]
+ for p in params:
+ p.grad = torch.ones_like(p)
+ optimizer.step()
+ assert optimizer.param_groups[0]["k"] == 1
+ for p, e in zip(params, expected):
+ assert torch.allclose(p, e)
+
+
+def test_chunk_safety_by_step_owner():
+ module = _module()
+ p = [torch.nn.Parameter(torch.randn(2))]
+ assert module._chunk_safe(torch.optim.AdamW(p))
+ assert not module._chunk_safe(_GroupCountingSGD(p, lr = 0.1))
+
+
+def test_unstepped_chunk_size_counts_amsgrad_moment():
+ module = _module()
+ n = 1000
+ params = [torch.nn.Parameter(torch.randn(n)) for _ in range(2)]
+ # 8 bytes per fp32 parameter would fit both in 20n; amsgrad holds 12n each.
+ plain = module._StateOffloader(torch.optim.AdamW(params), chunk_bytes = 20 * n)
+ assert len(plain._chunks(params)) == 1
+ amsgrad = module._StateOffloader(torch.optim.AdamW(params, amsgrad = True), chunk_bytes = 20 * n)
+ assert len(amsgrad._chunks(params)) == 2
+
+
+class _Projector:
+ __slots__ = ("ortho_matrix", "rank")
+
+ def __init__(self, ortho_matrix):
+ self.ortho_matrix = ortho_matrix
+ self.rank = 2
+
+
+def test_nested_state_tensors_are_sized():
+ module = _module()
+ param = torch.nn.Parameter(torch.randn(4))
+ optimizer = torch.optim.AdamW([param])
+ optimizer.state[param]["projector"] = _Projector(torch.zeros(1000))
+ offloader = module._StateOffloader(optimizer, chunk_bytes = 1)
+ assert [name for name, _ in module._tensor_attrs(optimizer.state[param]["projector"])] == [
+ "ortho_matrix"
+ ]
+ other = torch.nn.Parameter(torch.randn(4))
+ assert len(offloader._chunks([param, other])) == 2
+
+
[email protected](not torch.cuda.is_available(), reason = "needs CUDA")
+def test_nested_state_object_round_trip():
+ module = _module()
+ param = torch.nn.Parameter(torch.randn(64, device = "cuda"))
+ optimizer = torch.optim.AdamW([param], lr = 1e-3)
+ param.grad = torch.ones_like(param)
+ optimizer.step()
+ ortho = torch.randn(64, 8, device = "cuda")
+ projector = _Projector(ortho.clone())
+ optimizer.state[param]["projector"] = projector
+ offloader = module.offload_optimizer_state(optimizer)
+ assert projector.ortho_matrix.device.type == "cpu"
+ assert offloader.offloaded_bytes >= ortho.numel() * 4
+ offloader.restore()
+ assert projector.ortho_matrix.device.type == "cuda"
+ assert torch.equal(projector.ortho_matrix, ortho)
+ param.grad = torch.ones_like(param)
+ optimizer.step()
+ assert projector.ortho_matrix.device.type == "cpu"
+
+
[email protected](not torch.cuda.is_available(), reason = "needs CUDA")
+def test_layerwise_optimizer_dict_is_offloaded():
+ module = _module()
+ LayerWiseDummyOptimizer = pytest.importorskip(
+ "transformers.trainer_pt_utils"
+ ).LayerWiseDummyOptimizer
+ params = [torch.nn.Parameter(torch.randn(256, device = "cuda")) for _ in range(2)]
+ optimizer_dict = {p: torch.optim.AdamW([p], lr = 1e-3) for p in params}
+ dummy = LayerWiseDummyOptimizer(optimizer_dict = optimizer_dict, lr = 1e-3)
+ offloaders = module.offload_optimizer_state(dummy)
+ assert offloaders is not None and len(offloaders) == 2
+ for p in params:
+ p.grad = torch.ones_like(p)
+ optimizer_dict[p].step()
+ for inner in optimizer_dict.values():
+ assert _state_devices(inner) == {"cpu"}
+
+
+def _hooked(module, optimizer, chunk_bytes):
+ offloader = module._StateOffloader(optimizer, chunk_bytes = chunk_bytes)
+ optimizer.register_step_pre_hook(offloader._pre_step)
+ optimizer.register_step_post_hook(offloader._post_step)
+ return offloader
+
+
+def test_chunked_step_synchronizes_chunk_copies(monkeypatch):
+ module = _module()
+ params = [torch.nn.Parameter(torch.randn(16)) for _ in range(3)]
+ offloader = _hooked(module, torch.optim.AdamW(params), chunk_bytes = 1)
+ offload = offloader.offload
+ # CPU tensors are never copied, so stand in for the device a chunk copy was issued on.
+ monkeypatch.setattr(
+ offloader, "offload", lambda params = None: offload(params) | ({"dev"} if params else set())
+ )
+ synced = []
+ monkeypatch.setattr(module, "_synchronize", synced.append)
+ for p in params:
+ p.grad = torch.ones_like(p)
+ offloader.optimizer.step()
+ assert synced == ["dev"]
+
+
+def test_unstepped_chunk_size_counts_centered_rmsprop_momentum():
+ module = _module()
+ n = 1000
+ params = [torch.nn.Parameter(torch.randn(n)) for _ in range(2)]
+ plain = module._StateOffloader(torch.optim.RMSprop(params, centered = True), chunk_bytes = 20 * n)
+ assert len(plain._chunks(params)) == 1
+ both = torch.optim.RMSprop(params, centered = True, momentum = 0.9)
+ assert len(module._StateOffloader(both, chunk_bytes = 20 * n)._chunks(params)) == 2
+
+
+def test_single_chunk_restores_only_active_state(monkeypatch):
+ module = _module()
+ active, idle = torch.nn.Parameter(torch.randn(16)), torch.nn.Parameter(torch.randn(16))
+ offloader = _hooked(module, torch.optim.AdamW([active, idle]), chunk_bytes = None)
+ restored = []
+ monkeypatch.setattr(offloader, "restore", lambda params = None: restored.append(params))
+ active.grad = torch.ones_like(active)
+ offloader.optimizer.step()
+ assert restored == [[active]]
+
+
+def test_chunked_step_runs_external_hooks_once():
+ module = _module()
+ params = [torch.nn.Parameter(torch.randn(16)) for _ in range(3)]
+ optimizer = torch.optim.AdamW(params, lr = 0.1)
+ _hooked(module, optimizer, chunk_bytes = 1)
+ calls = []
+ optimizer.register_step_pre_hook(lambda *a: calls.append("pre"))
+ optimizer.register_step_post_hook(lambda *a: calls.append("post"))
+ reference = torch.optim.AdamW([torch.nn.Parameter(p.detach().clone()) for p in params], lr = 0.1)
+ for p, q in zip(params, reference.param_groups[0]["params"]):
+ p.grad = torch.ones_like(p)
+ q.grad = torch.ones_like(q)
+ optimizer.step()
+ reference.step()
+ assert calls == ["pre", "post"]
+ for p, q in zip(params, reference.param_groups[0]["params"]):
+ assert torch.equal(p, q)
+
+
+def test_list_state_tensors_are_sized():
+ module = _module()
+ params = [torch.nn.Parameter(torch.randn(4)) for _ in range(2)]
+ optimizer = torch.optim.AdamW(params)
+ optimizer.state[params[0]]["old_dirs"] = [torch.zeros(1000)]
+ assert len(module._StateOffloader(optimizer, chunk_bytes = 100)._chunks(params)) == 2
+
+
[email protected](not torch.cuda.is_available(), reason = "needs CUDA")
+def test_lbfgs_history_is_offloaded():
+ module = _module()
+ torch.manual_seed(0)
+ target = torch.randn(64, device = "cuda")
+
+ def run(offload):
+ p = torch.nn.Parameter(torch.zeros(64, device = "cuda"))
+ optimizer = torch.optim.LBFGS([p], lr = 0.5, history_size = 4, max_iter = 3)
+ if offload:
+ module.offload_optimizer_state(optimizer)
+
+ def closure():
+ optimizer.zero_grad()
+ loss = ((p - target) ** 2).sum()
+ loss.backward()
+ return loss
+
+ for _ in range(3):
+ optimizer.step(closure)
+ return p.detach(), optimizer.state[p]
+
+ base, _ = run(False)
+ head, state = run(True)
+ assert torch.allclose(base, head)
+ assert state["old_dirs"] and all(
+ t.device.type == "cpu" for t in state["old_dirs"] + state["ro"]
+ )
+
+
+def test_chunked_step_runs_after_later_pre_hooks():
+ module = _module()
+ params = [torch.nn.Parameter(torch.randn(16)) for _ in range(3)]
+ reference = [torch.nn.Parameter(p.detach().clone()) for p in params]
+
+ def halve_and_fill(optimizer, args, kwargs):
+ # Clips existing gradients and creates the missing one, as a later library hook might.
+ for group in optimizer.param_groups:
+ group["lr"] = 0.05
+ for p in group["params"]:
+ p.grad = torch.full_like(p, 0.5) if p.grad is None else p.grad * 0.5
+
+ optimizer = torch.optim.AdamW(params, lr = 0.1)
+ offloader = _hooked(module, optimizer, chunk_bytes = 1)
+ optimizer.register_step_pre_hook(halve_and_fill)
+ plain = torch.optim.AdamW(reference, lr = 0.1)
+ plain.register_step_pre_hook(halve_and_fill)
+ for p, q in zip(params[:2], reference[:2]):
+ p.grad = torch.ones_like(p)
+ q.grad = torch.ones_like(q)
+ optimizer.step()
+ plain.step()
+ for p, q in zip(params, reference):
+ assert torch.equal(p, q)
+ assert torch.equal(optimizer.state[p]["exp_avg"], plain.state[q]["exp_avg"])
+ assert offloader.hidden is None and offloader.steps == 1
+
+
+def test_positional_none_closure_still_chunks():
+ # accelerate calls optimizer.step(closure) with closure = None.
+ module = _module()
+ params = [torch.nn.Parameter(torch.randn(16)) for _ in range(3)]
+ reference = torch.optim.AdamW([torch.nn.Parameter(p.detach().clone()) for p in params], lr = 0.1)
+ optimizer = torch.optim.AdamW(params, lr = 0.1)
+ chunked = []
+ offloader = _hooked(module, optimizer, chunk_bytes = 1)
+ restore = offloader.restore
+ offloader.restore = lambda params = None: (chunked.append(params), restore(params))
+ for p, q in zip(params, reference.param_groups[0]["params"]):
+ p.grad = torch.ones_like(p)
+ q.grad = torch.ones_like(q)
+ assert optimizer.step(None) is None
+ reference.step()
+ assert chunked == [[p] for p in params]
+ for p, q in zip(params, reference.param_groups[0]["params"]):
+ assert torch.equal(p, q)
+
+
+def _big_params(n = 4, numel = 1 << 20):
+ return [torch.nn.Parameter(torch.randn(numel, device = "cuda")) for _ in range(n)]
+
+
[email protected](not torch.cuda.is_available(), reason = "needs CUDA")
[email protected]("fused", [False, True])
+def test_load_state_dict_keeps_state_on_host(fused):
+ module = _module()
+ params = _big_params()
+ source = torch.optim.AdamW(params, lr = 1e-3, fused = fused)
+ for p in params:
+ p.grad = torch.ones_like(p)
+ source.step()
+ saved = {
+ "state": {
+ k: {n: t.cpu().clone() for n, t in v.items()}
+ for k, v in source.state_dict()["state"].items()
+ },
+ "param_groups": source.state_dict()["param_groups"],
+ }
+ clones = [torch.nn.Parameter(p.detach().clone()) for p in params]
+ optimizer = torch.optim.AdamW(clones, lr = 1e-3, fused = fused)
+ offloader = module.offload_optimizer_state(optimizer, chunk_bytes = 9 << 20)
+ torch.cuda.synchronize()
+ base = torch.cuda.memory_allocated()
+ torch.cuda.reset_peak_memory_stats()
+ optimizer.load_state_dict(saved)
+ torch.cuda.synchronize()
+ # All four parameters' moments would be 32 MiB on device.
+ assert torch.cuda.max_memory_allocated() - base < 1 << 20
+ assert _state_devices(optimizer) == {"cpu"}
+ # Fused steps keep "step" on the parameter's device, the others on the host.
+ step_device = offloader.offloaded.get((clones[0], "step"))
+ assert (step_device.type if step_device is not None else "cpu") == ("cuda" if fused else "cpu")
+ for p, q in zip(params, clones):
+ p.grad = torch.ones_like(p)
+ q.grad = torch.ones_like(q)
+ source.step()
+ optimizer.step()
+ for p, q in zip(params, clones):
+ assert torch.equal(p, q)
+ assert _state_devices(optimizer) == {"cpu"}
+
+
[email protected](not torch.cuda.is_available(), reason = "needs CUDA")
+def test_resumed_bitsandbytes_first_step_stays_chunked():
+ bnb = pytest.importorskip("bitsandbytes")
+ module = _module()
+ params = _big_params()
+ source = bnb.optim.AdamW8bit(params, lr = 1e-3)
+ for p in params:
+ p.grad = torch.ones_like(p)
+ source.step()
+ buffer = io.BytesIO()
+ torch.save(source.state_dict(), buffer)
+ buffer.seek(0)
+ saved = torch.load(buffer, map_location = "cpu", weights_only = False)
+ clones = [torch.nn.Parameter(p.detach().clone()) for p in params]
+ optimizer = bnb.optim.AdamW8bit(clones, lr = 1e-3)
+ module.offload_optimizer_state(optimizer, chunk_bytes = 3 << 20)
+ optimizer.load_state_dict(saved)
+ for p, q in zip(params, clones):
+ p.grad = torch.ones_like(p)
+ q.grad = torch.ones_like(q)
+ torch.cuda.synchronize()
+ base = torch.cuda.memory_allocated()
+ torch.cuda.reset_peak_memory_stats()
+ optimizer.step()
+ peak = torch.cuda.max_memory_allocated() - base
+ source.step()
+ # One parameter's 8-bit moments (about 2 MiB) at a time, not all four (about 8 MiB).
+ assert peak < 5 << 20
+ for p, q in zip(params, clones):
+ assert torch.equal(p, q)
+ assert _state_devices(optimizer) == {"cpu"}
+
+
+def test_unstepped_chunk_size_counts_the_ademamix_double_moment():
+ module = _module()
+ n = 1000
+ params = [torch.nn.Parameter(torch.randn(n)) for _ in range(2)]
+
+ class AdEMAMix(torch.optim.AdamW):
+ pass
+
+ plain = module._StateOffloader(torch.optim.AdamW(params), chunk_bytes = 20 * n)
+ assert len(plain._chunks(params)) == 1
+ ademamix = module._StateOffloader(AdEMAMix(params), chunk_bytes = 20 * n)
+ assert len(ademamix._chunks(params)) == 2
+
+
+def test_mps_synchronize_takes_no_device(monkeypatch):
+ module = _module()
+ calls = []
+ monkeypatch.setattr(
+ torch, "mps", types.SimpleNamespace(synchronize = lambda *a: calls.append(a)), raising = False
+ )
+ module._synchronize(torch.device("mps"))
+ assert calls == [()]
diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py
index b1ebe0ac311..93656d1695c 100644
--- a/unsloth/models/_utils.py
+++ b/unsloth/models/_utils.py
@@ -5557,6 +5557,9 @@ def patch_gradient_accumulation_fix(Trainer):
_replan_auto_offload_safely(self)
_block_swap_for_trainer_backend(self)
_enable_ddp_batched_grad_copy(self)
+ if os.environ.get("UNSLOTH_OFFLOAD_OPTIMIZER_STATE", "0") != "0":
+ from ..optimizers.state_offload import install_optimizer_state_offload
+ install_optimizer_state_offload(self)
_unsloth_trainer_init.__wrapped__ = _original_trainer_init
Trainer.__init__ = _unsloth_trainer_init
diff --git a/unsloth/optimizers/state_offload.py b/unsloth/optimizers/state_offload.py
new file mode 100644
index 00000000000..d3c04d458de
--- /dev/null
+++ b/unsloth/optimizers/state_offload.py
@@ -0,0 +1,468 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""Optimizer-state-only CPU offload (UNSLOTH_OFFLOAD_OPTIMIZER_STATE=1).
+
+State lives in pinned host memory between steps and visits the GPU only inside optimizer.step,
+so peak memory is max(activations, optimizer state) instead of their sum. The step runs in chunks
+of UNSLOTH_OFFLOAD_OPTIMIZER_STATE_CHUNK_MB (default 1024) of state, so the step itself never holds
+all of it on device either (parameter-local optimizers only, see _CHUNK_SAFE_MODULES).
+"""
+
+import copy
+import os
+
+import torch
+
+__all__ = [
+ "optimizer_state_offload_enabled",
+ "offload_optimizer_state",
+ "install_optimizer_state_offload",
+]
+
+_ENV = "UNSLOTH_OFFLOAD_OPTIMIZER_STATE"
+
+try:
+ from torch.distributed.tensor import DTensor
+except Exception:
+ DTensor = None
+
+
+def _chunk_bytes():
+ try:
+ mb = float(os.environ.get(_ENV + "_CHUNK_MB", "1024"))
+ except ValueError:
+ mb = 1024.0
+ return int(mb * 2**20) if mb > 0 else float("inf")
+
+
+def optimizer_state_offload_enabled():
+ return os.environ.get(_ENV, "0").strip().lower() in ("1", "true", "yes", "on")
+
+
+def _innermost(optimizer):
+ # accelerate's AcceleratedOptimizer is itself an Optimizer subclass wrapping .optimizer.
+ while isinstance(getattr(optimizer, "optimizer", None), torch.optim.Optimizer):
+ optimizer = optimizer.optimizer
+ return optimizer
+
+
+# Classes whose step only touches parameters that have a gradient, so stepping one chunk at a time
+# is the same update. Others (schedulefree advances group["k"] on every call) step unchunked.
+_CHUNK_SAFE_MODULES = (
+ "torch.optim.",
+ "bitsandbytes.",
+ "transformers.optimization",
+ "unsloth.optimizers.",
+)
+
+
+def _chunk_safe(optimizer):
+ for cls in type(optimizer).__mro__:
+ if "step" in vars(cls):
+ return cls.__module__.startswith(_CHUNK_SAFE_MODULES)
+ return False
+
+
+def _tensor_attrs(value):
+ # Tensors held by state objects such as GaLoreProjector (__slots__, no __dict__).
+ if isinstance(value, (torch.Tensor, dict, list, tuple, str, int, float, bool)) or value is None:
+ return ()
+ names = set(getattr(value, "__dict__", ()))
+ for cls in type(value).__mro__:
+ slots = vars(cls).get("__slots__", ())
+ names.update((slots,) if isinstance(slots, str) else slots)
+ return [
+ (name, getattr(value, name))
+ for name in sorted(names)
+ if isinstance(getattr(value, name, None), torch.Tensor)
+ ]
+
+
+def _held_tensors(value):
+ if isinstance(value, torch.Tensor):
+ return [value]
+ # LBFGS keeps its history (old_dirs, old_stps, ro) as lists of tensors.
+ if isinstance(value, (list, tuple)):
+ return [v for v in value if isinstance(v, torch.Tensor)]
+ return [t for _, t in _tensor_attrs(value)]
+
+
+def _source_device(local, param, key, device_of):
+ if local.device.type != "cpu":
+ return local.device
+ return device_of(param, key, local) if device_of is not None else None
+
+
+def _synchronize(device):
+ module = getattr(torch, device.type, None)
+ sync = getattr(module, "synchronize", None)
+ if sync is not None:
+ # torch.mps.synchronize takes no device argument.
+ sync() if device.type == "mps" else sync(device)
+
+
+class _StateOffloader:
+ def __init__(
+ self,
+ optimizer,
+ pin_memory = True,
+ chunk_bytes = None,
+ ):
+ self.optimizer = optimizer
+ self.pin_memory = pin_memory
+ self.chunk_bytes = chunk_bytes if chunk_bytes is not None else _chunk_bytes()
+ self.buffers = {}
+ self.offloaded = {}
+ self.offloaded_bytes = 0
+ self.steps = 0
+ self.inside = False
+ self.hidden = None
+ self.pending = set()
+
+ def _host_buffer(self, param, key, src):
+ buffer = self.buffers.get((param, key))
+ if buffer is None or buffer.shape != src.shape or buffer.dtype != src.dtype:
+ try:
+ buffer = torch.empty(
+ src.shape, dtype = src.dtype, device = "cpu", pin_memory = self.pin_memory
+ )
+ except RuntimeError:
+ # No pinnable host memory (some WSL / container setups): pageable still works.
+ self.pin_memory = False
+ buffer = torch.empty(src.shape, dtype = src.dtype, device = "cpu")
+ self.buffers[(param, key)] = buffer
+ buffer.copy_(src, non_blocking = self.pin_memory)
+ return buffer
+
+ def _states(self, params):
+ if params is None:
+ return list(self.optimizer.state.items())
+ state = self.optimizer.state
+ return [(p, state[p]) for p in params if p in state]
+
+ def offload(
+ self,
+ params = None,
+ device_of = None,
+ ):
+ # device_of(param, key, tensor) names the device of state that is already on the host
+ # (a checkpoint just loaded there), None to leave it host-only.
+ devices = set()
+ for param, state in self._states(params):
+ for key, value in list(state.items()):
+ if isinstance(value, (list, tuple)):
+ items = list(value)
+ for i, local in enumerate(items):
+ if not isinstance(local, torch.Tensor):
+ continue
+ device = _source_device(local, param, (key, i), device_of)
+ if device is not None:
+ devices.add(local.device)
+ items[i] = self._host_buffer(param, (key, i), local)
+ self.offloaded[(param, (key, i))] = device
+ if isinstance(value, list):
+ value[:] = items
+ else:
+ state[key] = type(value)(items)
+ continue
+ for name, local in _tensor_attrs(value):
+ device = _source_device(local, param, (key, name), device_of)
+ if device is None:
+ continue
+ devices.add(local.device)
+ setattr(value, name, self._host_buffer(param, (key, name), local))
+ self.offloaded[(param, (key, name))] = device
+ is_dtensor = DTensor is not None and isinstance(value, DTensor)
+ local = value._local_tensor if is_dtensor else value
+ if not isinstance(local, torch.Tensor):
+ continue
+ device = _source_device(local, param, key, device_of)
+ if device is None:
+ continue
+ devices.add(local.device)
+ host = self._host_buffer(param, key, local)
+ self.offloaded[(param, key)] = device
+ if is_dtensor:
+ value = copy.copy(value)
+ value._local_tensor = host
+ state[key] = value
+ else:
+ state[key] = host
+ return devices
+
+ def finish(self, devices):
+ # The host copies must be complete before anything reads them (torch.save, the next step).
+ for device in devices:
+ _synchronize(device)
+ self.offloaded_bytes = sum(b.numel() * b.element_size() for b in self.buffers.values())
+
+ def restore(self, params = None):
+ for param, state in self._states(params):
+ for key, value in list(state.items()):
+ if isinstance(value, (list, tuple)):
+ items = list(value)
+ for i, local in enumerate(items):
+ device = self.offloaded.get((param, (key, i)))
+ if (
+ device is not None
+ and isinstance(local, torch.Tensor)
+ and local.device != device
+ ):
+ items[i] = local.to(device, non_blocking = True)
+ if isinstance(value, list):
+ value[:] = items
+ else:
+ state[key] = type(value)(items)
+ continue
+ for name, local in _tensor_attrs(value):
+ device = self.offloaded.get((param, (key, name)))
+ if device is not None and local.device != device:
+ setattr(value, name, local.to(device, non_blocking = True))
+ device = self.offloaded.get((param, key))
+ if device is None:
+ continue
+ is_dtensor = DTensor is not None and isinstance(value, DTensor)
+ local = value._local_tensor if is_dtensor else value
+ if not isinstance(local, torch.Tensor) or local.device == device:
+ continue
+ local = local.to(device, non_blocking = True)
+ if is_dtensor:
+ value = copy.copy(value)
+ value._local_tensor = local
+ state[key] = value
+ else:
+ state[key] = local
+
+ def _chunks(self, params):
+ # Optimizers update only parameters that have a gradient, so stepping one chunk at a time
+ # (other gradients hidden) bounds the on-device state to chunk_bytes instead of all of it.
+ chunks, chunk, size = [], [], 0
+ state = self.optimizer.state
+ groups = {p: g for g in self.optimizer.param_groups for p in g["params"]}
+ for p in params:
+ held = [t for v in state.get(p, {}).values() for t in _held_tensors(v)]
+ if held:
+ nbytes = sum(v.numel() * v.element_size() for v in held)
+ else:
+ # Not stepped yet: two moments, a third with amsgrad or centered RMSprop with
+ # momentum (square_avg, momentum_buffer, grad_avg), at least fp32 each.
+ group = groups.get(p, {})
+ # bitsandbytes AdEMAMix keeps a double-sized state1 next to state2.
+ third = (
+ group.get("amsgrad")
+ or (group.get("centered") and group.get("momentum", 0) > 0)
+ or "ademamix" in type(self.optimizer).__name__.lower()
+ )
+ moments = 3 if third else 2
+ nbytes = moments * max(p.element_size(), 4) * p.numel()
+ if chunk and size + nbytes > self.chunk_bytes:
+ chunks.append(chunk)
+ chunk, size = [], 0
+ chunk.append(p)
+ size += nbytes
+ if chunk:
+ chunks.append(chunk)
+ return chunks
+
+ def _pre_step(self, optimizer, args, kwargs):
+ if self.inside:
+ return
+ # torch passes the step's own args to hooks, self included.
+ has_self = bool(args) and args[0] is optimizer
+ rest = args[1:] if has_self else args
+ closure = rest[0] if rest else kwargs.get("closure")
+ # A closure recomputes the loss on every call, so it must run once over all parameters.
+ if closure is not None or not _chunk_safe(optimizer):
+ self.restore()
+ return
+
+ # Chunk-safe steps call a closure first, before any update, so the chunked updates run
+ # after every step pre hook (registered later ones too), as a plain step would.
+ def _chunked_updates():
+ self._step_chunks(optimizer, rest[1:], dict(kwargs))
+ return None
+
+ if rest:
+ rest = (_chunked_updates,) + tuple(rest[1:])
+ else:
+ kwargs = dict(kwargs, closure = _chunked_updates)
+ return ((optimizer,) + tuple(rest) if has_self else tuple(rest)), kwargs
+
+ def _step_chunks(self, optimizer, args, kwargs):
+ kwargs.pop("closure", None)
+ params = [
+ p for group in optimizer.param_groups for p in group["params"] if p.grad is not None
+ ]
+ chunks = self._chunks(params)
+ # Chunk-safe steps never touch parameters without a gradient, so their state stays on host.
+ if len(chunks) <= 1:
+ self.restore(params)
+ return
+ # Step each chunk through the unhooked class step, so registered step hooks run once for
+ # the logical step, then hide every gradient so the caller's own step finds nothing left.
+ step = type(optimizer).step
+ if getattr(step, "hooked", False):
+ step = getattr(step, "__wrapped__", step)
+ grads = [p.grad for p in params]
+ devices = set()
+ self.inside = True
+ try:
+ for chunk in chunks:
+ members = set(chunk)
+ for p, g in zip(params, grads):
+ p.grad = g if p in members else None
+ self.restore(chunk)
+ step(optimizer, *args, **kwargs)
+ devices |= self.offload(chunk)
+ except BaseException:
+ for p, g in zip(params, grads):
+ p.grad = g
+ self.finish(devices | self.offload())
+ raise
+ finally:
+ self.inside = False
+ for p in params:
+ p.grad = None
+ self.hidden = (params, grads)
+ # The chunk copies are still in flight; the post hook's offload finds nothing left to copy.
+ self.pending = devices
+
+ def _post_step(self, optimizer, args, kwargs):
+ if self.inside:
+ return
+ if self.hidden is not None:
+ for p, g in zip(*self.hidden):
+ p.grad = g
+ self.hidden = None
+ self.steps += 1
+ devices, self.pending = self.pending, set()
+ self.finish(devices | self.offload())
+
+
+def offload_optimizer_state(
+ optimizer,
+ pin_memory = True,
+ chunk_bytes = None,
+):
+ """Keep `optimizer`'s state on the host between steps. Idempotent; returns the offloader or None."""
+ optimizer = _innermost(optimizer)
+ if not isinstance(optimizer, torch.optim.Optimizer):
+ return None
+ # transformers' LayerWiseDummyOptimizer (galore/apollo *_layerwise): the real per-parameter
+ # optimizers live in optimizer_dict and step from gradient hooks.
+ layerwise = getattr(optimizer, "optimizer_dict", None)
+ if isinstance(layerwise, dict):
+ unique = {id(o): o for o in layerwise.values()}.values()
+ offloaders = [offload_optimizer_state(o, pin_memory, chunk_bytes) for o in unique]
+ offloaders = [o for o in offloaders if o is not None]
+ return offloaders or None
+ existing = getattr(optimizer, "_unsloth_state_offloader", None)
+ if existing is not None:
+ return existing
+ # Paged bitsandbytes state already spills to host through unified memory.
+ if getattr(optimizer, "is_paged", False):
+ return None
+ if not any(p.device.type != "cpu" for group in optimizer.param_groups for p in group["params"]):
+ return None
+
+ offloader = _StateOffloader(optimizer, pin_memory = pin_memory, chunk_bytes = chunk_bytes)
+ # Hooks, not an optimizer.step override: accelerate swaps the instance's step attribute on
+ # every fp16 step, and hooks run whichever bound step a caller kept.
+ optimizer.register_step_pre_hook(offloader._pre_step)
+ optimizer.register_step_post_hook(offloader._post_step)
+
+ # Instance override, not register_load_state_dict_post_hook: bitsandbytes replaces
+ # Optimizer.load_state_dict and does not run torch's post hooks. Loading moves every
+ # state tensor to its parameter's device, so offload again right after.
+ load_state_dict = optimizer.load_state_dict
+
+ def _load_state_dict_and_offload(state_dict, *args, **kwargs):
+ moved = {}
+ original = vars(torch.optim.Optimizer).get("_process_value_according_to_param_policy")
+ if isinstance(original, staticmethod):
+ policy = original.__func__
+
+ # torch casts every loaded state tensor onto its parameter's device; keep it on the
+ # host instead so a resume never holds the whole state on device at once.
+ def _host_policy(param, value, *policy_args, **policy_kwargs):
+ if param.device.type in ("cpu", "meta"):
+ return policy(param, value, *policy_args, **policy_kwargs)
+ stand_in = torch.empty(0, dtype = param.dtype, device = "meta")
+ out = policy(stand_in, value, *policy_args, **policy_kwargs)
+ if out.device.type != "meta":
+ return out
+ out = value.to(dtype = out.dtype)
+ moved[id(out)] = param.device
+ return out
+
+ torch.optim.Optimizer._process_value_according_to_param_policy = staticmethod(
+ _host_policy
+ )
+ try:
+ result = load_state_dict(state_dict, *args, **kwargs)
+ finally:
+ if isinstance(original, staticmethod):
+ torch.optim.Optimizer._process_value_according_to_param_policy = original
+ if moved:
+ device_of = lambda param, key, local: moved.get(id(local))
+ elif callable(getattr(optimizer, "to_gpu", None)) and hasattr(optimizer, "initialized"):
+ # bitsandbytes loads state on the host and its first step moves all of it to the device
+ # (to_gpu), defeating the chunks: do its first-step setup here, minus that move.
+ if not optimizer.initialized:
+ if callable(getattr(optimizer, "check_overrides", None)):
+ optimizer.check_overrides()
+ optimizer.initialized = True
+ device_of = lambda param, key, local: (
+ param.device
+ if not isinstance(key, tuple)
+ and param.device.type != "cpu"
+ and not getattr(local, "is_paged", False)
+ else None
+ )
+ else:
+ device_of = None
+ offloader.finish(offloader.offload(device_of = device_of))
+ return result
+
+ optimizer.load_state_dict = _load_state_dict_and_offload
+ optimizer._unsloth_state_offloader = offloader
+ offloader.finish(offloader.offload())
+ return offloader
+
+
+_PRINTED = False
+
+
+def install_optimizer_state_offload(trainer):
+ """Offload the trainer's optimizer state once its optimizer is final (inside create_scheduler)."""
+ if not optimizer_state_offload_enabled():
+ return False
+ if getattr(trainer, "is_deepspeed_enabled", False):
+ print(
+ "Unsloth: UNSLOTH_OFFLOAD_OPTIMIZER_STATE is ignored under DeepSpeed (use its offload_optimizer)."
+ )
+ return False
+ create_scheduler = trainer.create_scheduler
+ if getattr(create_scheduler, "_unsloth_state_offload", False):
+ return True
+
+ def _create_scheduler_offload_state(*args, **kwargs):
+ global _PRINTED
+ scheduler = create_scheduler(*args, **kwargs)
+ optimizer = kwargs.get("optimizer", args[1] if len(args) > 1 else None)
+ optimizer = optimizer if optimizer is not None else getattr(trainer, "optimizer", None)
+ if (
+ optimizer is not None
+ and offload_optimizer_state(optimizer) is not None
+ and not _PRINTED
+ ):
+ _PRINTED = True
+ print(
+ "Unsloth: Optimizer state is offloaded to CPU between steps (UNSLOTH_OFFLOAD_OPTIMIZER_STATE=1)."
+ )
+ return scheduler
+
+ _create_scheduler_offload_state._unsloth_state_offload = True
+ trainer.create_scheduler = _create_scheduler_offload_state
+ return True