Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

GRPO: opt-in parking of optimizer state and gradients on host during rollout (#13866)

Daniel Hanc7824b0bb250
diff --git a/tests/test_grpo_park_optimizer_during_rollout.py b/tests/test_grpo_park_optimizer_during_rollout.py
new file mode 100644
index 00000000000..ebc09599bef
--- /dev/null
+++ b/tests/test_grpo_park_optimizer_during_rollout.py
@@ -0,0 +1,211 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""UNSLOTH_PARK_OPTIMIZER_DURING_ROLLOUT=1 moves optimizer state and grads to host around GRPO
+generation and puts them back unchanged. The helpers are lifted from rl.py with ast and run on
+CPU: a torch shim stands in for pinned memory and CUDA sync, and every tensor counts as parkable.
+"""
+
+from __future__ import annotations
+
+import sys
+import types
+from pathlib import Path
+
+import pytest
+import torch
+
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+
+from _rl_source import load_rl_wrapper # noqa: E402
+
+
+NAMES = (
+ "_park_optimizer_enabled",
+ "_is_parkable_tensor",
+ "_optimizer_state_is_flat",
+ "_park_optimizer_to_host",
+ "_restore_optimizer_from_host",
+ "_wrap_grpo_generate_and_score",
+)
+
+
+def _load():
+ fns = load_rl_wrapper(NAMES)
+ namespace = fns["_wrap_grpo_generate_and_score"].__globals__
+ cuda = types.SimpleNamespace(
+ calls = [],
+ is_available = lambda: True,
+ )
+ cuda.synchronize = lambda device = None: cuda.calls.append("synchronize")
+ cuda.empty_cache = lambda: cuda.calls.append("empty_cache")
+
+ def empty_like(
+ tensor,
+ pin_memory = False,
+ **kwargs,
+ ):
+ return torch.empty_like(tensor, **kwargs)
+
+ namespace["torch"] = types.SimpleNamespace(
+ Tensor = torch.Tensor, empty_like = empty_like, cuda = cuda, strided = torch.strided
+ )
+ namespace["_is_parkable_tensor"] = lambda value: isinstance(value, torch.Tensor)
+ return fns, cuda
+
+
+class _Model:
+ training = True
+
+
+def _trainer_after_one_step():
+ torch.manual_seed(0)
+ model = torch.nn.Linear(4, 3)
+ optimizer = torch.optim.AdamW(model.parameters(), lr = 1e-2)
+ model(torch.randn(2, 4)).sum().backward()
+ optimizer.step()
+ # Mid-accumulation: a fresh grad is pending while the next rollout happens.
+ model(torch.randn(2, 4)).sum().backward()
+ trainer = types.SimpleNamespace(model = _Model(), optimizer = optimizer)
+ return trainer, model, optimizer
+
+
+def _snapshot(optimizer, model):
+ state = {
+ (id(p), k): v.clone()
+ for p, s in optimizer.state.items()
+ for k, v in s.items()
+ if isinstance(v, torch.Tensor)
+ }
+ grads = {id(p): p.grad.clone() for p in model.parameters()}
+ return state, grads
+
+
+def _run(monkeypatch, enabled):
+ fns, cuda = _load()
+ trainer, model, optimizer = _trainer_after_one_step()
+ before = _snapshot(optimizer, model)
+ original_state = {(id(p), k): v for p, s in optimizer.state.items() for k, v in s.items()}
+ seen = {}
+
+ class Trainer:
+ def _generate_and_score_completions(self, inputs):
+ seen["grads"] = [p.grad for p in model.parameters()]
+ seen["state_objects_kept"] = all(
+ v is original_state[(id(p), k)]
+ for p, s in optimizer.state.items()
+ for k, v in s.items()
+ )
+ return inputs
+
+ fns["_wrap_grpo_generate_and_score"](Trainer)
+ instance = Trainer()
+ instance.__dict__.update(vars(trainer))
+ if enabled:
+ monkeypatch.setenv("UNSLOTH_PARK_OPTIMIZER_DURING_ROLLOUT", "1")
+ else:
+ monkeypatch.delenv("UNSLOTH_PARK_OPTIMIZER_DURING_ROLLOUT", raising = False)
+ assert instance._generate_and_score_completions("x") == "x"
+ return instance, model, optimizer, before, seen, cuda
+
+
+def test_park_moves_state_and_grads_then_restores_exact_values(monkeypatch):
+ instance, model, optimizer, (state_before, grads_before), seen, cuda = _run(monkeypatch, True)
+ assert getattr(instance, "_unsloth_park_count", 0) == 1
+ assert seen["grads"] == [None, None], "grads must be released during generation"
+ assert seen["state_objects_kept"] is False, "state tensors must be swapped for host buffers"
+ assert cuda.calls == ["synchronize", "empty_cache"]
+ for p, s in optimizer.state.items():
+ for k, v in s.items():
+ if isinstance(v, torch.Tensor):
+ assert torch.equal(v, state_before[(id(p), k)])
+ for p in model.parameters():
+ assert torch.equal(p.grad, grads_before[id(p)])
+
+ optimizer.step()
+
+
+def test_park_off_by_default_leaves_everything_alone(monkeypatch):
+ instance, model, optimizer, _, seen, cuda = _run(monkeypatch, False)
+ assert getattr(instance, "_unsloth_park_count", 0) == 0
+ assert all(g is not None for g in seen["grads"])
+ assert seen["state_objects_kept"] is True
+ assert cuda.calls == []
+
+
+def test_shared_state_tensor_stays_shared(monkeypatch):
+ fns, _ = _load()
+ params = [torch.nn.Parameter(torch.zeros(3)) for _ in range(2)]
+ optimizer = torch.optim.SGD(params, lr = 0.1)
+ qmap = torch.arange(4.0)
+ for p in params:
+ optimizer.state[p] = {"qmap1": qmap, "state1": torch.ones(3)}
+ trainer = types.SimpleNamespace(optimizer = optimizer)
+ record = fns["_park_optimizer_to_host"](trainer)
+ fns["_restore_optimizer_from_host"](record)
+ a, b = (optimizer.state[p]["qmap1"] for p in params)
+ assert a is b and torch.equal(a, qmap)
+
+
+def test_first_step_without_state_or_grads_is_a_noop(monkeypatch):
+ fns, cuda = _load()
+ model = torch.nn.Linear(2, 2)
+ trainer = types.SimpleNamespace(optimizer = torch.optim.AdamW(model.parameters()))
+ assert fns["_park_optimizer_to_host"](trainer) is None
+ assert cuda.calls == []
+
+
+def test_gate_skips_without_optimizer_or_under_deepspeed():
+ fns, _ = _load()
+ enabled = fns["_park_optimizer_enabled"]
+ assert enabled(types.SimpleNamespace(optimizer = None)) is False
+ assert enabled(types.SimpleNamespace(optimizer = object(), is_deepspeed_enabled = True)) is False
+ assert enabled(types.SimpleNamespace(optimizer = object())) is True
+ multi = types.SimpleNamespace(world_size = 2)
+ assert enabled(types.SimpleNamespace(optimizer = object(), args = multi)) is False
+
+
+def test_no_parked_tensor_is_alive_at_empty_cache():
+ import weakref
+
+ fns, cuda = _load()
+ trainer, model, optimizer = _trainer_after_one_step()
+ refs = [weakref.ref(p.grad) for p in model.parameters()]
+ refs += [
+ weakref.ref(v)
+ for s in optimizer.state.values()
+ for v in s.values()
+ if isinstance(v, torch.Tensor)
+ ]
+ alive = []
+ cuda.empty_cache = lambda: alive.append(sum(r() is not None for r in refs))
+ record = fns["_park_optimizer_to_host"](trainer)
+ assert record is not None and alive == [0]
+
+
+def test_restored_state_keeps_the_source_strides():
+ fns, _ = _load()
+ trainer, model, optimizer = _trainer_after_one_step()
+ param = model.weight
+ state = optimizer.state[param]
+ state["exp_avg"] = state["exp_avg"].t().contiguous().t()
+ strides = state["exp_avg"].stride()
+ record = fns["_park_optimizer_to_host"](trainer)
+ assert state["exp_avg"].stride() == strides
+ fns["_restore_optimizer_from_host"](record)
+ assert state["exp_avg"].stride() == strides
+
+
+def test_nested_or_layerwise_optimizer_state_is_declined():
+ fns, cuda = _load()
+ trainer, model, optimizer = _trainer_after_one_step()
+ optimizer.state[model.weight]["projector"] = types.SimpleNamespace(ortho_matrix = torch.ones(2))
+ before = _snapshot(optimizer, model)
+ assert fns["_park_optimizer_to_host"](trainer) is None
+ assert cuda.calls == []
+ after = _snapshot(optimizer, model)
+ assert all(torch.equal(before[0][k], after[0][k]) for k in before[0])
+ layerwise = types.SimpleNamespace(optimizer_dict = {}, param_groups = [], state = {})
+ trainer = types.SimpleNamespace(model = _Model(), optimizer = layerwise)
+ assert fns["_park_optimizer_to_host"](trainer) is None
diff --git a/unsloth/models/rl.py b/unsloth/models/rl.py
index 0bbd6eeffe6..5a7e9c76d5e 100644
--- a/unsloth/models/rl.py
+++ b/unsloth/models/rl.py
@@ -835,6 +835,114 @@ def _wrap_ppo_train(trainer_cls):
trainer_cls.train = wrapped
+def _park_optimizer_enabled(trainer):
+ if not torch.cuda.is_available():
+ return False
+ # DeepSpeed / FSDP own their optimizer state layout; moving it behind their back is unsafe.
+ if getattr(trainer, "is_deepspeed_enabled", False) or getattr(
+ trainer, "is_fsdp_enabled", False
+ ):
+ return False
+ # DDP may keep grads as bucket views; replacing them would detach the views.
+ if getattr(getattr(trainer, "args", None), "world_size", 1) > 1:
+ return False
+ return getattr(trainer, "optimizer", None) is not None
+
+
+def _is_parkable_tensor(value):
+ # bitsandbytes paged state lives in unified memory and is paged by bnb itself.
+ return (
+ isinstance(value, torch.Tensor)
+ and value.is_cuda
+ and value.layout == torch.strided
+ and not getattr(value, "is_paged", False)
+ )
+
+
+def _optimizer_state_is_flat(optimizer):
+ # Layer-wise optimizers keep their real state in optimizer_dict, Q-GaLore keeps CUDA tensors inside
+ # a projector object: parking only the visible tensors would free part of it, so decline instead.
+ if hasattr(optimizer, "optimizer_dict"):
+ return False
+ for state in optimizer.state.values():
+ for value in state.values():
+ if not isinstance(value, (torch.Tensor, int, float, bool, type(None))):
+ return False
+ return True
+
+
+def _park_optimizer_to_host(trainer):
+ """Move every CUDA optimizer state tensor and .grad to pinned host buffers kept on the trainer.
+ Returns the record needed by _restore_optimizer_from_host, or None if nothing was parked."""
+ optimizer = trainer.optimizer
+ optimizer = getattr(optimizer, "optimizer", optimizer)
+ if not _optimizer_state_is_flat(optimizer):
+ if not getattr(trainer, "_unsloth_park_declined", False):
+ trainer._unsloth_park_declined = True
+ print(
+ f"Unsloth: UNSLOTH_PARK_OPTIMIZER_DURING_ROLLOUT does not support {type(optimizer).__name__} "
+ "(nested optimizer state); the optimizer stays on the GPU during rollouts."
+ )
+ return None
+ buffers = trainer.__dict__.setdefault("_unsloth_park_buffers", {})
+ # Shared tensors (bitsandbytes qmap1/qmap2) are parked once so the sharing survives the restore.
+ parked = {}
+ states = []
+ grads = []
+
+ def park(tensor, key):
+ if id(tensor) in parked:
+ return parked[id(tensor)][0]
+ host = buffers.get(key)
+ if (
+ host is None
+ or host.shape != tensor.shape
+ or host.dtype != tensor.dtype
+ or host.stride() != tensor.stride()
+ ):
+ # Same strides as the source: fused AdamW needs param, grad and moments in one layout.
+ host = torch.empty_like(tensor, device = "cpu", pin_memory = True)
+ buffers[key] = host
+ host.copy_(tensor, non_blocking = True)
+ parked[id(tensor)] = (host, tensor.device)
+ return host
+
+ for group in optimizer.param_groups:
+ for param in group["params"]:
+ state = optimizer.state.get(param)
+ if state:
+ for name, value in state.items():
+ if _is_parkable_tensor(value):
+ states.append((state, name, park(value, (id(param), name))))
+ grad = getattr(param, "grad", None)
+ if _is_parkable_tensor(grad):
+ grads.append((param, park(grad, (id(param), ".grad"))))
+ if not states and not grads:
+ return None
+ for state, name, host in states:
+ state[name] = host
+ for param, _ in grads:
+ param.grad = None
+ # Loop locals still hold the last CUDA tensors; drop them so empty_cache can release those blocks.
+ value = grad = None
+ # D2H copies must land on every device before the CUDA blocks are handed to vLLM / released.
+ for device in {device for _, device in parked.values()}:
+ torch.cuda.synchronize(device)
+ torch.cuda.empty_cache()
+ return (states, grads, parked)
+
+
+def _restore_optimizer_from_host(record):
+ states, grads, parked = record
+ restored = {}
+ for host, device in parked.values():
+ restored[id(host)] = host.to(device, non_blocking = True)
+ for state, name, host in states:
+ state[name] = restored[id(host)]
+ for param, host in grads:
+ param.grad = restored[id(host)]
+
+
def _wrap_grpo_generate_and_score(trainer_cls):
if not hasattr(trainer_cls, "_generate_and_score_completions"):
return
@@ -844,9 +952,18 @@ def _wrap_grpo_generate_and_score(trainer_cls):
def wrapped(self, *args, **kwargs):
was_training = getattr(getattr(self, "model", None), "training", None)
+ parked = None
+ if os.environ.get(
+ "UNSLOTH_PARK_OPTIMIZER_DURING_ROLLOUT", "0"
+ ) == "1" and _park_optimizer_enabled(self):
+ parked = _park_optimizer_to_host(self)
+ if parked is not None:
+ self._unsloth_park_count = getattr(self, "_unsloth_park_count", 0) + 1
try:
return original(self, *args, **kwargs)
finally:
+ if parked is not None:
+ _restore_optimizer_from_host(parked)
if (
was_training is False
and hasattr(self, "model")