Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
GRPO: opt-in parking of optimizer state and gradients on host during rollout (#13866)
Daniel Hanc7824b0bb250
diff --git a/tests/test_grpo_park_optimizer_during_rollout.py b/tests/test_grpo_park_optimizer_during_rollout.pynew file mode 100644index 00000000000..ebc09599bef--- /dev/null+++ b/tests/test_grpo_park_optimizer_during_rollout.py@@ -0,0 +1,211 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""UNSLOTH_PARK_OPTIMIZER_DURING_ROLLOUT=1 moves optimizer state and grads to host around GRPO+generation and puts them back unchanged. The helpers are lifted from rl.py with ast and run on+CPU: a torch shim stands in for pinned memory and CUDA sync, and every tensor counts as parkable.+"""++from __future__ import annotations++import sys+import types+from pathlib import Path++import pytest+import torch+++sys.path.insert(0, str(Path(__file__).resolve().parent))++from _rl_source import load_rl_wrapper # noqa: E402+++NAMES = (+ "_park_optimizer_enabled",+ "_is_parkable_tensor",+ "_optimizer_state_is_flat",+ "_park_optimizer_to_host",+ "_restore_optimizer_from_host",+ "_wrap_grpo_generate_and_score",+)+++def _load():+ fns = load_rl_wrapper(NAMES)+ namespace = fns["_wrap_grpo_generate_and_score"].__globals__+ cuda = types.SimpleNamespace(+ calls = [],+ is_available = lambda: True,+ )+ cuda.synchronize = lambda device = None: cuda.calls.append("synchronize")+ cuda.empty_cache = lambda: cuda.calls.append("empty_cache")++ def empty_like(+ tensor,+ pin_memory = False,+ **kwargs,+ ):+ return torch.empty_like(tensor, **kwargs)++ namespace["torch"] = types.SimpleNamespace(+ Tensor = torch.Tensor, empty_like = empty_like, cuda = cuda, strided = torch.strided+ )+ namespace["_is_parkable_tensor"] = lambda value: isinstance(value, torch.Tensor)+ return fns, cuda+++class _Model:+ training = True+++def _trainer_after_one_step():+ torch.manual_seed(0)+ model = torch.nn.Linear(4, 3)+ optimizer = torch.optim.AdamW(model.parameters(), lr = 1e-2)+ model(torch.randn(2, 4)).sum().backward()+ optimizer.step()+ # Mid-accumulation: a fresh grad is pending while the next rollout happens.+ model(torch.randn(2, 4)).sum().backward()+ trainer = types.SimpleNamespace(model = _Model(), optimizer = optimizer)+ return trainer, model, optimizer+++def _snapshot(optimizer, model):+ state = {+ (id(p), k): v.clone()+ for p, s in optimizer.state.items()+ for k, v in s.items()+ if isinstance(v, torch.Tensor)+ }+ grads = {id(p): p.grad.clone() for p in model.parameters()}+ return state, grads+++def _run(monkeypatch, enabled):+ fns, cuda = _load()+ trainer, model, optimizer = _trainer_after_one_step()+ before = _snapshot(optimizer, model)+ original_state = {(id(p), k): v for p, s in optimizer.state.items() for k, v in s.items()}+ seen = {}++ class Trainer:+ def _generate_and_score_completions(self, inputs):+ seen["grads"] = [p.grad for p in model.parameters()]+ seen["state_objects_kept"] = all(+ v is original_state[(id(p), k)]+ for p, s in optimizer.state.items()+ for k, v in s.items()+ )+ return inputs++ fns["_wrap_grpo_generate_and_score"](Trainer)+ instance = Trainer()+ instance.__dict__.update(vars(trainer))+ if enabled:+ monkeypatch.setenv("UNSLOTH_PARK_OPTIMIZER_DURING_ROLLOUT", "1")+ else:+ monkeypatch.delenv("UNSLOTH_PARK_OPTIMIZER_DURING_ROLLOUT", raising = False)+ assert instance._generate_and_score_completions("x") == "x"+ return instance, model, optimizer, before, seen, cuda+++def test_park_moves_state_and_grads_then_restores_exact_values(monkeypatch):+ instance, model, optimizer, (state_before, grads_before), seen, cuda = _run(monkeypatch, True)+ assert getattr(instance, "_unsloth_park_count", 0) == 1+ assert seen["grads"] == [None, None], "grads must be released during generation"+ assert seen["state_objects_kept"] is False, "state tensors must be swapped for host buffers"+ assert cuda.calls == ["synchronize", "empty_cache"]+ for p, s in optimizer.state.items():+ for k, v in s.items():+ if isinstance(v, torch.Tensor):+ assert torch.equal(v, state_before[(id(p), k)])+ for p in model.parameters():+ assert torch.equal(p.grad, grads_before[id(p)])++ optimizer.step()+++def test_park_off_by_default_leaves_everything_alone(monkeypatch):+ instance, model, optimizer, _, seen, cuda = _run(monkeypatch, False)+ assert getattr(instance, "_unsloth_park_count", 0) == 0+ assert all(g is not None for g in seen["grads"])+ assert seen["state_objects_kept"] is True+ assert cuda.calls == []+++def test_shared_state_tensor_stays_shared(monkeypatch):+ fns, _ = _load()+ params = [torch.nn.Parameter(torch.zeros(3)) for _ in range(2)]+ optimizer = torch.optim.SGD(params, lr = 0.1)+ qmap = torch.arange(4.0)+ for p in params:+ optimizer.state[p] = {"qmap1": qmap, "state1": torch.ones(3)}+ trainer = types.SimpleNamespace(optimizer = optimizer)+ record = fns["_park_optimizer_to_host"](trainer)+ fns["_restore_optimizer_from_host"](record)+ a, b = (optimizer.state[p]["qmap1"] for p in params)+ assert a is b and torch.equal(a, qmap)+++def test_first_step_without_state_or_grads_is_a_noop(monkeypatch):+ fns, cuda = _load()+ model = torch.nn.Linear(2, 2)+ trainer = types.SimpleNamespace(optimizer = torch.optim.AdamW(model.parameters()))+ assert fns["_park_optimizer_to_host"](trainer) is None+ assert cuda.calls == []+++def test_gate_skips_without_optimizer_or_under_deepspeed():+ fns, _ = _load()+ enabled = fns["_park_optimizer_enabled"]+ assert enabled(types.SimpleNamespace(optimizer = None)) is False+ assert enabled(types.SimpleNamespace(optimizer = object(), is_deepspeed_enabled = True)) is False+ assert enabled(types.SimpleNamespace(optimizer = object())) is True+ multi = types.SimpleNamespace(world_size = 2)+ assert enabled(types.SimpleNamespace(optimizer = object(), args = multi)) is False+++def test_no_parked_tensor_is_alive_at_empty_cache():+ import weakref++ fns, cuda = _load()+ trainer, model, optimizer = _trainer_after_one_step()+ refs = [weakref.ref(p.grad) for p in model.parameters()]+ refs += [+ weakref.ref(v)+ for s in optimizer.state.values()+ for v in s.values()+ if isinstance(v, torch.Tensor)+ ]+ alive = []+ cuda.empty_cache = lambda: alive.append(sum(r() is not None for r in refs))+ record = fns["_park_optimizer_to_host"](trainer)+ assert record is not None and alive == [0]+++def test_restored_state_keeps_the_source_strides():+ fns, _ = _load()+ trainer, model, optimizer = _trainer_after_one_step()+ param = model.weight+ state = optimizer.state[param]+ state["exp_avg"] = state["exp_avg"].t().contiguous().t()+ strides = state["exp_avg"].stride()+ record = fns["_park_optimizer_to_host"](trainer)+ assert state["exp_avg"].stride() == strides+ fns["_restore_optimizer_from_host"](record)+ assert state["exp_avg"].stride() == strides+++def test_nested_or_layerwise_optimizer_state_is_declined():+ fns, cuda = _load()+ trainer, model, optimizer = _trainer_after_one_step()+ optimizer.state[model.weight]["projector"] = types.SimpleNamespace(ortho_matrix = torch.ones(2))+ before = _snapshot(optimizer, model)+ assert fns["_park_optimizer_to_host"](trainer) is None+ assert cuda.calls == []+ after = _snapshot(optimizer, model)+ assert all(torch.equal(before[0][k], after[0][k]) for k in before[0])+ layerwise = types.SimpleNamespace(optimizer_dict = {}, param_groups = [], state = {})+ trainer = types.SimpleNamespace(model = _Model(), optimizer = layerwise)+ assert fns["_park_optimizer_to_host"](trainer) is Nonediff --git a/unsloth/models/rl.py b/unsloth/models/rl.pyindex 0bbd6eeffe6..5a7e9c76d5e 100644--- a/unsloth/models/rl.py+++ b/unsloth/models/rl.py@@ -835,6 +835,114 @@ def _wrap_ppo_train(trainer_cls):trainer_cls.train = wrapped+def _park_optimizer_enabled(trainer):+ if not torch.cuda.is_available():+ return False+ # DeepSpeed / FSDP own their optimizer state layout; moving it behind their back is unsafe.+ if getattr(trainer, "is_deepspeed_enabled", False) or getattr(+ trainer, "is_fsdp_enabled", False+ ):+ return False+ # DDP may keep grads as bucket views; replacing them would detach the views.+ if getattr(getattr(trainer, "args", None), "world_size", 1) > 1:+ return False+ return getattr(trainer, "optimizer", None) is not None+++def _is_parkable_tensor(value):+ # bitsandbytes paged state lives in unified memory and is paged by bnb itself.+ return (+ isinstance(value, torch.Tensor)+ and value.is_cuda+ and value.layout == torch.strided+ and not getattr(value, "is_paged", False)+ )+++def _optimizer_state_is_flat(optimizer):+ # Layer-wise optimizers keep their real state in optimizer_dict, Q-GaLore keeps CUDA tensors inside+ # a projector object: parking only the visible tensors would free part of it, so decline instead.+ if hasattr(optimizer, "optimizer_dict"):+ return False+ for state in optimizer.state.values():+ for value in state.values():+ if not isinstance(value, (torch.Tensor, int, float, bool, type(None))):+ return False+ return True+++def _park_optimizer_to_host(trainer):+ """Move every CUDA optimizer state tensor and .grad to pinned host buffers kept on the trainer.+ Returns the record needed by _restore_optimizer_from_host, or None if nothing was parked."""+ optimizer = trainer.optimizer+ optimizer = getattr(optimizer, "optimizer", optimizer)+ if not _optimizer_state_is_flat(optimizer):+ if not getattr(trainer, "_unsloth_park_declined", False):+ trainer._unsloth_park_declined = True+ print(+ f"Unsloth: UNSLOTH_PARK_OPTIMIZER_DURING_ROLLOUT does not support {type(optimizer).__name__} "+ "(nested optimizer state); the optimizer stays on the GPU during rollouts."+ )+ return None+ buffers = trainer.__dict__.setdefault("_unsloth_park_buffers", {})+ # Shared tensors (bitsandbytes qmap1/qmap2) are parked once so the sharing survives the restore.+ parked = {}+ states = []+ grads = []++ def park(tensor, key):+ if id(tensor) in parked:+ return parked[id(tensor)][0]+ host = buffers.get(key)+ if (+ host is None+ or host.shape != tensor.shape+ or host.dtype != tensor.dtype+ or host.stride() != tensor.stride()+ ):+ # Same strides as the source: fused AdamW needs param, grad and moments in one layout.+ host = torch.empty_like(tensor, device = "cpu", pin_memory = True)+ buffers[key] = host+ host.copy_(tensor, non_blocking = True)+ parked[id(tensor)] = (host, tensor.device)+ return host++ for group in optimizer.param_groups:+ for param in group["params"]:+ state = optimizer.state.get(param)+ if state:+ for name, value in state.items():+ if _is_parkable_tensor(value):+ states.append((state, name, park(value, (id(param), name))))+ grad = getattr(param, "grad", None)+ if _is_parkable_tensor(grad):+ grads.append((param, park(grad, (id(param), ".grad"))))+ if not states and not grads:+ return None+ for state, name, host in states:+ state[name] = host+ for param, _ in grads:+ param.grad = None+ # Loop locals still hold the last CUDA tensors; drop them so empty_cache can release those blocks.+ value = grad = None+ # D2H copies must land on every device before the CUDA blocks are handed to vLLM / released.+ for device in {device for _, device in parked.values()}:+ torch.cuda.synchronize(device)+ torch.cuda.empty_cache()+ return (states, grads, parked)+++def _restore_optimizer_from_host(record):+ states, grads, parked = record+ restored = {}+ for host, device in parked.values():+ restored[id(host)] = host.to(device, non_blocking = True)+ for state, name, host in states:+ state[name] = restored[id(host)]+ for param, host in grads:+ param.grad = restored[id(host)]++def _wrap_grpo_generate_and_score(trainer_cls):if not hasattr(trainer_cls, "_generate_and_score_completions"):return@@ -844,9 +952,18 @@ def _wrap_grpo_generate_and_score(trainer_cls):def wrapped(self, *args, **kwargs):was_training = getattr(getattr(self, "model", None), "training", None)+ parked = None+ if os.environ.get(+ "UNSLOTH_PARK_OPTIMIZER_DURING_ROLLOUT", "0"+ ) == "1" and _park_optimizer_enabled(self):+ parked = _park_optimizer_to_host(self)+ if parked is not None:+ self._unsloth_park_count = getattr(self, "_unsloth_park_count", 0) + 1try:return original(self, *args, **kwargs)finally:+ if parked is not None:+ _restore_optimizer_from_host(parked)if (was_training is Falseand hasattr(self, "model")