Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Fix PPOTrainer with an Unsloth policy: wrapper GC toggles, policy generate, RoPE on value / reward models (#13067)
Daniel Hanbf0a464bf047
diff --git a/tests/test_ppo_trainer_patches.py b/tests/test_ppo_trainer_patches.pynew file mode 100644index 00000000000..fda142f7c82--- /dev/null+++ b/tests/test_ppo_trainer_patches.py@@ -0,0 +1,203 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.+"""PPOTrainer with an Unsloth policy (#884).++TRL < 1.0 PolicyAndValueWrapper copies is_gradient_checkpointing from the policy without the toggles+unwrap_model_for_generation calls, and has no generate() for Unsloth's unwrap to wrap. The PPO value+and reward models are plain transformers models built after Unsloth patched the rotary class, so on+transformers v5 their inv_freq buffer is uninitialized memory that extend_rope_embedding read.+The rl.py helpers are lifted out with ``ast`` so those checks run without ``import unsloth``.+"""++from __future__ import annotations++import ast+import sys+from contextlib import contextmanager+from pathlib import Path+from types import ModuleType, SimpleNamespace++import pytest++torch = pytest.importorskip("torch")++RL_PY = Path(__file__).resolve().parents[1] / "unsloth" / "models" / "rl.py"+++def _lift(+ name,+ namespace,+ parent = None,+):+ tree = ast.parse(RL_PY.read_text(encoding = "utf-8"))+ scope = tree+ if parent is not None:+ scope = next(n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name == parent)+ node = next(n for n in ast.walk(scope) if isinstance(n, ast.FunctionDef) and n.name == name)+ exec(compile(ast.Module(body = [node], type_ignores = []), str(RL_PY), "exec"), namespace)+ return namespace[name]+++class _Policy(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.is_gradient_checkpointing = True+ self.calls = []++ def gradient_checkpointing_disable(self):+ self.calls.append("disable")+ self.is_gradient_checkpointing = False++ def gradient_checkpointing_enable(self, **kwargs):+ self.calls.append("enable")+ self.is_gradient_checkpointing = True++ def generate(self, *args, **kwargs):+ with torch.inference_mode():+ return torch.ones(1, 3, dtype = torch.long)+++def _old_trl_wrapper_class():+ # TRL 0.18 - 0.29 PolicyAndValueWrapper, minus the critic backbone.+ class PolicyAndValueWrapper(torch.nn.Module):+ def __init__(self, policy, value_model):+ super().__init__()+ self.policy = policy+ self.value_model = value_model+ self.is_gradient_checkpointing = policy.is_gradient_checkpointing++ return PolicyAndValueWrapper+++@contextmanager+def _trl_unwrap(+ model,+ accelerator = None,+ **kwargs,+):+ # trl.models.utils.unwrap_model_for_generation without DeepSpeed.+ is_gradient_checkpointing = model.is_gradient_checkpointing+ if is_gradient_checkpointing:+ model.gradient_checkpointing_disable()+ yield model+ if is_gradient_checkpointing:+ model.gradient_checkpointing_enable()+++def test_old_wrapper_gets_gradient_checkpointing_toggles():+ patch = _lift("_patch_ppo_policy_value_wrapper", {"sys": sys})+ module = ModuleType("fake_ppo_trainer")+ module.PolicyAndValueWrapper = _old_trl_wrapper_class()+ wrapper = module.PolicyAndValueWrapper(_Policy(), torch.nn.Linear(1, 1))+ with pytest.raises(AttributeError):+ with _trl_unwrap(wrapper):+ pass++ patch(module)+ patch(module)+ with _trl_unwrap(wrapper):+ assert wrapper.is_gradient_checkpointing is False+ assert wrapper.policy.calls == ["disable", "enable"]+ assert wrapper.is_gradient_checkpointing is True+++def test_wrapper_found_through_the_trainer_module_and_upstream_toggles_kept():+ patch = _lift("_patch_ppo_policy_value_wrapper", {"sys": sys})+ real = ModuleType("fake_trl_experimental_ppo_trainer")+ real.PolicyAndValueWrapper = _old_trl_wrapper_class()+ real.PPOTrainer = type("PPOTrainer", (), {"__module__": real.__name__})+ sys.modules[real.__name__] = real+ try:+ shim = ModuleType("fake_trl_trainer_ppo_trainer")+ shim.PPOTrainer = real.PPOTrainer+ patch(shim)+ assert hasattr(real.PolicyAndValueWrapper, "gradient_checkpointing_disable")+ finally:+ del sys.modules[real.__name__]++ upstream = ModuleType("fake_trl_1x")+ upstream.PolicyAndValueWrapper = _old_trl_wrapper_class()+ own = lambda self: None+ upstream.PolicyAndValueWrapper.gradient_checkpointing_disable = own+ patch(upstream)+ assert upstream.PolicyAndValueWrapper.gradient_checkpointing_disable is own+++def _unsloth_unwrap(calls):+ fast = SimpleNamespace(+ for_inference = lambda model: calls.append("for_inference"),+ for_training = lambda model, use_gradient_checkpointing: calls.append(+ ("for_training", use_gradient_checkpointing)+ ),+ )+ namespace = {+ "torch": torch,+ "contextmanager": contextmanager,+ "unwrap_model_for_generation": _trl_unwrap,+ "FastLanguageModel": fast,+ }+ return _lift("unsloth_unwrap_model_for_generation", namespace, parent = "PatchRL")+++def test_unwrap_generates_through_the_ppo_policy():+ patch = _lift("_patch_ppo_policy_value_wrapper", {"sys": sys})+ module = ModuleType("fake_ppo_trainer")+ module.PolicyAndValueWrapper = _old_trl_wrapper_class()+ patch(module)+ policy = _Policy()+ policy.gradient_checkpointing = "unsloth"+ wrapper = module.PolicyAndValueWrapper(policy, torch.nn.Linear(1, 1))++ calls = []+ with _unsloth_unwrap(calls)(wrapper) as unwrapped:+ assert unwrapped is wrapper+ out = unwrapped.policy.generate()+ assert not out.is_inference()+ assert "generate" not in vars(wrapper)+ assert calls == ["for_inference", ("for_training", "unsloth")]+++def test_unwrap_still_wraps_a_model_with_generate():+ policy = _Policy()+ calls = []+ with _unsloth_unwrap(calls)(policy) as unwrapped:+ out = unwrapped.generate()+ assert not out.is_inference()+++def _has_real_gpu():+ try:+ torch.zeros(1).to("cuda")+ return True+ except Exception:+ return False+++ not _has_real_gpu(), reason = "LlamaRotaryEmbedding builds per-device caches in __init__"+)[email protected]("garbage", [float("nan"), 1e30])+def test_rope_extension_ignores_a_refilled_inv_freq(garbage):+ from transformers import LlamaConfig+ from unsloth.models.llama import LlamaRotaryEmbedding++ config = LlamaConfig(+ hidden_size = 256,+ num_attention_heads = 4,+ max_position_embeddings = 65536,+ rope_theta = 500000.0,+ )+ reference = LlamaRotaryEmbedding(config = config)+ rope = LlamaRotaryEmbedding(config = config)+ # What transformers v5 meta loading leaves in a non-persistent buffer it does not know how to init.+ rope.inv_freq.fill_(garbage)+ x = torch.zeros(1, device = "cuda", dtype = torch.float32)+ reference.extend_rope_embedding(x, config.max_position_embeddings)+ rope.extend_rope_embedding(x, config.max_position_embeddings)+ cos, sin = rope.get_cached(device_index = x.device.index)+ ref_cos, ref_sin = reference.get_cached(device_index = x.device.index)+ assert cos.shape[0] >= config.max_position_embeddings+ assert torch.isfinite(cos).all() and torch.isfinite(sin).all()+ torch.testing.assert_close(cos, ref_cos, rtol = 0, atol = 0)+ torch.testing.assert_close(sin, ref_sin, rtol = 0, atol = 0)+ torch.testing.assert_close(rope.inv_freq, reference.inv_freq, rtol = 0, atol = 0)diff --git a/unsloth/models/llama.py b/unsloth/models/llama.pyindex dac5d1bc72e..ba07f05caf8 100644--- a/unsloth/models/llama.py+++ b/unsloth/models/llama.py@@ -2000,6 +2000,10 @@ class LlamaRotaryEmbedding(torch.nn.Module):return tdef _set_cos_sin_cache(self, seq_len, device, dtype):+ # Rebuild inv_freq instead of reading the buffer: transformers v5 refills it with uninitialized+ # memory on models _fix_rope_inv_freq never sees, e.g. a PPO value / reward model loaded after+ # Unsloth patched the classes, so extend_rope_embedding built NaN or garbage cos/sin (#884).+ self.inv_freq = self._unsloth_recompute_inv_freq().to(self.inv_freq.device)# The original Llama codebase creates these on the target device in FP32 and multiplies in FP32.self.current_rope_size = seq_lent = torch.arange(diff --git a/unsloth/models/rl.py b/unsloth/models/rl.pyindex 5b256b94b7f..34bea619d8c 100644--- a/unsloth/models/rl.py+++ b/unsloth/models/rl.py@@ -204,8 +204,13 @@ def PatchRL(FastLanguageModel):with unwrap_model_for_generation(model, *args, **kwargs) as unwrapped_model:FastLanguageModel.for_inference(model)+ # PPO's PolicyAndValueWrapper has no generate(); TRL generates through its .policy.+ generating_model = unwrapped_model+ if not hasattr(generating_model, "generate"):+ generating_model = getattr(unwrapped_model, "policy", unwrapped_model)+# .clone is required because inference_mode is forced here; no_grad would have been the better choice.- original_generate = unwrapped_model.generate+ original_generate = generating_model.generatedef generate_with_clone(*args, **kwargs):out = original_generate(*args, **kwargs)@@ -213,12 +218,12 @@ def PatchRL(FastLanguageModel):return out.clone()return out- unwrapped_model.generate = generate_with_clone+ generating_model.generate = generate_with_clonetry:yield unwrapped_modelfinally:- unwrapped_model.generate = original_generate+ generating_model.generate = original_generateFastLanguageModel.for_training(model,use_gradient_checkpointing = use_gradient_checkpointing,@@ -2039,6 +2044,32 @@ def _backport_vision_dataset_gate(RLTrainer_source):return RLTrainer_source+def _patch_ppo_policy_value_wrapper(module):+ # TRL < 1.0 copies is_gradient_checkpointing onto PolicyAndValueWrapper without the toggles+ # unwrap_model_for_generation then calls on it; same fix as huggingface/trl#5245.+ wrapper = getattr(module, "PolicyAndValueWrapper", None)+ if wrapper is None:+ trainer_class = getattr(module, "PPOTrainer", None)+ wrapper = getattr(+ sys.modules.get(getattr(trainer_class, "__module__", "")),+ "PolicyAndValueWrapper",+ None,+ )+ if wrapper is None or hasattr(wrapper, "gradient_checkpointing_disable"):+ return++ def gradient_checkpointing_enable(self, **kwargs):+ self.policy.gradient_checkpointing_enable(**kwargs)+ self.is_gradient_checkpointing = True++ def gradient_checkpointing_disable(self):+ self.policy.gradient_checkpointing_disable()+ self.is_gradient_checkpointing = False++ wrapper.gradient_checkpointing_enable = gradient_checkpointing_enable+ wrapper.gradient_checkpointing_disable = gradient_checkpointing_disable++def _patch_trl_rl_trainers(trainer_file = "grpo_trainer"):# Defensive wrapper matching patch_trl_rl_trainers()'s try/except, so direct callers do not see exceptions from the impl on TRL versions that rename or move classes.try:@@ -2063,6 +2094,9 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):logger.info(f"Unsloth: Could not import trl.trainer.{trainer_file}: {error}")return+ if trainer_file == "ppo_trainer":+ _patch_ppo_policy_value_wrapper(trainer)+name = [xfor x in dir(trainer)