Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

13.6 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
"""PPO value/reward models built after the rotary patch read uninitialized inv_freq on transformers 5."""

from __future__ import annotations

import ast
import copy
import functools
import importlib.util
import inspect
import sys
from contextlib import contextmanager
from pathlib import Path
from types import ModuleType, SimpleNamespace

import pytest

torch = pytest.importorskip("torch")

RL_PY = Path(__file__).resolve().parents[1] / "unsloth" / "models" / "rl.py"


def _lift(
    name,
    namespace,
    parent = None,
):
    tree = ast.parse(RL_PY.read_text(encoding = "utf-8"))
    scope = tree
    if parent is not None:
        scope = next(n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name == parent)
    node = next(n for n in ast.walk(scope) if isinstance(n, ast.FunctionDef) and n.name == name)
    exec(compile(ast.Module(body = [node], type_ignores = []), str(RL_PY), "exec"), namespace)
    return namespace[name]


class _Policy(torch.nn.Module):
    def __init__(self):
        super().__init__()
        self.is_gradient_checkpointing = True
        self.calls = []

    def gradient_checkpointing_disable(self):
        self.calls.append("disable")
        self.is_gradient_checkpointing = False

    def gradient_checkpointing_enable(self, **kwargs):
        self.calls.append("enable")
        self.is_gradient_checkpointing = True

    def generate(self, *args, **kwargs):
        with torch.inference_mode():
            return torch.ones(1, 3, dtype = torch.long)


def _old_trl_wrapper_class():
    # TRL 0.18 - 0.29 PolicyAndValueWrapper, minus the critic backbone.
    class PolicyAndValueWrapper(torch.nn.Module):
        def __init__(self, policy, value_model):
            super().__init__()
            self.policy = policy
            self.value_model = value_model
            self.is_gradient_checkpointing = policy.is_gradient_checkpointing

    return PolicyAndValueWrapper


@contextmanager
def _trl_unwrap(
    model,
    accelerator = None,
    **kwargs,
):
    # trl.models.utils.unwrap_model_for_generation without DeepSpeed.
    is_gradient_checkpointing = model.is_gradient_checkpointing
    if is_gradient_checkpointing:
        model.gradient_checkpointing_disable()
    yield model
    if is_gradient_checkpointing:
        model.gradient_checkpointing_enable()


def test_old_wrapper_gets_gradient_checkpointing_toggles():
    patch = _lift("_patch_ppo_policy_value_wrapper", {"sys": sys})
    module = ModuleType("fake_ppo_trainer")
    module.PolicyAndValueWrapper = _old_trl_wrapper_class()
    wrapper = module.PolicyAndValueWrapper(_Policy(), torch.nn.Linear(1, 1))
    with pytest.raises(AttributeError):
        with _trl_unwrap(wrapper):
            pass

    patch(module)
    patch(module)
    with _trl_unwrap(wrapper):
        assert wrapper.is_gradient_checkpointing is False
    assert wrapper.policy.calls == ["disable", "enable"]
    assert wrapper.is_gradient_checkpointing is True


def test_wrapper_found_through_the_trainer_module_and_upstream_toggles_kept():
    patch = _lift("_patch_ppo_policy_value_wrapper", {"sys": sys})
    real = ModuleType("fake_trl_experimental_ppo_trainer")
    real.PolicyAndValueWrapper = _old_trl_wrapper_class()
    real.PPOTrainer = type("PPOTrainer", (), {"__module__": real.__name__})
    sys.modules[real.__name__] = real
    try:
        shim = ModuleType("fake_trl_trainer_ppo_trainer")
        shim.PPOTrainer = real.PPOTrainer
        patch(shim)
        assert hasattr(real.PolicyAndValueWrapper, "gradient_checkpointing_disable")
    finally:
        del sys.modules[real.__name__]

    upstream = ModuleType("fake_trl_1x")
    upstream.PolicyAndValueWrapper = _old_trl_wrapper_class()
    own = lambda self: None
    upstream.PolicyAndValueWrapper.gradient_checkpointing_disable = own
    patch(upstream)
    assert upstream.PolicyAndValueWrapper.gradient_checkpointing_disable is own


def _decode_graphs(calls):
    """rl.py's _decode_graphs with its real rl_generation scope, loaded by file path (importing
    unsloth needs an accelerator); entering the scope and releasing for training are recorded."""
    spec = importlib.util.spec_from_file_location(
        "_unsloth_decode_graphs_ppo", RL_PY.with_name("_decode_graphs.py")
    )
    real = importlib.util.module_from_spec(spec)
    spec.loader.exec_module(real)

    class rl_generation(real.rl_generation):
        __slots__ = ()

        def __enter__(self):
            calls.append("rl_generation")
            return super().__enter__()

    return SimpleNamespace(
        rl_generation = rl_generation,
        release_for_training = lambda model: calls.append("release_for_training"),
    )


def _unsloth_unwrap(calls):
    fast = SimpleNamespace(
        for_inference = lambda model: calls.append("for_inference"),
        for_training = lambda model, use_gradient_checkpointing: calls.append(
            ("for_training", use_gradient_checkpointing)
        ),
    )
    namespace = {
        "torch": torch,
        "contextmanager": contextmanager,
        "unwrap_model_for_generation": _trl_unwrap,
        "FastLanguageModel": fast,
        "copy": copy,
        "inspect": inspect,
        "_decode_graphs": _decode_graphs(calls),
    }
    _lift("_generate_accepts_use_model_defaults", namespace)
    _lift("_caller_sampling_only", namespace)
    return _lift("unsloth_unwrap_model_for_generation", namespace, parent = "PatchRL")


def test_unwrap_generates_through_the_ppo_policy():
    patch = _lift("_patch_ppo_policy_value_wrapper", {"sys": sys})
    module = ModuleType("fake_ppo_trainer")
    module.PolicyAndValueWrapper = _old_trl_wrapper_class()
    patch(module)
    policy = _Policy()
    policy.gradient_checkpointing = "unsloth"
    wrapper = module.PolicyAndValueWrapper(policy, torch.nn.Linear(1, 1))

    calls = []
    with _unsloth_unwrap(calls)(wrapper) as unwrapped:
        assert unwrapped is wrapper
        out = unwrapped.policy.generate()
    assert not out.is_inference()
    assert "generate" not in vars(wrapper)
    # Generation runs inside the RL scope; the decode buffers go before training resumes.
    assert calls == [
        "for_inference",
        "rl_generation",
        "release_for_training",
        ("for_training", "unsloth"),
    ]


def test_unwrap_still_wraps_a_model_with_generate():
    policy = _Policy()
    calls = []
    with _unsloth_unwrap(calls)(policy) as unwrapped:
        out = unwrapped.generate()
    assert not out.is_inference()
    assert calls.index("rl_generation") < calls.index("release_for_training")


def _has_real_gpu():
    try:
        torch.zeros(1).to("cuda")
        return True
    except Exception:
        return False


@pytest.mark.skipif(
    not _has_real_gpu(), reason = "LlamaRotaryEmbedding builds per-device caches in __init__"
)
@pytest.mark.parametrize("garbage", [float("nan"), 1e30])
def test_rope_extension_ignores_a_refilled_inv_freq(garbage):
    from transformers import LlamaConfig
    from unsloth.models.llama import LlamaRotaryEmbedding

    config = LlamaConfig(
        hidden_size = 256,
        num_attention_heads = 4,
        max_position_embeddings = 65536,
        rope_theta = 500000.0,
    )
    reference = LlamaRotaryEmbedding(config = config)
    rope = LlamaRotaryEmbedding(config = config)
    # What transformers v5 meta loading leaves in a non-persistent buffer it does not know how to init.
    rope.inv_freq.fill_(garbage)
    x = torch.zeros(1, device = "cuda", dtype = torch.float32)
    reference.extend_rope_embedding(x, config.max_position_embeddings)
    rope.extend_rope_embedding(x, config.max_position_embeddings)
    cos, sin = rope.get_cached(device_index = x.device.index)
    ref_cos, ref_sin = reference.get_cached(device_index = x.device.index)
    assert cos.shape[0] >= config.max_position_embeddings
    assert torch.isfinite(cos).all() and torch.isfinite(sin).all()
    torch.testing.assert_close(cos, ref_cos, rtol = 0, atol = 0)
    torch.testing.assert_close(sin, ref_sin, rtol = 0, atol = 0)
    torch.testing.assert_close(rope.inv_freq, reference.inv_freq, rtol = 0, atol = 0)


def _rollout_helpers():
    namespace = {"copy": copy, "functools": functools, "inspect": inspect}
    for name in (
        "_generate_accepts_use_model_defaults",
        "_caller_sampling_only",
        "_ppo_padding_mask_modules",
        "_wrap_ppo_train",
    ):
        _lift(name, namespace)
    return namespace


NS = _rollout_helpers()


class _Generator:
    def __init__(self, generation_config):
        self.generation_config = generation_config


def test_ppo_rollouts_keep_caller_sampling():
    transformers = pytest.importorskip("transformers")
    model_config = transformers.GenerationConfig(
        top_p = 0.8, top_k = 20, temperature = 0.7, min_p = 0.05, eos_token_id = 5, pad_token_id = 7
    )
    trl_config = transformers.GenerationConfig(
        max_new_tokens = 8, temperature = 0.7, top_k = 0, top_p = 1.0, do_sample = True
    )
    kwargs = {"input_ids": "ids", "generation_config": trl_config}
    out = NS["_caller_sampling_only"](_Generator(model_config), kwargs)
    if not NS["_generate_accepts_use_model_defaults"]():
        # transformers 5 only fills fields left unset, and TRL sets its sampling fields explicitly.
        assert out is kwargs
        return
    assert out["use_model_defaults"] is False and out["input_ids"] == "ids"
    passed = out["generation_config"]
    assert (passed.top_p, passed.top_k, passed.min_p) == (1.0, 0, None)
    # Stopping still follows the model (TRL writes its stop token there).
    assert (passed.eos_token_id, passed.pad_token_id) == (5, 7)
    assert trl_config.eos_token_id is None and model_config.top_p == 0.8


def test_ppo_rollouts_respect_explicit_use_model_defaults():
    transformers = pytest.importorskip("transformers")
    kwargs = {"generation_config": transformers.GenerationConfig(), "use_model_defaults": True}
    assert (
        NS["_caller_sampling_only"](_Generator(transformers.GenerationConfig()), kwargs) is kwargs
    )
    assert NS["_caller_sampling_only"](_Generator(None), {"max_new_tokens": 4}) == {
        "max_new_tokens": 4
    }


def test_ppo_train_wrapped_once_and_mask_cleared_on_error():
    base = _Module(base = True)

    class _Trainer:
        policy_model = _Module(base)

        def train(self):
            assert base._unsloth_keep_padding_mask is True
            raise RuntimeError("oom")

    NS["_wrap_ppo_train"](_Trainer)
    first = _Trainer.train
    NS["_wrap_ppo_train"](_Trainer)
    assert _Trainer.train is first
    with pytest.raises(RuntimeError):
        _Trainer().train()
    assert base._unsloth_keep_padding_mask is False


def test_rollout_logits_freed_after_scoring():
    ppo = pytest.importorskip("trl.trainer.ppo_trainer")
    if not hasattr(ppo, "PPOTrainer"):
        pytest.skip("TRL without trl.trainer.ppo_trainer.PPOTrainer")
    path = RL_PY.with_name("rl_replacements.py")
    tree = ast.parse(path.read_text(encoding = "utf-8"))
    node = next(
        n
        for n in tree.body
        if isinstance(n, ast.FunctionDef) and n.name == "ppo_trainer_free_rollout_logits"
    )
    namespace = {}
    exec(compile(ast.Module(body = [node], type_ignores = []), str(path), "exec"), namespace)
    edit = namespace["ppo_trainer_free_rollout_logits"]

    source = inspect.getsource(ppo.PPOTrainer.train)
    patched = edit("train", source)
    assert "unwrapped_model, logitss)" in patched
    # Nothing past the rollout reads them.
    tail = patched.split("unwrapped_model, logitss)", 1)[1]
    assert "logitss" not in tail
    assert edit("generate_completions", source) == source


class _Module:
    def __init__(
        self,
        *children,
        base = False,
    ):
        self.children = children
        if base:
            self.embed_tokens = object()

    def modules(self):
        yield self
        for child in self.children:
            yield from child.modules()


def test_ppo_training_keeps_padding_masks_then_restores():
    transformers = pytest.importorskip("transformers")
    policy_base, value_base, other_base = _Module(base = True), _Module(base = True), _Module(base = True)
    seen = {}

    class _Trainer:
        def __init__(self):
            self.policy_model = _Module(_Module(policy_base))
            self.policy_model.generation_config = transformers.GenerationConfig()
            self.value_model = _Module(value_base)
            self.ref_model = None
            self.reward_model = _Module(other_base)

        def train(self):
            seen["policy"] = policy_base._unsloth_keep_padding_mask
            seen["value"] = value_base._unsloth_keep_padding_mask
            seen["reward"] = getattr(other_base, "_unsloth_keep_padding_mask", None)

    NS["_wrap_ppo_train"](_Trainer)
    _Trainer().train()
    # The reward model runs in eval mode, which already keeps the mask.
    assert seen == {"policy": True, "value": True, "reward": None}
    assert policy_base._unsloth_keep_padding_mask is False
    assert value_base._unsloth_keep_padding_mask is False


def test_llama_training_forward_mask_is_opt_in():
    # Every other trainer keeps today's behaviour: training mode drops the mask unless the flag is
    # set or a label-less batch is left padded (test_llama_left_padding_training_mask.py).
    source = (RL_PY.parent / "llama.py").read_text(encoding = "utf-8")
    assert (
        "    elif (\n"
        "        self.training\n"
        '        and not getattr(self, "_unsloth_keep_padding_mask", False)\n'
        '        and not (getattr(self, "_has_no_labels", False) and _is_left_padded(attention_mask))\n'
        "    ):\n"
        "        attention_mask = None\n"
    ) in source