Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

19.1 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
"""PPOTrainer's rollout, reference and training log-probs (and entropy metric) scored from hidden states by the zoo
fused linear kernel; the value head keeps reading the critic backbone."""

from __future__ import annotations

import ast
import copy
import importlib
import os
from types import SimpleNamespace

import pytest

torch = pytest.importorskip("torch")
os.environ.setdefault("UNSLOTH_COMPILE_DISABLE", "1")
import unsloth  # noqa: F401,E402
import unsloth.models.rl_replacements as rl  # noqa: E402
from unsloth.models import _fused_rl_logprobs as frl  # noqa: E402

fused_linear = pytest.importorskip("unsloth_zoo.fused_losses.fused_linear")


def _ppo_module():
    for name in ("trl.trainer.ppo_trainer", "trl.experimental.ppo.ppo_trainer"):
        try:
            module = importlib.import_module(name)
        except Exception:
            continue
        if "class PPOTrainer" in open(module.__file__, encoding = "utf-8").read():
            return module
    pytest.skip("this TRL has no PPOTrainer")


def _trl_train_source(module):
    text = open(module.__file__, encoding = "utf-8").read()
    cls = next(
        n for n in ast.parse(text).body if isinstance(n, ast.ClassDef) and n.name == "PPOTrainer"
    )
    fn = next(n for n in cls.body if isinstance(n, ast.FunctionDef) and n.name == "train")
    return "\n".join(text.split("\n")[fn.lineno - 1 : fn.end_lineno]) + "\n"


H, V, CL = 16, 40, 4


class _Policy(torch.nn.Module):
    def __init__(
        self,
        dtype = torch.float64,
        honours_flag = True,
    ):
        super().__init__()
        from transformers import Qwen3Config

        self.config = Qwen3Config(hidden_size = H, vocab_size = V, num_hidden_layers = 1)
        gen = torch.Generator().manual_seed(0)
        self.embed = torch.nn.Embedding(V, H, dtype = dtype)
        self.embed.weight.data = torch.randn(V, H, generator = gen, dtype = dtype)
        self.mix = torch.nn.Linear(H, H, dtype = dtype)
        self.mix.weight.data = torch.randn(H, H, generator = gen, dtype = dtype) / H**0.5
        self.mix.bias.data = torch.zeros(H, dtype = dtype)
        self.lm_head = torch.nn.Linear(H, V, bias = False, dtype = dtype)
        self.lm_head.weight.data = torch.randn(V, H, generator = gen, dtype = dtype)
        self.honours_flag = honours_flag
        self.forwards = []

    def get_output_embeddings(self):
        return self.lm_head

    def forward(
        self,
        input_ids,
        attention_mask = None,
        **kwargs,
    ):
        h = torch.tanh(self.mix(self.embed(input_ids)))
        hidden = self.honours_flag and os.environ.get("UNSLOTH_RETURN_HIDDEN_STATES", "0") == "1"
        self.forwards.append("hidden" if hidden else "logits")
        return SimpleNamespace(logits = h if hidden else self.lm_head(h), hidden_states = (h,))

    def generate(
        self,
        input_ids,
        attention_mask,
        generation_config = None,
        return_dict_in_generate = False,
        output_scores = False,
        **kwargs,
    ):
        new = (input_ids[:, -1:] * 7 + torch.arange(1, 6)) % V
        out = SimpleNamespace(sequences = torch.cat([input_ids, new], 1))
        if output_scores:
            out.scores = [
                torch.randn(input_ids.shape[0], V, dtype = torch.float64) for _ in range(new.shape[1])
            ]
        return out


class _Value(torch.nn.Module):
    base_model_prefix = "backbone"

    def __init__(self):
        super().__init__()
        self.backbone = _Policy()
        self.score = torch.nn.Linear(H, 1, dtype = torch.float64)


def _wrapper(module, policy):
    value = _Value()

    class _Backbone(torch.nn.Module):
        def __init__(self, inner):
            super().__init__()
            self.inner = inner

        def forward(self, **kwargs):
            out = self.inner(**kwargs)
            return SimpleNamespace(hidden_states = out.hidden_states)

    value.backbone = _Backbone(value.backbone)
    policy.is_gradient_checkpointing = False
    return module.PolicyAndValueWrapper(policy, value)


def _route(model):
    trainer = SimpleNamespace(
        model = model,
        accelerator = SimpleNamespace(unwrap_model = lambda m, **k: m),
        is_fsdp_enabled = False,
        is_deepspeed_enabled = False,
    )
    return frl.ppo_route(trainer), trainer


def _query_responses(B = 3, L = 10):
    gen = torch.Generator().manual_seed(2)
    qr = torch.randint(1, V, (B, L), generator = gen)
    qr[0, 0] = 0  # left pad
    return qr


def test_ppo_train_rewrite_applies_to_installed_trl():
    module = _ppo_module()
    source = _trl_train_source(module)
    new = rl.ppo_trainer_fused_logprobs("train", source)
    assert new != source, "PPO train rewrite did not apply to the installed TRL"
    ast.parse("class A:\n" + new)
    for needle in (
        "_unsloth_ppo_route = _unsloth_ppo_route_of(self)",
        "query_responses, logitss = _unsloth_ppo_batch_generation(",
        "if _unsloth_ppo_route is not None and logitss is None:",
        "logprob = _unsloth_ppo_policy_logprobs(",
        "ref_output = _unsloth_ppo_forward(",
        "output, vpred_temp = _unsloth_ppo_forward(",
        "new_logprobs, _unsloth_entropy = _unsloth_scored",
        "entropy = _unsloth_entropy",
    ):
        assert needle in new, needle
    for line in (
        "new_logprobs = selective_log_softmax(logits, mb_responses)",
        "ref_logprob = selective_log_softmax(ref_logits, response)",
        "logprob = selective_log_softmax(logits, response)",
    ):
        assert line in new
    assert rl.ppo_trainer_fused_logprobs("train", new) == new
    assert rl.ppo_trainer_fused_logprobs("generate_completions", source) == source
    drifted = source.replace(
        "new_logprobs = selective_log_softmax(logits, mb_responses)", "new_logprobs = f(logits)"
    )
    assert rl.ppo_trainer_fused_logprobs("train", drifted) == drifted
    assert rl.ppo_trainer_fused_logprobs in rl.RL_FUNCTIONS["ppo_trainer"]


def _stock_train_logps(module, model, qr, T):
    output, vpred_temp = module.forward(model, qr, 0)
    logits = output.logits[:, CL - 1 : -1]
    logits /= T
    logps = module.selective_log_softmax(logits, qr[:, CL:])
    prob_dist = torch.nn.functional.softmax(logits, dim = -1)
    entropy = torch.logsumexp(logits, dim = -1) - torch.sum(prob_dist * logits, dim = -1)
    return logps, entropy.detach(), vpred_temp


def _fused_train_logps(module, route, model, qr, T):
    output, vpred_temp = frl.ppo_forward(route, module.forward, model, qr, 0)
    out = frl.ppo_score(route, model, output.logits[:, CL - 1 : -1], qr[:, CL:], T, entropy = True)
    assert out is not None
    return out[0], out[1], vpred_temp


def test_ppo_training_pass_matches_trl_fp64():
    module = _ppo_module()
    T = 0.7 + 1e-7
    model = _wrapper(module, _Policy())
    route, trainer = _route(model)
    assert route is not None
    qr = _query_responses()
    calls = fused_linear.CALLS["logprobs"]
    results = []
    for fn in (
        lambda: _fused_train_logps(module, route, model, qr, T),
        lambda: _stock_train_logps(module, model, qr, T),
    ):
        model.zero_grad()
        logps, ent, vpred = fn()
        adv = torch.linspace(-1, 1, logps.shape[1], dtype = torch.float64)
        ratio = torch.exp(logps - logps.detach())
        loss = (-adv * ratio).mean() + 0.1 * vpred.square().mean()
        loss.backward()
        grads = [
            p.grad.detach().clone()
            for p in (
                model.policy.mix.weight,
                model.policy.embed.weight,
                model.value_model.score.weight,
            )
        ]
        results.append((logps.detach(), ent, vpred.detach(), loss.detach(), grads))
    assert fused_linear.CALLS["logprobs"] > calls and trainer._unsloth_ppo_fused_calls == 1
    (a, b) = results
    for x, y in zip(a[:4], b[:4]):
        torch.testing.assert_close(x.double(), y.double(), atol = 1e-6, rtol = 1e-6)
    for x, y in zip(a[4], b[4]):
        torch.testing.assert_close(x.double(), y.double(), atol = 1e-6, rtol = 1e-5)
    assert not a[1].requires_grad


def test_ppo_rollout_and_reference_logprobs_match_trl_fp64():
    module = _ppo_module()
    T = 0.9 + 1e-7
    policy = _Policy()
    model = _wrapper(module, policy)
    route, _ = _route(model)
    qr = _query_responses()
    with torch.no_grad():
        got = frl.ppo_policy_logprobs(
            route, module.forward, module.selective_log_softmax, model, qr, qr[:, CL:], CL, 0, T
        )
        logits = module.forward(policy, qr, 0).logits[:, CL - 1 : -1] / T
        want = module.selective_log_softmax(logits, qr[:, CL:])
    torch.testing.assert_close(got.double(), want.double(), atol = 1e-6, rtol = 1e-6)
    # A reference model whose forward ignores the flag: TRL's lines run on its real logits.
    ref = _Policy(honours_flag = False)
    out = frl.ppo_forward(route, module.forward, ref, qr, 0)
    assert frl.ppo_score(route, ref, out.logits[:, CL - 1 : -1], qr[:, CL:], T) is None
    got = frl.ppo_policy_logprobs(
        route, module.forward, module.selective_log_softmax, ref, qr, qr[:, CL:], CL, 0, T
    )
    want = module.selective_log_softmax(
        module.forward(ref, qr, 0).logits[:, CL - 1 : -1] / T, qr[:, CL:]
    )
    torch.testing.assert_close(got, want)


def test_ppo_bf16_training_logps_error_not_worse_than_stock():
    module = _ppo_module()
    T = 0.7 + 1e-7
    exact = _wrapper(module, _Policy())
    qr = _query_responses()
    ref, _, _ = _stock_train_logps(module, exact, qr, T)
    low = copy.deepcopy(exact).to(torch.bfloat16)
    route, _ = _route(low)
    fused, _, _ = _fused_train_logps(module, route, low, qr, T)
    stock, _, _ = _stock_train_logps(module, low, qr, T)
    # Mean, not max: one bf16-rounded hidden row dominates a max either way.
    err_fused = (fused.double() - ref.double()).abs().mean()
    err_stock = (stock.double() - ref.double()).abs().mean()
    assert err_fused <= err_stock + 1e-6, (err_fused, err_stock)


def _batch_generation(module):
    batch_generation = getattr(module, "batch_generation", None)
    if batch_generation is None:
        from trl.trainer.utils import batch_generation
    return batch_generation


def test_ppo_generation_without_generate_helpers_keeps_trl():
    module = _ppo_module()
    batch_generation = _batch_generation(module)
    policy = _Policy()
    route, trainer = _route(_wrapper(module, policy))
    queries = _query_responses(B = 5, L = 4)
    want, logitss = batch_generation(policy, queries, 2, 0, None)
    got, scores = frl.ppo_batch_generation(route, batch_generation, policy, queries, 2, 0, None)
    assert scores is not None and scores.shape == logitss.shape
    torch.testing.assert_close(got, want)
    assert (
        "generation logits processors besides temperature" in trainer._unsloth_ppo_fused_fallbacks
    )
    got, scores = frl.ppo_batch_generation(None, batch_generation, policy, queries, 2, 0, None)
    assert scores is not None


def _hidden_gpt2():
    from transformers import GPT2Config, GPT2LMHeadModel

    class _HiddenGPT2(GPT2LMHeadModel):
        # position_ids in the signature: generate then derives them from the left-padded attention mask.
        def forward(
            self,
            input_ids = None,
            attention_mask = None,
            position_ids = None,
            **kwargs,
        ):
            hidden = os.environ.get("UNSLOTH_RETURN_HIDDEN_STATES") == "1"
            if hidden:
                kwargs["output_hidden_states"] = True
            out = super().forward(
                input_ids = input_ids,
                attention_mask = attention_mask,
                position_ids = position_ids,
                **kwargs,
            )
            if hidden:
                out.logits = out.hidden_states[-1]
            return out

    torch.manual_seed(42)
    config = GPT2Config(
        vocab_size = 41,
        n_embd = 16,
        n_layer = 1,
        n_head = 2,
        n_positions = 64,
        resid_pdrop = 0.0,
        embd_pdrop = 0.0,
        attn_pdrop = 0.0,
        bos_token_id = 1,
        eos_token_id = None,
        pad_token_id = 0,
    )
    return _HiddenGPT2(config).double().eval()


@pytest.mark.parametrize(
    "inherited",
    [{}, {"repetition_penalty": 2.0}, {"no_repeat_ngram_size": 2}, {"suppress_tokens": [3, 5]}],
)
def test_ppo_rollout_logprobs_match_trl_generation_scores(inherited):
    from transformers import GenerationConfig

    module = _ppo_module()
    batch_generation = _batch_generation(module)
    model = _hidden_gpt2()
    for key, value in inherited.items():
        setattr(model.generation_config, key, value)
    T = 0.7 + 1e-7
    generation_config = GenerationConfig(
        max_new_tokens = 6, temperature = T, top_k = 0.0, top_p = 1.0, do_sample = True
    )
    trainer = SimpleNamespace(
        model = SimpleNamespace(policy = model),
        accelerator = SimpleNamespace(unwrap_model = lambda m, **k: m),
        is_fsdp_enabled = False,
        is_deepspeed_enabled = False,
    )
    route = frl.ppo_route(trainer)
    assert route is not None
    queries = torch.tensor([[1, 2, 3, 4], [1, 5, 6, 7], [0, 1, 8, 9]])
    torch.manual_seed(123)
    want, trl_scores = batch_generation(model, queries, 2, 0, generation_config)
    torch.manual_seed(123)
    got, scores = frl.ppo_batch_generation(
        route, batch_generation, model, queries, 2, 0, generation_config
    )
    torch.testing.assert_close(got, want)
    responses = want[:, queries.shape[1] :]
    trl_old = module.selective_log_softmax(trl_scores, responses)
    if scores is None:
        calls = fused_linear.CALLS["logprobs"]
        old = frl.ppo_policy_logprobs(
            route,
            module.forward,
            module.selective_log_softmax,
            trainer.model,
            want,
            responses,
            queries.shape[1],
            0,
            T,
        )
        assert fused_linear.CALLS["logprobs"] > calls
    else:
        old = module.selective_log_softmax(scores, responses)
    torch.testing.assert_close(old, trl_old, atol = 1e-5, rtol = 1e-5)
    fallbacks = getattr(trainer, "_unsloth_ppo_fused_fallbacks", {})
    if inherited:
        assert (
            scores is not None and "generation logits processors besides temperature" in fallbacks
        )
        tempered = module.forward(model, want, 0).logits[:, queries.shape[1] - 1 : -1] / T
        assert (module.selective_log_softmax(tempered, responses) - trl_old).abs().max() > 1e-3
    else:
        assert scores is None and not fallbacks


def test_ppo_rollout_check_follows_the_patched_generate():
    from transformers import GenerationConfig
    from unsloth.models.rl import _caller_sampling_only, _generate_accepts_use_model_defaults

    module = _ppo_module()
    batch_generation = _batch_generation(module)
    model = _hidden_gpt2()
    model.generation_config.top_p = 0.8
    original_generate = model.generate

    def generate_with_clone(*args, **kwargs):
        return original_generate(*args, **_caller_sampling_only(model, kwargs))

    generate_with_clone._unsloth_caller_sampling_only = True
    model.generate = generate_with_clone
    T = 0.7 + 1e-7
    generation_config = GenerationConfig(
        max_new_tokens = 6, temperature = T, top_k = 0.0, top_p = 1.0, do_sample = True
    )
    trainer = SimpleNamespace(
        model = SimpleNamespace(policy = model),
        accelerator = SimpleNamespace(unwrap_model = lambda m, **k: m),
        is_fsdp_enabled = False,
        is_deepspeed_enabled = False,
    )
    route = frl.ppo_route(trainer)
    queries = torch.tensor([[1, 2, 3, 4], [1, 5, 6, 7], [0, 1, 8, 9]])
    torch.manual_seed(123)
    want, trl_scores = batch_generation(model, queries, 2, 0, generation_config)
    torch.manual_seed(123)
    got, scores = frl.ppo_batch_generation(
        route, batch_generation, model, queries, 2, 0, generation_config
    )
    torch.testing.assert_close(got, want)
    responses = want[:, queries.shape[1] :]
    trl_old = module.selective_log_softmax(trl_scores, responses)
    if scores is None:
        old = frl.ppo_policy_logprobs(
            route,
            module.forward,
            module.selective_log_softmax,
            trainer.model,
            want,
            responses,
            queries.shape[1],
            0,
            T,
        )
    else:
        old = module.selective_log_softmax(scores, responses)
    torch.testing.assert_close(old, trl_old, atol = 1e-5, rtol = 1e-5)
    if _generate_accepts_use_model_defaults():
        assert scores is None


def test_ppo_kill_switch_keeps_trl(monkeypatch):
    module = _ppo_module()
    model = _wrapper(module, _Policy())
    monkeypatch.setenv("UNSLOTH_FUSED_LINEAR", "0")
    route, trainer = _route(model)
    assert route is None
    assert trainer._unsloth_ppo_fused_fallbacks == {"UNSLOTH_FUSED_LINEAR=0": 1}
    calls = fused_linear.CALLS["logprobs"]
    model.policy.forwards.clear()
    output, _ = frl.ppo_forward(route, module.forward, model, _query_responses(), 0)
    assert output.logits.shape[-1] == V and model.policy.forwards == ["logits"]
    assert frl.ppo_score(route, model, output.logits, _query_responses(), 1.0) is None
    assert fused_linear.CALLS["logprobs"] == calls


def test_ppo_reference_with_hooked_head_keeps_trl():
    module = _ppo_module()
    T = 0.9 + 1e-7
    route, trainer = _route(_wrapper(module, _Policy()))
    assert route is not None
    ref = _Policy()
    ref.lm_head.register_forward_hook(lambda *a: None)
    qr = _query_responses()
    with torch.no_grad():
        out = frl.ppo_forward(route, module.forward, ref, qr, 0)
        assert ref.forwards == ["logits"] and out.logits.shape[-1] == V
        assert frl.ppo_score(route, ref, out.logits[:, CL - 1 : -1], qr[:, CL:], T) is None
        got = frl.ppo_policy_logprobs(
            route, module.forward, module.selective_log_softmax, ref, qr, qr[:, CL:], CL, 0, T
        )
        want = module.selective_log_softmax(
            module.forward(ref, qr, 0).logits[:, CL - 1 : -1] / T, qr[:, CL:]
        )
    torch.testing.assert_close(got, want)
    assert "output head has hooks or a patched forward" in trainer._unsloth_ppo_fused_fallbacks


def test_ppo_hooked_head_keeps_trl():
    module = _ppo_module()
    model = _wrapper(module, _Policy())
    model.policy.lm_head.register_forward_hook(lambda *a: None)
    route, trainer = _route(model)
    assert route is None
    assert "output head has hooks or a patched forward" in trainer._unsloth_ppo_fused_fallbacks


@pytest.mark.parametrize(
    "register",
    [
        "register_module_forward_hook",
        "register_module_forward_pre_hook",
        "register_module_full_backward_hook",
        "register_module_full_backward_pre_hook",
    ],
)
def test_ppo_global_module_hooks_keep_trl(register):
    """torch's global module hooks would never see a head the fused kernel reads directly."""
    from torch.nn.modules import module as module_hooks

    module = _ppo_module()
    model = _wrapper(module, _Policy())
    assert _route(model)[0] is not None
    handle = getattr(module_hooks, register)(lambda *a: None)
    try:
        route, trainer = _route(model)
    finally:
        handle.remove()
    assert route is None
    assert "output head has hooks or a patched forward" in trainer._unsloth_ppo_fused_fallbacks
    assert _route(model)[0] is not None