Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

19.1 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team.
"""ORPO / CPO concatenated_forward scores from hidden states through the chunked log-prob head and matches TRL."""

from __future__ import annotations

import ast
import functools
import importlib.util
import os
import sys
import textwrap
import types

import pytest
import torch

os.environ.setdefault("UNSLOTH_COMPILE_DISABLE", "1")
import unsloth.models.rl_replacements as rl

trl = pytest.importorskip("trl")

if rl.unsloth_chunked_logprobs is None:
    # Skip because the kernel ships in unsloth_zoo (unslothai/unsloth-zoo#1803).
    pytest.skip(
        reason = "installed unsloth_zoo predates unsloth_chunked_logprobs", allow_module_level = True
    )


def _trl_class(kind):
    """TRL's own class and the source of its concatenated_forward, read from the file (importing unsloth may
    already have rebound the class to the patched one)."""
    for name in (f"trl.experimental.{kind}.{kind}_trainer", f"trl.trainer.{kind}_trainer"):
        try:
            spec = importlib.util.find_spec(name)
        except Exception:
            spec = None
        if spec is None or spec.origin is None:
            continue
        text = open(spec.origin, encoding = "utf-8").read()
        node = next(
            (
                n
                for n in ast.walk(ast.parse(text))
                if isinstance(n, ast.FunctionDef) and n.name == "concatenated_forward"
            ),
            None,
        )
        if node is None:
            continue
        module = importlib.import_module(name)
        cls = getattr(module, "ORPOTrainer" if kind == "orpo" else "CPOTrainer")
        return cls, ast.get_source_segment(text, node, padded = True)
    # Skip because this TRL ships no such trainer (CPO / ORPO left trl.trainer in 0.29).
    pytest.skip(reason = f"no {kind} trainer in the installed TRL")


def _compile(source, rows_per_chunk = None):
    ns = dict(vars(rl))
    ns["nn"] = torch.nn
    if rows_per_chunk is not None:
        ns["unsloth_chunked_logprobs"] = functools.partial(
            rl.unsloth_chunked_logprobs, rows_per_chunk = rows_per_chunk
        )
    exec(textwrap.dedent(source), ns)
    return ns["concatenated_forward"]


class _HiddenStatesLM(torch.nn.Module):
    """Wraps a causal LM the way Unsloth's forwards behave: with UNSLOTH_RETURN_HIDDEN_STATES=1 the final-norm hidden
    states come back as ``.logits``."""

    __UNSLOTH_SUPPORTS_RETURN_HIDDEN_STATES__ = True

    def __init__(
        self,
        lm,
        honour = True,
    ):
        super().__init__()
        self.lm = lm
        self.honour = honour
        self.config = lm.config

    def get_output_embeddings(self):
        return self.lm.get_output_embeddings()

    def forward(
        self,
        input_ids,
        attention_mask = None,
        use_cache = False,
    ):
        if self.honour and os.environ.get("UNSLOTH_RETURN_HIDDEN_STATES", "0") == "1":
            return types.SimpleNamespace(logits = self.lm.hidden(input_ids, attention_mask))
        return types.SimpleNamespace(logits = self.lm.logits(input_ids, attention_mask))


class _QwenLM(torch.nn.Module):
    def __init__(self, model):
        super().__init__()
        self.model = model
        self.config = model.config

    def get_output_embeddings(self):
        return self.model.get_output_embeddings()

    def hidden(self, input_ids, attention_mask):
        body = self.model.get_base_model().model
        return body(input_ids = input_ids, attention_mask = attention_mask).last_hidden_state

    def logits(self, input_ids, attention_mask):
        return self.model(
            input_ids = input_ids, attention_mask = attention_mask, use_cache = False
        ).logits


class _ToyLM(torch.nn.Module):
    """Head with a bias, a logit scale and a soft cap, applied in the model's own order (scale, then cap)."""

    def __init__(
        self,
        vocab = 61,
        hidden = 12,
    ):
        super().__init__()
        g = torch.Generator().manual_seed(1)
        self.embed = torch.nn.Parameter(torch.randn(vocab, hidden, generator = g))
        self.mix = torch.nn.Parameter(torch.randn(hidden, hidden, generator = g) / hidden**0.5)
        self.lm_head = torch.nn.Linear(hidden, vocab, bias = True)
        with torch.no_grad():
            self.lm_head.weight.copy_(torch.randn(vocab, hidden, generator = g))
            self.lm_head.bias.copy_(torch.randn(vocab, generator = g))
        self.config = types.SimpleNamespace(
            model_type = "cohere", logit_scale = 0.7, final_logit_softcapping = 3.0
        )

    def get_output_embeddings(self):
        return self.lm_head

    def hidden(self, input_ids, attention_mask):
        return torch.tanh(self.embed[input_ids] @ self.mix)

    def logits(self, input_ids, attention_mask):
        z = self.lm_head(self.hidden(input_ids, attention_mask)) * self.config.logit_scale
        cap = self.config.final_logit_softcapping
        return cap * torch.tanh(z / cap)


def _pin_stock_peft_lora(monkeypatch):
    """The reference is TRL on stock PEFT. An Unsloth load earlier in the worker swaps PEFT's LoRA forwards for the
    whole process, which moves these gradients past the tolerance below."""
    for name, module in list(sys.modules.items()):
        if not name.startswith("peft.tuners.lora") or module is None:
            continue
        for cls in list(vars(module).values()):
            forward = vars(cls).get("forward") if isinstance(cls, type) else None
            original = getattr(forward, "_unsloth_original_forward", None)
            if original is not None:
                monkeypatch.setattr(cls, "forward", original)


def _qwen_lora():
    transformers = pytest.importorskip("transformers")
    peft = pytest.importorskip("peft")
    try:
        model = transformers.AutoModelForCausalLM.from_pretrained(
            "trl-internal-testing/tiny-Qwen3ForCausalLM", torch_dtype = torch.float32
        )
    except Exception as error:
        # Skip because the offline checkpoint is missing, not a code failure.
        pytest.skip(reason = f"tiny Qwen3 checkpoint unavailable: {error}")
    torch.manual_seed(0)
    model = peft.get_peft_model(
        model, peft.LoraConfig(r = 4, lora_alpha = 8, target_modules = ["q_proj", "v_proj", "down_proj"])
    )
    # Non-zero B so LoRA A also gets a gradient.
    with torch.no_grad():
        for name, p in model.named_parameters():
            if "lora_B" in name:
                p.normal_(0, 0.05)
    return _QwenLM(model)


def _batch(vocab, seed = 0):
    """ORPO/CPO collator layout: right-padded rows, prompt labels -100, chosen and rejected of different widths."""
    g = torch.Generator().manual_seed(seed)
    B, prompt = 3, 4
    out = {}
    for side, lengths, width in (("chosen", (9, 7, 11), 11), ("rejected", (12, 6, 10), 12)):
        ids = torch.zeros(B, width, dtype = torch.long)
        mask = torch.zeros(B, width, dtype = torch.long)
        labels = torch.full((B, width), -100, dtype = torch.long)
        for i, n in enumerate(lengths):
            ids[i, :n] = torch.randint(1, vocab, (n,), generator = g)
            mask[i, :n] = 1
            labels[i, prompt:n] = ids[i, prompt:n]
        out[f"{side}_input_ids"], out[f"{side}_attention_mask"], out[f"{side}_labels"] = (
            ids,
            mask,
            labels,
        )
    return out


def _trainer(
    cls,
    kind,
    loss_type = "sigmoid",
    cpo_alpha = 1.0,
    alpha = 0.0,
):
    trainer = cls.__new__(cls)
    trainer.__dict__.update(
        is_encoder_decoder = False,
        label_pad_token_id = -100,
        padding_value = 0,
        pad_token_id = 0,
        aux_loss_enabled = False,
        aux_loss_coef = 0.0,
        beta = 0.2,
        loss_type = loss_type,
        cpo_alpha = cpo_alpha,
        alpha = alpha,
        simpo_gamma = 0.5,
        label_smoothing = 0.1 if loss_type == "sigmoid" else 0.0,
        accelerator = types.SimpleNamespace(
            device = torch.device("cpu"),
            unwrap_model = lambda m: m,
            gather_for_metrics = lambda x: x,
        ),
    )
    return trainer


def _run(trainer, forward, model, batch, params):
    trainer.concatenated_forward = types.MethodType(forward, trainer)
    loss, metrics = trainer.get_batch_loss_metrics(model, batch)
    grads = torch.autograd.grad(loss, params)
    return loss.detach(), metrics, grads


CASES = [
    ("orpo", {}),
    ("cpo", {"loss_type": "sigmoid"}),
    ("cpo", {"loss_type": "hinge"}),
    ("cpo", {"loss_type": "ipo"}),
    ("cpo", {"loss_type": "simpo"}),
    ("cpo", {"loss_type": "simpo", "cpo_alpha": 0.0, "alpha": 0.25}),  # AlphaPO
    ("cpo", {"loss_type": "sigmoid", "cpo_alpha": 0.0}),
]


@pytest.mark.parametrize("model_name", ["qwen3_lora", "toy_bias_scale_softcap"])
@pytest.mark.parametrize("kind,kwargs", CASES)
def test_matches_trl(kind, kwargs, model_name, monkeypatch):
    monkeypatch.delenv("UNSLOTH_RETURN_HIDDEN_STATES", raising = False)
    cls, source = _trl_class(kind)
    patched_source = getattr(rl, f"{kind}_trainer_concatenated_forward")(
        "concatenated_forward", source
    )
    assert (
        patched_source != source
    ), "rewrite did not recognise the installed TRL concatenated_forward"
    stock, patched = _compile(source), _compile(patched_source, rows_per_chunk = 5)

    if model_name == "qwen3_lora":
        _pin_stock_peft_lora(monkeypatch)
    lm = _qwen_lora() if model_name == "qwen3_lora" else _ToyLM()
    model = _HiddenStatesLM(lm)
    params = [p for p in model.parameters() if p.requires_grad]
    vocab = model.get_output_embeddings().weight.shape[0]
    batch = _batch(min(vocab, 1000))

    trainer = _trainer(cls, kind, **kwargs)
    want_loss, want_metrics, want_grads = _run(trainer, stock, model, batch, params)
    assert getattr(trainer, "_unsloth_pref_chunked_calls", 0) == 0
    got_loss, got_metrics, got_grads = _run(trainer, patched, model, batch, params)
    assert trainer._unsloth_pref_chunked_calls == 1, "chunked path did not run"
    assert "UNSLOTH_RETURN_HIDDEN_STATES" not in os.environ, "flag not restored"

    torch.testing.assert_close(got_loss, want_loss, rtol = 1e-5, atol = 1e-6)
    assert got_metrics.keys() == want_metrics.keys()
    for key in want_metrics:
        torch.testing.assert_close(
            torch.tensor(float(got_metrics[key])),
            torch.tensor(float(want_metrics[key])),
            # AlphaPO margins subtract two rewards of magnitude ~5: a few fp32 ulps.
            rtol = 1e-5,
            atol = 1e-5,
            msg = lambda m: f"{key}: {m}",
        )
    for g, w in zip(got_grads, want_grads):
        torch.testing.assert_close(g, w, rtol = 1e-4, atol = 1e-6)


def test_forward_ignoring_the_flag_runs_trl_math(monkeypatch):
    monkeypatch.setenv("UNSLOTH_RETURN_HIDDEN_STATES", "0")
    cls, source = _trl_class("orpo")
    stock = _compile(source)
    patched = _compile(rl.orpo_trainer_concatenated_forward("concatenated_forward", source))
    model = _HiddenStatesLM(_ToyLM(), honour = False)
    params = list(model.parameters())
    batch = _batch(61)
    trainer = _trainer(cls, "orpo")
    want = _run(trainer, stock, model, batch, params)
    got = _run(trainer, patched, model, batch, params)
    assert getattr(trainer, "_unsloth_pref_chunked_calls", 0) == 0
    assert os.environ["UNSLOTH_RETURN_HIDDEN_STATES"] == "0", "prior flag value not restored"
    assert torch.equal(got[0], want[0]) and got[1] == want[1]
    for g, w in zip(got[2], want[2]):
        assert torch.equal(g, w)


def test_labels_differing_from_input_ids_take_a_second_nll_pass(monkeypatch):
    """ORPO's NLL reads input_ids, the log-probs read labels; a row where they disagree must score both."""
    monkeypatch.delenv("UNSLOTH_RETURN_HIDDEN_STATES", raising = False)
    cls, source = _trl_class("orpo")
    stock = _compile(source)
    patched = _compile(rl.orpo_trainer_concatenated_forward("concatenated_forward", source))
    model = _HiddenStatesLM(_ToyLM())
    params = list(model.parameters())
    batch = _batch(61)
    batch["chosen_labels"][0, 6] = (batch["chosen_input_ids"][0, 6] + 1) % 61
    trainer = _trainer(cls, "orpo")
    want = _run(trainer, stock, model, batch, params)
    got = _run(trainer, patched, model, batch, params)
    assert trainer._unsloth_pref_chunked_calls == 1
    torch.testing.assert_close(got[0], want[0], rtol = 1e-5, atol = 1e-6)
    for key in want[1]:
        assert abs(got[1][key] - want[1][key]) <= 1e-5 * max(1.0, abs(want[1][key])), key
    for g, w in zip(got[2], want[2]):
        torch.testing.assert_close(g, w, rtol = 1e-4, atol = 1e-6)


def test_declines_without_kernel_or_on_unsupported_setups(monkeypatch):
    stock, source = _trl_class("orpo")
    cls = type(
        "Patched",
        (stock,),
        {
            "concatenated_forward": _compile(
                rl.orpo_trainer_concatenated_forward("concatenated_forward", source)
            )
        },
    )
    model = _HiddenStatesLM(_ToyLM())
    assert rl._unsloth_pref_chunked_head(_trainer(cls, "orpo"), model) is not None
    for attr, value in (
        ("is_encoder_decoder", True),
        ("aux_loss_enabled", True),
        ("label_pad_token_id", -1),
        ("is_deepspeed_enabled", True),
    ):
        trainer = _trainer(cls, "orpo")
        setattr(trainer, attr, value)
        assert rl._unsloth_pref_chunked_head(trainer, model) is None, attr
    # A subclass reading the full chosen / rejected logits, or scoring its own log-probs, keeps TRL's path.
    for hook in ("get_batch_loss_metrics", "get_batch_logps", "concatenated_forward"):
        sub = type("Sub", (cls,), {hook: lambda self, *a, **k: None})
        assert rl._unsloth_pref_chunked_head(_trainer(sub, "orpo"), model) is None, hook
    # A global module hook (register_module_forward_hook) fires for the head only when it is called.
    from torch.nn.modules import module as module_hooks

    handle = module_hooks.register_module_forward_hook(lambda m, i, o: None)
    try:
        assert rl._unsloth_pref_chunked_head(_trainer(cls, "orpo"), model) is None
    finally:
        handle.remove()
    assert rl._unsloth_pref_chunked_head(_trainer(cls, "orpo"), model) is not None
    monkeypatch.setenv("UNSLOTH_PREFERENCE_CHUNKED", "0")
    assert rl._unsloth_pref_chunked_head(_trainer(cls, "orpo"), model) is None
    monkeypatch.delenv("UNSLOTH_PREFERENCE_CHUNKED")
    monkeypatch.setattr(rl, "unsloth_chunked_logprobs", None)
    assert rl._unsloth_pref_chunked_head(_trainer(cls, "orpo"), model) is None


@pytest.mark.parametrize("kind", ["orpo", "cpo"])
def test_rewrite_is_idempotent_and_leaves_other_layouts(kind):
    _, source = _trl_class(kind)
    rewrite = getattr(rl, f"{kind}_trainer_concatenated_forward")
    once = rewrite("concatenated_forward", source)
    assert rewrite("concatenated_forward", once) == once
    assert rewrite("get_batch_loss_metrics", source) == source
    changed = source.replace("average_log_prob=", "average_log_prob=not ", 1)
    assert rewrite("concatenated_forward", changed) == changed


def test_generated_items_are_self_contained():
    for trainer in ("orpo_trainer", "cpo_trainer"):
        pre = "\n".join(rl.RL_PRE_ITEMS[trainer])
        for name in (
            "_unsloth_pref_chunked_head",
            "_unsloth_pref_chunked_forward",
            "_unsloth_pref_logit_transforms",
            "_unsloth_gkd_dense_head",
            "_unsloth_global_module_hooks",
            "_unsloth_grpo_returns_hidden_states",
            "_unsloth_grpo_hidden_states_signal",
            "_unsloth_pref_returns_hidden_states",
            "_unsloth_get_model_config",
            "_unsloth_text_configs",
        ):
            assert f"def {name}(" in pre, (trainer, name)
        assert (
            "from unsloth_zoo.fused_losses.chunked_logprob import unsloth_chunked_logprobs" in pre
        )


def test_square_head_needs_an_affirmative_hidden_state_signal():
    # vocab_size == hidden_size: the width cannot tell hidden states from logits. A forward with no marker ignored
    # UNSLOTH_RETURN_HIDDEN_STATES, so projecting its output again would score logits through the head twice.
    class Plain(torch.nn.Module):
        def forward(self, x):
            return x

    square, wide = torch.zeros(8, 8), torch.zeros(10, 8)
    out = torch.zeros(1, 2, 8)
    plain = Plain()
    assert rl._unsloth_pref_returns_hidden_states(plain, out, square) is False
    assert rl._unsloth_pref_returns_hidden_states(plain, out, wide) is True
    assert rl._unsloth_pref_returns_hidden_states(plain, torch.zeros(1, 2, 10), wide) is False
    marked = Plain()
    marked.__UNSLOTH_SUPPORTS_RETURN_HIDDEN_STATES__ = True
    assert rl._unsloth_pref_returns_hidden_states(marked, out, square) is True
    # GRPO keeps its leniency for a zoo too old to write the marker.
    assert rl._unsloth_grpo_returns_hidden_states(plain, out, square) is True


def test_every_generated_trainer_with_a_head_guard_defines_the_global_hook_check():
    # A trainer that copies a chunked-head guard into its source must copy the helper it calls.
    for trainer, items in rl.RL_PRE_ITEMS.items():
        pre = "\n".join(items)
        if any(
            f"def {guard}(" in pre
            for guard in (
                "_unsloth_dpo_chunked_head",
                "_unsloth_pref_chunked_head",
                "_unsloth_gkd_chunked_loss",
            )
        ):
            assert "def _unsloth_global_module_hooks(" in pre, trainer


_GLOBAL_HOOKS = {
    "forward": "register_module_forward_hook",
    "forward_pre": "register_module_forward_pre_hook",
    "full_backward": "register_module_full_backward_hook",
    "full_backward_pre": "register_module_full_backward_pre_hook",
}


@pytest.mark.parametrize("hook", list(_GLOBAL_HOOKS))
def test_global_module_hooks_see_the_head_through_trl_math(hook, monkeypatch):
    """Every torch global module hook must still fire on the head: the chunked path declines while one is registered,
    gives TRL's numbers, and engages again once it is removed."""
    from torch.nn.modules import module as module_hooks

    monkeypatch.delenv("UNSLOTH_RETURN_HIDDEN_STATES", raising = False)
    cls, source = _trl_class("orpo")
    stock = _compile(source)
    patched = _compile(rl.orpo_trainer_concatenated_forward("concatenated_forward", source))
    model = _HiddenStatesLM(_ToyLM())
    head = model.get_output_embeddings()
    params = list(model.parameters())
    batch = _batch(61)
    seen = []
    handle = getattr(module_hooks, _GLOBAL_HOOKS[hook])(
        lambda m, *a: seen.append(1) if m is head else None
    )
    try:
        want = _run(_trainer(cls, "orpo"), stock, model, batch, params)
        assert seen, "control: the hook fires on the head under TRL's own forward"
        seen.clear()
        trainer = _trainer(cls, "orpo")
        got = _run(trainer, patched, model, batch, params)
    finally:
        handle.remove()
    assert getattr(trainer, "_unsloth_pref_chunked_calls", 0) == 0
    assert seen, "the chunked path skipped a global module hook on the head"
    assert torch.equal(got[0], want[0])
    for g, w in zip(got[2], want[2]):
        assert torch.equal(g, w)
    trainer = _trainer(cls, "orpo")
    _run(trainer, patched, model, batch, params)
    assert (
        trainer._unsloth_pref_chunked_calls == 1
    ), "removing the hook must restore the chunked path"