Skip to content

armgit / Soup

Public Imported from GitHub
0

Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.

42.7 KB
Raw Blame
"""Native MLX DPO trainer: tokenization, batching, loss and the training loop.

The tokenization tests run everywhere with a character-level fake tokenizer.
The loss, log-prob and training-loop tests need ``mlx`` and skip without it.
"""

from __future__ import annotations

import json
import math
import re

import pytest


def _strip_ansi(text: str) -> str:
    """Rich styles numbers and wraps at the console width; compare plain words."""
    return " ".join(re.sub(r"\x1b\[[0-9;]*m", "", text).split())


# --------------------------------------------------------------------------
# A character-level tokenizer with a deterministic chat template
# --------------------------------------------------------------------------

BOS = 1
EOS = 3


class FakeTokenizer:
    """One token per character, BOS prepended, ``\\x03`` as EOS."""

    eos_token = "\x03"
    eos_token_id = EOS
    pad_token_id = 0

    def encode(self, text: str) -> list[int]:
        return [BOS] + [ord(c) for c in text]

    def _render(self, messages, add_generation_prompt: bool) -> str:
        out = "".join(f"<{m['role']}>{m['content']}\x03" for m in messages)
        if add_generation_prompt:
            out += "<assistant>"
        return out

    def apply_chat_template(
        self, messages, add_generation_prompt=False, tokenize=True, return_dict=False
    ):
        text = self._render(messages, add_generation_prompt)
        if not tokenize:
            return text
        ids = self.encode(text)
        return {"input_ids": ids} if return_dict else ids


def _decode(ids: list[int]) -> str:
    return "".join(chr(i) for i in ids if i != BOS)


# --------------------------------------------------------------------------
# build_preference_example
# --------------------------------------------------------------------------


class TestBuildPreferenceExample:
    def _build(self, row, max_length=1024, max_prompt_length=512, mode="keep_start"):
        from soup_cli.trainer.mlx_dpo import build_preference_example

        return build_preference_example(
            row,
            FakeTokenizer(),
            max_length=max_length,
            max_prompt_length=max_prompt_length,
            truncation_mode=mode,
        )

    def test_text_row_matches_trl_layout(self):
        """Text rows: prompt tokens, then prompt+completion+EOS sliced after them."""
        ex = self._build({"prompt": "Q:", "chosen": " yes", "rejected": " no"})
        assert ex.prompt_ids == [BOS] + [ord(c) for c in "Q:"]
        assert _decode(ex.chosen_ids) == " yes\x03"
        assert _decode(ex.rejected_ids) == " no\x03"

    def test_text_row_does_not_double_eos(self):
        ex = self._build({"prompt": "Q:", "chosen": " yes\x03", "rejected": " no"})
        assert _decode(ex.chosen_ids) == " yes\x03"

    def test_conversational_row_uses_chat_template(self):
        row = {
            "prompt": [{"role": "user", "content": "hi"}],
            "chosen": [{"role": "assistant", "content": "hello"}],
            "rejected": [{"role": "assistant", "content": "go away"}],
        }
        ex = self._build(row)
        assert _decode(ex.prompt_ids) == "<user>hi\x03<assistant>"
        # The completion is whatever follows the generation prompt.
        assert _decode(ex.chosen_ids) == "hello\x03"
        assert _decode(ex.rejected_ids) == "go away\x03"

    def test_implicit_prompt_in_full_conversations(self):
        """Soup's bundled DPO sample: a string prompt plus full chosen/rejected
        conversations that repeat the user turn. The shared leading turns are
        the prompt, not part of the supervised completion."""
        user = {"role": "user", "content": "hi"}
        row = {
            "prompt": "hi",
            "chosen": [user, {"role": "assistant", "content": "hello"}],
            "rejected": [user, {"role": "assistant", "content": "nope"}],
        }
        ex = self._build(row)
        assert _decode(ex.prompt_ids) == "<user>hi\x03<assistant>"
        assert _decode(ex.chosen_ids) == "hello\x03"
        assert _decode(ex.rejected_ids) == "nope\x03"

    def test_string_prompt_with_assistant_only_lists_becomes_user_turn(self):
        row = {
            "prompt": "hi",
            "chosen": [{"role": "assistant", "content": "hello"}],
            "rejected": [{"role": "assistant", "content": "nope"}],
        }
        ex = self._build(row)
        assert _decode(ex.prompt_ids) == "<user>hi\x03<assistant>"

    def test_prompt_list_repeated_inside_chosen_is_not_duplicated(self):
        user = {"role": "user", "content": "hi"}
        row = {
            "prompt": [user],
            "chosen": [user, {"role": "assistant", "content": "a"}],
            "rejected": [user, {"role": "assistant", "content": "b"}],
        }
        ex = self._build(row)
        assert _decode(ex.prompt_ids) == "<user>hi\x03<assistant>"
        assert _decode(ex.chosen_ids) == "a\x03"

    def test_mixed_string_and_list_completions_refused(self):
        from soup_cli.trainer.mlx_dpo import PreferenceRowError

        with pytest.raises(PreferenceRowError, match="chosen"):
            self._build(
                {
                    "prompt": "hi",
                    "chosen": "hello",
                    "rejected": [{"role": "assistant", "content": "x"}],
                }
            )

    def test_identical_conversations_flagged(self):
        """No turn differs, so there is no completion to prefer."""
        from soup_cli.trainer.mlx_dpo import IdenticalPairError

        user = {"role": "user", "content": "hi"}
        same = [user, {"role": "assistant", "content": "a"}]
        with pytest.raises(IdenticalPairError, match="identical"):
            self._build({"prompt": "hi", "chosen": same, "rejected": list(same)})

    def test_identical_text_completions_flagged(self):
        from soup_cli.trainer.mlx_dpo import IdenticalPairError

        with pytest.raises(IdenticalPairError):
            self._build({"prompt": "Q", "chosen": "a", "rejected": "a"})

    def test_template_that_is_not_prefix_stable_refused(self):
        """A completion that would be sliced at the wrong offset trains the
        wrong tokens; refuse instead of warning as TRL does."""
        from soup_cli.trainer.mlx_dpo import PreferenceRowError

        class Unstable(FakeTokenizer):
            def apply_chat_template(self, messages, add_generation_prompt=False, **kw):
                ids = super().apply_chat_template(messages, add_generation_prompt, **kw)
                if not add_generation_prompt:
                    ids = [ids[0], 999] + ids[1:]
                return ids

        from soup_cli.trainer.mlx_dpo import build_preference_example

        row = {
            "prompt": [{"role": "user", "content": "hi"}],
            "chosen": [{"role": "assistant", "content": "a"}],
            "rejected": [{"role": "assistant", "content": "b"}],
        }
        with pytest.raises(PreferenceRowError, match="prefix"):
            build_preference_example(
                row, Unstable(), max_length=64, max_prompt_length=32
            )

    def test_prompt_capped_keep_start(self):
        ex = self._build(
            {"prompt": "abcdefgh", "chosen": "x", "rejected": "y"},
            max_length=100,
            max_prompt_length=4,
        )
        assert ex.prompt_ids == [BOS, ord("a"), ord("b"), ord("c")]

    def test_prompt_capped_keep_end(self):
        ex = self._build(
            {"prompt": "abcdefgh", "chosen": "x", "rejected": "y"},
            max_length=100,
            max_prompt_length=3,
            mode="keep_end",
        )
        assert _decode(ex.prompt_ids) == "fgh"

    def test_completion_capped_to_remaining_length(self):
        ex = self._build(
            {"prompt": "ab", "chosen": "0123456789", "rejected": "xy"},
            max_length=6,
            max_prompt_length=3,
        )
        assert len(ex.prompt_ids) == 3
        assert _decode(ex.chosen_ids) == "012"
        assert _decode(ex.rejected_ids) == "xy\x03"

    def test_null_field_refused(self):
        from soup_cli.trainer.mlx_dpo import PreferenceRowError

        with pytest.raises(PreferenceRowError, match="rejected"):
            self._build({"prompt": "a", "chosen": "b", "rejected": None})


# --------------------------------------------------------------------------
# Batching
# --------------------------------------------------------------------------


def _example(p, c, r):
    from soup_cli.trainer.mlx_dpo import PreferenceExample

    return PreferenceExample(prompt_ids=p, chosen_ids=c, rejected_ids=r)


class TestBatchPlan:
    def test_plan_is_reproducible_for_a_seed(self):
        from soup_cli.trainer.mlx_dpo import plan_batches

        a = plan_batches(10, batch_size=2, epochs=3, seed=42)
        b = plan_batches(10, batch_size=2, epochs=3, seed=42)
        assert a == b

    def test_different_seeds_shuffle_differently(self):
        from soup_cli.trainer.mlx_dpo import plan_batches

        assert plan_batches(50, 2, 1, seed=1) != plan_batches(50, 2, 1, seed=2)

    def test_each_epoch_covers_every_full_batch_once(self):
        from soup_cli.trainer.mlx_dpo import plan_batches

        plan = plan_batches(9, batch_size=2, epochs=2, seed=0)
        assert len(plan) == 8  # 4 full batches per epoch, the odd row dropped
        first_epoch = sorted(i for batch in plan[:4] for i in batch)
        assert len(set(first_epoch)) == 8

    def test_fewer_rows_than_batch_size_refused(self):
        from soup_cli.trainer.mlx_dpo import plan_batches

        with pytest.raises(ValueError, match="batch_size"):
            plan_batches(1, batch_size=2, epochs=1, seed=0)


class TestCollate:
    def test_chosen_then_rejected_with_completion_mask(self):
        from soup_cli.trainer.mlx_dpo import collate_pairs

        tokens, mask = collate_pairs(
            [_example([1, 2], [5, 6, 7], [8]), _example([1], [9], [4, 4])]
        )
        assert tokens == [
            [1, 2, 5, 6, 7],
            [1, 9, 0, 0, 0],
            [1, 2, 8, 0, 0],
            [1, 4, 4, 0, 0],
        ]
        assert mask == [
            [0, 0, 1, 1, 1],
            [0, 1, 0, 0, 0],
            [0, 0, 1, 0, 0],
            [0, 1, 1, 0, 0],
        ]


# --------------------------------------------------------------------------
# Loss + log-probs (need mlx)
# --------------------------------------------------------------------------


def _sigmoid(x):
    return 1.0 / (1.0 + math.exp(-x))


class TestDpoLoss:
    def test_matches_closed_form(self):
        mx = pytest.importorskip("mlx.core")
        from soup_cli.trainer.mlx_dpo import dpo_loss

        pc, pr = mx.array([-1.0, -2.0]), mx.array([-3.0, -1.0])
        rc, rr = mx.array([-1.5, -2.0]), mx.array([-2.5, -1.5])
        beta = 0.1
        loss, metrics = dpo_loss(pc, pr, rc, rr, beta)

        expected = []
        for i in range(2):
            z = beta * ((pc[i] - rc[i]).item() - (pr[i] - rr[i]).item())
            expected.append(-math.log(_sigmoid(z)))
        assert loss.item() == pytest.approx(sum(expected) / 2, rel=1e-5)
        # chosen reward 0.05 vs rejected -0.05 for row 0; 0.0 vs 0.05 for row 1
        assert metrics["reward_accuracy"].item() == pytest.approx(0.5)
        assert metrics["reward_margin"].item() == pytest.approx(
            ((0.05 - -0.05) + (0.0 - 0.05)) / 2, rel=1e-5
        )

    def test_equal_policy_and_reference_is_log2(self):
        mx = pytest.importorskip("mlx.core")
        from soup_cli.trainer.mlx_dpo import dpo_loss

        x = mx.array([-4.0, -7.0, -1.0])
        loss, _ = dpo_loss(x, x, x, x, 0.5)
        assert loss.item() == pytest.approx(math.log(2), rel=1e-6)

    def test_stable_for_large_margins(self):
        mx = pytest.importorskip("mlx.core")
        from soup_cli.trainer.mlx_dpo import dpo_loss

        loss, _ = dpo_loss(
            mx.array([0.0]), mx.array([-1e5]), mx.array([0.0]), mx.array([0.0]), 1.0
        )
        assert math.isfinite(loss.item()) and loss.item() == pytest.approx(0.0, abs=1e-6)
        loss, _ = dpo_loss(
            mx.array([-1e5]), mx.array([0.0]), mx.array([0.0]), mx.array([0.0]), 1.0
        )
        assert loss.item() == pytest.approx(1e5, rel=1e-4)


class TestSequenceLogps:
    def test_sums_only_masked_target_tokens(self):
        mx = pytest.importorskip("mlx.core")
        nn = pytest.importorskip("mlx.nn")
        from soup_cli.trainer.mlx_dpo import sequence_logps

        vocab = 5

        class Uniform(nn.Module):
            def __call__(self, x):
                return mx.zeros((*x.shape, vocab))

        tokens = mx.array([[1, 2, 3, 4], [1, 2, 0, 0]])
        mask = mx.array([[0, 0, 1, 1], [0, 1, 0, 0]])
        out = sequence_logps(Uniform(), tokens, mask)
        # Uniform logits: every target token has log-prob -log(5).
        assert out.tolist() == pytest.approx([-2 * math.log(vocab), -math.log(vocab)])


class TestChunkedLogps:
    """The memory path: backbone once, then the vocab head in checkpointed
    chunks starting at the first completion token. It must equal the plain
    full-model path, value and gradient."""

    def _batch(self):
        mx = pytest.importorskip("mlx.core")
        tokens = mx.array([[5, 6, 7, 8, 9, 10, 11], [5, 6, 12, 13, 0, 0, 0]])
        mask = mx.array([[0, 0, 0, 1, 1, 1, 1], [0, 0, 1, 1, 0, 0, 0]])
        return tokens, mask

    @pytest.mark.parametrize("tied", [True, False])
    def test_matches_full_model(self, tied):
        pytest.importorskip("mlx.core")
        from soup_cli.trainer.mlx_dpo import resolve_head_split, sequence_logps

        model = _tiny_llama(tie=tied)
        model.freeze()
        tokens, mask = self._batch()
        split = resolve_head_split(model)
        assert split is not None
        full = sequence_logps(model, tokens, mask)
        chunked = sequence_logps(model, tokens, mask, head_split=split, chunk_size=2)
        assert chunked.tolist() == pytest.approx(full.tolist(), rel=1e-5, abs=1e-5)

    def test_gradients_match_full_model(self):
        mx = pytest.importorskip("mlx.core")
        nn = pytest.importorskip("mlx.nn")
        from mlx.utils import tree_flatten

        from soup_cli.trainer.mlx_dpo import resolve_head_split, sequence_logps

        model = _tiny_llama()
        # As in LoRA training: the vocab head (here the tied embedding) is
        # frozen, everything inside the backbone layers trains.
        model.freeze()
        model.model.layers[0].unfreeze()
        tokens, mask = self._batch()
        split = resolve_head_split(model)
        assert split is not None

        def loss(m, **kw):
            return sequence_logps(m, tokens, mask, **kw).sum()

        _, g_full = nn.value_and_grad(model, loss)(model)
        _, g_chunk = nn.value_and_grad(
            model, lambda m: loss(m, head_split=split, chunk_size=3)
        )(model)
        for (name, a), (_, b) in zip(tree_flatten(g_full), tree_flatten(g_chunk)):
            assert mx.allclose(a, b, rtol=1e-4, atol=1e-5).item(), name

    @pytest.mark.parametrize("tied", [True, False])
    def test_trainable_head_has_no_split(self, tied):
        """The checkpointed head sees its weights as constants, so a head that
        trains would lose its gradient: refuse the split, use the full path."""
        pytest.importorskip("mlx.core")
        from soup_cli.trainer.mlx_dpo import resolve_head_split

        model = _tiny_llama(tie=tied)  # nothing frozen
        assert resolve_head_split(model) is None

    def test_unknown_architecture_has_no_split(self):
        nn = pytest.importorskip("mlx.nn")
        from soup_cli.trainer.mlx_dpo import resolve_head_split

        class Opaque(nn.Module):
            def __call__(self, x):
                return x

        assert resolve_head_split(Opaque()) is None


# --------------------------------------------------------------------------
# Config gate
# --------------------------------------------------------------------------

_MLX_DPO_YAML = """
base: mlx-community/Qwen2.5-0.5B-Instruct-4bit
task: dpo
backend: mlx
data:
  train: ./x.jsonl
  format: dpo
training:
  epochs: 1
  lr: 1e-6
output: ./output
"""


class TestConfig:
    def test_mlx_dpo_config_loads(self):
        from soup_cli.config.loader import load_config_from_string

        cfg = load_config_from_string(_MLX_DPO_YAML)
        assert cfg.task == "dpo" and cfg.backend == "mlx"

    def test_add_new_tokens_refused_on_mlx_dpo(self):
        """The vocab-expansion validator used to wave every non-SFT MLX task
        through, trusting the task gate to refuse it. DPO passes that gate now,
        so the field must be refused for MLX's own reason, not dropped."""
        from soup_cli.config.loader import load_config_from_string

        yaml_str = _MLX_DPO_YAML.replace(
            "  format: dpo\n", "  format: dpo\n  add_new_tokens: ['<x>']\n"
        )
        with pytest.raises(Exception, match="not applied on backend='mlx'"):
            load_config_from_string(yaml_str)

    def test_doctor_reports_mlx_dpo_gaps_but_not_the_seed(self):
        """`soup doctor --config` reads backend_support; the seed is honoured
        on MLX DPO, so only the genuinely ignored setting is reported."""
        from soup_cli.config.backend_support import check_config
        from soup_cli.config.loader import load_config_from_string

        cfg = load_config_from_string(
            _MLX_DPO_YAML.replace("  lr: 1e-6\n", "  lr: 1e-6\n  seed: 3\n  use_galore: true\n")
        )
        fields = {entry.field for entry in check_config(cfg)}
        assert fields == {"training.use_galore"}

    # orpo, simpo and kto have their own MLX trainers since #2004 / #1999 / #2000
    # (tests/test_mlx_orpo.py, tests/test_mlx_simpo.py, tests/test_mlx_kto.py).
    @pytest.mark.parametrize("task", ["ipo", "ppo"])
    def test_other_preference_tasks_still_refused(self, task):
        from soup_cli.config.loader import load_config_from_string

        yaml_str = _MLX_DPO_YAML.replace("task: dpo", f"task: {task}")
        with pytest.raises(Exception, match="MLX backend"):
            load_config_from_string(yaml_str)

    def test_the_preference_alias_is_refused_naming_task_dpo(self):
        from soup_cli.config.loader import load_config_from_string

        alias = (_MLX_DPO_YAML.replace("task: dpo", "task: preference")
                 .replace("  lr: 1e-6\n", "  lr: 1e-6\n  preference_loss: dpo\n"))
        with pytest.raises(Exception, match="task='preference' has no trainer") as info:
            load_config_from_string(alias)
        assert "Use task: dpo for the native MLX trainer." in str(info.value)


# --------------------------------------------------------------------------
# End-to-end on a tiny random model (needs mlx + mlx_lm)
# --------------------------------------------------------------------------


def _tiny_llama(tie: bool = True):
    pytest.importorskip("mlx.core")
    pytest.importorskip("mlx_lm")
    from mlx_lm.models import llama

    args = llama.ModelArgs(
        model_type="llama",
        hidden_size=32,
        num_hidden_layers=2,
        intermediate_size=64,
        num_attention_heads=4,
        num_key_value_heads=2,
        rms_norm_eps=1e-5,
        vocab_size=200,
        tie_word_embeddings=tie,
    )
    return llama.Model(args)


def _wrapper(tmp_path, monkeypatch, **training):
    from soup_cli.config.loader import load_config_from_string
    from soup_cli.trainer.mlx_dpo import MLXDPOTrainerWrapper

    tmp_path.mkdir(parents=True, exist_ok=True)
    monkeypatch.chdir(tmp_path)
    extra = "".join(f"  {k}: {v}\n" for k, v in training.items())
    cfg = load_config_from_string(
        _MLX_DPO_YAML.replace("  lr: 1e-6\n", "  lr: 5e-3\n" + extra)
    )
    wrapper = MLXDPOTrainerWrapper(cfg)
    wrapper.model = _tiny_llama()
    wrapper.tokenizer = FakeTokenizer()
    rows = [
        {"prompt": f"q{i}:", "chosen": " good", "rejected": " bad"} for i in range(8)
    ]
    wrapper._dataset = {"train": rows, "val": rows[:2]}
    return wrapper


class TestTrainLoop:
    def test_tied_pairs_are_skipped_with_one_warning(self, tmp_path, monkeypatch, capsys):
        """UltraFeedback-style ties carry zero DPO gradient: skip, do not abort."""
        wrapper = _wrapper(tmp_path, monkeypatch, batch_size=2, seed=1)
        tie = {"prompt": "q:", "chosen": " same", "rejected": " same"}
        wrapper._dataset["train"] = wrapper._dataset["train"] + [tie, dict(tie)]
        result = wrapper.train()
        assert result["skipped_identical_pairs"] == 2
        out = _strip_ansi(capsys.readouterr().out)
        assert "skipped 2 dpo row(s) with identical chosen and rejected" in out

    def test_preference_is_learned_and_artifacts_written(self, tmp_path, monkeypatch):
        wrapper = _wrapper(
            tmp_path,
            monkeypatch,
            epochs=4,
            batch_size=2,
            gradient_accumulation_steps=1,
            warmup_ratio=0.0,
            seed=7,
        )
        result = wrapper.train()

        assert result["total_steps"] == 16
        assert result["final_loss"] < math.log(2)  # learned to prefer "good"
        assert result["val_reward_accuracy"] == pytest.approx(1.0)
        out = tmp_path / "output"
        assert (out / "adapters.safetensors").exists()
        meta = json.loads((out / "adapter_config.json").read_text())
        assert meta["fine_tune_type"] == "lora"
        assert meta["task"] == "dpo"
        assert meta["dpo_beta"] == pytest.approx(0.1)
        assert meta["seed"] == 7
        assert "lora_parameters" in meta

    def test_same_seed_reproduces_losses(self, tmp_path, monkeypatch):
        first = _wrapper(tmp_path / "a", monkeypatch, seed=3, batch_size=2)
        losses_a = first.train()["losses"]
        second = _wrapper(tmp_path / "b", monkeypatch, seed=3, batch_size=2)
        losses_b = second.train()["losses"]
        assert losses_a == pytest.approx(losses_b, rel=1e-5)

    def test_reference_logps_are_the_untrained_policy(self, tmp_path, monkeypatch):
        """LoRA B starts at zero, so the reference precomputed before the first
        update equals the base model; the first-step loss is exactly log 2."""
        wrapper = _wrapper(tmp_path, monkeypatch, batch_size=2, seed=0)
        result = wrapper.train()
        assert result["losses"][0] == pytest.approx(math.log(2), rel=1e-4)


# --------------------------------------------------------------------------
# Review of #1846: the preference-row contract is the loader's (#1866)
# --------------------------------------------------------------------------

_U = {"role": "user", "content": "hi"}
_A = {"role": "assistant", "content": "a"}
_B = {"role": "assistant", "content": "b"}


def _build_row(row, tokenizer=None, max_length=1024, max_prompt_length=512):
    from soup_cli.trainer.mlx_dpo import build_preference_example

    return build_preference_example(
        row, tokenizer or FakeTokenizer(),
        max_length=max_length, max_prompt_length=max_prompt_length,
    )


class TestTheLoaderLayoutsAreAccepted:
    @pytest.mark.parametrize("name", ["dpo_sample.jsonl", "chat_preferences.jsonl"])
    def test_every_bundled_preference_row_tokenizes(self, name):
        """Through the real loader converter, as `load_dataset` hands rows over."""
        from pathlib import Path

        from soup_cli.data.formats import format_to_messages

        path = Path(__file__).resolve().parents[1] / "examples" / "data" / name
        raws = [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines()
                if line.strip()]
        assert raws
        for raw in raws:
            row = format_to_messages(raw, "dpo")
            assert row is not None
            ex = _build_row(row, max_length=4096, max_prompt_length=2048)
            user = raw["chosen"][0]["content"]
            assert _decode(ex.prompt_ids) == f"<user>{user}\x03<assistant>"
            assert _decode(ex.chosen_ids) == raw["chosen"][-1]["content"] + "\x03"
            assert _decode(ex.rejected_ids) == raw["rejected"][-1]["content"] + "\x03"

    def test_message_list_prompt_with_string_answers(self):
        """Review item 1: each string answer is one assistant turn."""
        ex = _build_row({"prompt": [_U], "chosen": "a", "rejected": "b"})
        assert _decode(ex.prompt_ids) == "<user>hi\x03<assistant>"
        assert _decode(ex.chosen_ids) == "a\x03"
        assert _decode(ex.rejected_ids) == "b\x03"

    def test_one_message_object_is_read_as_a_list_of_one(self):
        """Review item 2: a dict prompt is a message, never `str(dict)`."""
        ex = _build_row({"prompt": _U, "chosen": [_A], "rejected": [_B]})
        assert _decode(ex.prompt_ids) == "<user>hi\x03<assistant>"
        assert "role" not in _decode(ex.prompt_ids)

    @pytest.mark.parametrize("prompt", [7, True, 1.5])
    def test_a_non_text_prompt_is_refused_by_name(self, prompt):
        from soup_cli.trainer.mlx_dpo import PreferenceRowError

        with pytest.raises(PreferenceRowError, match="dpo.prompt"):
            _build_row({"prompt": prompt, "chosen": [_A], "rejected": [_B]})

    def test_a_conversation_that_continues_past_the_other(self):
        """Review item 3: the shorter side's last assistant turn is its answer."""
        u2 = {"role": "user", "content": "more"}
        ex = _build_row({"prompt": "hi", "chosen": [_U, _A], "rejected": [_U, _A, u2, _B]})
        assert _decode(ex.prompt_ids) == "<user>hi\x03<assistant>"
        assert _decode(ex.chosen_ids) == "a\x03"
        assert _decode(ex.rejected_ids).startswith("a\x03<user>more")

    def test_a_side_with_no_answer_is_a_data_error_not_a_tokenizer_one(self):
        from soup_cli.trainer.mlx_dpo import PreferenceRowError, PrefixUnstableError

        with pytest.raises(PreferenceRowError, match="no chosen answer") as excinfo:
            _build_row({"prompt": "hi", "chosen": [_U], "rejected": [_U, _B]})
        assert not isinstance(excinfo.value, PrefixUnstableError)

    def test_no_prompt_uses_the_shared_turns(self):
        """Review item 4: rows handed straight to setup() read like the loader's."""
        ex = _build_row({"chosen": [_U, _A], "rejected": [_U, _B]})
        assert _decode(ex.prompt_ids) == "<user>hi\x03<assistant>"

    def test_messages_column_is_the_prompt(self):
        ex = _build_row({"messages": [_U], "chosen": [_A], "rejected": [_B]})
        assert _decode(ex.prompt_ids) == "<user>hi\x03<assistant>"


class _Unstable(FakeTokenizer):
    def apply_chat_template(self, messages, add_generation_prompt=False, **kw):
        ids = super().apply_chat_template(messages, add_generation_prompt, **kw)
        if not add_generation_prompt and messages[0]["content"] == "bad":
            ids = [ids[0], 999] + ids[1:]
        return ids


def _bare_wrapper(**training):
    """A wrapper with a tokenizer and no model: enough for `_examples`."""
    from soup_cli.config.loader import load_config_from_string
    from soup_cli.trainer.mlx_dpo import MLXDPOTrainerWrapper

    extra = "".join(f"  {k}: {v}\n" for k, v in training.items())
    yaml_str = _MLX_DPO_YAML.replace("  lr: 1e-6\n", "  lr: 1e-6\n" + extra)
    return MLXDPOTrainerWrapper(load_config_from_string(yaml_str))


class TestRowsTheTemplateCannotSlice:
    def test_skipped_with_a_count_and_the_first_offender(self, capsys):
        wrapper = _bare_wrapper()
        wrapper.tokenizer = _Unstable()
        bad = {"prompt": [{"role": "user", "content": "bad"}], "chosen": [_A], "rejected": [_B]}
        good = {"prompt": [_U], "chosen": [_A], "rejected": [_B]}
        examples, ties = wrapper._examples([good, bad, good, bad])
        assert (len(examples), ties) == (2, 0)
        out = _strip_ansi(capsys.readouterr().out)
        assert "skipped 2 train row(s)" in out and "first: row 1" in out

    def test_a_malformed_row_still_stops_with_its_index(self):
        from soup_cli.trainer.mlx_dpo import PreferenceRowError

        wrapper = _bare_wrapper()
        wrapper.tokenizer = FakeTokenizer()
        good = {"prompt": "q", "chosen": "a", "rejected": "b"}
        with pytest.raises(PreferenceRowError, match="dpo train row 1"):
            wrapper._examples([good, {"prompt": "q", "chosen": "a", "rejected": [_B]}])


class TestPromptCapIsWired:
    def test_examples_cap_the_prompt_at_half_max_length(self):
        wrapper = _bare_wrapper()
        wrapper.config.data.max_length = 40
        wrapper.tokenizer = FakeTokenizer()
        long_prompt = "x" * 100
        (example,), _ = wrapper._examples([{"prompt": long_prompt, "chosen": "a", "rejected": "b"}])
        assert len(example.prompt_ids) == 20
        assert len(example.prompt_ids) + len(example.chosen_ids) <= 40


class TestThePlanHooksKeepDpoAndOrpoBehaviour:
    """Pins the 5ce31005 refactor (no MLX): DPO and ORPO still plan seeded,
    shuffled, whole batches and run ``auto`` at 1; only KTO overrides the hooks."""

    def _wrapper(self, cls, yaml_text):
        from soup_cli.config.loader import load_config_from_string

        return cls(load_config_from_string(yaml_text))

    def _both(self):
        from soup_cli.trainer.mlx_dpo import MLXDPOTrainerWrapper
        from soup_cli.trainer.mlx_orpo import MLXORPOTrainerWrapper
        from tests.test_mlx_orpo import _ORPO_YAML

        return [self._wrapper(MLXDPOTrainerWrapper, _MLX_DPO_YAML),
                self._wrapper(MLXORPOTrainerWrapper, _ORPO_YAML)]

    def test_the_plan_is_seeded_shuffled_and_drops_the_partial_batch(self):
        for wrapper in self._both():
            plan = wrapper._plan_batches(7, 2, 2, 3)
            assert plan == wrapper._plan_batches(7, 2, 2, 3)
            assert plan != wrapper._plan_batches(7, 2, 2, 4)
            assert len(plan) == 6 and all(len(b) == 2 for b in plan)
            for epoch in (plan[:3], plan[3:]):  # 6 distinct rows; the 7th never trains
                rows = [i for b in epoch for i in b]
                assert len(set(rows)) == 6 and set(rows) <= set(range(7))
            sequential = [[0, 1], [2, 3], [4, 5]] * 2
            assert any(wrapper._plan_batches(7, 2, 2, s) != sequential for s in range(3))

    def test_auto_batch_size_is_one(self):
        for wrapper in self._both():
            wrapper.config.training.batch_size = "auto"
            assert wrapper._resolve_batch_size() == 1
            wrapper.config.training.batch_size = 3
            assert wrapper._resolve_batch_size() == 3

    def test_only_kto_overrides_the_hooks(self):
        from soup_cli.trainer.mlx_dpo import MLXDPOTrainerWrapper
        from soup_cli.trainer.mlx_kto import MLXKTOTrainerWrapper
        from soup_cli.trainer.mlx_orpo import MLXORPOTrainerWrapper

        for hook in ("_plan_batches", "_resolve_batch_size", "_check_train_rows",
                     "_on_plan_cut"):
            base = getattr(MLXDPOTrainerWrapper, hook)
            assert getattr(MLXORPOTrainerWrapper, hook) is base, hook
            assert getattr(MLXKTOTrainerWrapper, hook) is not base, hook

    def test_fewer_pairs_than_a_batch_is_still_refused(self):
        wrapper = self._both()[0]
        with pytest.raises(ValueError, match=r"3 training pair\(s\) remain, fewer than "
                                             r"training.batch_size \(4\), after skipping 1"):
            wrapper._check_train_rows(3, 4, 1)


class TestConfigRefusals:
    def _load(self, extra_top="", extra_training=""):
        from soup_cli.config.loader import load_config_from_string

        yaml_str = _MLX_DPO_YAML.replace("backend: mlx\n", "backend: mlx\n" + extra_top)
        yaml_str = yaml_str.replace("  lr: 1e-6\n", "  lr: 1e-6\n" + extra_training)
        return load_config_from_string(yaml_str)

    def test_lora_rank_zero_is_refused(self):
        with pytest.raises(Exception, match="lora.r must be at least 1"):
            self._load(extra_training="  lora: {r: 0}\n")

    @pytest.mark.parametrize("modality", ["vision", "audio"])
    def test_non_text_modality_is_refused(self, modality):
        with pytest.raises(Exception, match=f"modality='{modality}'"):
            self._load(extra_top=f"modality: {modality}\n")

    def test_loraplus_is_still_refused_on_mlx_dpo(self):
        with pytest.raises(Exception, match="loraplus_lr_ratio"):
            self._load(extra_training="  loraplus_lr_ratio: 16\n")

    def test_the_ignored_settings_are_printed(self, monkeypatch):
        from io import StringIO

        from rich.console import Console

        from soup_cli.trainer import mlx_dpo

        buffer = StringIO()
        monkeypatch.setattr(mlx_dpo, "console", Console(file=buffer, width=400))
        cfg = self._load(extra_training="  use_galore: true\n")
        mlx_dpo.MLXDPOTrainerWrapper(cfg)._check_unsupported()
        assert "MLX backend ignores: GaLore" in _strip_ansi(buffer.getvalue())


# --------------------------------------------------------------------------
# sequence_logps scores the NEXT token
# --------------------------------------------------------------------------


class _Oracle:
    """Logits that put all mass on ``tokens[t + 1]`` at position ``t``
    (``shift=1``) or on ``tokens[t]`` (``shift=0``: copies its input)."""

    def __new__(cls, tokens, vocab, shift):
        nn = pytest.importorskip("mlx.nn")
        mx = pytest.importorskip("mlx.core")

        class Backbone(nn.Module):
            def __call__(self, x):
                width = x.shape[1]
                targets = tokens[:, shift : shift + width]
                return mx.eye(vocab)[targets]

        class Model(nn.Module):
            def __init__(self):
                super().__init__()
                self.model = Backbone()
                self.lm_head = nn.Linear(vocab, vocab, bias=False)
                self.lm_head.weight = mx.eye(vocab) * 50.0
                self.args = type("Args", (), {"tie_word_embeddings": False})()

            def __call__(self, x):
                return self.lm_head(self.model(x))

        model = Model()
        model.freeze()
        return model


class TestTargetAlignment:
    @pytest.mark.parametrize("chunked", [False, True])
    def test_the_real_continuation_scores_near_zero(self, chunked):
        mx = pytest.importorskip("mlx.core")
        from soup_cli.trainer.mlx_dpo import resolve_head_split, sequence_logps

        tokens = mx.array([[1, 5, 9, 3, 7, 2]])
        mask = mx.array([[0, 0, 1, 1, 1, 1]])
        right = _Oracle(tokens, 16, shift=1)
        copier = _Oracle(tokens, 16, shift=0)
        kw = {}
        if chunked:
            assert resolve_head_split(right) is not None
            kw = {"head_split": resolve_head_split(right), "chunk_size": 2}
        assert sequence_logps(right, tokens, mask, **kw).item() == pytest.approx(0.0, abs=1e-3)
        kw_copy = {"head_split": resolve_head_split(copier), "chunk_size": 2} if chunked else {}
        assert sequence_logps(copier, tokens, mask, **kw_copy).item() < -100


# --------------------------------------------------------------------------
# Train-loop guarantees the docs state
# --------------------------------------------------------------------------


def _seeded_wrapper(tmp_path, monkeypatch, model_seed=0, **training):
    mx = pytest.importorskip("mlx.core")
    wrapper = _wrapper(tmp_path, monkeypatch, **training)
    mx.random.seed(model_seed)
    wrapper.model = _tiny_llama()
    return wrapper


class TestResume:
    def test_reference_is_taken_before_the_checkpoint_loads(self, tmp_path, monkeypatch):
        from soup_cli.trainer import mlx_dpo

        first = _seeded_wrapper(tmp_path / "a", monkeypatch, batch_size=2, epochs=3, seed=1)
        first.train()
        checkpoint = tmp_path / "a" / "output" / "adapters.safetensors"
        assert checkpoint.exists()

        order = []
        real_ref = mlx_dpo.MLXDPOTrainerWrapper._reference_logps
        real_load = mlx_dpo.MLXDPOTrainerWrapper._load_checkpoint_weights
        monkeypatch.setattr(mlx_dpo.MLXDPOTrainerWrapper, "_reference_logps",
                            lambda self, *a: order.append("reference") or real_ref(self, *a))
        monkeypatch.setattr(mlx_dpo.MLXDPOTrainerWrapper, "_load_checkpoint_weights",
                            lambda self, *a: order.append("load") or real_load(self, *a))
        resumed = _seeded_wrapper(tmp_path / "b", monkeypatch, batch_size=2, seed=1)
        result = resumed.train(resume_from_checkpoint=str(checkpoint))

        assert order[-1] == "load" and "reference" in order[:-1]
        # The policy starts trained, the reference is the base model, so the
        # first loss is already below log 2 (it is exactly log 2 from scratch).
        assert result["losses"][0] < math.log(2) - 1e-3


class TestGradientAccumulation:
    def test_two_micro_batches_of_one_equal_one_batch_of_two(self, tmp_path, monkeypatch):
        mx = pytest.importorskip("mlx.core")
        from mlx.utils import tree_flatten

        from soup_cli.trainer import mlx_dpo

        def run(path, batch_size, accum, plan):
            monkeypatch.setattr(mlx_dpo, "plan_batches", lambda *a, **k: [list(b) for b in plan])
            wrapper = _seeded_wrapper(
                path, monkeypatch, batch_size=batch_size, gradient_accumulation_steps=accum,
                optimizer="sgd", warmup_ratio=0.0, scheduler="constant", seed=4,
            )
            wrapper.config.training.lora.dropout = 0.0
            result = wrapper.train()
            return result, dict(tree_flatten(wrapper.model.trainable_parameters()))

        one, w_one = run(tmp_path / "a", 2, 1, [[0, 1]])
        two, w_two = run(tmp_path / "b", 1, 2, [[0], [1]])
        assert len(one["losses"]) == len(two["losses"]) == 1
        for name in w_one:
            assert mx.allclose(w_one[name], w_two[name], atol=1e-6).item(), name

    def test_updates_are_counted_per_accumulation_group(self, tmp_path, monkeypatch):
        wrapper = _seeded_wrapper(tmp_path, monkeypatch, batch_size=2,
                                  gradient_accumulation_steps=2, seed=1)
        result = wrapper.train()
        assert result["total_steps"] == 4  # 8 rows / batch 2, one epoch
        assert len(result["losses"]) == 2  # one loss per optimizer update


class TestDataSeed:
    def test_data_seed_owns_the_batch_order(self, tmp_path, monkeypatch):
        from soup_cli.trainer import mlx_dpo

        plans = []
        real = mlx_dpo.plan_batches

        def recording(*args, **kwargs):
            plan = real(*args, **kwargs)
            plans.append(plan)
            return plan

        monkeypatch.setattr(mlx_dpo, "plan_batches", recording)
        for path, training in (("a", {"seed": 1, "data_seed": 2}),
                               ("b", {"seed": 3, "data_seed": 2}),
                               ("c", {"seed": 3})):
            _seeded_wrapper(tmp_path / path, monkeypatch, batch_size=2, epochs=2,
                            **training).train()
        assert plans[0] == plans[1]
        assert plans[1] != plans[2]


class TestValidation:
    def test_one_val_row_with_batch_two_is_evaluated(self, tmp_path, monkeypatch):
        """The #1851 shape: a partial last chunk, not a crash."""
        wrapper = _seeded_wrapper(tmp_path, monkeypatch, batch_size=2, seed=1)
        wrapper._dataset["val"] = wrapper._dataset["val"][:1]
        result = wrapper.train()
        assert "val_loss" in result and math.isfinite(result["val_loss"])

    def test_val_loss_is_the_mean_over_rows_not_over_chunks(self, tmp_path, monkeypatch):
        pytest.importorskip("mlx.core")
        from soup_cli.trainer import mlx_dpo

        refs = []
        real = mlx_dpo.MLXDPOTrainerWrapper._reference_logps
        monkeypatch.setattr(mlx_dpo.MLXDPOTrainerWrapper, "_reference_logps",
                            lambda self, ex, bs: refs.append(real(self, ex, bs)) or refs[-1])
        wrapper = _seeded_wrapper(tmp_path, monkeypatch, batch_size=2, epochs=3, seed=2)
        # Three rows, one reversed: chunks of 2 + 1 with different losses.
        wrapper._dataset["val"] = [
            {"prompt": "v0:", "chosen": " good", "rejected": " bad"},
            {"prompt": "v1:", "chosen": " good", "rejected": " bad"},
            {"prompt": "v2:", "chosen": " bad", "rejected": " good"},
        ]
        result = wrapper.train()

        examples, _ = wrapper._examples(wrapper._dataset["val"], "val")
        ref_c, ref_r = refs[1]
        wrapper.model.eval()
        per_row = []
        for i, example in enumerate(examples):
            tokens, mask = wrapper._to_arrays([example])
            lp = mlx_dpo.sequence_logps(wrapper.model, tokens, mask, head_split=wrapper._head_split)
            loss, _ = mlx_dpo.dpo_loss(lp[:1], lp[1:], ref_c[i : i + 1], ref_r[i : i + 1], 0.1)
            per_row.append(loss.item())
        assert result["val_loss"] == pytest.approx(sum(per_row) / 3, rel=1e-4)


class TestCheckpoints:
    def test_save_steps_files_are_found_by_resume_auto(self, tmp_path, monkeypatch):
        from soup_cli.commands.train import _resolve_mlx_checkpoint

        wrapper = _seeded_wrapper(tmp_path, monkeypatch, batch_size=2, epochs=2,
                                  save_steps=2, seed=1)
        wrapper.train()
        out = tmp_path / "output"
        numbered = sorted(p.name for p in out.glob("*_adapters.safetensors"))
        assert numbered == [f"{n:07d}_adapters.safetensors" for n in (2, 4, 6, 8)]
        resolved = _resolve_mlx_checkpoint("auto", str(out))
        assert resolved is not None and resolved.endswith("0000008_adapters.safetensors")


class TestHeadSplitFallback:
    def test_a_mismatching_head_falls_back_with_a_warning(self, tmp_path, monkeypatch):
        from io import StringIO

        from rich.console import Console

        from soup_cli.trainer import mlx_dpo

        real = mlx_dpo.resolve_head_split

        def scaled(model):
            split = real(model)
            backbone, head = split
            return backbone, (lambda h: head(h) * 2.0)

        monkeypatch.setattr(mlx_dpo, "resolve_head_split", scaled)
        buffer = StringIO()
        monkeypatch.setattr(mlx_dpo, "console", Console(file=buffer, width=400))
        wrapper = _seeded_wrapper(tmp_path, monkeypatch, batch_size=2, seed=1)
        result = wrapper.train()
        assert wrapper._head_split is None
        assert "using the full-vocabulary path" in _strip_ansi(buffer.getvalue())
        assert result["losses"][0] == pytest.approx(math.log(2), rel=1e-4)


class TestRefusalsAndReporting:
    def test_too_few_pairs_after_ties_names_batch_size_and_ties(self, tmp_path, monkeypatch):
        wrapper = _seeded_wrapper(tmp_path, monkeypatch, batch_size=4, seed=1)
        tie = {"prompt": "q:", "chosen": " same", "rejected": " same"}
        wrapper._dataset["train"] = wrapper._dataset["train"][:3] + [tie, dict(tie)]
        with pytest.raises(ValueError, match=r"3 training pair.*batch_size \(4\).*2 tie"):
            wrapper.train()

    def test_metrics_reach_the_live_stream_with_a_display(self, tmp_path, monkeypatch):
        from soup_cli.utils import train_event_buffer

        events = []
        monkeypatch.setattr(train_event_buffer, "push_train_event", events.append)

        class Display:
            def start(self, total):
                pass

            def update(self, **kw):
                pass

            def stop(self):
                pass

        wrapper = _seeded_wrapper(tmp_path, monkeypatch, batch_size=2, seed=1)
        wrapper.train(display=Display())
        assert events and all(e.type == "metric" and e.loss is not None for e in events)
        assert any(e.val_loss is not None for e in events)