Skip to content

armgit / Soup

Public Imported from GitHub
0

Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.

59.8 KB
Raw Blame
"""v0.71.31 — Judge-in-the-loop suite.

Covers the shared pairwise-judge layer (``eval/judge.pairwise_compare`` /
``pairwise_winrate`` / ``make_soup_pairwise_judge``), ``soup ship --task-mode
pairwise`` (#284), ``task='online_dpo'`` (schema + trainer + routing),
``soup data best-of-n``, and ``soup data evolve``.
"""

from __future__ import annotations

import re

import pytest

_ANSI_RE = re.compile(r"\x1b\[[0-9;]*m")


def _plain(text: str) -> str:
    """Return ANSI-stripped, whitespace-collapsed CLI output."""
    return " ".join(_ANSI_RE.sub("", text).split())


def _local_base_model_dir() -> str:
    """A tiny GPT-2 + tokenizer built locally (no download), standing in for
    hf-internal-testing/tiny-random-gpt2 -- #1356 found the one test here
    that actually calls wrapper.setup() (building a real trainer) pulling it
    from the Hub for nothing that test checks depends on real pretrained
    weights. Shared and cached across every file that needs one
    (tests/_tiny_hf_models.py)."""
    from tests._tiny_hf_models import tiny_model_dir

    return tiny_model_dir("gpt2")


# ---------------------------------------------------------------------------
# Shared test doubles
# ---------------------------------------------------------------------------


def _trl_has_judges():
    """True when the installed OnlineDPOTrainer accepts pairwise ``judge=``."""
    from soup_cli.trainer.online_dpo import _trl_has_judges as has_judges

    return has_judges()


_TRL_HAS_JUDGES = _trl_has_judges()


class _FakeJudge:
    """Synthetic length-preferring Soup evaluator: prefers the LONGER response.

    Provides BOTH shapes so it drives either trl API: ``compare_pair`` (pairwise,
    trl 0.19.x) and ``evaluate`` (pointwise, trl 1.x reward-func path).
    """

    def __init__(self, rubric=None):
        self.rubric = rubric or {"scale": {"min": 1, "max": 5}, "criteria": []}

    def compare_pair(self, prompt, resp_a, resp_b):
        if len(resp_a) == len(resp_b):
            return -1
        return 0 if len(resp_a) > len(resp_b) else 1

    def evaluate(self, prompt, response, category="default"):
        from soup_cli.eval.judge import JudgeScore

        return JudgeScore(
            prompt=prompt, response=response, weighted_score=float(len(response))
        )


class _PosBias:
    """A biased judge that ALWAYS says the first response is best."""

    def compare_pair(self, prompt, resp_a, resp_b):
        return 0


# ---------------------------------------------------------------------------
# Task 1 — pairwise_compare / pairwise_winrate
# ---------------------------------------------------------------------------


class TestPairwiseCompare:
    def test_a_preferred(self):
        from soup_cli.eval.judge import pairwise_compare

        assert pairwise_compare("p", "longer response", "short", _FakeJudge(), swap=True) == 0

    def test_b_preferred(self):
        from soup_cli.eval.judge import pairwise_compare

        assert pairwise_compare("p", "short", "longer response", _FakeJudge(), swap=True) == 1

    def test_tie_on_equal(self):
        from soup_cli.eval.judge import pairwise_compare

        assert pairwise_compare("p", "aaaa", "bbbb", _FakeJudge(), swap=True) == -1

    def test_swap_debias_disagreement_is_tie(self):
        from soup_cli.eval.judge import pairwise_compare

        # A judge that ALWAYS says "first is best" disagrees under swap -> tie.
        assert pairwise_compare("p", "x", "y", _PosBias(), swap=True) == -1

    def test_no_swap_uses_single_call(self):
        from soup_cli.eval.judge import pairwise_compare

        assert pairwise_compare("p", "x", "y", _PosBias(), swap=False) == 0

    def test_one_side_failure_is_tie(self):
        from soup_cli.eval.judge import pairwise_compare

        # Definite verdict on the first order, failure (-1) on the swap -> the
        # single unconfirmed verdict must NOT be trusted; result is a tie.
        class _Flaky:
            def __init__(self):
                self.calls = 0

            def compare_pair(self, prompt, resp_a, resp_b):
                self.calls += 1
                return 0 if self.calls == 1 else -1

        assert pairwise_compare("p", "x", "y", _Flaky(), swap=True) == -1


class TestPairwiseWinrate:
    def test_tuned_always_wins(self):
        from soup_cli.eval.judge import pairwise_winrate

        # base short, tuned long -> _FakeJudge prefers tuned every time -> 1.0
        pairs = [("p", "s", "longer"), ("q", "s", "longer")]
        assert pairwise_winrate(pairs, _FakeJudge()) == 1.0

    def test_all_ties_is_half(self):
        from soup_cli.eval.judge import pairwise_winrate

        pairs = [("p", "aaa", "bbb")]  # equal length -> tie -> 0.5
        assert pairwise_winrate(pairs, _FakeJudge()) == 0.5

    def test_empty_pairs_is_half(self):
        from soup_cli.eval.judge import pairwise_winrate

        assert pairwise_winrate([], _FakeJudge()) == 0.5

    def test_mixed_winrate(self):
        from soup_cli.eval.judge import pairwise_winrate

        # tuned wins (long), tuned loses (short), tie (equal) -> (1 + 0 + 0.5)/3
        pairs = [("a", "s", "longer"), ("b", "longer", "s"), ("c", "xx", "yy")]
        assert pairwise_winrate(pairs, _FakeJudge()) == (1.0 + 0.0 + 0.5) / 3


class TestParsePairwise:
    def test_json_winner_a(self):
        from soup_cli.eval.judge import _parse_pairwise

        assert _parse_pairwise('{"winner": "A"}') == 0

    def test_json_winner_b_lowercase(self):
        from soup_cli.eval.judge import _parse_pairwise

        assert _parse_pairwise('the answer is {"winner": "b"}') == 1

    def test_bare_token_fallback(self):
        from soup_cli.eval.judge import _parse_pairwise

        assert _parse_pairwise("A is clearly better") == 0
        assert _parse_pairwise("B wins here") == 1

    def test_unparseable_is_tie(self):
        from soup_cli.eval.judge import _parse_pairwise

        assert _parse_pairwise("I cannot decide") == -1
        assert _parse_pairwise("") == -1


class TestCompareePairMethod:
    def test_compare_pair_parses_llm_reply(self, monkeypatch):
        from soup_cli.eval.judge import JudgeEvaluator

        ev = JudgeEvaluator(provider="ollama", model="m")
        monkeypatch.setattr(ev, "_call_llm", lambda prompt: '{"winner": "B"}')
        assert ev.compare_pair("p", "x", "y") == 1

    def test_compare_pair_llm_failure_propagates(self, monkeypatch):
        # #1447: a failed judge call is not a tie. It used to return -1 here,
        # which scored a dead judge as a measured 0.5 win-rate downstream.
        from soup_cli.eval.judge import JudgeEvaluator, JudgeUnavailableError

        ev = JudgeEvaluator(provider="ollama", model="m")

        def _boom(prompt):
            raise JudgeUnavailableError("network down", url="http://localhost:11434")

        monkeypatch.setattr(ev, "_call_llm", _boom)
        with pytest.raises(JudgeUnavailableError):
            ev.compare_pair("p", "x", "y")


# ---------------------------------------------------------------------------
# Task 2 — make_soup_pairwise_judge (trl <1 pairwise BasePairwiseJudge
# adapter). trl 1.x removed judges -> the 1.x-equivalent coverage is
# TestJudgeRewardFunc below (the pointwise reward-func adapter). Together they
# cover whichever adapter the installed trl actually exposes.
# ---------------------------------------------------------------------------


@pytest.mark.skipif(
    not _TRL_HAS_JUDGES, reason="pairwise BasePairwiseJudge adapter is trl<1.0 only"
)
class TestSoupPairwiseJudge:
    def test_judge_returns_best_index(self):
        from soup_cli.eval.judge import make_soup_pairwise_judge

        j = make_soup_pairwise_judge(_FakeJudge())  # prefers longer
        # prompt0: [short, long] -> B(1); prompt1: [long, short] -> A(0)
        out = j.judge(["p0", "p1"], [["s", "longer"], ["longer", "s"]])
        assert out == [1, 0]

    def test_judge_tie_returns_minus_one(self):
        from soup_cli.eval.judge import make_soup_pairwise_judge

        j = make_soup_pairwise_judge(_FakeJudge())
        assert j.judge(["p"], [["aaaa", "bbbb"]]) == [-1]

    def test_shuffle_order_false_no_swap(self):
        from soup_cli.eval.judge import make_soup_pairwise_judge

        j = make_soup_pairwise_judge(_PosBias())
        assert j.judge(["p"], [["x", "y"]], shuffle_order=False) == [0]

    def test_malformed_pair_returns_minus_one(self):
        from soup_cli.eval.judge import make_soup_pairwise_judge

        j = make_soup_pairwise_judge(_FakeJudge())
        assert j.judge(["p"], [["only-one"]]) == [-1]

    def test_is_trl_base_pairwise_judge(self):
        from soup_cli.eval.judge import (
            _base_pairwise_judge_cls,
            make_soup_pairwise_judge,
        )

        assert isinstance(
            make_soup_pairwise_judge(_FakeJudge()), _base_pairwise_judge_cls()
        )


# ---------------------------------------------------------------------------
# make_judge_reward_func (trl 1.x pointwise reward-func adapter) — version-
# independent (uses evaluator.evaluate), so it runs on every trl.
# ---------------------------------------------------------------------------


class TestJudgeRewardFunc:
    def test_named_and_scores_by_evaluate(self):
        from soup_cli.eval.judge import make_judge_reward_func

        fn = make_judge_reward_func(_FakeJudge())
        assert fn.__name__ == "soup_judge"
        # _FakeJudge.evaluate -> weighted_score == len(response)
        scores = fn(["p", "p"], ["ab", "abcd"])
        assert scores == [2.0, 4.0]

    def test_custom_name(self):
        from soup_cli.eval.judge import make_judge_reward_func

        assert make_judge_reward_func(_FakeJudge(), name="mine").__name__ == "mine"

    def test_conversational_prompt_and_completion(self):
        from soup_cli.eval.judge import make_judge_reward_func

        fn = make_judge_reward_func(_FakeJudge())
        prompts = [[{"role": "user", "content": "hi"}]]
        completions = [[{"role": "assistant", "content": "abcd"}]]
        assert fn(prompts, completions) == [4.0]

    def test_evaluate_failure_scores_zero(self):
        from soup_cli.eval.judge import make_judge_reward_func

        class _Boom:
            def evaluate(self, prompt, response, category="default"):
                raise RuntimeError("network down")

        assert make_judge_reward_func(_Boom())(["p"], ["x"]) == [0.0]


# ---------------------------------------------------------------------------
# Task 3 — soup ship --task-mode pairwise (#284)
# ---------------------------------------------------------------------------


class _Task:
    def __init__(self, prompt):
        self.prompt = prompt
        self.category = "default"


class TestShipPairwise:
    def test_pairwise_in_supported(self):
        from soup_cli.utils.ship_verdict import SUPPORTED_TASK_MODES

        assert "pairwise" in SUPPORTED_TASK_MODES

    def test_leg1_pairwise_builds_taskwin(self, monkeypatch):
        from soup_cli.commands import ship as ship_cmd

        monkeypatch.setattr(
            "soup_cli.eval.custom.load_eval_tasks",
            lambda path: [_Task("q1"), _Task("q2")],
        )
        monkeypatch.setattr(
            "soup_cli.eval.gate._parse_judge_url",
            lambda url: ("ollama", "m", None),
        )
        monkeypatch.setattr(
            "soup_cli.eval.judge.JudgeEvaluator", lambda **kw: _FakeJudge()
        )
        # base_gen short, tuned_gen long; _FakeJudge prefers long -> winrate 1.0
        tw = ship_cmd._leg1_pairwise(
            lambda p: "s", lambda p: "longer", "x.jsonl", "ollama://m"
        )
        assert tw.mode == "pairwise"
        assert tw.base == 0.5
        assert tw.tuned == 1.0
        assert tw.won is True

    def test_evidence_pairwise_accepted(self):
        import json
        import os

        from typer.testing import CliRunner

        from soup_cli.commands.ship import app

        path = os.path.join(os.getcwd(), "_ev_pairwise_v07131.json")
        with open(path, "w", encoding="utf-8") as fh:
            json.dump(
                {
                    "task": {"mode": "pairwise", "base": 0.5, "tuned": 0.7},
                    "benchmarks": {"mini_mmlu": {"base": 0.8, "tuned": 0.8}},
                },
                fh,
            )
        try:
            result = CliRunner().invoke(app, ["--evidence", path])
            assert result.exit_code == 0, (result.output, repr(result.exception))
            assert "SHIP" in result.output
        finally:
            os.remove(path)

    def test_evidence_pairwise_tie_is_dont_ship(self):
        import json
        import os

        from typer.testing import CliRunner

        from soup_cli.commands.ship import app

        path = os.path.join(os.getcwd(), "_ev_pairwise_tie_v07131.json")
        with open(path, "w", encoding="utf-8") as fh:
            json.dump(
                {
                    "task": {"mode": "pairwise", "base": 0.5, "tuned": 0.5},
                    "benchmarks": {"mini_mmlu": {"base": 0.8, "tuned": 0.8}},
                },
                fh,
            )
        try:
            result = CliRunner().invoke(app, ["--evidence", path])
            assert result.exit_code == 2, (result.output, repr(result.exception))
            assert "DON'T SHIP" in result.output
        finally:
            os.remove(path)


# ---------------------------------------------------------------------------
# Task 4 — online_dpo schema (task literal + fields + cross-validator)
# ---------------------------------------------------------------------------

_ODPO = """
base: sshleifer/tiny-gpt2
task: online_dpo
data:
  train: x.jsonl
training:
  online_dpo_judge: "ollama://llama3"
"""


class TestOnlineDpoSchema:
    def test_happy_parse(self):
        from soup_cli.config.loader import load_config_from_string

        cfg = load_config_from_string(_ODPO)
        assert cfg.task == "online_dpo"
        assert cfg.training.online_dpo_judge == "ollama://llama3"
        assert cfg.training.online_dpo_loss_type == "sigmoid"
        assert cfg.training.online_dpo_max_new_tokens == 64

    def test_reward_model_leg_parses(self):
        from soup_cli.config.loader import load_config_from_string

        cfg = load_config_from_string(
            "base: sshleifer/tiny-gpt2\ntask: online_dpo\n"
            "data:\n  train: x.jsonl\n"
            "training:\n  reward_model: some/rm\n"
        )
        assert cfg.training.reward_model == "some/rm"
        assert cfg.training.online_dpo_judge is None

    def test_reject_both_judge_and_reward(self):
        import pytest

        from soup_cli.config.loader import load_config_from_string

        with pytest.raises(Exception, match="exactly one"):
            load_config_from_string(_ODPO + "  reward_model: some/rm\n")

    def test_reject_neither(self):
        import pytest

        from soup_cli.config.loader import load_config_from_string

        with pytest.raises(Exception, match="judge|reward_model"):
            load_config_from_string(
                "base: sshleifer/tiny-gpt2\ntask: online_dpo\n"
                "data:\n  train: x.jsonl\n"
            )

    def test_reject_mlx_backend(self):
        import pytest

        from soup_cli.config.loader import load_config_from_string

        with pytest.raises(Exception, match="transformers"):
            load_config_from_string(
                "base: sshleifer/tiny-gpt2\ntask: online_dpo\nbackend: mlx\n"
                "data:\n  train: x.jsonl\n"
                "training:\n  online_dpo_judge: \"ollama://m\"\n"
            )

    def test_footgun_field_without_task(self):
        import pytest

        from soup_cli.config.loader import load_config_from_string

        with pytest.raises(Exception, match="online_dpo"):
            load_config_from_string(
                "base: sshleifer/tiny-gpt2\ntask: sft\n"
                "data:\n  train: x.jsonl\n"
                "training:\n  online_dpo_judge: \"ollama://m\"\n"
            )

    def test_reject_empty_judge(self):
        import pytest

        from soup_cli.config.loader import load_config_from_string

        with pytest.raises(Exception, match="non-empty|empty"):
            load_config_from_string(
                "base: sshleifer/tiny-gpt2\ntask: online_dpo\n"
                "data:\n  train: x.jsonl\n"
                "training:\n  online_dpo_judge: \"\"\n"
            )

    def test_loss_type_and_max_new_tokens(self):
        from soup_cli.config.loader import load_config_from_string

        cfg = load_config_from_string(
            _ODPO + "  online_dpo_loss_type: ipo\n  online_dpo_max_new_tokens: 128\n"
        )
        assert cfg.training.online_dpo_loss_type == "ipo"
        assert cfg.training.online_dpo_max_new_tokens == 128

    def test_reject_vision_modality(self):
        import pytest

        from soup_cli.config.loader import load_config_from_string

        with pytest.raises(Exception, match="modality"):
            load_config_from_string(
                "base: hf-internal-testing/tiny-random-gpt2\ntask: online_dpo\n"
                "modality: vision\ndata:\n  train: x.jsonl\n"
                "training:\n  online_dpo_judge: \"ollama://m\"\n"
            )

    def test_footgun_loss_type_without_task(self):
        import pytest

        from soup_cli.config.loader import load_config_from_string

        with pytest.raises(Exception, match="online_dpo"):
            load_config_from_string(
                "base: m\ntask: sft\ndata:\n  train: x.jsonl\n"
                "training:\n  online_dpo_loss_type: ipo\n"
            )

    def test_field_validator_direct(self):
        import pytest

        from soup_cli.config.schema import TrainingConfig

        with pytest.raises(Exception, match="null"):
            TrainingConfig(online_dpo_judge="ollama://\x00m")
        with pytest.raises(Exception, match="512"):
            TrainingConfig(online_dpo_judge="x" * 513)
        with pytest.raises(Exception, match="string"):
            TrainingConfig(online_dpo_judge=123)

    def test_max_new_tokens_bounds(self):
        import pytest

        from soup_cli.config.schema import TrainingConfig

        with pytest.raises(Exception):
            TrainingConfig(online_dpo_max_new_tokens=0)
        with pytest.raises(Exception):
            TrainingConfig(online_dpo_max_new_tokens=4097)


# ---------------------------------------------------------------------------
# Task 5 — OnlineDPOTrainerWrapper
# ---------------------------------------------------------------------------


class TestOnlineDpoWrapper:
    def test_prompt_rows_from_messages(self):
        from soup_cli.trainer.online_dpo import OnlineDPOTrainerWrapper

        rows = [
            {
                "messages": [
                    {"role": "user", "content": "hi"},
                    {"role": "assistant", "content": "yo"},
                ]
            }
        ]
        out = OnlineDPOTrainerWrapper._to_prompt_rows(rows)
        assert out == [{"prompt": [{"role": "user", "content": "hi"}]}]

    def test_prompt_rows_from_prompt_field(self):
        from soup_cli.trainer.online_dpo import OnlineDPOTrainerWrapper

        out = OnlineDPOTrainerWrapper._to_prompt_rows([{"prompt": "hello"}])
        assert out == [{"prompt": [{"role": "user", "content": "hello"}]}]

    def test_prompt_rows_keeps_system(self):
        from soup_cli.trainer.online_dpo import OnlineDPOTrainerWrapper

        rows = [
            {
                "messages": [
                    {"role": "system", "content": "be nice"},
                    {"role": "user", "content": "hi"},
                    {"role": "assistant", "content": "yo"},
                ]
            }
        ]
        out = OnlineDPOTrainerWrapper._to_prompt_rows(rows)
        assert out == [
            {"prompt": [{"role": "system", "content": "be nice"},
                        {"role": "user", "content": "hi"}]}
        ]

    def test_prompt_rows_multiturn_keeps_alternation(self):
        from soup_cli.trainer.online_dpo import OnlineDPOTrainerWrapper

        rows = [
            {
                "messages": [
                    {"role": "user", "content": "u1"},
                    {"role": "assistant", "content": "a1"},
                    {"role": "user", "content": "u2"},
                    {"role": "assistant", "content": "a2"},
                ]
            }
        ]
        out = OnlineDPOTrainerWrapper._to_prompt_rows(rows)
        # Interleaved assistant turn kept; conversation ends on the last user
        # turn (trailing assistant dropped -> model generates it on-policy).
        assert out == [
            {
                "prompt": [
                    {"role": "user", "content": "u1"},
                    {"role": "assistant", "content": "a1"},
                    {"role": "user", "content": "u2"},
                ]
            }
        ]

    def test_prompt_rows_no_user_skipped(self):
        from soup_cli.trainer.online_dpo import OnlineDPOTrainerWrapper

        out = OnlineDPOTrainerWrapper._to_prompt_rows(
            [{"messages": [{"role": "assistant", "content": "x"}]}]
        )
        assert out == []

    def test_synthetic_evaluator_double_both_shapes(self):
        # The seam is a Soup evaluator (compare_pair + evaluate), adapted per
        # trl version. Verify both shapes on the length-preferring double.
        ev = _FakeJudge()
        assert ev.compare_pair("p", "longer", "s") == 0
        assert ev.compare_pair("p", "s", "longer") == 1
        assert ev.evaluate("p", "abcd").weighted_score == 4.0

    def test_setup_builds_trainer_with_synthetic_judge(self):
        # Parametrized over trl version by the wrapper itself: the seam evaluator
        # is adapted to judge= (trl 0.19.x) or reward_funcs= (trl 1.x). Builds a
        # real trainer on either.
        from pathlib import Path

        import soup_cli.trainer.online_dpo as od
        from soup_cli.config.loader import load_config_from_string
        from soup_cli.trainer.online_dpo import OnlineDPOTrainerWrapper

        cfg = load_config_from_string(
            f"base: {Path(_local_base_model_dir()).as_posix()}\ntask: online_dpo\n"
            "data:\n  train: x.jsonl\n  max_length: 64\n"
            "training:\n  online_dpo_judge: \"ollama://m\"\n"
            "  epochs: 1\n  batch_size: 2\n  online_dpo_max_new_tokens: 8\n"
        )
        od._ONLINE_DPO_JUDGE_OVERRIDE = _FakeJudge()
        try:
            wrapper = OnlineDPOTrainerWrapper(cfg, device="cpu")
            wrapper.setup(
                {"train": [{"messages": [{"role": "user", "content": "hi there"}]},
                           {"messages": [{"role": "user", "content": "hello"}]}]}
            )
            assert wrapper.trainer is not None
        finally:
            od._ONLINE_DPO_JUDGE_OVERRIDE = None


class _Tcfg:
    """Minimal tcfg stub for _build_judge_or_reward (reads two attrs)."""

    def __init__(self, judge=None, reward=None):
        self.online_dpo_judge = judge
        self.reward_model = reward


def _online_dpo_wrapper():
    from soup_cli.config.loader import load_config_from_string
    from soup_cli.trainer.online_dpo import OnlineDPOTrainerWrapper

    cfg = load_config_from_string(
        "base: hf-internal-testing/tiny-random-gpt2\ntask: online_dpo\n"
        "data:\n  train: x.jsonl\n"
        "training:\n  online_dpo_judge: \"ollama://m\"\n"
    )
    return OnlineDPOTrainerWrapper(cfg, device="cpu")


class TestBuildJudgeOrReward:
    def test_judge_url_branch_adapts_to_trl_version(self):
        import soup_cli.trainer.online_dpo as od

        od._ONLINE_DPO_JUDGE_OVERRIDE = None
        result = _online_dpo_wrapper()._build_judge_or_reward(_Tcfg(judge="ollama://m"))
        if _TRL_HAS_JUDGES:  # trl <1 -> pairwise judge=
            from soup_cli.eval.judge import _base_pairwise_judge_cls

            assert isinstance(result["judge"], _base_pairwise_judge_cls())
            assert "reward_funcs" not in result
        else:  # trl 1.x -> pointwise reward_funcs=
            assert callable(result["reward_funcs"][0])
            assert "judge" not in result

    def test_reward_model_branch_adapts_to_trl_version(self, monkeypatch):
        import soup_cli.trainer.online_dpo as od

        od._ONLINE_DPO_JUDGE_OVERRIDE = None
        monkeypatch.setattr(
            "transformers.AutoModelForSequenceClassification.from_pretrained",
            lambda *a, **k: object(),
        )
        monkeypatch.setattr(
            "transformers.AutoTokenizer.from_pretrained", lambda *a, **k: object()
        )
        result = _online_dpo_wrapper()._build_judge_or_reward(_Tcfg(reward="some/rm"))
        # #300 — this used to branch on `_TRL_HAS_JUDGES`, the SAME predicate the
        # production code branched on, so it asserted that the code returned what
        # its own condition said and could not fail. It stayed green while
        # `reward_model=` was being passed to a trl that removed it at 0.25.0.
        #
        # It now asks the installed trl instead. `tests/test_issue300_online_dpo_kwargs.py`
        # carries the stronger form (binding every built kwarg against the real
        # signature); this one keeps the shape assertion honest.
        from soup_cli.trainer.online_dpo import _trl_accepts

        if _trl_accepts("reward_model"):
            assert set(result.keys()) == {"reward_model", "reward_processing_class"}
        else:
            assert set(result.keys()) == {"reward_funcs", "reward_processing_classes"}
            assert len(result["reward_funcs"]) == 1
        assert not [k for k in result if not _trl_accepts(k)], (
            f"built {sorted(result)} but this trl accepts none of the missing ones"
        )

    def test_precedence_override_wins(self):
        import soup_cli.trainer.online_dpo as od

        od._ONLINE_DPO_JUDGE_OVERRIDE = _FakeJudge()
        try:
            # A URL that _parse_judge_url would REJECT if the URL branch ran.
            # No exception -> the seam took precedence (URL never parsed).
            result = _online_dpo_wrapper()._build_judge_or_reward(
                _Tcfg(judge="not-a-valid-url")
            )
            assert "judge" in result or "reward_funcs" in result
        finally:
            od._ONLINE_DPO_JUDGE_OVERRIDE = None

    def test_neither_raises(self):
        import soup_cli.trainer.online_dpo as od

        od._ONLINE_DPO_JUDGE_OVERRIDE = None
        with pytest.raises(ValueError, match="judge|reward_model"):
            _online_dpo_wrapper()._build_judge_or_reward(_Tcfg())


# ---------------------------------------------------------------------------
# Task 6 — train.py routing
# ---------------------------------------------------------------------------


class TestOnlineDpoRouting:
    def test_train_routes_online_dpo(self):
        # A distinct-string `task == ...` branch (cannot be shadowed by another
        # equality branch) that instantiates the wrapper. #1213 moved the chain
        # into the dispatch both commands call, so the branch is read there.
        import inspect

        from soup_cli.commands import train as train_cmd
        from soup_cli.trainer import dispatch as dispatch_mod

        assert "build_trainer(" in inspect.getsource(train_cmd)
        src = inspect.getsource(dispatch_mod)
        assert 'task == "online_dpo"' in src
        assert "OnlineDPOTrainerWrapper(cfg, **trainer_kwargs)" in src


# ---------------------------------------------------------------------------
# Task 7 — utils/best_of_n.py
# ---------------------------------------------------------------------------


class _ScoreJudge:
    """evaluate(prompt, response) -> JudgeScore(weighted_score=len(response))."""

    def evaluate(self, prompt, response, category="default"):
        from soup_cli.eval.judge import JudgeScore

        return JudgeScore(
            prompt=prompt, response=response, weighted_score=float(len(response))
        )


class TestBestOfN:
    def test_provider_generate_fn_is_called_n_times(self):
        from soup_cli.utils.best_of_n import sample_candidates

        calls = []

        def generate(prompt):
            calls.append(prompt)
            return f"  answer {len(calls)}  "

        candidates = sample_candidates(
            None,
            None,
            "question",
            n=3,
            temperature=0.7,
            max_new_tokens=64,
            generate_fn=generate,
        )

        assert calls == ["question", "question", "question"]
        assert candidates == ["answer 1", "answer 2", "answer 3"]

    def test_pick_best_argmax(self):
        from soup_cli.utils.best_of_n import judge_pick_best

        pick = judge_pick_best("p", ["a", "abcd", "ab"], _ScoreJudge())
        assert pick.winner_idx == 1
        assert pick.winner == "abcd"
        assert pick.scores == (1.0, 4.0, 2.0)

    def test_pick_best_ties_first(self):
        from soup_cli.utils.best_of_n import judge_pick_best

        pick = judge_pick_best("p", ["ab", "cd"], _ScoreJudge())
        assert pick.winner_idx == 0

    def test_pick_best_empty_raises(self):
        import pytest

        from soup_cli.utils.best_of_n import judge_pick_best

        with pytest.raises(ValueError, match="candidate"):
            judge_pick_best("p", [], _ScoreJudge())

    def test_pick_best_rejects_non_finite_scores_in_every_position(self):
        import math

        import pytest

        from soup_cli.utils.best_of_n import judge_pick_best

        class _Judge:
            def __init__(self, scores):
                self.scores = iter(scores)

            def evaluate(self, prompt, response):
                from soup_cli.eval.judge import JudgeScore

                return JudgeScore(
                    prompt=prompt,
                    response=response,
                    weighted_score=next(self.scores),
                )

        candidates = ["first", "middle", "last"]
        for bad_score in (float("nan"), float("inf"), float("-inf")):
            for bad_index in range(len(candidates)):
                scores = [1.0, 2.0, 3.0]
                scores[bad_index] = bad_score
                with pytest.raises(ValueError, match=rf"candidate {bad_index}.*finite"):
                    judge_pick_best("private prompt", candidates, _Judge(scores))
                assert not math.isfinite(bad_score)

    def test_pick_best_rejects_boolean_score(self):
        import pytest

        from soup_cli.utils.best_of_n import judge_pick_best

        class _BoolJudge:
            def evaluate(self, prompt, response):
                from soup_cli.eval.judge import JudgeScore

                return JudgeScore(prompt=prompt, response=response, weighted_score=True)

        with pytest.raises(ValueError, match=r"candidate 0.*finite number"):
            judge_pick_best("private prompt", ["candidate"], _BoolJudge())

    def test_build_sft_row(self):
        from soup_cli.utils.best_of_n import BestOfNPick, build_sft_row

        row = build_sft_row(
            "p", BestOfNPick(1, "win", (1.0, 3.0)), judge_model="ollama://m"
        )
        assert row["messages"] == [
            {"role": "user", "content": "p"},
            {"role": "assistant", "content": "win"},
        ]
        assert row["_best_of_n"]["winner_idx"] == 1
        assert row["_best_of_n"]["judge_model"] == "ollama://m"
        assert row["_best_of_n"]["n"] == 2
        assert row["_best_of_n"]["scores"] == [1.0, 3.0]

    def test_build_dpo_pair(self):
        from soup_cli.utils.best_of_n import BestOfNPick, build_dpo_pair

        pair = build_dpo_pair("p", BestOfNPick(1, "win", (1.0, 3.0)), ["lose", "win"])
        assert pair == {"prompt": "p", "chosen": "win", "rejected": "lose"}

    def test_build_dpo_pair_all_equal_none(self):
        from soup_cli.utils.best_of_n import BestOfNPick, build_dpo_pair

        assert build_dpo_pair("p", BestOfNPick(0, "x", (2.0, 2.0)), ["x", "x"]) is None

    def test_three_candidates_non_boundary_winner_and_loser(self):
        from soup_cli.utils.best_of_n import build_dpo_pair, judge_pick_best

        # scores by length: ["xx"(2), "xxxx"(4), "x"(1)] -> winner idx1, loser idx2
        cands = ["xx", "xxxx", "x"]
        pick = judge_pick_best("p", cands, _ScoreJudge())
        assert pick.winner_idx == 1 and pick.winner == "xxxx"
        pair = build_dpo_pair("p", pick, cands)
        assert pair == {"prompt": "p", "chosen": "xxxx", "rejected": "x"}

    def test_no_top_level_torch(self):
        import ast
        import pathlib

        import soup_cli

        src = (
            pathlib.Path(soup_cli.__file__).parent / "utils" / "best_of_n.py"
        ).read_text(encoding="utf-8")
        tree = ast.parse(src)
        top = {
            n.names[0].name.split(".")[0]
            for n in ast.walk(tree)
            if isinstance(n, ast.Import) and n.col_offset == 0
        }
        top |= {
            n.module.split(".")[0]
            for n in ast.walk(tree)
            if isinstance(n, ast.ImportFrom) and n.col_offset == 0 and n.module
        }
        assert "torch" not in top
        assert "transformers" not in top


# ---------------------------------------------------------------------------
# Task 8 — soup data best-of-n command
# ---------------------------------------------------------------------------


class TestBestOfNCli:
    @staticmethod
    def _invoke_guard(monkeypatch, tmp_path, *sampler_args):
        from typer.testing import CliRunner

        from soup_cli.commands.data import app

        monkeypatch.chdir(tmp_path)
        monkeypatch.setattr("soup_cli.eval.judge.JudgeEvaluator", lambda **kw: _ScoreJudge())
        prompts = tmp_path / "prompts.jsonl"
        prompts.write_text('{"prompt": "question"}\n', encoding="utf-8")
        return CliRunner().invoke(
            app,
            [
                "best-of-n",
                "--prompts",
                str(prompts),
                "--judge",
                "ollama://judge",
                "--plan-only",
                *sampler_args,
            ],
        )

    def test_help(self):
        from typer.testing import CliRunner

        from soup_cli.commands.data import app

        result = CliRunner().invoke(app, ["best-of-n", "--help"])
        assert result.exit_code == 0, (result.output, repr(result.exception))
        output = _plain(result.output)
        assert "best-of-n" in output.lower() or "best of n" in output.lower()
        assert "--provider" in output
        assert "--model" in output
        assert "--base-url" in output

    def test_rejects_local_base_with_provider(self, monkeypatch, tmp_path):
        result = self._invoke_guard(
            monkeypatch, tmp_path, "--base", "local-model", "--provider", "ollama"
        )
        assert result.exit_code == 2, (result.output, repr(result.exception))
        assert "mutually exclusive" in _plain(result.output).lower()

    def test_rejects_local_only_flags_with_provider(self, monkeypatch, tmp_path):
        result = self._invoke_guard(
            monkeypatch,
            tmp_path,
            "--provider",
            "ollama",
            "--model",
            "sampler-model",
            "--seed",
            "42",
        )
        assert result.exit_code == 2, (result.output, repr(result.exception))
        output = _plain(result.output).lower()
        assert "--seed" in output
        assert "only to local --base" in output

    def test_rejects_provider_options_without_provider(self, monkeypatch, tmp_path):
        result = self._invoke_guard(
            monkeypatch, tmp_path, "--base", "local-model", "--model", "sampler-model"
        )
        assert result.exit_code == 2, (result.output, repr(result.exception))
        assert "require --provider" in _plain(result.output).lower()

    def test_requires_local_base_or_provider(self, monkeypatch, tmp_path):
        result = self._invoke_guard(monkeypatch, tmp_path)
        assert result.exit_code == 2, (result.output, repr(result.exception))
        assert "either local --base or --provider" in _plain(result.output).lower()

    def test_provider_path_calls_sampler_and_records_provenance(self, monkeypatch):
        import json
        import os

        from typer.testing import CliRunner

        import soup_cli.commands.data as data_cmd
        from soup_cli.commands.data import app

        factory_calls = []
        sample_calls = []

        def fake_make(provider, **kwargs):
            factory_calls.append((provider, kwargs))

            def generate(prompt):
                sample_calls.append(prompt)
                return ("a", "longest", "mid")[len(sample_calls) - 1]

            return generate

        def fail_local_load(*_args, **_kwargs):
            raise AssertionError("provider mode must not load the local model")

        monkeypatch.setattr("soup_cli.utils.magpie.make_magpie_generate_fn", fake_make)
        monkeypatch.setattr(data_cmd, "_load_bon_model", fail_local_load)
        monkeypatch.setattr("soup_cli.eval.judge.JudgeEvaluator", lambda **kw: _ScoreJudge())

        prompt_path = os.path.join(os.getcwd(), "_bon_provider_prompts.jsonl")
        output_path = os.path.join(os.getcwd(), "_bon_provider_out.jsonl")
        with open(prompt_path, "w", encoding="utf-8") as handle:
            handle.write('{"prompt": "question"}\n')
        try:
            result = CliRunner().invoke(
                app,
                [
                    "best-of-n",
                    "--provider",
                    "ollama",
                    "--model",
                    "sampler-model",
                    "--base-url",
                    "http://localhost:11434",
                    "--prompts",
                    prompt_path,
                    "--n",
                    "3",
                    "--temperature",
                    "0.7",
                    "--max-new-tokens",
                    "64",
                    "--judge",
                    "ollama://judge",
                    "-o",
                    output_path,
                ],
            )

            assert result.exit_code == 0, (result.output, repr(result.exception))
            assert factory_calls == [
                (
                    "ollama",
                    {
                        "model": "sampler-model",
                        "base_url": "http://localhost:11434",
                        "temperature": 0.7,
                        "max_tokens": 64,
                    },
                )
            ]
            assert sample_calls == ["question", "question", "question"]
            row = json.loads(open(output_path, encoding="utf-8").read())
            assert row["messages"][1]["content"] == "longest"
            assert row["_best_of_n"]["sampler"] == {
                "provider": "ollama",
                "model": "sampler-model",
            }
        finally:
            for path in (
                prompt_path,
                output_path,
                output_path + ".checkpoint.jsonl",
                output_path + ".manifest.json",
            ):
                if os.path.exists(path):
                    os.remove(path)

    def test_provider_rejects_anthropic_by_name(self, monkeypatch):
        import os

        from typer.testing import CliRunner

        from soup_cli.commands.data import app

        monkeypatch.setattr("soup_cli.eval.judge.JudgeEvaluator", lambda **kw: _ScoreJudge())
        path = os.path.join(os.getcwd(), "_bon_provider_anthropic.jsonl")
        with open(path, "w", encoding="utf-8") as handle:
            handle.write('{"prompt": "question"}\n')
        try:
            result = CliRunner().invoke(
                app,
                [
                    "best-of-n",
                    "--provider",
                    "anthropic",
                    "--model",
                    "claude",
                    "--prompts",
                    path,
                    "--judge",
                    "ollama://judge",
                    "--plan-only",
                ],
            )
            assert result.exit_code == 2, (result.output, repr(result.exception))
            assert "anthropic" in result.output.lower()
            assert "ollama or vllm" in result.output.lower()
        finally:
            os.remove(path)

    def test_provider_base_url_is_ssrf_validated(self, monkeypatch):
        import os

        from typer.testing import CliRunner

        from soup_cli.commands.data import app

        monkeypatch.setattr("soup_cli.eval.judge.JudgeEvaluator", lambda **kw: _ScoreJudge())
        path = os.path.join(os.getcwd(), "_bon_provider_ssrf.jsonl")
        with open(path, "w", encoding="utf-8") as handle:
            handle.write('{"prompt": "question"}\n')
        try:
            result = CliRunner().invoke(
                app,
                [
                    "best-of-n",
                    "--provider",
                    "ollama",
                    "--model",
                    "sampler-model",
                    "--base-url",
                    "http://evil.example.com:11434",
                    "--prompts",
                    path,
                    "--judge",
                    "ollama://judge",
                    "--plan-only",
                ],
            )
            assert result.exit_code == 2, (result.output, repr(result.exception))
            assert "localhost" in result.output.lower()
        finally:
            os.remove(path)

    def test_reject_bad_n(self):
        import os

        from typer.testing import CliRunner

        from soup_cli.commands.data import app

        path = os.path.join(os.getcwd(), "_bon_prompts_bad_n.jsonl")
        with open(path, "w", encoding="utf-8") as fh:
            fh.write('{"prompt": "hi"}\n')
        try:
            result = CliRunner().invoke(
                app,
                ["best-of-n", "--base", "m", "--prompts", path, "--n", "1",
                 "--judge", "ollama://m", "-o", "o.jsonl"],
            )
            assert result.exit_code == 2, (result.output, repr(result.exception))
            assert "n must be" in _plain(result.output) or "between 2" in _plain(result.output)
        finally:
            os.remove(path)

    def test_reject_ssrf_judge(self):
        import os

        from typer.testing import CliRunner

        from soup_cli.commands.data import app

        path = os.path.join(os.getcwd(), "_bon_prompts_ssrf.jsonl")
        with open(path, "w", encoding="utf-8") as fh:
            fh.write('{"prompt": "hi"}\n')
        try:
            result = CliRunner().invoke(
                app,
                ["best-of-n", "--base", "m", "--prompts", path, "--n", "4",
                 "--judge", "http://evil.example.com/m", "-o", "o.jsonl"],
            )
            assert result.exit_code == 2, (result.output, repr(result.exception))
        finally:
            os.remove(path)

    def test_reject_output_outside_cwd(self):
        import os

        from typer.testing import CliRunner

        from soup_cli.commands.data import app

        path = os.path.join(os.getcwd(), "_bon_prompts_outcwd.jsonl")
        with open(path, "w", encoding="utf-8") as fh:
            fh.write('{"prompt": "hi"}\n')
        try:
            result = CliRunner().invoke(
                app,
                ["best-of-n", "--base", "m", "--prompts", path, "--n", "4",
                 "--judge", "ollama://m", "-o", "../escape.jsonl"],
            )
            assert result.exit_code == 2, (result.output, repr(result.exception))
        finally:
            os.remove(path)

    def test_happy_path(self, monkeypatch):
        import json
        import os

        from typer.testing import CliRunner

        import soup_cli.commands.data as data_cmd
        import soup_cli.utils.best_of_n as bon
        from soup_cli.commands.data import app

        monkeypatch.setattr(data_cmd, "_load_bon_model", lambda base, device, trust: (None, None))
        monkeypatch.setattr(
            bon, "sample_candidates",
            lambda model, tok, prompt, **kw: ["a", "abcd", "xy"],
        )
        monkeypatch.setattr(
            "soup_cli.utils.magpie.make_magpie_generate_fn",
            lambda *_args, **_kwargs: (_ for _ in ()).throw(
                AssertionError("local mode must not construct a provider sampler")
            ),
        )
        monkeypatch.setattr("soup_cli.eval.judge.JudgeEvaluator", lambda **kw: _ScoreJudge())

        ppath = os.path.join(os.getcwd(), "_bon_prompts_ok.jsonl")
        opath = os.path.join(os.getcwd(), "_bon_out.jsonl")
        dpath = os.path.join(os.getcwd(), "_bon_pairs.jsonl")
        with open(ppath, "w", encoding="utf-8") as fh:
            fh.write('{"prompt": "q1"}\n{"prompt": "q2"}\n')
        try:
            result = CliRunner().invoke(
                app,
                ["best-of-n", "--base", "m", "--prompts", ppath, "--n", "3",
                 "--judge", "ollama://m", "-o", opath, "--emit-pairs", dpath],
            )
            assert result.exit_code == 0, (result.output, repr(result.exception))
            rows = [json.loads(x) for x in open(opath, encoding="utf-8") if x.strip()]
            assert len(rows) == 2
            assert rows[0]["messages"][1]["content"] == "abcd"  # longest wins
            assert rows[0]["_best_of_n"]["n"] == 3
            assert set(rows[0]["_best_of_n"]) == {
                "n",
                "winner_idx",
                "judge_model",
                "scores",
                "source_line",
            }
            assert [row["_best_of_n"]["source_line"] for row in rows] == [1, 2]
            pairs = [json.loads(x) for x in open(dpath, encoding="utf-8") if x.strip()]
            assert pairs[0] == {"prompt": "q1", "chosen": "abcd", "rejected": "a"}
        finally:
            for p in (
                ppath,
                opath,
                dpath,
                opath + ".checkpoint.jsonl",
                opath + ".manifest.json",
            ):
                if os.path.exists(p):
                    os.remove(p)

    def test_non_finite_judge_score_does_not_publish_outputs(self, monkeypatch, tmp_path):
        from types import SimpleNamespace

        from typer.testing import CliRunner

        import soup_cli.commands.data as data_cmd
        import soup_cli.utils.best_of_n as bon
        from soup_cli.commands.data import app

        class _NonFiniteJudge:
            def evaluate(self, _prompt, response):
                score = float("nan") if response == "bad" else 1.0
                return SimpleNamespace(weighted_score=score)

        monkeypatch.chdir(tmp_path)
        monkeypatch.setattr(
            data_cmd, "_load_bon_model", lambda base, device, trust: (None, None)
        )
        monkeypatch.setattr(
            bon,
            "sample_candidates",
            lambda model, tok, prompt, **kwargs: ["bad", "good"],
        )
        monkeypatch.setattr(
            "soup_cli.eval.judge.JudgeEvaluator", lambda **kwargs: _NonFiniteJudge()
        )
        prompts = tmp_path / "prompts.jsonl"
        prompts.write_text('{"prompt":"private"}\n', encoding="utf-8")

        result = CliRunner().invoke(
            app,
            [
                "best-of-n",
                "--base",
                "model",
                "--prompts",
                str(prompts),
                "--n",
                "2",
                "--judge",
                "ollama://judge",
                "--output",
                "sft.jsonl",
                "--emit-pairs",
                "dpo.jsonl",
            ],
        )

        assert result.exit_code == 1
        assert isinstance(result.exception, ValueError)
        assert "candidate 0" in str(result.exception)
        assert "private" not in str(result.exception)
        assert not (tmp_path / "sft.jsonl").exists()
        assert not (tmp_path / "dpo.jsonl").exists()

    def test_plan_only(self):
        import os

        from typer.testing import CliRunner

        from soup_cli.commands.data import app

        path = os.path.join(os.getcwd(), "_bon_prompts_plan.jsonl")
        with open(path, "w", encoding="utf-8") as fh:
            fh.write('{"prompt": "hi"}\n')
        try:
            result = CliRunner().invoke(
                app,
                ["best-of-n", "--base", "m", "--prompts", path, "--n", "4",
                 "--judge", "ollama://m", "--plan-only"],
            )
            assert result.exit_code == 0, (result.output, repr(result.exception))
        finally:
            os.remove(path)

    def test_prompt_loader_fails_closed_with_line_number(self, monkeypatch, tmp_path):
        from typer.testing import CliRunner

        from soup_cli.commands.data import app

        monkeypatch.chdir(tmp_path)
        monkeypatch.setattr("soup_cli.eval.judge.JudgeEvaluator", lambda **kw: _ScoreJudge())
        invalid_rows = (
            "{not json}",
            "[]",
            '{"messages":[{"role":"assistant","content":"private answer"}]}',
            '{"prompt":""}',
            '{"prompt":"   "}',
        )

        for index, invalid_row in enumerate(invalid_rows):
            prompts = tmp_path / f"invalid-{index}.jsonl"
            prompts.write_text(
                '{"prompt":"valid private prompt"}\n' + invalid_row + "\n",
                encoding="utf-8",
            )
            result = CliRunner().invoke(
                app,
                [
                    "best-of-n",
                    "--base",
                    "model",
                    "--prompts",
                    str(prompts),
                    "--n",
                    "2",
                    "--judge",
                    "ollama://judge",
                    "--plan-only",
                ],
            )

            assert result.exit_code == 2
            assert "line 2" in _plain(result.output)
            assert "private" not in result.output

    def test_blank_lines_are_ignored_but_source_lines_remain_physical(
        self, monkeypatch, tmp_path
    ):
        from soup_cli.commands.data import _bon_load_prompt_records

        monkeypatch.chdir(tmp_path)
        prompts = tmp_path / "blank-lines.jsonl"
        prompts.write_text(
            '\n{"prompt":"first"}\n\n{"instruction":"second"}\n',
            encoding="utf-8",
        )

        assert _bon_load_prompt_records(str(prompts)) == [("first", 2), ("second", 4)]


# ---------------------------------------------------------------------------
# Task 9 — utils/evolve.py
# ---------------------------------------------------------------------------


class TestEvolve:
    def test_depth_round_produces_lineage(self):
        from soup_cli.utils.evolve import run_evolve

        # A generate_fn that returns a fresh, distinct instruction each call
        # (so no evolution is eliminated as "unchanged").
        counter = {"i": 0}

        def _gen(prompt):
            counter["i"] += 1
            return f"evolved instruction number {counter['i']}"

        rows = run_evolve(["write a poem"], "depth", 2, _gen)
        assert len(rows) == 2  # one per round
        assert rows[0].round == 1 and rows[1].round == 2
        assert rows[0].seed == "write a poem"
        assert rows[1].seed == "evolved instruction number 1"  # lineage chains
        assert all(r.strategy == "depth" for r in rows)

    def test_breadth_strategy(self):
        from soup_cli.utils.evolve import run_evolve

        rows = run_evolve(["x"], "breadth", 1, lambda p: "brand new instruction")
        assert rows and rows[0].strategy == "breadth"
        assert rows[0].instruction == "brand new instruction"

    def test_unchanged_is_eliminated(self):
        from soup_cli.utils.evolve import run_evolve

        rows = run_evolve(["same"], "depth", 1, lambda p: "same")  # echoes seed
        assert rows == []

    def test_empty_is_eliminated(self):
        from soup_cli.utils.evolve import run_evolve

        assert run_evolve(["x"], "depth", 1, lambda p: "   ") == []

    def test_meta_prompt_echo_eliminated(self):
        from soup_cli.utils.evolve import run_evolve

        rows = run_evolve(["x"], "depth", 1, lambda p: "#Given Prompt#: x")
        assert rows == []

    def test_bad_strategy(self):
        import pytest

        from soup_cli.utils.evolve import run_evolve

        with pytest.raises(ValueError, match="strategy"):
            run_evolve(["x"], "sideways", 1, lambda p: "y")

    def test_bad_rounds(self):
        import pytest

        from soup_cli.utils.evolve import run_evolve

        with pytest.raises(ValueError, match="rounds"):
            run_evolve(["x"], "depth", 0, lambda p: "y")
        with pytest.raises(ValueError, match="rounds"):
            run_evolve(["x"], "depth", 6, lambda p: "y")

    def test_full_elimination_carries_forward_seed(self):
        from soup_cli.utils.evolve import run_evolve

        calls = {"i": 0}

        def _gen(prompt):
            calls["i"] += 1
            # Round 1 echoes the seed (eliminated for all); round 2+ is valid.
            return "seedX" if calls["i"] == 1 else "evolved-real"

        rows = run_evolve(["seedX"], "depth", 2, _gen)
        assert len(rows) == 1
        assert rows[0].round == 2  # round 1 fully eliminated
        assert rows[0].seed == "seedX"  # original seed carried forward

    def test_evolve_instruction_renders_seed(self):
        from soup_cli.utils.evolve import evolve_instruction

        captured = {}

        def _gen(prompt):
            captured["prompt"] = prompt
            return "evolved"

        out = evolve_instruction("my seed", "depth", _gen)
        assert out == "evolved"
        assert "my seed" in captured["prompt"]

    def test_no_top_level_torch(self):
        import ast
        import pathlib

        import soup_cli

        tree = ast.parse(
            (pathlib.Path(soup_cli.__file__).parent / "utils" / "evolve.py").read_text(
                encoding="utf-8"
            )
        )
        top = {
            n.module.split(".")[0]
            for n in ast.walk(tree)
            if isinstance(n, ast.ImportFrom) and n.col_offset == 0 and n.module
        }
        assert "torch" not in top


# ---------------------------------------------------------------------------
# Task 10 — soup data evolve command
# ---------------------------------------------------------------------------


def _write_seeds(name):
    import os

    path = os.path.join(os.getcwd(), name)
    with open(path, "w", encoding="utf-8") as fh:
        fh.write('{"prompt": "write a poem"}\n{"instruction": "sort a list"}\n')
    return path


class TestEvolveCli:
    def test_help(self):
        from typer.testing import CliRunner

        from soup_cli.commands.data import app

        result = CliRunner().invoke(app, ["evolve", "--help"])
        assert result.exit_code == 0, (result.output, repr(result.exception))
        assert "evolve" in result.output.lower()

    def test_reject_bad_strategy(self):
        import os

        from typer.testing import CliRunner

        from soup_cli.commands.data import app

        path = _write_seeds("_evolve_badstrat.jsonl")
        try:
            result = CliRunner().invoke(
                app,
                ["evolve", "--input", path, "--provider", "ollama", "--model", "m",
                 "--strategy", "sideways", "-o", "o.jsonl"],
            )
            assert result.exit_code == 2, (result.output, repr(result.exception))
            assert "strategy" in result.output.lower()
        finally:
            os.remove(path)

    def test_reject_bad_rounds(self):
        import os

        from typer.testing import CliRunner

        from soup_cli.commands.data import app

        path = _write_seeds("_evolve_badrounds.jsonl")
        try:
            result = CliRunner().invoke(
                app,
                ["evolve", "--input", path, "--provider", "ollama", "--model", "m",
                 "--rounds", "9", "-o", "o.jsonl"],
            )
            assert result.exit_code == 2, (result.output, repr(result.exception))
            assert "rounds" in result.output.lower()
        finally:
            os.remove(path)

    def test_reject_anthropic(self):
        import os

        from typer.testing import CliRunner

        from soup_cli.commands.data import app

        path = _write_seeds("_evolve_anthropic.jsonl")
        try:
            result = CliRunner().invoke(
                app,
                ["evolve", "--input", path, "--provider", "anthropic", "--model", "m",
                 "-o", "o.jsonl"],
            )
            assert result.exit_code == 2, (result.output, repr(result.exception))
            assert "anthropic" in result.output.lower()
        finally:
            os.remove(path)

    def test_plan_only(self):
        import os

        from typer.testing import CliRunner

        from soup_cli.commands.data import app

        path = _write_seeds("_evolve_plan.jsonl")
        try:
            result = CliRunner().invoke(
                app,
                ["evolve", "--input", path, "--provider", "ollama", "--model", "m",
                 "--plan-only"],
            )
            assert result.exit_code == 0, (result.output, repr(result.exception))
        finally:
            os.remove(path)

    def test_reject_output_outside_cwd(self):
        import os

        from typer.testing import CliRunner

        from soup_cli.commands.data import app

        path = _write_seeds("_evolve_outcwd.jsonl")
        try:
            result = CliRunner().invoke(
                app,
                ["evolve", "--input", path, "--provider", "ollama", "--model", "m",
                 "-o", "../escape.jsonl"],
            )
            assert result.exit_code == 2, (result.output, repr(result.exception))
        finally:
            os.remove(path)

    def test_happy_path(self, monkeypatch):
        import json
        import os

        from typer.testing import CliRunner

        from soup_cli.commands.data import app

        counter = {"i": 0}

        def _fake_make(*args, **kwargs):
            def _gen(prompt):
                counter["i"] += 1
                return f"evolved {counter['i']}"

            return _gen

        monkeypatch.setattr("soup_cli.utils.magpie.make_magpie_generate_fn", _fake_make)

        ipath = _write_seeds("_evolve_ok_in.jsonl")
        opath = os.path.join(os.getcwd(), "_evolve_ok_out.jsonl")
        try:
            result = CliRunner().invoke(
                app,
                ["evolve", "--input", ipath, "--provider", "ollama", "--model", "m",
                 "--strategy", "depth", "--rounds", "1", "-o", opath],
            )
            assert result.exit_code == 0, (result.output, repr(result.exception))
            rows = [json.loads(x) for x in open(opath, encoding="utf-8") if x.strip()]
            assert len(rows) == 2  # 2 seeds, 1 round
            assert rows[0]["messages"][0]["role"] == "user"
            assert rows[0]["messages"][0]["content"].startswith("evolved")
            assert rows[0]["_evolve"]["strategy"] == "depth"
            assert rows[0]["_evolve"]["round"] == 1
        finally:
            for p in (ipath, opath):
                if os.path.exists(p):
                    os.remove(p)

    def test_malformed_seed_rows_remain_skipped(self, monkeypatch, tmp_path):
        import json

        from typer.testing import CliRunner

        from soup_cli.commands.data import app

        counter = {"i": 0}

        def _fake_make(*args, **kwargs):
            def _gen(prompt):
                counter["i"] += 1
                return f"evolved {counter['i']}"

            return _gen

        monkeypatch.chdir(tmp_path)
        monkeypatch.setattr("soup_cli.utils.magpie.make_magpie_generate_fn", _fake_make)
        seeds = tmp_path / "mixed-seeds.jsonl"
        seeds.write_text(
            '{"prompt":"good seed one"}\n'
            "{not valid json}\n"
            "[]\n"
            '{"messages":[{"role":"assistant","content":"no user seed"}]}\n'
            '{"prompt":"   "}\n'
            '{"instruction":"good seed two"}\n',
            encoding="utf-8",
        )
        output = tmp_path / "evolved.jsonl"

        result = CliRunner().invoke(
            app,
            [
                "evolve",
                "--input",
                str(seeds),
                "--provider",
                "ollama",
                "--model",
                "m",
                "--rounds",
                "1",
                "--output",
                str(output),
            ],
        )

        assert result.exit_code == 0, (result.output, repr(result.exception))
        rows = [json.loads(line) for line in output.read_text(encoding="utf-8").splitlines()]
        assert [row["_evolve"]["seed"] for row in rows] == [
            "good seed one",
            "good seed two",
        ]