Skip to content

armgit / Soup

Public Imported from GitHub
0

Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.

85.1 KB
Raw Blame
"""v0.53.7 — Data Forge + Pipeline live wave 1.

Tests for:
- #88  markdown heading-aware ingest split
- #112 decontaminate --benchmark-file
- #87  prompt_strategy live runtime
- #86  soup data preprocess live tokenize
- #111 soup data forge --judge-provider
- #75  QA log entry (file-presence assertion only)
- #106 utils/recipe_run.run_recipe live
- #105 utils/trainer_plugins.instantiate_trainer_plugins live
- #103 /v1/tools/{python, bash, web_search} live
- #102 /v1/messages on vLLM backend + streaming SSE
"""

from __future__ import annotations

import json
import sys
from pathlib import Path
from unittest.mock import MagicMock, patch

import pytest

# ----- #88 markdown heading split ----------------------------------------


class TestMarkdownHeadingSplit:
    def test_empty_string_returns_empty_list(self):
        from soup_cli.utils.data_pipeline import split_markdown_by_headings
        assert split_markdown_by_headings("") == []

    def test_preamble_only_returns_single_row_with_none_section(self):
        from soup_cli.utils.data_pipeline import split_markdown_by_headings
        rows = split_markdown_by_headings("first paragraph\nsecond line")
        assert len(rows) == 1
        assert rows[0]["section"] is None
        assert rows[0]["level"] is None
        assert "first paragraph" in rows[0]["text"]

    def test_three_headings_yield_three_sections(self):
        from soup_cli.utils.data_pipeline import split_markdown_by_headings
        md = (
            "# Intro\nIntro body\n## Sub\nSub body\n### Deep\nDeep body\n"
        )
        rows = split_markdown_by_headings(md)
        assert len(rows) == 3
        assert rows[0]["section"] == "Intro"
        assert rows[0]["level"] == 1
        assert rows[1]["section"] == "Sub"
        assert rows[1]["level"] == 2
        assert rows[2]["section"] == "Deep"
        assert rows[2]["level"] == 3

    def test_preamble_plus_headings_yield_preamble_row(self):
        from soup_cli.utils.data_pipeline import split_markdown_by_headings
        md = "preamble text\n# Heading\nbody"
        rows = split_markdown_by_headings(md)
        assert len(rows) == 2
        assert rows[0]["section"] is None
        assert rows[0]["level"] is None
        assert "preamble text" in rows[0]["text"]
        assert rows[1]["section"] == "Heading"
        assert rows[1]["level"] == 1

    def test_atx_levels_1_through_6_accepted(self):
        from soup_cli.utils.data_pipeline import split_markdown_by_headings
        md = "\n".join(f"{'#' * n} L{n}\nbody{n}" for n in range(1, 7))
        rows = split_markdown_by_headings(md)
        assert [r["level"] for r in rows] == [1, 2, 3, 4, 5, 6]

    def test_seven_hashes_not_a_heading(self):
        from soup_cli.utils.data_pipeline import split_markdown_by_headings
        rows = split_markdown_by_headings("####### not a heading\nbody")
        assert len(rows) == 1
        assert rows[0]["section"] is None

    def test_hash_without_space_not_a_heading(self):
        from soup_cli.utils.data_pipeline import split_markdown_by_headings
        rows = split_markdown_by_headings("#NoSpace\nbody")
        assert len(rows) == 1
        assert rows[0]["section"] is None

    def test_non_string_input_raises_typeerror(self):
        from soup_cli.utils.data_pipeline import split_markdown_by_headings
        with pytest.raises(TypeError):
            split_markdown_by_headings(123)
        with pytest.raises(TypeError):
            split_markdown_by_headings(None)

    def test_ingest_cli_emits_one_row_per_heading(self, tmp_path, monkeypatch):
        from typer.testing import CliRunner

        from soup_cli.cli import app

        monkeypatch.chdir(tmp_path)
        md_path = tmp_path / "doc.md"
        md_path.write_text(
            "# H1\nfirst\n## H2\nsecond\n## H3\nthird\n",
            encoding="utf-8",
        )
        out = tmp_path / "out.jsonl"
        runner = CliRunner()
        result = runner.invoke(
            app,
            ["data", "ingest", str(md_path), "-o", str(out)],
        )
        assert result.exit_code == 0, result.output
        lines = out.read_text(encoding="utf-8").strip().splitlines()
        assert len(lines) == 3
        first = json.loads(lines[0])
        assert first["section"] == "H1"
        assert first["level"] == 1

    def test_ingest_cli_preamble_creates_extra_row(self, tmp_path, monkeypatch):
        from typer.testing import CliRunner

        from soup_cli.cli import app

        monkeypatch.chdir(tmp_path)
        md_path = tmp_path / "doc.md"
        md_path.write_text(
            "preamble\n# Heading\nbody\n",
            encoding="utf-8",
        )
        out = tmp_path / "out.jsonl"
        runner = CliRunner()
        result = runner.invoke(
            app,
            ["data", "ingest", str(md_path), "-o", str(out)],
        )
        assert result.exit_code == 0, result.output
        rows = [json.loads(line) for line in out.read_text(
            encoding="utf-8"
        ).strip().splitlines()]
        assert len(rows) == 2
        assert rows[0]["section"] is None
        assert rows[1]["section"] == "Heading"


# ----- #112 decontaminate --benchmark-file -------------------------------


class TestDecontaminateBenchmarkFile:
    def test_loads_operator_corpus_and_filters(self, tmp_path, monkeypatch):
        from typer.testing import CliRunner

        from soup_cli.cli import app

        monkeypatch.chdir(tmp_path)
        # The decontaminate row text overlaps the benchmark text by n-grams.
        bench_path = tmp_path / "bench.jsonl"
        bench_path.write_text(
            json.dumps({"text": (
                "the quick brown fox jumps over the lazy dog "
                "many times in the morning sun every day"
            )}) + "\n",
            encoding="utf-8",
        )
        rows_path = tmp_path / "input.jsonl"
        rows_path.write_text(
            json.dumps({"text": (
                "the quick brown fox jumps over the lazy dog "
                "many times in the morning sun every day"
            )}) + "\n"
            + json.dumps({"text": "completely unrelated content here"}) + "\n",
            encoding="utf-8",
        )
        out_path = tmp_path / "clean.jsonl"
        runner = CliRunner()
        result = runner.invoke(
            app,
            [
                "data", "decontaminate",
                "-i", str(rows_path),
                "--benchmark-file", str(bench_path),
                "-o", str(out_path),
                "--n", "4",
                "--threshold", "0.3",
            ],
        )
        assert result.exit_code == 0, result.output
        kept = [
            json.loads(line)
            for line in out_path.read_text(encoding="utf-8").strip().splitlines()
        ]
        # Overlapping row removed, unrelated kept.
        assert len(kept) == 1
        assert "unrelated" in kept[0]["text"]

    def test_missing_benchmarks_and_file_exits(self, tmp_path, monkeypatch):
        from typer.testing import CliRunner

        from soup_cli.cli import app

        monkeypatch.chdir(tmp_path)
        rows_path = tmp_path / "input.jsonl"
        rows_path.write_text(json.dumps({"text": "hi"}) + "\n", encoding="utf-8")
        runner = CliRunner()
        result = runner.invoke(
            app,
            ["data", "decontaminate", "-i", str(rows_path), "-o", "out.jsonl"],
        )
        assert result.exit_code != 0

    def test_bad_benchmark_file_path_rejected(self, tmp_path, monkeypatch):
        from typer.testing import CliRunner

        from soup_cli.cli import app

        monkeypatch.chdir(tmp_path)
        rows_path = tmp_path / "input.jsonl"
        rows_path.write_text(json.dumps({"text": "hi"}) + "\n", encoding="utf-8")
        runner = CliRunner()
        result = runner.invoke(
            app,
            [
                "data", "decontaminate",
                "-i", str(rows_path),
                "--benchmark-file", "/etc/does-not-exist.jsonl",
                "-o", "out.jsonl",
            ],
        )
        assert result.exit_code != 0

    def test_benchmarks_flag_still_accepted_with_label(self, tmp_path, monkeypatch):
        from typer.testing import CliRunner

        from soup_cli.cli import app

        monkeypatch.chdir(tmp_path)
        rows_path = tmp_path / "input.jsonl"
        rows_path.write_text(json.dumps({"text": "any"}) + "\n", encoding="utf-8")
        out_path = tmp_path / "out.jsonl"
        runner = CliRunner()
        result = runner.invoke(
            app,
            [
                "data", "decontaminate",
                "-i", str(rows_path),
                "-b", "mmlu",
                "-o", str(out_path),
            ],
        )
        assert result.exit_code == 0, result.output


# ----- #87 prompt_strategy live runtime ----------------------------------


class TestPromptStrategyRuntime:
    def test_resolve_finds_callable(self):
        from soup_cli.utils.data_pipeline import resolve_prompt_strategy
        # json.dumps takes a positional arg
        spec = "json:dumps"
        fn = resolve_prompt_strategy(spec)
        assert callable(fn)

    def test_resolve_missing_module_raises(self):
        from soup_cli.utils.data_pipeline import resolve_prompt_strategy
        with pytest.raises(ValueError, match="could not be imported"):
            resolve_prompt_strategy("definitely_not_a_module_xyz:fn")

    def test_resolve_missing_attribute_raises(self):
        from soup_cli.utils.data_pipeline import resolve_prompt_strategy
        with pytest.raises(ValueError, match="no attribute"):
            resolve_prompt_strategy("json:not_a_real_attr_zzz")

    def test_resolve_non_callable_raises(self):
        from soup_cli.utils.data_pipeline import resolve_prompt_strategy
        # sys.maxsize is an int, not callable
        with pytest.raises(ValueError, match="not callable"):
            resolve_prompt_strategy("sys:maxsize")

    def test_resolve_bad_shape_raises(self):
        from soup_cli.utils.data_pipeline import resolve_prompt_strategy
        with pytest.raises(ValueError):
            resolve_prompt_strategy("no_colon")
        with pytest.raises(ValueError):
            resolve_prompt_strategy("")

    def test_resolve_non_string_raises(self):
        from soup_cli.utils.data_pipeline import resolve_prompt_strategy
        with pytest.raises(ValueError):
            resolve_prompt_strategy(123)  # type: ignore[arg-type]

    def test_apply_with_none_returns_row(self):
        from soup_cli.utils.data_pipeline import apply_prompt_strategy
        row = {"a": 1}
        assert apply_prompt_strategy(None, row) is row

    def test_apply_callable_transforms_row(self, tmp_path, monkeypatch):
        # Create a tiny module with an upper-casing transform
        mod_dir = tmp_path / "tfm_mod"
        mod_dir.mkdir()
        (mod_dir / "__init__.py").write_text("", encoding="utf-8")
        (mod_dir / "t.py").write_text(
            "def upper(row):\n"
            "    return {k: v.upper() if isinstance(v, str) else v for k, v in row.items()}\n",
            encoding="utf-8",
        )
        monkeypatch.syspath_prepend(str(tmp_path))
        # Clear lru_cache so spec resolves with new module
        from soup_cli.utils.data_pipeline import (
            apply_prompt_strategy,
            resolve_prompt_strategy,
        )
        resolve_prompt_strategy.cache_clear()
        spec = "tfm_mod.t:upper"
        out = apply_prompt_strategy(spec, {"text": "hi"})
        assert out["text"] == "HI"

    def test_apply_callable_exception_falls_through(self, tmp_path, monkeypatch):
        mod_dir = tmp_path / "tfm_mod2"
        mod_dir.mkdir()
        (mod_dir / "__init__.py").write_text("", encoding="utf-8")
        (mod_dir / "t.py").write_text(
            "def boom(row):\n    raise RuntimeError('nope')\n",
            encoding="utf-8",
        )
        monkeypatch.syspath_prepend(str(tmp_path))
        from soup_cli.utils.data_pipeline import (
            apply_prompt_strategy,
            resolve_prompt_strategy,
        )
        resolve_prompt_strategy.cache_clear()
        spec = "tfm_mod2.t:boom"
        row = {"text": "x"}
        out = apply_prompt_strategy(spec, row)
        assert out == row  # silent-degrade per CrossDocCollator policy

    def test_apply_non_mapping_return_falls_through(self, tmp_path, monkeypatch):
        mod_dir = tmp_path / "tfm_mod3"
        mod_dir.mkdir()
        (mod_dir / "__init__.py").write_text("", encoding="utf-8")
        (mod_dir / "t.py").write_text(
            "def to_str(row):\n    return 'not a mapping'\n",
            encoding="utf-8",
        )
        monkeypatch.syspath_prepend(str(tmp_path))
        from soup_cli.utils.data_pipeline import (
            apply_prompt_strategy,
            resolve_prompt_strategy,
        )
        resolve_prompt_strategy.cache_clear()
        row = {"text": "x"}
        out = apply_prompt_strategy("tfm_mod3.t:to_str", row)
        assert out == row

    def test_sft_format_threads_prompt_strategy(self, tmp_path, monkeypatch):
        """build_format_row routes through prompt_strategy when set."""
        mod_dir = tmp_path / "tfm_mod_sft"
        mod_dir.mkdir()
        (mod_dir / "__init__.py").write_text("", encoding="utf-8")
        (mod_dir / "t.py").write_text(
            "def attach(row):\n"
            "    return {**row, '_attached': True}\n",
            encoding="utf-8",
        )
        monkeypatch.syspath_prepend(str(tmp_path))
        from soup_cli.utils.data_pipeline import resolve_prompt_strategy
        resolve_prompt_strategy.cache_clear()

        from soup_cli.config.schema import DataConfig
        from soup_cli.data.sft_format import build_format_row

        data_cfg = DataConfig(
            train="dummy.jsonl",
            prompt_strategy="tfm_mod_sft.t:attach",
            train_on_responses_only=False,
            chat_template="chatml",
        )
        tokenizer = MagicMock()
        tokenizer.chat_template = "{% for msg in messages %}{{msg['content']}}{% endfor %}"
        # #785/#788: the legacy path now pre-tokenizes (apply_chat_template
        # tokenize=True returns ids). No EOS is appended here not because the mock
        # is uncallable (a MagicMock IS callable) but because its eos_token_id does
        # not resolve to an int, so append_training_eos is a no-op.
        tokenizer.apply_chat_template = MagicMock(return_value=[7, 8, 9])
        fn = build_format_row(tokenizer, data_cfg)
        # When invoked, the inner row should pass through the attach transform
        result = fn({"messages": [{"role": "user", "content": "hi"}]})
        # Verify the legacy formatter ran (pre-tokenized) through the wrapper.
        assert tokenizer.apply_chat_template.called
        assert result.get("input_ids") == [7, 8, 9]


# ----- #86 soup data preprocess live tokenize ----------------------------


class TestPreprocessTokenize:
    def test_help_advertises_live_loop(self):
        from typer.testing import CliRunner

        from soup_cli.cli import app

        runner = CliRunner()
        result = runner.invoke(app, ["data", "preprocess", "--help"])
        assert result.exit_code == 0

    def test_load_pretokenized_dataset_rejects_outside_cwd(self, tmp_path, monkeypatch):
        from soup_cli.utils.data_pipeline import load_pretokenized_dataset

        monkeypatch.chdir(tmp_path)
        # An absolute outside-cwd path
        with pytest.raises(ValueError, match="under cwd"):
            load_pretokenized_dataset("/etc/some_dir")

    def test_load_pretokenized_dataset_rejects_null_byte(self, tmp_path, monkeypatch):
        from soup_cli.utils.data_pipeline import load_pretokenized_dataset

        monkeypatch.chdir(tmp_path)
        with pytest.raises(ValueError, match="null byte"):
            load_pretokenized_dataset("foo\x00bar")

    def test_load_pretokenized_dataset_rejects_empty(self):
        from soup_cli.utils.data_pipeline import load_pretokenized_dataset
        with pytest.raises(ValueError, match="non-empty"):
            load_pretokenized_dataset("")

    def test_load_pretokenized_dataset_rejects_non_string(self):
        from soup_cli.utils.data_pipeline import load_pretokenized_dataset
        with pytest.raises(TypeError, match="string"):
            load_pretokenized_dataset(123)  # type: ignore[arg-type]

    def test_load_pretokenized_dataset_rejects_missing_path(self, tmp_path, monkeypatch):
        from soup_cli.utils.data_pipeline import load_pretokenized_dataset

        monkeypatch.chdir(tmp_path)
        with pytest.raises(ValueError, match="not found"):
            load_pretokenized_dataset("nonexistent_dir")

    @pytest.mark.requires_symlink
    def test_load_pretokenized_dataset_rejects_symlink(self, tmp_path, monkeypatch):
        from soup_cli.utils.data_pipeline import load_pretokenized_dataset

        monkeypatch.chdir(tmp_path)
        real_dir = tmp_path / "real"
        real_dir.mkdir()
        link = tmp_path / "link"
        link.symlink_to(real_dir)
        with pytest.raises(ValueError, match="symlink"):
            load_pretokenized_dataset(str(link))

    def test_load_pretokenized_dataset_rejects_cache_key_mismatch(
        self, tmp_path, monkeypatch
    ):
        from soup_cli.utils.data_pipeline import load_pretokenized_dataset

        pytest.importorskip("datasets")
        monkeypatch.chdir(tmp_path)
        target = tmp_path / "tokenized"
        target.mkdir()
        (target / "metadata.json").write_text(
            json.dumps({"cache_key": "DIFFERENT"}), encoding="utf-8"
        )
        with pytest.raises(ValueError, match="cache_key mismatch"):
            load_pretokenized_dataset(
                str(target), expected_cache_key="EXPECTED"
            )


# ----- #111 forge --judge-provider ---------------------------------------


class TestForgeJudgeProvider:
    def test_make_judge_provider_fn_unknown_rejected(self):
        from soup_cli.utils.data_forge import make_judge_provider_fn
        with pytest.raises(ValueError, match="unknown judge provider"):
            make_judge_provider_fn("openai")

    def test_make_judge_provider_fn_non_string_rejected(self):
        from soup_cli.utils.data_forge import make_judge_provider_fn
        with pytest.raises(TypeError):
            make_judge_provider_fn(123)  # type: ignore[arg-type]

    def test_make_judge_provider_fn_bad_model_rejected(self):
        from soup_cli.utils.data_forge import make_judge_provider_fn
        with pytest.raises(ValueError, match="model"):
            make_judge_provider_fn("ollama", model="")
        with pytest.raises(ValueError, match="NUL-free"):
            make_judge_provider_fn("ollama", model="x\x00y")

    def test_make_judge_provider_fn_bool_timeout_rejected(self):
        from soup_cli.utils.data_forge import make_judge_provider_fn
        with pytest.raises(TypeError):
            make_judge_provider_fn("ollama", timeout_seconds=True)

    def test_make_judge_provider_fn_bad_timeout_rejected(self):
        from soup_cli.utils.data_forge import make_judge_provider_fn
        with pytest.raises(ValueError):
            make_judge_provider_fn("ollama", timeout_seconds=0)
        with pytest.raises(ValueError):
            make_judge_provider_fn("ollama", timeout_seconds=1000)

    def test_anthropic_requires_env_var(self, monkeypatch):
        from soup_cli.utils.data_forge import make_judge_provider_fn
        monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False)
        with pytest.raises(ValueError, match="ANTHROPIC_API_KEY"):
            make_judge_provider_fn("anthropic")

    def test_ollama_rejects_remote_url(self):
        from soup_cli.utils.data_forge import make_judge_provider_fn
        with pytest.raises(ValueError):
            make_judge_provider_fn(
                "ollama", base_url="http://10.0.0.1:11434"
            )

    def test_ollama_judge_returns_text_on_success(self, monkeypatch):
        # Monkeypatch httpx.post in the module
        import httpx

        from soup_cli.utils.data_forge import make_judge_provider_fn

        mock_response = MagicMock()
        mock_response.status_code = 200
        mock_response.json.return_value = {
            "choices": [{"message": {"content": "hello world"}}]
        }
        monkeypatch.setattr(httpx, "post", lambda *a, **k: mock_response)
        judge = make_judge_provider_fn("ollama")
        reply = judge("prompt")
        assert reply["text"] == "hello world"

    def test_ollama_judge_swallows_http_error(self, monkeypatch):
        import httpx

        from soup_cli.utils.data_forge import make_judge_provider_fn

        def _boom(*a, **k):
            raise httpx.ConnectError("nope")

        monkeypatch.setattr(httpx, "post", _boom)
        judge = make_judge_provider_fn("ollama")
        reply = judge("prompt")
        assert reply["text"] == ""

    def test_vllm_judge_validates_url(self):
        from soup_cli.utils.data_forge import make_judge_provider_fn
        with pytest.raises(ValueError):
            make_judge_provider_fn(
                "vllm", base_url="ftp://localhost:8000"
            )

    def test_forge_provider_constants(self):
        from soup_cli.utils.data_forge import JUDGE_PROVIDERS
        assert JUDGE_PROVIDERS == frozenset({"ollama", "anthropic", "vllm"})

    def test_forge_cli_rejects_unknown_provider(self, tmp_path, monkeypatch):
        from typer.testing import CliRunner

        from soup_cli.cli import app

        monkeypatch.chdir(tmp_path)
        docs = tmp_path / "docs"
        docs.mkdir()
        (docs / "a.txt").write_text("hello world", encoding="utf-8")
        runner = CliRunner()
        result = runner.invoke(
            app,
            [
                "data", "forge",
                "--docs", str(docs),
                "--task", "sft",
                "--target-rows", "1",
                "--judge-provider", "openai",
                "-o", "out.jsonl",
                "-p", "prov.json",
            ],
        )
        assert result.exit_code == 2


# ----- #75 QA log entry --------------------------------------------------


class TestQALogEntry:
    def test_qa_log_mentions_v0537(self):
        qa_log = (
            Path(__file__).parent / "qa" / "v053_qa.md"
        )
        assert qa_log.is_file()
        text = qa_log.read_text(encoding="utf-8")
        assert "v0.53.7" in text
        assert "#75" in text


# ----- #106 run_recipe live ----------------------------------------------


class TestRunRecipeLive:
    def test_stub_gone(self, tmp_path, monkeypatch):
        from soup_cli.utils.recipe_dag import parse_recipe
        from soup_cli.utils.recipe_run import run_recipe

        monkeypatch.chdir(tmp_path)
        dag = parse_recipe(
            {
                "nodes": [
                    {"name": "seed1", "kind": "seed", "config": {"path": "doesnotexist.jsonl"}},
                ],
                "edges": [],
            }
        )
        # Should NOT raise NotImplementedError now (was the v0.53.6 stub).
        # Instead should fail on missing seed path.
        with pytest.raises(ValueError):
            run_recipe(dag, output_dir=str(tmp_path / "out"))

    def test_seed_node_loads_jsonl(self, tmp_path, monkeypatch):
        from soup_cli.utils.recipe_dag import parse_recipe
        from soup_cli.utils.recipe_run import run_recipe

        monkeypatch.chdir(tmp_path)
        seed_path = tmp_path / "seed.jsonl"
        seed_path.write_text(
            json.dumps({"text": "a"}) + "\n" + json.dumps({"text": "b"}) + "\n",
            encoding="utf-8",
        )
        out_dir = tmp_path / "out"
        dag = parse_recipe(
            {
                "nodes": [
                    {"name": "seed1", "kind": "seed", "config": {"path": str(seed_path)}},
                    {"name": "sink", "kind": "sampler", "config": {}},
                ],
                "edges": [["seed1", "sink"]],
            }
        )
        result = run_recipe(dag, output_dir=str(out_dir))
        assert result["status"] == "completed"
        assert "seed1" in result["completed_nodes"]
        assert "sink" in result["completed_nodes"]
        # sampler wrote the rows
        sink_jsonl = out_dir / "sink.jsonl"
        assert sink_jsonl.is_file()
        lines = sink_jsonl.read_text(encoding="utf-8").strip().splitlines()
        assert len(lines) == 2

    def test_validator_node_regex_filter(self, tmp_path, monkeypatch):
        from soup_cli.utils.recipe_dag import parse_recipe
        from soup_cli.utils.recipe_run import run_recipe

        monkeypatch.chdir(tmp_path)
        seed_path = tmp_path / "seed.jsonl"
        seed_path.write_text(
            json.dumps({"text": "hello world"}) + "\n"
            + json.dumps({"text": "skip me"}) + "\n",
            encoding="utf-8",
        )
        out_dir = tmp_path / "out"
        dag = parse_recipe(
            {
                "nodes": [
                    {"name": "seed1", "kind": "seed", "config": {"path": str(seed_path)}},
                    {"name": "vchk", "kind": "validator", "config": {"regex": "hello"}},
                    {"name": "sink", "kind": "sampler", "config": {}},
                ],
                "edges": [["seed1", "vchk"], ["vchk", "sink"]],
            }
        )
        result = run_recipe(dag, output_dir=str(out_dir))
        assert result["node_row_counts"]["vchk"] == 1
        assert result["node_row_counts"]["sink"] == 1

    def test_llm_text_explicit_offline_stub(self, tmp_path, monkeypatch):
        from soup_cli.utils.recipe_dag import parse_recipe
        from soup_cli.utils.recipe_run import run_recipe

        monkeypatch.chdir(tmp_path)
        seed_path = tmp_path / "seed.jsonl"
        seed_path.write_text(json.dumps({"text": "x"}) + "\n", encoding="utf-8")
        out_dir = tmp_path / "out"
        dag = parse_recipe(
            {
                "nodes": [
                    {"name": "seed1", "kind": "seed", "config": {"path": str(seed_path)}},
                    {
                        "name": "llm",
                        "kind": "llm_text",
                        "config": {"prompt": "tell me about {text}"},
                    },
                    {"name": "sink", "kind": "sampler", "config": {}},
                ],
                "edges": [["seed1", "llm"], ["llm", "sink"]],
            }
        )
        result = run_recipe(dag, output_dir=str(out_dir), offline=True)
        assert result["status"] == "completed"
        # Offline stub injects llm column
        sink = (out_dir / "sink.jsonl").read_text(encoding="utf-8").strip().splitlines()
        first = json.loads(sink[0])
        assert "llm" in first

    def test_judge_node_explicit_offline_keeps_all(self, tmp_path, monkeypatch):
        from soup_cli.utils.recipe_dag import parse_recipe
        from soup_cli.utils.recipe_run import run_recipe

        monkeypatch.chdir(tmp_path)
        seed_path = tmp_path / "seed.jsonl"
        seed_path.write_text(
            json.dumps({"text": "a"}) + "\n" + json.dumps({"text": "b"}) + "\n",
            encoding="utf-8",
        )
        out_dir = tmp_path / "out"
        dag = parse_recipe(
            {
                "nodes": [
                    {"name": "seed1", "kind": "seed", "config": {"path": str(seed_path)}},
                    {"name": "j", "kind": "judge", "config": {}},
                    {"name": "sink", "kind": "sampler", "config": {}},
                ],
                "edges": [["seed1", "j"], ["j", "sink"]],
            }
        )
        result = run_recipe(dag, output_dir=str(out_dir), offline=True)
        assert result["node_row_counts"]["j"] == 2

    def test_checkpoint_written_on_completion(self, tmp_path, monkeypatch):
        from soup_cli.utils.recipe_dag import parse_recipe
        from soup_cli.utils.recipe_run import run_recipe

        monkeypatch.chdir(tmp_path)
        seed_path = tmp_path / "seed.jsonl"
        seed_path.write_text(json.dumps({"text": "x"}) + "\n", encoding="utf-8")
        out_dir = tmp_path / "out"
        dag = parse_recipe(
            {
                "nodes": [
                    {"name": "seed1", "kind": "seed", "config": {"path": str(seed_path)}},
                ],
                "edges": [],
            }
        )
        run_recipe(dag, output_dir=str(out_dir))
        cp = out_dir / ".checkpoint.json"
        assert cp.is_file()
        state = json.loads(cp.read_text(encoding="utf-8"))
        assert state["status"] == "completed"
        assert "seed1" in state["completed_nodes"]

    def test_rejects_non_recipedag(self, tmp_path, monkeypatch):
        from soup_cli.utils.recipe_run import run_recipe
        monkeypatch.chdir(tmp_path)
        with pytest.raises(TypeError):
            run_recipe("not a dag", output_dir="x")  # type: ignore[arg-type]

    def test_output_dir_outside_cwd_rejected(self, tmp_path, monkeypatch):
        from soup_cli.utils.recipe_dag import parse_recipe
        from soup_cli.utils.recipe_run import run_recipe

        monkeypatch.chdir(tmp_path)
        dag = parse_recipe({
            "nodes": [{"name": "n1", "kind": "sampler", "config": {}}],
            "edges": [],
        })
        with pytest.raises(ValueError, match="under cwd"):
            run_recipe(dag, output_dir="/etc/foo")

    def test_validator_requires_regex_or_schema(self, tmp_path, monkeypatch):
        from soup_cli.utils.recipe_dag import parse_recipe
        from soup_cli.utils.recipe_run import run_recipe

        monkeypatch.chdir(tmp_path)
        seed_path = tmp_path / "seed.jsonl"
        seed_path.write_text(json.dumps({"text": "x"}) + "\n", encoding="utf-8")
        out_dir = tmp_path / "out"
        dag = parse_recipe({
            "nodes": [
                {"name": "seed1", "kind": "seed", "config": {"path": str(seed_path)}},
                {"name": "v", "kind": "validator", "config": {}},
            ],
            "edges": [["seed1", "v"]],
        })
        with pytest.raises(ValueError, match="regex.*schema"):
            run_recipe(dag, output_dir=str(out_dir))

    def test_code_node_requires_code_field(self, tmp_path, monkeypatch):
        from soup_cli.utils.recipe_dag import parse_recipe
        from soup_cli.utils.recipe_run import run_recipe

        monkeypatch.chdir(tmp_path)
        seed_path = tmp_path / "seed.jsonl"
        seed_path.write_text(json.dumps({"text": "x"}) + "\n", encoding="utf-8")
        out_dir = tmp_path / "out"
        dag = parse_recipe({
            "nodes": [
                {"name": "seed1", "kind": "seed", "config": {"path": str(seed_path)}},
                {"name": "c", "kind": "code", "config": {}},
            ],
            "edges": [["seed1", "c"]],
        })
        with pytest.raises(ValueError, match="code"):
            run_recipe(dag, output_dir=str(out_dir))

    def test_recipe_cli_executes_dag(self, tmp_path, monkeypatch):
        from typer.testing import CliRunner

        from soup_cli.cli import app

        monkeypatch.chdir(tmp_path)
        seed_path = tmp_path / "seed.jsonl"
        seed_path.write_text(json.dumps({"text": "x"}) + "\n", encoding="utf-8")
        recipe_path = tmp_path / "recipe.yaml"
        recipe_path.write_text(
            "nodes:\n"
            f"  - {{name: seed1, kind: seed, config: {{path: {seed_path.name}}}}}\n"
            "  - {name: sink, kind: sampler, config: {}}\n"
            "edges:\n"
            "  - [seed1, sink]\n",
            encoding="utf-8",
        )
        runner = CliRunner()
        result = runner.invoke(
            app,
            [
                "data", "recipe",
                str(recipe_path),
                "--execute",
                "--output", "out",
            ],
        )
        assert result.exit_code == 0, result.output
        # v0.53.7 marker should NOT appear in success path
        assert "v0.53.7" not in result.output or "Re-run" not in result.output


# ----- #105 instantiate_trainer_plugins live ----------------------------


class TestInstantiateTrainerPlugins:
    def test_cce_plugin_returns_advisory_callback(self):
        from soup_cli.utils.trainer_plugins import instantiate_trainer_plugins

        out = instantiate_trainer_plugins(["cce_plugin"])
        assert len(out) == 1
        cb = out[0]
        assert hasattr(cb, "plugin_name")
        assert cb.plugin_name == "cce_plugin"

    def test_unknown_name_rejected_before_instantiation(self):
        from soup_cli.utils.trainer_plugins import instantiate_trainer_plugins
        with pytest.raises(ValueError):
            instantiate_trainer_plugins(["definitely_unknown_plugin"])

    def test_non_sequence_rejected(self):
        from soup_cli.utils.trainer_plugins import instantiate_trainer_plugins
        with pytest.raises(TypeError):
            instantiate_trainer_plugins("grokfast")  # bare string

    def test_empty_list_returns_empty_tuple(self):
        from soup_cli.utils.trainer_plugins import instantiate_trainer_plugins
        out = instantiate_trainer_plugins([])
        assert out == ()

    def test_no_longer_raises_notimplementederror_for_cce(self):
        from soup_cli.utils.trainer_plugins import instantiate_trainer_plugins
        # cce_plugin has no required upstream dep; should NOT raise.
        out = instantiate_trainer_plugins(["cce_plugin"])
        assert out  # non-empty

    def test_missing_upstream_pkg_raises_importerror(self):
        """grokfast / spectrum etc. may not be installed in CI."""
        from soup_cli.utils.trainer_plugins import instantiate_trainer_plugins
        # If the package is installed, this test asserts no ImportError.
        # If absent (the common case), ImportError fires with friendly hint.
        try:
            out = instantiate_trainer_plugins(["grokfast"])
            # If installed, just verify we got something callable / module-like
            assert out
        except ImportError as exc:
            assert "grokfast" in str(exc)
            assert "pip install" in str(exc)


# ----- #103 /v1/tools/{python,bash,web_search} live ---------------------


# /v1/tools/python and /v1/tools/bash answer 401 without a token on every bind
# (tests/test_serve_code_tool_token.py), so the apps below are built with one by
# default and the live-route tests send it.
_TOOL_TOKEN = "tool-test-token"
_TOOL_AUTH = {"Authorization": f"Bearer {_TOOL_TOKEN}"}


def _create_test_app(host: str = "127.0.0.1", auth_token: str | None = _TOOL_TOKEN, **kwargs):
    """Build a test FastAPI app via _create_app (lazy fastapi import)."""
    try:
        import fastapi  # noqa: F401
    except ImportError:
        pytest.skip("FastAPI not installed")

    from soup_cli.commands.serve import _create_app

    return _create_app(
        model_obj=MagicMock(),
        tokenizer=MagicMock(),
        device="cpu",
        model_name="test-model",
        max_tokens_default=128,
        host=host,
        auth_token=auth_token,
        **kwargs,
    )


def _allow_live_python_tool(monkeypatch) -> None:
    """Let a live /v1/tools/python run proceed whatever this host's isolation.

    The route answers 501 under the ``best-effort`` isolation strategy (Windows,
    Linux without ``os.unshare``) and when the namespace ``unshare`` fails at run
    time (a kernel or container that refuses unprivileged user namespaces).
    These tests exercise the live sandbox run, not the isolation, so the
    ``unshare`` the sandbox child calls is a no-op that succeeds.
    """
    import os

    from soup_cli.trainer import rewards

    if rewards._get_isolation_strategy() == "best-effort":
        monkeypatch.setattr(rewards, "_get_isolation_strategy", lambda: "namespaces")
    monkeypatch.setattr(os, "unshare", lambda *args, **kwargs: None, raising=False)


class TestToolEndpointsLive:
    @pytest.fixture(autouse=True)
    def _require_fastapi(self):
        pytest.importorskip("fastapi")
        pytest.importorskip("httpx")

    def test_python_tool_runs_simple_code(self, monkeypatch):
        from fastapi.testclient import TestClient
        _allow_live_python_tool(monkeypatch)
        app = _create_test_app()
        client = TestClient(app, base_url="http://127.0.0.1", headers=_TOOL_AUTH)
        resp = client.post(
            "/v1/tools/python",
            json={"code": "print('hello')"},
        )
        # The sandbox may print warnings on first run; should still return 200.
        assert resp.status_code == 200, resp.text
        data = resp.json()
        assert "stdout" in data
        assert "exit_code" in data
        assert "timed_out" in data

    def test_python_tool_rejects_missing_code(self):
        from fastapi.testclient import TestClient
        app = _create_test_app()
        client = TestClient(app, base_url="http://127.0.0.1", headers=_TOOL_AUTH)
        resp = client.post("/v1/tools/python", json={})
        assert resp.status_code == 400

    def test_python_tool_rejects_oversize_code(self):
        from fastapi.testclient import TestClient
        app = _create_test_app()
        client = TestClient(app, base_url="http://127.0.0.1", headers=_TOOL_AUTH)
        oversize = "x" * (64 * 1024 + 1)
        resp = client.post("/v1/tools/python", json={"code": oversize})
        assert resp.status_code == 400

    def test_python_tool_non_dict_rejected(self):
        from fastapi.testclient import TestClient
        app = _create_test_app()
        client = TestClient(app, base_url="http://127.0.0.1", headers=_TOOL_AUTH)
        # FastAPI body parsing accepts list as dict-typed param? It coerces.
        # Use empty code which is rejected as ValueError-equivalent.
        resp = client.post("/v1/tools/python", json={"code": ""})
        assert resp.status_code == 400

    def test_bash_tool_executes_simple_command(self, monkeypatch):
        if sys.platform == "win32":
            pytest.skip("bash sandbox not supported on Windows")
        from fastapi.testclient import TestClient

        from soup_cli.trainer.rewards import SandboxProcessResult

        app = _create_test_app()
        client = TestClient(app, base_url="http://127.0.0.1", headers=_TOOL_AUTH)
        monkeypatch.setattr(
            "soup_cli.trainer.rewards._get_isolation_strategy", lambda: "namespaces"
        )
        monkeypatch.setattr(
            "soup_cli.trainer.rewards._run_bash_sandbox",
            lambda cmd: SandboxProcessResult(returncode=0, stdout="hello\n", stderr=""),
        )
        resp = client.post(
            "/v1/tools/bash",
            json={"command": "echo hello"},
        )
        assert resp.status_code == 200, resp.text
        data = resp.json()
        assert data["stdout"] == "hello\n"
        assert data["exit_code"] == 0
        assert not data["timed_out"]

    def test_bash_tool_rejects_missing_command(self):
        from fastapi.testclient import TestClient
        app = _create_test_app()
        client = TestClient(app, base_url="http://127.0.0.1", headers=_TOOL_AUTH)
        resp = client.post("/v1/tools/bash", json={})
        assert resp.status_code == 400

    def test_bash_tool_rejects_oversize_command(self):
        from fastapi.testclient import TestClient
        app = _create_test_app()
        client = TestClient(app, base_url="http://127.0.0.1", headers=_TOOL_AUTH)
        oversize = "x" * (64 * 1024 + 1)
        resp = client.post("/v1/tools/bash", json={"command": oversize})
        assert resp.status_code == 400

    def test_bash_tool_times_out(self, monkeypatch):
        if sys.platform == "win32":
            pytest.skip("bash sandbox not supported on Windows")
        from fastapi.testclient import TestClient

        from soup_cli.trainer.rewards import SandboxProcessResult

        app = _create_test_app()
        client = TestClient(app, base_url="http://127.0.0.1", headers=_TOOL_AUTH)
        monkeypatch.setattr(
            "soup_cli.trainer.rewards._get_isolation_strategy", lambda: "namespaces"
        )
        monkeypatch.setattr(
            "soup_cli.trainer.rewards._run_bash_sandbox",
            lambda cmd: SandboxProcessResult(returncode=None, stdout="", stderr="", timed_out=True),
        )
        resp = client.post(
            "/v1/tools/bash",
            json={"command": "sleep 10"},
        )
        assert resp.status_code == 200
        data = resp.json()
        assert data["timed_out"] is True
        assert data["exit_code"] == 124

    def test_bash_tool_blocks_network_namespace(self):
        if sys.platform != "linux":
            pytest.skip("network namespace test only valid on Linux")
        from soup_cli.trainer.rewards import _get_isolation_strategy, _run_bash_sandbox
        if _get_isolation_strategy() != "namespaces":
            pytest.skip("namespaces not available")

        cmd = "curl -s --connect-timeout 1 http://169.254.169.254/ || echo 'BLOCKED'"
        result = _run_bash_sandbox(cmd)
        assert "BLOCKED" in result.stdout or result.returncode != 0

    def test_bash_tool_windows_returns_501(self, monkeypatch):
        from fastapi.testclient import TestClient
        app = _create_test_app()
        client = TestClient(app, base_url="http://127.0.0.1", headers=_TOOL_AUTH)
        monkeypatch.setattr(
            "soup_cli.trainer.rewards._get_isolation_strategy", lambda: "best-effort"
        )
        resp = client.post(
            "/v1/tools/bash",
            json={"command": "echo hello"},
        )
        assert resp.status_code == 501
        assert "OS-level isolation" in resp.json()["detail"]

    def test_bash_tool_auth_required_on_non_loopback_host(self):
        from fastapi.testclient import TestClient
        app = _create_test_app(host="0.0.0.0", auth_token=None)
        client = TestClient(app, base_url="http://127.0.0.1")
        resp = client.post(
            "/v1/tools/bash",
            json={"command": "echo hello"},
        )
        assert resp.status_code == 401
        assert "Authentication required" in resp.json()["detail"]

    def test_bash_tool_bearer_auth_validation(self, monkeypatch):
        from fastapi.testclient import TestClient

        from soup_cli.trainer.rewards import SandboxProcessResult

        app = _create_test_app(host="0.0.0.0", auth_token="secret123")
        client = TestClient(app, base_url="http://127.0.0.1")
        # Invalid token -> 401
        resp = client.post(
            "/v1/tools/bash",
            json={"command": "echo hello"},
            headers={"Authorization": "Bearer wrong"},
        )
        assert resp.status_code == 401

        # Valid token -> 200 (mocked sandbox)
        monkeypatch.setattr(
            "soup_cli.trainer.rewards._get_isolation_strategy", lambda: "namespaces"
        )
        monkeypatch.setattr(
            "soup_cli.trainer.rewards._run_bash_sandbox",
            lambda cmd: SandboxProcessResult(returncode=0, stdout="ok", stderr=""),
        )
        resp = client.post(
            "/v1/tools/bash",
            json={"command": "echo hello"},
            headers={"Authorization": "Bearer secret123"},
        )
        assert resp.status_code == 200

    def test_bash_tool_bounded_streaming_kills_massive_output(self):
        if sys.platform == "win32":
            pytest.skip("bash sandbox not supported on Windows")
        from soup_cli.trainer.rewards import _run_sandboxed_subprocess
        # Generate >100KB of output, should be killed and output_exceeded set
        argv = ["python3", "-c", "import sys; sys.stdout.write('A' * 200_000); sys.stdout.flush()"]
        result = _run_sandboxed_subprocess(argv, max_output_bytes=10_000)
        assert result.output_exceeded is True
        assert "exceeded limit" in result.stderr

    def test_bash_tool_environment_secret_isolation(self, monkeypatch):
        if sys.platform == "win32":
            pytest.skip("bash sandbox not supported on Windows")
        from soup_cli.trainer.rewards import _run_sandboxed_subprocess
        monkeypatch.setenv("SECRET_TOKEN", "supersecret12345")
        argv = ["python3", "-c", "import os; print(os.environ.get('SECRET_TOKEN', 'ISOLATED'))"]
        result = _run_sandboxed_subprocess(argv)
        assert result.stdout == "ISOLATED"

    def test_bash_sandbox_raises_on_strict_namespace_failure(self, monkeypatch):
        import os
        import shutil
        import subprocess
        import sys

        # Cross-platform mock: trick the sandbox into thinking it's Linux and has namespaces
        monkeypatch.setattr(sys, "platform", "linux")
        monkeypatch.setattr(
            "soup_cli.trainer.rewards._get_isolation_strategy",
            lambda: "namespaces",
        )

        # Prevent preexec_fn from applying CPU/Memory limits to the pytest test runner!
        try:
            import resource
            monkeypatch.setattr(resource, "setrlimit", lambda *args: None)
        except ImportError:
            pass

        def mock_unshare(*args, **kwargs):
            raise PermissionError("Operation not permitted")

        monkeypatch.setattr(os, "unshare", mock_unshare, raising=False)

        # Mock Popen to execute preexec_fn and mimic POSIX behavior by raising SubprocessError

        def mock_popen(*args, **kwargs):
            preexec = kwargs.get("preexec_fn")
            if preexec:
                try:
                    preexec()
                except Exception as e:
                    raise subprocess.SubprocessError(f"Exception occurred in preexec_fn: {e}")
            raise OSError("Mock prevents actual execution")

        monkeypatch.setattr(subprocess, "Popen", mock_popen)
        monkeypatch.setattr(shutil, "which", lambda x: "/bin/bash")

        from soup_cli.trainer.rewards import _run_bash_sandbox

        with pytest.raises(PermissionError, match="Operation not permitted"):
            _run_bash_sandbox("echo hello")

    def test_bash_sandbox_runs_on_strict_namespace_success(self, monkeypatch):
        import os
        import sys
        if sys.platform == "win32":
            pytest.skip("bash sandbox not supported on Windows")
        if sys.platform != "linux":
            pytest.skip("unshare is a Linux-only syscall")

        monkeypatch.setattr(
            "soup_cli.trainer.rewards._get_isolation_strategy",
            lambda: "namespaces",
        )
        monkeypatch.setattr(os, "unshare", lambda *args, **kwargs: None, raising=False)

        from soup_cli.trainer.rewards import _run_bash_sandbox

        result = _run_bash_sandbox("echo hello")
        assert result.returncode == 0
        assert "hello" in result.stdout

    def test_bash_sandbox_requests_strict_namespaces_by_name(self, monkeypatch):
        """The preexec hook must ask for STRICT namespace isolation, by keyword.

        Every other test on this path asserts on an exception message raised
        three wrappings below the call, so rewording any of those messages would
        leave the suite green while the sandbox quietly degraded to best-effort
        isolation. `strict_namespaces` appeared nowhere in tests/ before this.

        Added by the maintainer after merge; raised twice in review and
        deferred, which was the wrong call for a guard on an endpoint that
        executes shell commands.
        """
        import sys

        if sys.platform == "win32":
            pytest.skip("bash sandbox is not supported on Windows")

        from soup_cli.trainer import rewards

        captured: dict = {}

        def _record_rlimit(*args, **kwargs):
            captured["args"] = args
            captured["kwargs"] = kwargs

        class _Result:
            launch_failed = False
            returncode = 0
            stdout = ""
            stderr = ""

        def _capture_preexec(argv, preexec):
            captured["argv"] = argv
            preexec()
            return _Result()

        monkeypatch.setattr(rewards, "_apply_rlimit", _record_rlimit)
        monkeypatch.setattr(rewards, "_run_sandboxed_subprocess", _capture_preexec)

        rewards._run_bash_sandbox("echo hello")

        assert captured["kwargs"] == {"strict_namespaces": True}, (
            "the bash sandbox must request strict namespace isolation by "
            f"keyword; got args={captured.get('args')} "
            f"kwargs={captured.get('kwargs')}"
        )

    def test_bash_tool_restricted_linux_fails_closed_501(self, monkeypatch):
        from fastapi.testclient import TestClient
        app = _create_test_app()
        client = TestClient(app, base_url="http://127.0.0.1", headers=_TOOL_AUTH)
        monkeypatch.setattr(
            "soup_cli.trainer.rewards._get_isolation_strategy", lambda: "namespaces"
        )
        def _failing_bash(cmd):
            raise PermissionError("unshare failed: Operation not permitted")
        monkeypatch.setattr(
            "soup_cli.trainer.rewards._run_bash_sandbox",
            _failing_bash,
        )
        resp = client.post(
            "/v1/tools/bash",
            json={"command": "echo hello"},
        )
        assert resp.status_code == 501
        assert "Operation not permitted" in resp.json()["detail"]

    def test_bash_tool_subprocess_error_maps_to_501(self, monkeypatch):
        import subprocess

        from fastapi.testclient import TestClient
        app = _create_test_app()
        client = TestClient(app, base_url="http://127.0.0.1", headers=_TOOL_AUTH)
        monkeypatch.setattr(
            "soup_cli.trainer.rewards._get_isolation_strategy", lambda: "namespaces"
        )
        def _failing_bash(cmd):
            raise subprocess.SubprocessError(
                "Exception occurred in preexec_fn: unshare namespace failed"
            )
        monkeypatch.setattr(
            "soup_cli.trainer.rewards._run_bash_sandbox",
            _failing_bash,
        )
        resp = client.post(
            "/v1/tools/bash",
            json={"command": "echo hello"},
        )
        assert resp.status_code == 501
        assert "preexec_fn" in resp.json()["detail"]

    def test_web_search_default_deny_all(self):
        from fastapi.testclient import TestClient
        app = _create_test_app()
        client = TestClient(app, base_url="http://127.0.0.1", headers=_TOOL_AUTH)
        resp = client.post(
            "/v1/tools/web_search",
            json={"query": "anything"},
        )
        # Empty allowlist => 403
        assert resp.status_code == 403

    def test_web_search_rejects_oversize_query(self):
        from fastapi.testclient import TestClient
        app = _create_test_app()
        client = TestClient(app, base_url="http://127.0.0.1", headers=_TOOL_AUTH)
        resp = client.post(
            "/v1/tools/web_search",
            json={"query": "x" * 2000},
        )
        assert resp.status_code == 400

    def test_web_search_rejects_bool_max_results(self):
        from fastapi.testclient import TestClient
        app = _create_test_app()
        client = TestClient(app, base_url="http://127.0.0.1", headers=_TOOL_AUTH)
        resp = client.post(
            "/v1/tools/web_search",
            json={"query": "q", "max_results": True},
        )
        assert resp.status_code == 400

    def test_web_search_with_allowlist_calls_backend(self):
        from fastapi.testclient import TestClient

        from soup_cli.commands.serve import _create_app
        from soup_cli.utils.server_tools import WebSearchConfig

        try:
            import fastapi  # noqa: F401
        except ImportError:
            pytest.skip("FastAPI not installed")

        def fake_backend(query, max_results, allowlist):
            return [
                {"url": "https://example.com/page1", "snippet": "snippet1"},
                {"url": "https://other.com/blocked", "snippet": "blocked"},
            ]

        cfg = WebSearchConfig(domain_allowlist=("example.com",))
        app = _create_app(
            model_obj=MagicMock(),
            tokenizer=MagicMock(),
            device="cpu",
            model_name="test-model",
            max_tokens_default=128,
            web_search_config=cfg,
            web_search_backend=fake_backend,
        )
        client = TestClient(app, base_url="http://127.0.0.1")
        resp = client.post(
            "/v1/tools/web_search",
            json={"query": "hello", "max_results": 5},
        )
        assert resp.status_code == 200
        data = resp.json()
        # Only the allowlist-matched URL should survive.
        urls = [r["url"] for r in data["results"]]
        assert any("example.com" in u for u in urls)
        assert all("other.com" not in u for u in urls)

    def test_tools_no_longer_return_501(self, monkeypatch):
        """v0.53.7 #103 regression: 501-stubs are gone."""
        from fastapi.testclient import TestClient
        _allow_live_python_tool(monkeypatch)
        app = _create_test_app()
        client = TestClient(app, base_url="http://127.0.0.1", headers=_TOOL_AUTH)
        # Python: should return 200 with sandbox response.
        resp = client.post("/v1/tools/python", json={"code": "print(1)"})
        assert resp.status_code != 501


# ----- #102 vLLM /v1/messages + streaming SSE -----------------------------


class TestAnthropicMessagesStreaming:
    @pytest.fixture(autouse=True)
    def _require_fastapi(self):
        pytest.importorskip("fastapi")
        pytest.importorskip("httpx")

    def test_stream_true_returns_sse(self):
        from fastapi.testclient import TestClient

        # Patch _generate_response so we get deterministic output without
        # actually invoking a model.
        with patch(
            "soup_cli.commands.serve._generate_response",
            return_value=("hello world", 3, 2),
        ):
            app = _create_test_app()
            client = TestClient(app)
            resp = client.post(
                "/v1/messages",
                json={
                    "model": "test-model",
                    "max_tokens": 16,
                    "stream": True,
                    "messages": [{"role": "user", "content": "hi"}],
                },
            )
            assert resp.status_code == 200, resp.text
            body = resp.text
            assert "event: message_start" in body
            assert "event: content_block_delta" in body
            assert "event: message_delta" in body
            assert "event: message_stop" in body
            assert "text_delta" in body

    def test_non_stream_returns_anthropic_envelope(self):
        from fastapi.testclient import TestClient

        with patch(
            "soup_cli.commands.serve._generate_response",
            return_value=("response", 1, 1),
        ):
            app = _create_test_app()
            client = TestClient(app)
            resp = client.post(
                "/v1/messages",
                json={
                    "model": "test-model",
                    "max_tokens": 16,
                    "messages": [{"role": "user", "content": "hi"}],
                },
            )
            assert resp.status_code == 200
            data = resp.json()
            assert data["type"] == "message"
            assert data["role"] == "assistant"
            assert data["stop_reason"] == "end_turn"
            assert data["content"][0]["type"] == "text"

    def test_stream_no_longer_returns_501(self):
        """Regression: v0.53.6 returned 501 on stream:true."""
        from fastapi.testclient import TestClient

        with patch(
            "soup_cli.commands.serve._generate_response",
            return_value=("x", 1, 1),
        ):
            app = _create_test_app()
            client = TestClient(app)
            resp = client.post(
                "/v1/messages",
                json={
                    "model": "test-model",
                    "max_tokens": 16,
                    "stream": True,
                    "messages": [{"role": "user", "content": "hi"}],
                },
            )
            assert resp.status_code != 501


class TestVllmAnthropicMessages:
    def test_vllm_app_has_v1_messages_route(self):
        """Source-level guard: vLLM create_vllm_app installs /v1/messages."""
        from pathlib import Path

        src = (
            Path(__file__).parent.parent
            / "src" / "soup_cli" / "utils" / "vllm.py"
        ).read_text(encoding="utf-8")
        # Verify the route + helper landed.
        assert "/v1/messages" in src
        assert "_stream_anthropic_messages_vllm" in src
        assert "validate_anthropic_payload" in src

    def test_vllm_stream_helper_emits_anthropic_events(self):
        from soup_cli.utils.vllm import _stream_anthropic_messages_vllm

        frames = list(
            _stream_anthropic_messages_vllm(
                msg_id="m1",
                model="test",
                text="hello world",
                input_tokens=2,
                output_tokens=2,
            )
        )
        joined = "".join(frames)
        assert "event: message_start" in joined
        assert "event: content_block_delta" in joined
        assert "event: message_delta" in joined
        assert "event: message_stop" in joined
        assert "text_delta" in joined

    def test_anthropic_streaming_helper_handles_empty_text(self):
        from soup_cli.utils.vllm import _stream_anthropic_messages_vllm

        frames = list(
            _stream_anthropic_messages_vllm(
                msg_id="m1",
                model="test",
                text="",
                input_tokens=0,
                output_tokens=0,
            )
        )
        joined = "".join(frames)
        # Even with empty text we still emit start / delta / stop events.
        assert "event: message_start" in joined
        assert "event: message_stop" in joined


class TestServeStreamHelper:
    def test_transformers_stream_helper_emits_anthropic_events(self):
        from soup_cli.commands.serve import _stream_anthropic_messages

        frames = list(
            _stream_anthropic_messages(
                msg_id="m1",
                model="test",
                text="hello world from soup",
                input_tokens=4,
                output_tokens=4,
            )
        )
        joined = "".join(frames)
        assert "event: message_start" in joined
        assert "event: content_block_delta" in joined
        assert "event: message_delta" in joined
        assert "event: message_stop" in joined
        # 4 words → at least 4 content_block_delta frames.
        assert joined.count("content_block_delta") >= 4


# ----- Cross-cutting / smoke ----------------------------------------------


class TestPublicSurface:
    def test_data_pipeline_exports(self):
        from soup_cli.utils import data_pipeline
        for name in (
            "split_markdown_by_headings",
            "resolve_prompt_strategy",
            "apply_prompt_strategy",
            "load_pretokenized_dataset",
            "validate_prompt_strategy",
        ):
            assert hasattr(data_pipeline, name), name

    def test_data_forge_exports(self):
        from soup_cli.utils import data_forge
        for name in ("make_judge_provider_fn", "JUDGE_PROVIDERS"):
            assert hasattr(data_forge, name), name

    def test_recipe_run_exports(self):
        from soup_cli.utils import recipe_run
        assert hasattr(recipe_run, "run_recipe")

    def test_trainer_plugins_exports(self):
        from soup_cli.utils import trainer_plugins
        assert hasattr(trainer_plugins, "instantiate_trainer_plugins")


# ============================================================================
# v0.53.7 review-fix tests
# ============================================================================


class TestReviewFixesRecipeRun:
    """Tests for review fixes H-B/H-C/H-D/M-C/M-E/M-F + H-J/H-K/M-O."""

    @pytest.mark.requires_symlink
    def test_seed_rejects_symlink_via_lstat_on_raw_path(self, tmp_path, monkeypatch):
        """v0.53.7 H-B: lstat the raw path BEFORE realpath."""
        import os as _os
        monkeypatch.chdir(tmp_path)
        target = tmp_path / "real.jsonl"
        target.write_text('{"x": 1}\n', encoding="utf-8")
        link = tmp_path / "link.jsonl"
        _os.symlink(str(target), str(link))
        from soup_cli.utils.recipe_dag import RecipeNode
        from soup_cli.utils.recipe_run import _node_seed
        node = RecipeNode(name="s", kind="seed", config={"path": "link.jsonl"})
        with pytest.raises(ValueError, match="symlink"):
            _node_seed(node, ())

    def test_redact_exc_message_strips_posix_paths(self):
        from soup_cli.utils.recipe_run import _redact_exc_message
        exc = FileNotFoundError("/etc/passwd: no such file")
        redacted = _redact_exc_message(exc)
        assert "/etc/passwd" not in redacted
        assert "FileNotFoundError" in redacted

    def test_redact_exc_message_truncates_long(self):
        from soup_cli.utils.recipe_run import _redact_exc_message
        exc = ValueError("x" * 1000)
        redacted = _redact_exc_message(exc, limit=64)
        assert len(redacted) <= 64

    def test_redact_exc_message_handles_windows_paths(self):
        from soup_cli.utils.recipe_run import _redact_exc_message
        exc = OSError(r"C:\Users\alice\secret.txt missing")
        redacted = _redact_exc_message(exc)
        assert "alice" not in redacted

    def test_node_code_row_injection_blocked_by_double_encode(self, tmp_path, monkeypatch):
        """v0.53.7 M-C: tricky row content cannot escape the Python literal."""
        from soup_cli.utils.recipe_dag import RecipeNode
        from soup_cli.utils.recipe_run import _node_code
        monkeypatch.chdir(tmp_path)
        evil = {"text": 'foo"); print("HIJACK"); ("'}
        node = RecipeNode(
            name="c",
            kind="code",
            config={"code": "import json\nprint(json.dumps({'L': len(_row['text'])}))"},
        )
        result = _node_code(node, ((evil,),))
        assert len(result) == 1
        out = result[0]["c"]
        # No HIJACK side effect; the literal survived intact.
        if isinstance(out, dict) and "L" in out:
            assert out["L"] == len(evil["text"])

    def test_run_recipe_resume_rehydrates_predecessor_outputs(self, tmp_path, monkeypatch):
        """v0.53.7 M-F: resume rehydrates from sidecar (was ``[]`` in v0.53.6)."""
        monkeypatch.chdir(tmp_path)
        from soup_cli.utils.recipe_dag import RecipeDAG, RecipeNode
        from soup_cli.utils.recipe_run import run_recipe

        seed = tmp_path / "seed.jsonl"
        seed.write_text('{"x": 1}\n{"x": 2}\n', encoding="utf-8")
        outdir = tmp_path / "out"

        nodes = (
            RecipeNode(name="s", kind="seed", config={"path": "seed.jsonl"}),
            RecipeNode(name="snk", kind="sampler", config={}),
        )
        dag = RecipeDAG(
            nodes=nodes, edges=(("s", "snk"),), topo_order=("s", "snk")
        )

        result1 = run_recipe(dag, output_dir=str(outdir))
        assert result1["status"] == "completed"
        assert result1["node_row_counts"]["snk"] == 2

        result2 = run_recipe(dag, output_dir=str(outdir), resume=True)
        assert result2["status"] == "completed"
        assert set(result2["completed_nodes"]) == {"s", "snk"}

    def test_run_recipe_failed_node_state_persisted(self, tmp_path, monkeypatch):
        """v0.53.7 M-O: mid-DAG failure persists status + failed_node."""
        monkeypatch.chdir(tmp_path)
        import json as _json

        from soup_cli.utils.recipe_dag import RecipeDAG, RecipeNode
        from soup_cli.utils.recipe_run import run_recipe

        outdir = tmp_path / "out"
        # seed node with a missing path raises ValueError BEFORE reaching
        # the sandbox-swallow path; this surfaces as a real failure.
        nodes = (
            RecipeNode(
                name="badseed",
                kind="seed",
                config={"path": "no_such_file_xyzzy.jsonl"},
            ),
        )
        dag = RecipeDAG(nodes=nodes, edges=(), topo_order=("badseed",))
        with pytest.raises(ValueError):
            run_recipe(dag, output_dir=str(outdir))
        ckpt = _json.loads((outdir / ".checkpoint.json").read_text(encoding="utf-8"))
        assert ckpt["status"] == "failed"
        assert ckpt["failed_node"] == "badseed"
        assert "ValueError" in ckpt["failed_reason"]

    def test_node_code_happy_path(self, tmp_path, monkeypatch):
        """v0.53.7 H-K: live code-node execution through sandbox."""
        monkeypatch.chdir(tmp_path)
        from soup_cli.utils.recipe_dag import RecipeDAG, RecipeNode
        from soup_cli.utils.recipe_run import run_recipe

        seed = tmp_path / "seed.jsonl"
        seed.write_text('{"x": 5}\n', encoding="utf-8")
        outdir = tmp_path / "out"
        nodes = (
            RecipeNode(name="s", kind="seed", config={"path": "seed.jsonl"}),
            RecipeNode(
                name="c",
                kind="code",
                config={"code": "import json\nprint(json.dumps({'doubled': _row['x']*2}))"},
            ),
            RecipeNode(name="snk", kind="sampler", config={}),
        )
        dag = RecipeDAG(
            nodes=nodes,
            edges=(("s", "c"), ("c", "snk")),
            topo_order=("s", "c", "snk"),
        )
        result = run_recipe(dag, output_dir=str(outdir))
        assert result["status"] == "completed"
        out_path = outdir / "snk.jsonl"
        if out_path.exists():
            content = out_path.read_text(encoding="utf-8")
            assert "doubled" in content or "10" in content

    def test_save_checkpoint_atomic_via_mkstemp(self, tmp_path, monkeypatch):
        """v0.53.7 H-C/M-E: mkstemp + os.replace pattern."""
        monkeypatch.chdir(tmp_path)
        from soup_cli.utils.recipe_run import _save_checkpoint
        outdir = tmp_path / "out"
        outdir.mkdir()
        _save_checkpoint(str(outdir), {"status": "running"})
        ckpt = outdir / ".checkpoint.json"
        assert ckpt.is_file()
        assert not ckpt.is_symlink()


class TestReviewFixesPromptStrategy:
    """v0.53.7 M-P + L-B + H-E coverage."""

    def setup_method(self):
        from soup_cli.utils.data_pipeline import resolve_prompt_strategy
        resolve_prompt_strategy.cache_clear()

    def test_resolve_null_byte_rejected(self):
        from soup_cli.utils.data_pipeline import resolve_prompt_strategy
        with pytest.raises(ValueError):
            resolve_prompt_strategy("mod\x00name:fn")

    def test_resolve_oversize_rejected(self):
        from soup_cli.utils.data_pipeline import resolve_prompt_strategy
        oversize = "a" * 1000 + ":fn"
        with pytest.raises(ValueError):
            resolve_prompt_strategy(oversize)

    def test_h_e_bad_signature_propagates(self, tmp_path, monkeypatch):
        """H-E: signature-shape ValueError must NOT be swallowed."""
        monkeypatch.chdir(tmp_path)
        monkeypatch.syspath_prepend(str(tmp_path))
        (tmp_path / "tfm_bad_sig.py").write_text(
            "def fn(a, b, c):\n    return {'ok': True}\n",
            encoding="utf-8",
        )
        from soup_cli.utils.data_pipeline import resolve_prompt_strategy
        resolve_prompt_strategy.cache_clear()
        with pytest.raises(ValueError, match="positional"):
            resolve_prompt_strategy("tfm_bad_sig:fn")


class TestReviewFixesForge:
    """v0.53.7 M-M temperature bool + M-L typing."""

    def test_make_judge_provider_fn_bool_temperature_rejected(self):
        from soup_cli.utils.data_forge import make_judge_provider_fn
        with pytest.raises(TypeError, match="temperature"):
            make_judge_provider_fn("ollama", temperature=True)

    def test_make_judge_provider_fn_temperature_out_of_range(self):
        from soup_cli.utils.data_forge import make_judge_provider_fn
        with pytest.raises(ValueError, match=r"\[0, 2\]"):
            make_judge_provider_fn("ollama", temperature=5.0)

    def test_judge_providers_typed_frozenset_of_str(self):
        from soup_cli.utils.data_forge import JUDGE_PROVIDERS
        assert isinstance(JUDGE_PROVIDERS, frozenset)
        assert all(isinstance(p, str) for p in JUDGE_PROVIDERS)


class TestReviewFixesVllmAnthropicLive:
    """v0.53.7 H-I + M-Q: live route tests with mocked AsyncLLMEngine."""

    @pytest.fixture(autouse=True)
    def _require_fastapi(self):
        pytest.importorskip("fastapi")
        pytest.importorskip("httpx")

    def _build_vllm_app(self):
        try:
            import fastapi  # noqa: F401
        except ImportError:
            pytest.skip("FastAPI not installed")
        try:
            import vllm  # noqa: F401
        except ImportError:
            pytest.skip("vLLM not installed")
        from soup_cli.utils.vllm import create_vllm_app

        engine = MagicMock()
        app = create_vllm_app(
            engine=engine,
            engine_model_name="test-model",
            model_name="test-model",
            max_tokens_default=128,
        )
        return app

    def test_messages_route_registered(self):
        app = self._build_vllm_app()
        paths = [r.path for r in app.routes if hasattr(r, "path")]
        assert "/v1/messages" in paths

    def test_messages_malformed_returns_400(self):
        from fastapi.testclient import TestClient
        app = self._build_vllm_app()
        client = TestClient(app)
        resp = client.post(
            "/v1/messages", json={"messages": [], "model": "x"}
        )
        assert resp.status_code == 400

    def test_messages_max_tokens_over_cap_returns_400(self):
        from fastapi.testclient import TestClient
        app = self._build_vllm_app()
        client = TestClient(app)
        resp = client.post(
            "/v1/messages",
            json={
                "messages": [{"role": "user", "content": "hi"}],
                "model": "x",
                "max_tokens": 999_999,
            },
        )
        assert resp.status_code == 400


class TestReviewFixesDataScore:
    """v0.53.7 H-H + M-J coverage."""

    @pytest.mark.requires_symlink
    def test_load_jsonl_rows_rejects_symlink(self, tmp_path, monkeypatch):
        import os as _os
        monkeypatch.chdir(tmp_path)
        target_outside = tmp_path.parent / "outside_bench.jsonl"
        try:
            target_outside.write_text('{"text": "x"}\n', encoding="utf-8")
            link = tmp_path / "link.jsonl"
            _os.symlink(str(target_outside), str(link))
            from soup_cli.utils.data_score import load_jsonl_rows
            with pytest.raises(ValueError, match="symlink|under cwd"):
                load_jsonl_rows("link.jsonl")
        finally:
            if target_outside.exists():
                target_outside.unlink()

    def test_load_jsonl_rows_outside_cwd_rejected(self, tmp_path, monkeypatch):
        monkeypatch.chdir(tmp_path)
        outside = tmp_path.parent / "evil_bench.jsonl"
        try:
            outside.write_text('{"x": 1}\n', encoding="utf-8")
            from soup_cli.utils.data_score import load_jsonl_rows
            with pytest.raises(ValueError, match="under cwd"):
                load_jsonl_rows(str(outside))
        finally:
            if outside.exists():
                outside.unlink()

    def test_extract_row_text_publicly_exposed(self):
        """v0.53.7 M-J: public alias is the documented import."""
        from soup_cli.utils import data_score
        from soup_cli.utils.data_score import extract_row_text
        assert callable(extract_row_text)
        assert "extract_row_text" in data_score.__all__


class TestReviewFixesTrainerPlugins:
    """v0.53.7 L-A + L-I dedup + L-E narrow catch."""

    def test_validate_trainer_plugin_list_rejects_duplicates(self):
        """L-I: duplicate names are explicitly rejected (not silently deduped)."""
        from soup_cli.utils.trainer_plugins import validate_trainer_plugin_list
        with pytest.raises(ValueError, match="duplicate"):
            validate_trainer_plugin_list(["grokfast", "grokfast"])

    def test_simple_plugin_returns_none_when_module_lacks_attrs(self, monkeypatch):
        """L-A: missing API surface returns None, not the bare module."""
        import types

        from soup_cli.utils import trainer_plugins

        dummy = types.ModuleType("grokfast")
        monkeypatch.setitem(sys.modules, "grokfast", dummy)
        result = trainer_plugins._instantiate_simple_plugin(
            "grokfast", "grokfast", ("GrokFastCallback", "Gradfilter")
        )
        assert result is None


class TestReviewFixesSSEHeaders:
    """v0.53.7 L-C + M-A: SSE no-store + header-injection sanitisation."""

    def test_sanitise_sse_field_strips_crlf_nul(self):
        from soup_cli.commands.serve import _sanitise_sse_field
        assert _sanitise_sse_field("hello\r\nworld", max_len=200) == "helloworld"
        assert _sanitise_sse_field("a\x00b", max_len=200) == "ab"
        assert _sanitise_sse_field("x" * 500, max_len=10) == "x" * 10

    def test_stream_anthropic_sanitises_msg_id(self):
        """Caller-controlled msg_id with CR/LF cannot create a new SSE event.

        After sanitisation the injected ``event:``/``data:`` text becomes a
        literal substring inside an existing JSON ``id`` value — no new
        ``\\n\\n`` boundary, so the SSE consumer treats the whole frame as
        one event with garbled-but-safe ``id`` content.
        """
        from soup_cli.commands.serve import _stream_anthropic_messages
        evil_id = "msg-1\nevent: hijack\ndata: {}\n"
        frames = list(_stream_anthropic_messages(
            msg_id=evil_id, model="m", text="hi",
            input_tokens=1, output_tokens=1,
        ))
        # Each frame is one SSE event terminated by exactly one "\n\n".
        # The CR/LFs from the evil id must have been stripped — count of
        # "\n\n" must equal the number of frames produced.
        joined = "".join(frames)
        assert joined.count("\n\n") == len(frames)
        # The injected literal "event: hijack" cannot start a new line; it
        # always appears as a substring inside an existing data line, never
        # at line-start, because no CR/LF separator survived.
        for line in joined.split("\n"):
            assert not line.startswith("event: hijack")
            assert not line.startswith("data: {}")


# TestReviewFixesC1BashStub replaced by TestToolEndpointsLive with OS sandbox isolation


class TestReviewFixesAuthToken:
    """v0.53.7 H-A: optional Bearer-token gate on tool endpoints."""

    def test_python_tool_with_auth_token_requires_bearer(self, monkeypatch):
        try:
            import fastapi  # noqa: F401
        except ImportError:
            pytest.skip("FastAPI not installed")
        from fastapi.testclient import TestClient

        _allow_live_python_tool(monkeypatch)

        from soup_cli.commands.serve import _create_app

        app = _create_app(
            model_obj=MagicMock(),
            tokenizer=MagicMock(),
            device="cpu",
            model_name="test-model",
            max_tokens_default=128,
            auth_token="s3cret",
        )
        client = TestClient(app, base_url="http://127.0.0.1")
        # No header -> 401.
        resp = client.post("/v1/tools/python", json={"code": "print(1)"})
        assert resp.status_code == 401
        # Wrong token -> 401.
        resp = client.post(
            "/v1/tools/python",
            json={"code": "print(1)"},
            headers={"Authorization": "Bearer wrong"},
        )
        assert resp.status_code == 401
        # Right token -> 200.
        resp = client.post(
            "/v1/tools/python",
            json={"code": "print(1)"},
            headers={"Authorization": "Bearer s3cret"},
        )
        assert resp.status_code == 200

    def test_auth_token_none_means_no_gate(self, monkeypatch):
        try:
            import fastapi  # noqa: F401
        except ImportError:
            pytest.skip("FastAPI not installed")
        from fastapi.testclient import TestClient

        _allow_live_python_tool(monkeypatch)

        from soup_cli.commands.serve import _create_app
        from soup_cli.utils.server_tools import WebSearchConfig

        # web_search keeps the opt-in gate on loopback; /v1/tools/python and
        # /v1/tools/bash take a token on every bind (test_serve_code_tool_token.py).
        app = _create_app(
            model_obj=MagicMock(),
            tokenizer=MagicMock(),
            device="cpu",
            model_name="test-model",
            max_tokens_default=128,
            web_search_config=WebSearchConfig(domain_allowlist=("example.com",)),
            web_search_backend=lambda query, max_results, allowlist: [],
        )
        client = TestClient(app, base_url="http://127.0.0.1")
        resp = client.post("/v1/tools/web_search", json={"query": "q"})
        assert resp.status_code == 200


class TestReviewFixesPreprocessAtomic:
    """v0.53.7 M-H + L-J source-grep guard."""

    def test_preprocess_save_uses_atomic_pattern(self):
        src = Path(__file__).parent.parent / "src" / "soup_cli" / "commands" / "data.py"
        body = src.read_text(encoding="utf-8")
        assert ".tmp_" in body
        assert "os.replace" in body


class TestReviewFixesVllmAnthropicCors:
    """v0.53.7 M-G: vLLM /v1/messages CORS restricted to loopback."""

    def test_vllm_app_cors_is_loopback_only(self):
        try:
            import fastapi  # noqa: F401
        except ImportError:
            pytest.skip("FastAPI not installed")
        try:
            import vllm  # noqa: F401
        except ImportError:
            pytest.skip("vLLM not installed")
        from soup_cli.utils.vllm import create_vllm_app

        engine = MagicMock()
        app = create_vllm_app(
            engine=engine,
            engine_model_name="test-model",
            model_name="test-model",
            max_tokens_default=128,
        )
        # Inspect installed middleware for the loopback regex.
        cors_options = []
        for m in app.user_middleware:
            cors_options.append(getattr(m, "options", {}))
        # Find the CORSMiddleware entry; its options should carry
        # ``allow_origin_regex`` not ``allow_origins=*``.
        found_loopback = False
        for opts in cors_options:
            regex = opts.get("allow_origin_regex", "")
            if regex and "localhost" in regex:
                found_loopback = True
        assert found_loopback, "vLLM /v1/messages CORS must be loopback-only"


# ----- #86 SFT + Pretrain pre_tokenized short-circuit --------------------
#
# These tests verify that SFTTrainerWrapper and PretrainTrainerWrapper
# actually call ``load_pretokenized_dataset`` when ``data.format =
# 'pre_tokenized'`` and ``data.tokenized_path`` is set, instead of running
# the normal tokenize pipeline. Schema-layer validators (e.g. the
# ``format='pre_tokenized'`` + missing-``tokenized_path`` cross-validator)
# are already covered by existing v0.42.0 tests; this class focuses on the
# wiring at the trainer setup() boundary.

class TestSftPretrainPreTokenizedShortCircuit:
    """v0.53.7 #86 — trainer-side short-circuit + cache-hash gate.

    Note: ``data.tokenized_path`` is cwd-containment-checked at schema load
    time, so every test ``monkeypatch.chdir(tmp_path)`` before loading the
    config and writes its Arrow fixture under ``tmp_path``.
    """

    def _write_arrow_dir(self, tmp_path: Path, cache_key: object) -> Path:
        """Save a 2-row pre-tokenized Arrow dataset under tmp_path/tokenized.

        ``cache_key`` is the value written into ``metadata.json``; pass
        ``...`` (Ellipsis) to skip writing the metadata file entirely.
        """
        pytest.importorskip("datasets")
        from datasets import Dataset

        target = tmp_path / "tokenized"
        rows = [
            {
                "input_ids": [1, 2, 3, 4],
                "labels": [1, 2, 3, 4],
                "attention_mask": [1, 1, 1, 1],
            },
            {
                "input_ids": [5, 6, 7, 8],
                "labels": [5, 6, 7, 8],
                "attention_mask": [1, 1, 1, 1],
            },
        ]
        ds = Dataset.from_list(rows)
        ds.save_to_disk(str(target))
        if cache_key is not Ellipsis:
            (target / "metadata.json").write_text(
                json.dumps({"cache_key": cache_key}), encoding="utf-8"
            )
        return target

    def _build_cfg_yaml(
        self,
        train_path: str,
        tokenized_path: str,
        *,
        max_length: int = 64,
        task: str = "sft",
    ) -> str:
        return (
            "base: TinyLlama/TinyLlama-1.1B-Chat-v1.0\n"
            f"task: {task}\n"
            "data:\n"
            f"  train: {train_path}\n"
            "  format: pre_tokenized\n"
            f"  tokenized_path: {tokenized_path}\n"
            f"  max_length: {max_length}\n"
            "output: ./out\n"
        )

    def test_sft_short_circuits_to_arrow_dataset(self, tmp_path, monkeypatch):
        """Happy path: SFT trainer loads pre-tokenized Arrow shards directly
        and skips ``Dataset.from_list(...).map(format_row)`` entirely."""
        from soup_cli.config.loader import load_config_from_string
        from soup_cli.utils.data_pipeline import (
            make_preprocess_cache_key,
            preprocess_dataset_key_input,
        )

        monkeypatch.chdir(tmp_path)
        # The cfg.data.train path doesn't need to exist for the short-circuit
        # path — we never load it.
        train_jsonl = "train.jsonl"
        target = self._write_arrow_dir(tmp_path, cache_key="placeholder")

        yaml = self._build_cfg_yaml(train_jsonl, "tokenized")
        cfg = load_config_from_string(yaml)

        # Compute matching cache_key so the gate passes.
        cache_key = make_preprocess_cache_key(
            dataset_path=preprocess_dataset_key_input(cfg.data),
            tokenizer_name="TinyLlama/TinyLlama-1.1B-Chat-v1.0",
            max_length=64,
            format_name="pre_tokenized",
            mask_mode="responses_only",
            task="sft",
        )
        (target / "metadata.json").write_text(
            json.dumps({"cache_key": cache_key}), encoding="utf-8"
        )

        # Exercise the helper directly — bypasses the full ``setup()`` (which
        # would also load a 1B model). The helper is the unit under test.
        from soup_cli.trainer.sft import _maybe_load_pretokenized

        captured = MagicMock()
        result = _maybe_load_pretokenized(cfg.data, cfg.base, captured, task="sft")
        assert result is not None
        train_ds, eval_ds = result
        assert len(train_ds) == 2
        assert eval_ds is None
        # Ensure the format_row tokenizer-call path was NOT taken — we do
        # this by counting console.print advisory lines (yellow advisory
        # for missing metadata.json was NOT printed since metadata exists).
        printed = "".join(str(c) for c in captured.print.call_args_list)
        assert "skipping tokenization" in printed
        assert "no metadata.json" not in printed

    def test_pretrain_short_circuits_to_arrow_dataset(self, tmp_path, monkeypatch):
        """Same as SFT but for the Pretrain wrapper — verifies pretrain.py
        also short-circuits, not just sft.py."""
        from soup_cli.config.loader import load_config_from_string
        from soup_cli.utils.data_pipeline import (
            make_preprocess_cache_key,
            preprocess_dataset_key_input,
        )

        monkeypatch.chdir(tmp_path)
        train_jsonl = "train.txt"
        target = self._write_arrow_dir(tmp_path, cache_key="placeholder")

        yaml = self._build_cfg_yaml(
            train_jsonl, "tokenized", task="pretrain",
        )
        cfg = load_config_from_string(yaml)

        cache_key = make_preprocess_cache_key(
            dataset_path=preprocess_dataset_key_input(cfg.data),
            tokenizer_name="TinyLlama/TinyLlama-1.1B-Chat-v1.0",
            max_length=64,
            format_name="pre_tokenized",
            mask_mode="responses_only",
            task="pretrain",
        )
        (target / "metadata.json").write_text(
            json.dumps({"cache_key": cache_key}), encoding="utf-8"
        )

        # The pretrain wrapper imports the helper from sft.py — verify that
        # import resolves and the function behaves identically.
        from soup_cli.trainer.sft import _maybe_load_pretokenized as helper

        captured = MagicMock()
        result = helper(cfg.data, cfg.base, captured, task=cfg.task)
        assert result is not None
        train_ds, _ = result
        assert len(train_ds) == 2

    def test_cache_hash_mismatch_raises(self, tmp_path, monkeypatch):
        """metadata.json carrying a stale cache_key (e.g. user changed
        ``max_length`` without re-running ``soup data preprocess``) must
        raise with the keyword ``cache hash mismatch`` so users know what
        to do."""
        from soup_cli.config.loader import load_config_from_string

        monkeypatch.chdir(tmp_path)
        self._write_arrow_dir(tmp_path, cache_key="STALE_KEY_FROM_OLD_RUN")

        yaml = self._build_cfg_yaml("train.jsonl", "tokenized")
        cfg = load_config_from_string(yaml)

        from soup_cli.trainer.sft import _maybe_load_pretokenized

        with pytest.raises(ValueError, match="cache hash mismatch"):
            _maybe_load_pretokenized(cfg.data, cfg.base, MagicMock(), task="sft")

    def test_missing_metadata_proceeds_with_yellow_advisory(
        self, tmp_path, monkeypatch,
    ):
        """When metadata.json is absent, the trainer falls back to "trusted"
        mode with a yellow advisory and still loads the Arrow shards."""
        from soup_cli.config.loader import load_config_from_string

        monkeypatch.chdir(tmp_path)
        # Ellipsis sentinel → skip writing metadata.json entirely.
        self._write_arrow_dir(tmp_path, cache_key=Ellipsis)

        yaml = self._build_cfg_yaml("train.jsonl", "tokenized")
        cfg = load_config_from_string(yaml)

        from soup_cli.trainer.sft import _maybe_load_pretokenized

        captured = MagicMock()
        result = _maybe_load_pretokenized(cfg.data, cfg.base, captured, task="sft")
        assert result is not None
        train_ds, _ = result
        assert len(train_ds) == 2

        # Yellow advisory must mention the user-actionable next step.
        printed = "".join(str(c) for c in captured.print.call_args_list)
        assert "no metadata.json" in printed
        assert "soup data preprocess" in printed

    # Schema layer already rejects ``format='pre_tokenized'`` without a
    # ``data.tokenized_path`` via ``DataConfig._validate_v042_pre_tokenized_path``
    # (see test_v0420.py). No re-test here — that's the schema gate's job.