Skip to content

armgit / Soup

Public Imported from GitHub
0

Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.

20.1 KB
Raw Blame
"""Tests for Issue #1435 (PR 2): live diagnose probe inputs from real dataset formats.

PR 1 reports a dataset the row builders cannot read as NOT_RUN. These tests pin that the
common training formats (ShareGPT, Alpaca with ``input``, plaintext, mixed) now reach the
probes as the model saw them, that vision rows and over-long prompts are handled
explicitly, and that skipped rows are reported.
"""

from __future__ import annotations

import json

import pytest

from soup_cli.utils.diagnose.memorization import split_prefix
from tests.conftest import strip_ansi
from tests.test_issue1435_diagnose_not_run import DATASET_MODES, _cli, _gens, _run_live

SKIPPABLE = ("format", "mode_collapse")


def _words(tag: str, count: int = 24) -> str:
    return " ".join(f"{tag}word{j}" for j in range(count))


def _sharegpt(i: int, answer: str | None = None) -> dict:
    return {
        "conversations": [
            {"from": "human", "value": f"question {i} {_words(f'q{i}', 12)}"},
            {"from": "gpt", "value": answer or f"answer {i} {_words(f'a{i}', 12)}"},
        ]
    }


def _not_run_modes(report) -> set:
    return {m for m, s in report.scores.items() if s.verdict == "NOT_RUN"}


class TestPlaintext:
    TEXTS = [_words(f"topic{i}") for i in range(12)]

    def _regurgitating(self):
        suffix = dict(split_prefix(text) for text in self.TEXTS)
        return _gens(adapter_gen=lambda prompt: suffix.get(prompt, "ok"))

    def test_memorization_runs_and_catches_regurgitation(self, tmp_path, monkeypatch):
        rows = [{"text": text} for text in self.TEXTS]
        report, _ = _run_live(tmp_path, monkeypatch, rows, self._regurgitating())
        assert report.scores["memorization"].verdict == "MAJOR"
        assert report.overall == "MAJOR"

    def test_probes_needing_a_prompt_answer_split_say_so(self, tmp_path, monkeypatch):
        rows = [{"text": text} for text in self.TEXTS]
        report, _ = _run_live(tmp_path, monkeypatch, rows, self._regurgitating())
        for mode in SKIPPABLE:
            score = report.scores[mode]
            assert score.verdict == "NOT_RUN", mode
            assert "plaintext" in score.evidence

    def test_cli_exit_2_on_regurgitating_plaintext(self, tmp_path, monkeypatch):
        rows = [{"text": text} for text in self.TEXTS]
        result, _, _ = _cli(tmp_path, monkeypatch, rows, self._regurgitating())
        assert result.exit_code == 2


class TestShareGPT:
    def test_probes_run(self, tmp_path, monkeypatch):
        rows = [_sharegpt(i) for i in range(12)]
        report, _ = _run_live(tmp_path, monkeypatch, rows, _gens())
        for mode in DATASET_MODES:
            assert report.scores[mode].verdict != "NOT_RUN", mode
        assert report.overall != "NOT_RUN"

    def test_conversation_text_reaches_memorization(self, tmp_path, monkeypatch):
        rows = [_sharegpt(i) for i in range(12)]
        report, _ = _run_live(tmp_path, monkeypatch, rows, _gens())
        assert "scanned=" in report.scores["memorization"].evidence

    def test_json_targets_are_seen_by_the_format_probe(self, tmp_path, monkeypatch):
        rows = [_sharegpt(i, answer=json.dumps({"n": i})) for i in range(12)]
        report, _ = _run_live(tmp_path, monkeypatch, rows, _gens())  # adapter never emits JSON
        assert report.scores["format"].verdict == "MAJOR"


class TestAlpaca:
    def test_input_is_joined_to_the_instruction(self, tmp_path, monkeypatch):
        seen: list = []

        def adapter_multi(prompt, k):
            seen.append(prompt)
            return [f"adapter{j} p{j}q r{j}s" for j in range(k)]

        rows = [
            {"instruction": "Translate to French:", "input": f"Good morning {i}",
             "output": f"Bonjour {i}"}
            for i in range(12)
        ]
        _run_live(tmp_path, monkeypatch, rows, _gens(adapter_multi=adapter_multi))
        assert "Translate to French:\nGood morning 0" in seen

    def test_alpaca_without_input_still_works(self, tmp_path, monkeypatch):
        rows = [{"instruction": f"Say {i}", "output": f"Said {i}"} for i in range(12)]
        report, _ = _run_live(tmp_path, monkeypatch, rows, _gens())
        assert not _not_run_modes(report)


class TestFlatAndMixedRows:
    def test_flat_question_answer_rows_keep_working(self, tmp_path, monkeypatch):
        rows = [{"question": f"What is {i}?", "answer": f"It is {i}."} for i in range(12)]
        report, _ = _run_live(tmp_path, monkeypatch, rows, _gens())
        assert not _not_run_modes(report)

    def test_flat_rows_reach_memorization(self, tmp_path, monkeypatch):
        rows = [
            {"question": f"What is {_words(f'n{i}', 10)}?", "answer": _words(f"a{i}", 12)}
            for i in range(12)
        ]
        report, _ = _run_live(tmp_path, monkeypatch, rows, _gens())
        assert "scanned=" in report.scores["memorization"].evidence

    def test_mixed_formats_in_one_file(self, tmp_path, monkeypatch):
        seen: list = []

        def adapter_multi(prompt, k):
            seen.append(prompt)
            return [f"adapter{j} p{j}q r{j}s" for j in range(k)]

        rows = []
        for i in range(6):
            rows.append(_sharegpt(i))
            rows.append({"instruction": f"Say {i}", "input": "now", "output": f"Said {i}"})
        report, _ = _run_live(tmp_path, monkeypatch, rows, _gens(adapter_multi=adapter_multi))
        assert not _not_run_modes(report)
        # Both halves reach the probes, not just the format that used to be readable.
        assert any(prompt.startswith("question 0") for prompt in seen)
        assert any(prompt.startswith("Say 0\nnow") for prompt in seen)


class TestVision:
    def test_llava_rows_are_not_run_with_the_image_reason(self, tmp_path, monkeypatch):
        rows = [dict(_sharegpt(i), image=f"img{i}.jpg") for i in range(12)]
        report, _ = _run_live(tmp_path, monkeypatch, rows, _gens())
        for mode in DATASET_MODES:
            score = report.scores[mode]
            assert score.verdict == "NOT_RUN", mode
            assert "image" in score.evidence


class TestLongPrompts:
    @staticmethod
    def _rows():
        return [
            {"prompt": f"Write story number {i}.", "completion": f"Once upon a time {i}."}
            for i in range(12)
        ]

    def test_one_long_prompt_is_skipped_not_fatal(self, tmp_path, monkeypatch):
        rows = self._rows()
        rows[0]["prompt"] = "x" * 9000
        collapsed = _gens(adapter_multi=lambda prompt, k: ["same reply"] * k)
        report, _ = _run_live(tmp_path, monkeypatch, rows, collapsed)
        assert report.scores["mode_collapse"].verdict == "MAJOR"  # other rows still scored

    def test_all_prompts_too_long_is_not_run_with_the_limit(self, tmp_path, monkeypatch):
        rows = [dict(row, prompt="x" * 9000) for row in self._rows()]
        report, _ = _run_live(tmp_path, monkeypatch, rows, _gens())
        for mode in SKIPPABLE:
            score = report.scores[mode]
            assert score.verdict == "NOT_RUN", mode
            assert "8192" in score.evidence

    def test_prompt_limit_constant_matches_require_prompts(self):
        from soup_cli.utils.diagnose import _common

        assert _common.MAX_PROMPT_CHARS == 8192
        with pytest.raises(ValueError, match="8192"):
            _common.require_prompts(["x" * (_common.MAX_PROMPT_CHARS + 1)])


class TestSkippedRows:
    def _rows(self):
        rows = [
            {"prompt": f"Write story number {i}.", "completion": f"Once upon a time {i}."}
            for i in range(12)
        ]
        rows[0]["prompt"] = "x" * 9000
        rows += [{"foo": 1}, {"foo": 2}]
        return rows

    def test_skips_are_counted_in_extras(self, tmp_path, monkeypatch):
        report, _ = _run_live(tmp_path, monkeypatch, self._rows(), _gens())
        assert report.extras["rows_skipped"] == "2 unreadable, 1 over-long"

    def test_no_extras_when_nothing_is_skipped(self, tmp_path, monkeypatch):
        rows = [{"prompt": f"Story {i}.", "completion": f"Once {i}."} for i in range(12)]
        report, _ = _run_live(tmp_path, monkeypatch, rows, _gens())
        assert "rows_skipped" not in report.extras

    def test_cli_prints_the_skipped_note(self, tmp_path, monkeypatch):
        _, out, _ = _cli(tmp_path, monkeypatch, self._rows(), _gens())
        assert "Rows skipped: 2 unreadable, 1 over-long" in strip_ansi(out)


def test_looks_like_json_dataset_keeps_its_call_shape():
    from soup_cli.utils.diagnose.live import _looks_like_json_dataset

    assert _looks_like_json_dataset([{"output": '{"a": 1}'}] * 5) is True
    assert _looks_like_json_dataset([{"output": "plain"}] * 5) is False
    # ShareGPT targets are read through the converters too.
    sharegpt_json = [_sharegpt(i, answer=json.dumps({"n": i})) for i in range(5)]
    assert _looks_like_json_dataset(sharegpt_json) is True


TOOL_ROW = {
    "messages": [
        {"role": "system", "content": "You can call tools."},
        {"role": "user", "content": "Weather in Paris?"},
        {"role": "assistant", "content": "", "tool_calls": [
            {"id": "c1", "type": "function",
             "function": {"name": "get_weather", "arguments": "{\"city\": \"Paris\"}"}}]},
        {"role": "tool", "tool_call_id": "c1", "content": "Sunny"},
        {"role": "assistant", "content": "It is sunny in Paris."},
    ],
    "tools": [{"type": "function", "function": {
        "name": "get_weather", "description": "d",
        "parameters": {"type": "object", "properties": {"city": {"type": "string"}}}}}],
}


def _recording(seen: list):
    def gen(prompt):
        seen.append(prompt)
        return "ok"

    return gen


def _recording_multi(seen: list):
    def gen(prompt, k):
        seen.append(prompt)
        return [f"adapter{j} p{j}q r{j}s" for j in range(k)]

    return gen


class TestRobustness:
    def test_malformed_tool_schema_row_does_not_crash(self, tmp_path, monkeypatch):
        bad = {"messages": [{"role": "user", "content": "hi"}],
               "tools": [{"function": "get_weather"}]}
        rows = [
            {"prompt": f"Write story number {i}.", "completion": f"Once upon a time {i}."}
            for i in range(12)
        ] + [bad]
        report, _ = _run_live(tmp_path, monkeypatch, rows, _gens())
        assert report.extras["rows_skipped"] == "1 without an answer"
        assert not _not_run_modes(report)

    def test_rows_without_an_answer_are_counted(self, tmp_path, monkeypatch):
        rows = [{"prompt": f"only a prompt {i}"} for i in range(12)]
        report, _ = _run_live(tmp_path, monkeypatch, rows, _gens())
        assert report.extras["rows_skipped"] == "12 without an answer"
        for mode in SKIPPABLE:
            assert report.scores[mode].verdict == "NOT_RUN"

    def test_tool_calling_rows_give_a_pair_from_the_final_reply(self, tmp_path, monkeypatch):
        seen: list = []
        rows = [TOOL_ROW] * 12
        gens = _gens(adapter_multi=_recording_multi(seen))
        report, _ = _run_live(tmp_path, monkeypatch, rows, gens)
        assert not _not_run_modes(report)
        assert any("Weather in Paris?" in prompt for prompt in seen)
        assert "rows_skipped" not in report.extras


class TestOtherFormats:
    def test_audio_rows_are_not_run_with_the_audio_reason(self, tmp_path, monkeypatch):
        rows = [{"audio": f"a{i}.wav", "messages": [
            {"role": "user", "content": "hi"}, {"role": "assistant", "content": "yo"}]}
            for i in range(12)]
        report, _ = _run_live(tmp_path, monkeypatch, rows, _gens())
        for mode in DATASET_MODES:
            score = report.scores[mode]
            assert score.verdict == "NOT_RUN", mode
            assert "audio" in score.evidence

    def test_dpo_rows_probe_with_the_prompt(self, tmp_path, monkeypatch):
        seen: list = []
        rows = [{"prompt": f"Pick one {i}", "chosen": f"good {i}", "rejected": f"bad {i}"}
                for i in range(12)]
        gens = _gens(adapter_multi=_recording_multi(seen))
        report, _ = _run_live(tmp_path, monkeypatch, rows, gens)
        assert "Pick one 0" in seen
        assert not _not_run_modes(report)

    def test_kto_rows_probe_with_the_prompt(self, tmp_path, monkeypatch):
        seen: list = []
        rows = [{"prompt": f"Rate {i}", "completion": f"fine {i}", "label": True}
                for i in range(12)]
        gens = _gens(adapter_multi=_recording_multi(seen))
        report, _ = _run_live(tmp_path, monkeypatch, rows, gens)
        assert "Rate 0" in seen
        assert not _not_run_modes(report)

    def test_multi_turn_prompt_stops_at_the_first_assistant_reply(self, tmp_path, monkeypatch):
        seen: list = []
        rows = [{"messages": [
            {"role": "user", "content": f"first question {i}"},
            {"role": "assistant", "content": f"first answer {i}"},
            {"role": "user", "content": f"second question {i}"},
            {"role": "assistant", "content": f"second answer {i}"}]} for i in range(12)]
        _run_live(tmp_path, monkeypatch, rows, _gens(adapter_multi=_recording_multi(seen)))
        assert "first question 0" in seen
        assert not any("second question" in prompt for prompt in seen)

    def test_alpaca_system_text_is_part_of_the_prompt(self, tmp_path, monkeypatch):
        seen: list = []
        rows = [{"instruction": f"Say {i}", "output": f"Said {i}", "system": "Be terse."}
                for i in range(12)]
        _run_live(tmp_path, monkeypatch, rows, _gens(adapter_multi=_recording_multi(seen)))
        assert "Be terse.\nSay 0" in seen

    def test_raft_rows_still_feed_citation_the_raw_rows(self, tmp_path, monkeypatch):
        from unittest import mock

        from soup_cli.utils.diagnose.runner import neutral_score

        rows = [{"query": f"What is {i}?", "golden_doc": f"doc {i}", "answer": f"It is {i}."}
                for i in range(12)]
        with mock.patch(
            "soup_cli.utils.diagnose.citation.score_citation",
            return_value=neutral_score("citation", "mock"),
        ) as citation:
            report, _ = _run_live(tmp_path, monkeypatch, rows, _gens())
        passed = citation.call_args.args[0]
        assert passed[0].get("golden_doc") == "doc 0"  # raw RAFT rows, not converted ones
        assert not _not_run_modes(report)  # the flat query/answer pair is still built


class TestReasons:
    def test_memorization_not_run_reason_names_the_missing_text(self, tmp_path, monkeypatch):
        rows = [{"foo": f"bar {i}"} for i in range(12)]
        report, _ = _run_live(tmp_path, monkeypatch, rows, _gens())
        assert "no row has text" in report.scores["memorization"].evidence

    def test_dominant_cause_picks_the_reason(self, tmp_path, monkeypatch):
        rows = [dict(_sharegpt(0), image="i.jpg")] + [
            {"text": _words(f"t{i}")} for i in range(11)
        ]
        report, _ = _run_live(tmp_path, monkeypatch, rows, _gens())
        for mode in SKIPPABLE:
            evidence = report.scores[mode].evidence
            assert "plaintext" in evidence and "image" not in evidence

    def test_no_answer_rows_can_be_the_dominant_cause(self, tmp_path, monkeypatch):
        long_row = {"prompt": "x" * 9000, "completion": "done"}
        rows = [long_row] + [{"prompt": f"only a prompt {i}"} for i in range(11)]
        report, _ = _run_live(tmp_path, monkeypatch, rows, _gens())
        for mode in SKIPPABLE:
            evidence = report.scores[mode].evidence
            assert "no prompt/answer pair" in evidence and "8192" not in evidence

    def test_no_answer_beats_a_lone_plaintext_row(self, tmp_path, monkeypatch):
        rows = [{"text": _words("t0")}] + [
            {"messages": [{"role": "user", "content": f"question {i}"}]} for i in range(11)
        ]
        report, _ = _run_live(tmp_path, monkeypatch, rows, _gens())
        for mode in SKIPPABLE:
            evidence = report.scores[mode].evidence
            assert "no prompt/answer pair" in evidence and "plaintext" not in evidence

    def test_unreadable_rows_beat_a_lone_media_row_for_memorization(
        self, tmp_path, monkeypatch
    ):
        rows = [dict(_sharegpt(0), image="i.jpg")] + [{"foo": f"bar {i}"} for i in range(11)]
        report, _ = _run_live(tmp_path, monkeypatch, rows, _gens())
        evidence = report.scores["memorization"].evidence
        assert "no row has text" in evidence and "image" not in evidence

    def test_mixed_media_kinds_are_named_together(self, tmp_path, monkeypatch):
        rows = [dict(_sharegpt(i), image=f"i{i}.jpg") for i in range(6)] + [
            {"audio": f"a{i}.wav", "messages": [
                {"role": "user", "content": "hi"}, {"role": "assistant", "content": "yo"}]}
            for i in range(6)
        ]
        report, _ = _run_live(tmp_path, monkeypatch, rows, _gens())
        assert report.extras["rows_skipped"] == "12 need audio or image"
        assert "an image or audio" in report.scores["format"].evidence

    def test_over_long_rows_still_reach_memorization(self, tmp_path, monkeypatch):
        seen: list = []
        long_prompt = " ".join(["word"] * 3000)  # ~15000 chars, over the prompt limit
        rows = [{"prompt": long_prompt, "completion": "done"} for _ in range(12)]
        _run_live(tmp_path, monkeypatch, rows, _gens(adapter_gen=_recording(seen)))
        assert max(len(prompt) for prompt in seen) > 3000


def test_dpo_target_is_the_chosen_answer(tmp_path, monkeypatch):
    # JSON in `chosen`, prose in `rejected`: the format probe only fires if chosen is the target.
    rows = [
        {"prompt": f"Give json {i}", "chosen": json.dumps({"n": i}), "rejected": f"plain {i}"}
        for i in range(12)
    ]
    report, _ = _run_live(tmp_path, monkeypatch, rows, _gens())  # adapter never emits JSON
    assert report.scores["format"].verdict == "MAJOR", report.scores["format"]


def test_rows_are_read_before_the_model_loads(tmp_path, monkeypatch):
    import soup_cli.utils.diagnose.live as live

    order: list = []
    real_analyse = live._analyse_rows

    def analyse(rows):
        order.append("rows")
        return real_analyse(rows)

    def fake_pair(base, adapter=None, **kw):
        order.append("model")
        return _gens()

    monkeypatch.chdir(tmp_path)
    (tmp_path / "d.jsonl").write_text(
        "\n".join(json.dumps({"prompt": f"p{i}", "completion": f"c{i}"}) for i in range(12)),
        encoding="utf-8",
    )
    monkeypatch.setattr(live, "_analyse_rows", analyse)
    monkeypatch.setattr(live, "load_adapter_pair", fake_pair)
    live.run_live_diagnose(run_id="r", base="b", adapter="a", dataset_path="d.jsonl")
    assert order == ["rows", "model"]


def test_a_null_byte_prompt_is_skipped_not_fatal(tmp_path, monkeypatch):
    rows = [
        {"prompt": f"Write story number {i}.", "completion": f"Once upon a time {i}."}
        for i in range(12)
    ]
    rows[0]["prompt"] = "nul\x00byte"
    collapsed = _gens(adapter_multi=lambda prompt, k: ["same reply"] * k)
    report, _ = _run_live(tmp_path, monkeypatch, rows, collapsed)
    assert report.scores["mode_collapse"].verdict == "MAJOR", report.scores["mode_collapse"]
    assert report.extras["rows_skipped"] == "1 unreadable"


def test_rows_skipped_from_an_evidence_file_has_no_control_bytes(tmp_path, monkeypatch):
    from typer.testing import CliRunner

    from soup_cli.cli import app

    monkeypatch.chdir(tmp_path)
    esc, csi = chr(0x1B), chr(0x9B)
    (tmp_path / "ev.json").write_text(
        json.dumps(
            {
                "scores": {"forgetting": {"score": 1.0, "verdict": "OK", "evidence": "fine"}},
                "extras": {"rows_skipped": f"{esc}]0;T{esc}[2J{csi}31m3 unreadable [bold red]x[/]"},
            }
        ),
        encoding="utf-8",
    )
    result = CliRunner().invoke(app, ["diagnose", "r1", "--evidence", "ev.json"])
    assert result.exit_code == 0, (result.output, repr(result.exception))
    assert f"{esc}]0;T" not in result.output
    assert f"{esc}[2J" not in result.output
    assert csi not in result.output
    flat = "".join(strip_ansi(result.output).split())
    assert "3unreadable[boldred]x[/]" in flat, result.output


def test_an_assistant_only_row_is_not_a_pair(tmp_path, monkeypatch):
    rows = [{"messages": [{"role": "assistant", "content": f"greeting {i}"}]} for i in range(12)]
    report, _ = _run_live(tmp_path, monkeypatch, rows, _gens())
    assert report.scores["mode_collapse"].verdict == "NOT_RUN"
    assert report.extras["rows_skipped"] == "12 without an answer"