Skip to content

armgit / Soup

Public Imported from GitHub
0

Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.

21.0 KB
Raw Blame
"""#1524 — a `continue` held by the 3x rule says so, with the two spreads.

#1419 holds `accept_h0` back at `continue` while the tested rows' pooled
standard deviation is more than `ACCEPT_HOLD_SPREAD_RATIO` (3) times the
held-out rows'. That fixed the wrong accepts after a saturated start, and left
the other side open: the held-out rows are FIXED, so once the hold fires, more
rows of the same kind never release it. A run that sits there prints "Insufficient
evidence. Collect more samples and re-run." with the confidence sequence for the
difference already inside `+-effect_size`, and that advice cannot help — the
held-out spread does not move.

So the verdict now carries `accept_held` plus the two spreads, and the panel
prints one line that names the ratio and what to do about it. There is no JSON
to gain keys in — `soup ab` has none, and a held run sends no webhook (the
webhook fires only on a terminal decision, and a hold is only legal on a
`continue`) — so the facts live on the panel and on the `MsprtVerdict`.

The two tests that matter are the pair below. `test_a_held_accept_says_why`
fails if the hold is removed (it asserts the field AND that the hold is what
caused it, by monkeypatching the ratio away), and
`test_an_unheld_continue_carries_no_note` pins the other direction: a `continue`
that is merely short of evidence must not borrow this explanation.
"""

from __future__ import annotations

import dataclasses
import json
import math
import random
import re

import pytest
from typer.testing import CliRunner

runner = CliRunner()

_ANSI_RE = re.compile(r"\x1b\[[0-9;]*m")
_BOX_RE = re.compile(f"[{chr(0x2500)}-{chr(0x257F)}|]")


def _plain(text: str) -> str:
    return " ".join(_BOX_RE.sub(" ", _ANSI_RE.sub("", text)).split())


def _step(control, treatment, metric="judge_score", **kwargs):
    from soup_cli.utils.ab_test import MsprtConfig, msprt_step

    return msprt_step(MsprtConfig(metric=metric, **kwargs), control=control, treatment=treatment)


def _saturated_warm_up(rows: int = 60, *, seed: int = 0, effect_size: float = 0.1):
    """The issue's repro shape: the first 5 rows of each arm nearly constant at
    1.0 (a saturated judge / warm cache), then ordinary rows drawn from the SAME
    distribution for both arms, so there is no real difference to find.

    The warm-up spread is tiny but non-zero. Exactly-equal rows would not work:
    `_held_out_rows` keeps holding out rows while both arms are still constant,
    so the held-out window would swallow the ordinary rows too and the two
    spreads would agree. A saturated *judge* returns near-1.0, not bit-equal 1.0.
    """
    rng = random.Random(seed)
    warm = [1.0 + rng.gauss(0.0, 0.002) for _ in range(5)]
    ordinary = [
        round(min(1.0, max(0.0, rng.gauss(0.82, 0.05))), 3) for _ in range(rows)
    ]
    return warm + ordinary, warm + ordinary


# ---------------------------------------------------------------------------
# The hold names itself
# ---------------------------------------------------------------------------


class TestAcceptHoldNote:
    """A `continue` held by the 3x rule reports the hold and its two spreads."""

    def test_a_held_accept_says_why(self):
        """Saturated warm-up: `continue`, `accept_held` true, both spreads real.

        The premise is that the accept boundary is already crossed, so the hold
        is the only thing keeping this at `continue` — otherwise the new field
        would be reporting a reason that is not the reason.
        """
        from soup_cli.utils.ab_test import PRIOR_SCALE_ROWS

        control, treatment = _saturated_warm_up(40, seed=2)

        verdict = _step(control, treatment)

        assert verdict.decision == "continue", verdict.decision
        assert verdict.accept_held is True
        # Both spreads are present, finite, and the tested rows really are the
        # wider ones — that is the whole content of the note.
        assert verdict.held_out_spread is not None
        assert verdict.tested_spread is not None
        assert math.isfinite(verdict.held_out_spread)
        assert math.isfinite(verdict.tested_spread)
        assert verdict.held_out_spread > 0.0
        assert verdict.tested_spread > 3.0 * verdict.held_out_spread
        # It names how many rows set the scale, so the panel's advice points at
        # real rows rather than at the default.
        assert verdict.held_out_rows == PRIOR_SCALE_ROWS

    def test_a_saturated_start_that_is_bit_equal_reports_the_wider_window(self):
        """The held-out window grows while both arms are still exactly constant.

        The panel has to name the rows that actually set the scale, which is not
        always `PRIOR_SCALE_ROWS`: `_held_out_rows` keeps holding rows out while
        both arms are still bit-equal, so the count is only knowable per run.
        """
        rng = random.Random(7)
        from soup_cli.utils.ab_test import PRIOR_SCALE_ROWS

        control = [1.0] * 7 + [0.999] + [
            round(rng.gauss(0.82, 0.05), 3) for _ in range(60)
        ]
        treatment = [1.0] * 7 + [0.9991] + [
            round(rng.gauss(0.82, 0.05), 3) for _ in range(60)
        ]

        verdict = _step(control, treatment)

        assert verdict.decision == "continue", verdict.decision
        assert verdict.accept_held is True
        # Wider than PRIOR_SCALE_ROWS: the report must not hardcode the default.
        assert verdict.held_out_rows == 8
        assert verdict.held_out_rows != PRIOR_SCALE_ROWS
        assert verdict.tested_spread > 3.0 * verdict.held_out_spread

    def test_the_note_is_caused_by_the_hold_not_by_the_data(self, monkeypatch):
        """Remove the hold and the same rows accept: the field tracks the hold.

        Without this the first test would pass on any `continue` that happened to
        carry the two spreads, which is what a hardcoded note would do.
        """
        from soup_cli.utils import ab_test

        control, treatment = _saturated_warm_up(40, seed=2)

        monkeypatch.setattr(ab_test, "ACCEPT_HOLD_SPREAD_RATIO", math.inf)
        verdict = _step(control, treatment)

        assert verdict.decision == "accept_h0", verdict.decision
        assert verdict.accept_held is False

    def test_more_rows_of_the_same_kind_do_not_release_it(self):
        """The reason the note matters: the run stays held as rows arrive.

        The held-out rows are fixed, so the tested spread converges and the ratio
        only ever grows. A note that advised "collect more samples" would be
        advising the one thing that provably does not help.

        The invariant is that the hold never *releases* once it has fired, not
        that it fires on literally every peek: a peek can also sit at `continue`
        for want of tested rows before the accept branch is reached at all.
        """
        control, treatment = _saturated_warm_up(200, seed=2)

        rows = [
            _step(control[:n], treatment[:n])
            for n in range(7, len(control) + 1)
        ]
        assert all(v.decision == "continue" for v in rows)
        assert any(v.accept_held for v in rows), "the hold never fired"
        # From the first held peek on, every later peek is still held.
        first_held = next(i for i, v in enumerate(rows) if v.accept_held)
        assert all(v.accept_held for v in rows[first_held:]), [
            (rows[i].n_control, v.decision, v.accept_held)
            for i, v in enumerate(rows)
            if i >= first_held and not v.accept_held
        ]
        # Still held with the whole dataset in hand — the issue's "no decision in
        # 300 rows", here at 200.
        last = rows[-1]
        assert last.accept_held is True
        assert last.tested_spread > 3.0 * last.held_out_spread

    def test_an_unheld_continue_carries_no_note(self):
        """A `continue` that is merely short of evidence borrows no explanation.

        The control for the false explanation: ordinary Gaussian rows where the
        held-out and tested spreads agree must not report a hold, and must carry
        no spreads either, so a consumer reading the field cannot print the note
        for a run it does not describe.
        """
        rng = random.Random(1_524)
        control = [rng.gauss(0.0, 1.0) for _ in range(30)]
        treatment = [rng.gauss(0.0, 1.0) for _ in range(30)]

        verdict = _step(control, treatment, effect_size=2.0)

        assert verdict.decision == "accept_h0", verdict.decision
        assert verdict.accept_held is False
        assert verdict.held_out_spread is None
        assert verdict.tested_spread is None

    def test_a_reject_is_never_held(self):
        """A `reject_h0` clears the boundary above the hold, so it cannot carry one."""
        rng = random.Random(1_524)
        control = [rng.gauss(0.0, 1.0) for _ in range(40)]
        treatment = [rng.gauss(3.0, 1.0) for _ in range(40)]

        verdict = _step(control, treatment, effect_size=0.1)

        assert verdict.decision == "reject_h0", verdict.decision
        assert verdict.accept_held is False
        assert verdict.held_out_spread is None
        assert verdict.held_out_rows == 0

    def test_too_few_tested_rows_is_continue_without_a_note(self):
        """Still inside the warm-up the verdict has no tested rows to compare."""
        verdict = _step([1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0], [1.0] * 7)

        assert verdict.decision == "continue"
        assert verdict.accept_held is False

    def test_the_note_is_the_holds_own_ratio(self, monkeypatch):
        """A different ratio changes which runs are held, not the note's rule."""
        from soup_cli.utils import ab_test

        control, treatment = _saturated_warm_up(40, seed=2)

        monkeypatch.setattr(ab_test, "ACCEPT_HOLD_SPREAD_RATIO", 1e9)
        verdict = _step(control, treatment)

        # Same verdict object shape, hold off: accepts, and says nothing.
        assert verdict.decision == "accept_h0"
        assert verdict.accept_held is False


# ---------------------------------------------------------------------------
# The verdict field is validated like the rest
# ---------------------------------------------------------------------------


class TestAcceptHeldValidation:
    """`accept_held` cannot be set on a decision the hold does not produce."""

    def test_only_continue_can_be_held(self):
        from soup_cli.utils.ab_test import MsprtVerdict

        for decision in ("accept_h0", "reject_h0"):
            kwargs = {"direction": "better"} if decision == "reject_h0" else {}
            with pytest.raises(ValueError, match="continue"):
                MsprtVerdict(
                    decision=decision,
                    log_likelihood_ratio=0.0,
                    n_control=10,
                    n_treatment=10,
                    mean_control=1.0,
                    mean_treatment=1.0,
                    accept_held=True,
                    **kwargs,
                )

    def test_a_held_verdict_must_carry_both_spreads(self):
        """`accept_held` without the numbers behind it would be an unfalsifiable note."""
        from soup_cli.utils.ab_test import MsprtVerdict

        for missing in ("held_out_spread", "tested_spread"):
            fields = {"held_out_spread": 0.01, "tested_spread": 0.5}
            fields[missing] = None
            with pytest.raises(ValueError, match=missing):
                MsprtVerdict(
                    decision="continue",
                    log_likelihood_ratio=-2.0,
                    n_control=20,
                    n_treatment=20,
                    mean_control=1.0,
                    mean_treatment=1.0,
                    accept_held=True,
                    **fields,
                )

    def test_spreads_are_non_negative_and_finite(self):
        from soup_cli.utils.ab_test import MsprtVerdict

        for bad in (-0.1, math.inf, math.nan):
            with pytest.raises(ValueError, match="spread"):
                MsprtVerdict(
                    decision="continue",
                    log_likelihood_ratio=-2.0,
                    n_control=20,
                    n_treatment=20,
                    mean_control=1.0,
                    mean_treatment=1.0,
                    accept_held=True,
                    held_out_spread=0.01,
                    tested_spread=bad,
                )

    def test_spreads_without_a_hold_are_refused(self):
        """No hold, no explanation: the two must not float free of each other."""
        from soup_cli.utils.ab_test import MsprtVerdict

        with pytest.raises(ValueError, match="accept_held"):
            MsprtVerdict(
                decision="continue",
                log_likelihood_ratio=-2.0,
                n_control=20,
                n_treatment=20,
                mean_control=1.0,
                mean_treatment=1.0,
                held_out_spread=0.01,
                tested_spread=0.5,
            )

    def test_the_verdict_is_still_frozen(self):
        """The new fields do not cost the verdict its immutability."""
        from soup_cli.utils.ab_test import MsprtVerdict

        verdict = MsprtVerdict(
            decision="continue",
            log_likelihood_ratio=-2.0,
            n_control=20,
            n_treatment=20,
            mean_control=1.0,
            mean_treatment=1.0,
        )
        with pytest.raises(dataclasses.FrozenInstanceError):
            verdict.accept_held = True  # type: ignore[misc]


# ---------------------------------------------------------------------------
# The panel
# ---------------------------------------------------------------------------


def _write_jsonl(path, control, treatment, metric="judge_score"):
    rows = []
    for c, t in zip(control, treatment):
        rows.append(json.dumps({"arm": "control", metric: c}))
        rows.append(json.dumps({"arm": "treatment", metric: t}))
    path.write_text("\n".join(rows), encoding="utf-8")


def test_the_panel_names_the_hold_and_its_ratio(tmp_path, monkeypatch):
    """One actionable line: the ratio, and what to do about it."""
    from soup_cli.cli import app
    from soup_cli.utils.ab_test import MsprtConfig, run_msprt

    monkeypatch.chdir(tmp_path)
    control, treatment = _saturated_warm_up(40, seed=2)
    inp = tmp_path / "ab.jsonl"
    _write_jsonl(inp, control, treatment)
    verdict = run_msprt(str(inp), config=MsprtConfig(metric="judge_score"))
    assert verdict.accept_held is True, verdict  # the premise

    result = runner.invoke(app, ["ab", "--input", str(inp), "--metric", "judge_score"])
    assert result.exit_code == 0, (result.output, repr(result.exception))

    out = _plain(result.output).lower()
    ratio = verdict.tested_spread / verdict.held_out_spread
    assert f"held back: the tested rows spread {ratio:.3g} times" in out, out
    assert (
        f"(held_out_spread {verdict.held_out_spread:.4g}, "
        f"tested_spread {verdict.tested_spread:.4g}, limit 3x)"
    ) in out, out
    # The three details the note's advice rests on, pinned.
    assert "already inside +-0.1." in out, out
    assert "accept_held true" in out, out
    # Each spread appears exactly twice: once as a table row, once in the panel
    # prose. A count of 1 means one of the two was dropped.
    assert out.count(f"held_out_spread {verdict.held_out_spread:.4g}") == 2, out
    assert out.count(f"tested_spread {verdict.tested_spread:.4g}") == 2, out
    # It must not ALSO print the advice it replaces.
    assert "insufficient evidence" not in out, out


def test_the_panel_of_an_unheld_continue_is_unchanged(tmp_path, monkeypatch):
    """The control: an unheld continue past the warm-up still prints the old line."""
    from soup_cli.cli import app
    from soup_cli.utils.ab_test import MsprtConfig, run_msprt

    monkeypatch.chdir(tmp_path)
    rng = random.Random(0)
    control = [round(rng.gauss(0.80, 0.05), 3) for _ in range(10)]
    treatment = [round(rng.gauss(0.80, 0.05), 3) for _ in range(10)]
    inp = tmp_path / "ab.jsonl"
    _write_jsonl(inp, control, treatment)
    verdict = run_msprt(str(inp), config=MsprtConfig(metric="judge_score"))
    # The premise: past the warm-up, undecided, and not held.
    assert (verdict.decision, verdict.accept_held) == ("continue", False), verdict

    result = runner.invoke(app, ["ab", "--input", str(inp), "--metric", "judge_score"])
    assert result.exit_code == 0, (result.output, repr(result.exception))

    out = _plain(result.output).lower()
    assert "insufficient evidence. collect more samples and re-run." in out, out
    assert "held back" not in out, out
    assert "accept_held" not in out, out


def test_the_panel_names_the_rows_that_actually_set_the_scale(tmp_path, monkeypatch):
    """A bit-equal warm-up holds out more than ``PRIOR_SCALE_ROWS`` rows.

    ``PRIOR_SCALE_ROWS`` is 5, so a panel that printed the constant would be
    right on every input but this one. ``_held_out_rows`` keeps holding rows out
    while both arms are still bit-equal, so the count is only knowable per run
    and the panel has to name the real one.
    """
    from soup_cli.cli import app
    from soup_cli.utils.ab_test import MsprtConfig, run_msprt

    monkeypatch.chdir(tmp_path)
    rng = random.Random(7)
    control = [1.0] * 7 + [0.999] + [round(rng.gauss(0.82, 0.05), 3) for _ in range(60)]
    treatment = [1.0] * 7 + [0.9991] + [round(rng.gauss(0.82, 0.05), 3) for _ in range(60)]
    inp = tmp_path / "ab.jsonl"
    _write_jsonl(inp, control, treatment)
    verdict = run_msprt(str(inp), config=MsprtConfig(metric="judge_score"))
    assert verdict.accept_held is True, verdict  # the premise
    assert verdict.held_out_rows != 5, verdict.held_out_rows  # the premise

    result = runner.invoke(app, ["ab", "--input", str(inp), "--metric", "judge_score"])
    assert result.exit_code == 0, (result.output, repr(result.exception))

    out = _plain(result.output).lower()
    assert f"held_out_rows {verdict.held_out_rows}" in out, out
    assert f"the first {verdict.held_out_rows} rows of each arm set the scale" in out, out


# ---------------------------------------------------------------------------
# JSON output stays backward compatible
# ---------------------------------------------------------------------------


def test_the_verdict_fields_are_added_not_renamed():
    """Every field #1265 shipped is still there, and the new ones are additive."""
    import dataclasses

    from soup_cli.utils.ab_test import MsprtVerdict

    names = {f.name for f in dataclasses.fields(MsprtVerdict)}
    assert {
        "decision",
        "log_likelihood_ratio",
        "n_control",
        "n_treatment",
        "mean_control",
        "mean_treatment",
        "direction",
    } <= names
    assert {"accept_held", "held_out_spread", "tested_spread", "held_out_rows"} <= names


def test_the_webhook_payload_gains_the_keys_without_losing_any(tmp_path, monkeypatch):
    """The POSTed verdict gains keys; every key #1265 shipped is still there.

    Run through the real command so the test fails if `ab.py` drops or renames a
    key, rather than only checking a dict the test wrote itself.
    """
    from soup_cli.cli import app
    from soup_cli.commands import ab as ab_cmd

    sent: list[dict] = []
    monkeypatch.setattr(
        ab_cmd, "emit_webhooks",
        lambda slack, discord, *, payload, console: sent.append(payload),
    )

    monkeypatch.chdir(tmp_path)
    rng = random.Random(1_524)
    control = [rng.gauss(0.0, 1.0) for _ in range(40)]
    treatment = [rng.gauss(3.0, 1.0) for _ in range(40)]
    inp = tmp_path / "ab.jsonl"
    _write_jsonl(inp, control, treatment)

    result = runner.invoke(
        app,
        [
            "ab", "--input", str(inp), "--metric", "judge_score",
            "--slack-url", "https://hooks.slack.com/services/T/B/X",
        ],
    )
    assert result.exit_code == 0, (result.output, repr(result.exception))
    assert len(sent) == 1, sent

    payload = sent[0]
    # Every key that was there before, unchanged. The four #1524 keys are NOT
    # here on purpose: a held run is a `continue` and the webhook only fires on
    # a terminal decision, so they could only ever be false / null.
    assert set(payload) >= {
        "command",
        "metric",
        "decision",
        "direction",
        "log_likelihood_ratio",
        "n_control",
        "n_treatment",
        "mean_control",
        "mean_treatment",
    }
    # Nothing renamed: the old names must not have become aliases.
    assert not {"spread_ratio", "held", "held_back"} & set(payload)
    # No hold keys: there is nothing to hold back on a terminal decision.
    assert not {"accept_held", "held_out_spread", "tested_spread", "held_out_rows"} & set(payload)
    assert payload["decision"] == "reject_h0"
    # Still JSON-serialisable end to end.
    json.dumps(payload)


def test_run_msprt_reports_the_hold_from_a_file(tmp_path, monkeypatch):
    """The JSONL path carries the note too, not just `msprt_step`."""
    from soup_cli.utils.ab_test import MsprtConfig, run_msprt

    monkeypatch.chdir(tmp_path)
    control, treatment = _saturated_warm_up(40, seed=2)
    inp = tmp_path / "ab.jsonl"
    _write_jsonl(inp, control, treatment)

    verdict = run_msprt(str(inp), config=MsprtConfig(metric="judge_score"))

    assert verdict.decision == "continue"
    assert verdict.accept_held is True
    assert verdict.tested_spread > 3.0 * verdict.held_out_spread