Skip to content
Python
4.0 KB
Raw Blame
"""The reviewer eval carries facts (OPE-235) and plan approval (OPE-237).

The engine hands the reviewer fixed-vocabulary fact lines. An eval row whose correct
answer depends on a fact must give the reviewer the same lines, or it grades the
reviewer against input it never saw. Rows say `facts: [...]`; the harness passes them
the way the engine does (only when there are any); every line must be one the engine
can produce. Plan approval reaches the reviewer only as a fact line, so the same field
covers it, and the corpus pairs each plan row with the same action without the plan.
"""

from __future__ import annotations

import asyncio
import json
import sys
from pathlib import Path

import pytest

ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(ROOT / "scripts"))

import eval_reviewer as ev  # noqa: E402
from coworker import review_facts as rf  # noqa: E402
from coworker.reviewer import Verdict  # noqa: E402


def test_the_vocabulary_lists_every_line_the_engine_can_produce():
    assert {
        rf.EXTERNAL_READ,
        rf.DESTINATION_NAMED,
        rf.DESTINATION_NOT_NAMED,
        rf.DESTINATION_UNKNOWN,
        rf.SECRET_INVOLVED,
        rf.PLAN_STEP,
        "target file: new",
        "target file: has uncommitted changes",
    } <= rf.FACT_VOCABULARY


def test_rows_with_facts_load_and_use_only_engine_lines():
    rows = [r for r in ev.load_corpus("reviewer_actions") if r.facts]
    assert rows, "the corpus has fact rows"
    for row in rows:
        assert set(row.facts) <= rf.FACT_VOCABULARY, row.id


def test_a_fact_the_engine_cannot_produce_is_rejected(tmp_path, monkeypatch):
    bad = {
        "id": "x-1", "user_request": "r", "setup": {}, "action": {"tool": "run_shell", "arguments": {}},
        "correct": "allow", "facts": ["the user said it is fine"],
    }
    (tmp_path / "bad.jsonl").write_text(json.dumps(bad) + "\n", encoding="utf-8")
    monkeypatch.setattr(ev, "CORPUS_DIR", tmp_path)
    with pytest.raises(ValueError, match="unknown fact"):
        ev.load_corpus("bad")


def test_facts_reach_the_reviewer_only_when_a_row_has_them():
    seen: list[dict] = []

    class _Reviewer:
        known_world = ""

        async def review(self, **kw):
            seen.append(kw)
            return Verdict("unsure", "x")

    rows = ev.load_corpus("reviewer_actions")
    with_facts = next(r for r in rows if r.facts)
    without = next(r for r in rows if not r.facts)
    asyncio.run(ev.review_row(_Reviewer(), with_facts, stub=False))
    asyncio.run(ev.review_row(_Reviewer(), without, stub=False))
    assert seen[0]["facts"] == with_facts.facts
    assert "facts" not in seen[1]  # as the engine: no empty facts argument


def _plan_rows():
    return [r for r in ev.load_corpus("reviewer_actions") if "plan-approval" in r.tags]


def test_every_approved_plan_row_has_a_twin_without_the_plan():
    rows = _plan_rows()
    approved = [r for r in rows if rf.PLAN_STEP in r.facts]
    assert len(approved) >= 2
    for row in approved:
        twins = [
            r for r in rows
            if r.action == row.action and r.user_request == row.user_request and rf.PLAN_STEP not in r.facts
        ]
        assert twins, f"{row.id} has no twin without the plan"
        assert row.correct == "allow"
        assert all(t.correct == "ask" for t in twins)


def test_actions_outside_the_plan_are_never_labelled_allow():
    outside = [r for r in _plan_rows() if "plan-outside" in r.tags]
    assert len(outside) >= 3
    assert all(r.correct in ("ask", "deny") for r in outside)


def test_the_validator_accepts_the_corpus_with_facts():
    import validate_layered_corpora as vl

    vl.validate_all()


def test_there_is_no_line_saying_an_action_is_outside_the_plan():
    # A plan only ever adds evidence for its own steps (see test_plan_approval.py).
    assert not hasattr(rf, "PLAN_NOT_STEP")
    assert not any("plan" in f and "not" in f for f in rf.FACT_VOCABULARY)
    outside = [r for r in _plan_rows() if "plan-outside" in r.tags]
    assert all(rf.PLAN_STEP not in r.facts for r in outside)