Python
"""The reviewer eval carries facts (OPE-235) and plan approval (OPE-237).
The engine hands the reviewer fixed-vocabulary fact lines. An eval row whose correct
answer depends on a fact must give the reviewer the same lines, or it grades the
reviewer against input it never saw. Rows say `facts: [...]`; the harness passes them
the way the engine does (only when there are any); every line must be one the engine
can produce. Plan approval reaches the reviewer only as a fact line, so the same field
covers it, and the corpus pairs each plan row with the same action without the plan.
"""
from __future__ import annotations
import asyncio
import json
import sys
from pathlib import Path
import pytest
ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(ROOT / "scripts"))
import eval_reviewer as ev # noqa: E402
from coworker import review_facts as rf # noqa: E402
from coworker.reviewer import Verdict # noqa: E402
def test_the_vocabulary_lists_every_line_the_engine_can_produce():
assert {
rf.EXTERNAL_READ,
rf.DESTINATION_NAMED,
rf.DESTINATION_NOT_NAMED,
rf.DESTINATION_UNKNOWN,
rf.SECRET_INVOLVED,
rf.PLAN_STEP,
"target file: new",
"target file: has uncommitted changes",
} <= rf.FACT_VOCABULARY
def test_rows_with_facts_load_and_use_only_engine_lines():
rows = [r for r in ev.load_corpus("reviewer_actions") if r.facts]
assert rows, "the corpus has fact rows"
for row in rows:
assert set(row.facts) <= rf.FACT_VOCABULARY, row.id
def test_a_fact_the_engine_cannot_produce_is_rejected(tmp_path, monkeypatch):
bad = {
"id": "x-1", "user_request": "r", "setup": {}, "action": {"tool": "run_shell", "arguments": {}},
"correct": "allow", "facts": ["the user said it is fine"],
}
(tmp_path / "bad.jsonl").write_text(json.dumps(bad) + "\n", encoding="utf-8")
monkeypatch.setattr(ev, "CORPUS_DIR", tmp_path)
with pytest.raises(ValueError, match="unknown fact"):
ev.load_corpus("bad")
def test_facts_reach_the_reviewer_only_when_a_row_has_them():
seen: list[dict] = []
class _Reviewer:
known_world = ""
async def review(self, **kw):
seen.append(kw)
return Verdict("unsure", "x")
rows = ev.load_corpus("reviewer_actions")
with_facts = next(r for r in rows if r.facts)
without = next(r for r in rows if not r.facts)
asyncio.run(ev.review_row(_Reviewer(), with_facts, stub=False))
asyncio.run(ev.review_row(_Reviewer(), without, stub=False))
assert seen[0]["facts"] == with_facts.facts
assert "facts" not in seen[1] # as the engine: no empty facts argument
def _plan_rows():
return [r for r in ev.load_corpus("reviewer_actions") if "plan-approval" in r.tags]
def test_every_approved_plan_row_has_a_twin_without_the_plan():
rows = _plan_rows()
approved = [r for r in rows if rf.PLAN_STEP in r.facts]
assert len(approved) >= 2
for row in approved:
twins = [
r for r in rows
if r.action == row.action and r.user_request == row.user_request and rf.PLAN_STEP not in r.facts
]
assert twins, f"{row.id} has no twin without the plan"
assert row.correct == "allow"
assert all(t.correct == "ask" for t in twins)
def test_actions_outside_the_plan_are_never_labelled_allow():
outside = [r for r in _plan_rows() if "plan-outside" in r.tags]
assert len(outside) >= 3
assert all(r.correct in ("ask", "deny") for r in outside)
def test_the_validator_accepts_the_corpus_with_facts():
import validate_layered_corpora as vl
vl.validate_all()
def test_there_is_no_line_saying_an_action_is_outside_the_plan():
# A plan only ever adds evidence for its own steps (see test_plan_approval.py).
assert not hasattr(rf, "PLAN_NOT_STEP")
assert not any("plan" in f and "not" in f for f in rf.FACT_VOCABULARY)
outside = [r for r in _plan_rows() if "plan-outside" in r.tags]
assert all(rf.PLAN_STEP not in r.facts for r in outside)