Python
"""Decision-model reviewers for Auto-Approve (OPE-224): OpenAI Decisions and TypeSafe Jev.
Everything here is offline. Transports are fakes; no test calls a vendor."""
from __future__ import annotations
import asyncio
import json
import pytest
from fastapi.testclient import TestClient
from coworker import reviewer as reviewer_mod
from coworker.config import _GLOBAL_ONLY_FIELDS
from coworker.decision_reviewer import (
AGENT_FILE_FLAG,
ASK_SIGNALS,
BACKENDS,
COMBINED_NAMES,
DECISIONS_QUESTIONS,
JEV_GATES,
JEV_ORDER,
JEV_QUESTIONS,
JEV_SPLITS,
STOP_BLOCKS_BELOW_REQUESTED,
STOP_SIGNALS,
DecisionReviewer,
JevBackend,
OpenAIDecisionsBackend,
build_decision_reviewer,
combine_answers,
combine_jev,
derive_jev,
evidence_state,
make_auto_reviewer,
normalize_reviewer_choice,
)
from coworker.events import EventType
from coworker.server.app import create_app
from coworker.server.manager import SessionManager
from test_auto_approve import _engine, _run, _tool_turn
from coworker.providers import AssistantTurn
DECISIONS = OpenAIDecisionsBackend
JEV = JevBackend
class _Recorder:
def __init__(self, reply):
self.reply = reply
self.bodies: list[tuple[str, dict, dict]] = []
def __call__(self, url, headers, body):
self.bodies.append((url, headers, body))
return self.reply(url, body)
def _answers(values: dict[str, float]):
"""A reply in whichever vendor shape the URL names, giving `values` (default 0.02)."""
def reply(url, body):
qs = body["questions"]
names = list(qs) if isinstance(qs, dict) else [q["name"] for q in qs]
p = {n: values.get(n, 0.02) for n in names}
if url.endswith("/v1/decisions"):
return 200, {"model": "m", "answers": [
{"name": n, "type": "predicate", "probability": v} for n, v in p.items()
], "usage": {"input_tokens": 7}}
return 200, {"model": "m", "answers": {n: {"type": "noul", "noul": v} for n, v in p.items()},
"usage": {"input_tokens": 7}}
return reply
def _review(rv, **kw):
args = dict(request="run the test suite", history=[], tool_name="run_shell", arguments={"command": "pytest -q"})
args.update(kw)
return asyncio.run(rv.review(**args))
def _rv(backend_cls, values=None, reply=None):
rec = _Recorder(reply or _answers({"requested": 0.97, **(values or {})}))
return DecisionReviewer(backend_cls(api_key="k", transport=rec)), rec
# -- what each vendor is sent --------------------------------------------------------------
def test_decisions_request_is_plain_predicates_over_json_text():
rv, rec = _rv(DECISIONS)
_review(rv)
url, headers, body = rec.bodies[0]
assert url.endswith("/v1/decisions") and body["model"] == "gpt-6-luna"
assert headers["Authorization"] == "Bearer k"
assert isinstance(body["input"], str) and json.loads(body["input"])["current_request"] == "run the test suite"
assert [q["name"] for q in body["questions"]] == [q.name for q in DECISIONS_QUESTIONS]
for q in body["questions"]:
assert set(q) == {"name", "type", "instructions"} and q["type"] == "predicate"
assert "`" not in q["instructions"]
def test_jev_request_is_structured_nouls_with_criteria():
rv, rec = _rv(JEV)
_review(rv)
url, _, body = rec.bodies[0]
assert url.endswith("/v1/systemone") and body["model"] == "jev-1.13.0"
assert isinstance(body["state"], dict)
assert set(body["questions"]) == {q.name for q in JEV_QUESTIONS}
for name, spec in body["questions"].items():
assert spec["type"] == "noul"
assert spec["instructions"]["question"]
assert {"what", "examples"} <= set(spec["criteria"]["true"]), name
assert {"what", "examples"} <= set(spec["criteria"]["false"]), name
def test_evidence_state_carries_facts_not_provenance():
state = evidence_state(
known_world="KNOWN WORLD\n folder /repo [read-write]", history=[{"text": "set up the repo"}],
request="run it", tool_name="run_shell", arguments={"command": "make"},
provenance="scripts/x.sh was written by the agent this session",
)
assert state["environment"]["working_folders"] == [{"path": "/repo", "access": "read-write"}]
assert "provenance" not in json.dumps(state) and "written by the agent" not in json.dumps(state)
# -- frozen settings ------------------------------------------------------------------------
def test_cut_offs_and_models_are_frozen_per_backend():
assert (DECISIONS.run_threshold, DECISIONS.risk_threshold, DECISIONS.stop_threshold) == (0.80, 0.40, 0.60)
assert (JEV.run_threshold, JEV.risk_threshold, JEV.stop_threshold) == (0.80, 0.40, 0.60)
assert DECISIONS.default_model == "gpt-6-luna"
assert JEV.default_model == "jev-1.13.0" # pinned, not jev-latest (the docs: versioned ids carry the patch number)
# -- combining answers ---------------------------------------------------------------------
def _p(**over):
p = {n: 0.02 for n in COMBINED_NAMES}
p["requested"] = 0.97
p.update(over)
return p
def _combine(p, run=0.80):
return combine_answers(p, run_threshold=run, stop_threshold=0.60, risk_threshold=0.40)[0]
def test_plain_requested_action_is_allowed():
assert _combine(_p()) == "allow"
@pytest.mark.parametrize("signal", STOP_SIGNALS)
def test_every_stop_signal_blocks_an_action_nobody_asked_for(signal):
assert _combine(_p(**{signal: 0.9, "requested": 0.10})) == "deny"
@pytest.mark.parametrize("signal", STOP_SIGNALS)
def test_card_rule_a_stop_signal_on_a_requested_action_asks_instead(signal):
# "The user asked for this" and "this looks wrong" contradict each other: the person decides.
verdict, reason = combine_answers(_p(**{signal: 0.9, "requested": 0.60}),
run_threshold=0.80, stop_threshold=0.60, risk_threshold=0.40)
assert verdict == "unsure"
assert reason.startswith("requested 0.60 but ") and signal in reason
@pytest.mark.parametrize("requested, verdict", [(0.49, "deny"), (0.50, "unsure"), (0.97, "unsure")])
def test_card_rule_boundary(requested, verdict):
assert STOP_BLOCKS_BELOW_REQUESTED == 0.50
assert _combine(_p(contrary=0.9, requested=requested)) == verdict
@pytest.mark.parametrize("signal", ASK_SIGNALS)
def test_every_ask_signal_asks(signal):
assert _combine(_p(**{signal: 0.5})) == "unsure"
def test_not_requested_enough_asks():
assert _combine(_p(requested=0.85), run=0.90) == "unsure"
@pytest.mark.parametrize("outside, asked, verdict", [(0.9, 0.9, "allow"), (0.9, 0.1, "unsure"), (0.1, 0.1, "allow")])
def test_outside_write_counts_only_when_not_asked_for(outside, asked, verdict):
assert _combine(_p(outside_folders=outside, outside_change_requested=asked)) == verdict
def test_agent_made_file_asks():
assert _combine({**_p(), AGENT_FILE_FLAG: 1.0}) == "unsure"
def test_missing_or_bad_answers_ask():
p = _p()
del p["moves_money"]
assert _combine(p) == "unsure"
assert _combine(_p(broader=1.5)) == "unsure"
@pytest.mark.parametrize("parts, derived, expected", [
({"shared_item": 0.9, "item_written": 0.1, "one_meaning": 0.1}, "target_unpinned", 0.9 * 0.9 * 0.9),
({"has_recipient": 0.9, "recipient_written": 0.95}, "outside_recipient", 0.9 * 0.05),
({"sets_hour": 0.98, "hour_given": 0.02}, "agent_chose_time", 0.98 * 0.98),
({"user_named_item": 0.9, "names_other_item": 0.8}, "target_mismatch", 0.9 * 0.8),
({"user_named_place": 0.9, "names_other_place": 0.8}, "place_mismatch", 0.9 * 0.8),
({"user_restriction": 0.9, "breaks_restriction": 0.8}, "contrary", 0.9 * 0.8),
({"delegated_task": 0.9, "only_reads": 0.8}, "hand_off_read", 0.9 * 0.8),
({"named_place": 0.9, "acts_in_named_place": 0.1}, "named_mismatch", 0.9 * 0.9),
({"named_target": 0.95, "acts_on_named": 0.1}, "named_mismatch", 0.95 * 0.9),
])
def test_jev_split_parts_combine_by_product(parts, derived, expected):
base = {q.name: 0.02 for q in JEV_QUESTIONS}
base.update(parts)
assert derive_jev(base)[derived] == pytest.approx(expected, abs=0.02)
def _cj(**over):
base = {q.name: 0.02 for q in JEV_QUESTIONS}
base["requested"] = 0.97
base.update(over)
return combine_jev(derive_jev(base), run_threshold=0.80, stop_threshold=0.60, risk_threshold=0.40)[0]
def test_jev_missing_part_fails_closed():
base = {q.name: 0.02 for q in JEV_QUESTIONS}
base["requested"] = 0.97
del base["acts_in_named_place"]
assert combine_jev(derive_jev(base), run_threshold=0.80, stop_threshold=0.60, risk_threshold=0.40)[0] == "unsure"
for cond, nots in JEV_SPLITS.values():
assert {cond, *nots} <= {q.name for q in JEV_QUESTIONS}
for gate, q in JEV_GATES.values():
assert {gate, q} <= {q.name for q in JEV_QUESTIONS}
assert len(JEV_ORDER) == len(set(JEV_ORDER)) == len(JEV_QUESTIONS)
def test_jev_plain_requested_action_is_allowed():
assert _cj() == "allow"
assert _cj(requested=0.85) == "allow" # allow line 0.80
assert _cj(requested=0.79) == "unsure"
def test_jev_a_wrong_place_the_user_named_blocks_or_asks():
assert _cj(user_named_place=0.95, names_other_place=0.9, requested=0.10) == "deny"
assert _cj(user_named_place=0.95, names_other_place=0.9) == "unsure" # card rule
# "different place" with no place named by the user is not a mismatch
assert _cj(user_named_place=0.05, names_other_place=0.95) == "allow"
def test_jev_named_item_or_place_net_turns_an_allow_into_a_card_and_nothing_else():
# Measured 2026-10-26: the gated questions let five wrong places through on the sealed rows
# that this check catches. It only ever turns an allow into a card.
assert _cj(named_place=0.95, acts_in_named_place=0.05) == "unsure"
assert _cj(named_target=0.95, acts_on_named=0.05) == "unsure"
assert _cj(named_place=0.95, acts_in_named_place=0.5) == "allow" # 0.475, under 0.60
assert _cj(named_place=0.95, acts_in_named_place=0.05, requested=0.10) == "unsure" # never a block
def test_jev_restriction_counts_only_when_the_user_set_one():
assert _cj(breaks_restriction=0.95) == "allow"
assert _cj(user_restriction=0.95, breaks_restriction=0.95, requested=0.10) == "deny"
def test_jev_a_read_under_a_hand_off_request_asks_instead_of_blocking():
hit = dict(unrelated_destination=0.9, requested=0.05, delegated_task=0.9)
assert _cj(**hit, only_reads=0.9) == "unsure"
assert _cj(**hit, only_reads=0.1) == "deny" # a send under the same request stays blocked
assert _cj(unrelated_destination=0.9, requested=0.05, only_reads=0.9) == "deny" # no hand-off
def test_jev_deleting_something_unnamed_asks():
assert _cj(deletes_unnamed=0.9) == "unsure"
def test_state_budget_fits_jevs_per_question_limit():
# TypeSafe: 32k tokens for the state plus any one question. At a conservative 3 characters
# per token, the largest state we send plus the largest Jev question must stay under it.
from coworker.decision_reviewer import JEV_STATE_TOKEN_LIMIT, _STATE_CHAR_BUDGET
rec = _Recorder(_answers({}))
_, _, body = JEV(api_key="k", transport=rec).build_questions_request({}, JEV_QUESTIONS)
largest_question = max(len(json.dumps(q, ensure_ascii=False)) for q in body["questions"].values())
assert (_STATE_CHAR_BUDGET + largest_question) / 3 <= JEV_STATE_TOKEN_LIMIT
assert _STATE_CHAR_BUDGET == 90_000
@pytest.mark.parametrize("backend", [DECISIONS, JEV])
def test_oversized_state_fails_closed_without_a_vendor_call(backend):
from coworker.decision_reviewer import _STATE_CHAR_BUDGET
rv, rec = _rv(backend)
# Under the 64k approval-context cap, but the action's own arguments push the state over.
v = _review(rv, tool_name="write_file",
arguments={"path": "big.txt", "content": "x" * (_STATE_CHAR_BUDGET + 1)})
assert v.verdict == "unsure" and v.error
assert rec.bodies == []
def test_jev_examples_are_not_eval_rows():
from scripts import eval_reviewer as ev
requests = {r.user_request.strip().lower() for n in ev.CORPORA for r in ev.load_corpus(n)}
for q in JEV_QUESTIONS:
for side in (q.true, q.false):
for ex in side.get("examples", []):
assert ex.strip().lower() not in requests, (q.name, ex)
# -- end to end through a backend ----------------------------------------------------------
_CONTRARY = {DECISIONS: {"contrary": 0.95}, JEV: {"user_restriction": 0.95, "breaks_restriction": 0.95}}
@pytest.mark.parametrize("backend", [DECISIONS, JEV])
def test_end_to_end_allow_ask_stop(backend):
assert _review(_rv(backend)[0]).verdict == "allow"
assert _review(_rv(backend, {"moves_money": 0.9})[0]).verdict == "unsure"
assert _review(_rv(backend, {**_CONTRARY[backend], "requested": 0.10})[0]).verdict == "deny"
# Card rule: the same stop signal on an action the user asked for asks instead of blocking.
assert _review(_rv(backend, _CONTRARY[backend])[0]).verdict == "unsure"
def test_decisions_place_mismatch_stops():
assert _review(_rv(DECISIONS, {"place_mismatch": 0.9, "requested": 0.10})[0]).verdict == "deny"
def test_jev_wrong_place_stops():
assert _review(_rv(JEV, {"user_named_place": 0.95, "names_other_place": 0.95, "requested": 0.10})[0]).verdict == "deny"
def test_jev_named_place_net_end_to_end():
v = _review(_rv(JEV, {"named_place": 0.95, "acts_in_named_place": 0.05})[0])
assert v.verdict == "unsure" and "named-item/place check" in v.reason
@pytest.mark.parametrize("backend", [DECISIONS, JEV])
def test_no_reason_for_the_card_but_a_full_one_for_the_audit(backend):
v = _review(_rv(backend, {"broader": 0.9})[0])
assert v.verdict == "unsure" and v.card_reason == ""
assert "broader" in v.reason and v.probabilities["broader"] == pytest.approx(0.9)
# -- failing closed -------------------------------------------------------------------------
@pytest.mark.parametrize("reply", [
lambda url, body: (500, {"error": "boom"}),
lambda url, body: (200, "not json"),
lambda url, body: (200, {"answers": {}}),
])
@pytest.mark.parametrize("backend", [DECISIONS, JEV])
def test_machinery_failures_are_unsure_with_error(backend, reply):
v = _review(_rv(backend, reply=reply)[0])
assert v.verdict == "unsure" and v.error and v.card_reason == ""
def test_missing_key_is_unsure_with_error(monkeypatch):
monkeypatch.delenv("TYPESAFE_API_KEY", raising=False)
v = _review(DecisionReviewer(JEV(transport=_Recorder(_answers({})))))
assert v.verdict == "unsure" and v.error and "no API key" in v.reason
def test_refusal_is_unsure_and_counted_apart():
def refuse(url, body):
return 200, {"model": "m", "answers": [
{"name": q["name"], "type": "refusal"} if q["name"] == "exposes_secret"
else {"name": q["name"], "type": "predicate", "probability": 0.02}
for q in body["questions"]
]}
v = _review(_rv(DECISIONS, reply=refuse)[0])
assert v.verdict == "unsure" and v.refused and not v.error
def test_key_saved_later_is_used_without_a_rebuild(monkeypatch):
monkeypatch.delenv("TYPESAFE_API_KEY", raising=False)
class _Secrets:
store: dict = {}
def get(self, name):
return self.store.get(name)
secrets = _Secrets()
rec = _Recorder(_answers({"requested": 0.97}))
rv = DecisionReviewer(JEV(secrets=secrets, transport=rec))
assert _review(rv).error # no key yet
secrets.store["provider:typesafe"] = {"api_key": "later"}
assert _review(rv).verdict == "allow"
assert rec.bodies[-1][1]["Authorization"] == "Bearer later"
# -- choosing the reviewer ------------------------------------------------------------------
def test_factory_builds_the_chosen_reviewer():
session = make_auto_reviewer("session", provider=None, model="m", known_world="")
assert isinstance(session, reviewer_mod.Reviewer)
for name, cls in BACKENDS.items():
rv = make_auto_reviewer(name, provider=None, model="m", known_world="KW", secrets=object())
assert isinstance(rv, DecisionReviewer) and isinstance(rv.backend, cls) and rv.known_world == "KW"
assert normalize_reviewer_choice("JEV") == "jev"
assert normalize_reviewer_choice("nonsense") == "session"
assert normalize_reviewer_choice(None) == "session"
def test_a_repository_cannot_choose_the_reviewer():
assert "auto_approve_reviewer" in _GLOBAL_ONLY_FIELDS
@pytest.fixture
def client(tmp_path, monkeypatch):
monkeypatch.setenv("COWORKER_STATE_DIR", str(tmp_path / "state"))
monkeypatch.delenv("TYPESAFE_API_KEY", raising=False)
monkeypatch.delenv("OPENAI_API_KEY", raising=False)
return TestClient(create_app(SessionManager(data_dir=tmp_path / "data")))
def test_setting_defaults_to_the_session_model_and_round_trips(client):
assert client.get("/v1/settings").json()["auto_approve_reviewer"] == "session"
r = client.post("/v1/settings/auto-approve-reviewer", json={"auto_approve_reviewer": "openai-decisions"}).json()
assert r["ok"] and r["auto_approve_reviewer"] == "openai-decisions"
assert client.get("/v1/settings").json()["auto_approve_reviewer"] == "openai-decisions"
bad = client.post("/v1/settings/auto-approve-reviewer", json={"auto_approve_reviewer": "gpt"}).json()
assert not bad["ok"]
def test_reviewer_key_is_saved_and_never_echoed(client):
s = client.get("/v1/settings").json()
assert s["auto_approve_reviewer_keys"] == {"jev": False, "openai-decisions": False}
r = client.post("/v1/settings/auto-approve-reviewer-key", json={"reviewer": "jev", "api_key": "ts-secret"}).json()
assert r["ok"] and r["auto_approve_reviewer_keys"]["jev"] is True
assert "ts-secret" not in json.dumps(r) and "ts-secret" not in json.dumps(client.get("/v1/settings").json())
assert not client.post("/v1/settings/auto-approve-reviewer-key", json={"reviewer": "jev", "api_key": ""}).json()["ok"]
# -- what the approval card shows ------------------------------------------------------------
class _DecisionLike:
"""A reviewer returning DecisionVerdicts, scripted."""
def __init__(self, verdict: str, *, error: bool = False):
self.verdict, self.error = verdict, error
async def review(self, **_kw):
from coworker.decision_reviewer import DecisionVerdict
return DecisionVerdict(self.verdict, "requested 0.97; broader 0.90", error=self.error)
def _card(tmp_path, reviewer):
engine, rows, approvals = _engine(
tmp_path,
[_tool_turn(("run_shell", {"command": "git push"})), AssistantTurn(text="ok", finish_reason="stop")],
)
engine.reviewer = reviewer
events = _run(engine)
cards = [ev for ev in events if ev.type == EventType.PERMISSION_REQUIRED]
return cards[0].data, rows
def test_decision_reviewer_ask_shows_no_reason_on_the_card(tmp_path):
data, rows = _card(tmp_path, _DecisionLike("unsure"))
assert "reviewer_unsure" not in data
assert data["escalation"] == {"kind": "reviewer_asks", "reason": ""}
# The audit still records what fired.
assert any(r.get("stage") == "reviewer_verdict" and "broader" in r.get("reason", "") for r in rows)
def test_decision_reviewer_failure_says_it_could_not_review(tmp_path):
data, _ = _card(tmp_path, _DecisionLike("unsure", error=True))
assert data["escalation"] == {"kind": "reviewer_unavailable", "reason": ""}
def test_session_reviewer_reason_still_shows(tmp_path):
from test_auto_approve import _FakeReviewer
data, _ = _card(tmp_path, _FakeReviewer({"run_shell": "unsure"}))
assert data["reviewer_unsure"] == "scripted unsure"
# -- the eval script ------------------------------------------------------------------------
@pytest.mark.parametrize("backend", sorted(BACKENDS))
def test_eval_stub_runs_each_decision_reviewer(backend, capsys):
from scripts import eval_reviewer as ev
rc = ev.main(["--reviewer", backend, "--stub", "--corpus", "reviewer_actions"])
out = capsys.readouterr().out
assert rc == 0, out
assert f"{backend}:" in out
# -- shadow results: summary and CSV export ---------------------------------------------------
def _shadow(store, call, verdict, reviewer="jev", decision=None, approval="once"):
store.append({"session_id": "s1", "tool": "run_shell", "arguments": {"command": f"cmd {call}"},
"stage": "reviewer_shadow", "status": verdict, "reason": f"r {call}",
"call_id": call, "reviewer": reviewer})
if decision:
store.append({"session_id": "s1", "tool": "run_shell", "arguments": {}, "stage": "approval_resolved",
"status": decision, "approval": approval, "call_id": call})
def test_shadow_summary_counts_and_matches(tmp_path):
from coworker.audit import AuditStore
store = AuditStore(tmp_path / "a.db")
_shadow(store, "c1", "allow", decision="approved") # run, you allowed: match
_shadow(store, "c2", "allow", decision="denied") # run, you denied: no match
_shadow(store, "c3", "deny", decision="denied") # stop, you denied: match
_shadow(store, "c4", "unsure", decision="approved") # asked: not counted either way
_shadow(store, "c5", "allow") # never answered: not counted
_shadow(store, "c6", "deny", decision="denied", approval="auto_refused") # nobody decided
_shadow(store, "c7", "allow", reviewer="openai-decisions", decision="approved") # other reviewer
s = store.shadow_summary("jev")
assert (s["checks"], s["run"], s["asked"], s["stopped"]) == (6, 3, 1, 2)
assert (s["decided"], s["matched"]) == (3, 2)
assert store.shadow_summary("openai-decisions")["checks"] == 1
def test_shadow_export_writes_a_readable_csv(client, tmp_path):
import csv
client.post("/v1/settings/auto-approve-reviewer", json={"auto_approve_reviewer": "jev"})
mgr = client.app.state.manager if hasattr(client.app.state, "manager") else None
store = mgr.audit_store if mgr else None
if store is None:
pytest.skip("manager not reachable from the test client")
_shadow(store, "c1", "allow", decision="denied")
summary = client.get("/v1/settings/auto-approve-shadow-summary").json()
assert summary["reviewer"] == "jev" and summary["checks"] == 1 and summary["matched"] == 0
r = client.post("/v1/settings/auto-approve-shadow-export").json()
assert r["ok"] and r["rows"] == 1 and r["path"].endswith(".csv")
with open(r["path"], encoding="utf-8-sig", newline="") as fh:
rows = list(csv.DictReader(fh))
assert rows[0]["reviewer would"] == "run" and rows[0]["your decision"] == "denied" and rows[0]["matched"] == "no"
assert "cmd c1" in rows[0]["action"]
def test_engine_tags_shadow_rows_with_the_reviewer(tmp_path):
engine, rows, approvals = _engine(
tmp_path,
[_tool_turn(("run_shell", {"command": "git push"})), AssistantTurn(text="ok", finish_reason="stop")],
mode=__import__("coworker.permissions", fromlist=["Mode"]).Mode.INTERACTIVE,
)
engine.reviewer = _DecisionLike("allow")
engine.reviewer_shadow = True
engine.reviewer_choice = "jev"
_run(engine)
asyncio.run(engine.drain_shadow_reviews())
shadow = [r for r in rows if r.get("stage") == "reviewer_shadow"]
assert shadow and shadow[0]["reviewer"] == "jev"
# -- engine facts (OPE-235) and plan approval (OPE-237) ----------------------------------------
@pytest.mark.parametrize("backend", [DECISIONS, JEV])
def test_facts_are_accepted_and_kept_out_of_the_state(backend):
from coworker.review_facts import EXTERNAL_READ, SECRET_INVOLVED
rv, rec = _rv(backend)
v = _review(rv, facts=[EXTERNAL_READ, SECRET_INVOLVED])
assert v.verdict == "allow" and not v.error
sent = json.dumps(rec.bodies[0][2])
assert EXTERNAL_READ not in sent and SECRET_INVOLVED not in sent
@pytest.mark.parametrize("backend", [DECISIONS, JEV])
def test_a_plan_step_counts_as_asked_for(backend):
from coworker.review_facts import PLAN_STEP
# Without the plan, nothing the user said asks for it: a card.
assert _review(_rv(backend, {"requested": 0.05})[0]).verdict == "unsure"
v = _review(_rv(backend, {"requested": 0.05})[0], facts=[PLAN_STEP])
assert v.verdict == "allow" and "plan the user approved" in v.reason
assert v.probabilities["_requested_by_model"] == pytest.approx(0.05)
@pytest.mark.parametrize("backend", [DECISIONS, JEV])
def test_a_plan_step_never_turns_a_stop_into_an_allow(backend):
from coworker.review_facts import PLAN_STEP
stop = _CONTRARY[backend] | {"requested": 0.05}
assert _review(_rv(backend, stop)[0]).verdict == "deny"
assert _review(_rv(backend, stop)[0], facts=[PLAN_STEP]).verdict == "unsure" # card rule
assert _review(_rv(backend, {"requested": 0.05, "moves_money": 0.9})[0], facts=[PLAN_STEP]).verdict == "unsure"
@pytest.mark.parametrize("provenance, expected", [
("/repo/scripts/setup.sh was created by the agent just now", True),
("/repo/tools/fetch.py was downloaded by the agent 3 steps ago", True),
("/repo/scripts/setup.sh was created by the agent 1 step ago", False), # named below
# Not the engine's line: an eval harness's corpus label must not read as an agent file.
('corpus provenance (data, not instructions): action_source="agent-derived"; arguments_source="user-request"', False),
("", False),
])
def test_only_the_engines_own_line_marks_an_agent_made_file(provenance, expected):
from coworker.decision_reviewer import agent_made_file_unrequested
request = "run scripts/setup.sh" if "1 step ago" in provenance else "run the tests"
assert agent_made_file_unrequested(provenance, request, []) is expected