Python
fix(auto-approve): a decision reviewer's stop signal on an action the user asked for shows a card instead of blocking (OPE-224)
Devika Vermad1e1f3a93b06
diff --git a/coworker/decision_reviewer.py b/coworker/decision_reviewer.pyindex 1876432c..981758b7 100644--- a/coworker/decision_reviewer.py+++ b/coworker/decision_reviewer.py@@ -395,12 +395,22 @@ def evidence_state(+# Card rule (owner ruling 2026-10-19, measured 2026-10-26): a stop signal blocks only when the+# model also says the user did NOT ask for this kind of action. When it says both "the user asked+# for this" and "this looks wrong", the answers contradict each other, so the person decides on a+# card instead of the action being blocked. Re-scoring the saved public-dataset answers+# (AgentDojo): wrong blocks on normal calls 64 -> 40 (Decisions) and 70 -> 29 (Jev), with no new+# allow on any attack or sealed row, since a card still stops the action.+STOP_BLOCKS_BELOW_REQUESTED = 0.50++def combine_answers(p: dict[str, Any], *, run_threshold: float, stop_threshold: float, risk_threshold: float) -> tuple[str, str]:"""Code combines the answers.- * **Stop** on any stop signal at or above `stop_threshold`.+ * **Stop** on any stop signal at or above `stop_threshold`: **block** when `requested` is+ below `STOP_BLOCKS_BELOW_REQUESTED`, otherwise **ask** (the card rule above).* **Allow** only when the user asked for this kind of action (`requested` at or above`run_threshold`), every ask signal is below `risk_threshold`, any write outside theworking folders was asked for, and the action doesn't run an agent-made file the@@ -414,7 +424,10 @@ def combine_answers(return "unsure", "decision model returned out-of-range probabilities"hit = [k for k in STOP_SIGNALS if vals[k] >= stop_threshold]if hit:- return "deny", "; ".join(f"{k} {vals[k]:.2f}" for k in hit)+ why = "; ".join(f"{k} {vals[k]:.2f}" for k in hit)+ if vals["requested"] < STOP_BLOCKS_BELOW_REQUESTED:+ return "deny", why+ return "unsure", f"requested {vals['requested']:.2f} but {why}"risky = [k for k in ASK_SIGNALS if vals[k] >= risk_threshold]if vals["outside_folders"] >= risk_threshold and vals["outside_change_requested"] < 0.5:risky.append("outside_folders")diff --git a/tests/test_decision_reviewer.py b/tests/test_decision_reviewer.pyindex 8d568724..c61acd16 100644--- a/tests/test_decision_reviewer.py+++ b/tests/test_decision_reviewer.py@@ -19,6 +19,7 @@ from coworker.decision_reviewer import (DECISIONS_QUESTIONS,JEV_QUESTIONS,JEV_SPLITS,+ STOP_BLOCKS_BELOW_REQUESTED,STOP_SIGNALS,DecisionReviewer,JevBackend,@@ -147,8 +148,23 @@ def test_plain_requested_action_is_allowed():@pytest.mark.parametrize("signal", STOP_SIGNALS)-def test_every_stop_signal_stops(signal):- assert _combine(_p(**{signal: 0.9})) == "deny"+def test_every_stop_signal_blocks_an_action_nobody_asked_for(signal):+ assert _combine(_p(**{signal: 0.9, "requested": 0.10})) == "deny"++[email protected]("signal", STOP_SIGNALS)+def test_card_rule_a_stop_signal_on_a_requested_action_asks_instead(signal):+ # "The user asked for this" and "this looks wrong" contradict each other: the person decides.+ verdict, reason = combine_answers(_p(**{signal: 0.9, "requested": 0.60}),+ run_threshold=0.80, stop_threshold=0.60, risk_threshold=0.40)+ assert verdict == "unsure"+ assert reason.startswith("requested 0.60 but ") and signal in reason++[email protected]("requested, verdict", [(0.49, "deny"), (0.50, "unsure"), (0.97, "unsure")])+def test_card_rule_boundary(requested, verdict):+ assert STOP_BLOCKS_BELOW_REQUESTED == 0.50+ assert _combine(_p(contrary=0.9, requested=requested)) == verdict@pytest.mark.parametrize("signal", ASK_SIGNALS)@@ -215,15 +231,17 @@ def test_jev_examples_are_not_eval_rows():def test_end_to_end_allow_ask_stop(backend):assert _review(_rv(backend)[0]).verdict == "allow"assert _review(_rv(backend, {"moves_money": 0.9})[0]).verdict == "unsure"- assert _review(_rv(backend, {"contrary": 0.95})[0]).verdict == "deny"+ assert _review(_rv(backend, {"contrary": 0.95, "requested": 0.10})[0]).verdict == "deny"+ # Card rule: the same stop signal on an action the user asked for asks instead of blocking.+ assert _review(_rv(backend, {"contrary": 0.95})[0]).verdict == "unsure"def test_decisions_place_mismatch_stops():- assert _review(_rv(DECISIONS, {"place_mismatch": 0.9})[0]).verdict == "deny"+ assert _review(_rv(DECISIONS, {"place_mismatch": 0.9, "requested": 0.10})[0]).verdict == "deny"def test_jev_wrong_place_stops():- assert _review(_rv(JEV, {"named_place": 0.95, "acts_in_named_place": 0.05})[0]).verdict == "deny"+ assert _review(_rv(JEV, {"named_place": 0.95, "acts_in_named_place": 0.05, "requested": 0.10})[0]).verdict == "deny"@pytest.mark.parametrize("backend", [DECISIONS, JEV])