Skip to content
Python

fix(auto-approve): a decision reviewer's stop signal on an action the user asked for shows a card instead of blocking (OPE-224)

Devika Vermad1e1f3a93b06
diff --git a/coworker/decision_reviewer.py b/coworker/decision_reviewer.py
index 1876432c..981758b7 100644
--- a/coworker/decision_reviewer.py
+++ b/coworker/decision_reviewer.py
@@ -395,12 +395,22 @@ def evidence_state(
+# Card rule (owner ruling 2026-10-19, measured 2026-10-26): a stop signal blocks only when the
+# model also says the user did NOT ask for this kind of action. When it says both "the user asked
+# for this" and "this looks wrong", the answers contradict each other, so the person decides on a
+# card instead of the action being blocked. Re-scoring the saved public-dataset answers
+# (AgentDojo): wrong blocks on normal calls 64 -> 40 (Decisions) and 70 -> 29 (Jev), with no new
+# allow on any attack or sealed row, since a card still stops the action.
+STOP_BLOCKS_BELOW_REQUESTED = 0.50
+
+
def combine_answers(
p: dict[str, Any], *, run_threshold: float, stop_threshold: float, risk_threshold: float
) -> tuple[str, str]:
"""Code combines the answers.
- * **Stop** on any stop signal at or above `stop_threshold`.
+ * **Stop** on any stop signal at or above `stop_threshold`: **block** when `requested` is
+ below `STOP_BLOCKS_BELOW_REQUESTED`, otherwise **ask** (the card rule above).
* **Allow** only when the user asked for this kind of action (`requested` at or above
`run_threshold`), every ask signal is below `risk_threshold`, any write outside the
working folders was asked for, and the action doesn't run an agent-made file the
@@ -414,7 +424,10 @@ def combine_answers(
return "unsure", "decision model returned out-of-range probabilities"
hit = [k for k in STOP_SIGNALS if vals[k] >= stop_threshold]
if hit:
- return "deny", "; ".join(f"{k} {vals[k]:.2f}" for k in hit)
+ why = "; ".join(f"{k} {vals[k]:.2f}" for k in hit)
+ if vals["requested"] < STOP_BLOCKS_BELOW_REQUESTED:
+ return "deny", why
+ return "unsure", f"requested {vals['requested']:.2f} but {why}"
risky = [k for k in ASK_SIGNALS if vals[k] >= risk_threshold]
if vals["outside_folders"] >= risk_threshold and vals["outside_change_requested"] < 0.5:
risky.append("outside_folders")
diff --git a/tests/test_decision_reviewer.py b/tests/test_decision_reviewer.py
index 8d568724..c61acd16 100644
--- a/tests/test_decision_reviewer.py
+++ b/tests/test_decision_reviewer.py
@@ -19,6 +19,7 @@ from coworker.decision_reviewer import (
DECISIONS_QUESTIONS,
JEV_QUESTIONS,
JEV_SPLITS,
+ STOP_BLOCKS_BELOW_REQUESTED,
STOP_SIGNALS,
DecisionReviewer,
JevBackend,
@@ -147,8 +148,23 @@ def test_plain_requested_action_is_allowed():
@pytest.mark.parametrize("signal", STOP_SIGNALS)
-def test_every_stop_signal_stops(signal):
- assert _combine(_p(**{signal: 0.9})) == "deny"
+def test_every_stop_signal_blocks_an_action_nobody_asked_for(signal):
+ assert _combine(_p(**{signal: 0.9, "requested": 0.10})) == "deny"
+
+
[email protected]("signal", STOP_SIGNALS)
+def test_card_rule_a_stop_signal_on_a_requested_action_asks_instead(signal):
+ # "The user asked for this" and "this looks wrong" contradict each other: the person decides.
+ verdict, reason = combine_answers(_p(**{signal: 0.9, "requested": 0.60}),
+ run_threshold=0.80, stop_threshold=0.60, risk_threshold=0.40)
+ assert verdict == "unsure"
+ assert reason.startswith("requested 0.60 but ") and signal in reason
+
+
[email protected]("requested, verdict", [(0.49, "deny"), (0.50, "unsure"), (0.97, "unsure")])
+def test_card_rule_boundary(requested, verdict):
+ assert STOP_BLOCKS_BELOW_REQUESTED == 0.50
+ assert _combine(_p(contrary=0.9, requested=requested)) == verdict
@pytest.mark.parametrize("signal", ASK_SIGNALS)
@@ -215,15 +231,17 @@ def test_jev_examples_are_not_eval_rows():
def test_end_to_end_allow_ask_stop(backend):
assert _review(_rv(backend)[0]).verdict == "allow"
assert _review(_rv(backend, {"moves_money": 0.9})[0]).verdict == "unsure"
- assert _review(_rv(backend, {"contrary": 0.95})[0]).verdict == "deny"
+ assert _review(_rv(backend, {"contrary": 0.95, "requested": 0.10})[0]).verdict == "deny"
+ # Card rule: the same stop signal on an action the user asked for asks instead of blocking.
+ assert _review(_rv(backend, {"contrary": 0.95})[0]).verdict == "unsure"
def test_decisions_place_mismatch_stops():
- assert _review(_rv(DECISIONS, {"place_mismatch": 0.9})[0]).verdict == "deny"
+ assert _review(_rv(DECISIONS, {"place_mismatch": 0.9, "requested": 0.10})[0]).verdict == "deny"
def test_jev_wrong_place_stops():
- assert _review(_rv(JEV, {"named_place": 0.95, "acts_in_named_place": 0.05})[0]).verdict == "deny"
+ assert _review(_rv(JEV, {"named_place": 0.95, "acts_in_named_place": 0.05, "requested": 0.10})[0]).verdict == "deny"
@pytest.mark.parametrize("backend", [DECISIONS, JEV])