Skip to content
Python

feat(auto-approve): shadow results summary for the chosen reviewer, with a readable CSV export (OPE-224)

Devika Vermacd63cbab05b3
diff --git a/coworker/audit.py b/coworker/audit.py
index 3c9770c5..a81a68f5 100644
--- a/coworker/audit.py
+++ b/coworker/audit.py
@@ -89,6 +89,10 @@ class AuditStore:
# this call ran in — "" for local/desktop or automated sessions.
("actor", "TEXT DEFAULT ''"),
("approved_by", "TEXT DEFAULT ''"),
+ # Which Auto-Approve reviewer produced a reviewer_verdict / reviewer_shadow row
+ # (OPE-224): "session", "openai-decisions" or "jev". Lets shadow results be
+ # summarised per reviewer instead of mixing them.
+ ("reviewer", "TEXT DEFAULT ''"),
):
try:
self._conn.execute(
@@ -133,8 +137,8 @@ class AuditStore:
self._conn.execute(
"""
INSERT INTO audit_events
- (session_id, agent, workspace, connector, tool, stage, status, approval, args, result_preview, reason, resource, call_id, tokens_in, tokens_out, cache_read, cache_write, actor, approved_by)
- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+ (session_id, agent, workspace, connector, tool, stage, status, approval, args, result_preview, reason, resource, call_id, tokens_in, tokens_out, cache_read, cache_write, actor, approved_by, reviewer)
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""",
(
event.get("session_id") or "",
@@ -156,6 +160,7 @@ class AuditStore:
int(event.get("cache_write") or 0),
str(event.get("actor") or ""),
str(event.get("approved_by") or ""),
+ str(event.get("reviewer") or ""),
),
)
self._conn.commit()
@@ -203,6 +208,52 @@ class AuditStore:
return {"live": _bucket("reviewer_verdict"), "shadow": _bucket("reviewer_shadow")}
+ def shadow_rows(self, reviewer: str, *, days: int = 30) -> list[dict[str, Any]]:
+ """Shadow verdicts of one reviewer over the last `days`, each joined to the human's
+ own decision on the same tool call (OPE-224). `decision` is "approved", "denied" or
+ "" when the card was never answered by a person (unattended auto-refusals and
+ superseded calls are not a person's decision)."""
+ with self._lock:
+ rows = self._conn.execute(
+ """
+ SELECT s.timestamp, s.session_id, s.tool, s.args, s.status AS verdict,
+ s.reason, s.call_id,
+ COALESCE((
+ SELECT d.status FROM audit_events d
+ WHERE d.call_id = s.call_id AND s.call_id != ''
+ AND d.stage = 'approval_resolved'
+ AND d.status IN ('approved', 'denied')
+ AND COALESCE(d.approval, '') != 'auto_refused'
+ ORDER BY d.id DESC LIMIT 1
+ ), '') AS decision
+ FROM audit_events s
+ WHERE s.stage = 'reviewer_shadow' AND s.reviewer = ?
+ AND s.timestamp >= datetime('now', ?)
+ ORDER BY s.id
+ """,
+ (reviewer, f"-{int(days)} days"),
+ ).fetchall()
+ return [dict(r) for r in rows]
+
+ def shadow_summary(self, reviewer: str, *, days: int = 30) -> dict[str, Any]:
+ """Counts for the Settings line. `matched` counts calls where the reviewer would
+ have decided (run or stop) the same way the person did; an "ask" verdict would have
+ left the call to the person anyway, so it is not counted either way."""
+ rows = self.shadow_rows(reviewer, days=days)
+ run = sum(r["verdict"] == "allow" for r in rows)
+ stopped = sum(r["verdict"] == "deny" for r in rows)
+ decided = [r for r in rows if r["verdict"] in ("allow", "deny") and r["decision"]]
+ matched = sum(
+ (r["verdict"] == "allow" and r["decision"] == "approved")
+ or (r["verdict"] == "deny" and r["decision"] == "denied")
+ for r in decided
+ )
+ return {
+ "reviewer": reviewer, "days": days, "checks": len(rows),
+ "run": run, "asked": len(rows) - run - stopped, "stopped": stopped,
+ "decided": len(decided), "matched": matched,
+ }
+
def list(
self,
*,
diff --git a/coworker/engine.py b/coworker/engine.py
index eab98b2c..716c262b 100644
--- a/coworker/engine.py
+++ b/coworker/engine.py
@@ -1761,6 +1761,7 @@ class TurnEngine:
stage="reviewer_shadow",
status=verdict.verdict,
reason=verdict.reason,
+ reviewer=getattr(self, "reviewer_choice", "session"),
call_id=tool_call.id,
tokens_in=verdict.tokens_in,
tokens_out=verdict.tokens_out,
@@ -1954,6 +1955,7 @@ class TurnEngine:
stage="reviewer_verdict",
status=verdict.verdict,
reason=verdict.reason,
+ reviewer=getattr(self, "reviewer_choice", "session"),
tokens_in=verdict.tokens_in,
tokens_out=verdict.tokens_out,
cache_read=verdict.cache_read,
diff --git a/coworker/server/app.py b/coworker/server/app.py
index b38cc093..e2bb9a99 100644
--- a/coworker/server/app.py
+++ b/coworker/server/app.py
@@ -2603,6 +2603,16 @@ def create_app(manager: SessionManager) -> FastAPI:
body = body or {}
return manager.set_reviewer_key(body.get("reviewer", ""), body.get("api_key", ""))
+ @app.get("/v1/settings/auto-approve-shadow-summary")
+ def settings_shadow_summary() -> dict[str, Any]:
+ # Shadow results of the chosen reviewer, last 30 days (OPE-224).
+ return manager.shadow_summary()
+
+ @app.post("/v1/settings/auto-approve-shadow-export")
+ def settings_shadow_export() -> dict[str, Any]:
+ # Every shadow check of the chosen reviewer as a readable CSV; returns its path.
+ return manager.export_shadow_csv()
+
@app.post("/v1/settings/pdf")
def settings_set_pdf(body: dict) -> dict[str, Any]:
# Token savings (owner ask, 2026-07-17): fallback mode for models without native
diff --git a/coworker/server/manager.py b/coworker/server/manager.py
index f9bfd9be..9aa5c4b3 100644
--- a/coworker/server/manager.py
+++ b/coworker/server/manager.py
@@ -5299,6 +5299,37 @@ class SessionManager:
result = {"ok": True, "auto_approve_reviewer_keys": self.reviewer_key_status()}
return result
+ def shadow_summary(self) -> dict[str, Any]:
+ """Shadow results of the currently chosen reviewer, last 30 days (OPE-224)."""
+ return self.audit_store.shadow_summary(self.auto_approve_reviewer())
+
+ def export_shadow_csv(self) -> dict[str, Any]:
+ """Write every shadow check of the chosen reviewer (last 30 days) to a readable CSV
+ and return its path, for a person to investigate in a spreadsheet (OPE-224)."""
+ import csv
+ from datetime import date
+
+ reviewer = self.auto_approve_reviewer()
+ rows = self.audit_store.shadow_rows(reviewer)
+ out_dir = self._data_base / "shadow-results"
+ out_dir.mkdir(parents=True, exist_ok=True)
+ path = out_dir / f"shadow-{reviewer}-{date.today().isoformat()}.csv"
+ would = {"allow": "run", "unsure": "ask you", "deny": "stop"}
+ yours = {"approved": "allowed", "denied": "denied", "": "not answered"}
+ with path.open("w", newline="", encoding="utf-8-sig") as fh:
+ w = csv.writer(fh)
+ w.writerow(["time (UTC)", "session", "tool", "action", "reviewer",
+ "reviewer would", "reviewer reason", "your decision", "matched"])
+ for r in rows:
+ v, d = r["verdict"], r["decision"]
+ if v == "unsure" or not d:
+ matched = ""
+ else:
+ matched = "yes" if (v, d) in (("allow", "approved"), ("deny", "denied")) else "no"
+ w.writerow([r["timestamp"], r["session_id"], r["tool"], r["args"], reviewer,
+ would.get(v, v), r["reason"], yours.get(d, d), matched])
+ return {"ok": True, "path": str(path), "rows": len(rows)}
+
def set_auto_approve(self, on: Any) -> dict[str, Any]:
self._prefs["auto_approve"] = bool(on)
self._save_prefs()
diff --git a/surfaces/gui/src/api.ts b/surfaces/gui/src/api.ts
index 929023f4..a0af6c6f 100644
--- a/surfaces/gui/src/api.ts
+++ b/surfaces/gui/src/api.ts
@@ -1410,6 +1410,29 @@ export async function setAutoApproveReviewer(
return res.json();
}
+export type ShadowSummary = {
+ reviewer: ReviewerChoice;
+ days: number;
+ checks: number;
+ run: number;
+ asked: number;
+ stopped: number;
+ decided: number;
+ matched: number;
+};
+
+/** Shadow results of the chosen reviewer, last 30 days (OPE-224). */
+export async function getShadowSummary(): Promise<ShadowSummary> {
+ const res = await fetch(`${httpBase()}/v1/settings/auto-approve-shadow-summary`);
+ return res.json();
+}
+
+/** Write every shadow check of the chosen reviewer to a CSV; returns where it was saved. */
+export async function exportShadowResults(): Promise<{ ok: boolean; path?: string; rows?: number; error?: string }> {
+ const res = await fetch(`${httpBase()}/v1/settings/auto-approve-shadow-export`, { method: "POST" });
+ return res.json();
+}
+
/** Save the API key a decision-model reviewer needs. The key is never sent back. */
export async function setAutoApproveReviewerKey(
reviewer: "openai-decisions" | "jev",
diff --git a/surfaces/gui/src/components/SettingsView.tsx b/surfaces/gui/src/components/SettingsView.tsx
index 75297295..88e49ae6 100644
--- a/surfaces/gui/src/components/SettingsView.tsx
+++ b/surfaces/gui/src/components/SettingsView.tsx
@@ -25,7 +25,10 @@ import {
setAutoApproveShadow,
setAutoApproveReviewer,
setAutoApproveReviewerKey,
+ getShadowSummary,
+ exportShadowResults,
type ReviewerChoice,
+ type ShadowSummary,
setCompactionSettings,
setContextBar,
setHideReasoningRunning,
@@ -1400,6 +1403,24 @@ function AutoApproveCard() {
const [keys, setKeys] = useState<Record<string, boolean>>({});
const [keyDraft, setKeyDraft] = useState("");
const [keyError, setKeyError] = useState("");
+ const [summary, setSummary] = useState<ShadowSummary | null>(null);
+ const [exported, setExported] = useState("");
+
+ // Shadow results follow the chosen reviewer: reload when shadow turns on or the
+ // reviewer changes, so the line never mixes reviewers (OPE-224).
+ useEffect(() => {
+ setExported("");
+ if (!shadow) {
+ setSummary(null);
+ return;
+ }
+ getShadowSummary().then(setSummary).catch(() => setSummary(null));
+ }, [shadow, reviewer]);
+
+ const exportResults = async () => {
+ const r = await exportShadowResults().catch(() => ({ ok: false } as { ok: boolean; path?: string }));
+ setExported(r.ok && r.path ? r.path : t("settingsx.auto_approve.shadow_export_failed"));
+ };
useEffect(() => {
getSettings()
@@ -1481,6 +1502,43 @@ function AutoApproveCard() {
</span>
</span>
</label>
+ {shadow && summary && (
+ <div className="pl-14 pb-1 text-meta text-muted" data-testid="shadow-summary">
+ <div className="text-ink">
+ {t("settingsx.auto_approve.shadow_summary_title", {
+ days: summary.days,
+ reviewer: t(`settingsx.auto_approve.reviewer_${choiceKey(summary.reviewer)}`),
+ })}
+ </div>
+ {summary.checks === 0 ? (
+ <div>{t("settingsx.auto_approve.shadow_summary_empty")}</div>
+ ) : (
+ <>
+ <div>
+ {t("settingsx.auto_approve.shadow_summary_counts", {
+ checks: summary.checks, run: summary.run, asked: summary.asked, stopped: summary.stopped,
+ })}
+ </div>
+ {summary.decided > 0 && (
+ <div>{t("settingsx.auto_approve.shadow_summary_matched", { matched: summary.matched, decided: summary.decided })}</div>
+ )}
+ <button
+ type="button"
+ className="mt-1 text-accent hover:underline"
+ data-testid="shadow-export"
+ onClick={exportResults}
+ >
+ {t("settingsx.auto_approve.shadow_export")}
+ </button>
+ {exported && (
+ <div className="select-all break-all" data-testid="shadow-export-path">
+ {t("settingsx.auto_approve.shadow_export_saved", { path: exported })}
+ </div>
+ )}
+ </>
+ )}
+ </div>
+ )}
{/* Which reviewer judges the checks (OPE-224). Shown only when something uses it:
Auto-approve mode, or shadow evaluation, which records that reviewer's verdicts. */}
{(on || shadow) && (
diff --git a/surfaces/gui/src/locales/en.json b/surfaces/gui/src/locales/en.json
index 166536e3..ffab749e 100644
--- a/surfaces/gui/src/locales/en.json
+++ b/surfaces/gui/src/locales/en.json
@@ -2501,7 +2501,14 @@
"reviewer_key_save": "Save key",
"reviewer_key_placeholder_decisions": "OpenAI API key",
"reviewer_key_placeholder_jev": "TypeSafe API key",
- "reviewer_key_failed": "Couldn’t save the key."
+ "reviewer_key_failed": "Couldn’t save the key.",
+ "shadow_summary_title": "Shadow results · last {{days}} days, all sessions, {{reviewer}}",
+ "shadow_summary_counts": "{{checks}} checks: would have run {{run}} · asked you {{asked}} · stopped {{stopped}}",
+ "shadow_summary_matched": "Matched your own decision {{matched}} of {{decided}} times.",
+ "shadow_summary_empty": "Nothing recorded yet. Keep working as usual; each approval card you answer is recorded.",
+ "shadow_export": "Export full results",
+ "shadow_export_saved": "Saved to {{path}}",
+ "shadow_export_failed": "Couldn’t export the results."
},
"sandbox": {
"title": "Sandbox",
diff --git a/surfaces/gui/src/locales/zh.json b/surfaces/gui/src/locales/zh.json
index 8e483299..09bf2a74 100644
--- a/surfaces/gui/src/locales/zh.json
+++ b/surfaces/gui/src/locales/zh.json
@@ -2457,7 +2457,14 @@
"reviewer_key_save": "保存密钥",
"reviewer_key_placeholder_decisions": "OpenAI API 密钥",
"reviewer_key_placeholder_jev": "TypeSafe API 密钥",
- "reviewer_key_failed": "无法保存密钥。"
+ "reviewer_key_failed": "无法保存密钥。",
+ "shadow_summary_title": "影子评估结果 · 最近 {{days}} 天,所有会话,{{reviewer}}",
+ "shadow_summary_counts": "{{checks}} 次检查:本会放行 {{run}} · 询问你 {{asked}} · 阻止 {{stopped}}",
+ "shadow_summary_matched": "与你自己的决定一致 {{matched}} / {{decided}} 次。",
+ "shadow_summary_empty": "尚无记录。照常使用即可;你处理的每张审批卡片都会被记录。",
+ "shadow_export": "导出完整结果",
+ "shadow_export_saved": "已保存到 {{path}}",
+ "shadow_export_failed": "无法导出结果。"
},
"sandbox": {
"title": "沙箱",
diff --git a/tests/test_decision_reviewer.py b/tests/test_decision_reviewer.py
index 6a1a416b..8d568724 100644
--- a/tests/test_decision_reviewer.py
+++ b/tests/test_decision_reviewer.py
@@ -384,3 +384,66 @@ def test_eval_stub_runs_each_decision_reviewer(backend, capsys):
out = capsys.readouterr().out
assert rc == 0, out
assert f"{backend}:" in out
+
+
+# -- shadow results: summary and CSV export ---------------------------------------------------
+
+
+def _shadow(store, call, verdict, reviewer="jev", decision=None, approval="once"):
+ store.append({"session_id": "s1", "tool": "run_shell", "arguments": {"command": f"cmd {call}"},
+ "stage": "reviewer_shadow", "status": verdict, "reason": f"r {call}",
+ "call_id": call, "reviewer": reviewer})
+ if decision:
+ store.append({"session_id": "s1", "tool": "run_shell", "arguments": {}, "stage": "approval_resolved",
+ "status": decision, "approval": approval, "call_id": call})
+
+
+def test_shadow_summary_counts_and_matches(tmp_path):
+ from coworker.audit import AuditStore
+
+ store = AuditStore(tmp_path / "a.db")
+ _shadow(store, "c1", "allow", decision="approved") # run, you allowed: match
+ _shadow(store, "c2", "allow", decision="denied") # run, you denied: no match
+ _shadow(store, "c3", "deny", decision="denied") # stop, you denied: match
+ _shadow(store, "c4", "unsure", decision="approved") # asked: not counted either way
+ _shadow(store, "c5", "allow") # never answered: not counted
+ _shadow(store, "c6", "deny", decision="denied", approval="auto_refused") # nobody decided
+ _shadow(store, "c7", "allow", reviewer="openai-decisions", decision="approved") # other reviewer
+ s = store.shadow_summary("jev")
+ assert (s["checks"], s["run"], s["asked"], s["stopped"]) == (6, 3, 1, 2)
+ assert (s["decided"], s["matched"]) == (3, 2)
+ assert store.shadow_summary("openai-decisions")["checks"] == 1
+
+
+def test_shadow_export_writes_a_readable_csv(client, tmp_path):
+ import csv
+
+ client.post("/v1/settings/auto-approve-reviewer", json={"auto_approve_reviewer": "jev"})
+ mgr = client.app.state.manager if hasattr(client.app.state, "manager") else None
+ store = mgr.audit_store if mgr else None
+ if store is None:
+ pytest.skip("manager not reachable from the test client")
+ _shadow(store, "c1", "allow", decision="denied")
+ summary = client.get("/v1/settings/auto-approve-shadow-summary").json()
+ assert summary["reviewer"] == "jev" and summary["checks"] == 1 and summary["matched"] == 0
+ r = client.post("/v1/settings/auto-approve-shadow-export").json()
+ assert r["ok"] and r["rows"] == 1 and r["path"].endswith(".csv")
+ with open(r["path"], encoding="utf-8-sig", newline="") as fh:
+ rows = list(csv.DictReader(fh))
+ assert rows[0]["reviewer would"] == "run" and rows[0]["your decision"] == "denied" and rows[0]["matched"] == "no"
+ assert "cmd c1" in rows[0]["action"]
+
+
+def test_engine_tags_shadow_rows_with_the_reviewer(tmp_path):
+ engine, rows, approvals = _engine(
+ tmp_path,
+ [_tool_turn(("run_shell", {"command": "git push"})), AssistantTurn(text="ok", finish_reason="stop")],
+ mode=__import__("coworker.permissions", fromlist=["Mode"]).Mode.INTERACTIVE,
+ )
+ engine.reviewer = _DecisionLike("allow")
+ engine.reviewer_shadow = True
+ engine.reviewer_choice = "jev"
+ _run(engine)
+ asyncio.run(engine.drain_shadow_reviews())
+ shadow = [r for r in rows if r.get("stage") == "reviewer_shadow"]
+ assert shadow and shadow[0]["reviewer"] == "jev"