Python
feat(auto-approve): shadow results summary for the chosen reviewer, with a readable CSV export (OPE-224)
Devika Vermacd63cbab05b3
diff --git a/coworker/audit.py b/coworker/audit.pyindex 3c9770c5..a81a68f5 100644--- a/coworker/audit.py+++ b/coworker/audit.py@@ -89,6 +89,10 @@ class AuditStore:# this call ran in — "" for local/desktop or automated sessions.("actor", "TEXT DEFAULT ''"),("approved_by", "TEXT DEFAULT ''"),+ # Which Auto-Approve reviewer produced a reviewer_verdict / reviewer_shadow row+ # (OPE-224): "session", "openai-decisions" or "jev". Lets shadow results be+ # summarised per reviewer instead of mixing them.+ ("reviewer", "TEXT DEFAULT ''"),):try:self._conn.execute(@@ -133,8 +137,8 @@ class AuditStore:self._conn.execute("""INSERT INTO audit_events- (session_id, agent, workspace, connector, tool, stage, status, approval, args, result_preview, reason, resource, call_id, tokens_in, tokens_out, cache_read, cache_write, actor, approved_by)- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)+ (session_id, agent, workspace, connector, tool, stage, status, approval, args, result_preview, reason, resource, call_id, tokens_in, tokens_out, cache_read, cache_write, actor, approved_by, reviewer)+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",(event.get("session_id") or "",@@ -156,6 +160,7 @@ class AuditStore:int(event.get("cache_write") or 0),str(event.get("actor") or ""),str(event.get("approved_by") or ""),+ str(event.get("reviewer") or ""),),)self._conn.commit()@@ -203,6 +208,52 @@ class AuditStore:return {"live": _bucket("reviewer_verdict"), "shadow": _bucket("reviewer_shadow")}+ def shadow_rows(self, reviewer: str, *, days: int = 30) -> list[dict[str, Any]]:+ """Shadow verdicts of one reviewer over the last `days`, each joined to the human's+ own decision on the same tool call (OPE-224). `decision` is "approved", "denied" or+ "" when the card was never answered by a person (unattended auto-refusals and+ superseded calls are not a person's decision)."""+ with self._lock:+ rows = self._conn.execute(+ """+ SELECT s.timestamp, s.session_id, s.tool, s.args, s.status AS verdict,+ s.reason, s.call_id,+ COALESCE((+ SELECT d.status FROM audit_events d+ WHERE d.call_id = s.call_id AND s.call_id != ''+ AND d.stage = 'approval_resolved'+ AND d.status IN ('approved', 'denied')+ AND COALESCE(d.approval, '') != 'auto_refused'+ ORDER BY d.id DESC LIMIT 1+ ), '') AS decision+ FROM audit_events s+ WHERE s.stage = 'reviewer_shadow' AND s.reviewer = ?+ AND s.timestamp >= datetime('now', ?)+ ORDER BY s.id+ """,+ (reviewer, f"-{int(days)} days"),+ ).fetchall()+ return [dict(r) for r in rows]++ def shadow_summary(self, reviewer: str, *, days: int = 30) -> dict[str, Any]:+ """Counts for the Settings line. `matched` counts calls where the reviewer would+ have decided (run or stop) the same way the person did; an "ask" verdict would have+ left the call to the person anyway, so it is not counted either way."""+ rows = self.shadow_rows(reviewer, days=days)+ run = sum(r["verdict"] == "allow" for r in rows)+ stopped = sum(r["verdict"] == "deny" for r in rows)+ decided = [r for r in rows if r["verdict"] in ("allow", "deny") and r["decision"]]+ matched = sum(+ (r["verdict"] == "allow" and r["decision"] == "approved")+ or (r["verdict"] == "deny" and r["decision"] == "denied")+ for r in decided+ )+ return {+ "reviewer": reviewer, "days": days, "checks": len(rows),+ "run": run, "asked": len(rows) - run - stopped, "stopped": stopped,+ "decided": len(decided), "matched": matched,+ }+def list(self,*,diff --git a/coworker/engine.py b/coworker/engine.pyindex eab98b2c..716c262b 100644--- a/coworker/engine.py+++ b/coworker/engine.py@@ -1761,6 +1761,7 @@ class TurnEngine:stage="reviewer_shadow",status=verdict.verdict,reason=verdict.reason,+ reviewer=getattr(self, "reviewer_choice", "session"),call_id=tool_call.id,tokens_in=verdict.tokens_in,tokens_out=verdict.tokens_out,@@ -1954,6 +1955,7 @@ class TurnEngine:stage="reviewer_verdict",status=verdict.verdict,reason=verdict.reason,+ reviewer=getattr(self, "reviewer_choice", "session"),tokens_in=verdict.tokens_in,tokens_out=verdict.tokens_out,cache_read=verdict.cache_read,diff --git a/coworker/server/app.py b/coworker/server/app.pyindex b38cc093..e2bb9a99 100644--- a/coworker/server/app.py+++ b/coworker/server/app.py@@ -2603,6 +2603,16 @@ def create_app(manager: SessionManager) -> FastAPI:body = body or {}return manager.set_reviewer_key(body.get("reviewer", ""), body.get("api_key", ""))+ @app.get("/v1/settings/auto-approve-shadow-summary")+ def settings_shadow_summary() -> dict[str, Any]:+ # Shadow results of the chosen reviewer, last 30 days (OPE-224).+ return manager.shadow_summary()++ @app.post("/v1/settings/auto-approve-shadow-export")+ def settings_shadow_export() -> dict[str, Any]:+ # Every shadow check of the chosen reviewer as a readable CSV; returns its path.+ return manager.export_shadow_csv()+@app.post("/v1/settings/pdf")def settings_set_pdf(body: dict) -> dict[str, Any]:# Token savings (owner ask, 2026-07-17): fallback mode for models without nativediff --git a/coworker/server/manager.py b/coworker/server/manager.pyindex f9bfd9be..9aa5c4b3 100644--- a/coworker/server/manager.py+++ b/coworker/server/manager.py@@ -5299,6 +5299,37 @@ class SessionManager:result = {"ok": True, "auto_approve_reviewer_keys": self.reviewer_key_status()}return result+ def shadow_summary(self) -> dict[str, Any]:+ """Shadow results of the currently chosen reviewer, last 30 days (OPE-224)."""+ return self.audit_store.shadow_summary(self.auto_approve_reviewer())++ def export_shadow_csv(self) -> dict[str, Any]:+ """Write every shadow check of the chosen reviewer (last 30 days) to a readable CSV+ and return its path, for a person to investigate in a spreadsheet (OPE-224)."""+ import csv+ from datetime import date++ reviewer = self.auto_approve_reviewer()+ rows = self.audit_store.shadow_rows(reviewer)+ out_dir = self._data_base / "shadow-results"+ out_dir.mkdir(parents=True, exist_ok=True)+ path = out_dir / f"shadow-{reviewer}-{date.today().isoformat()}.csv"+ would = {"allow": "run", "unsure": "ask you", "deny": "stop"}+ yours = {"approved": "allowed", "denied": "denied", "": "not answered"}+ with path.open("w", newline="", encoding="utf-8-sig") as fh:+ w = csv.writer(fh)+ w.writerow(["time (UTC)", "session", "tool", "action", "reviewer",+ "reviewer would", "reviewer reason", "your decision", "matched"])+ for r in rows:+ v, d = r["verdict"], r["decision"]+ if v == "unsure" or not d:+ matched = ""+ else:+ matched = "yes" if (v, d) in (("allow", "approved"), ("deny", "denied")) else "no"+ w.writerow([r["timestamp"], r["session_id"], r["tool"], r["args"], reviewer,+ would.get(v, v), r["reason"], yours.get(d, d), matched])+ return {"ok": True, "path": str(path), "rows": len(rows)}+def set_auto_approve(self, on: Any) -> dict[str, Any]:self._prefs["auto_approve"] = bool(on)self._save_prefs()diff --git a/surfaces/gui/src/api.ts b/surfaces/gui/src/api.tsindex 929023f4..a0af6c6f 100644--- a/surfaces/gui/src/api.ts+++ b/surfaces/gui/src/api.ts@@ -1410,6 +1410,29 @@ export async function setAutoApproveReviewer(return res.json();}+export type ShadowSummary = {+ reviewer: ReviewerChoice;+ days: number;+ checks: number;+ run: number;+ asked: number;+ stopped: number;+ decided: number;+ matched: number;+};++/** Shadow results of the chosen reviewer, last 30 days (OPE-224). */+export async function getShadowSummary(): Promise<ShadowSummary> {+ const res = await fetch(`${httpBase()}/v1/settings/auto-approve-shadow-summary`);+ return res.json();+}++/** Write every shadow check of the chosen reviewer to a CSV; returns where it was saved. */+export async function exportShadowResults(): Promise<{ ok: boolean; path?: string; rows?: number; error?: string }> {+ const res = await fetch(`${httpBase()}/v1/settings/auto-approve-shadow-export`, { method: "POST" });+ return res.json();+}+/** Save the API key a decision-model reviewer needs. The key is never sent back. */export async function setAutoApproveReviewerKey(reviewer: "openai-decisions" | "jev",diff --git a/surfaces/gui/src/components/SettingsView.tsx b/surfaces/gui/src/components/SettingsView.tsxindex 75297295..88e49ae6 100644--- a/surfaces/gui/src/components/SettingsView.tsx+++ b/surfaces/gui/src/components/SettingsView.tsx@@ -25,7 +25,10 @@ import {setAutoApproveShadow,setAutoApproveReviewer,setAutoApproveReviewerKey,+ getShadowSummary,+ exportShadowResults,type ReviewerChoice,+ type ShadowSummary,setCompactionSettings,setContextBar,setHideReasoningRunning,@@ -1400,6 +1403,24 @@ function AutoApproveCard() {const [keys, setKeys] = useState<Record<string, boolean>>({});const [keyDraft, setKeyDraft] = useState("");const [keyError, setKeyError] = useState("");+ const [summary, setSummary] = useState<ShadowSummary | null>(null);+ const [exported, setExported] = useState("");++ // Shadow results follow the chosen reviewer: reload when shadow turns on or the+ // reviewer changes, so the line never mixes reviewers (OPE-224).+ useEffect(() => {+ setExported("");+ if (!shadow) {+ setSummary(null);+ return;+ }+ getShadowSummary().then(setSummary).catch(() => setSummary(null));+ }, [shadow, reviewer]);++ const exportResults = async () => {+ const r = await exportShadowResults().catch(() => ({ ok: false } as { ok: boolean; path?: string }));+ setExported(r.ok && r.path ? r.path : t("settingsx.auto_approve.shadow_export_failed"));+ };useEffect(() => {getSettings()@@ -1481,6 +1502,43 @@ function AutoApproveCard() {</span></span></label>+ {shadow && summary && (+ <div className="pl-14 pb-1 text-meta text-muted" data-testid="shadow-summary">+ <div className="text-ink">+ {t("settingsx.auto_approve.shadow_summary_title", {+ days: summary.days,+ reviewer: t(`settingsx.auto_approve.reviewer_${choiceKey(summary.reviewer)}`),+ })}+ </div>+ {summary.checks === 0 ? (+ <div>{t("settingsx.auto_approve.shadow_summary_empty")}</div>+ ) : (+ <>+ <div>+ {t("settingsx.auto_approve.shadow_summary_counts", {+ checks: summary.checks, run: summary.run, asked: summary.asked, stopped: summary.stopped,+ })}+ </div>+ {summary.decided > 0 && (+ <div>{t("settingsx.auto_approve.shadow_summary_matched", { matched: summary.matched, decided: summary.decided })}</div>+ )}+ <button+ type="button"+ className="mt-1 text-accent hover:underline"+ data-testid="shadow-export"+ onClick={exportResults}+ >+ {t("settingsx.auto_approve.shadow_export")}+ </button>+ {exported && (+ <div className="select-all break-all" data-testid="shadow-export-path">+ {t("settingsx.auto_approve.shadow_export_saved", { path: exported })}+ </div>+ )}+ </>+ )}+ </div>+ )}{/* Which reviewer judges the checks (OPE-224). Shown only when something uses it:Auto-approve mode, or shadow evaluation, which records that reviewer's verdicts. */}{(on || shadow) && (diff --git a/surfaces/gui/src/locales/en.json b/surfaces/gui/src/locales/en.jsonindex 166536e3..ffab749e 100644--- a/surfaces/gui/src/locales/en.json+++ b/surfaces/gui/src/locales/en.json@@ -2501,7 +2501,14 @@"reviewer_key_save": "Save key","reviewer_key_placeholder_decisions": "OpenAI API key","reviewer_key_placeholder_jev": "TypeSafe API key",- "reviewer_key_failed": "Couldn’t save the key."+ "reviewer_key_failed": "Couldn’t save the key.",+ "shadow_summary_title": "Shadow results · last {{days}} days, all sessions, {{reviewer}}",+ "shadow_summary_counts": "{{checks}} checks: would have run {{run}} · asked you {{asked}} · stopped {{stopped}}",+ "shadow_summary_matched": "Matched your own decision {{matched}} of {{decided}} times.",+ "shadow_summary_empty": "Nothing recorded yet. Keep working as usual; each approval card you answer is recorded.",+ "shadow_export": "Export full results",+ "shadow_export_saved": "Saved to {{path}}",+ "shadow_export_failed": "Couldn’t export the results."},"sandbox": {"title": "Sandbox",diff --git a/surfaces/gui/src/locales/zh.json b/surfaces/gui/src/locales/zh.jsonindex 8e483299..09bf2a74 100644--- a/surfaces/gui/src/locales/zh.json+++ b/surfaces/gui/src/locales/zh.json@@ -2457,7 +2457,14 @@"reviewer_key_save": "保存密钥","reviewer_key_placeholder_decisions": "OpenAI API 密钥","reviewer_key_placeholder_jev": "TypeSafe API 密钥",- "reviewer_key_failed": "无法保存密钥。"+ "reviewer_key_failed": "无法保存密钥。",+ "shadow_summary_title": "影子评估结果 · 最近 {{days}} 天,所有会话,{{reviewer}}",+ "shadow_summary_counts": "{{checks}} 次检查:本会放行 {{run}} · 询问你 {{asked}} · 阻止 {{stopped}}",+ "shadow_summary_matched": "与你自己的决定一致 {{matched}} / {{decided}} 次。",+ "shadow_summary_empty": "尚无记录。照常使用即可;你处理的每张审批卡片都会被记录。",+ "shadow_export": "导出完整结果",+ "shadow_export_saved": "已保存到 {{path}}",+ "shadow_export_failed": "无法导出结果。"},"sandbox": {"title": "沙箱",diff --git a/tests/test_decision_reviewer.py b/tests/test_decision_reviewer.pyindex 6a1a416b..8d568724 100644--- a/tests/test_decision_reviewer.py+++ b/tests/test_decision_reviewer.py@@ -384,3 +384,66 @@ def test_eval_stub_runs_each_decision_reviewer(backend, capsys):out = capsys.readouterr().outassert rc == 0, outassert f"{backend}:" in out+++# -- shadow results: summary and CSV export ---------------------------------------------------+++def _shadow(store, call, verdict, reviewer="jev", decision=None, approval="once"):+ store.append({"session_id": "s1", "tool": "run_shell", "arguments": {"command": f"cmd {call}"},+ "stage": "reviewer_shadow", "status": verdict, "reason": f"r {call}",+ "call_id": call, "reviewer": reviewer})+ if decision:+ store.append({"session_id": "s1", "tool": "run_shell", "arguments": {}, "stage": "approval_resolved",+ "status": decision, "approval": approval, "call_id": call})+++def test_shadow_summary_counts_and_matches(tmp_path):+ from coworker.audit import AuditStore++ store = AuditStore(tmp_path / "a.db")+ _shadow(store, "c1", "allow", decision="approved") # run, you allowed: match+ _shadow(store, "c2", "allow", decision="denied") # run, you denied: no match+ _shadow(store, "c3", "deny", decision="denied") # stop, you denied: match+ _shadow(store, "c4", "unsure", decision="approved") # asked: not counted either way+ _shadow(store, "c5", "allow") # never answered: not counted+ _shadow(store, "c6", "deny", decision="denied", approval="auto_refused") # nobody decided+ _shadow(store, "c7", "allow", reviewer="openai-decisions", decision="approved") # other reviewer+ s = store.shadow_summary("jev")+ assert (s["checks"], s["run"], s["asked"], s["stopped"]) == (6, 3, 1, 2)+ assert (s["decided"], s["matched"]) == (3, 2)+ assert store.shadow_summary("openai-decisions")["checks"] == 1+++def test_shadow_export_writes_a_readable_csv(client, tmp_path):+ import csv++ client.post("/v1/settings/auto-approve-reviewer", json={"auto_approve_reviewer": "jev"})+ mgr = client.app.state.manager if hasattr(client.app.state, "manager") else None+ store = mgr.audit_store if mgr else None+ if store is None:+ pytest.skip("manager not reachable from the test client")+ _shadow(store, "c1", "allow", decision="denied")+ summary = client.get("/v1/settings/auto-approve-shadow-summary").json()+ assert summary["reviewer"] == "jev" and summary["checks"] == 1 and summary["matched"] == 0+ r = client.post("/v1/settings/auto-approve-shadow-export").json()+ assert r["ok"] and r["rows"] == 1 and r["path"].endswith(".csv")+ with open(r["path"], encoding="utf-8-sig", newline="") as fh:+ rows = list(csv.DictReader(fh))+ assert rows[0]["reviewer would"] == "run" and rows[0]["your decision"] == "denied" and rows[0]["matched"] == "no"+ assert "cmd c1" in rows[0]["action"]+++def test_engine_tags_shadow_rows_with_the_reviewer(tmp_path):+ engine, rows, approvals = _engine(+ tmp_path,+ [_tool_turn(("run_shell", {"command": "git push"})), AssistantTurn(text="ok", finish_reason="stop")],+ mode=__import__("coworker.permissions", fromlist=["Mode"]).Mode.INTERACTIVE,+ )+ engine.reviewer = _DecisionLike("allow")+ engine.reviewer_shadow = True+ engine.reviewer_choice = "jev"+ _run(engine)+ asyncio.run(engine.drain_shadow_reviews())+ shadow = [r for r in rows if r.get("stage") == "reviewer_shadow"]+ assert shadow and shadow[0]["reviewer"] == "jev"