Skip to content
Python

fix(observability): Auto-approve is one table, a row per reviewer (T3-6)

Devika Verma07f394db990e
diff --git a/coworker/observability/store.py b/coworker/observability/store.py
index 308423ae..35d889f4 100644
--- a/coworker/observability/store.py
+++ b/coworker/observability/store.py
@@ -901,7 +901,7 @@ class RecordingStore:
self.flush()
with self._lock:
rows = self._conn.execute(
- "SELECT s.kind, s.shadow, s.duration_ms, s.attrs, s.parent_step_id FROM steps s "
+ "SELECT s.kind, s.shadow, s.duration_ms, s.attrs, s.parent_step_id, t.reviewer FROM steps s "
"JOIN turns t ON t.turn_id = s.turn_id WHERE t.started_at >= ? AND t.started_at < ? "
"AND (s.kind = 'auto_approver' OR (s.kind = 'tool' AND s.attrs LIKE '%allow_anyway%'))"
# one coworker: by the ledger's coworker id, which stands in for 1.2 turns
@@ -909,7 +909,26 @@ class RecordingStore:
if persona_id else ""),
(start, end, persona_id) if persona_id else (start, end)).fetchall()
return [{"kind": r["kind"], "shadow": bool(r["shadow"]), "duration_ms": r["duration_ms"],
- "parent_step_id": r["parent_step_id"], "attrs": json.loads(r["attrs"] or "{}")} for r in rows]
+ "parent_step_id": r["parent_step_id"], "attrs": json.loads(r["attrs"] or "{}"),
+ "turn_reviewer": json.loads(r["reviewer"]) if r["reviewer"] else None} for r in rows]
+
+ def approver_tokens_by_reviewer(self, start: int, end: int, *,
+ persona_id: Optional[str] = None) -> dict[str, int]:
+ """The live auto-approver model calls' tokens of turns that started in the period, by the
+ reviewer kind the turn ran with (T3-6, one row per reviewer). Shadow checks are left out:
+ the strip's total, from the ledger, has them."""
+ self.flush()
+ with self._lock:
+ rows = self._conn.execute(
+ "SELECT json_extract(t.reviewer, '$.kind') AS rkind, SUM(COALESCE(s.input_fresh,0)"
+ "+COALESCE(s.cache_read,0)+COALESCE(s.cache_write,0)+COALESCE(s.output,0)) AS tok "
+ "FROM steps s JOIN turns t ON t.turn_id = s.turn_id WHERE t.started_at >= ? AND t.started_at < ? "
+ "AND s.kind = 'model' AND s.shadow = 0 AND s.usage_reported = 1 "
+ "AND s.purpose = 'auto_approver'"
+ + (" AND EXISTS (SELECT 1 FROM turn_usage u WHERE u.turn_id = t.turn_id AND u.persona_id = ?)"
+ if persona_id else "") + " GROUP BY rkind",
+ (start, end, persona_id) if persona_id else (start, end)).fetchall()
+ return {r["rkind"] or "unknown": int(r["tok"] or 0) for r in rows}
def outside_calls_between(self, start: int, end: int) -> dict[str, int]:
"""Model steps with no turn (proxy calls) in [start, end): in no pill, so outside the
diff --git a/coworker/observability/usage_api.py b/coworker/observability/usage_api.py
index 96b2e5b5..f10fe01e 100644
--- a/coworker/observability/usage_api.py
+++ b/coworker/observability/usage_api.py
@@ -101,9 +101,15 @@ def approver(store: Any, *, start: int, end: int, persona_id: Optional[str] = No
steps = store.approver_steps(start, end, persona_id=persona_id)
live = [s for s in steps if s["kind"] == "auto_approver" and not s["shadow"] and s["parent_step_id"] is None]
shadow = [s for s in steps if s["kind"] == "auto_approver" and s["shadow"]]
- overridden = sum(1 for s in steps if s["kind"] == "tool" and any(
+ overrides = [s for s in steps if s["kind"] == "tool" and any(
x.get("stage") == "grant" and x.get("outcome") == "allow_anyway"
- for x in (s["attrs"].get("decision") or {}).get("path") or []))
+ for x in (s["attrs"].get("decision") or {}).get("path") or [])]
+ # T3-6: an override counts for the reviewer its turn ran with (the one whose block was
+ # let through); the step itself names none, as no reviewer is asked again.
+ overridden_by: dict[str, int] = {}
+ for s in overrides:
+ k = (s.get("turn_reviewer") or {}).get("kind") or "unknown"
+ overridden_by[k] = overridden_by.get(k, 0) + 1
def counts(rows: list[dict]) -> dict[str, Any]:
res = [r["attrs"].get("result") for r in rows]
@@ -117,8 +123,10 @@ def approver(store: Any, *, start: int, end: int, persona_id: Optional[str] = No
tokens = sum(g["total"] for g in store.usage_rollup(["purpose"], start, end,
where={"persona_id": persona_id} if persona_id else None)
if g["key"]["purpose"] == "auto_approver")
- return {**counts(live), "overridden": overridden, "tokens": tokens, "shadow_checks": len(shadow),
- "by_reviewer": [{"kind": k, **counts(v)} for k, v in sorted(by_kind.items())],
+ tokens_by = store.approver_tokens_by_reviewer(start, end, persona_id=persona_id)
+ return {**counts(live), "overridden": len(overrides), "tokens": tokens, "shadow_checks": len(shadow),
+ "by_reviewer": [{"kind": k, **counts(v), "overridden": overridden_by.get(k, 0),
+ "tokens": tokens_by.get(k, 0)} for k, v in sorted(by_kind.items())],
"shown": bool(live)}
diff --git a/surfaces/gui/src/api.ts b/surfaces/gui/src/api.ts
index 39a91eeb..9dc672f8 100644
--- a/surfaces/gui/src/api.ts
+++ b/surfaces/gui/src/api.ts
@@ -4733,7 +4733,9 @@ export interface ApproverStrip extends ApproverCounts {
overridden: number;
tokens: number;
shadow_checks: number;
- by_reviewer: (ApproverCounts & { kind: string })[];
+ // T3-6: each reviewer's own overrides and model tokens (live checks only; the strip's
+ // `tokens` total has the shadow checks too)
+ by_reviewer: (ApproverCounts & { kind: string; overridden?: number; tokens?: number })[];
shown: boolean;
}
diff --git a/surfaces/gui/src/components/UsagePage.test.tsx b/surfaces/gui/src/components/UsagePage.test.tsx
index c490328c..09ed2527 100644
--- a/surfaces/gui/src/components/UsagePage.test.tsx
+++ b/surfaces/gui/src/components/UsagePage.test.tsx
@@ -91,13 +91,37 @@ describe("ApproverStripView", () => {
tokens: 3796, shadow_checks: 2, by_reviewer: [{ kind: "session_model", checks: 1, allowed: 1, asked: 0, blocked: 0, could_not_run: 0, median_ms: 1000 }], shown: true };
render(<ApproverStripView ap={ap} t={t} />);
const text = screen.getByTestId("usage-approver").textContent || "";
- expect(text).toContain("allowed 1");
expect(text).not.toContain("%");
- expect(text).toContain("session model");
- expect(text).toContain("incl. 2 shadow checks");
+ expect(screen.getByTestId("usage-approver-session_model").textContent).toContain("Session model");
+ expect(screen.getByTestId("usage-approver-total").textContent).toContain("incl. 2 shadow checks");
+ expect(screen.getByTestId("usage-approver-total").textContent).toContain("3.8k");
expect(text).toContain("Auto-approve");
expect(text).not.toContain("Safety reviewer");
});
+
+ it("one row per reviewer, named in full, with its own overrides and tokens, and a Total row (T3-6)", () => {
+ const row = (kind: string, checks: number, allowed: number, asked: number, blocked: number, overridden: number, tokens: number) =>
+ ({ kind, checks, allowed, asked, blocked, could_not_run: 0, median_ms: 1000, overridden, tokens });
+ const ap = { checks: 98, allowed: 80, asked: 10, blocked: 8, could_not_run: 0, median_ms: 1000, overridden: 3,
+ tokens: 457_000, shadow_checks: 2, shown: true,
+ by_reviewer: [row("openai_decisions", 2, 1, 1, 0, 0, 0), row("session_model", 95, 79, 8, 8, 3, 450_000), row("typesafe_jev", 1, 0, 1, 0, 0, 0)] };
+ render(<ApproverStripView ap={ap} t={t} />);
+ const cells = (id: string) => [...screen.getByTestId(id).querySelectorAll("td")].map((c) => c.textContent);
+ expect(cells("usage-approver-openai_decisions")[0]).toBe("Decision model · OpenAI Decisions");
+ expect(cells("usage-approver-typesafe_jev")[0]).toBe("Decision model · Jev");
+ // overridden is each reviewer's own, not the period's total on every line
+ expect(cells("usage-approver-session_model")[6]).toBe("3");
+ expect(cells("usage-approver-typesafe_jev")[6]).toBe("0");
+ // 95 checks: shares; 2 checks: counts
+ expect(cells("usage-approver-session_model")[2]).toBe("83%");
+ expect(cells("usage-approver-openai_decisions")[2]).toBe("1");
+ const total = cells("usage-approver-total");
+ expect(total[0]).toContain("Total");
+ expect(total[0]).toContain("incl. 2 shadow checks");
+ expect(total[1]).toBe("98");
+ expect(total[6]).toBe("3");
+ expect(total[8]).toBe("457k");
+ });
});
describe("one coworker", () => {
diff --git a/surfaces/gui/src/components/UsagePage.tsx b/surfaces/gui/src/components/UsagePage.tsx
index a84e9f7b..8b7361ad 100644
--- a/surfaces/gui/src/components/UsagePage.tsx
+++ b/surfaces/gui/src/components/UsagePage.tsx
@@ -322,36 +322,57 @@ export function coworkerTable(groups: RollupGroup[], names: Record<string, strin
return [...rows.values()].sort((a, b) => b.total - a.total);
}
+/** T3-6: one row per reviewer, named in full, and a Total row; the point is comparing
+ * reviewers, so they stay apart. Under 10 checks a cell is a count, from 10 a share (M3-R5). */
export function ApproverStripView({ ap, t }: { ap: ApproverStrip; t: T }) {
- const lines = ap.by_reviewer.length ? ap.by_reviewer : [{ ...ap, kind: "" }];
+ const rows = ap.by_reviewer.length ? ap.by_reviewer : [{ ...ap, kind: "unknown", tokens: undefined }];
+ const part = (n: number, checks: number) => {
+ const p = approverPart(n, checks);
+ return p.share ? t("p3.ap.share", { n: p.value }) : String(p.value);
+ };
+ const cells = (l: { checks: number; allowed: number; asked: number; blocked: number; could_not_run: number; median_ms: number | null },
+ overridden: number | undefined, tokens: number | undefined) => (
+ <>
+ <td className="n">{l.checks}</td>
+ <td className="n c-ok">{part(l.allowed, l.checks)}</td>
+ <td className="n c-warn">{part(l.asked, l.checks)}</td>
+ <td className="n c-err">{part(l.blocked, l.checks)}</td>
+ <td className="n">{l.could_not_run}</td>
+ <td className="n">{overridden ?? "—"}</td>
+ <td className="n">{formatDuration(l.median_ms)}</td>
+ <td className="n">{tokens == null ? "—" : formatTokens(tokens)}</td>
+ </>
+ );
+ const anyShare = rows.some((r) => r.checks >= 10) || ap.checks >= 10;
return (
- <div className={CARD + " p3-card mt-3 flex flex-col gap-1.5"} data-testid="usage-approver">
- {lines.map((l) => {
- const part = (n: number, key: string) => {
- const p = approverPart(n, l.checks);
- return t(`p3.ap.${key}_${p.share ? "share" : "count"}`, { n: p.value });
- };
- return (
- <div key={l.kind} className="flex items-center gap-3 flex-wrap text-meta">
- <b className="p3-h">{t("p3.ap.title")}</b>
- <span className="text-muted">
- {t("p3.ap.checks", { count: l.checks })}
- {l.kind ? " · " + t(`p3.ap.kind.${l.kind}`, { defaultValue: l.kind }) : ""}
- </span>
- <span className="p3-chip ok">{part(l.allowed, "allowed")}</span>
- <span className="p3-chip warn">{part(l.asked, "asked")}</span>
- <span className="p3-chip err">{part(l.blocked, "blocked")}</span>
- <span className="text-muted">
- {t("p3.ap.could_not_run", { n: l.could_not_run })} · {t("p3.ap.overridden", { n: ap.overridden })} ·{" "}
- {t("p3.ap.median", { time: formatDuration(l.median_ms) })}
- </span>
- </div>
- );
- })}
- <div className="text-meta text-muted">
- {t("p3.ap.tokens", { tokens: formatTokens(ap.tokens) })}
- {ap.shadow_checks > 0 ? " · " + t("p3.ap.incl_shadow", { count: ap.shadow_checks }) : ""}
- </div>
+ <div className={CARD + " p3-card mt-3"} data-testid="usage-approver">
+ <b className="p3-h">{t("p3.ap.title")}</b>
+ <table className="p3-t p3-ap" data-testid="usage-approver-table">
+ <thead>
+ <tr>
+ <th>{t("p3.ap.col.reviewer")}</th>
+ {(["checks", "allowed", "asked", "blocked", "could_not_run", "overridden", "median", "tokens"] as const).map((c) => (
+ <th key={c} className="n">{t(`p3.ap.col.${c}`)}</th>
+ ))}
+ </tr>
+ </thead>
+ <tbody>
+ {rows.map((l) => (
+ <tr key={l.kind} data-testid={`usage-approver-${l.kind}`}>
+ <td>{t(`p3.ap.name.${l.kind}`, { defaultValue: l.kind })}</td>
+ {cells(l, l.overridden, l.tokens)}
+ </tr>
+ ))}
+ <tr className="p3-total" data-testid="usage-approver-total">
+ <td>
+ <b>{t("p3.ap.total")}</b>
+ {ap.shadow_checks > 0 && <div className="text-faint">{t("p3.ap.incl_shadow", { count: ap.shadow_checks })}</div>}
+ </td>
+ {cells(ap, ap.overridden, ap.tokens)}
+ </tr>
+ </tbody>
+ </table>
+ {anyShare && <div className="text-meta text-faint mt-1">{t("p3.ap.shares_note")}</div>}
</div>
);
}
diff --git a/surfaces/gui/src/locales/en.json b/surfaces/gui/src/locales/en.json
index 2c6367e0..5978fd88 100644
--- a/surfaces/gui/src/locales/en.json
+++ b/surfaces/gui/src/locales/en.json
@@ -3968,23 +3968,26 @@
},
"ap": {
"title": "Auto-approve",
- "checks_one": "{{count}} check",
- "checks_other": "{{count}} checks",
- "kind": {
- "session_model": "session model",
- "openai_decisions": "decision model",
- "typesafe_jev": "decision model"
+ "col": {
+ "reviewer": "Reviewer",
+ "checks": "Checks",
+ "allowed": "Allowed",
+ "asked": "Asked you",
+ "blocked": "Blocked",
+ "could_not_run": "Could not run",
+ "overridden": "Overridden",
+ "median": "Median time",
+ "tokens": "Tokens"
},
- "allowed_count": "allowed {{n}}",
- "allowed_share": "allowed {{n}}%",
- "asked_count": "asked you {{n}}",
- "asked_share": "asked you {{n}}%",
- "blocked_count": "blocked {{n}}",
- "blocked_share": "blocked {{n}}%",
- "could_not_run": "could not run {{n}}",
- "overridden": "overridden {{n}}",
- "median": "{{time}} median",
- "tokens": "{{tokens}} tokens",
+ "name": {
+ "session_model": "Session model",
+ "openai_decisions": "Decision model · OpenAI Decisions",
+ "typesafe_jev": "Decision model · Jev",
+ "unknown": "Unknown reviewer"
+ },
+ "total": "Total",
+ "share": "{{n}}%",
+ "shares_note": "From 10 checks, a row shows shares of its checks; under 10, counts.",
"incl_shadow_one": "incl. {{count}} shadow check",
"incl_shadow_other": "incl. {{count}} shadow checks"
},
diff --git a/surfaces/gui/src/locales/zh.json b/surfaces/gui/src/locales/zh.json
index fe779169..e8f837c9 100644
--- a/surfaces/gui/src/locales/zh.json
+++ b/surfaces/gui/src/locales/zh.json
@@ -3908,22 +3908,26 @@
},
"ap": {
"title": "自动审批",
- "checks_other": "{{count}} 次检查",
- "kind": {
+ "col": {
+ "reviewer": "审核者",
+ "checks": "检查",
+ "allowed": "允许",
+ "asked": "询问你",
+ "blocked": "阻止",
+ "could_not_run": "无法运行",
+ "overridden": "被你放行",
+ "median": "中位用时",
+ "tokens": "令牌"
+ },
+ "name": {
"session_model": "会话模型",
- "openai_decisions": "决策模型",
- "typesafe_jev": "决策模型"
+ "openai_decisions": "决策模型 · OpenAI Decisions",
+ "typesafe_jev": "决策模型 · Jev",
+ "unknown": "未知审核者"
},
- "allowed_count": "允许 {{n}}",
- "allowed_share": "允许 {{n}}%",
- "asked_count": "询问你 {{n}}",
- "asked_share": "询问你 {{n}}%",
- "blocked_count": "阻止 {{n}}",
- "blocked_share": "阻止 {{n}}%",
- "could_not_run": "无法运行 {{n}}",
- "overridden": "被你放行 {{n}}",
- "median": "中位数 {{time}}",
- "tokens": "{{tokens}} 令牌",
+ "total": "合计",
+ "share": "{{n}}%",
+ "shares_note": "一行有 10 次及以上检查时显示占比,不足 10 次时显示次数。",
"incl_shadow_other": "含 {{count}} 次影子检查"
},
"foot": {
diff --git a/surfaces/gui/src/styles.css b/surfaces/gui/src/styles.css
index d5ceb078..ce359554 100644
--- a/surfaces/gui/src/styles.css
+++ b/surfaces/gui/src/styles.css
@@ -2644,6 +2644,9 @@ html[data-theme="dark"] { --bar-model:#4c8dff; --bar-tool:#3fb8ad; --bar-wait:#3
.p3-chip.ok { background:var(--ok-soft); color:var(--ok); }
.p3-chip.warn { background:var(--warn-soft); color:var(--warn-ink); }
.p3-chip.err { background:var(--danger-soft); color:var(--danger); }
+/* T3-6: the Auto-approve table, one row per reviewer and a Total row */
+.p3-ap td.c-ok { color:var(--ok); } .p3-ap td.c-warn { color:var(--warn-ink); } .p3-ap td.c-err { color:var(--danger); }
+.p3-ap tr.p3-total td { border-top:1px solid var(--line-strong, var(--line)); }
.p3-callout { border-radius:9px; padding:8px 12px; font-size:var(--fs-meta); margin:8px 0; }
.p3-callout.err { background:var(--danger-soft); color:var(--danger); }
.p3-event { display:flex; align-items:center; gap:8px; width:100%; background:none; border:0; padding:3px 0; font-size:var(--fs-meta); color:var(--ink); cursor:pointer; }
diff --git a/tests/test_observability_phase3_manual_fixes.py b/tests/test_observability_phase3_manual_fixes.py
new file mode 100644
index 00000000..479e9a5e
--- /dev/null
+++ b/tests/test_observability_phase3_manual_fixes.py
@@ -0,0 +1,71 @@
+"""Phase 3 manual testing (5 Nov), the server side of rows T3-6 and T3-10: the Auto-approve table
+counts "overridden" per reviewer, and the Ratings card's numbers."""
+
+from __future__ import annotations
+
+import time
+
+from coworker.observability import RecordingStore, usage_api
+
+# Turns an hour old: a store with the background writer sweeps on its first tick.
+RECENT = int(time.time() * 1000) - 3_600_000
+
+
+def _turn(tid, sid, seq, at, reviewer_kind=None):
+ return {"turn_id": tid, "session_id": sid, "seq": seq, "kind": "run", "trigger": "user", "started_at": at,
+ "duration_ms": 1000, "status": "completed", "incomplete": False, "approval_mode": "auto_approve",
+ "attended": True, "reviewer": {"kind": reviewer_kind, "active": True, "shadow": False} if reviewer_kind else None,
+ "stamp": {"machine": {"kind": "this_machine", "id": None}}, "first_message_id": None,
+ "last_message_id": None, "schema_version": 1, "wake": {"reason": "user"}}
+
+
+def _step(tid, sid, n, at, kind, attrs):
+ return {"step_id": f"{tid}-s{n}", "turn_id": tid, "session_id": sid, "parent_step_id": None, "seq": n,
+ "kind": kind, "name": kind, "started_at": at, "duration_ms": 100, "status": "ok", "call_id": None,
+ "call_index": None, "message_id": None, "shadow": False, "after_turn": False, "attrs": attrs,
+ "content": {}}
+
+
+def _check(result, kind):
+ return {"result": result, "reviewer": {"kind": kind}}
+
+
+OVERRIDE = {"decision": {"path": [{"stage": "auto_approver", "outcome": "deny"}, {"stage": "grant", "outcome": "allow_anyway"}]}}
+
+
+def test_overridden_is_counted_per_reviewer(tmp_path):
+ store = RecordingStore(tmp_path / "recordings.db", background=False)
+ turns = [_turn("A", "s1", 1, RECENT, "session_model"), _turn("B", "s2", 1, RECENT + 10, "typesafe_jev"),
+ _turn("C", "s2", 2, RECENT + 20, "typesafe_jev"), _turn("D", "s3", 1, RECENT + 30, "openai_decisions")]
+ steps = [
+ _step("A", "s1", 1, RECENT, "auto_approver", _check("allow", "session_model")),
+ _step("A", "s1", 2, RECENT + 1, "auto_approver", _check("allow", "session_model")),
+ _step("B", "s2", 1, RECENT + 10, "auto_approver", _check("deny", "typesafe_jev")),
+ # the person let Jev's block through, twice
+ _step("C", "s2", 1, RECENT + 20, "tool", OVERRIDE),
+ _step("C", "s2", 2, RECENT + 21, "tool", OVERRIDE),
+ _step("D", "s3", 1, RECENT + 30, "auto_approver", _check("unsure", "openai_decisions")),
+ # the reviewers' own model calls: tokens per reviewer
+ _step("A", "s1", 3, RECENT + 2, "model", {"purpose": "auto_approver", "model": "model-x", "usage": {
+ "reported": True, "input_fresh": 300, "cache_read": 0, "cache_write": 0, "output": 20}}),
+ _step("D", "s3", 2, RECENT + 31, "model", {"purpose": "auto_approver", "model": "model-y", "usage": {
+ "reported": True, "input_fresh": 50, "cache_read": 0, "cache_write": 0, "output": 5}}),
+ _step("A", "s1", 4, RECENT + 3, "model", {"purpose": "main", "model": "model-x", "usage": {
+ "reported": True, "input_fresh": 9000, "cache_read": 0, "cache_write": 0, "output": 100}}),
+ ]
+ for t in turns:
+ store.write_turn(t)
+ for s in steps:
+ store.write_step(s)
+ store.flush()
+ got = usage_api.approver(store, start=RECENT - 1, end=RECENT + 1000)
+ assert got["overridden"] == 2 # the total, as before
+ rows = {r["kind"]: r for r in got["by_reviewer"]}
+ assert set(rows) == {"session_model", "typesafe_jev", "openai_decisions"}
+ assert rows["typesafe_jev"]["overridden"] == 2
+ assert rows["session_model"]["overridden"] == 0 and rows["openai_decisions"]["overridden"] == 0
+ assert rows["session_model"]["checks"] == 2 and rows["openai_decisions"]["asked"] == 1
+ # each reviewer's own tokens; main work is not in them
+ assert rows["session_model"]["tokens"] == 320 and rows["openai_decisions"]["tokens"] == 55
+ assert rows["typesafe_jev"]["tokens"] == 0
+ store.close()