Python
fix(observability): Auto-approve is one table, a row per reviewer (T3-6)
Devika Verma07f394db990e
diff --git a/coworker/observability/store.py b/coworker/observability/store.pyindex 308423ae..35d889f4 100644--- a/coworker/observability/store.py+++ b/coworker/observability/store.py@@ -901,7 +901,7 @@ class RecordingStore:self.flush()with self._lock:rows = self._conn.execute(- "SELECT s.kind, s.shadow, s.duration_ms, s.attrs, s.parent_step_id FROM steps s "+ "SELECT s.kind, s.shadow, s.duration_ms, s.attrs, s.parent_step_id, t.reviewer FROM steps s ""JOIN turns t ON t.turn_id = s.turn_id WHERE t.started_at >= ? AND t.started_at < ? ""AND (s.kind = 'auto_approver' OR (s.kind = 'tool' AND s.attrs LIKE '%allow_anyway%'))"# one coworker: by the ledger's coworker id, which stands in for 1.2 turns@@ -909,7 +909,26 @@ class RecordingStore:if persona_id else ""),(start, end, persona_id) if persona_id else (start, end)).fetchall()return [{"kind": r["kind"], "shadow": bool(r["shadow"]), "duration_ms": r["duration_ms"],- "parent_step_id": r["parent_step_id"], "attrs": json.loads(r["attrs"] or "{}")} for r in rows]+ "parent_step_id": r["parent_step_id"], "attrs": json.loads(r["attrs"] or "{}"),+ "turn_reviewer": json.loads(r["reviewer"]) if r["reviewer"] else None} for r in rows]++ def approver_tokens_by_reviewer(self, start: int, end: int, *,+ persona_id: Optional[str] = None) -> dict[str, int]:+ """The live auto-approver model calls' tokens of turns that started in the period, by the+ reviewer kind the turn ran with (T3-6, one row per reviewer). Shadow checks are left out:+ the strip's total, from the ledger, has them."""+ self.flush()+ with self._lock:+ rows = self._conn.execute(+ "SELECT json_extract(t.reviewer, '$.kind') AS rkind, SUM(COALESCE(s.input_fresh,0)"+ "+COALESCE(s.cache_read,0)+COALESCE(s.cache_write,0)+COALESCE(s.output,0)) AS tok "+ "FROM steps s JOIN turns t ON t.turn_id = s.turn_id WHERE t.started_at >= ? AND t.started_at < ? "+ "AND s.kind = 'model' AND s.shadow = 0 AND s.usage_reported = 1 "+ "AND s.purpose = 'auto_approver'"+ + (" AND EXISTS (SELECT 1 FROM turn_usage u WHERE u.turn_id = t.turn_id AND u.persona_id = ?)"+ if persona_id else "") + " GROUP BY rkind",+ (start, end, persona_id) if persona_id else (start, end)).fetchall()+ return {r["rkind"] or "unknown": int(r["tok"] or 0) for r in rows}def outside_calls_between(self, start: int, end: int) -> dict[str, int]:"""Model steps with no turn (proxy calls) in [start, end): in no pill, so outside thediff --git a/coworker/observability/usage_api.py b/coworker/observability/usage_api.pyindex 96b2e5b5..f10fe01e 100644--- a/coworker/observability/usage_api.py+++ b/coworker/observability/usage_api.py@@ -101,9 +101,15 @@ def approver(store: Any, *, start: int, end: int, persona_id: Optional[str] = Nosteps = store.approver_steps(start, end, persona_id=persona_id)live = [s for s in steps if s["kind"] == "auto_approver" and not s["shadow"] and s["parent_step_id"] is None]shadow = [s for s in steps if s["kind"] == "auto_approver" and s["shadow"]]- overridden = sum(1 for s in steps if s["kind"] == "tool" and any(+ overrides = [s for s in steps if s["kind"] == "tool" and any(x.get("stage") == "grant" and x.get("outcome") == "allow_anyway"- for x in (s["attrs"].get("decision") or {}).get("path") or []))+ for x in (s["attrs"].get("decision") or {}).get("path") or [])]+ # T3-6: an override counts for the reviewer its turn ran with (the one whose block was+ # let through); the step itself names none, as no reviewer is asked again.+ overridden_by: dict[str, int] = {}+ for s in overrides:+ k = (s.get("turn_reviewer") or {}).get("kind") or "unknown"+ overridden_by[k] = overridden_by.get(k, 0) + 1def counts(rows: list[dict]) -> dict[str, Any]:res = [r["attrs"].get("result") for r in rows]@@ -117,8 +123,10 @@ def approver(store: Any, *, start: int, end: int, persona_id: Optional[str] = Notokens = sum(g["total"] for g in store.usage_rollup(["purpose"], start, end,where={"persona_id": persona_id} if persona_id else None)if g["key"]["purpose"] == "auto_approver")- return {**counts(live), "overridden": overridden, "tokens": tokens, "shadow_checks": len(shadow),- "by_reviewer": [{"kind": k, **counts(v)} for k, v in sorted(by_kind.items())],+ tokens_by = store.approver_tokens_by_reviewer(start, end, persona_id=persona_id)+ return {**counts(live), "overridden": len(overrides), "tokens": tokens, "shadow_checks": len(shadow),+ "by_reviewer": [{"kind": k, **counts(v), "overridden": overridden_by.get(k, 0),+ "tokens": tokens_by.get(k, 0)} for k, v in sorted(by_kind.items())],"shown": bool(live)}diff --git a/surfaces/gui/src/api.ts b/surfaces/gui/src/api.tsindex 39a91eeb..9dc672f8 100644--- a/surfaces/gui/src/api.ts+++ b/surfaces/gui/src/api.ts@@ -4733,7 +4733,9 @@ export interface ApproverStrip extends ApproverCounts {overridden: number;tokens: number;shadow_checks: number;- by_reviewer: (ApproverCounts & { kind: string })[];+ // T3-6: each reviewer's own overrides and model tokens (live checks only; the strip's+ // `tokens` total has the shadow checks too)+ by_reviewer: (ApproverCounts & { kind: string; overridden?: number; tokens?: number })[];shown: boolean;}diff --git a/surfaces/gui/src/components/UsagePage.test.tsx b/surfaces/gui/src/components/UsagePage.test.tsxindex c490328c..09ed2527 100644--- a/surfaces/gui/src/components/UsagePage.test.tsx+++ b/surfaces/gui/src/components/UsagePage.test.tsx@@ -91,13 +91,37 @@ describe("ApproverStripView", () => {tokens: 3796, shadow_checks: 2, by_reviewer: [{ kind: "session_model", checks: 1, allowed: 1, asked: 0, blocked: 0, could_not_run: 0, median_ms: 1000 }], shown: true };render(<ApproverStripView ap={ap} t={t} />);const text = screen.getByTestId("usage-approver").textContent || "";- expect(text).toContain("allowed 1");expect(text).not.toContain("%");- expect(text).toContain("session model");- expect(text).toContain("incl. 2 shadow checks");+ expect(screen.getByTestId("usage-approver-session_model").textContent).toContain("Session model");+ expect(screen.getByTestId("usage-approver-total").textContent).toContain("incl. 2 shadow checks");+ expect(screen.getByTestId("usage-approver-total").textContent).toContain("3.8k");expect(text).toContain("Auto-approve");expect(text).not.toContain("Safety reviewer");});++ it("one row per reviewer, named in full, with its own overrides and tokens, and a Total row (T3-6)", () => {+ const row = (kind: string, checks: number, allowed: number, asked: number, blocked: number, overridden: number, tokens: number) =>+ ({ kind, checks, allowed, asked, blocked, could_not_run: 0, median_ms: 1000, overridden, tokens });+ const ap = { checks: 98, allowed: 80, asked: 10, blocked: 8, could_not_run: 0, median_ms: 1000, overridden: 3,+ tokens: 457_000, shadow_checks: 2, shown: true,+ by_reviewer: [row("openai_decisions", 2, 1, 1, 0, 0, 0), row("session_model", 95, 79, 8, 8, 3, 450_000), row("typesafe_jev", 1, 0, 1, 0, 0, 0)] };+ render(<ApproverStripView ap={ap} t={t} />);+ const cells = (id: string) => [...screen.getByTestId(id).querySelectorAll("td")].map((c) => c.textContent);+ expect(cells("usage-approver-openai_decisions")[0]).toBe("Decision model · OpenAI Decisions");+ expect(cells("usage-approver-typesafe_jev")[0]).toBe("Decision model · Jev");+ // overridden is each reviewer's own, not the period's total on every line+ expect(cells("usage-approver-session_model")[6]).toBe("3");+ expect(cells("usage-approver-typesafe_jev")[6]).toBe("0");+ // 95 checks: shares; 2 checks: counts+ expect(cells("usage-approver-session_model")[2]).toBe("83%");+ expect(cells("usage-approver-openai_decisions")[2]).toBe("1");+ const total = cells("usage-approver-total");+ expect(total[0]).toContain("Total");+ expect(total[0]).toContain("incl. 2 shadow checks");+ expect(total[1]).toBe("98");+ expect(total[6]).toBe("3");+ expect(total[8]).toBe("457k");+ });});describe("one coworker", () => {diff --git a/surfaces/gui/src/components/UsagePage.tsx b/surfaces/gui/src/components/UsagePage.tsxindex a84e9f7b..8b7361ad 100644--- a/surfaces/gui/src/components/UsagePage.tsx+++ b/surfaces/gui/src/components/UsagePage.tsx@@ -322,36 +322,57 @@ export function coworkerTable(groups: RollupGroup[], names: Record<string, strinreturn [...rows.values()].sort((a, b) => b.total - a.total);}+/** T3-6: one row per reviewer, named in full, and a Total row; the point is comparing+ * reviewers, so they stay apart. Under 10 checks a cell is a count, from 10 a share (M3-R5). */export function ApproverStripView({ ap, t }: { ap: ApproverStrip; t: T }) {- const lines = ap.by_reviewer.length ? ap.by_reviewer : [{ ...ap, kind: "" }];+ const rows = ap.by_reviewer.length ? ap.by_reviewer : [{ ...ap, kind: "unknown", tokens: undefined }];+ const part = (n: number, checks: number) => {+ const p = approverPart(n, checks);+ return p.share ? t("p3.ap.share", { n: p.value }) : String(p.value);+ };+ const cells = (l: { checks: number; allowed: number; asked: number; blocked: number; could_not_run: number; median_ms: number | null },+ overridden: number | undefined, tokens: number | undefined) => (+ <>+ <td className="n">{l.checks}</td>+ <td className="n c-ok">{part(l.allowed, l.checks)}</td>+ <td className="n c-warn">{part(l.asked, l.checks)}</td>+ <td className="n c-err">{part(l.blocked, l.checks)}</td>+ <td className="n">{l.could_not_run}</td>+ <td className="n">{overridden ?? "—"}</td>+ <td className="n">{formatDuration(l.median_ms)}</td>+ <td className="n">{tokens == null ? "—" : formatTokens(tokens)}</td>+ </>+ );+ const anyShare = rows.some((r) => r.checks >= 10) || ap.checks >= 10;return (- <div className={CARD + " p3-card mt-3 flex flex-col gap-1.5"} data-testid="usage-approver">- {lines.map((l) => {- const part = (n: number, key: string) => {- const p = approverPart(n, l.checks);- return t(`p3.ap.${key}_${p.share ? "share" : "count"}`, { n: p.value });- };- return (- <div key={l.kind} className="flex items-center gap-3 flex-wrap text-meta">- <b className="p3-h">{t("p3.ap.title")}</b>- <span className="text-muted">- {t("p3.ap.checks", { count: l.checks })}- {l.kind ? " · " + t(`p3.ap.kind.${l.kind}`, { defaultValue: l.kind }) : ""}- </span>- <span className="p3-chip ok">{part(l.allowed, "allowed")}</span>- <span className="p3-chip warn">{part(l.asked, "asked")}</span>- <span className="p3-chip err">{part(l.blocked, "blocked")}</span>- <span className="text-muted">- {t("p3.ap.could_not_run", { n: l.could_not_run })} · {t("p3.ap.overridden", { n: ap.overridden })} ·{" "}- {t("p3.ap.median", { time: formatDuration(l.median_ms) })}- </span>- </div>- );- })}- <div className="text-meta text-muted">- {t("p3.ap.tokens", { tokens: formatTokens(ap.tokens) })}- {ap.shadow_checks > 0 ? " · " + t("p3.ap.incl_shadow", { count: ap.shadow_checks }) : ""}- </div>+ <div className={CARD + " p3-card mt-3"} data-testid="usage-approver">+ <b className="p3-h">{t("p3.ap.title")}</b>+ <table className="p3-t p3-ap" data-testid="usage-approver-table">+ <thead>+ <tr>+ <th>{t("p3.ap.col.reviewer")}</th>+ {(["checks", "allowed", "asked", "blocked", "could_not_run", "overridden", "median", "tokens"] as const).map((c) => (+ <th key={c} className="n">{t(`p3.ap.col.${c}`)}</th>+ ))}+ </tr>+ </thead>+ <tbody>+ {rows.map((l) => (+ <tr key={l.kind} data-testid={`usage-approver-${l.kind}`}>+ <td>{t(`p3.ap.name.${l.kind}`, { defaultValue: l.kind })}</td>+ {cells(l, l.overridden, l.tokens)}+ </tr>+ ))}+ <tr className="p3-total" data-testid="usage-approver-total">+ <td>+ <b>{t("p3.ap.total")}</b>+ {ap.shadow_checks > 0 && <div className="text-faint">{t("p3.ap.incl_shadow", { count: ap.shadow_checks })}</div>}+ </td>+ {cells(ap, ap.overridden, ap.tokens)}+ </tr>+ </tbody>+ </table>+ {anyShare && <div className="text-meta text-faint mt-1">{t("p3.ap.shares_note")}</div>}</div>);}diff --git a/surfaces/gui/src/locales/en.json b/surfaces/gui/src/locales/en.jsonindex 2c6367e0..5978fd88 100644--- a/surfaces/gui/src/locales/en.json+++ b/surfaces/gui/src/locales/en.json@@ -3968,23 +3968,26 @@},"ap": {"title": "Auto-approve",- "checks_one": "{{count}} check",- "checks_other": "{{count}} checks",- "kind": {- "session_model": "session model",- "openai_decisions": "decision model",- "typesafe_jev": "decision model"+ "col": {+ "reviewer": "Reviewer",+ "checks": "Checks",+ "allowed": "Allowed",+ "asked": "Asked you",+ "blocked": "Blocked",+ "could_not_run": "Could not run",+ "overridden": "Overridden",+ "median": "Median time",+ "tokens": "Tokens"},- "allowed_count": "allowed {{n}}",- "allowed_share": "allowed {{n}}%",- "asked_count": "asked you {{n}}",- "asked_share": "asked you {{n}}%",- "blocked_count": "blocked {{n}}",- "blocked_share": "blocked {{n}}%",- "could_not_run": "could not run {{n}}",- "overridden": "overridden {{n}}",- "median": "{{time}} median",- "tokens": "{{tokens}} tokens",+ "name": {+ "session_model": "Session model",+ "openai_decisions": "Decision model · OpenAI Decisions",+ "typesafe_jev": "Decision model · Jev",+ "unknown": "Unknown reviewer"+ },+ "total": "Total",+ "share": "{{n}}%",+ "shares_note": "From 10 checks, a row shows shares of its checks; under 10, counts.","incl_shadow_one": "incl. {{count}} shadow check","incl_shadow_other": "incl. {{count}} shadow checks"},diff --git a/surfaces/gui/src/locales/zh.json b/surfaces/gui/src/locales/zh.jsonindex fe779169..e8f837c9 100644--- a/surfaces/gui/src/locales/zh.json+++ b/surfaces/gui/src/locales/zh.json@@ -3908,22 +3908,26 @@},"ap": {"title": "自动审批",- "checks_other": "{{count}} 次检查",- "kind": {+ "col": {+ "reviewer": "审核者",+ "checks": "检查",+ "allowed": "允许",+ "asked": "询问你",+ "blocked": "阻止",+ "could_not_run": "无法运行",+ "overridden": "被你放行",+ "median": "中位用时",+ "tokens": "令牌"+ },+ "name": {"session_model": "会话模型",- "openai_decisions": "决策模型",- "typesafe_jev": "决策模型"+ "openai_decisions": "决策模型 · OpenAI Decisions",+ "typesafe_jev": "决策模型 · Jev",+ "unknown": "未知审核者"},- "allowed_count": "允许 {{n}}",- "allowed_share": "允许 {{n}}%",- "asked_count": "询问你 {{n}}",- "asked_share": "询问你 {{n}}%",- "blocked_count": "阻止 {{n}}",- "blocked_share": "阻止 {{n}}%",- "could_not_run": "无法运行 {{n}}",- "overridden": "被你放行 {{n}}",- "median": "中位数 {{time}}",- "tokens": "{{tokens}} 令牌",+ "total": "合计",+ "share": "{{n}}%",+ "shares_note": "一行有 10 次及以上检查时显示占比,不足 10 次时显示次数。","incl_shadow_other": "含 {{count}} 次影子检查"},"foot": {diff --git a/surfaces/gui/src/styles.css b/surfaces/gui/src/styles.cssindex d5ceb078..ce359554 100644--- a/surfaces/gui/src/styles.css+++ b/surfaces/gui/src/styles.css@@ -2644,6 +2644,9 @@ html[data-theme="dark"] { --bar-model:#4c8dff; --bar-tool:#3fb8ad; --bar-wait:#3.p3-chip.ok { background:var(--ok-soft); color:var(--ok); }.p3-chip.warn { background:var(--warn-soft); color:var(--warn-ink); }.p3-chip.err { background:var(--danger-soft); color:var(--danger); }+/* T3-6: the Auto-approve table, one row per reviewer and a Total row */+.p3-ap td.c-ok { color:var(--ok); } .p3-ap td.c-warn { color:var(--warn-ink); } .p3-ap td.c-err { color:var(--danger); }+.p3-ap tr.p3-total td { border-top:1px solid var(--line-strong, var(--line)); }.p3-callout { border-radius:9px; padding:8px 12px; font-size:var(--fs-meta); margin:8px 0; }.p3-callout.err { background:var(--danger-soft); color:var(--danger); }.p3-event { display:flex; align-items:center; gap:8px; width:100%; background:none; border:0; padding:3px 0; font-size:var(--fs-meta); color:var(--ink); cursor:pointer; }diff --git a/tests/test_observability_phase3_manual_fixes.py b/tests/test_observability_phase3_manual_fixes.pynew file mode 100644index 00000000..479e9a5e--- /dev/null+++ b/tests/test_observability_phase3_manual_fixes.py@@ -0,0 +1,71 @@+"""Phase 3 manual testing (5 Nov), the server side of rows T3-6 and T3-10: the Auto-approve table+counts "overridden" per reviewer, and the Ratings card's numbers."""++from __future__ import annotations++import time++from coworker.observability import RecordingStore, usage_api++# Turns an hour old: a store with the background writer sweeps on its first tick.+RECENT = int(time.time() * 1000) - 3_600_000+++def _turn(tid, sid, seq, at, reviewer_kind=None):+ return {"turn_id": tid, "session_id": sid, "seq": seq, "kind": "run", "trigger": "user", "started_at": at,+ "duration_ms": 1000, "status": "completed", "incomplete": False, "approval_mode": "auto_approve",+ "attended": True, "reviewer": {"kind": reviewer_kind, "active": True, "shadow": False} if reviewer_kind else None,+ "stamp": {"machine": {"kind": "this_machine", "id": None}}, "first_message_id": None,+ "last_message_id": None, "schema_version": 1, "wake": {"reason": "user"}}+++def _step(tid, sid, n, at, kind, attrs):+ return {"step_id": f"{tid}-s{n}", "turn_id": tid, "session_id": sid, "parent_step_id": None, "seq": n,+ "kind": kind, "name": kind, "started_at": at, "duration_ms": 100, "status": "ok", "call_id": None,+ "call_index": None, "message_id": None, "shadow": False, "after_turn": False, "attrs": attrs,+ "content": {}}+++def _check(result, kind):+ return {"result": result, "reviewer": {"kind": kind}}+++OVERRIDE = {"decision": {"path": [{"stage": "auto_approver", "outcome": "deny"}, {"stage": "grant", "outcome": "allow_anyway"}]}}+++def test_overridden_is_counted_per_reviewer(tmp_path):+ store = RecordingStore(tmp_path / "recordings.db", background=False)+ turns = [_turn("A", "s1", 1, RECENT, "session_model"), _turn("B", "s2", 1, RECENT + 10, "typesafe_jev"),+ _turn("C", "s2", 2, RECENT + 20, "typesafe_jev"), _turn("D", "s3", 1, RECENT + 30, "openai_decisions")]+ steps = [+ _step("A", "s1", 1, RECENT, "auto_approver", _check("allow", "session_model")),+ _step("A", "s1", 2, RECENT + 1, "auto_approver", _check("allow", "session_model")),+ _step("B", "s2", 1, RECENT + 10, "auto_approver", _check("deny", "typesafe_jev")),+ # the person let Jev's block through, twice+ _step("C", "s2", 1, RECENT + 20, "tool", OVERRIDE),+ _step("C", "s2", 2, RECENT + 21, "tool", OVERRIDE),+ _step("D", "s3", 1, RECENT + 30, "auto_approver", _check("unsure", "openai_decisions")),+ # the reviewers' own model calls: tokens per reviewer+ _step("A", "s1", 3, RECENT + 2, "model", {"purpose": "auto_approver", "model": "model-x", "usage": {+ "reported": True, "input_fresh": 300, "cache_read": 0, "cache_write": 0, "output": 20}}),+ _step("D", "s3", 2, RECENT + 31, "model", {"purpose": "auto_approver", "model": "model-y", "usage": {+ "reported": True, "input_fresh": 50, "cache_read": 0, "cache_write": 0, "output": 5}}),+ _step("A", "s1", 4, RECENT + 3, "model", {"purpose": "main", "model": "model-x", "usage": {+ "reported": True, "input_fresh": 9000, "cache_read": 0, "cache_write": 0, "output": 100}}),+ ]+ for t in turns:+ store.write_turn(t)+ for s in steps:+ store.write_step(s)+ store.flush()+ got = usage_api.approver(store, start=RECENT - 1, end=RECENT + 1000)+ assert got["overridden"] == 2 # the total, as before+ rows = {r["kind"]: r for r in got["by_reviewer"]}+ assert set(rows) == {"session_model", "typesafe_jev", "openai_decisions"}+ assert rows["typesafe_jev"]["overridden"] == 2+ assert rows["session_model"]["overridden"] == 0 and rows["openai_decisions"]["overridden"] == 0+ assert rows["session_model"]["checks"] == 2 and rows["openai_decisions"]["asked"] == 1+ # each reviewer's own tokens; main work is not in them+ assert rows["session_model"]["tokens"] == 320 and rows["openai_decisions"]["tokens"] == 55+ assert rows["typesafe_jev"]["tokens"] == 0+ store.close()