Python
feat(observability): a Ratings card on Usage (T3-10)
Devika Verma77653c0ff7a6
diff --git a/coworker/observability/store.py b/coworker/observability/store.pyindex 35d889f4..6826d93c 100644--- a/coworker/observability/store.py+++ b/coworker/observability/store.py@@ -1100,6 +1100,19 @@ class RecordingStore:self._conn.execute("DELETE FROM feedback WHERE turn_id = ?", (turn_id,))self._conn.execute("UPDATE turn_usage SET rating = NULL WHERE turn_id = ?", (turn_id,))+ def feedback_for(self, turn_ids: list[str]) -> dict[str, dict[str, Any]]:+ """The ratings of these turns, by turn id (the Usage page's Ratings card, T3-10)."""+ out: dict[str, dict[str, Any]] = {}+ ids = list(dict.fromkeys(turn_ids))+ with self._lock:+ for i in range(0, len(ids), 400):+ chunk = ids[i:i + 400]+ rows = self._conn.execute(+ f"SELECT * FROM feedback WHERE turn_id IN ({','.join('?' * len(chunk))})", chunk).fetchall()+ for r in rows:+ out[r["turn_id"]] = _feedback_record(r)+ return out+def session_feedback(self, session_id: str) -> list[dict[str, Any]]:with self._lock:rows = self._conn.execute("SELECT * FROM feedback WHERE session_id = ? ORDER BY rated_at",diff --git a/coworker/observability/usage_api.py b/coworker/observability/usage_api.pyindex f10fe01e..6add2259 100644--- a/coworker/observability/usage_api.py+++ b/coworker/observability/usage_api.py@@ -130,6 +130,43 @@ def approver(store: Any, *, start: int, end: int, persona_id: Optional[str] = No"shown": bool(live)}+RATED_DOWN_SHOWN = 50+++def ratings(store: Any, *, start: int, end: int, persona_id: Optional[str] = None,+ title_of: Optional[Callable[[str], Optional[str]]] = None) -> dict[str, Any]:+ """`GET /v1/observability/ratings`: the Usage page's Ratings card (T3-10, lean). The 👍 and+ 👎 of the turns that started in the period (the same turns the page counts), how often each+ reason was picked, and the 👎 turns, newest first, with their note and how many of their+ steps the auto-approver let through. No charts or trends: those stay with Langfuse."""+ where = {"persona_id": persona_id} if persona_id else None+ rated = [t for t in store.usage_turns(start, end, where=where) if t.get("rating") in ("up", "down")]+ down = sorted((t for t in rated if t["rating"] == "down"), key=lambda t: (-(t["started_at"] or 0), t["turn_id"]))+ feedback = store.feedback_for([t["turn_id"] for t in down])+ reasons: dict[str, int] = {}+ for f in feedback.values():+ for r in f.get("reasons") or []:+ reasons[r] = reasons.get(r, 0) + 1+ shown = down[:RATED_DOWN_SHOWN]+ seqs = store.turn_seqs([t["turn_id"] for t in shown]) if shown else {}+ turns = []+ for t in shown:+ f = feedback.get(t["turn_id"]) or {}+ title = None+ if title_of is not None and not t["session_deleted"]:+ try:+ title = title_of(t["session_id"])+ except Exception: # noqa: BLE001+ title = None+ turns.append({"turn_id": t["turn_id"], "session_id": t["session_id"], "title": title,+ "seq": seqs.get(t["turn_id"]), "session_deleted": bool(t["session_deleted"]),+ "persona_id": t["persona_id"], "started_at": t["started_at"],+ "reasons": f.get("reasons") or [], "note": (f.get("content") or {}).get("note"),+ "auto_approved": len(f.get("auto_approved") or [])})+ return {"up": sum(1 for t in rated if t["rating"] == "up"), "down": len(down), "reasons": reasons,+ "down_turns": turns, "more": max(0, len(down) - len(shown))}++# -- OB-15: how a coworker has been doing ------------------------------------------------------diff --git a/coworker/server/app.py b/coworker/server/app.pyindex 23cfcee1..56b19e28 100644--- a/coworker/server/app.py+++ b/coworker/server/app.py@@ -926,6 +926,15 @@ def create_app(manager: SessionManager) -> FastAPI:return JSONResponse({"error": str(exc)}, status_code=400)return manager.usage_approver(start=start, end=end, persona_id=persona_id)+ @app.get("/v1/observability/ratings")+ def observability_ratings(request: Request, to: Optional[int] = None, persona_id: Optional[str] = None):+ # T3-10: the Ratings card on Usage+ try:+ start, end, _ = _period(request.query_params.get("from"), to, None)+ except (ValueError, TypeError) as exc:+ return JSONResponse({"error": str(exc)}, status_code=400)+ return manager.usage_ratings(start=start, end=end, persona_id=persona_id)+@app.get("/v1/personas/{persona_id}/doing")def persona_doing(persona_id: str, days: int = 30, to: Optional[int] = None) -> dict[str, Any]:return manager.persona_doing(persona_id, days=days, end=to)diff --git a/coworker/server/manager.py b/coworker/server/manager.pyindex f7723c1c..e999b35d 100644--- a/coworker/server/manager.py+++ b/coworker/server/manager.py@@ -7028,6 +7028,13 @@ class SessionManager:def usage_approver(self, *, start: int, end: int, persona_id: Optional[str] = None) -> dict[str, Any]:return usage_api.approver(self.recordings_store, start=start, end=end, persona_id=persona_id)+ def usage_ratings(self, *, start: int, end: int, persona_id: Optional[str] = None) -> dict[str, Any]:+ """The Ratings card on Usage (T3-10)."""+ out = usage_api.ratings(self.recordings_store, start=start, end=end, persona_id=persona_id,+ title_of=self.session_store.display_title)+ out["names"] = self.usage_names()+ return out+def current_persona_fp(self, persona_id: str) -> Optional[str]:"""The coworker's fingerprint now, worked out as the session build does (E16)."""entry = self.personas.get(persona_id)diff --git a/surfaces/gui/src/api.ts b/surfaces/gui/src/api.tsindex 9dc672f8..68269cb6 100644--- a/surfaces/gui/src/api.ts+++ b/surfaces/gui/src/api.ts@@ -4739,6 +4739,35 @@ export interface ApproverStrip extends ApproverCounts {shown: boolean;}+// T3-10: the Ratings card on Usage (counts, reasons, and the turns rated 👎)+export interface RatedDownTurn {+ turn_id: string;+ session_id: string;+ title: string | null;+ seq: number | null;+ session_deleted: boolean;+ persona_id: string | null;+ started_at: number;+ reasons: string[];+ note: string | null;+ auto_approved: number;+}+export interface RatingsSummary {+ up: number;+ down: number;+ reasons: Record<string, number>;+ down_turns: RatedDownTurn[];+ more: number;+ names?: Record<string, string>;+}++export async function getRatings(from: number, to: number, personaId?: string | null): Promise<RatingsSummary> {+ const who = personaId ? `&persona_id=${encodeURIComponent(personaId)}` : "";+ const res = await fetch(`${engineBase(null)}/v1/observability/ratings?from=${from}&to=${to}${who}`);+ if (!res.ok) throw new Error(`HTTP ${res.status}`);+ return res.json();+}+export async function getApproverStrip(from: number, to: number, machineId?: string | null,personaId?: string | null): Promise<ApproverStrip> {const who = personaId ? `&persona_id=${encodeURIComponent(personaId)}` : "";diff --git a/surfaces/gui/src/components/UsagePage.test.tsx b/surfaces/gui/src/components/UsagePage.test.tsxindex 9cde72cc..78f4ba4e 100644--- a/surfaces/gui/src/components/UsagePage.test.tsx+++ b/surfaces/gui/src/components/UsagePage.test.tsx@@ -2,7 +2,7 @@import { afterEach, describe, expect, it, vi } from "vitest";import { cleanup, fireEvent, render, screen } from "@testing-library/react";import i18n from "i18next";-import { ApproverStripView, coworkerChoices, coworkerTable, loadUsage, UsageBody, type UsageData } from "./UsagePage";+import { ApproverStripView, coworkerChoices, coworkerTable, loadUsage, RatingsCard, UsageBody, type UsageData } from "./UsagePage";import type { UsageRollup } from "../api";afterEach(cleanup);@@ -181,3 +181,39 @@ describe("one coworker", () => {expect(narrowed).toHaveLength(usage.length - 1);});});++describe("RatingsCard (T3-10)", () => {+ const ratings = {+ up: 4, down: 2, reasons: { too_slow: 1, wrong_answer: 2, did_unasked: 1 }, more: 0,+ down_turns: [+ { turn_id: "t7", session_id: "s1", title: "Fix the build", seq: 7, session_deleted: false, persona_id: "cowork",+ started_at: 2, reasons: ["wrong_answer", "did_unasked"], note: "ran a command I didn't want", auto_approved: 2 },+ { turn_id: "t3", session_id: "gone", title: null, seq: null, session_deleted: true, persona_id: "cowork",+ started_at: 1, reasons: [], note: null, auto_approved: 0 },+ ],+ };++ it("counts the thumbs, the reasons in their fixed order, and lists the 👎 turns with their note", () => {+ const open = vi.fn();+ render(<RatingsCard r={ratings} names={{ cowork: "OpenWorker" }} t={t} onOpenTurn={open} />);+ expect(screen.getByTestId("usage-ratings-counts").textContent).toBe("👍 4👎 2");+ const reasons = screen.getByTestId("usage-ratings-reasons").textContent || "";+ expect(reasons.indexOf("Wrong answer 2")).toBeLessThan(reasons.indexOf("Took too long 1"));+ expect(reasons).toContain("Did something I didn't ask 1");+ const table = screen.getByTestId("usage-ratings-down").textContent || "";+ expect(table).toContain("Fix the build");+ expect(table).toContain("OpenWorker");+ expect(screen.getAllByTestId("usage-ratings-note")[0].textContent).toBe("ran a command I didn't want");+ // a deleted session's turn has no Trace link+ const links = screen.getAllByTestId("usage-ratings-open");+ expect(links).toHaveLength(1);+ expect(links[0].textContent).toBe("Trace");+ fireEvent.click(links[0]);+ expect(open).toHaveBeenCalledWith("s1", "t7");+ });++ it("says so when nothing was rated", () => {+ render(<RatingsCard r={{ up: 0, down: 0, reasons: {}, down_turns: [], more: 0 }} names={{}} t={t} />);+ expect(screen.getByTestId("usage-ratings").textContent).toContain("No turns of this period were rated.");+ });+});diff --git a/surfaces/gui/src/components/UsagePage.tsx b/surfaces/gui/src/components/UsagePage.tsxindex 222843c7..e3e50547 100644--- a/surfaces/gui/src/components/UsagePage.tsx+++ b/surfaces/gui/src/components/UsagePage.tsx@@ -9,8 +9,10 @@ import {getApproverStrip,getHeaviestTurns,getMachines,+ getRatings,getUsageRollup,type ApproverStrip,+ type RatingsSummary,type HeavyTurn,type Machine,type UsageRollup,@@ -26,6 +28,7 @@ import {overheadByDay,previousDays,purposeByDay,+ REASONS,SERIES,type Period,type RollupGroup,@@ -51,6 +54,8 @@ export interface UsageData {heavy: HeavyTurn[];approver: ApproverStrip | null;machines: MachineRow[];+ /** T3-10; null when the engine has no ratings endpoint (older) or it failed */+ ratings?: RatingsSummary | null;}/** The coworkers of the period for the picker: each id once, by its name (M3-R1), A to Z. */@@ -77,7 +82,7 @@ export async function loadUsage(period: Period, now = new Date(), personaId: str// One coworker, when picked: every number on the page narrows to its turns. The By// coworker table keeps listing all of them, so the picker always has its choices.const who = personaId ? { persona_id: personaId } : {};- const [total, prev, dayPurpose, coworkers, heavy, approver, fleet] = await Promise.all([+ const [total, prev, dayPurpose, coworkers, heavy, approver, fleet, ratings] = await Promise.all([getUsageRollup({ from, to, group: "all", ...who }),getUsageRollup({ from: prevDays[0][1], to: prevDays[prevDays.length - 1][2], group: "all", ...who }).catch(() => null),getUsageRollup({ from, to, days, group: "day,purpose", ...who }),@@ -85,6 +90,7 @@ export async function loadUsage(period: Period, now = new Date(), personaId: strgetHeaviestTurns(from, to, null, 20, personaId).then((r) => r.turns).catch(() => [] as HeavyTurn[]),getApproverStrip(from, to, null, personaId).catch(() => null),getMachines().then((r) => r.machines || []).catch(() => [] as Machine[]),+ getRatings(from, to, personaId).catch(() => null),]);const here = sumAll(total);const machines: MachineRow[] = [@@ -104,7 +110,7 @@ export async function loadUsage(period: Period, now = new Date(), personaId: str}}),);- return { total, prev, dayPurpose, coworkers, heavy, approver, machines: [...machines, ...remote] };+ return { total, prev, dayPurpose, coworkers, heavy, approver, machines: [...machines, ...remote], ratings };}export function UsagePage({ onOpenTurn, onOpenPersona }: {@@ -307,6 +313,8 @@ export function UsageBody({ data, period, t, onOpenTurn, onOpenPersona, picked ={ap && ap.shown && <ApproverStripView ap={ap} t={t} />}+ {data.ratings && <RatingsCard r={data.ratings} names={names} t={t} onOpenTurn={onOpenTurn} />}+{data.total.no_turn.calls > 0 && (<div className="text-meta text-faint mt-2" data-testid="usage-no-turn">{t("p3.usage.no_turn", { calls: data.total.no_turn.calls, tokens: data.total.no_turn.total.toLocaleString() })}@@ -380,6 +388,71 @@ export function coworkerTable(groups: RollupGroup[], names: Record<string, strinreturn [...rows.values()].sort((a, b) => b.total - a.total);}+/** T3-10 (lean): the period's 👍 and 👎, how often each reason was picked, and the turns rated+ * 👎 with their note, each opening its trace. No charts or trends: those stay with Langfuse. */+export function RatingsCard({ r, names, t, onOpenTurn }: {+ r: RatingsSummary; names: Record<string, string>; t: T; onOpenTurn?: (sessionId: string, turnId: string) => void;+}) {+ const reasonLabel = (k: string) => t(`p3.fb.reason.${k}`, { defaultValue: k });+ const picked = [...REASONS.filter((k) => r.reasons[k]), ...Object.keys(r.reasons).filter((k) => !(REASONS as readonly string[]).includes(k))];+ return (+ <div className={CARD + " p3-card mt-3"} data-testid="usage-ratings">+ <b className="p3-h">{t("p3.ratings.title")}</b>+ {r.up + r.down === 0 ? (+ <div className="text-meta text-faint mt-1">{t("p3.ratings.none")}</div>+ ) : (+ <>+ <div className="flex items-center gap-4 mt-1 text-ui" data-testid="usage-ratings-counts">+ <span>👍 {r.up}</span>+ <span>👎 {r.down}</span>+ </div>+ {picked.length > 0 && (+ <div className="flex gap-1.5 flex-wrap mt-2 items-center text-meta" data-testid="usage-ratings-reasons">+ <span className="text-faint">{t("p3.ratings.reasons")}</span>+ {picked.map((k) => (+ <span key={k} className="p3-chip">{reasonLabel(k)} {r.reasons[k]}</span>+ ))}+ </div>+ )}+ {r.down_turns.length > 0 && (+ <table className="p3-t p3-heavy mt-2" data-testid="usage-ratings-down">+ <colgroup>+ <col style={{ width: "26%" }} /><col style={{ width: "14%" }} /><col style={{ width: "22%" }} />+ <col style={{ width: "24%" }} /><col style={{ width: "7%" }} /><col style={{ width: "7%" }} />+ </colgroup>+ <thead><tr><th>{t("p3.usage.session")}</th><th>{t("p3.usage.coworker")}</th><th>{t("p3.ratings.col_reasons")}</th>+ <th>{t("p3.ratings.col_note")}</th><th className="n" title={t("p3.ratings.auto_title")}>{t("p3.ratings.col_auto")}</th><th></th></tr></thead>+ <tbody>+ {r.down_turns.map((d) => {+ const title = d.session_deleted ? t("p3.usage.deleted_session") : (d.title || d.session_id);+ const turn = d.seq != null ? t("p3.usage.turn_n", { n: d.seq }) : "";+ const who = (d.persona_id && names[d.persona_id]) || d.persona_id || "—";+ const reasons = d.reasons.map(reasonLabel).join(", ");+ return (+ <tr key={d.turn_id}>+ <td><span className="p3-cut" title={turn ? `${title} · ${turn}` : title}>+ {d.session_deleted ? <span className="text-faint">{title}</span> : title}+ {turn && <span className="text-faint"> · {turn}</span>}</span></td>+ <td><span className="p3-cut" title={who}>{who}</span></td>+ <td><span className="p3-cut" title={reasons}>{reasons || <span className="text-faint">—</span>}</span></td>+ <td><span className="p3-cut" title={d.note || ""} data-testid="usage-ratings-note">{d.note || <span className="text-faint">—</span>}</span></td>+ <td className="n">{d.auto_approved || "—"}</td>+ <td>{!d.session_deleted && onOpenTurn && (+ <button className="p3-link" data-testid="usage-ratings-open" onClick={() => onOpenTurn(d.session_id, d.turn_id)}>{t("steps.open")}</button>+ )}</td>+ </tr>+ );+ })}+ </tbody>+ </table>+ )}+ {r.more > 0 && <div className="text-meta text-faint mt-1">{t("p3.ratings.more", { count: r.more })}</div>}+ </>+ )}+ </div>+ );+}+/** T3-6: one row per reviewer, named in full, and a Total row; the point is comparing* reviewers, so they stay apart. Under 10 checks a cell is a count, from 10 a share (M3-R5). */export function ApproverStripView({ ap, t }: { ap: ApproverStrip; t: T }) {diff --git a/surfaces/gui/src/locales/en.json b/surfaces/gui/src/locales/en.jsonindex 58ce44bf..3f40a531 100644--- a/surfaces/gui/src/locales/en.json+++ b/surfaces/gui/src/locales/en.json@@ -3997,6 +3997,17 @@"incl_shadow_one": "incl. {{count}} shadow check","incl_shadow_other": "incl. {{count}} shadow checks"},+ "ratings": {+ "title": "Ratings",+ "none": "No turns of this period were rated.",+ "reasons": "Reasons picked:",+ "col_reasons": "Reasons",+ "col_note": "Note",+ "col_auto": "Auto-approved",+ "auto_title": "Steps of that turn the auto-approver let through",+ "more_one": "and {{count}} more turn rated 👎",+ "more_other": "and {{count}} more turns rated 👎"+ },"foot": {"tokens": "{{tokens}} tokens this turn","issues_one": "{{count}} issue",diff --git a/surfaces/gui/src/locales/zh.json b/surfaces/gui/src/locales/zh.jsonindex a5f7f995..fc2f8e1f 100644--- a/surfaces/gui/src/locales/zh.json+++ b/surfaces/gui/src/locales/zh.json@@ -3936,6 +3936,16 @@"shares_note": "一行有 10 次及以上检查时显示占比,不足 10 次时显示次数。","incl_shadow_other": "含 {{count}} 次影子检查"},+ "ratings": {+ "title": "评价",+ "none": "这段时间没有被评价的轮次。",+ "reasons": "选择的原因:",+ "col_reasons": "原因",+ "col_note": "备注",+ "col_auto": "自动批准",+ "auto_title": "该轮次中由自动审批放行的步骤",+ "more_other": "另有 {{count}} 个被评为 👎 的轮次"+ },"foot": {"tokens": "本轮 {{tokens}} 令牌","issues_other": "{{count}} 个问题",diff --git a/tests/test_observability_phase3_manual_fixes.py b/tests/test_observability_phase3_manual_fixes.pyindex 1fc0f53f..7d81507b 100644--- a/tests/test_observability_phase3_manual_fixes.py+++ b/tests/test_observability_phase3_manual_fixes.py@@ -107,4 +107,45 @@ def test_the_default_coworkers_page_and_history_answer(tmp_path, monkeypatch):body = doing.json()assert body["persona_id"] == DEFAULT_PERSONA_ID and body["runs"] == 1, bodyassert sum(r["turns"] for r in body["rows"]) == 1+ # T3-10's route answers with the card's fields+ now = int(time.time() * 1000)+ got = client.get("/v1/observability/ratings", params={"from": now - 86_400_000, "to": now + 60_000}).json()+ assert (got["up"], got["down"], got["down_turns"], got["more"]) == (0, 0, [], 0)manager.observability.close()+++# -- T3-10 ----------------------------------------------------------------------------------------+def test_the_ratings_card_counts_reasons_and_lists_the_down_turns(tmp_path):+ from coworker.observability import signals++ store = RecordingStore(tmp_path / "recordings.db", background=False)+ main = {"purpose": "main", "model": "model-x",+ "usage": {"reported": True, "input_fresh": 10, "cache_read": 0, "cache_write": 0, "output": 1}}+ for i, (sid, persona) in enumerate([("s1", "cowork"), ("s1", "cowork"), ("s2", "sec"), ("s2", "sec"), ("s3", "cowork")]):+ t = _turn(f"T{i}", sid, i + 1, RECENT + i * 1000)+ t["stamp"]["persona_id"] = persona+ store.write_turn(t)+ store.write_step(_step(f"T{i}", sid, 1, RECENT + i * 1000, "model", main))+ store.write_issues(f"T{i}", {"signals_version": signals.version(), "warn": 0, "info": 0, "issues": [], "fine": []},+ seen=True)+ store.flush()+ fb = lambda tid, sid, rating, reasons=(), note=None, auto=(): store.put_feedback( # noqa: E731+ {"turn_id": tid, "session_id": sid, "rating": rating, "reasons": list(reasons), "note": note,+ "auto_approved": list(auto)})+ fb("T0", "s1", "up")+ fb("T1", "s1", "down", ["wrong_answer", "too_slow"], "it guessed")+ fb("T2", "s2", "down", ["did_unasked"], None, ["T2-s4", "T2-s5"])+ fb("T3", "s2", "up")+ # T4 is not rated+ got = usage_api.ratings(store, start=RECENT - 1, end=RECENT + 60_000, title_of=lambda sid: f"title {sid}")+ assert (got["up"], got["down"]) == (2, 2)+ assert got["reasons"] == {"wrong_answer": 1, "too_slow": 1, "did_unasked": 1}+ assert [t["turn_id"] for t in got["down_turns"]] == ["T2", "T1"] # newest first+ t2, t1 = got["down_turns"]+ assert t1["note"] == "it guessed" and t1["title"] == "title s1" and t1["reasons"] == ["wrong_answer", "too_slow"]+ assert t2["auto_approved"] == 2 and t2["note"] is None and t2["seq"] == 3+ assert got["more"] == 0+ # one coworker narrows it, as the rest of the page+ one = usage_api.ratings(store, start=RECENT - 1, end=RECENT + 60_000, persona_id="sec")+ assert (one["up"], one["down"]) == (1, 1) and one["down_turns"][0]["turn_id"] == "T2"+ store.close()