Skip to content
Python

feat(observability): a Ratings card on Usage (T3-10)

Devika Verma77653c0ff7a6
diff --git a/coworker/observability/store.py b/coworker/observability/store.py
index 35d889f4..6826d93c 100644
--- a/coworker/observability/store.py
+++ b/coworker/observability/store.py
@@ -1100,6 +1100,19 @@ class RecordingStore:
self._conn.execute("DELETE FROM feedback WHERE turn_id = ?", (turn_id,))
self._conn.execute("UPDATE turn_usage SET rating = NULL WHERE turn_id = ?", (turn_id,))
+ def feedback_for(self, turn_ids: list[str]) -> dict[str, dict[str, Any]]:
+ """The ratings of these turns, by turn id (the Usage page's Ratings card, T3-10)."""
+ out: dict[str, dict[str, Any]] = {}
+ ids = list(dict.fromkeys(turn_ids))
+ with self._lock:
+ for i in range(0, len(ids), 400):
+ chunk = ids[i:i + 400]
+ rows = self._conn.execute(
+ f"SELECT * FROM feedback WHERE turn_id IN ({','.join('?' * len(chunk))})", chunk).fetchall()
+ for r in rows:
+ out[r["turn_id"]] = _feedback_record(r)
+ return out
+
def session_feedback(self, session_id: str) -> list[dict[str, Any]]:
with self._lock:
rows = self._conn.execute("SELECT * FROM feedback WHERE session_id = ? ORDER BY rated_at",
diff --git a/coworker/observability/usage_api.py b/coworker/observability/usage_api.py
index f10fe01e..6add2259 100644
--- a/coworker/observability/usage_api.py
+++ b/coworker/observability/usage_api.py
@@ -130,6 +130,43 @@ def approver(store: Any, *, start: int, end: int, persona_id: Optional[str] = No
"shown": bool(live)}
+RATED_DOWN_SHOWN = 50
+
+
+def ratings(store: Any, *, start: int, end: int, persona_id: Optional[str] = None,
+ title_of: Optional[Callable[[str], Optional[str]]] = None) -> dict[str, Any]:
+ """`GET /v1/observability/ratings`: the Usage page's Ratings card (T3-10, lean). The 👍 and
+ 👎 of the turns that started in the period (the same turns the page counts), how often each
+ reason was picked, and the 👎 turns, newest first, with their note and how many of their
+ steps the auto-approver let through. No charts or trends: those stay with Langfuse."""
+ where = {"persona_id": persona_id} if persona_id else None
+ rated = [t for t in store.usage_turns(start, end, where=where) if t.get("rating") in ("up", "down")]
+ down = sorted((t for t in rated if t["rating"] == "down"), key=lambda t: (-(t["started_at"] or 0), t["turn_id"]))
+ feedback = store.feedback_for([t["turn_id"] for t in down])
+ reasons: dict[str, int] = {}
+ for f in feedback.values():
+ for r in f.get("reasons") or []:
+ reasons[r] = reasons.get(r, 0) + 1
+ shown = down[:RATED_DOWN_SHOWN]
+ seqs = store.turn_seqs([t["turn_id"] for t in shown]) if shown else {}
+ turns = []
+ for t in shown:
+ f = feedback.get(t["turn_id"]) or {}
+ title = None
+ if title_of is not None and not t["session_deleted"]:
+ try:
+ title = title_of(t["session_id"])
+ except Exception: # noqa: BLE001
+ title = None
+ turns.append({"turn_id": t["turn_id"], "session_id": t["session_id"], "title": title,
+ "seq": seqs.get(t["turn_id"]), "session_deleted": bool(t["session_deleted"]),
+ "persona_id": t["persona_id"], "started_at": t["started_at"],
+ "reasons": f.get("reasons") or [], "note": (f.get("content") or {}).get("note"),
+ "auto_approved": len(f.get("auto_approved") or [])})
+ return {"up": sum(1 for t in rated if t["rating"] == "up"), "down": len(down), "reasons": reasons,
+ "down_turns": turns, "more": max(0, len(down) - len(shown))}
+
+
# -- OB-15: how a coworker has been doing ------------------------------------------------------
diff --git a/coworker/server/app.py b/coworker/server/app.py
index 23cfcee1..56b19e28 100644
--- a/coworker/server/app.py
+++ b/coworker/server/app.py
@@ -926,6 +926,15 @@ def create_app(manager: SessionManager) -> FastAPI:
return JSONResponse({"error": str(exc)}, status_code=400)
return manager.usage_approver(start=start, end=end, persona_id=persona_id)
+ @app.get("/v1/observability/ratings")
+ def observability_ratings(request: Request, to: Optional[int] = None, persona_id: Optional[str] = None):
+ # T3-10: the Ratings card on Usage
+ try:
+ start, end, _ = _period(request.query_params.get("from"), to, None)
+ except (ValueError, TypeError) as exc:
+ return JSONResponse({"error": str(exc)}, status_code=400)
+ return manager.usage_ratings(start=start, end=end, persona_id=persona_id)
+
@app.get("/v1/personas/{persona_id}/doing")
def persona_doing(persona_id: str, days: int = 30, to: Optional[int] = None) -> dict[str, Any]:
return manager.persona_doing(persona_id, days=days, end=to)
diff --git a/coworker/server/manager.py b/coworker/server/manager.py
index f7723c1c..e999b35d 100644
--- a/coworker/server/manager.py
+++ b/coworker/server/manager.py
@@ -7028,6 +7028,13 @@ class SessionManager:
def usage_approver(self, *, start: int, end: int, persona_id: Optional[str] = None) -> dict[str, Any]:
return usage_api.approver(self.recordings_store, start=start, end=end, persona_id=persona_id)
+ def usage_ratings(self, *, start: int, end: int, persona_id: Optional[str] = None) -> dict[str, Any]:
+ """The Ratings card on Usage (T3-10)."""
+ out = usage_api.ratings(self.recordings_store, start=start, end=end, persona_id=persona_id,
+ title_of=self.session_store.display_title)
+ out["names"] = self.usage_names()
+ return out
+
def current_persona_fp(self, persona_id: str) -> Optional[str]:
"""The coworker's fingerprint now, worked out as the session build does (E16)."""
entry = self.personas.get(persona_id)
diff --git a/surfaces/gui/src/api.ts b/surfaces/gui/src/api.ts
index 9dc672f8..68269cb6 100644
--- a/surfaces/gui/src/api.ts
+++ b/surfaces/gui/src/api.ts
@@ -4739,6 +4739,35 @@ export interface ApproverStrip extends ApproverCounts {
shown: boolean;
}
+// T3-10: the Ratings card on Usage (counts, reasons, and the turns rated 👎)
+export interface RatedDownTurn {
+ turn_id: string;
+ session_id: string;
+ title: string | null;
+ seq: number | null;
+ session_deleted: boolean;
+ persona_id: string | null;
+ started_at: number;
+ reasons: string[];
+ note: string | null;
+ auto_approved: number;
+}
+export interface RatingsSummary {
+ up: number;
+ down: number;
+ reasons: Record<string, number>;
+ down_turns: RatedDownTurn[];
+ more: number;
+ names?: Record<string, string>;
+}
+
+export async function getRatings(from: number, to: number, personaId?: string | null): Promise<RatingsSummary> {
+ const who = personaId ? `&persona_id=${encodeURIComponent(personaId)}` : "";
+ const res = await fetch(`${engineBase(null)}/v1/observability/ratings?from=${from}&to=${to}${who}`);
+ if (!res.ok) throw new Error(`HTTP ${res.status}`);
+ return res.json();
+}
+
export async function getApproverStrip(from: number, to: number, machineId?: string | null,
personaId?: string | null): Promise<ApproverStrip> {
const who = personaId ? `&persona_id=${encodeURIComponent(personaId)}` : "";
diff --git a/surfaces/gui/src/components/UsagePage.test.tsx b/surfaces/gui/src/components/UsagePage.test.tsx
index 9cde72cc..78f4ba4e 100644
--- a/surfaces/gui/src/components/UsagePage.test.tsx
+++ b/surfaces/gui/src/components/UsagePage.test.tsx
@@ -2,7 +2,7 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import { cleanup, fireEvent, render, screen } from "@testing-library/react";
import i18n from "i18next";
-import { ApproverStripView, coworkerChoices, coworkerTable, loadUsage, UsageBody, type UsageData } from "./UsagePage";
+import { ApproverStripView, coworkerChoices, coworkerTable, loadUsage, RatingsCard, UsageBody, type UsageData } from "./UsagePage";
import type { UsageRollup } from "../api";
afterEach(cleanup);
@@ -181,3 +181,39 @@ describe("one coworker", () => {
expect(narrowed).toHaveLength(usage.length - 1);
});
});
+
+describe("RatingsCard (T3-10)", () => {
+ const ratings = {
+ up: 4, down: 2, reasons: { too_slow: 1, wrong_answer: 2, did_unasked: 1 }, more: 0,
+ down_turns: [
+ { turn_id: "t7", session_id: "s1", title: "Fix the build", seq: 7, session_deleted: false, persona_id: "cowork",
+ started_at: 2, reasons: ["wrong_answer", "did_unasked"], note: "ran a command I didn't want", auto_approved: 2 },
+ { turn_id: "t3", session_id: "gone", title: null, seq: null, session_deleted: true, persona_id: "cowork",
+ started_at: 1, reasons: [], note: null, auto_approved: 0 },
+ ],
+ };
+
+ it("counts the thumbs, the reasons in their fixed order, and lists the 👎 turns with their note", () => {
+ const open = vi.fn();
+ render(<RatingsCard r={ratings} names={{ cowork: "OpenWorker" }} t={t} onOpenTurn={open} />);
+ expect(screen.getByTestId("usage-ratings-counts").textContent).toBe("👍 4👎 2");
+ const reasons = screen.getByTestId("usage-ratings-reasons").textContent || "";
+ expect(reasons.indexOf("Wrong answer 2")).toBeLessThan(reasons.indexOf("Took too long 1"));
+ expect(reasons).toContain("Did something I didn't ask 1");
+ const table = screen.getByTestId("usage-ratings-down").textContent || "";
+ expect(table).toContain("Fix the build");
+ expect(table).toContain("OpenWorker");
+ expect(screen.getAllByTestId("usage-ratings-note")[0].textContent).toBe("ran a command I didn't want");
+ // a deleted session's turn has no Trace link
+ const links = screen.getAllByTestId("usage-ratings-open");
+ expect(links).toHaveLength(1);
+ expect(links[0].textContent).toBe("Trace");
+ fireEvent.click(links[0]);
+ expect(open).toHaveBeenCalledWith("s1", "t7");
+ });
+
+ it("says so when nothing was rated", () => {
+ render(<RatingsCard r={{ up: 0, down: 0, reasons: {}, down_turns: [], more: 0 }} names={{}} t={t} />);
+ expect(screen.getByTestId("usage-ratings").textContent).toContain("No turns of this period were rated.");
+ });
+});
diff --git a/surfaces/gui/src/components/UsagePage.tsx b/surfaces/gui/src/components/UsagePage.tsx
index 222843c7..e3e50547 100644
--- a/surfaces/gui/src/components/UsagePage.tsx
+++ b/surfaces/gui/src/components/UsagePage.tsx
@@ -9,8 +9,10 @@ import {
getApproverStrip,
getHeaviestTurns,
getMachines,
+ getRatings,
getUsageRollup,
type ApproverStrip,
+ type RatingsSummary,
type HeavyTurn,
type Machine,
type UsageRollup,
@@ -26,6 +28,7 @@ import {
overheadByDay,
previousDays,
purposeByDay,
+ REASONS,
SERIES,
type Period,
type RollupGroup,
@@ -51,6 +54,8 @@ export interface UsageData {
heavy: HeavyTurn[];
approver: ApproverStrip | null;
machines: MachineRow[];
+ /** T3-10; null when the engine has no ratings endpoint (older) or it failed */
+ ratings?: RatingsSummary | null;
}
/** The coworkers of the period for the picker: each id once, by its name (M3-R1), A to Z. */
@@ -77,7 +82,7 @@ export async function loadUsage(period: Period, now = new Date(), personaId: str
// One coworker, when picked: every number on the page narrows to its turns. The By
// coworker table keeps listing all of them, so the picker always has its choices.
const who = personaId ? { persona_id: personaId } : {};
- const [total, prev, dayPurpose, coworkers, heavy, approver, fleet] = await Promise.all([
+ const [total, prev, dayPurpose, coworkers, heavy, approver, fleet, ratings] = await Promise.all([
getUsageRollup({ from, to, group: "all", ...who }),
getUsageRollup({ from: prevDays[0][1], to: prevDays[prevDays.length - 1][2], group: "all", ...who }).catch(() => null),
getUsageRollup({ from, to, days, group: "day,purpose", ...who }),
@@ -85,6 +90,7 @@ export async function loadUsage(period: Period, now = new Date(), personaId: str
getHeaviestTurns(from, to, null, 20, personaId).then((r) => r.turns).catch(() => [] as HeavyTurn[]),
getApproverStrip(from, to, null, personaId).catch(() => null),
getMachines().then((r) => r.machines || []).catch(() => [] as Machine[]),
+ getRatings(from, to, personaId).catch(() => null),
]);
const here = sumAll(total);
const machines: MachineRow[] = [
@@ -104,7 +110,7 @@ export async function loadUsage(period: Period, now = new Date(), personaId: str
}
}),
);
- return { total, prev, dayPurpose, coworkers, heavy, approver, machines: [...machines, ...remote] };
+ return { total, prev, dayPurpose, coworkers, heavy, approver, machines: [...machines, ...remote], ratings };
}
export function UsagePage({ onOpenTurn, onOpenPersona }: {
@@ -307,6 +313,8 @@ export function UsageBody({ data, period, t, onOpenTurn, onOpenPersona, picked =
{ap && ap.shown && <ApproverStripView ap={ap} t={t} />}
+ {data.ratings && <RatingsCard r={data.ratings} names={names} t={t} onOpenTurn={onOpenTurn} />}
+
{data.total.no_turn.calls > 0 && (
<div className="text-meta text-faint mt-2" data-testid="usage-no-turn">
{t("p3.usage.no_turn", { calls: data.total.no_turn.calls, tokens: data.total.no_turn.total.toLocaleString() })}
@@ -380,6 +388,71 @@ export function coworkerTable(groups: RollupGroup[], names: Record<string, strin
return [...rows.values()].sort((a, b) => b.total - a.total);
}
+/** T3-10 (lean): the period's 👍 and 👎, how often each reason was picked, and the turns rated
+ * 👎 with their note, each opening its trace. No charts or trends: those stay with Langfuse. */
+export function RatingsCard({ r, names, t, onOpenTurn }: {
+ r: RatingsSummary; names: Record<string, string>; t: T; onOpenTurn?: (sessionId: string, turnId: string) => void;
+}) {
+ const reasonLabel = (k: string) => t(`p3.fb.reason.${k}`, { defaultValue: k });
+ const picked = [...REASONS.filter((k) => r.reasons[k]), ...Object.keys(r.reasons).filter((k) => !(REASONS as readonly string[]).includes(k))];
+ return (
+ <div className={CARD + " p3-card mt-3"} data-testid="usage-ratings">
+ <b className="p3-h">{t("p3.ratings.title")}</b>
+ {r.up + r.down === 0 ? (
+ <div className="text-meta text-faint mt-1">{t("p3.ratings.none")}</div>
+ ) : (
+ <>
+ <div className="flex items-center gap-4 mt-1 text-ui" data-testid="usage-ratings-counts">
+ <span>👍 {r.up}</span>
+ <span>👎 {r.down}</span>
+ </div>
+ {picked.length > 0 && (
+ <div className="flex gap-1.5 flex-wrap mt-2 items-center text-meta" data-testid="usage-ratings-reasons">
+ <span className="text-faint">{t("p3.ratings.reasons")}</span>
+ {picked.map((k) => (
+ <span key={k} className="p3-chip">{reasonLabel(k)} {r.reasons[k]}</span>
+ ))}
+ </div>
+ )}
+ {r.down_turns.length > 0 && (
+ <table className="p3-t p3-heavy mt-2" data-testid="usage-ratings-down">
+ <colgroup>
+ <col style={{ width: "26%" }} /><col style={{ width: "14%" }} /><col style={{ width: "22%" }} />
+ <col style={{ width: "24%" }} /><col style={{ width: "7%" }} /><col style={{ width: "7%" }} />
+ </colgroup>
+ <thead><tr><th>{t("p3.usage.session")}</th><th>{t("p3.usage.coworker")}</th><th>{t("p3.ratings.col_reasons")}</th>
+ <th>{t("p3.ratings.col_note")}</th><th className="n" title={t("p3.ratings.auto_title")}>{t("p3.ratings.col_auto")}</th><th></th></tr></thead>
+ <tbody>
+ {r.down_turns.map((d) => {
+ const title = d.session_deleted ? t("p3.usage.deleted_session") : (d.title || d.session_id);
+ const turn = d.seq != null ? t("p3.usage.turn_n", { n: d.seq }) : "";
+ const who = (d.persona_id && names[d.persona_id]) || d.persona_id || "—";
+ const reasons = d.reasons.map(reasonLabel).join(", ");
+ return (
+ <tr key={d.turn_id}>
+ <td><span className="p3-cut" title={turn ? `${title} · ${turn}` : title}>
+ {d.session_deleted ? <span className="text-faint">{title}</span> : title}
+ {turn && <span className="text-faint"> · {turn}</span>}</span></td>
+ <td><span className="p3-cut" title={who}>{who}</span></td>
+ <td><span className="p3-cut" title={reasons}>{reasons || <span className="text-faint">—</span>}</span></td>
+ <td><span className="p3-cut" title={d.note || ""} data-testid="usage-ratings-note">{d.note || <span className="text-faint">—</span>}</span></td>
+ <td className="n">{d.auto_approved || "—"}</td>
+ <td>{!d.session_deleted && onOpenTurn && (
+ <button className="p3-link" data-testid="usage-ratings-open" onClick={() => onOpenTurn(d.session_id, d.turn_id)}>{t("steps.open")}</button>
+ )}</td>
+ </tr>
+ );
+ })}
+ </tbody>
+ </table>
+ )}
+ {r.more > 0 && <div className="text-meta text-faint mt-1">{t("p3.ratings.more", { count: r.more })}</div>}
+ </>
+ )}
+ </div>
+ );
+}
+
/** T3-6: one row per reviewer, named in full, and a Total row; the point is comparing
* reviewers, so they stay apart. Under 10 checks a cell is a count, from 10 a share (M3-R5). */
export function ApproverStripView({ ap, t }: { ap: ApproverStrip; t: T }) {
diff --git a/surfaces/gui/src/locales/en.json b/surfaces/gui/src/locales/en.json
index 58ce44bf..3f40a531 100644
--- a/surfaces/gui/src/locales/en.json
+++ b/surfaces/gui/src/locales/en.json
@@ -3997,6 +3997,17 @@
"incl_shadow_one": "incl. {{count}} shadow check",
"incl_shadow_other": "incl. {{count}} shadow checks"
},
+ "ratings": {
+ "title": "Ratings",
+ "none": "No turns of this period were rated.",
+ "reasons": "Reasons picked:",
+ "col_reasons": "Reasons",
+ "col_note": "Note",
+ "col_auto": "Auto-approved",
+ "auto_title": "Steps of that turn the auto-approver let through",
+ "more_one": "and {{count}} more turn rated 👎",
+ "more_other": "and {{count}} more turns rated 👎"
+ },
"foot": {
"tokens": "{{tokens}} tokens this turn",
"issues_one": "{{count}} issue",
diff --git a/surfaces/gui/src/locales/zh.json b/surfaces/gui/src/locales/zh.json
index a5f7f995..fc2f8e1f 100644
--- a/surfaces/gui/src/locales/zh.json
+++ b/surfaces/gui/src/locales/zh.json
@@ -3936,6 +3936,16 @@
"shares_note": "一行有 10 次及以上检查时显示占比,不足 10 次时显示次数。",
"incl_shadow_other": "含 {{count}} 次影子检查"
},
+ "ratings": {
+ "title": "评价",
+ "none": "这段时间没有被评价的轮次。",
+ "reasons": "选择的原因:",
+ "col_reasons": "原因",
+ "col_note": "备注",
+ "col_auto": "自动批准",
+ "auto_title": "该轮次中由自动审批放行的步骤",
+ "more_other": "另有 {{count}} 个被评为 👎 的轮次"
+ },
"foot": {
"tokens": "本轮 {{tokens}} 令牌",
"issues_other": "{{count}} 个问题",
diff --git a/tests/test_observability_phase3_manual_fixes.py b/tests/test_observability_phase3_manual_fixes.py
index 1fc0f53f..7d81507b 100644
--- a/tests/test_observability_phase3_manual_fixes.py
+++ b/tests/test_observability_phase3_manual_fixes.py
@@ -107,4 +107,45 @@ def test_the_default_coworkers_page_and_history_answer(tmp_path, monkeypatch):
body = doing.json()
assert body["persona_id"] == DEFAULT_PERSONA_ID and body["runs"] == 1, body
assert sum(r["turns"] for r in body["rows"]) == 1
+ # T3-10's route answers with the card's fields
+ now = int(time.time() * 1000)
+ got = client.get("/v1/observability/ratings", params={"from": now - 86_400_000, "to": now + 60_000}).json()
+ assert (got["up"], got["down"], got["down_turns"], got["more"]) == (0, 0, [], 0)
manager.observability.close()
+
+
+# -- T3-10 ----------------------------------------------------------------------------------------
+def test_the_ratings_card_counts_reasons_and_lists_the_down_turns(tmp_path):
+ from coworker.observability import signals
+
+ store = RecordingStore(tmp_path / "recordings.db", background=False)
+ main = {"purpose": "main", "model": "model-x",
+ "usage": {"reported": True, "input_fresh": 10, "cache_read": 0, "cache_write": 0, "output": 1}}
+ for i, (sid, persona) in enumerate([("s1", "cowork"), ("s1", "cowork"), ("s2", "sec"), ("s2", "sec"), ("s3", "cowork")]):
+ t = _turn(f"T{i}", sid, i + 1, RECENT + i * 1000)
+ t["stamp"]["persona_id"] = persona
+ store.write_turn(t)
+ store.write_step(_step(f"T{i}", sid, 1, RECENT + i * 1000, "model", main))
+ store.write_issues(f"T{i}", {"signals_version": signals.version(), "warn": 0, "info": 0, "issues": [], "fine": []},
+ seen=True)
+ store.flush()
+ fb = lambda tid, sid, rating, reasons=(), note=None, auto=(): store.put_feedback( # noqa: E731
+ {"turn_id": tid, "session_id": sid, "rating": rating, "reasons": list(reasons), "note": note,
+ "auto_approved": list(auto)})
+ fb("T0", "s1", "up")
+ fb("T1", "s1", "down", ["wrong_answer", "too_slow"], "it guessed")
+ fb("T2", "s2", "down", ["did_unasked"], None, ["T2-s4", "T2-s5"])
+ fb("T3", "s2", "up")
+ # T4 is not rated
+ got = usage_api.ratings(store, start=RECENT - 1, end=RECENT + 60_000, title_of=lambda sid: f"title {sid}")
+ assert (got["up"], got["down"]) == (2, 2)
+ assert got["reasons"] == {"wrong_answer": 1, "too_slow": 1, "did_unasked": 1}
+ assert [t["turn_id"] for t in got["down_turns"]] == ["T2", "T1"] # newest first
+ t2, t1 = got["down_turns"]
+ assert t1["note"] == "it guessed" and t1["title"] == "title s1" and t1["reasons"] == ["wrong_answer", "too_slow"]
+ assert t2["auto_approved"] == 2 and t2["note"] is None and t2["seq"] == 3
+ assert got["more"] == 0
+ # one coworker narrows it, as the rest of the page
+ one = usage_api.ratings(store, start=RECENT - 1, end=RECENT + 60_000, persona_id="sec")
+ assert (one["up"], one["down"]) == (1, 1) and one["down_turns"][0]["turn_id"] == "T2"
+ store.close()