Python
feat(observability): read endpoints, screen numbers and live step messages (OPE-244)
Devika Verma4dc6dbad44cc
diff --git a/coworker/observability/calculator.py b/coworker/observability/calculator.pynew file mode 100644index 00000000..433e40c3--- /dev/null+++ b/coworker/observability/calculator.py@@ -0,0 +1,154 @@+"""Every number the Phase 1 screens show, from a recording (technical design §6.3, §7.1).++Works on the recording JSON (`{"turns": [...], "steps": [...]}`, recording format 1.0), as+the store's `load_recording()` and the recorder's `as_dict()` give it. The server computes+the numbers once, so the token pill, the popover and later pages always agree. The shared+golden test's `expected-numbers.json` checks these functions.+"""++from __future__ import annotations++from typing import Any, Iterable, Optional++BUCKET_ORDER = ["waiting", "tools", "model", "auto_approver", "summary"]+++# -- tokens (§6.3) -------------------------------------------------------------------------------+def input_total(u: dict[str, Any]) -> int:+ """Fresh input plus cache read and cache write; a part not reported adds nothing."""+ return (u.get("input_fresh") or 0) + (u.get("cache_read") or 0) + (u.get("cache_write") or 0)+++def step_total(step: dict[str, Any]) -> Optional[int]:+ u = (step.get("attrs") or {}).get("usage") or {}+ if not u.get("reported"):+ return None+ return input_total(u) + (u.get("output") or 0)+++def usage(recording: dict[str, Any], session_id: Optional[str] = None,+ turn_id: Optional[str] = None) -> dict[str, Any]:+ """The session's token numbers: the pill, the "≥" marker, failed and over-limit calls,+ by purpose, by model, cache share. With `turn_id`, the same over one turn (its+ `after_turn` steps included)."""+ steps = recording.get("steps") or []+ models = [s for s in steps if s.get("kind") == "model"+ and (session_id is None or s.get("session_id") == session_id)+ and (turn_id is None or s.get("turn_id") == turn_id)+ and (s.get("attrs") or {}).get("purpose") != "proxy"] # proxy calls are in no pill+ total = unreported = failed = overflow = 0+ by_purpose: dict[str, int] = {}+ by_model: dict[str, int] = {}+ cache_read = inp = 0+ unreported_ids: list[str] = []+ for s in models:+ attrs = s.get("attrs") or {}+ err = attrs.get("error")+ if err and s.get("shadow"):+ continue # a failed shadow check is never shown as a failure (M-17)+ if err:+ if err.get("type") == "context_length":+ overflow += 1 # its own line, not a failed call (F-5)+ else:+ failed += 1 # V-02+ continue+ t = step_total(s)+ if t is None:+ unreported += 1 # only completed model steps count toward "≥" (V-01)+ unreported_ids.append(s.get("step_id"))+ continue+ total += t+ p = attrs.get("purpose", "main")+ by_purpose[p] = by_purpose.get(p, 0) + t+ key = attrs.get("model", "?") + ("" if p == "main" else f" ({p})")+ by_model[key] = by_model.get(key, 0) + t+ u = attrs["usage"]+ if u.get("cache_read") is not None: # a call with no cache count stays out of the share+ cache_read += u["cache_read"]+ inp += input_total(u)+ reported_parents = {s.get("parent_step_id") for s in models+ if ((s.get("attrs") or {}).get("usage") or {}).get("reported")}+ not_reported_checks = sum(+ 1 for s in steps+ if s.get("kind") == "auto_approver"+ and ((s.get("attrs") or {}).get("reviewer") or {}).get("kind") not in (None, "session_model")+ and s.get("step_id") not in reported_parents+ and (session_id is None or s.get("session_id") == session_id)+ and (turn_id is None or s.get("turn_id") == turn_id))+ return {"pill": total, "at_least": unreported > 0, "unreported_calls": unreported,+ "failed_calls": failed, "overflow_calls": overflow, "by_purpose": by_purpose,+ "by_model": by_model, "cache_share": round(cache_read / inp, 3) if inp else None,+ "auto_approver_checks_not_reported": not_reported_checks,+ "unreported_step_ids": unreported_ids}+++# -- time buckets (§7.1) ---------------------------------------------------------------------------+def _merge(intervals: Iterable[tuple[int, int]]) -> list[tuple[int, int]]:+ out: list[tuple[int, int]] = []+ for a, b in sorted(i for i in intervals if i[1] > i[0]):+ if out and a <= out[-1][1]:+ out[-1] = (out[-1][0], max(out[-1][1], b))+ else:+ out.append((a, b))+ return out+++def _subtract(a: list[tuple[int, int]], b: list[tuple[int, int]]) -> list[tuple[int, int]]:+ out = []+ for s, e in a:+ cur = s+ for bs, be in b:+ if be <= cur or bs >= e:+ continue+ if bs > cur:+ out.append((cur, bs))+ cur = max(cur, be)+ if cur < e:+ out.append((cur, e))+ return out+++def _length(iv: list[tuple[int, int]]) -> int:+ return sum(b - a for a, b in iv)+++def _bucket_of(step: dict[str, Any]) -> Optional[str]:+ if step.get("shadow") or step.get("after_turn") or step.get("status") == "unknown":+ return None+ kind = step.get("kind")+ if kind == "wait":+ return "waiting"+ if kind == "tool":+ return "tools"+ if kind == "model" and (step.get("attrs") or {}).get("purpose") in ("main", "helper"):+ return "model"+ if kind == "auto_approver":+ return "auto_approver"+ if kind == "summary":+ return "summary"+ return None+++def buckets(recording: dict[str, Any], turn: dict[str, Any], now: Optional[int] = None) -> dict[str, Any]:+ """Where the turn's time went: each bucket is the union of its steps' time, overlaps go+ to the first bucket in the order waiting, tools, model, auto-approver, summary; Other is+ the rest, shown when it is 1% of the total or more (M-R5)."""+ start = turn["started_at"]+ end_of_turn = (start + turn["duration_ms"] if turn.get("duration_ms") is not None+ else (now if now is not None else start))+ steps = [s for s in recording.get("steps") or [] if s.get("turn_id") == turn["turn_id"]]+ # A step still open is counted up to the turn's end (or now, for a running turn; TD-12).+ spans = [(s, s["started_at"], s["started_at"] + s["duration_ms"] if s.get("duration_ms") is not None+ else end_of_turn) for s in steps]+ raw = {b: _merge((a, e) for s, a, e in spans if _bucket_of(s) == b) for b in BUCKET_ORDER}+ taken: list[tuple[int, int]] = []+ out: dict[str, Any] = {}+ for b in BUCKET_ORDER:+ own = _subtract(raw[b], taken)+ out[b] = _length(own)+ taken = _merge(taken + own)+ total = end_of_turn - start+ out["total"] = total+ out["other"] = total - _length(taken)+ out["other_shown"] = bool(total) and out["other"] > 0 and out["other"] >= 0.01 * total+ return outdiff --git a/coworker/observability/read_api.py b/coworker/observability/read_api.pynew file mode 100644index 00000000..9d4486ba--- /dev/null+++ b/coworker/observability/read_api.py@@ -0,0 +1,99 @@+"""What the app's screens read (observability §12): turns, a turn's steps, one step in full,+and the session's token numbers. Served by the engine's local web server, and through the+existing machine proxy for remote and cloud machines.++- `turns`: every turn of the session, with its stamp, time buckets (§7.1) and usage (§6.3).+- `turn_steps`: a light list of one turn's steps, in canonical order, without content.+- `step`: one step in full: content, the exchange rebuilt from its pieces, the transcript+ message it points at (cut at 4 KB), the audit rows with its call id, and its child steps.+- `usage`: the session's token numbers for the pill and its popover, and context fill.+"""++from __future__ import annotations++import json+import time+from typing import Any, Callable, Optional++from . import calculator+from .capture import rebuild_request++MESSAGE_CUT = 4096+++def _light(step: dict[str, Any]) -> dict[str, Any]:+ attrs = {k: v for k, v in (step.get("attrs") or {}).items() if k != "provider_meta"}+ return {**{k: v for k, v in step.items() if k not in ("attrs", "content")}, "attrs": attrs}+++def turns(recording: dict[str, Any]) -> list[dict[str, Any]]:+ now = int(time.time() * 1000)+ out = []+ for t in recording.get("turns") or []:+ out.append({+ **t,+ "buckets": calculator.buckets(recording, t, now=now),+ "usage": calculator.usage(recording, turn_id=t["turn_id"]),+ })+ return out+++def turn_steps(recording: dict[str, Any], turn_id: str) -> list[dict[str, Any]]:+ return [_light(s) for s in recording.get("steps") or [] if s.get("turn_id") == turn_id]+++def step(recording: dict[str, Any], step_id: str, *, get_piece: Callable[[str], Optional[bytes]],+ messages: list[dict[str, Any]], audit_rows: Callable[[str], list[dict[str, Any]]]) -> Optional[dict[str, Any]]:+ steps = recording.get("steps") or []+ found = next((s for s in steps if s.get("step_id") == step_id), None)+ if found is None:+ return None+ out = dict(found)+ exchange = (found.get("content") or {}).get("exchange")+ if exchange:+ out["exchange_rebuilt"] = [_rebuilt(x, get_piece) for x in exchange]+ mid = found.get("message_id")+ if mid:+ message = next((m for m in messages if m.get("mid") == mid), None)+ if message is not None:+ out["message"] = _cut(message)+ out["audit"] = audit_rows(found["call_id"]) if found.get("call_id") else []+ out["children"] = [_light(s) for s in steps if s.get("parent_step_id") == step_id]+ return out+++def _rebuilt(entry: dict[str, Any], get_piece: Callable[[str], Optional[bytes]]) -> dict[str, Any]:+ body = get_piece((entry.get("response") or {}).get("body_blob") or "") or b""+ try:+ request = rebuild_request(entry, get_piece)+ except Exception: # noqa: BLE001 - a piece already swept: show what is left+ request = None+ if isinstance(request, bytes):+ request = request.decode("utf-8", "replace")+ return {"auxiliary": bool(entry.get("auxiliary")), "request": request,+ "response": body.decode("utf-8", "replace")}+++def _cut(message: dict[str, Any]) -> dict[str, Any]:+ """The transcript message, its content cut at 4 KB ("Open full result" reads the rest)."""+ content = message.get("content")+ text = content if isinstance(content, str) else json.dumps(content, ensure_ascii=False, default=str)+ cut = len(text) > MESSAGE_CUT+ return {"role": message.get("role"), "mid": message.get("mid"), "tool_call_id": message.get("tool_call_id"),+ "content": text[:MESSAGE_CUT], "cut": cut}+++def usage(recording: dict[str, Any], session_id: str) -> dict[str, Any]:+ out = calculator.usage(recording, session_id)+ out["recorded"] = bool(recording.get("turns"))+ # Context fill: the last main call's total input against its model's context window (M-R18).+ last = next((s for s in reversed(recording.get("steps") or [])+ if s.get("kind") == "model" and (s.get("attrs") or {}).get("purpose") == "main"+ and ((s.get("attrs") or {}).get("usage") or {}).get("reported")), None)+ if last is not None:+ a = last["attrs"]+ out["context"] = {"tokens": calculator.input_total(a["usage"]), "window": a.get("context_window"),+ "model": a.get("model")}+ else:+ out["context"] = None+ return outdiff --git a/coworker/server/app.py b/coworker/server/app.pyindex a0c20f98..834fcb59 100644--- a/coworker/server/app.py+++ b/coworker/server/app.py@@ -826,6 +826,23 @@ def create_app(manager: SessionManager) -> FastAPI:def session_messages(session_id: str) -> dict[str, Any]:return {"messages": manager.session_messages(session_id)}+ # Observability (§12): the recorded turns and steps of a session.+ @app.get("/v1/sessions/{session_id}/turns")+ def session_turns(session_id: str) -> dict[str, Any]:+ return manager.recording_turns(session_id)++ @app.get("/v1/sessions/{session_id}/turns/{turn_id}/steps")+ def session_turn_steps(session_id: str, turn_id: str) -> dict[str, Any]:+ return manager.recording_turn_steps(session_id, turn_id)++ @app.get("/v1/sessions/{session_id}/steps/{step_id}")+ def session_step(session_id: str, step_id: str) -> dict[str, Any]:+ return manager.recording_step(session_id, step_id)++ @app.get("/v1/sessions/{session_id}/usage")+ def session_usage(session_id: str) -> dict[str, Any]:+ return manager.recording_usage(session_id)+@app.patch("/v1/sessions/{session_id}")def session_patch(session_id: str, body: dict) -> dict[str, Any]:body = body or {}diff --git a/coworker/server/manager.py b/coworker/server/manager.pyindex 6a271bc1..e1fddf3d 100644--- a/coworker/server/manager.py+++ b/coworker/server/manager.py@@ -44,6 +44,7 @@ from ..unrouted import UnroutedStorefrom ..unattended import UnattendedRegistryfrom ..audit import AuditStorefrom ..observability import Recorder, RecordingStore+from ..observability import read_api as recording_apifrom ..observability.provider_view import RecordingProvider, tag as tag_providerfrom ..config import load_config, workspace_allowed_commandsfrom ..conversations import ConversationStore, title_from@@ -387,6 +388,11 @@ class SessionManager:# coworker.db so they can be pruned and reset on their own.self.recordings_store = RecordingStore.beside(base)self.recorder = Recorder(store=self.recordings_store)+ # Live: the same recording events go to the session's sockets, batched (§12).+ self._live_steps: dict[str, list[dict[str, Any]]] = {}+ self._live_steps_lock = threading.Lock()+ self._live_loop: Optional[asyncio.AbstractEventLoop] = None+ self.recorder.add_listener(self._forward_recording_event)self.session_store = ConversationStore(base)self.session_store.canonicalize_workspaces() # collapse /tmp vs /private/tmp etc.if self.default_workspace:@@ -6308,6 +6314,60 @@ class SessionManager:def has_session_clients(self, session_id: str) -> bool:return bool(self._session_clients.get(session_id))+ # -- observability: reading recordings (§12) -------------------------------------------+ LIVE_STEP_BATCH_S = 0.1++ def recording_turns(self, session_id: str) -> dict[str, Any]:+ return {"turns": recording_api.turns(self.recordings_store.load_recording(session_id))}++ def recording_turn_steps(self, session_id: str, turn_id: str) -> dict[str, Any]:+ rec = self.recordings_store.load_recording(session_id)+ return {"steps": recording_api.turn_steps(rec, turn_id)}++ def recording_step(self, session_id: str, step_id: str) -> dict[str, Any]:+ rec = self.recordings_store.load_recording(session_id)+ found = recording_api.step(+ rec, step_id, get_piece=self.recordings_store.get_piece,+ messages=self.session_messages(session_id),+ audit_rows=lambda call_id: self.audit_store.list(session_id=session_id, call_id=call_id),+ )+ return {"step": found} if found is not None else {"step": None, "error": "no such step"}++ def recording_usage(self, session_id: str) -> dict[str, Any]:+ return recording_api.usage(self.recordings_store.load_recording(session_id), session_id)++ def _forward_recording_event(self, event: dict[str, Any]) -> None:+ """Queue one recording event for the session's sockets; a batch goes out every+ 100 ms as `{"type": "step", "events": [...]}`, so a burst of steps does not flood+ the connection (P-5). Older app versions ignore the unknown type."""+ session_id = event.get("session_id")+ loop = self._loop+ if loop is None:+ try:+ loop = asyncio.get_running_loop() # a turn's start runs on the server's loop+ except RuntimeError:+ loop = self._live_loop+ if not session_id or loop is None:+ return+ self._live_loop = loop+ with self._live_steps_lock:+ batch = self._live_steps.setdefault(session_id, [])+ batch.append(event)+ first = len(batch) == 1+ if first:+ try:+ loop.call_soon_threadsafe(+ loop.call_later, self.LIVE_STEP_BATCH_S, self._send_live_steps, session_id+ )+ except RuntimeError:+ pass # the loop is closed: the server is stopping++ def _send_live_steps(self, session_id: str) -> None:+ with self._live_steps_lock:+ events = self._live_steps.pop(session_id, [])+ if events and self.has_session_clients(session_id):+ asyncio.ensure_future(self.broadcast_session(session_id, {"type": "step", "events": events}))+def _inbox_item_changed(self, item: Any) -> None:"""A card was added or moved to the Inbox: tell the waiting engine where it is shown(observability §5.4, a wait step's `where`)."""diff --git a/tests/corpora/observability/README.md b/tests/corpora/observability/README.mdindex 975f098b..ddf679fa 100644--- a/tests/corpora/observability/README.md+++ b/tests/corpora/observability/README.md@@ -9,5 +9,6 @@ Do not edit these files here. A change to the format needs a new version in `ocw(its `tracing-spec.md` §12), and then a fresh copy of the folder here.`tests/test_observability_golden.py` replays every scenario's `events.jsonl` into the-recorder and compares the result with `expected-recording.json`. `expected-numbers.json`-(the screen numbers) is checked once the read endpoints are built.+recorder and compares the result with `expected-recording.json`, and computes the screen+numbers (`coworker/observability/calculator.py`) and compares them with+`expected-numbers.json`.diff --git a/tests/test_observability_golden.py b/tests/test_observability_golden.pyindex 4a72dff9..d04bc740 100644--- a/tests/test_observability_golden.py+++ b/tests/test_observability_golden.py@@ -82,3 +82,25 @@ def test_the_store_gives_back_the_same_recording(scenario, tmp_path):assert store.load_recording(session) == wantfinally:store.close()++[email protected]("scenario", SCENARIOS)+def test_the_screen_numbers(scenario):+ """`expected-numbers.json`: every token number and time bucket the screens show."""+ from coworker.observability import calculator++ expected = json.loads((GOLDEN / scenario / "expected-numbers.json").read_text("utf-8"))+ rec = _replay(_events(scenario)).as_dict()+ sessions = sorted({t["session_id"] for t in rec["turns"] if t["session_id"]}+ | {s["session_id"] for s in rec["steps"] if s["session_id"]})++ def usage(sid=None):+ u = calculator.usage(rec, sid)+ u.pop("unreported_step_ids") # an addition for the "≥" links (M-R19), not in the golden+ return u++ got = {"usage": usage()}+ if len(sessions) > 1:+ got["usage_by_session"] = {sid: usage(sid) for sid in sessions}+ got["buckets_by_turn"] = {t["turn_id"]: calculator.buckets(rec, t) for t in rec["turns"]}+ assert got == expecteddiff --git a/tests/test_observability_read_api.py b/tests/test_observability_read_api.pynew file mode 100644index 00000000..4b41d4e3--- /dev/null+++ b/tests/test_observability_read_api.py@@ -0,0 +1,89 @@+"""Observability §12: what the app's screens read, through the real server.++A turn runs over the session's websocket; the recorded turns, steps, one step in full and+the session's token numbers are then read back over HTTP, and the same recording events+arrive live on the socket as `{"type": "step", "events": [...]}`.+"""++from __future__ import annotations++from fastapi.testclient import TestClient++from coworker.providers import AssistantTurn, ModelCapabilities, ProviderClient, ToolCall+from coworker.providers.base import TokenUsage+from coworker.server import SessionManager, create_app+++class _Scripted(ProviderClient):+ def __init__(self, turns):+ self._turns = list(turns)++ def complete(self, *, model, messages, tools=None, **settings):+ if str(messages[0].get("content") or "").startswith("You title"):+ return AssistantTurn(text="Notes", usage=TokenUsage(input=40, output=2)) # the session title+ return self._turns.pop(0)++ def capabilities(self, model):+ return ModelCapabilities()+++def test_a_recorded_turn_reads_back_through_the_endpoints(tmp_path):+ (tmp_path / "notes.txt").write_text("Friday", encoding="utf-8")+ provider = _Scripted([+ AssistantTurn(tool_calls=[ToolCall(id="c1", name="read_file", arguments={"path": "notes.txt"})],+ finish_reason="tool_calls", usage=TokenUsage(input=100, output=10, cache_read=900)),+ AssistantTurn(text="It says Friday.", finish_reason="stop", usage=TokenUsage(input=20, output=5, cache_read=1000)),+ ])+ manager = SessionManager(workspace=tmp_path, provider=provider)+ client = TestClient(create_app(manager))+ live: list[dict] = []+ with client.websocket_connect("/ws/session/r1") as ws:+ assert ws.receive_json()["type"] == "ready"+ ws.send_json({"type": "user_message", "text": "what does notes.txt say?"})+ while True:+ msg = ws.receive_json()+ if msg["type"] == "step":+ live.extend(msg["events"])+ if msg["type"] == "turn_done":+ break+ # The last batch may arrive just after the turn's end.+ import time+ deadline = time.time() + 2+ while time.time() < deadline and not any(e["type"] == "turn_ended" for e in live):+ msg = ws.receive_json()+ if msg["type"] == "step":+ live.extend(msg["events"])++ [turn] = client.get("/v1/sessions/r1/turns").json()["turns"]+ assert (turn["kind"], turn["trigger"], turn["status"]) == ("run", "user", "completed")+ # The two main calls, and the session title (40 + 2): every purpose with a session counts.+ assert turn["usage"]["pill"] == 100 + 900 + 10 + 20 + 1000 + 5 + 40 + 2+ assert turn["usage"]["by_purpose"] == {"main": 2035, "title": 42}+ assert turn["buckets"]["total"] == turn["duration_ms"]++ steps = client.get(f"/v1/sessions/r1/turns/{turn['turn_id']}/steps").json()["steps"]+ mains = [s for s in steps if s["kind"] != "context" and s["attrs"].get("purpose") != "title"]+ assert [s["kind"] for s in mains] == ["model", "tool", "model"]+ assert all("content" not in s for s in steps)++ tool = next(s for s in steps if s["kind"] == "tool")+ detail = client.get(f"/v1/sessions/r1/steps/{tool['step_id']}").json()["step"]+ assert detail["message"]["role"] == "tool" and "Friday" in detail["message"]["content"]+ assert detail["message"]["mid"] == tool["message_id"]+ assert any(row.get("call_id") == "c1" for row in detail["audit"])++ usage = client.get("/v1/sessions/r1/usage").json()+ assert usage["pill"] == turn["usage"]["pill"] and usage["at_least"] is False+ assert usage["recorded"] is True+ assert usage["cache_share"] == round(1900 / (100 + 900 + 20 + 1000 + 40), 3)+ assert usage["context"]["tokens"] == 1020++ types = [e["type"] for e in live]+ assert types[0] == "turn_started" and "turn_ended" in types and "step_ended" in types+++def test_a_session_recorded_before_phase_1_says_so(tmp_path):+ manager = SessionManager(workspace=tmp_path, provider=_Scripted([]))+ client = TestClient(create_app(manager))+ assert client.get("/v1/sessions/old/turns").json() == {"turns": []}+ assert client.get("/v1/sessions/old/usage").json()["recorded"] is False