Skip to content
Python

feat(observability): read endpoints, screen numbers and live step messages (OPE-244)

Devika Verma4dc6dbad44cc
diff --git a/coworker/observability/calculator.py b/coworker/observability/calculator.py
new file mode 100644
index 00000000..433e40c3
--- /dev/null
+++ b/coworker/observability/calculator.py
@@ -0,0 +1,154 @@
+"""Every number the Phase 1 screens show, from a recording (technical design §6.3, §7.1).
+
+Works on the recording JSON (`{"turns": [...], "steps": [...]}`, recording format 1.0), as
+the store's `load_recording()` and the recorder's `as_dict()` give it. The server computes
+the numbers once, so the token pill, the popover and later pages always agree. The shared
+golden test's `expected-numbers.json` checks these functions.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Iterable, Optional
+
+BUCKET_ORDER = ["waiting", "tools", "model", "auto_approver", "summary"]
+
+
+# -- tokens (§6.3) -------------------------------------------------------------------------------
+def input_total(u: dict[str, Any]) -> int:
+ """Fresh input plus cache read and cache write; a part not reported adds nothing."""
+ return (u.get("input_fresh") or 0) + (u.get("cache_read") or 0) + (u.get("cache_write") or 0)
+
+
+def step_total(step: dict[str, Any]) -> Optional[int]:
+ u = (step.get("attrs") or {}).get("usage") or {}
+ if not u.get("reported"):
+ return None
+ return input_total(u) + (u.get("output") or 0)
+
+
+def usage(recording: dict[str, Any], session_id: Optional[str] = None,
+ turn_id: Optional[str] = None) -> dict[str, Any]:
+ """The session's token numbers: the pill, the "≥" marker, failed and over-limit calls,
+ by purpose, by model, cache share. With `turn_id`, the same over one turn (its
+ `after_turn` steps included)."""
+ steps = recording.get("steps") or []
+ models = [s for s in steps if s.get("kind") == "model"
+ and (session_id is None or s.get("session_id") == session_id)
+ and (turn_id is None or s.get("turn_id") == turn_id)
+ and (s.get("attrs") or {}).get("purpose") != "proxy"] # proxy calls are in no pill
+ total = unreported = failed = overflow = 0
+ by_purpose: dict[str, int] = {}
+ by_model: dict[str, int] = {}
+ cache_read = inp = 0
+ unreported_ids: list[str] = []
+ for s in models:
+ attrs = s.get("attrs") or {}
+ err = attrs.get("error")
+ if err and s.get("shadow"):
+ continue # a failed shadow check is never shown as a failure (M-17)
+ if err:
+ if err.get("type") == "context_length":
+ overflow += 1 # its own line, not a failed call (F-5)
+ else:
+ failed += 1 # V-02
+ continue
+ t = step_total(s)
+ if t is None:
+ unreported += 1 # only completed model steps count toward "≥" (V-01)
+ unreported_ids.append(s.get("step_id"))
+ continue
+ total += t
+ p = attrs.get("purpose", "main")
+ by_purpose[p] = by_purpose.get(p, 0) + t
+ key = attrs.get("model", "?") + ("" if p == "main" else f" ({p})")
+ by_model[key] = by_model.get(key, 0) + t
+ u = attrs["usage"]
+ if u.get("cache_read") is not None: # a call with no cache count stays out of the share
+ cache_read += u["cache_read"]
+ inp += input_total(u)
+ reported_parents = {s.get("parent_step_id") for s in models
+ if ((s.get("attrs") or {}).get("usage") or {}).get("reported")}
+ not_reported_checks = sum(
+ 1 for s in steps
+ if s.get("kind") == "auto_approver"
+ and ((s.get("attrs") or {}).get("reviewer") or {}).get("kind") not in (None, "session_model")
+ and s.get("step_id") not in reported_parents
+ and (session_id is None or s.get("session_id") == session_id)
+ and (turn_id is None or s.get("turn_id") == turn_id))
+ return {"pill": total, "at_least": unreported > 0, "unreported_calls": unreported,
+ "failed_calls": failed, "overflow_calls": overflow, "by_purpose": by_purpose,
+ "by_model": by_model, "cache_share": round(cache_read / inp, 3) if inp else None,
+ "auto_approver_checks_not_reported": not_reported_checks,
+ "unreported_step_ids": unreported_ids}
+
+
+# -- time buckets (§7.1) ---------------------------------------------------------------------------
+def _merge(intervals: Iterable[tuple[int, int]]) -> list[tuple[int, int]]:
+ out: list[tuple[int, int]] = []
+ for a, b in sorted(i for i in intervals if i[1] > i[0]):
+ if out and a <= out[-1][1]:
+ out[-1] = (out[-1][0], max(out[-1][1], b))
+ else:
+ out.append((a, b))
+ return out
+
+
+def _subtract(a: list[tuple[int, int]], b: list[tuple[int, int]]) -> list[tuple[int, int]]:
+ out = []
+ for s, e in a:
+ cur = s
+ for bs, be in b:
+ if be <= cur or bs >= e:
+ continue
+ if bs > cur:
+ out.append((cur, bs))
+ cur = max(cur, be)
+ if cur < e:
+ out.append((cur, e))
+ return out
+
+
+def _length(iv: list[tuple[int, int]]) -> int:
+ return sum(b - a for a, b in iv)
+
+
+def _bucket_of(step: dict[str, Any]) -> Optional[str]:
+ if step.get("shadow") or step.get("after_turn") or step.get("status") == "unknown":
+ return None
+ kind = step.get("kind")
+ if kind == "wait":
+ return "waiting"
+ if kind == "tool":
+ return "tools"
+ if kind == "model" and (step.get("attrs") or {}).get("purpose") in ("main", "helper"):
+ return "model"
+ if kind == "auto_approver":
+ return "auto_approver"
+ if kind == "summary":
+ return "summary"
+ return None
+
+
+def buckets(recording: dict[str, Any], turn: dict[str, Any], now: Optional[int] = None) -> dict[str, Any]:
+ """Where the turn's time went: each bucket is the union of its steps' time, overlaps go
+ to the first bucket in the order waiting, tools, model, auto-approver, summary; Other is
+ the rest, shown when it is 1% of the total or more (M-R5)."""
+ start = turn["started_at"]
+ end_of_turn = (start + turn["duration_ms"] if turn.get("duration_ms") is not None
+ else (now if now is not None else start))
+ steps = [s for s in recording.get("steps") or [] if s.get("turn_id") == turn["turn_id"]]
+ # A step still open is counted up to the turn's end (or now, for a running turn; TD-12).
+ spans = [(s, s["started_at"], s["started_at"] + s["duration_ms"] if s.get("duration_ms") is not None
+ else end_of_turn) for s in steps]
+ raw = {b: _merge((a, e) for s, a, e in spans if _bucket_of(s) == b) for b in BUCKET_ORDER}
+ taken: list[tuple[int, int]] = []
+ out: dict[str, Any] = {}
+ for b in BUCKET_ORDER:
+ own = _subtract(raw[b], taken)
+ out[b] = _length(own)
+ taken = _merge(taken + own)
+ total = end_of_turn - start
+ out["total"] = total
+ out["other"] = total - _length(taken)
+ out["other_shown"] = bool(total) and out["other"] > 0 and out["other"] >= 0.01 * total
+ return out
diff --git a/coworker/observability/read_api.py b/coworker/observability/read_api.py
new file mode 100644
index 00000000..9d4486ba
--- /dev/null
+++ b/coworker/observability/read_api.py
@@ -0,0 +1,99 @@
+"""What the app's screens read (observability §12): turns, a turn's steps, one step in full,
+and the session's token numbers. Served by the engine's local web server, and through the
+existing machine proxy for remote and cloud machines.
+
+- `turns`: every turn of the session, with its stamp, time buckets (§7.1) and usage (§6.3).
+- `turn_steps`: a light list of one turn's steps, in canonical order, without content.
+- `step`: one step in full: content, the exchange rebuilt from its pieces, the transcript
+ message it points at (cut at 4 KB), the audit rows with its call id, and its child steps.
+- `usage`: the session's token numbers for the pill and its popover, and context fill.
+"""
+
+from __future__ import annotations
+
+import json
+import time
+from typing import Any, Callable, Optional
+
+from . import calculator
+from .capture import rebuild_request
+
+MESSAGE_CUT = 4096
+
+
+def _light(step: dict[str, Any]) -> dict[str, Any]:
+ attrs = {k: v for k, v in (step.get("attrs") or {}).items() if k != "provider_meta"}
+ return {**{k: v for k, v in step.items() if k not in ("attrs", "content")}, "attrs": attrs}
+
+
+def turns(recording: dict[str, Any]) -> list[dict[str, Any]]:
+ now = int(time.time() * 1000)
+ out = []
+ for t in recording.get("turns") or []:
+ out.append({
+ **t,
+ "buckets": calculator.buckets(recording, t, now=now),
+ "usage": calculator.usage(recording, turn_id=t["turn_id"]),
+ })
+ return out
+
+
+def turn_steps(recording: dict[str, Any], turn_id: str) -> list[dict[str, Any]]:
+ return [_light(s) for s in recording.get("steps") or [] if s.get("turn_id") == turn_id]
+
+
+def step(recording: dict[str, Any], step_id: str, *, get_piece: Callable[[str], Optional[bytes]],
+ messages: list[dict[str, Any]], audit_rows: Callable[[str], list[dict[str, Any]]]) -> Optional[dict[str, Any]]:
+ steps = recording.get("steps") or []
+ found = next((s for s in steps if s.get("step_id") == step_id), None)
+ if found is None:
+ return None
+ out = dict(found)
+ exchange = (found.get("content") or {}).get("exchange")
+ if exchange:
+ out["exchange_rebuilt"] = [_rebuilt(x, get_piece) for x in exchange]
+ mid = found.get("message_id")
+ if mid:
+ message = next((m for m in messages if m.get("mid") == mid), None)
+ if message is not None:
+ out["message"] = _cut(message)
+ out["audit"] = audit_rows(found["call_id"]) if found.get("call_id") else []
+ out["children"] = [_light(s) for s in steps if s.get("parent_step_id") == step_id]
+ return out
+
+
+def _rebuilt(entry: dict[str, Any], get_piece: Callable[[str], Optional[bytes]]) -> dict[str, Any]:
+ body = get_piece((entry.get("response") or {}).get("body_blob") or "") or b""
+ try:
+ request = rebuild_request(entry, get_piece)
+ except Exception: # noqa: BLE001 - a piece already swept: show what is left
+ request = None
+ if isinstance(request, bytes):
+ request = request.decode("utf-8", "replace")
+ return {"auxiliary": bool(entry.get("auxiliary")), "request": request,
+ "response": body.decode("utf-8", "replace")}
+
+
+def _cut(message: dict[str, Any]) -> dict[str, Any]:
+ """The transcript message, its content cut at 4 KB ("Open full result" reads the rest)."""
+ content = message.get("content")
+ text = content if isinstance(content, str) else json.dumps(content, ensure_ascii=False, default=str)
+ cut = len(text) > MESSAGE_CUT
+ return {"role": message.get("role"), "mid": message.get("mid"), "tool_call_id": message.get("tool_call_id"),
+ "content": text[:MESSAGE_CUT], "cut": cut}
+
+
+def usage(recording: dict[str, Any], session_id: str) -> dict[str, Any]:
+ out = calculator.usage(recording, session_id)
+ out["recorded"] = bool(recording.get("turns"))
+ # Context fill: the last main call's total input against its model's context window (M-R18).
+ last = next((s for s in reversed(recording.get("steps") or [])
+ if s.get("kind") == "model" and (s.get("attrs") or {}).get("purpose") == "main"
+ and ((s.get("attrs") or {}).get("usage") or {}).get("reported")), None)
+ if last is not None:
+ a = last["attrs"]
+ out["context"] = {"tokens": calculator.input_total(a["usage"]), "window": a.get("context_window"),
+ "model": a.get("model")}
+ else:
+ out["context"] = None
+ return out
diff --git a/coworker/server/app.py b/coworker/server/app.py
index a0c20f98..834fcb59 100644
--- a/coworker/server/app.py
+++ b/coworker/server/app.py
@@ -826,6 +826,23 @@ def create_app(manager: SessionManager) -> FastAPI:
def session_messages(session_id: str) -> dict[str, Any]:
return {"messages": manager.session_messages(session_id)}
+ # Observability (§12): the recorded turns and steps of a session.
+ @app.get("/v1/sessions/{session_id}/turns")
+ def session_turns(session_id: str) -> dict[str, Any]:
+ return manager.recording_turns(session_id)
+
+ @app.get("/v1/sessions/{session_id}/turns/{turn_id}/steps")
+ def session_turn_steps(session_id: str, turn_id: str) -> dict[str, Any]:
+ return manager.recording_turn_steps(session_id, turn_id)
+
+ @app.get("/v1/sessions/{session_id}/steps/{step_id}")
+ def session_step(session_id: str, step_id: str) -> dict[str, Any]:
+ return manager.recording_step(session_id, step_id)
+
+ @app.get("/v1/sessions/{session_id}/usage")
+ def session_usage(session_id: str) -> dict[str, Any]:
+ return manager.recording_usage(session_id)
+
@app.patch("/v1/sessions/{session_id}")
def session_patch(session_id: str, body: dict) -> dict[str, Any]:
body = body or {}
diff --git a/coworker/server/manager.py b/coworker/server/manager.py
index 6a271bc1..e1fddf3d 100644
--- a/coworker/server/manager.py
+++ b/coworker/server/manager.py
@@ -44,6 +44,7 @@ from ..unrouted import UnroutedStore
from ..unattended import UnattendedRegistry
from ..audit import AuditStore
from ..observability import Recorder, RecordingStore
+from ..observability import read_api as recording_api
from ..observability.provider_view import RecordingProvider, tag as tag_provider
from ..config import load_config, workspace_allowed_commands
from ..conversations import ConversationStore, title_from
@@ -387,6 +388,11 @@ class SessionManager:
# coworker.db so they can be pruned and reset on their own.
self.recordings_store = RecordingStore.beside(base)
self.recorder = Recorder(store=self.recordings_store)
+ # Live: the same recording events go to the session's sockets, batched (§12).
+ self._live_steps: dict[str, list[dict[str, Any]]] = {}
+ self._live_steps_lock = threading.Lock()
+ self._live_loop: Optional[asyncio.AbstractEventLoop] = None
+ self.recorder.add_listener(self._forward_recording_event)
self.session_store = ConversationStore(base)
self.session_store.canonicalize_workspaces() # collapse /tmp vs /private/tmp etc.
if self.default_workspace:
@@ -6308,6 +6314,60 @@ class SessionManager:
def has_session_clients(self, session_id: str) -> bool:
return bool(self._session_clients.get(session_id))
+ # -- observability: reading recordings (§12) -------------------------------------------
+ LIVE_STEP_BATCH_S = 0.1
+
+ def recording_turns(self, session_id: str) -> dict[str, Any]:
+ return {"turns": recording_api.turns(self.recordings_store.load_recording(session_id))}
+
+ def recording_turn_steps(self, session_id: str, turn_id: str) -> dict[str, Any]:
+ rec = self.recordings_store.load_recording(session_id)
+ return {"steps": recording_api.turn_steps(rec, turn_id)}
+
+ def recording_step(self, session_id: str, step_id: str) -> dict[str, Any]:
+ rec = self.recordings_store.load_recording(session_id)
+ found = recording_api.step(
+ rec, step_id, get_piece=self.recordings_store.get_piece,
+ messages=self.session_messages(session_id),
+ audit_rows=lambda call_id: self.audit_store.list(session_id=session_id, call_id=call_id),
+ )
+ return {"step": found} if found is not None else {"step": None, "error": "no such step"}
+
+ def recording_usage(self, session_id: str) -> dict[str, Any]:
+ return recording_api.usage(self.recordings_store.load_recording(session_id), session_id)
+
+ def _forward_recording_event(self, event: dict[str, Any]) -> None:
+ """Queue one recording event for the session's sockets; a batch goes out every
+ 100 ms as `{"type": "step", "events": [...]}`, so a burst of steps does not flood
+ the connection (P-5). Older app versions ignore the unknown type."""
+ session_id = event.get("session_id")
+ loop = self._loop
+ if loop is None:
+ try:
+ loop = asyncio.get_running_loop() # a turn's start runs on the server's loop
+ except RuntimeError:
+ loop = self._live_loop
+ if not session_id or loop is None:
+ return
+ self._live_loop = loop
+ with self._live_steps_lock:
+ batch = self._live_steps.setdefault(session_id, [])
+ batch.append(event)
+ first = len(batch) == 1
+ if first:
+ try:
+ loop.call_soon_threadsafe(
+ loop.call_later, self.LIVE_STEP_BATCH_S, self._send_live_steps, session_id
+ )
+ except RuntimeError:
+ pass # the loop is closed: the server is stopping
+
+ def _send_live_steps(self, session_id: str) -> None:
+ with self._live_steps_lock:
+ events = self._live_steps.pop(session_id, [])
+ if events and self.has_session_clients(session_id):
+ asyncio.ensure_future(self.broadcast_session(session_id, {"type": "step", "events": events}))
+
def _inbox_item_changed(self, item: Any) -> None:
"""A card was added or moved to the Inbox: tell the waiting engine where it is shown
(observability §5.4, a wait step's `where`)."""
diff --git a/tests/corpora/observability/README.md b/tests/corpora/observability/README.md
index 975f098b..ddf679fa 100644
--- a/tests/corpora/observability/README.md
+++ b/tests/corpora/observability/README.md
@@ -9,5 +9,6 @@ Do not edit these files here. A change to the format needs a new version in `ocw
(its `tracing-spec.md` §12), and then a fresh copy of the folder here.
`tests/test_observability_golden.py` replays every scenario's `events.jsonl` into the
-recorder and compares the result with `expected-recording.json`. `expected-numbers.json`
-(the screen numbers) is checked once the read endpoints are built.
+recorder and compares the result with `expected-recording.json`, and computes the screen
+numbers (`coworker/observability/calculator.py`) and compares them with
+`expected-numbers.json`.
diff --git a/tests/test_observability_golden.py b/tests/test_observability_golden.py
index 4a72dff9..d04bc740 100644
--- a/tests/test_observability_golden.py
+++ b/tests/test_observability_golden.py
@@ -82,3 +82,25 @@ def test_the_store_gives_back_the_same_recording(scenario, tmp_path):
assert store.load_recording(session) == want
finally:
store.close()
+
+
[email protected]("scenario", SCENARIOS)
+def test_the_screen_numbers(scenario):
+ """`expected-numbers.json`: every token number and time bucket the screens show."""
+ from coworker.observability import calculator
+
+ expected = json.loads((GOLDEN / scenario / "expected-numbers.json").read_text("utf-8"))
+ rec = _replay(_events(scenario)).as_dict()
+ sessions = sorted({t["session_id"] for t in rec["turns"] if t["session_id"]}
+ | {s["session_id"] for s in rec["steps"] if s["session_id"]})
+
+ def usage(sid=None):
+ u = calculator.usage(rec, sid)
+ u.pop("unreported_step_ids") # an addition for the "≥" links (M-R19), not in the golden
+ return u
+
+ got = {"usage": usage()}
+ if len(sessions) > 1:
+ got["usage_by_session"] = {sid: usage(sid) for sid in sessions}
+ got["buckets_by_turn"] = {t["turn_id"]: calculator.buckets(rec, t) for t in rec["turns"]}
+ assert got == expected
diff --git a/tests/test_observability_read_api.py b/tests/test_observability_read_api.py
new file mode 100644
index 00000000..4b41d4e3
--- /dev/null
+++ b/tests/test_observability_read_api.py
@@ -0,0 +1,89 @@
+"""Observability §12: what the app's screens read, through the real server.
+
+A turn runs over the session's websocket; the recorded turns, steps, one step in full and
+the session's token numbers are then read back over HTTP, and the same recording events
+arrive live on the socket as `{"type": "step", "events": [...]}`.
+"""
+
+from __future__ import annotations
+
+from fastapi.testclient import TestClient
+
+from coworker.providers import AssistantTurn, ModelCapabilities, ProviderClient, ToolCall
+from coworker.providers.base import TokenUsage
+from coworker.server import SessionManager, create_app
+
+
+class _Scripted(ProviderClient):
+ def __init__(self, turns):
+ self._turns = list(turns)
+
+ def complete(self, *, model, messages, tools=None, **settings):
+ if str(messages[0].get("content") or "").startswith("You title"):
+ return AssistantTurn(text="Notes", usage=TokenUsage(input=40, output=2)) # the session title
+ return self._turns.pop(0)
+
+ def capabilities(self, model):
+ return ModelCapabilities()
+
+
+def test_a_recorded_turn_reads_back_through_the_endpoints(tmp_path):
+ (tmp_path / "notes.txt").write_text("Friday", encoding="utf-8")
+ provider = _Scripted([
+ AssistantTurn(tool_calls=[ToolCall(id="c1", name="read_file", arguments={"path": "notes.txt"})],
+ finish_reason="tool_calls", usage=TokenUsage(input=100, output=10, cache_read=900)),
+ AssistantTurn(text="It says Friday.", finish_reason="stop", usage=TokenUsage(input=20, output=5, cache_read=1000)),
+ ])
+ manager = SessionManager(workspace=tmp_path, provider=provider)
+ client = TestClient(create_app(manager))
+ live: list[dict] = []
+ with client.websocket_connect("/ws/session/r1") as ws:
+ assert ws.receive_json()["type"] == "ready"
+ ws.send_json({"type": "user_message", "text": "what does notes.txt say?"})
+ while True:
+ msg = ws.receive_json()
+ if msg["type"] == "step":
+ live.extend(msg["events"])
+ if msg["type"] == "turn_done":
+ break
+ # The last batch may arrive just after the turn's end.
+ import time
+ deadline = time.time() + 2
+ while time.time() < deadline and not any(e["type"] == "turn_ended" for e in live):
+ msg = ws.receive_json()
+ if msg["type"] == "step":
+ live.extend(msg["events"])
+
+ [turn] = client.get("/v1/sessions/r1/turns").json()["turns"]
+ assert (turn["kind"], turn["trigger"], turn["status"]) == ("run", "user", "completed")
+ # The two main calls, and the session title (40 + 2): every purpose with a session counts.
+ assert turn["usage"]["pill"] == 100 + 900 + 10 + 20 + 1000 + 5 + 40 + 2
+ assert turn["usage"]["by_purpose"] == {"main": 2035, "title": 42}
+ assert turn["buckets"]["total"] == turn["duration_ms"]
+
+ steps = client.get(f"/v1/sessions/r1/turns/{turn['turn_id']}/steps").json()["steps"]
+ mains = [s for s in steps if s["kind"] != "context" and s["attrs"].get("purpose") != "title"]
+ assert [s["kind"] for s in mains] == ["model", "tool", "model"]
+ assert all("content" not in s for s in steps)
+
+ tool = next(s for s in steps if s["kind"] == "tool")
+ detail = client.get(f"/v1/sessions/r1/steps/{tool['step_id']}").json()["step"]
+ assert detail["message"]["role"] == "tool" and "Friday" in detail["message"]["content"]
+ assert detail["message"]["mid"] == tool["message_id"]
+ assert any(row.get("call_id") == "c1" for row in detail["audit"])
+
+ usage = client.get("/v1/sessions/r1/usage").json()
+ assert usage["pill"] == turn["usage"]["pill"] and usage["at_least"] is False
+ assert usage["recorded"] is True
+ assert usage["cache_share"] == round(1900 / (100 + 900 + 20 + 1000 + 40), 3)
+ assert usage["context"]["tokens"] == 1020
+
+ types = [e["type"] for e in live]
+ assert types[0] == "turn_started" and "turn_ended" in types and "step_ended" in types
+
+
+def test_a_session_recorded_before_phase_1_says_so(tmp_path):
+ manager = SessionManager(workspace=tmp_path, provider=_Scripted([]))
+ client = TestClient(create_app(manager))
+ assert client.get("/v1/sessions/old/turns").json() == {"turns": []}
+ assert client.get("/v1/sessions/old/usage").json()["recorded"] is False