Python
fix(observability): Phase 3 review before merging (P3B-2 to P3B-7)
Devika Verma847aca2f8495
diff --git a/coworker/observability/export.py b/coworker/observability/export.pyindex fc2e94b6..068148de 100644--- a/coworker/observability/export.py+++ b/coworker/observability/export.py@@ -383,8 +383,9 @@ class Exporter:rows = getattr(self.store, "feedback_since", None)if rows is None:return 0- todo = [fb for fb in rows(cfg.since_ms, limit=100)- if self._feedback_mark(fb) not in self.store.export_marked(dest, fb["turn_id"])]+ # The unsent ones are picked in the query, not after it: filtering after a LIMIT left the+ # queue stuck on the first 100 once those had gone (P3B-2).+ todo = rows(cfg.since_ms, limit=100, unsent_for=dest)sent = 0for fb in todo:mark = [(self._feedback_mark(fb), fb["turn_id"])]diff --git a/coworker/observability/scopes.py b/coworker/observability/scopes.pyindex c782bae2..944222d5 100644--- a/coworker/observability/scopes.py+++ b/coworker/observability/scopes.py@@ -169,6 +169,12 @@ def same_approval_twice(cards: list[dict]) -> list[dict]:"""Task check over the team's approval cards in order: the same tool and args_hash askedagain after an earlier card for it was approved. `cards`: [{tool, args_hash, session,outcome}]."""+ return [f for f, _card in same_approval_repeats(cards)]+++def same_approval_repeats(cards: list[dict]) -> list[tuple[dict, dict]]:+ """`same_approval_twice` with the card that asked again: (finding, repeat card). Both the+ Issues tab and TeamView's event line point at that card, the second ask (P3B-4)."""approved: dict[tuple, dict] = {}fired = []for c in cards:@@ -176,7 +182,7 @@ def same_approval_twice(cards: list[dict]) -> list[dict]:continue # arguments that never parsed match nothing (build: not in the prototype)k = (c["tool"], c["args_hash"])if k in approved:- fired.append({"tool": c["tool"], "first_by": approved[k]["session"], "again_by": c["session"]})+ fired.append(({"tool": c["tool"], "first_by": approved[k]["session"], "again_by": c["session"]}, c))if c.get("outcome") == "approved":approved.setdefault(k, c)return fired@@ -321,14 +327,11 @@ def failed_in_a_row_issue(runs: list[dict], last_step: Optional[dict] = None) ->def same_approval_issues(cards: list[dict]) -> list[dict]:- out = []- for i, f in enumerate(same_approval_twice(cards)):- step = next((c.get("step") for c in cards- if c["tool"] == f["tool"] and c["session"] == f["again_by"] and c.get("step")), None)- out.append(_issue("same_approval_twice",- f"Same approval asked twice: {f['tool']} ({f['first_by']}, then {f['again_by']}).",- step, **f))- return out+ """Each repeat as an issue on the card that asked again (P3B-4), not the first one."""+ return [_issue("same_approval_twice",+ f"Same approval asked twice: {f['tool']} ({f['first_by']}, then {f['again_by']}).",+ card.get("step"), **f)+ for f, card in same_approval_repeats(cards)]def reviewed_before_prereq_issues(events: list[dict], links: Iterable[tuple[Any, Any]]) -> list[dict]:diff --git a/coworker/observability/store.py b/coworker/observability/store.pyindex ad2ad0b1..308423ae 100644--- a/coworker/observability/store.py+++ b/coworker/observability/store.py@@ -236,6 +236,9 @@ class RecordingStore:self._pending_steps: dict[str, dict[str, Any]] = {}self._pending_refresh: set[str] = set() # turns whose ledger rows to work out againself._pending_lock = threading.Lock()+ # How many writes touched each session since start (P3B-5): a team view keys its+ # cache on its members' numbers, so it is worked out again only when one changed.+ self._generation: dict[str, int] = {}self._owner = False # this process called close_open_turns: it owns the data folderself._closed = Falseself._last_sweep = 0.0@@ -292,11 +295,14 @@ class RecordingStore:"INSERT OR IGNORE INTO step_pieces (step_id, sha256) VALUES (?, ?)",[(s["step_id"], r) for r in refs],)- # Phase 3: the ledger rows of turns that ended or changed, from what was- # just written (the turn, its steps and its Issues result).- self._refresh_usage(refresh)except Exception:log.exception("recordings: write failed (%d turns, %d steps)", len(turns), len(steps))+ self._touch_sessions({t.get("session_id") for t in turns.values()} | {x.get("session_id") for x in steps.values()})+ # Phase 3: the ledger rows of turns that ended or changed, from what was just+ # written (the turn, its steps and its Issues result). After the recording's own+ # transaction, one turn at a time: a ledger error loses that turn's rows only,+ # never the recorded steps (P3B-6).+ self._refresh_usage(refresh)def _tick(self) -> None:"""One turn of the shared writer: write what is pending, and sweep once a day."""@@ -418,9 +424,23 @@ class RecordingStore:return len(open_turns)# -- deleting -----------------------------------------------------------------------------+ def _touch_sessions(self, session_ids: Iterable[Optional[str]]) -> None:+ with self._pending_lock:+ for sid in session_ids:+ if sid:+ self._generation[sid] = self._generation.get(sid, 0) + 1++ def generation(self, session_ids: Iterable[str]) -> tuple[int, ...]:+ """How many writes have touched each of these sessions since start (P3B-5). Writes+ still pending are written first, so an unchanged answer means nothing new."""+ self.flush()+ with self._pending_lock:+ return tuple(self._generation.get(sid, 0) for sid in session_ids)+def delete_session(self, session_id: str) -> None:"""The session-delete cascade (§11): its turns, steps and the pieces only they used."""self.flush() # the ledger rows of its turns are worked out before the steps go+ self._touch_sessions({session_id})with self._lock, self._conn:with self._pending_lock:self._pending_turns = {@@ -723,41 +743,51 @@ class RecordingStore:_writer.wake()def _refresh_usage(self, turn_ids: Iterable[str]) -> None:- """Inside the writer's transaction: replace each finished turn's ledger rows."""- from .rollups import LEDGER_COLUMNS, ledger_rows-+ """With the connection lock held: replace each finished turn's ledger rows, each turn in+ a transaction of its own. A turn whose rows cannot be worked out keeps its old rows and+ is logged; the others go on (P3B-6)."""now = int(time.time() * 1000)for tid in turn_ids:- t = self._conn.execute("SELECT * FROM turns WHERE turn_id = ?", (tid,)).fetchone()- if t is None or t["status"] in (None, "running"):- continue- steps = [_step_record(r) for r in self._conn.execute(- "SELECT * FROM steps WHERE turn_id = ? AND kind IN ('model', 'wait', 'auto_approver', 'tool')",- (tid,))]try:- issues = json.loads(t["issues"]) if t["issues"] else None- except ValueError:- issues = None- prior = self._conn.execute(- "SELECT rating, session_deleted, persona_id FROM turn_usage WHERE turn_id = ? AND first = 1",- (tid,)).fetchone()- fb = self._conn.execute("SELECT rating FROM feedback WHERE turn_id = ?", (tid,)).fetchone()- persona = prior["persona_id"] if prior is not None else None- if persona is None and self.persona_of is not None:- try:- persona = self.persona_of(t["session_id"])- except Exception: # noqa: BLE001 - a stand-in never breaks the write- persona = None- rating = fb["rating"] if fb is not None else (prior["rating"] if prior is not None else None)- rows = ledger_rows(_turn_record(t), steps, issues, persona_id=persona, rating=rating, now_ms=now)- if prior is not None and prior["session_deleted"]:- for r in rows:- r["session_deleted"] = 1- self._conn.execute("DELETE FROM turn_usage WHERE turn_id = ?", (tid,))- self._conn.executemany(- f"INSERT INTO turn_usage ({', '.join(LEDGER_COLUMNS)}) "- f"VALUES ({', '.join('?' * len(LEDGER_COLUMNS))})",- [tuple(r.get(c) for c in LEDGER_COLUMNS) for r in rows])+ with self._conn:+ self._refresh_turn_usage(tid, now)+ except Exception:+ log.exception("recordings: usage ledger not updated for turn %s", tid)++ def _refresh_turn_usage(self, tid: str, now: int) -> None:+ from .rollups import LEDGER_COLUMNS, ledger_rows++ t = self._conn.execute("SELECT * FROM turns WHERE turn_id = ?", (tid,)).fetchone()+ if t is None or t["status"] in (None, "running"):+ return+ self._touch_sessions({t["session_id"]}) # its Issues result or ledger rows changed+ steps = [_step_record(r) for r in self._conn.execute(+ "SELECT * FROM steps WHERE turn_id = ? AND kind IN ('model', 'wait', 'auto_approver', 'tool')",+ (tid,))]+ try:+ issues = json.loads(t["issues"]) if t["issues"] else None+ except ValueError:+ issues = None+ prior = self._conn.execute(+ "SELECT rating, session_deleted, persona_id FROM turn_usage WHERE turn_id = ? AND first = 1",+ (tid,)).fetchone()+ fb = self._conn.execute("SELECT rating FROM feedback WHERE turn_id = ?", (tid,)).fetchone()+ persona = prior["persona_id"] if prior is not None else None+ if persona is None and self.persona_of is not None:+ try:+ persona = self.persona_of(t["session_id"])+ except Exception: # noqa: BLE001 - a stand-in never breaks the write+ persona = None+ rating = fb["rating"] if fb is not None else (prior["rating"] if prior is not None else None)+ rows = ledger_rows(_turn_record(t), steps, issues, persona_id=persona, rating=rating, now_ms=now)+ if prior is not None and prior["session_deleted"]:+ for r in rows:+ r["session_deleted"] = 1+ self._conn.execute("DELETE FROM turn_usage WHERE turn_id = ?", (tid,))+ self._conn.executemany(+ f"INSERT INTO turn_usage ({', '.join(LEDGER_COLUMNS)}) "+ f"VALUES ({', '.join('?' * len(LEDGER_COLUMNS))})",+ [tuple(r.get(c) for c in LEDGER_COLUMNS) for r in rows])def sweep_ledger(self, *, now_ms: Optional[int] = None) -> int:"""Ledger rows older than 90 days go (TD3-5), whatever "Keep for" says."""@@ -919,6 +949,34 @@ class RecordingStore:"wake": json.loads(r["wake"]) if r["wake"] else None,"stamp": json.loads(r["stamp"] or "{}")} for r in rows]+ def last_turn_automation(self, session_id: str) -> Optional[dict[str, Any]]:+ """The session's latest turn's `turn.automation` (1.3), or None when that turn was not+ an automation run's own (a follow-up typed later, or no turn yet)."""+ self.flush()+ with self._lock:+ r = self._conn.execute("SELECT automation FROM turns WHERE session_id = ? "+ "ORDER BY started_at DESC, seq DESC LIMIT 1", (session_id,)).fetchone()+ if r is None or not r["automation"]:+ return None+ try:+ return json.loads(r["automation"])+ except ValueError:+ return None++ def last_tool_step(self, session_id: str, name: str, *, before_ms: Optional[int] = None) -> Optional[dict[str, Any]]:+ """The session's last tool step of this name started at or before `before_ms` (any+ time when None): {step_id, seq, turn_id}. What a hand-off's `transition` step is found+ by, without loading the whole recording (P3B-5)."""+ self.flush()+ sql = "SELECT step_id, seq, turn_id FROM steps WHERE session_id = ? AND kind = 'tool' AND name = ?"+ args: list[Any] = [session_id, name]+ if before_ms is not None:+ sql += " AND COALESCE(started_at, 0) <= ?"+ args.append(before_ms)+ with self._lock:+ r = self._conn.execute(sql + " ORDER BY started_at DESC, seq DESC LIMIT 1", args).fetchone()+ return None if r is None else {"step_id": r["step_id"], "seq": r["seq"], "turn_id": r["turn_id"]}+def team_approval_cards(self, session_ids: list[str]) -> list[dict[str, Any]]:"""The members' approval cards in order: each wait step of kind approval with itstool step's name and args_hash (`same_approval_twice`, signals 0.6)."""@@ -932,14 +990,24 @@ class RecordingStore:"JOIN turns t ON t.turn_id = s.turn_id "f"WHERE s.kind = 'wait' AND s.session_id IN ({marks}) ORDER BY s.started_at, s.step_id",list(session_ids)).fetchall()- tools = {(r["session_id"], r["call_id"]): r for r in self._conn.execute(- "SELECT session_id, call_id, name, attrs FROM steps "- f"WHERE kind = 'tool' AND session_id IN ({marks})", list(session_ids)).fetchall()}+ approvals = []+ for w in waits:+ a = json.loads(w["attrs"] or "{}")+ if a.get("kind") == "approval" and a.get("for_call_id"):+ approvals.append((w, a))+ # Only the tool steps the cards ask about, by (session, call id): not every tool+ # step of every member (P3B-5).+ tools = {}+ wanted = sorted({(w["session_id"], a["for_call_id"]) for w, a in approvals})+ for i in range(0, len(wanted), 400):+ part = wanted[i:i + 400]+ where = " OR ".join("(session_id = ? AND call_id = ?)" for _ in part)+ for r in self._conn.execute(+ f"SELECT session_id, call_id, name, attrs FROM steps WHERE kind = 'tool' AND ({where})",+ [v for pair in part for v in pair]).fetchall():+ tools[(r["session_id"], r["call_id"])] = rout = []- for w in waits:- a = json.loads(w["attrs"] or "{}")- if a.get("kind") != "approval":- continue+ for w, a in approvals:tool = tools.get((w["session_id"], a.get("for_call_id")))if tool is None:continue@@ -1019,13 +1087,22 @@ class RecordingStore:(session_id,)).fetchall()return [_feedback_record(r) for r in rows]- def feedback_since(self, since_ms: int, limit: int = 100) -> list[dict[str, Any]]:+ def feedback_since(self, since_ms: int, limit: int = 100, *,+ unsent_for: Optional[str] = None) -> list[dict[str, Any]]:"""Ratings given or changed at or after `since_ms`, oldest change first: the- exporter's queue (E27b)."""+ exporter's queue (E27b). With `unsent_for`, only the versions with no export mark for+ that destination yet (mark `feedback|<turn_id>|<changed_at or rated_at>`), so the+ queue keeps moving past the first `limit` once those have gone (P3B-2)."""+ sql = "SELECT * FROM feedback f WHERE COALESCE(f.changed_at, f.rated_at) >= ?"+ args: list[Any] = [since_ms]+ if unsent_for is not None:+ sql += (" AND NOT EXISTS (SELECT 1 FROM export_marks m WHERE m.destination = ? AND m.turn_id = f.turn_id"+ " AND m.item_id = 'feedback|' || f.turn_id || '|' || COALESCE(f.changed_at, f.rated_at))")+ args.append(unsent_for)+ sql += " ORDER BY COALESCE(f.changed_at, f.rated_at) LIMIT ?"+ args.append(limit)with self._lock:- rows = self._conn.execute(- "SELECT * FROM feedback WHERE COALESCE(changed_at, rated_at) >= ? "- "ORDER BY COALESCE(changed_at, rated_at) LIMIT ?", (since_ms, limit)).fetchall()+ rows = self._conn.execute(sql, args).fetchall()return [_feedback_record(r) for r in rows]# -- coworker versions (Phase 3, E25c) -------------------------------------------------------diff --git a/coworker/observability/usage_api.py b/coworker/observability/usage_api.pyindex 45030fa2..96b2e5b5 100644--- a/coworker/observability/usage_api.py+++ b/coworker/observability/usage_api.py@@ -265,12 +265,14 @@ def team_wakes(store: Any, session_ids: Iterable[str], *, start: int = 0, end: idef team_events(store: Any, *, lead_session: str, worker_sessions: dict[str, str],- board_events: list[dict[str, Any]], links: list[tuple[int, int]], limit: int = 50) -> list[dict[str, Any]]:+ board_events: list[dict[str, Any]], links: list[tuple[int, int]], limit: int = 50,+ turns: Optional[list[dict[str, Any]]] = None) -> list[dict[str, Any]]:""""Team events that used to be silent" (§11.4), newest first: cap hits, long deferrals,watchdog nudges, lead polling, the same approval asked twice and an item reviewed before- its prerequisite. Each names the session and turn whose Steps it opens."""+ its prerequisite. Each names the session and turn whose Steps it opens. `turns`: the+ members' turns when the caller has read them already (P3B-5)."""sessions = {lead_session: "lead", **{sid: actor for actor, sid in worker_sessions.items()}}- rows = store.team_turns(list(sessions))+ rows = store.team_turns(list(sessions)) if turns is None else turnsevents: list[dict[str, Any]] = []for t in rows:wake = t.get("wake") or {}@@ -290,10 +292,9 @@ def team_events(store: Any, *, lead_session: str, worker_sessions: dict[str, strevents.append({"session_id": lead_session, "turn_id": t["turn_id"], "seq": t["seq"], "at": t["started_at"],"who": "lead", "kind": "lead_polling", "severity": "warn", "count": f["count"]})cards = store.team_approval_cards(list(sessions))- for f in scopes.same_approval_twice(cards):- c = next((c for c in reversed(cards) if c["tool"] == f["tool"] and c["session"] == f["again_by"]), None)- events.append({"session_id": f["again_by"], "turn_id": (c or {}).get("turn_id"), "seq": (c or {}).get("seq"),- "at": (c or {}).get("at"), "who": sessions.get(f["again_by"], "?"), "kind": "same_approval_twice",+ for f, c in scopes.same_approval_repeats(cards): # the repeat card, as the Issues tab (P3B-4)+ events.append({"session_id": f["again_by"], "turn_id": c.get("turn_id"), "seq": c.get("seq"),+ "at": c.get("at"), "who": sessions.get(f["again_by"], "?"), "kind": "same_approval_twice","severity": "warn", "tool": f["tool"], "first_by": sessions.get(f["first_by"], f["first_by"]),"again_by": sessions.get(f["again_by"], f["again_by"])})transitions = [{"seq": e["seq"], "item": e.get("item_id"), "to": (e.get("payload") or {}).get("to"),diff --git a/coworker/remote/acceptor.py b/coworker/remote/acceptor.pyindex 145ec4ce..0b0313e4 100644--- a/coworker/remote/acceptor.py+++ b/coworker/remote/acceptor.py@@ -814,10 +814,11 @@ def mount_acceptor()return {"ok": True, "revoked": names}- def _cached_rollup(machine_id: str, grouping: str):- """An offline machine's last rollup, labelled with when it was fetched (TD3-6). The+ def _cached_rollup(machine_id: str, query: str):+ """An offline machine's last answer to this same rollup request, labelled with when it+ was fetched (TD3-6). Only the same period, grouping and coworker count (P3B-3). Thedesktop never adds it to the live totals without that label."""- found = registry.last_rollup(machine_id, grouping)+ found = registry.last_rollup(machine_id, query)if found is None:return Nonetry:@@ -841,9 +842,8 @@ def mount_acceptor(_touch(machine_id)live = await _live_or_wake(machine_id)rollup = request.method == "GET" and path == "v1/observability/rollup"- grouping = request.query_params.get("group") or "all"if live is None:- cached = _cached_rollup(machine_id, grouping) if rollup else None+ cached = _cached_rollup(machine_id, request.url.query) if rollup else Nonereturn cached or JSONResponse({"error": "machine offline"}, status_code=503)target = "/" + pathif request.url.query:@@ -863,12 +863,12 @@ def mount_acceptor(except asyncio.TimeoutError:return JSONResponse({"error": "machine timed out"}, status_code=504)except ConnectionError:- cached = _cached_rollup(machine_id, grouping) if rollup else None+ cached = _cached_rollup(machine_id, request.url.query) if rollup else Nonereturn cached or JSONResponse({"error": "machine disconnected"}, status_code=503)- # Phase 3 (E28): each machine's last Usage rollup, per grouping, for when it is off.+ # Phase 3 (E28): each machine's last answer per Usage request, for when it is off.if rollup and status == 200:try:- registry.save_rollup(machine_id, grouping, request.url.query, payload.decode("utf-8"))+ registry.save_rollup(machine_id, request.url.query, payload.decode("utf-8"))except Exception:pass# Tier-2 snapshot: every successful unfiltered session-list fetch doublesdiff --git a/coworker/remote/registry.py b/coworker/remote/registry.pyindex facc1e1d..0b9e2414 100644--- a/coworker/remote/registry.py+++ b/coworker/remote/registry.py@@ -48,24 +48,27 @@ class MachinesRegistry:)""")- # Tier-2 snapshot (remote-home-design.md data path): the last-known- # session-LIST METADATA per machine — what keeps an offline machine's- # sessions greyed-but-visible instead of vanished.# Observability Phase 3 (E28, TD3-6): the last Usage rollup each machine answered,- # per grouping, so an offline machine still shows its numbers on the Usage page,- # labelled "as of". Counts only (the ledger is content-free).+ # per request (period, grouping, coworker, ...), so an offline machine still shows+ # its numbers on the Usage page, labelled "as of", and only for the question that+ # was asked (P3B-3). Counts only (the ledger is content-free). The first build kept+ # one row per grouping, whatever the period; that table is dropped.+ self._conn.execute("DROP TABLE IF EXISTS machine_rollups")self._conn.execute("""- CREATE TABLE IF NOT EXISTS machine_rollups (+ CREATE TABLE IF NOT EXISTS machine_rollup_answers (machine_id TEXT NOT NULL,- grouping TEXT NOT NULL,+ query_key TEXT NOT NULL,query TEXT NOT NULL,payload TEXT NOT NULL,updated_at REAL NOT NULL,- PRIMARY KEY (machine_id, grouping)+ PRIMARY KEY (machine_id, query_key))""")+ # Tier-2 snapshot (remote-home-design.md data path): the last-known+ # session-LIST METADATA per machine — what keeps an offline machine's+ # sessions greyed-but-visible instead of vanished.self._conn.execute("""CREATE TABLE IF NOT EXISTS machine_sessions (@@ -289,22 +292,43 @@ class MachinesRegistry:).fetchall()return [dict(r) for r in rows]- def save_rollup(self, machine_id: str, grouping: str, query: str, payload: str) -> None:+ ROLLUPS_KEPT = 50 # per machine: the few periods and coworkers a person looks at++ @staticmethod+ def rollup_key(query: str) -> str:+ """A rollup request's cache key: its query parameters, sorted, so the same question+ asked with its parameters in another order finds the same answer."""+ from urllib.parse import parse_qsl, urlencode++ return urlencode(sorted(parse_qsl(query or "", keep_blank_values=True)))++ def save_rollup(self, machine_id: str, query: str, payload: str) -> None:+ """Keep a machine's answer to one rollup request (P3B-3: the whole request is the key,+ not just its grouping). Only the newest `ROLLUPS_KEPT` per machine stay."""with self._lock:self._conn.execute(- "INSERT INTO machine_rollups (machine_id, grouping, query, payload, updated_at)"- " VALUES (?, ?, ?, ?, ?) ON CONFLICT(machine_id, grouping) DO UPDATE SET"+ "INSERT INTO machine_rollup_answers (machine_id, query_key, query, payload, updated_at)"+ " VALUES (?, ?, ?, ?, ?) ON CONFLICT(machine_id, query_key) DO UPDATE SET"" query = excluded.query, payload = excluded.payload, updated_at = excluded.updated_at",- (machine_id, grouping, query, payload, time.time()),+ (machine_id, self.rollup_key(query), query, payload, time.time()),+ )+ self._conn.execute(+ "DELETE FROM machine_rollup_answers WHERE machine_id = ? AND query_key NOT IN ("+ " SELECT query_key FROM machine_rollup_answers WHERE machine_id = ?"+ " ORDER BY updated_at DESC LIMIT ?)",+ (machine_id, machine_id, self.ROLLUPS_KEPT),)self._conn.commit()- def last_rollup(self, machine_id: str, grouping: str) -> Optional[dict]:- """The machine's last Usage rollup for this grouping: {payload, query, updated_at}."""+ def last_rollup(self, machine_id: str, query: str) -> Optional[dict]:+ """The machine's last answer to this same rollup request: {payload, query, updated_at},+ or None when that request was never answered (another period or coworker is not a+ stand-in)."""with self._lock:row = self._conn.execute(- "SELECT payload, query, updated_at FROM machine_rollups WHERE machine_id = ? AND grouping = ?",- (machine_id, grouping),+ "SELECT payload, query, updated_at FROM machine_rollup_answers"+ " WHERE machine_id = ? AND query_key = ?",+ (machine_id, self.rollup_key(query)),).fetchone()return dict(row) if row else Nonediff --git a/coworker/server/manager.py b/coworker/server/manager.pyindex 7f59a089..f7723c1c 100644--- a/coworker/server/manager.py+++ b/coworker/server/manager.py@@ -454,6 +454,11 @@ class SessionManager:self._scope_pool = concurrent.futures.ThreadPoolExecutor(max_workers=1, thread_name_prefix="obs-scopes")self.recorder.add_listener(self._scopes_on_turn_end)+ # P3B-5: a team's Phase 3 numbers and task checks, kept until a member's recording,+ # the board or the journal changes (team_id -> what they were worked out from, result).+ self._team_obs_cache: dict[str, tuple[Any, Any]] = {}+ self._team_checked: dict[str, Any] = {}+ self._team_obs_lock = threading.Lock()# No explicit provider injected → route by the model's `provider:` prefix (OpenAI default,# Ollama, …). Tests inject a provider directly and bypass the router. The same router is# shared by every engine and the `/v1/chat/completions` proxy.@@ -2057,9 +2062,20 @@ class SessionManager:current = self.inbox.get(item.id) if getattr(item, "id", None) else Noneby_raw = str(getattr(current or item, "resolved_by", "") or "")by = "lead" if by_raw.startswith(("lead", "agent")) else "user"+ # P3B-7 (TD3-23): the resume finishes the turn that asked, so when that turn was an+ # automation run's own, the resumed turn is the run's too: its tokens and health count+ # in the run's row. The card is the latest turn's (the turn stopped at it); a follow-up+ # typed into the run's session later carries no `automation`, and neither does its resume.+ automation = None+ if item.session_id.startswith("__run__"):+ try:+ automation = self.recordings_store.last_turn_automation(item.session_id)+ except Exception: # noqa: BLE001 - a recording fact never blocks the resume+ logger.exception("could not tell whether %s resumes a run's turn", item.session_id)try:async for _event in engine.resume(trigger="user", team=self._team_stamp(item.session_id), wake={- "reason": "answer", "by": by, "board_seq": self._board_seq(item.session_id)}):+ "reason": "answer", "by": by, "board_seq": self._board_seq(item.session_id)},+ automation=automation):passself.save(item.session_id, engine)finally:@@ -2748,29 +2764,10 @@ class SessionManager:out = make_summary(self, team, time.time())# Phase 3 (OB-17, E19c): why each team turn started, from the ledger of every member# session (its total equals their pills), and the team events that used to be silent.+ # Worked out again only when something they read has changed (P3B-5).try:- members = {w.actor: w.session_id for w in team.workers}- sids = [team.lead_session, *members.values()]- out["wakes"] = usage_api.team_wakes(self.recordings_store, sids)- from ..teams.summary import all_events-- board = list(all_events(self.team_store, team.space))- links = []- for item in self.team_store.list_items(team.space, self._user_actor()):- for link in item.get("links") or []:- if link.get("kind") == "blocked_by" and link.get("item") is not None:- links.append((int(link["item"]), int(item["id"])))- out["events"] = usage_api.team_events(self.recordings_store, lead_session=team.lead_session,- worker_sessions=members, board_events=board, links=links)- out["wake_explained"] = self._wake_explained(sids)- # OB-16: "Coverage for this task" on each item a coworker that declares coverage- # worked on, with its findings' evidence links.- cov = {}- for item in self.team_store.list_items(team.space, self._user_actor()):- c = self.item_coverage(team, item, board)- if c is not None:- cov[str(item["id"])] = c- out["coverage"] = cov+ out.update(self._team_obs(team))+ cov = out["coverage"]by_session = {team.lead_session: team.lead_actor, **{w.session_id: w.actor for w in team.workers}}codes = {name: list(sc.get("findings_exit") or []) for name, sc in((recording_signals.signals().get("scanners") or {}).get("list") or {}).items()}@@ -2802,9 +2799,64 @@ class SessionManager:out.setdefault("events", [])return out- def _wake_explained(self, session_ids: list[str]) -> dict[str, int]:+ def _team_inputs(self, team) -> dict[str, Any]:+ """What a team's Phase 3 numbers and task checks are worked out from, read once: the+ board, its items, and a mark of everything they depend on (P3B-5). The mark is the+ members' write counts, the board's last event, the journal's last entry and the items+ themselves; when it is unchanged, so is the answer."""+ from ..teams.summary import all_events++ members = {w.actor: w.session_id for w in team.workers}+ sids = [team.lead_session, *members.values()]+ board = list(all_events(self.team_store, team.space))+ items = self.team_store.list_items(team.space, self._user_actor())+ js = self.journal_store+ with js._lock:+ row = js._conn.execute("SELECT MAX(seq) AS m FROM journal_entries WHERE space = ?",+ (team.space,)).fetchone()+ mark = (tuple(sids), self.recordings_store.generation(sids), len(board),+ board[-1]["seq"] if board else None, row["m"] if row else None,+ json.dumps(items, sort_keys=True, default=str))+ return {"members": members, "sids": sids, "board": board, "items": items, "mark": mark}++ def _team_obs(self, team) -> dict[str, Any]:+ """A team's wakes, events, wake gate and per-item coverage, kept until its inputs+ change: a refresh of an idle team rereads only the board and the counts (P3B-5)."""+ inputs = self._team_inputs(team)+ with self._team_obs_lock:+ hit = self._team_obs_cache.get(team.team_id)+ if hit is not None and hit[0] == inputs["mark"]:+ return hit[1]+ members, sids, board, items = inputs["members"], inputs["sids"], inputs["board"], inputs["items"]+ turns = self.recordings_store.team_turns(sids)+ links = []+ for item in items:+ for link in item.get("links") or []:+ if link.get("kind") == "blocked_by" and link.get("item") is not None:+ links.append((int(link["item"]), int(item["id"])))+ res: dict[str, Any] = {+ "wakes": usage_api.team_wakes(self.recordings_store, sids),+ "events": usage_api.team_events(self.recordings_store, lead_session=team.lead_session,+ worker_sessions=members, board_events=board, links=links,+ turns=turns),+ "wake_explained": self._wake_explained(sids, turns),+ }+ # OB-16: "Coverage for this task" on each item a coworker that declares coverage+ # worked on, with its findings' evidence links.+ cov = {}+ for item in items:+ c = self.item_coverage(team, item, board, turns=turns)+ if c is not None:+ cov[str(item["id"])] = c+ res["coverage"] = cov+ with self._team_obs_lock:+ self._team_obs_cache[team.team_id] = (inputs["mark"], res)+ return res++ def _wake_explained(self, session_ids: list[str],+ rows: Optional[list[dict[str, Any]]] = None) -> dict[str, int]:"""The 95% gate on a real team (V3-08): team turns, and those whose wake is known."""- rows = self.recordings_store.team_turns(session_ids)+ rows = self.recordings_store.team_turns(session_ids) if rows is None else rowsteam_rows = [t for t in rows if (t.get("stamp") or {}).get("team")]known = sum(1 for t in team_rows if (t.get("wake") or {}).get("reason") not in (None, "unknown"))return {"turns": len(team_rows), "explained": known}@@ -6805,10 +6857,12 @@ class SessionManager:found.setdefault(i.get("turn_id") or last, []).append(i)self._store_scope_issues([t["turn_id"] for t in done], found, ["scan_missing", "scanner_failed"])- def item_coverage(self, team, item: dict[str, Any], events: list[dict[str, Any]]) -> Optional[dict[str, Any]]:+ def item_coverage(self, team, item: dict[str, Any], events: list[dict[str, Any]], *,+ turns: Optional[list[dict[str, Any]]] = None) -> Optional[dict[str, Any]]:""""Coverage for this task" (OB-16, §10): a board item of a worker that declarescoverage, its scans across the item's sessions, the hand-off's report, its findings- with their evidence, and the fix it verifies. None when nobody on it promised a scan."""+ with their evidence, and the fix it verifies. None when nobody on it promised a scan.+ `turns`: the members' turns, when the caller has read them already (P3B-5)."""by_actor = {w.actor: w.session_id for w in team.workers}by_actor[team.lead_actor] = team.lead_sessionmine = [e for e in events if e.get("item_id") == item["id"]]@@ -6819,7 +6873,9 @@ class SessionManager:if not sessions:return Nonepromised: list[str] = []- for t in self.recordings_store.team_turns(sessions):+ mine_turns = ([t for t in turns if t["session_id"] in sessions] if turns is not None+ else self.recordings_store.team_turns(sessions))+ for t in mine_turns:for a in (t.get("stamp") or {}).get("persona_coverage") or []:if a not in promised:promised.append(a)@@ -6885,13 +6941,9 @@ class SessionManager:def _handoff_step(self, session_id: str, at_ms: Optional[int]) -> Optional[dict[str, Any]]:"""The `transition` tool step that handed the item off: the last one of the session ator before the board event's time."""- rec = self.recordings_store.load_recording(session_id)- steps = [s for s in rec.get("steps") or [] if s.get("kind") == "tool" and s.get("name") == "transition"- and (at_ms is None or (s.get("started_at") or 0) <= at_ms + 1000)]- if not steps:- return None- s = steps[-1]- return {"step_id": s["step_id"], "seq": s.get("seq"), "turn_id": s.get("turn_id")}+ # One indexed read, not the whole recording (P3B-5).+ return self.recordings_store.last_tool_step(session_id, "transition",+ before_ms=None if at_ms is None else at_ms + 1000)def _scope_team_tasks(self, team_id: str) -> None:"""A team's task checks: coverage of each handed-off item of a worker that declares@@ -6899,13 +6951,15 @@ class SessionManager:team = self.teams.get(team_id)if team is None:return- from ..teams.summary import all_events-- events = list(all_events(self.team_store, team.space))- sids = [team.lead_session, *(w.session_id for w in team.workers)]+ inputs = self._team_inputs(team)+ with self._team_obs_lock:+ if self._team_checked.get(team_id) == inputs["mark"]:+ return # nothing the checks read has changed since the last run (P3B-5)+ events, sids, items = inputs["board"], inputs["sids"], inputs["items"]+ turns = self.recordings_store.team_turns(sids)found: dict[str, list] = {}- for item in self.team_store.list_items(team.space, self._user_actor()):- cov = self.item_coverage(team, item, events)+ for item in items:+ cov = self.item_coverage(team, item, events, turns=turns)if cov is None or not cov["handed_off"]:continuefor i in cov["issues"]:@@ -6916,7 +6970,7 @@ class SessionManager:if i.get("turn_id"):found.setdefault(i["turn_id"], []).append(i)links = []- for item in self.team_store.list_items(team.space, self._user_actor()):+ for item in items:for link in item.get("links") or []:if link.get("kind") == "blocked_by":links.append((int(link["item"]), int(item["id"])))@@ -6931,8 +6985,12 @@ class SessionManager:if step and step.get("turn_id"):i.update({"step_id": step["step_id"], "step_seq": step["seq"], "turn_id": step["turn_id"]})found.setdefault(step["turn_id"], []).append(i)- turn_ids = [t["turn_id"] for t in self.recordings_store.team_turns(sids)]+ turn_ids = [t["turn_id"] for t in turns]self._store_scope_issues(turn_ids, found, list(_obs_scopes.TASK_CHECKS))+ # Storing the issues bumps the members' write counts: the mark to compare next time is+ # the one after this run's own writes.+ with self._team_obs_lock:+ self._team_checked[team_id] = self._team_inputs(team)["mark"]# -- observability Phase 3: rollups over many turns (E24b, E24c, E26b, E27a) ---------------def usage_names(self) -> dict[str, str]:diff --git a/tests/test_observability_phase3_review_fixes.py b/tests/test_observability_phase3_review_fixes.pynew file mode 100644index 00000000..5827a1b7--- /dev/null+++ b/tests/test_observability_phase3_review_fixes.py@@ -0,0 +1,221 @@+"""Phase 3 code review before merging (4 Nov), one test per fix: the rating export keeps moving+past 100 (P3B-2), the same approval asked twice points at the repeat in both views (P3B-4), a+team's numbers are read once and kept until something changes (P3B-5), a ledger error keeps the+recorded steps (P3B-6), and a resumed automation turn is the run's (P3B-7). P3B-3 (the offline+machine's cached Usage answer) is in test_remote_home.py."""++from __future__ import annotations++import asyncio+import json+from types import SimpleNamespace++from coworker.observability import RecordingStore, rollups, scopes, signals, usage_api+from coworker.observability import export as ex+from coworker.server.manager import SessionManager+from coworker.sessions import SessionRecord+from coworker.teams.model import Actor, Role+from coworker.teams.registry import TeamWorker+++def _turn(tid, sid, seq, at, *, automation=None, status="completed"):+ t = {"turn_id": tid, "session_id": sid, "seq": seq, "kind": "run", "trigger": "user", "started_at": at,+ "duration_ms": 1000, "status": status, "incomplete": False, "approval_mode": "interactive",+ "attended": True, "reviewer": None, "stamp": {"machine": {"kind": "this_machine", "id": None}},+ "first_message_id": None, "last_message_id": None, "schema_version": 1, "wake": {"reason": "user"}}+ if automation:+ t["automation"] = automation+ return t+++def _step(tid, sid, n, at, *, kind="model", name="model", call_id=None, attrs=None):+ return {"step_id": f"{tid}-s{n}", "turn_id": tid, "session_id": sid, "parent_step_id": None, "seq": n,+ "kind": kind, "name": name, "started_at": at, "duration_ms": 100, "status": "ok",+ "call_id": call_id, "call_index": None, "message_id": None, "shadow": False, "after_turn": False,+ "attrs": attrs if attrs is not None else+ {"purpose": "main", "model": "model-x",+ "usage": {"reported": True, "input_fresh": 10, "cache_read": 0, "cache_write": 0, "output": 1}},+ "content": {}}+++def _write(store, turns, steps):+ for t in turns:+ store.write_turn(t)+ for s in steps:+ store.write_step(s)+ store.flush()+++# -- P3B-2 ----------------------------------------------------------------------------------------+def test_ratings_keep_going_out_after_the_first_100(tmp_path):+ store = RecordingStore(tmp_path / "recordings.db", background=False)+ for i in range(130):+ store.put_feedback({"turn_id": f"T{i:03d}", "session_id": "s", "rating": "up", "reasons": [],+ "note": None, "auto_approved": []})+ sent = []++ def post(url, body, headers, timeout):+ sent.append(json.loads(body)["traceId"])+ return 200, ""++ cfg = ex.ExportConfig(destination=ex.Destination(kind="langfuse", host="http://lf.test"), headers={},+ include={}, since_ms=0)+ exp = ex.Exporter(store, cfg, post=post, secret_values=lambda: [])+ exp.flush_once()+ assert len(sent) == 100 # one batch+ exp.flush_once()+ assert len(sent) == 130 and len(set(sent)) == 130 # the next batch is the other 30, not the first 100 again+ exp.flush_once()+ assert len(sent) == 130 # nothing left+ # a rating changed after all that still goes out+ store.put_feedback({"turn_id": "T000", "session_id": "s", "rating": "down", "reasons": [],+ "note": None, "auto_approved": []})+ exp.flush_once()+ assert len(sent) == 131+ store.close()+++# -- P3B-4 ----------------------------------------------------------------------------------------+def _approval(store, tid, sid, seq, at, call_id, args_hash, outcome):+ """A turn with one tool step and its approval card (wait step)."""+ _write(store, [_turn(tid, sid, seq, at)], [+ _step(tid, sid, 1, at, kind="tool", name="run_shell", call_id=call_id, attrs={"args_hash": args_hash}),+ _step(tid, sid, 2, at + 1, kind="wait", name="approval",+ attrs={"kind": "approval", "for_call_id": call_id, "outcome": outcome}),+ ])+++def test_the_same_approval_twice_points_at_the_repeat_in_both_views(tmp_path):+ store = RecordingStore(tmp_path / "recordings.db", background=False)+ _approval(store, "L1", "lead-s", 1, 1_000, "c1", "h-same", "approved")+ # maya's first card is for the same tool with other arguments: not a repeat+ _approval(store, "M1", "maya-s", 1, 2_000, "c2", "h-other", "denied")+ # her second card asks again what the lead already approved: the repeat+ _approval(store, "M2", "maya-s", 2, 3_000, "c3", "h-same", "approved")+ cards = store.team_approval_cards(["lead-s", "maya-s"])+ [issue] = scopes.same_approval_issues(cards)+ assert issue["turn_id"] == "M2" and issue["step_id"] == "M2-s2"+ events = usage_api.team_events(store, lead_session="lead-s", worker_sessions={"maya": "maya-s"},+ board_events=[], links=[])+ [ev] = [e for e in events if e["kind"] == "same_approval_twice"]+ assert ev["turn_id"] == issue["turn_id"] == "M2"+ # the golden check's own answer is unchanged+ assert scopes.same_approval_twice(cards) == [{"tool": "run_shell", "first_by": "lead-s", "again_by": "maya-s"}]+ store.close()+++# -- P3B-6 ----------------------------------------------------------------------------------------+def test_a_ledger_error_keeps_the_recorded_steps(tmp_path, monkeypatch):+ store = RecordingStore(tmp_path / "recordings.db", background=False)+ real = rollups.ledger_rows++ def ledger_rows(turn, *args, **kwargs):+ if turn["turn_id"] == "BAD":+ raise RuntimeError("ledger bug")+ return real(turn, *args, **kwargs)++ monkeypatch.setattr(rollups, "ledger_rows", ledger_rows)+ good, bad = _turn("GOOD", "s1", 1, 1_000), _turn("BAD", "s2", 1, 1_000)+ for t in (good, bad):+ store.write_turn(t)+ store.write_step(_step(t["turn_id"], t["session_id"], 1, 1_000))+ store.write_issues(t["turn_id"], {"signals_version": signals.version(), "warn": 0, "info": 0,+ "issues": [], "fine": []}, seen=True)+ store.flush() # one batch with both turns: the ledger error must not throw it away+ for sid, tid in (("s1", "GOOD"), ("s2", "BAD")):+ rec = store.load_recording(sid)+ assert [t["turn_id"] for t in rec["turns"]] == [tid]+ assert [s["step_id"] for s in rec["steps"]] == [f"{tid}-s1"]+ # the other turn's ledger rows were still worked out+ assert [t["turn_id"] for t in store.usage_turns(0, 2**62)] == ["GOOD"]+ store.close()+++# -- P3B-5 ----------------------------------------------------------------------------------------+def test_a_team_summary_reads_once_and_works_out_again_only_on_a_change(tmp_path, monkeypatch):+ monkeypatch.setenv("COWORKER_STATE_DIR", str(tmp_path / "state"))+ manager = SessionManager(workspace=tmp_path / "repo", data_dir=tmp_path / "data")+ manager.session_store.save(SessionRecord(session_id="lead-session", workspace=str(tmp_path / "repo"),+ model="sample", mode="interactive", agent="swe-lead"))+ team = manager.teams.create(space=manager._board_space("lead-session"), lead_session="lead-session",+ lead_actor="lead",+ workers=[TeamWorker(actor="sam", persona="swe-worker", session_id="worker-session")])+ store = manager.recordings_store+ _write(store, [_turn("W1", "worker-session", 1, 1_000)],+ [_step("W1", "worker-session", 1, 1_000),+ _step("W1", "worker-session", 2, 1_100, kind="tool", name="transition", call_id="t1", attrs={})])++ reads = {"team_turns": 0, "load_recording": 0}+ for name in reads:+ real = getattr(store, name)++ def counted(*a, _real=real, _name=name, **k):+ reads[_name] += 1+ return _real(*a, **k)++ monkeypatch.setattr(store, name, counted)+ # the hand-off step is one indexed read, not the whole recording+ assert manager._handoff_step("worker-session", 2_000) == {"step_id": "W1-s2", "seq": 2, "turn_id": "W1"}+ assert manager._handoff_step("worker-session", 0) is None # before the step+ assert reads["load_recording"] == 0++ first = manager.team_summary(team.team_id)+ assert first["wake_explained"] == {"turns": 0, "explained": 0}+ after_first = reads["team_turns"]+ assert after_first == 1 # once for the whole summary, not once per item and per gate+ again = manager.team_summary(team.team_id)+ assert reads["team_turns"] == after_first # nothing changed: the kept answer+ assert again["wakes"] == first["wakes"] and again["events"] == first["events"]+ # a member's new turn is a change+ _write(store, [_turn("W2", "worker-session", 2, 5_000)], [_step("W2", "worker-session", 1, 5_000)])+ manager.team_summary(team.team_id)+ assert reads["team_turns"] == after_first + 1+ # so is a board change+ manager.team_store.create_item(team.space, Actor(id="lead", role=Role.LEAD), title="New", criteria="Done")+ manager.team_summary(team.team_id)+ assert reads["team_turns"] == after_first + 2+ # the task checks skip a rerun when nothing changed since the last one+ manager._scope_team_tasks(team.team_id)+ n = reads["team_turns"]+ manager._scope_team_tasks(team.team_id)+ assert reads["team_turns"] == n+ manager.observability.close()+++# -- P3B-7 ----------------------------------------------------------------------------------------+class _Resumable:+ def __init__(self):+ self.kwargs = None++ async def resume(self, **kwargs):+ self.kwargs = kwargs+ if False:+ yield None+++def _resume(manager, session_id):+ engine = _Resumable()+ manager.get_engine = lambda sid, *a, **k: engine+ manager.save = lambda *a, **k: None+ asyncio.run(manager._durable_resume(SimpleNamespace(id=None, session_id=session_id, tool_call_id="c1",+ resolved_by="")))+ return engine.kwargs+++def test_a_resumed_automation_turn_is_tagged_with_its_run(tmp_path, monkeypatch):+ monkeypatch.setenv("COWORKER_STATE_DIR", str(tmp_path / "state"))+ manager = SessionManager(workspace=tmp_path / "repo", data_dir=tmp_path / "data")+ store = manager.recordings_store+ run = {"task_id": "task-1", "run_id": "run-1", "trigger": "schedule"}+ # the run's own turn stopped at an approval; the app restarted; the card is answered+ _write(store, [_turn("R1", "__run__run-1", 1, 1_000, automation=run, status="unknown")],+ [_step("R1", "__run__run-1", 1, 1_000)])+ got = _resume(manager, "__run__run-1")+ assert got["automation"] == run and got["wake"]["reason"] == "answer"+ # a follow-up typed into the run's session later is not the run's, nor is its resume (TD3-23)+ _write(store, [_turn("R2", "__run__run-1", 2, 9_000, status="unknown")], [_step("R2", "__run__run-1", 1, 9_000)])+ assert _resume(manager, "__run__run-1")["automation"] is None+ # an ordinary session's resume carries none+ _write(store, [_turn("S1", "plain", 1, 1_000, automation=run)], [_step("S1", "plain", 1, 1_000)])+ assert _resume(manager, "plain")["automation"] is None+ manager.observability.close()diff --git a/tests/test_remote_home.py b/tests/test_remote_home.pyindex 21458814..82210179 100644--- a/tests/test_remote_home.py+++ b/tests/test_remote_home.py@@ -2503,7 +2503,7 @@ async def test_browser_sealed_managed_grant_lands_on_the_box(tmp_path):async def test_offline_machine_serves_its_last_usage_rollup_as_of(tmp_path):"""Observability Phase 3 (E28, TD3-6): the desktop keeps each machine's last Usage rollup- per grouping; while the machine is offline the proxy answers with it, labelled `cached`+ per request; while the machine is offline the proxy answers the same request with it, labelled `cached`with the time it was fetched, never as live numbers."""async with _Server(_controller(tmp_path)) as server:join_url = await _arm(server.base)@@ -2540,6 +2540,18 @@ async def test_offline_machine_serves_its_last_usage_rollup_as_of(tmp_path):none = await client.get(f"{server.base}/v1/machines/{mid}/p/v1/observability/rollup",params={"group": "day"})assert none.status_code == 503+ # P3B-3: the same grouping for another period or another coworker is another+ # question; the copy for "from 0" never stands in for it+ other_period = await client.get(f"{server.base}/v1/machines/{mid}/p/v1/observability/rollup",+ params={"group": "all", "from": 1000})+ assert other_period.status_code == 503+ other_coworker = await client.get(f"{server.base}/v1/machines/{mid}/p/v1/observability/rollup",+ params={"group": "all", "from": 0, "persona_id": "p-x"})+ assert other_coworker.status_code == 503+ # the same question with its parameters in another order still finds its copy+ reordered = await client.get(f"{server.base}/v1/machines/{mid}/p/v1/observability/rollup",+ params={"from": 0, "group": "all"})+ assert reordered.status_code == 200 and reordered.json()["cached"]["as_of"] > 0async def test_a_joined_machine_stamps_its_turns_with_the_controllers_id(tmp_path, monkeypatch):