Python
feat(observability): wait steps, summary steps and each tool call's decision path (OPE-244)
Devika Verma5eed698c651c
diff --git a/coworker/engine.py b/coworker/engine.pyindex c5d0d028..149ede99 100644--- a/coworker/engine.py+++ b/coworker/engine.py@@ -33,6 +33,7 @@ from . import toolchain as _toolchainfrom . import toolresultfrom . import unattended as _attendancefrom .events import Event, EventType+from .observability.decision import DecisionPathfrom .observability.engine_hooks import TurnRecordingfrom .observability.provider_view import tag as _tag_provider, view_for_enginefrom .observability.ids import new_ulid@@ -238,6 +239,54 @@ def _records_turn(kind: str) -> Callable[[Callable[..., AsyncIterator[Event]]],return deco+# `_wait_tool` kind by tool (observability §5.4); any other wait is an approval card.+_WAIT_KINDS = {+ "ask_user": "question",+ "propose_plan": "plan",+ "propose_team": "team",+ "propose_work_items": "items",+ "request_connector": "connector",+ "grant_connector": "connector",+ "request_tool": "tool_install",+ "request_directory": "directory",+}+++def _reviewer_skip(engine: "TurnEngine", decision: Any) -> str:+ """Why the auto-approver was not consulted on a call that needed a person (§8.2)."""+ if decision.human_only:+ return "skipped_human_only"+ if engine.reviewer is not None and engine._reviewer_denials >= _REVIEWER_TRIP:+ return "skipped_paused"+ try:+ attended = bool(engine.is_attended and engine.is_attended())+ except Exception: # noqa: BLE001+ attended = False+ if engine.reviewer is not None and engine.reviewer_enabled and not attended:+ return "skipped_unattended"+ return "skipped_off"+++def _wait_outcome(result: Any, interrupted: Any, kind: str, *, stopped: bool) -> str:+ """A wait step's `outcome`: approved, denied, answered or interrupted (§5.4). An approval+ card's "interrupted" value is the same DENY a person can click, so Stop is told apart by+ whether it was pressed."""+ if result is interrupted and stopped:+ return "interrupted"+ if isinstance(result, ApprovalOutcome):+ if result is ApprovalOutcome.DENY:+ return "denied"+ return "answered" if result is ApprovalOutcome.SUPERSEDED else "approved"+ if isinstance(result, dict):+ if result.get("error") == "interrupted by user":+ return "interrupted"+ if kind == "question":+ return "answered" if result.get("answer") or result.get("answers") else "denied"+ ok = result.get("approved") or result.get("granted") or result.get("installed")+ return "approved" if ok else "denied"+ return "answered" if result else "denied"++def _action_failed(action: dict[str, Any]) -> bool:"""A listed action that changed nothing: the call errored, the command exitednon-zero, or it timed out. A background start is neither done nor failed."""@@ -489,6 +538,13 @@ class TurnEngine:# it makes is recorded as a model step of the current turn, purpose `main`.self.provider = view_for_engine(self.provider, self._obs)self._call_index: dict[str, int] = {}+ # Who answered a waiting call's card (observability §5.4): set by the server, which+ # reads it from the Inbox.+ self.resolver_of: Optional[Callable[[str], str]] = None+ # Each tool call's decision path, built in `_authorize`, attached to its tool step+ # (observability E4, §8.3); and where its card was shown.+ self._decisions: dict[str, DecisionPath] = {}+ self._card_where: dict[str, str] = {}if instructions and not (self.messages and self.messages[0].get("role") == "system"):@@ -538,12 +594,51 @@ class TurnEngine:finally:cancel_wait.cancel()- async def _wait_tool(self, tool_call, coro, interrupted):+ async def _wait_tool(self, tool_call, coro, interrupted, kind: Optional[str] = None):started, clock = time.time(), time.monotonic()+ # Observability E20: a wait step from the moment the card is shown to the answer, so+ # a slow answer is never mistaken for a slow agent (§5.4).+ kind = kind or _WAIT_KINDS.get(tool_call.name, "approval")try:- return await self._interruptible(coro, interrupted)+ attended = bool(self.is_attended and self.is_attended())+ except Exception: # noqa: BLE001 - a broken getter must not break the wait+ attended = False+ step_id = self._obs.start_step(+ kind="wait", name=kind, call_id=tool_call.id,+ attrs={"for_call_id": tool_call.id, "kind": kind, "attended": attended,+ "where": "session" if attended else "inbox"},+ )+ self._obs.track_wait(tool_call.id, step_id)+ result: Any = interrupted+ failed = False+ try:+ result = await self._interruptible(coro, interrupted)+ return result+ except BaseException:+ failed = True+ raisefinally:self._tool_timings.setdefault(tool_call.id, {}).update(waited_started=started, waited_ms=(time.monotonic() - clock) * 1000)+ self._obs.end_wait(tool_call.id)+ if step_id is not None:+ who = ""+ if self.resolver_of is not None:+ try:+ who = str(self.resolver_of(tool_call.id) or "")+ except Exception: # noqa: BLE001+ who = ""+ self._obs.end_step(+ step_id, status="error" if failed else "ok",+ attrs={"outcome": "interrupted" if failed else _wait_outcome(+ result, interrupted, kind, stopped=self._cancel.is_set())},+ content={"resolved_by": who} if who else {},+ )++ def note_wait_where(self, call_id: Optional[str], where: str) -> None:+ """Where a waiting call's card is shown, `session` or `inbox` (observability §5.4)."""+ if call_id:+ self._card_where[call_id] = where+ self._obs.wait_where(call_id, where)@propertydef recorder(self) -> Any:@@ -606,9 +701,13 @@ class TurnEngine:if m.get("role") == "tool" and m.get("tool_call_id") == call_id),None,)+ attrs: dict[str, Any] = {"risk_class": self._risk_class(str(event.data.get("name") or ""))}+ path = self._decisions.pop(call_id, None)+ if path is not None and obs.step_for_call(call_id) is None:+ attrs["decision"] = path.finish(ran=event.data.get("status") == "ok")obs.tool_finished(event.data, message_id=mid, call_index=self._call_index.get(call_id),- attrs={"risk_class": self._risk_class(str(event.data.get("name") or ""))},+ attrs=attrs,)step_id = obs.step_for_call(call_id)if step_id:@@ -1248,11 +1347,38 @@ class TurnEngine:cfg.get("summary_max_tokens") or _compaction.SUMMARY_MAX_TOKENS)model = str(cfg.get("model") or "") or self.model+ # Observability E4: a summary step, written only if a summary or trim happens (§5.7);+ # its model calls are nested under it.+ summary_step = self._obs.reserve_step()+ tokens_before = self._last_context_tokens+ try:+ notice = await self._compact_with(summary_step, force=force, cfg=cfg, trigger=trigger,+ keep=keep, user_budget=user_budget,+ summary_max=summary_max, model=model)+ except BaseException:+ self._obs.drop_reserved(summary_step)+ raise+ children = self._obs.model_children(summary_step)+ if notice is not None or children:+ trimmed = notice is not None and notice.startswith("Context trimmed")+ self._obs.finish_reserved(+ summary_step, kind="summary", name="trim" if trimmed else "summary",+ status="ok" if notice is not None else "error",+ attrs={"reason": "overflow" if force else "threshold", "tokens_before": tokens_before,+ "model_step_id": None if trimmed or not children else children[-1]},+ wait_for_tokens_after=notice is not None,+ )+ else:+ self._obs.drop_reserved(summary_step)+ return notice+ async def _compact_with(self, summary_step: Optional[str], *, force: bool, cfg: dict[str, Any],+ trigger: int, keep: int, user_budget: int, summary_max: int,+ model: str) -> Optional[str]:def _build() -> Optional[_compaction.CompactionState]:return _compaction.build_state(self.messages,- provider=_tag_provider(self.provider, purpose="summary"),+ provider=_tag_provider(self.provider, purpose="summary", parent=summary_step),model=model,keep_tokens=keep,prior=self.compaction_state,@@ -1271,6 +1397,11 @@ class TurnEngine:failed = Trueif failed and self.question_asker is not None and self.is_attended and self.is_attended():while True:+ # The "Retry or trim?" question is a wait, nested under the summary (E20).+ wait_step = self._obs.start_step(+ kind="wait", name="question", parent_step_id=summary_step,+ attrs={"for_call_id": None, "kind": "question", "attended": True, "where": "session"},+ )answer = await self._interruptible(self.question_asker({@@ -1286,6 +1417,7 @@ class TurnEngine:),interrupted=None,)+ self._obs.end_step(wait_step, attrs={"outcome": "answered" if answer else "interrupted"})if not answer or answer.get("answer") != "Retry":breaktry:@@ -1999,6 +2131,12 @@ class TurnEngine:human_only=True,)+ path = self._decisions[tool_call.id] = DecisionPath()+ path.permission(+ allowed=allowed, needs_user=decision.needs_user, human_only=decision.human_only,+ reason=reason, tier=decision.tier if allowed else None,+ )+# OPE-233: an Auto-Approve gate tier let this call run without the reviewer —# recorded, never invisible (transcript chip + audit row), like every other# cardless origin.@@ -2075,6 +2213,7 @@ class TurnEngine:# deny card. One-shot — consumed above; a different action never matches.allowed = Truereason = "approved by user (allow anyway)"+ path.add("grant", "allow_anyway")self._audit(tool_call, stage="auto_allowed", status="allowed", reason=reason)# A lead DENYING one of its workers' waiting calls runs nothing: the worker is told@@ -2092,6 +2231,7 @@ class TurnEngine:):allowed = Truereason = "a lead's denial of a worker's call runs nothing"+ path.add("grant", "lead_denial")self._audit(tool_call, stage="auto_allowed", status="allowed", reason=reason)consulted_live = False@@ -2114,10 +2254,13 @@ class TurnEngine:consulted_reviewer = self.reviewersettings_epoch = self.reviewer_settings_epochverdict = await self._consult_reviewer(tool_call)- if (self.reviewer is not consulted_reviewer- or self.reviewer_settings_epoch != settings_epoch- or not self._reviewer_active()):+ settings_changed = (self.reviewer is not consulted_reviewer+ or self.reviewer_settings_epoch != settings_epoch+ or not self._reviewer_active())+ if settings_changed:verdict = _settings_changed(verdict)+ path.add("auto_approver", "context_changed" if settings_changed+ else "could_not_run" if getattr(verdict, "error", False) else verdict.verdict)self._audit(tool_call,stage="reviewer_verdict",@@ -2184,6 +2327,10 @@ class TurnEngine:if not card_note:unsure_silent_kind = "reviewer_unavailable" if verdict.error else "reviewer_asks"+ if (not allowed and decision.needs_user and not consulted_live+ and self.permissions.mode is Mode.AUTO_APPROVE):+ path.add("auto_approver", _reviewer_skip(self, decision))+if not allowed and decision.needs_user and self._nobody_answers_cards():# Attendance "auto": nobody can answer a card, and the mode did not allow# the call — so the answer is no, recorded, never a hang. (The dangerous mode@@ -2203,6 +2350,7 @@ class TurnEngine:reason=reason,)decision = replace(decision, needs_user=False)+ path.add("card", "auto_refused")if not allowed and decision.needs_user:escalation = (@@ -2286,7 +2434,7 @@ class TurnEngine:call_id=tool_call.id,)outcome = await self._wait_tool(tool_call,- self.approver(+ kind="approval", coro=self.approver(PermissionRequest(tool_name=tool_call.name,arguments=tool_call.arguments,@@ -2304,6 +2452,11 @@ class TurnEngine:),interrupted=ApprovalOutcome.DENY,)+ path.add("card", "parked_inbox" if self._card_where.pop(tool_call.id, "") == "inbox" else "shown_session")+ path.resolved(+ outcome.value if isinstance(outcome, ApprovalOutcome) else str(outcome),+ interrupted=outcome is ApprovalOutcome.DENY and self._cancel.is_set(),+ )if outcome is ApprovalOutcome.SUPERSEDED:result = {"skipped": True,@@ -2466,6 +2619,9 @@ class TurnEngine:def _tool_step_attrs(self, tool_call: ToolCall, result: Any) -> dict[str, Any]:attrs: dict[str, Any] = {"risk_class": self._risk_class(tool_call.name)}+ path = self._decisions.pop(tool_call.id, None)+ if path is not None:+ attrs["decision"] = path.finish(ran=True)try:attrs["result_bytes"] = len(result.encode("utf-8") if isinstance(result, str)diff --git a/coworker/inbox.py b/coworker/inbox.pyindex 39a4cb0b..7938a934 100644--- a/coworker/inbox.py+++ b/coworker/inbox.py@@ -27,7 +27,7 @@ import uuidfrom dataclasses import asdict, dataclass, fieldfrom datetime import datetime, timezonefrom pathlib import Path-from typing import Any, Optional+from typing import Any, Callable, OptionalKIND_APPROVAL = "approval"KIND_QUESTION = "question"@@ -114,8 +114,19 @@ class InboxStore:self._items: dict[str, InboxItem] = {}self._waiters: dict[str, asyncio.Event] = {}self._waiter_loops: dict[str, asyncio.AbstractEventLoop] = {}+ # Told of every new item and every item moved to the Inbox (observability: where a+ # waiting call's card is shown). Best-effort: its failures never touch the store.+ self.on_change: Optional[Callable[[InboxItem], None]] = Noneself._load()+ def _changed(self, item: "InboxItem") -> None:+ if self.on_change is None:+ return+ try:+ self.on_change(item)+ except Exception: # noqa: BLE001+ pass+# -- persistence ------------------------------------------------------------def _load(self) -> None:if self.path and self.path.is_file():@@ -180,6 +191,7 @@ class InboxStore:if source is not None and source.state == STATE_RESOLVED:self._supersede_locked(item, source)self._save()+ self._changed(item)return itemdef for_tool_call(self, session_id: str, tool_call_id: str) -> Optional[InboxItem]:@@ -476,6 +488,8 @@ class InboxStore:changed.append(item)if changed:self._save()+ for item in changed:+ self._changed(item)return changed# -- resume reconciliation --------------------------------------------------diff --git a/coworker/observability/decision.py b/coworker/observability/decision.pynew file mode 100644index 00000000..35d741fd--- /dev/null+++ b/coworker/observability/decision.py@@ -0,0 +1,125 @@+"""The decision path of a tool call: how it was allowed or refused, stage by stage+(observability E4, technical design §8.3).++`_authorize` in the engine records each stage it passes through a `DecisionPath`; the+tool step gets `attrs.decision = {path, origin}`. Every value is an id from a fixed list;+rule or reason text never goes here.++Stages, in order: `permission`, `grant`, `auto_approver`, `card`, `resolved`. `origin` names+the single stage that decided.+"""++from __future__ import annotations++from typing import Any, Optional++# Permission engine allow reasons -> grant ids (§8.3). Matched as prefixes.+_GRANTS = (+ ("command on allowlist", "config_allowlist"),+ ("domain on allowlist", "config_allowlist"),+ ("auto-allowed by config", "config_allowlist"),+ ("site on the allowed sites", "config_allowlist"),+ ("already on the allowed sites", "config_allowlist"),+ ("command allowed for session", "session_grant"),+ ("read-only command (session grant)", "session_grant"),+ ("tool allowed for session", "session_grant"),+ ("tool allowed for this request", "run_grant"),+ ("trusted MCP tool (user trust rule)", "trusted_rule"),+ ("trusted MCP tool", "trusted_server"),+ ("allowed by standing rule", "standing_rule"),+)++# Outcomes of an approval card -> the `resolved` stage (§8.3).+_RESOLVED = {+ "once": ("approved_once", None),+ "always_tool": ("approved_session", "always_tool"),+ "always_command": ("approved_session", "always_command"),+ "always_domain": ("approved_session", "always_domain"),+ "always_site": ("approved_session", "always_site"),+ "readonly_session": ("approved_session", "readonly_session"),+ "this_run": ("approved_run", None),+ "always_trust": ("approved_always", None),+ "deny": ("denied", None),+ "superseded": ("superseded", None),+}++# A grant stage that decided a call that ran is its origin.+_GRANT_ORIGINS = {"config_allowlist", "session_grant", "standing_rule", "trusted_rule",+ "trusted_server", "run_grant", "allow_anyway", "lead_denial",+ "readonly_tier", "gate_tier"}+++class DecisionPath:+ def __init__(self) -> None:+ self.path: list[dict[str, Any]] = []++ def add(self, stage: str, outcome: str, detail: Optional[str] = None) -> None:+ entry: dict[str, Any] = {"stage": stage, "outcome": outcome}+ if detail:+ entry["detail"] = detail+ self.path.append(entry)++ def permission(self, *, allowed: bool, needs_user: bool, human_only: bool, reason: str,+ tier: Optional[str] = None) -> None:+ """The permission engine's answer, and the grant behind an allow."""+ if allowed:+ if tier:+ # An Auto-Approve gate tier (OPE-233/234) cleared it without the reviewer.+ self.add("permission", "allowed")+ self.add("grant", "readonly_tier" if tier == "readonly_command" else "gate_tier")+ return+ if reason == "low risk":+ self.add("permission", "allowed", "low_risk")+ elif reason == "full access":+ self.add("permission", "allowed", "bypass")+ elif reason.startswith("cleared by mode"):+ self.add("permission", "allowed", "cleared_by_mode")+ else:+ grant = next((g for prefix, g in _GRANTS if reason.startswith(prefix)), None)+ self.add("permission", "allowed")+ if grant:+ self.add("grant", grant)+ return+ if needs_user:+ self.add("permission", "human_only" if human_only else "needs_user")+ else:+ self.add("permission", "denied")++ def resolved(self, outcome_value: str, *, interrupted: bool = False) -> None:+ if interrupted:+ self.add("resolved", "interrupted")+ return+ outcome, detail = _RESOLVED.get(outcome_value, ("approved_once", None))+ self.add("resolved", outcome, detail)++ def stage(self, name: str) -> Optional[dict[str, Any]]:+ return next((s for s in reversed(self.path) if s["stage"] == name), None)++ def finish(self, *, ran: bool) -> dict[str, Any]:+ """`{path, origin}` for the tool step."""+ return {"path": list(self.path), "origin": self._origin(ran)}++ def _origin(self, ran: bool) -> Optional[str]:+ resolved, reviewer = self.stage("resolved"), self.stage("auto_approver")+ card, grant, perm = self.stage("card"), self.stage("grant"), self.stage("permission")+ if ran:+ if resolved and resolved["outcome"].startswith("approved"):+ return "user"+ if reviewer and reviewer["outcome"] == "allow":+ return "reviewer"+ if grant and grant["outcome"] in _GRANT_ORIGINS:+ return grant["outcome"]+ if perm and perm.get("detail") in ("low_risk", "bypass"):+ return perm["detail"]+ if perm and perm.get("detail") == "cleared_by_mode":+ return "bypass"+ return None+ if reviewer and reviewer["outcome"] == "deny":+ return "reviewer_denied"+ if card and card["outcome"] == "auto_refused":+ return "unattended_auto"+ if resolved:+ return "user"+ if perm and perm["outcome"] == "denied":+ return "denied_by_rule"+ return Nonediff --git a/coworker/observability/engine_hooks.py b/coworker/observability/engine_hooks.pyindex 7836fafd..d4cc9469 100644--- a/coworker/observability/engine_hooks.py+++ b/coworker/observability/engine_hooks.py@@ -48,6 +48,9 @@ class TurnRecording:self._tool_steps: dict[str, str] = {} # call_id -> step_id, for this turn's toolsself._tool_ends: dict[str, dict[str, Any]] = {} # call_id -> step_ended awaiting its midself._reply_end: Optional[dict[str, Any]] = None # a main model step awaiting its reply's mid+ self._waits: dict[str, str] = {} # call_id -> its open wait step (E20)+ self._reserved: dict[str, dict[str, Any]] = {} # summary steps written at their end (E4)+ self._tokens_after_for: Optional[str] = None # a summary waiting for the next main call@propertydef enabled(self) -> bool:@@ -89,6 +92,7 @@ class TurnRecording:self._tool_steps.clear()self._tool_ends.clear()self._reply_end = None+ self._waits.clear()ev = self._base("turn_started", self.turn_id, self._turn_session)ev.update({"kind": kind, "trigger": trigger, "first_message_id": first_message_id,"attrs": copy.deepcopy(attrs), "stamp": copy.deepcopy(stamp)})@@ -182,9 +186,97 @@ class TurnRecording:}if kind == "tool" and call_id:self._tool_steps[call_id] = step_id+ reserved = self._reserved.get(parent_step_id or "")+ if reserved is not None and kind == "model":+ reserved["children"].append(step_id)self._send(ev)return step_id+ def update_step(self, step_id: Optional[str], *, attrs: Optional[dict[str, Any]] = None,+ content: Optional[dict[str, Any]] = None) -> None:+ """`step_updated`: a fact that arrives after the step started."""+ if not self.enabled or not step_id:+ return+ with self._lock:+ idn = self._ident.get(step_id) or {}+ ev = self._base("step_updated", idn.get("turn_id", self.turn_id), idn.get("session_id", self._turn_session))+ ev["step_id"] = step_id+ if attrs is not None:+ ev["attrs"] = attrs+ if content is not None:+ ev["content"] = content+ self._send(ev)++ # -- wait steps (E20) -----------------------------------------------------------------------+ def track_wait(self, call_id: str, step_id: Optional[str]) -> None:+ if step_id:+ with self._lock:+ self._waits[call_id] = step_id++ def end_wait(self, call_id: str) -> Optional[str]:+ with self._lock:+ return self._waits.pop(call_id, None)++ def wait_where(self, call_id: Optional[str], where: str) -> None:+ """Where the card for a waiting call is shown: `session` or `inbox` (§5.4). Reported+ by the Inbox when the card is added or moved there."""+ if not call_id:+ return+ with self._lock:+ step_id = self._waits.get(call_id)+ self.update_step(step_id, attrs={"where": where})++ # -- summary steps (E4) ---------------------------------------------------------------------+ def reserve_step(self) -> Optional[str]:+ """An id for a step written only at its end, from its identity copy (§3.2): a summary+ step exists only if a summary or trim happened, which is known only at the end. Model+ calls made inside it name it as their parent."""+ if not self.enabled:+ return None+ with self._lock:+ step_id = new_ulid()+ tid = self.turn_id if self.in_turn else None+ self._reserved[step_id] = {+ "at_ms": now_ms()[0], "turn_id": tid,+ "session_id": self._turn_session if tid else (self._session_id() or ""),+ "children": [],+ }+ return step_id++ def model_children(self, step_id: Optional[str]) -> list[str]:+ with self._lock:+ return list((self._reserved.get(step_id or "") or {}).get("children", []))++ def finish_reserved(self, step_id: Optional[str], *, kind: str, name: str, status: str,+ attrs: dict[str, Any], wait_for_tokens_after: bool = False) -> None:+ """Write a reserved step whole, from its end alone; or drop it with `status=None`."""+ if not self.enabled or not step_id:+ return+ with self._lock:+ r = self._reserved.pop(step_id, None)+ if r is None:+ return+ ev = self._base("step_ended", r["turn_id"], r["session_id"])+ ev.update({"step_id": step_id, "status": status, "message_id": None, "attrs": attrs,+ "content": {}, "kind": kind, "name": name, "parent_step_id": None,+ "call_id": None, "call_index": None, "shadow": False, "after_turn": False,+ "started_at": r["at_ms"]})+ self._send(ev)+ if wait_for_tokens_after:+ self._tokens_after_for = step_id++ def drop_reserved(self, step_id: Optional[str]) -> None:+ with self._lock:+ self._reserved.pop(step_id or "", None)++ def note_main_usage(self, usage: dict[str, Any]) -> None:+ """The first main call after a summary gives its `tokens_after` (§5.7)."""+ with self._lock:+ step_id, self._tokens_after_for = self._tokens_after_for, None+ if step_id and usage.get("reported"):+ after = (usage.get("input_fresh") or 0) + (usage.get("cache_read") or 0) + (usage.get("cache_write") or 0)+ self.update_step(step_id, attrs={"tokens_after": after})+def _started_at(self, ev: dict[str, Any]) -> int:# The recorder anchors a step's start to its turn's start (§7.3); the identity copy# uses the wall clock, which is what a recorder without the turn would use.diff --git a/coworker/observability/provider_view.py b/coworker/observability/provider_view.pyindex 62772c71..b2626bb7 100644--- a/coworker/observability/provider_view.py+++ b/coworker/observability/provider_view.py@@ -305,6 +305,8 @@ class _ModelCall:self.step_id, status="ok", attrs=attrs, content=content,wait_for_reply_message=self.tags.purpose == "main",)+ if self.tags.purpose == "main":+ self.sink.note_main_usage(attrs["usage"])except Exception: # noqa: BLE001log.exception("recording: could not end a model step")diff --git a/coworker/observability/store.py b/coworker/observability/store.pyindex b279d12f..ae19c55e 100644--- a/coworker/observability/store.py+++ b/coworker/observability/store.py@@ -10,7 +10,8 @@ of recording-format.md; `load_recording()` gives it back.Writing never makes a turn wait. `write_turn` and `write_step` only put the record in apending map keyed by its id, so a step written three times before the writer runs is written-once, with its latest state; one background thread writes the map in a single transaction+once, with its latest state; one background thread, shared by every store in the process,+writes the map in a single transactionevery `FLUSH_MS`. Nothing is dropped, and no queue can grow past the number of records thatchanged since the last write."""@@ -22,6 +23,7 @@ import loggingimport sqlite3import threadingimport time+import weakrefimport zlibfrom hashlib import sha256from pathlib import Path@@ -175,15 +177,11 @@ class RecordingStore:self._pending_turns: dict[str, dict[str, Any]] = {}self._pending_steps: dict[str, dict[str, Any]] = {}self._pending_lock = threading.Lock()- self._wake = threading.Event()- self._closed = threading.Event()+ self._closed = Falseself._last_sweep = 0.0- self._thread: Optional[threading.Thread] = None+ self._background = backgroundif background:- self._thread = threading.Thread(- target=self._run, name="recordings-writer", daemon=True- )- self._thread.start()+ _writer.add(self)@classmethoddef beside(cls, coworker_db_dir: str | Path, **kw: Any) -> "RecordingStore":@@ -194,12 +192,14 @@ class RecordingStore:def write_turn(self, record: dict[str, Any]) -> None:with self._pending_lock:self._pending_turns[record["turn_id"]] = record- self._wake.set()+ if self._background:+ _writer.wake()def write_step(self, record: dict[str, Any]) -> None:with self._pending_lock:self._pending_steps[record["step_id"]] = record- self._wake.set()+ if self._background:+ _writer.wake()def flush(self) -> None:"""Write everything pending now, on the calling thread."""@@ -225,32 +225,25 @@ class RecordingStore:except Exception:log.exception("recordings: write failed (%d turns, %d steps)", len(turns), len(steps))- def _run(self) -> None:- while not self._closed.is_set():- # Asleep until there is something to write (or a day passes, for the sweep).- self._wake.wait(timeout=SWEEP_EVERY_S)- self._wake.clear()- self.flush()- if (- self.retention_days is not None- and time.time() - self._last_sweep >= SWEEP_EVERY_S- ):- self._last_sweep = time.time()- try:- self.sweep()- except Exception:- log.exception("recordings: retention sweep failed")- self._closed.wait(FLUSH_MS / 1000)+ def _tick(self) -> None:+ """One turn of the shared writer: write what is pending, and sweep once a day."""+ if self._closed:+ returnself.flush()+ if self.retention_days is not None and time.time() - self._last_sweep >= SWEEP_EVERY_S:+ self._last_sweep = time.time()+ try:+ self.sweep()+ except Exception:+ log.exception("recordings: retention sweep failed")def close(self) -> None:- self._closed.set()- self._wake.set()- if self._thread is not None:- self._thread.join(timeout=5)- else:- self.flush()+ _writer.remove(self)with self._lock:+ if self._closed:+ return+ self.flush()+ self._closed = Trueself._conn.close()# -- exchange pieces (TD-17) --------------------------------------------------------------@@ -389,6 +382,51 @@ class RecordingStore:return {"turns": [_turn_record(t) for t in turns], "steps": [_step_record(s) for s in steps]}+class _SharedWriter:+ """One background thread for every open store in the process. It sleeps until a store+ has something to write (or an hour passes, to check the daily sweep), then writes each+ store's pending records in one transaction. Stores are held weakly, so a store nobody+ uses any more is not kept alive by the writer (tests make thousands of them)."""++ def __init__(self) -> None:+ self._stores: "weakref.WeakSet[RecordingStore]" = weakref.WeakSet()+ self._lock = threading.Lock()+ self._wake = threading.Event()+ self._thread: Optional[threading.Thread] = None++ def add(self, store: "RecordingStore") -> None:+ with self._lock:+ self._stores.add(store)+ if self._thread is None or not self._thread.is_alive():+ self._thread = threading.Thread(target=self._run, name="recordings-writer", daemon=True)+ self._thread.start()+ self._wake.set() # a new store's first sweep runs soon++ def remove(self, store: "RecordingStore") -> None:+ with self._lock:+ self._stores.discard(store)++ def wake(self) -> None:+ self._wake.set()++ def _run(self) -> None:+ while True:+ self._wake.wait(timeout=3600)+ self._wake.clear()+ with self._lock:+ stores = list(self._stores)+ for store in stores:+ try:+ store._tick()+ except Exception:+ log.exception("recordings: writer failed")+ del stores+ time.sleep(FLUSH_MS / 1000) # gather the next burst of writes into one transaction+++_writer = _SharedWriter()++def _turn_record(r: sqlite3.Row) -> dict[str, Any]:d = {k: r[k] for k in _TURN_COLUMNS}d["incomplete"] = bool(d["incomplete"])diff --git a/coworker/server/manager.py b/coworker/server/manager.pyindex 24ae59a3..6a271bc1 100644--- a/coworker/server/manager.py+++ b/coworker/server/manager.py@@ -505,6 +505,7 @@ class SessionManager:# Inbox (cross-session human-attention queue), routing (named inboxes + Slack/Telegram# bindings), the Unattended toggle, and self-wake records.self.inbox = InboxStore(base / "inbox.json")+ self.inbox.on_change = self._inbox_item_changedself.inbox_routing = InboxRouting(base / "inbox_routing.json")self.unattended = UnattendedRegistry(base / "unattended.json")self.wakes = WakeStore(base / "wakes.json")@@ -1013,6 +1014,7 @@ class SessionManager:)# Observability (E1): this engine's turns and steps go to the server's recorder.engine.recorder = self.recorder+ engine.resolver_of = lambda call_id, _sid=session_id: self.inbox.resolver_of(_sid, call_id)engine.delegated_approval = lambda args: self.worker_review_context(session_id, args)engine.reviewer_context = lambda: self.approval_context(session_id)engine.reviewer_owner_history = lambda: self.approval_owner_history(session_id, engine)@@ -6306,6 +6308,14 @@ class SessionManager:def has_session_clients(self, session_id: str) -> bool:return bool(self._session_clients.get(session_id))+ def _inbox_item_changed(self, item: Any) -> None:+ """A card was added or moved to the Inbox: tell the waiting engine where it is shown+ (observability §5.4, a wait step's `where`)."""+ engine = self._engines.get(item.session_id)+ note = getattr(engine, "note_wait_where", None)+ if note is not None and item.tool_call_id:+ note(item.tool_call_id, "inbox" if item.visibility == VIS_INBOX else "session")+async def promote_pending_prompts(self, session_id: str) -> int:"""The last viewer left while the agent waits on an inline prompt: move it to theInbox and mirror it, so the question is visible somewhere (see@@ -6969,6 +6979,7 @@ class SessionManager:)# Observability (E1): this engine's turns and steps go to the server's recorder.engine.recorder = self.recorder+ engine.resolver_of = lambda call_id, _sid=session_id: self.inbox.resolver_of(_sid, call_id)self._seed_task_permissions(engine, task)return enginediff --git a/tests/test_observability_store.py b/tests/test_observability_store.pyindex 54eefebd..eaad44c0 100644--- a/tests/test_observability_store.py+++ b/tests/test_observability_store.py@@ -273,3 +273,13 @@ def test_the_step_clock_ticks_every_millisecond():while (tick := now_ms()[1]) == first and time.perf_counter() < deadline:passassert tick - first == 1+++def test_every_store_shares_one_writer_thread(tmp_path):+ stores = [RecordingStore(tmp_path / f"r{i}.db") for i in range(20)]+ try:+ writers = [t for t in threading.enumerate() if t.name == "recordings-writer"]+ assert len(writers) == 1+ finally:+ for s in stores:+ s.close()diff --git a/tests/test_observability_waits.py b/tests/test_observability_waits.pynew file mode 100644index 00000000..c997ab5b--- /dev/null+++ b/tests/test_observability_waits.py@@ -0,0 +1,210 @@+"""Observability E20 and E4: wait steps, decision paths and summary steps.++- E20: an approval card is a wait step from the moment it is shown to the answer, linked to+ its tool call; its outcome and who answered are recorded.+- E4: every tool call that went through the permission flow carries its decision path+ (`attrs.decision = {path, origin}`); a summary or trim is a summary step with its model+ call nested inside it.+"""++from __future__ import annotations++import asyncio+from types import SimpleNamespace++from coworker import compaction as _compaction+from coworker.engine import ApprovalOutcome, TurnEngine+from coworker.observability import Recorder+from coworker.permissions import Mode, PermissionEngine+from coworker.providers import AssistantTurn, ModelCapabilities, ProviderClient, ToolCall+from coworker.providers.base import TokenUsage+from coworker.tools import ToolRegistry+++class _Meta:+ category: str = ""+ risk_level: str = "high"+ requires_approval: bool = False+++class _ReadMeta:+ category: str = ""+ risk_level: str = "low"+ requires_approval: bool = False+++class _Scripted(ProviderClient):+ def __init__(self, turns):+ self._turns = list(turns)++ def complete(self, *, model, messages, tools=None, **settings):+ return self._turns.pop(0)++ def capabilities(self, model):+ return ModelCapabilities()+++def run_shell(command: str) -> dict:+ """Run a command.++ Args:+ command: the command line+ """+ return {"command": command, "exit_code": 0, "output": ""}+++def look(path: str) -> dict:+ """Read something.++ Args:+ path: what to read+ """+ return {"path": path, "text": "ok"}+++def _engine(tmp_path, turns, *, answer=ApprovalOutcome.ONCE, mode=Mode.INTERACTIVE):+ registry = ToolRegistry()+ registry.register(run_shell, metadata=_Meta())+ registry.register(look, metadata=_ReadMeta())++ async def approver(request):+ return answer++ engine = TurnEngine(+ provider=_Scripted(turns),+ registry=registry,+ permissions=PermissionEngine(workspace_root=tmp_path, mode=mode),+ model="openai:model-a",+ approver=approver,+ )+ engine.audit_context = {"session_id": "sess_w"}+ engine.is_attended = lambda: True+ engine.resolver_of = lambda call_id: "devika"+ rec = Recorder()+ engine.recorder = rec+ return engine, rec+++def _call(cid, name, args):+ return AssistantTurn(tool_calls=[ToolCall(id=cid, name=name, arguments=args)], finish_reason="tool_calls")+++def _run(engine, text="go"):+ async def go():+ return [ev async for ev in engine.run(text, trigger="user")]++ return asyncio.run(go())+++def _steps(rec, kind):+ return [s for s in rec.as_dict()["steps"] if s["kind"] == kind]+++def test_an_approved_card_is_a_wait_step_and_the_call_records_its_path(tmp_path):+ engine, rec = _engine(tmp_path, [_call("c1", "run_shell", {"command": "make build"}),+ AssistantTurn(text="built")])+ _run(engine)+ [wait] = _steps(rec, "wait")+ assert (wait["name"], wait["call_id"], wait["status"]) == ("approval", "c1", "ok")+ assert wait["attrs"] == {"for_call_id": "c1", "kind": "approval", "attended": True,+ "where": "session", "outcome": "approved"}+ assert wait["content"] == {"resolved_by": "devika"}+ [tool] = _steps(rec, "tool")+ assert tool["status"] == "ok"+ assert tool["attrs"]["decision"] == {+ "path": [{"stage": "permission", "outcome": "needs_user"},+ {"stage": "card", "outcome": "shown_session"},+ {"stage": "resolved", "outcome": "approved_once"}],+ "origin": "user",+ }+ # The wait comes before the tool in the canonical order.+ kinds = [s["kind"] for s in rec.as_dict()["steps"]]+ assert kinds.index("wait") < kinds.index("tool")+++def test_a_denied_card_is_a_zero_length_denied_step_refused_by_the_user(tmp_path):+ engine, rec = _engine(tmp_path, [_call("c1", "run_shell", {"command": "rm -rf build"}),+ AssistantTurn(text="ok, not doing it")],+ answer=ApprovalOutcome.DENY)+ _run(engine)+ [wait] = _steps(rec, "wait")+ assert wait["attrs"]["outcome"] == "denied"+ [tool] = _steps(rec, "tool")+ assert (tool["status"], tool["duration_ms"]) == ("denied", 0)+ assert tool["attrs"]["decision"]["origin"] == "user"+ assert tool["attrs"]["decision"]["path"][-1] == {"stage": "resolved", "outcome": "denied"}+++def test_a_session_grant_is_recorded_as_its_origin(tmp_path):+ engine, rec = _engine(tmp_path, [_call("c1", "run_shell", {"command": "pytest"}),+ _call("c2", "run_shell", {"command": "pytest"}),+ AssistantTurn(text="done")],+ answer=ApprovalOutcome.ALWAYS_COMMAND)+ _run(engine)+ first, second = _steps(rec, "tool")+ assert first["attrs"]["decision"]["path"][-1] == {+ "stage": "resolved", "outcome": "approved_session", "detail": "always_command"}+ assert second["attrs"]["decision"] == {+ "path": [{"stage": "permission", "outcome": "allowed"},+ {"stage": "grant", "outcome": "session_grant"}],+ "origin": "session_grant",+ }+ assert len(_steps(rec, "wait")) == 1 # no card the second time+++def test_a_low_risk_read_needs_no_one(tmp_path):+ engine, rec = _engine(tmp_path, [_call("c1", "look", {"path": "x"}), AssistantTurn(text="ok")])+ _run(engine)+ [tool] = _steps(rec, "tool")+ assert tool["attrs"]["decision"] == {+ "path": [{"stage": "permission", "outcome": "allowed", "detail": "low_risk"}],+ "origin": "low_risk",+ }+ assert _steps(rec, "wait") == []+++def test_full_access_is_bypass(tmp_path):+ engine, rec = _engine(tmp_path, [_call("c1", "run_shell", {"command": "make"}), AssistantTurn(text="ok")],+ mode=Mode.BYPASS_APPROVALS)+ _run(engine)+ [tool] = _steps(rec, "tool")+ assert tool["attrs"]["decision"]["origin"] == "bypass"+++def test_a_summary_is_a_step_with_its_model_call_inside(tmp_path, monkeypatch):+ engine, rec = _engine(tmp_path, [+ AssistantTurn(text="a short summary", usage=TokenUsage(input=900, output=40)),+ AssistantTurn(text="next", usage=TokenUsage(input=300, output=5, cache_read=200)),+ ])+ engine._last_context_tokens = 47000++ def fake_build_state(messages, *, provider, model, **kw):+ provider.complete(model=model, messages=[{"role": "user", "content": "summarize"}])+ return SimpleNamespace(boundary_index=0, carries_outside=False, transcript_path=None)++ monkeypatch.setattr(_compaction, "build_state", fake_build_state)++ async def go():+ return await engine._compact_now()++ assert asyncio.run(go()) is not None+ engine.provider.complete(model=engine.model, messages=[]) # the next main call+ [summary] = _steps(rec, "summary")+ [model] = [s for s in rec.as_dict()["steps"] if s["kind"] == "model" and s["attrs"]["purpose"] == "summary"]+ assert (summary["name"], summary["status"]) == ("summary", "ok")+ assert model["parent_step_id"] == summary["step_id"]+ assert summary["attrs"]["model_step_id"] == model["step_id"]+ assert summary["attrs"]["reason"] == "threshold"+ assert summary["attrs"]["tokens_before"] == 47000+ assert summary["attrs"]["tokens_after"] == 500+++def test_no_summary_step_when_nothing_was_summarised(tmp_path, monkeypatch):+ engine, rec = _engine(tmp_path, [])+ monkeypatch.setattr(_compaction, "build_state", lambda messages, **kw: None)++ async def go():+ return await engine._compact_now()++ assert asyncio.run(go()) is None+ assert _steps(rec, "summary") == []