Skip to content
Python

feat(observability): wait steps, summary steps and each tool call's decision path (OPE-244)

Devika Verma5eed698c651c
diff --git a/coworker/engine.py b/coworker/engine.py
index c5d0d028..149ede99 100644
--- a/coworker/engine.py
+++ b/coworker/engine.py
@@ -33,6 +33,7 @@ from . import toolchain as _toolchain
from . import toolresult
from . import unattended as _attendance
from .events import Event, EventType
+from .observability.decision import DecisionPath
from .observability.engine_hooks import TurnRecording
from .observability.provider_view import tag as _tag_provider, view_for_engine
from .observability.ids import new_ulid
@@ -238,6 +239,54 @@ def _records_turn(kind: str) -> Callable[[Callable[..., AsyncIterator[Event]]],
return deco
+# `_wait_tool` kind by tool (observability §5.4); any other wait is an approval card.
+_WAIT_KINDS = {
+ "ask_user": "question",
+ "propose_plan": "plan",
+ "propose_team": "team",
+ "propose_work_items": "items",
+ "request_connector": "connector",
+ "grant_connector": "connector",
+ "request_tool": "tool_install",
+ "request_directory": "directory",
+}
+
+
+def _reviewer_skip(engine: "TurnEngine", decision: Any) -> str:
+ """Why the auto-approver was not consulted on a call that needed a person (§8.2)."""
+ if decision.human_only:
+ return "skipped_human_only"
+ if engine.reviewer is not None and engine._reviewer_denials >= _REVIEWER_TRIP:
+ return "skipped_paused"
+ try:
+ attended = bool(engine.is_attended and engine.is_attended())
+ except Exception: # noqa: BLE001
+ attended = False
+ if engine.reviewer is not None and engine.reviewer_enabled and not attended:
+ return "skipped_unattended"
+ return "skipped_off"
+
+
+def _wait_outcome(result: Any, interrupted: Any, kind: str, *, stopped: bool) -> str:
+ """A wait step's `outcome`: approved, denied, answered or interrupted (§5.4). An approval
+ card's "interrupted" value is the same DENY a person can click, so Stop is told apart by
+ whether it was pressed."""
+ if result is interrupted and stopped:
+ return "interrupted"
+ if isinstance(result, ApprovalOutcome):
+ if result is ApprovalOutcome.DENY:
+ return "denied"
+ return "answered" if result is ApprovalOutcome.SUPERSEDED else "approved"
+ if isinstance(result, dict):
+ if result.get("error") == "interrupted by user":
+ return "interrupted"
+ if kind == "question":
+ return "answered" if result.get("answer") or result.get("answers") else "denied"
+ ok = result.get("approved") or result.get("granted") or result.get("installed")
+ return "approved" if ok else "denied"
+ return "answered" if result else "denied"
+
+
def _action_failed(action: dict[str, Any]) -> bool:
"""A listed action that changed nothing: the call errored, the command exited
non-zero, or it timed out. A background start is neither done nor failed."""
@@ -489,6 +538,13 @@ class TurnEngine:
# it makes is recorded as a model step of the current turn, purpose `main`.
self.provider = view_for_engine(self.provider, self._obs)
self._call_index: dict[str, int] = {}
+ # Who answered a waiting call's card (observability §5.4): set by the server, which
+ # reads it from the Inbox.
+ self.resolver_of: Optional[Callable[[str], str]] = None
+ # Each tool call's decision path, built in `_authorize`, attached to its tool step
+ # (observability E4, §8.3); and where its card was shown.
+ self._decisions: dict[str, DecisionPath] = {}
+ self._card_where: dict[str, str] = {}
if instructions and not (
self.messages and self.messages[0].get("role") == "system"
):
@@ -538,12 +594,51 @@ class TurnEngine:
finally:
cancel_wait.cancel()
- async def _wait_tool(self, tool_call, coro, interrupted):
+ async def _wait_tool(self, tool_call, coro, interrupted, kind: Optional[str] = None):
started, clock = time.time(), time.monotonic()
+ # Observability E20: a wait step from the moment the card is shown to the answer, so
+ # a slow answer is never mistaken for a slow agent (§5.4).
+ kind = kind or _WAIT_KINDS.get(tool_call.name, "approval")
try:
- return await self._interruptible(coro, interrupted)
+ attended = bool(self.is_attended and self.is_attended())
+ except Exception: # noqa: BLE001 - a broken getter must not break the wait
+ attended = False
+ step_id = self._obs.start_step(
+ kind="wait", name=kind, call_id=tool_call.id,
+ attrs={"for_call_id": tool_call.id, "kind": kind, "attended": attended,
+ "where": "session" if attended else "inbox"},
+ )
+ self._obs.track_wait(tool_call.id, step_id)
+ result: Any = interrupted
+ failed = False
+ try:
+ result = await self._interruptible(coro, interrupted)
+ return result
+ except BaseException:
+ failed = True
+ raise
finally:
self._tool_timings.setdefault(tool_call.id, {}).update(waited_started=started, waited_ms=(time.monotonic() - clock) * 1000)
+ self._obs.end_wait(tool_call.id)
+ if step_id is not None:
+ who = ""
+ if self.resolver_of is not None:
+ try:
+ who = str(self.resolver_of(tool_call.id) or "")
+ except Exception: # noqa: BLE001
+ who = ""
+ self._obs.end_step(
+ step_id, status="error" if failed else "ok",
+ attrs={"outcome": "interrupted" if failed else _wait_outcome(
+ result, interrupted, kind, stopped=self._cancel.is_set())},
+ content={"resolved_by": who} if who else {},
+ )
+
+ def note_wait_where(self, call_id: Optional[str], where: str) -> None:
+ """Where a waiting call's card is shown, `session` or `inbox` (observability §5.4)."""
+ if call_id:
+ self._card_where[call_id] = where
+ self._obs.wait_where(call_id, where)
@property
def recorder(self) -> Any:
@@ -606,9 +701,13 @@ class TurnEngine:
if m.get("role") == "tool" and m.get("tool_call_id") == call_id),
None,
)
+ attrs: dict[str, Any] = {"risk_class": self._risk_class(str(event.data.get("name") or ""))}
+ path = self._decisions.pop(call_id, None)
+ if path is not None and obs.step_for_call(call_id) is None:
+ attrs["decision"] = path.finish(ran=event.data.get("status") == "ok")
obs.tool_finished(
event.data, message_id=mid, call_index=self._call_index.get(call_id),
- attrs={"risk_class": self._risk_class(str(event.data.get("name") or ""))},
+ attrs=attrs,
)
step_id = obs.step_for_call(call_id)
if step_id:
@@ -1248,11 +1347,38 @@ class TurnEngine:
cfg.get("summary_max_tokens") or _compaction.SUMMARY_MAX_TOKENS
)
model = str(cfg.get("model") or "") or self.model
+ # Observability E4: a summary step, written only if a summary or trim happens (§5.7);
+ # its model calls are nested under it.
+ summary_step = self._obs.reserve_step()
+ tokens_before = self._last_context_tokens
+ try:
+ notice = await self._compact_with(summary_step, force=force, cfg=cfg, trigger=trigger,
+ keep=keep, user_budget=user_budget,
+ summary_max=summary_max, model=model)
+ except BaseException:
+ self._obs.drop_reserved(summary_step)
+ raise
+ children = self._obs.model_children(summary_step)
+ if notice is not None or children:
+ trimmed = notice is not None and notice.startswith("Context trimmed")
+ self._obs.finish_reserved(
+ summary_step, kind="summary", name="trim" if trimmed else "summary",
+ status="ok" if notice is not None else "error",
+ attrs={"reason": "overflow" if force else "threshold", "tokens_before": tokens_before,
+ "model_step_id": None if trimmed or not children else children[-1]},
+ wait_for_tokens_after=notice is not None,
+ )
+ else:
+ self._obs.drop_reserved(summary_step)
+ return notice
+ async def _compact_with(self, summary_step: Optional[str], *, force: bool, cfg: dict[str, Any],
+ trigger: int, keep: int, user_budget: int, summary_max: int,
+ model: str) -> Optional[str]:
def _build() -> Optional[_compaction.CompactionState]:
return _compaction.build_state(
self.messages,
- provider=_tag_provider(self.provider, purpose="summary"),
+ provider=_tag_provider(self.provider, purpose="summary", parent=summary_step),
model=model,
keep_tokens=keep,
prior=self.compaction_state,
@@ -1271,6 +1397,11 @@ class TurnEngine:
failed = True
if failed and self.question_asker is not None and self.is_attended and self.is_attended():
while True:
+ # The "Retry or trim?" question is a wait, nested under the summary (E20).
+ wait_step = self._obs.start_step(
+ kind="wait", name="question", parent_step_id=summary_step,
+ attrs={"for_call_id": None, "kind": "question", "attended": True, "where": "session"},
+ )
answer = await self._interruptible(
self.question_asker(
{
@@ -1286,6 +1417,7 @@ class TurnEngine:
),
interrupted=None,
)
+ self._obs.end_step(wait_step, attrs={"outcome": "answered" if answer else "interrupted"})
if not answer or answer.get("answer") != "Retry":
break
try:
@@ -1999,6 +2131,12 @@ class TurnEngine:
human_only=True,
)
+ path = self._decisions[tool_call.id] = DecisionPath()
+ path.permission(
+ allowed=allowed, needs_user=decision.needs_user, human_only=decision.human_only,
+ reason=reason, tier=decision.tier if allowed else None,
+ )
+
# OPE-233: an Auto-Approve gate tier let this call run without the reviewer —
# recorded, never invisible (transcript chip + audit row), like every other
# cardless origin.
@@ -2075,6 +2213,7 @@ class TurnEngine:
# deny card. One-shot — consumed above; a different action never matches.
allowed = True
reason = "approved by user (allow anyway)"
+ path.add("grant", "allow_anyway")
self._audit(tool_call, stage="auto_allowed", status="allowed", reason=reason)
# A lead DENYING one of its workers' waiting calls runs nothing: the worker is told
@@ -2092,6 +2231,7 @@ class TurnEngine:
):
allowed = True
reason = "a lead's denial of a worker's call runs nothing"
+ path.add("grant", "lead_denial")
self._audit(tool_call, stage="auto_allowed", status="allowed", reason=reason)
consulted_live = False
@@ -2114,10 +2254,13 @@ class TurnEngine:
consulted_reviewer = self.reviewer
settings_epoch = self.reviewer_settings_epoch
verdict = await self._consult_reviewer(tool_call)
- if (self.reviewer is not consulted_reviewer
- or self.reviewer_settings_epoch != settings_epoch
- or not self._reviewer_active()):
+ settings_changed = (self.reviewer is not consulted_reviewer
+ or self.reviewer_settings_epoch != settings_epoch
+ or not self._reviewer_active())
+ if settings_changed:
verdict = _settings_changed(verdict)
+ path.add("auto_approver", "context_changed" if settings_changed
+ else "could_not_run" if getattr(verdict, "error", False) else verdict.verdict)
self._audit(
tool_call,
stage="reviewer_verdict",
@@ -2184,6 +2327,10 @@ class TurnEngine:
if not card_note:
unsure_silent_kind = "reviewer_unavailable" if verdict.error else "reviewer_asks"
+ if (not allowed and decision.needs_user and not consulted_live
+ and self.permissions.mode is Mode.AUTO_APPROVE):
+ path.add("auto_approver", _reviewer_skip(self, decision))
+
if not allowed and decision.needs_user and self._nobody_answers_cards():
# Attendance "auto": nobody can answer a card, and the mode did not allow
# the call — so the answer is no, recorded, never a hang. (The dangerous mode
@@ -2203,6 +2350,7 @@ class TurnEngine:
reason=reason,
)
decision = replace(decision, needs_user=False)
+ path.add("card", "auto_refused")
if not allowed and decision.needs_user:
escalation = (
@@ -2286,7 +2434,7 @@ class TurnEngine:
call_id=tool_call.id,
)
outcome = await self._wait_tool(tool_call,
- self.approver(
+ kind="approval", coro=self.approver(
PermissionRequest(
tool_name=tool_call.name,
arguments=tool_call.arguments,
@@ -2304,6 +2452,11 @@ class TurnEngine:
),
interrupted=ApprovalOutcome.DENY,
)
+ path.add("card", "parked_inbox" if self._card_where.pop(tool_call.id, "") == "inbox" else "shown_session")
+ path.resolved(
+ outcome.value if isinstance(outcome, ApprovalOutcome) else str(outcome),
+ interrupted=outcome is ApprovalOutcome.DENY and self._cancel.is_set(),
+ )
if outcome is ApprovalOutcome.SUPERSEDED:
result = {
"skipped": True,
@@ -2466,6 +2619,9 @@ class TurnEngine:
def _tool_step_attrs(self, tool_call: ToolCall, result: Any) -> dict[str, Any]:
attrs: dict[str, Any] = {"risk_class": self._risk_class(tool_call.name)}
+ path = self._decisions.pop(tool_call.id, None)
+ if path is not None:
+ attrs["decision"] = path.finish(ran=True)
try:
attrs["result_bytes"] = len(
result.encode("utf-8") if isinstance(result, str)
diff --git a/coworker/inbox.py b/coworker/inbox.py
index 39a4cb0b..7938a934 100644
--- a/coworker/inbox.py
+++ b/coworker/inbox.py
@@ -27,7 +27,7 @@ import uuid
from dataclasses import asdict, dataclass, field
from datetime import datetime, timezone
from pathlib import Path
-from typing import Any, Optional
+from typing import Any, Callable, Optional
KIND_APPROVAL = "approval"
KIND_QUESTION = "question"
@@ -114,8 +114,19 @@ class InboxStore:
self._items: dict[str, InboxItem] = {}
self._waiters: dict[str, asyncio.Event] = {}
self._waiter_loops: dict[str, asyncio.AbstractEventLoop] = {}
+ # Told of every new item and every item moved to the Inbox (observability: where a
+ # waiting call's card is shown). Best-effort: its failures never touch the store.
+ self.on_change: Optional[Callable[[InboxItem], None]] = None
self._load()
+ def _changed(self, item: "InboxItem") -> None:
+ if self.on_change is None:
+ return
+ try:
+ self.on_change(item)
+ except Exception: # noqa: BLE001
+ pass
+
# -- persistence ------------------------------------------------------------
def _load(self) -> None:
if self.path and self.path.is_file():
@@ -180,6 +191,7 @@ class InboxStore:
if source is not None and source.state == STATE_RESOLVED:
self._supersede_locked(item, source)
self._save()
+ self._changed(item)
return item
def for_tool_call(self, session_id: str, tool_call_id: str) -> Optional[InboxItem]:
@@ -476,6 +488,8 @@ class InboxStore:
changed.append(item)
if changed:
self._save()
+ for item in changed:
+ self._changed(item)
return changed
# -- resume reconciliation --------------------------------------------------
diff --git a/coworker/observability/decision.py b/coworker/observability/decision.py
new file mode 100644
index 00000000..35d741fd
--- /dev/null
+++ b/coworker/observability/decision.py
@@ -0,0 +1,125 @@
+"""The decision path of a tool call: how it was allowed or refused, stage by stage
+(observability E4, technical design §8.3).
+
+`_authorize` in the engine records each stage it passes through a `DecisionPath`; the
+tool step gets `attrs.decision = {path, origin}`. Every value is an id from a fixed list;
+rule or reason text never goes here.
+
+Stages, in order: `permission`, `grant`, `auto_approver`, `card`, `resolved`. `origin` names
+the single stage that decided.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Optional
+
+# Permission engine allow reasons -> grant ids (§8.3). Matched as prefixes.
+_GRANTS = (
+ ("command on allowlist", "config_allowlist"),
+ ("domain on allowlist", "config_allowlist"),
+ ("auto-allowed by config", "config_allowlist"),
+ ("site on the allowed sites", "config_allowlist"),
+ ("already on the allowed sites", "config_allowlist"),
+ ("command allowed for session", "session_grant"),
+ ("read-only command (session grant)", "session_grant"),
+ ("tool allowed for session", "session_grant"),
+ ("tool allowed for this request", "run_grant"),
+ ("trusted MCP tool (user trust rule)", "trusted_rule"),
+ ("trusted MCP tool", "trusted_server"),
+ ("allowed by standing rule", "standing_rule"),
+)
+
+# Outcomes of an approval card -> the `resolved` stage (§8.3).
+_RESOLVED = {
+ "once": ("approved_once", None),
+ "always_tool": ("approved_session", "always_tool"),
+ "always_command": ("approved_session", "always_command"),
+ "always_domain": ("approved_session", "always_domain"),
+ "always_site": ("approved_session", "always_site"),
+ "readonly_session": ("approved_session", "readonly_session"),
+ "this_run": ("approved_run", None),
+ "always_trust": ("approved_always", None),
+ "deny": ("denied", None),
+ "superseded": ("superseded", None),
+}
+
+# A grant stage that decided a call that ran is its origin.
+_GRANT_ORIGINS = {"config_allowlist", "session_grant", "standing_rule", "trusted_rule",
+ "trusted_server", "run_grant", "allow_anyway", "lead_denial",
+ "readonly_tier", "gate_tier"}
+
+
+class DecisionPath:
+ def __init__(self) -> None:
+ self.path: list[dict[str, Any]] = []
+
+ def add(self, stage: str, outcome: str, detail: Optional[str] = None) -> None:
+ entry: dict[str, Any] = {"stage": stage, "outcome": outcome}
+ if detail:
+ entry["detail"] = detail
+ self.path.append(entry)
+
+ def permission(self, *, allowed: bool, needs_user: bool, human_only: bool, reason: str,
+ tier: Optional[str] = None) -> None:
+ """The permission engine's answer, and the grant behind an allow."""
+ if allowed:
+ if tier:
+ # An Auto-Approve gate tier (OPE-233/234) cleared it without the reviewer.
+ self.add("permission", "allowed")
+ self.add("grant", "readonly_tier" if tier == "readonly_command" else "gate_tier")
+ return
+ if reason == "low risk":
+ self.add("permission", "allowed", "low_risk")
+ elif reason == "full access":
+ self.add("permission", "allowed", "bypass")
+ elif reason.startswith("cleared by mode"):
+ self.add("permission", "allowed", "cleared_by_mode")
+ else:
+ grant = next((g for prefix, g in _GRANTS if reason.startswith(prefix)), None)
+ self.add("permission", "allowed")
+ if grant:
+ self.add("grant", grant)
+ return
+ if needs_user:
+ self.add("permission", "human_only" if human_only else "needs_user")
+ else:
+ self.add("permission", "denied")
+
+ def resolved(self, outcome_value: str, *, interrupted: bool = False) -> None:
+ if interrupted:
+ self.add("resolved", "interrupted")
+ return
+ outcome, detail = _RESOLVED.get(outcome_value, ("approved_once", None))
+ self.add("resolved", outcome, detail)
+
+ def stage(self, name: str) -> Optional[dict[str, Any]]:
+ return next((s for s in reversed(self.path) if s["stage"] == name), None)
+
+ def finish(self, *, ran: bool) -> dict[str, Any]:
+ """`{path, origin}` for the tool step."""
+ return {"path": list(self.path), "origin": self._origin(ran)}
+
+ def _origin(self, ran: bool) -> Optional[str]:
+ resolved, reviewer = self.stage("resolved"), self.stage("auto_approver")
+ card, grant, perm = self.stage("card"), self.stage("grant"), self.stage("permission")
+ if ran:
+ if resolved and resolved["outcome"].startswith("approved"):
+ return "user"
+ if reviewer and reviewer["outcome"] == "allow":
+ return "reviewer"
+ if grant and grant["outcome"] in _GRANT_ORIGINS:
+ return grant["outcome"]
+ if perm and perm.get("detail") in ("low_risk", "bypass"):
+ return perm["detail"]
+ if perm and perm.get("detail") == "cleared_by_mode":
+ return "bypass"
+ return None
+ if reviewer and reviewer["outcome"] == "deny":
+ return "reviewer_denied"
+ if card and card["outcome"] == "auto_refused":
+ return "unattended_auto"
+ if resolved:
+ return "user"
+ if perm and perm["outcome"] == "denied":
+ return "denied_by_rule"
+ return None
diff --git a/coworker/observability/engine_hooks.py b/coworker/observability/engine_hooks.py
index 7836fafd..d4cc9469 100644
--- a/coworker/observability/engine_hooks.py
+++ b/coworker/observability/engine_hooks.py
@@ -48,6 +48,9 @@ class TurnRecording:
self._tool_steps: dict[str, str] = {} # call_id -> step_id, for this turn's tools
self._tool_ends: dict[str, dict[str, Any]] = {} # call_id -> step_ended awaiting its mid
self._reply_end: Optional[dict[str, Any]] = None # a main model step awaiting its reply's mid
+ self._waits: dict[str, str] = {} # call_id -> its open wait step (E20)
+ self._reserved: dict[str, dict[str, Any]] = {} # summary steps written at their end (E4)
+ self._tokens_after_for: Optional[str] = None # a summary waiting for the next main call
@property
def enabled(self) -> bool:
@@ -89,6 +92,7 @@ class TurnRecording:
self._tool_steps.clear()
self._tool_ends.clear()
self._reply_end = None
+ self._waits.clear()
ev = self._base("turn_started", self.turn_id, self._turn_session)
ev.update({"kind": kind, "trigger": trigger, "first_message_id": first_message_id,
"attrs": copy.deepcopy(attrs), "stamp": copy.deepcopy(stamp)})
@@ -182,9 +186,97 @@ class TurnRecording:
}
if kind == "tool" and call_id:
self._tool_steps[call_id] = step_id
+ reserved = self._reserved.get(parent_step_id or "")
+ if reserved is not None and kind == "model":
+ reserved["children"].append(step_id)
self._send(ev)
return step_id
+ def update_step(self, step_id: Optional[str], *, attrs: Optional[dict[str, Any]] = None,
+ content: Optional[dict[str, Any]] = None) -> None:
+ """`step_updated`: a fact that arrives after the step started."""
+ if not self.enabled or not step_id:
+ return
+ with self._lock:
+ idn = self._ident.get(step_id) or {}
+ ev = self._base("step_updated", idn.get("turn_id", self.turn_id), idn.get("session_id", self._turn_session))
+ ev["step_id"] = step_id
+ if attrs is not None:
+ ev["attrs"] = attrs
+ if content is not None:
+ ev["content"] = content
+ self._send(ev)
+
+ # -- wait steps (E20) -----------------------------------------------------------------------
+ def track_wait(self, call_id: str, step_id: Optional[str]) -> None:
+ if step_id:
+ with self._lock:
+ self._waits[call_id] = step_id
+
+ def end_wait(self, call_id: str) -> Optional[str]:
+ with self._lock:
+ return self._waits.pop(call_id, None)
+
+ def wait_where(self, call_id: Optional[str], where: str) -> None:
+ """Where the card for a waiting call is shown: `session` or `inbox` (§5.4). Reported
+ by the Inbox when the card is added or moved there."""
+ if not call_id:
+ return
+ with self._lock:
+ step_id = self._waits.get(call_id)
+ self.update_step(step_id, attrs={"where": where})
+
+ # -- summary steps (E4) ---------------------------------------------------------------------
+ def reserve_step(self) -> Optional[str]:
+ """An id for a step written only at its end, from its identity copy (§3.2): a summary
+ step exists only if a summary or trim happened, which is known only at the end. Model
+ calls made inside it name it as their parent."""
+ if not self.enabled:
+ return None
+ with self._lock:
+ step_id = new_ulid()
+ tid = self.turn_id if self.in_turn else None
+ self._reserved[step_id] = {
+ "at_ms": now_ms()[0], "turn_id": tid,
+ "session_id": self._turn_session if tid else (self._session_id() or ""),
+ "children": [],
+ }
+ return step_id
+
+ def model_children(self, step_id: Optional[str]) -> list[str]:
+ with self._lock:
+ return list((self._reserved.get(step_id or "") or {}).get("children", []))
+
+ def finish_reserved(self, step_id: Optional[str], *, kind: str, name: str, status: str,
+ attrs: dict[str, Any], wait_for_tokens_after: bool = False) -> None:
+ """Write a reserved step whole, from its end alone; or drop it with `status=None`."""
+ if not self.enabled or not step_id:
+ return
+ with self._lock:
+ r = self._reserved.pop(step_id, None)
+ if r is None:
+ return
+ ev = self._base("step_ended", r["turn_id"], r["session_id"])
+ ev.update({"step_id": step_id, "status": status, "message_id": None, "attrs": attrs,
+ "content": {}, "kind": kind, "name": name, "parent_step_id": None,
+ "call_id": None, "call_index": None, "shadow": False, "after_turn": False,
+ "started_at": r["at_ms"]})
+ self._send(ev)
+ if wait_for_tokens_after:
+ self._tokens_after_for = step_id
+
+ def drop_reserved(self, step_id: Optional[str]) -> None:
+ with self._lock:
+ self._reserved.pop(step_id or "", None)
+
+ def note_main_usage(self, usage: dict[str, Any]) -> None:
+ """The first main call after a summary gives its `tokens_after` (§5.7)."""
+ with self._lock:
+ step_id, self._tokens_after_for = self._tokens_after_for, None
+ if step_id and usage.get("reported"):
+ after = (usage.get("input_fresh") or 0) + (usage.get("cache_read") or 0) + (usage.get("cache_write") or 0)
+ self.update_step(step_id, attrs={"tokens_after": after})
+
def _started_at(self, ev: dict[str, Any]) -> int:
# The recorder anchors a step's start to its turn's start (§7.3); the identity copy
# uses the wall clock, which is what a recorder without the turn would use.
diff --git a/coworker/observability/provider_view.py b/coworker/observability/provider_view.py
index 62772c71..b2626bb7 100644
--- a/coworker/observability/provider_view.py
+++ b/coworker/observability/provider_view.py
@@ -305,6 +305,8 @@ class _ModelCall:
self.step_id, status="ok", attrs=attrs, content=content,
wait_for_reply_message=self.tags.purpose == "main",
)
+ if self.tags.purpose == "main":
+ self.sink.note_main_usage(attrs["usage"])
except Exception: # noqa: BLE001
log.exception("recording: could not end a model step")
diff --git a/coworker/observability/store.py b/coworker/observability/store.py
index b279d12f..ae19c55e 100644
--- a/coworker/observability/store.py
+++ b/coworker/observability/store.py
@@ -10,7 +10,8 @@ of recording-format.md; `load_recording()` gives it back.
Writing never makes a turn wait. `write_turn` and `write_step` only put the record in a
pending map keyed by its id, so a step written three times before the writer runs is written
-once, with its latest state; one background thread writes the map in a single transaction
+once, with its latest state; one background thread, shared by every store in the process,
+writes the map in a single transaction
every `FLUSH_MS`. Nothing is dropped, and no queue can grow past the number of records that
changed since the last write.
"""
@@ -22,6 +23,7 @@ import logging
import sqlite3
import threading
import time
+import weakref
import zlib
from hashlib import sha256
from pathlib import Path
@@ -175,15 +177,11 @@ class RecordingStore:
self._pending_turns: dict[str, dict[str, Any]] = {}
self._pending_steps: dict[str, dict[str, Any]] = {}
self._pending_lock = threading.Lock()
- self._wake = threading.Event()
- self._closed = threading.Event()
+ self._closed = False
self._last_sweep = 0.0
- self._thread: Optional[threading.Thread] = None
+ self._background = background
if background:
- self._thread = threading.Thread(
- target=self._run, name="recordings-writer", daemon=True
- )
- self._thread.start()
+ _writer.add(self)
@classmethod
def beside(cls, coworker_db_dir: str | Path, **kw: Any) -> "RecordingStore":
@@ -194,12 +192,14 @@ class RecordingStore:
def write_turn(self, record: dict[str, Any]) -> None:
with self._pending_lock:
self._pending_turns[record["turn_id"]] = record
- self._wake.set()
+ if self._background:
+ _writer.wake()
def write_step(self, record: dict[str, Any]) -> None:
with self._pending_lock:
self._pending_steps[record["step_id"]] = record
- self._wake.set()
+ if self._background:
+ _writer.wake()
def flush(self) -> None:
"""Write everything pending now, on the calling thread."""
@@ -225,32 +225,25 @@ class RecordingStore:
except Exception:
log.exception("recordings: write failed (%d turns, %d steps)", len(turns), len(steps))
- def _run(self) -> None:
- while not self._closed.is_set():
- # Asleep until there is something to write (or a day passes, for the sweep).
- self._wake.wait(timeout=SWEEP_EVERY_S)
- self._wake.clear()
- self.flush()
- if (
- self.retention_days is not None
- and time.time() - self._last_sweep >= SWEEP_EVERY_S
- ):
- self._last_sweep = time.time()
- try:
- self.sweep()
- except Exception:
- log.exception("recordings: retention sweep failed")
- self._closed.wait(FLUSH_MS / 1000)
+ def _tick(self) -> None:
+ """One turn of the shared writer: write what is pending, and sweep once a day."""
+ if self._closed:
+ return
self.flush()
+ if self.retention_days is not None and time.time() - self._last_sweep >= SWEEP_EVERY_S:
+ self._last_sweep = time.time()
+ try:
+ self.sweep()
+ except Exception:
+ log.exception("recordings: retention sweep failed")
def close(self) -> None:
- self._closed.set()
- self._wake.set()
- if self._thread is not None:
- self._thread.join(timeout=5)
- else:
- self.flush()
+ _writer.remove(self)
with self._lock:
+ if self._closed:
+ return
+ self.flush()
+ self._closed = True
self._conn.close()
# -- exchange pieces (TD-17) --------------------------------------------------------------
@@ -389,6 +382,51 @@ class RecordingStore:
return {"turns": [_turn_record(t) for t in turns], "steps": [_step_record(s) for s in steps]}
+class _SharedWriter:
+ """One background thread for every open store in the process. It sleeps until a store
+ has something to write (or an hour passes, to check the daily sweep), then writes each
+ store's pending records in one transaction. Stores are held weakly, so a store nobody
+ uses any more is not kept alive by the writer (tests make thousands of them)."""
+
+ def __init__(self) -> None:
+ self._stores: "weakref.WeakSet[RecordingStore]" = weakref.WeakSet()
+ self._lock = threading.Lock()
+ self._wake = threading.Event()
+ self._thread: Optional[threading.Thread] = None
+
+ def add(self, store: "RecordingStore") -> None:
+ with self._lock:
+ self._stores.add(store)
+ if self._thread is None or not self._thread.is_alive():
+ self._thread = threading.Thread(target=self._run, name="recordings-writer", daemon=True)
+ self._thread.start()
+ self._wake.set() # a new store's first sweep runs soon
+
+ def remove(self, store: "RecordingStore") -> None:
+ with self._lock:
+ self._stores.discard(store)
+
+ def wake(self) -> None:
+ self._wake.set()
+
+ def _run(self) -> None:
+ while True:
+ self._wake.wait(timeout=3600)
+ self._wake.clear()
+ with self._lock:
+ stores = list(self._stores)
+ for store in stores:
+ try:
+ store._tick()
+ except Exception:
+ log.exception("recordings: writer failed")
+ del stores
+ time.sleep(FLUSH_MS / 1000) # gather the next burst of writes into one transaction
+
+
+_writer = _SharedWriter()
+
+
def _turn_record(r: sqlite3.Row) -> dict[str, Any]:
d = {k: r[k] for k in _TURN_COLUMNS}
d["incomplete"] = bool(d["incomplete"])
diff --git a/coworker/server/manager.py b/coworker/server/manager.py
index 24ae59a3..6a271bc1 100644
--- a/coworker/server/manager.py
+++ b/coworker/server/manager.py
@@ -505,6 +505,7 @@ class SessionManager:
# Inbox (cross-session human-attention queue), routing (named inboxes + Slack/Telegram
# bindings), the Unattended toggle, and self-wake records.
self.inbox = InboxStore(base / "inbox.json")
+ self.inbox.on_change = self._inbox_item_changed
self.inbox_routing = InboxRouting(base / "inbox_routing.json")
self.unattended = UnattendedRegistry(base / "unattended.json")
self.wakes = WakeStore(base / "wakes.json")
@@ -1013,6 +1014,7 @@ class SessionManager:
)
# Observability (E1): this engine's turns and steps go to the server's recorder.
engine.recorder = self.recorder
+ engine.resolver_of = lambda call_id, _sid=session_id: self.inbox.resolver_of(_sid, call_id)
engine.delegated_approval = lambda args: self.worker_review_context(session_id, args)
engine.reviewer_context = lambda: self.approval_context(session_id)
engine.reviewer_owner_history = lambda: self.approval_owner_history(session_id, engine)
@@ -6306,6 +6308,14 @@ class SessionManager:
def has_session_clients(self, session_id: str) -> bool:
return bool(self._session_clients.get(session_id))
+ def _inbox_item_changed(self, item: Any) -> None:
+ """A card was added or moved to the Inbox: tell the waiting engine where it is shown
+ (observability §5.4, a wait step's `where`)."""
+ engine = self._engines.get(item.session_id)
+ note = getattr(engine, "note_wait_where", None)
+ if note is not None and item.tool_call_id:
+ note(item.tool_call_id, "inbox" if item.visibility == VIS_INBOX else "session")
+
async def promote_pending_prompts(self, session_id: str) -> int:
"""The last viewer left while the agent waits on an inline prompt: move it to the
Inbox and mirror it, so the question is visible somewhere (see
@@ -6969,6 +6979,7 @@ class SessionManager:
)
# Observability (E1): this engine's turns and steps go to the server's recorder.
engine.recorder = self.recorder
+ engine.resolver_of = lambda call_id, _sid=session_id: self.inbox.resolver_of(_sid, call_id)
self._seed_task_permissions(engine, task)
return engine
diff --git a/tests/test_observability_store.py b/tests/test_observability_store.py
index 54eefebd..eaad44c0 100644
--- a/tests/test_observability_store.py
+++ b/tests/test_observability_store.py
@@ -273,3 +273,13 @@ def test_the_step_clock_ticks_every_millisecond():
while (tick := now_ms()[1]) == first and time.perf_counter() < deadline:
pass
assert tick - first == 1
+
+
+def test_every_store_shares_one_writer_thread(tmp_path):
+ stores = [RecordingStore(tmp_path / f"r{i}.db") for i in range(20)]
+ try:
+ writers = [t for t in threading.enumerate() if t.name == "recordings-writer"]
+ assert len(writers) == 1
+ finally:
+ for s in stores:
+ s.close()
diff --git a/tests/test_observability_waits.py b/tests/test_observability_waits.py
new file mode 100644
index 00000000..c997ab5b
--- /dev/null
+++ b/tests/test_observability_waits.py
@@ -0,0 +1,210 @@
+"""Observability E20 and E4: wait steps, decision paths and summary steps.
+
+- E20: an approval card is a wait step from the moment it is shown to the answer, linked to
+ its tool call; its outcome and who answered are recorded.
+- E4: every tool call that went through the permission flow carries its decision path
+ (`attrs.decision = {path, origin}`); a summary or trim is a summary step with its model
+ call nested inside it.
+"""
+
+from __future__ import annotations
+
+import asyncio
+from types import SimpleNamespace
+
+from coworker import compaction as _compaction
+from coworker.engine import ApprovalOutcome, TurnEngine
+from coworker.observability import Recorder
+from coworker.permissions import Mode, PermissionEngine
+from coworker.providers import AssistantTurn, ModelCapabilities, ProviderClient, ToolCall
+from coworker.providers.base import TokenUsage
+from coworker.tools import ToolRegistry
+
+
+class _Meta:
+ category: str = ""
+ risk_level: str = "high"
+ requires_approval: bool = False
+
+
+class _ReadMeta:
+ category: str = ""
+ risk_level: str = "low"
+ requires_approval: bool = False
+
+
+class _Scripted(ProviderClient):
+ def __init__(self, turns):
+ self._turns = list(turns)
+
+ def complete(self, *, model, messages, tools=None, **settings):
+ return self._turns.pop(0)
+
+ def capabilities(self, model):
+ return ModelCapabilities()
+
+
+def run_shell(command: str) -> dict:
+ """Run a command.
+
+ Args:
+ command: the command line
+ """
+ return {"command": command, "exit_code": 0, "output": ""}
+
+
+def look(path: str) -> dict:
+ """Read something.
+
+ Args:
+ path: what to read
+ """
+ return {"path": path, "text": "ok"}
+
+
+def _engine(tmp_path, turns, *, answer=ApprovalOutcome.ONCE, mode=Mode.INTERACTIVE):
+ registry = ToolRegistry()
+ registry.register(run_shell, metadata=_Meta())
+ registry.register(look, metadata=_ReadMeta())
+
+ async def approver(request):
+ return answer
+
+ engine = TurnEngine(
+ provider=_Scripted(turns),
+ registry=registry,
+ permissions=PermissionEngine(workspace_root=tmp_path, mode=mode),
+ model="openai:model-a",
+ approver=approver,
+ )
+ engine.audit_context = {"session_id": "sess_w"}
+ engine.is_attended = lambda: True
+ engine.resolver_of = lambda call_id: "devika"
+ rec = Recorder()
+ engine.recorder = rec
+ return engine, rec
+
+
+def _call(cid, name, args):
+ return AssistantTurn(tool_calls=[ToolCall(id=cid, name=name, arguments=args)], finish_reason="tool_calls")
+
+
+def _run(engine, text="go"):
+ async def go():
+ return [ev async for ev in engine.run(text, trigger="user")]
+
+ return asyncio.run(go())
+
+
+def _steps(rec, kind):
+ return [s for s in rec.as_dict()["steps"] if s["kind"] == kind]
+
+
+def test_an_approved_card_is_a_wait_step_and_the_call_records_its_path(tmp_path):
+ engine, rec = _engine(tmp_path, [_call("c1", "run_shell", {"command": "make build"}),
+ AssistantTurn(text="built")])
+ _run(engine)
+ [wait] = _steps(rec, "wait")
+ assert (wait["name"], wait["call_id"], wait["status"]) == ("approval", "c1", "ok")
+ assert wait["attrs"] == {"for_call_id": "c1", "kind": "approval", "attended": True,
+ "where": "session", "outcome": "approved"}
+ assert wait["content"] == {"resolved_by": "devika"}
+ [tool] = _steps(rec, "tool")
+ assert tool["status"] == "ok"
+ assert tool["attrs"]["decision"] == {
+ "path": [{"stage": "permission", "outcome": "needs_user"},
+ {"stage": "card", "outcome": "shown_session"},
+ {"stage": "resolved", "outcome": "approved_once"}],
+ "origin": "user",
+ }
+ # The wait comes before the tool in the canonical order.
+ kinds = [s["kind"] for s in rec.as_dict()["steps"]]
+ assert kinds.index("wait") < kinds.index("tool")
+
+
+def test_a_denied_card_is_a_zero_length_denied_step_refused_by_the_user(tmp_path):
+ engine, rec = _engine(tmp_path, [_call("c1", "run_shell", {"command": "rm -rf build"}),
+ AssistantTurn(text="ok, not doing it")],
+ answer=ApprovalOutcome.DENY)
+ _run(engine)
+ [wait] = _steps(rec, "wait")
+ assert wait["attrs"]["outcome"] == "denied"
+ [tool] = _steps(rec, "tool")
+ assert (tool["status"], tool["duration_ms"]) == ("denied", 0)
+ assert tool["attrs"]["decision"]["origin"] == "user"
+ assert tool["attrs"]["decision"]["path"][-1] == {"stage": "resolved", "outcome": "denied"}
+
+
+def test_a_session_grant_is_recorded_as_its_origin(tmp_path):
+ engine, rec = _engine(tmp_path, [_call("c1", "run_shell", {"command": "pytest"}),
+ _call("c2", "run_shell", {"command": "pytest"}),
+ AssistantTurn(text="done")],
+ answer=ApprovalOutcome.ALWAYS_COMMAND)
+ _run(engine)
+ first, second = _steps(rec, "tool")
+ assert first["attrs"]["decision"]["path"][-1] == {
+ "stage": "resolved", "outcome": "approved_session", "detail": "always_command"}
+ assert second["attrs"]["decision"] == {
+ "path": [{"stage": "permission", "outcome": "allowed"},
+ {"stage": "grant", "outcome": "session_grant"}],
+ "origin": "session_grant",
+ }
+ assert len(_steps(rec, "wait")) == 1 # no card the second time
+
+
+def test_a_low_risk_read_needs_no_one(tmp_path):
+ engine, rec = _engine(tmp_path, [_call("c1", "look", {"path": "x"}), AssistantTurn(text="ok")])
+ _run(engine)
+ [tool] = _steps(rec, "tool")
+ assert tool["attrs"]["decision"] == {
+ "path": [{"stage": "permission", "outcome": "allowed", "detail": "low_risk"}],
+ "origin": "low_risk",
+ }
+ assert _steps(rec, "wait") == []
+
+
+def test_full_access_is_bypass(tmp_path):
+ engine, rec = _engine(tmp_path, [_call("c1", "run_shell", {"command": "make"}), AssistantTurn(text="ok")],
+ mode=Mode.BYPASS_APPROVALS)
+ _run(engine)
+ [tool] = _steps(rec, "tool")
+ assert tool["attrs"]["decision"]["origin"] == "bypass"
+
+
+def test_a_summary_is_a_step_with_its_model_call_inside(tmp_path, monkeypatch):
+ engine, rec = _engine(tmp_path, [
+ AssistantTurn(text="a short summary", usage=TokenUsage(input=900, output=40)),
+ AssistantTurn(text="next", usage=TokenUsage(input=300, output=5, cache_read=200)),
+ ])
+ engine._last_context_tokens = 47000
+
+ def fake_build_state(messages, *, provider, model, **kw):
+ provider.complete(model=model, messages=[{"role": "user", "content": "summarize"}])
+ return SimpleNamespace(boundary_index=0, carries_outside=False, transcript_path=None)
+
+ monkeypatch.setattr(_compaction, "build_state", fake_build_state)
+
+ async def go():
+ return await engine._compact_now()
+
+ assert asyncio.run(go()) is not None
+ engine.provider.complete(model=engine.model, messages=[]) # the next main call
+ [summary] = _steps(rec, "summary")
+ [model] = [s for s in rec.as_dict()["steps"] if s["kind"] == "model" and s["attrs"]["purpose"] == "summary"]
+ assert (summary["name"], summary["status"]) == ("summary", "ok")
+ assert model["parent_step_id"] == summary["step_id"]
+ assert summary["attrs"]["model_step_id"] == model["step_id"]
+ assert summary["attrs"]["reason"] == "threshold"
+ assert summary["attrs"]["tokens_before"] == 47000
+ assert summary["attrs"]["tokens_after"] == 500
+
+
+def test_no_summary_step_when_nothing_was_summarised(tmp_path, monkeypatch):
+ engine, rec = _engine(tmp_path, [])
+ monkeypatch.setattr(_compaction, "build_state", lambda messages, **kw: None)
+
+ async def go():
+ return await engine._compact_now()
+
+ assert asyncio.run(go()) is None
+ assert _steps(rec, "summary") == []