Python
feat(observability): the engine records turns and tool steps (OPE-244)
Devika Vermaa08e856a5f5e
diff --git a/coworker/engine.py b/coworker/engine.pyindex 6effd282..d5d6ec4a 100644--- a/coworker/engine.py+++ b/coworker/engine.py@@ -33,6 +33,7 @@ from . import toolchain as _toolchainfrom . import toolresultfrom . import unattended as _attendancefrom .events import Event, EventType+from .observability.engine_hooks import TurnRecordingfrom .observability.ids import new_ulid# §8.4 retry guard: the reviewer pauses for the rest of the turn after this many denials@@ -191,6 +192,51 @@ def _summarizes_turn(gen_fn: Callable[..., AsyncIterator[Event]]) -> Callable[..return wrapper+# The engine's reviewer choice -> the recorded reviewer kind (recording format 1.0).+_REVIEWER_KINDS = {"session": "session_model", "openai-decisions": "openai_decisions", "jev": "typesafe_jev"}+++def _records_turn(kind: str) -> Callable[[Callable[..., AsyncIterator[Event]]], Callable[..., AsyncIterator[Event]]]:+ """Record a turn entry point (run / retry / resume) for observability (E1).++ `turn_started` goes out when the entry point emits TURN_START, `turn_ended` at its+ TURN_END, INTERRUPTED or ERROR, and, for the paths that end with none of these (a+ resume stopped mid-tools, a generator the surface abandoned), in the `finally`: then the+ status is `interrupted` if Stop was pressed, else `unknown`. The caller says who started+ the turn with `trigger=` (user, channel, schedule, team_wake; `unknown` if it does not+ say). While recording, UI events also carry `turn_id`, `ts_ms`, and a tool's `step_id`."""++ def deco(gen_fn: Callable[..., AsyncIterator[Event]]) -> Callable[..., AsyncIterator[Event]]:+ @functools.wraps(gen_fn)+ async def wrapper(self: "TurnEngine", *args: Any, trigger: str = "unknown", **kwargs: Any) -> AsyncIterator[Event]:+ inner = gen_fn(self, *args, **kwargs)+ obs = getattr(self, "_obs", None)+ if obs is None or not obs.enabled:+ try:+ async for event in inner:+ yield event+ finally:+ await inner.aclose()+ return+ started = False+ try:+ async for event in inner:+ if event.type is EventType.TURN_START and not started:+ started = True+ self._begin_recorded_turn(kind, trigger)+ if started and obs.in_turn:+ self._observe_event(event)+ yield event+ finally:+ await inner.aclose()+ if started:+ obs.end_turn("interrupted" if self._cancel.is_set() else "unknown")++ return wrapper++ return deco++def _action_failed(action: dict[str, Any]) -> bool:"""A listed action that changed nothing: the call errored, the command exitednon-zero, or it timed out. A background start is neither done nor failed."""@@ -435,6 +481,10 @@ class TurnEngine:self._step = 0self._last_context_tokens: Optional[int] = Noneself.audit_context: dict[str, Any] = {}+ # Observability (E1): the turn and step events this engine sends to its recorder.+ # Off until a recorder is set (`engine.recorder = ...`), like the other surface hooks.+ self._obs = TurnRecording(None, lambda: str(self.audit_context.get("session_id") or ""))+ self._call_index: dict[str, int] = {}if instructions and not (self.messages and self.messages[0].get("role") == "system"):@@ -491,6 +541,90 @@ class TurnEngine:finally:self._tool_timings.setdefault(tool_call.id, {}).update(waited_started=started, waited_ms=(time.monotonic() - clock) * 1000)+ @property+ def recorder(self) -> Any:+ return self._obs.recorder++ @recorder.setter+ def recorder(self, value: Any) -> None:+ self._obs.recorder = value++ def _begin_recorded_turn(self, kind: str, trigger: str) -> None:+ """`turn_started` (E1, technical design §4). For `run` the turn opens with your+ message; for `resume`, with the assistant message whose calls are resumed; a `retry`+ opens with the first message it adds, so its start waits for that."""+ first: Optional[str] = None+ if kind == "run":+ first = next((m.get("mid") for m in reversed(self.messages) if m.get("role") == "user"), None)+ elif kind == "resume":+ first = next(+ (m.get("mid") for m in reversed(self.messages)+ if m.get("role") == "assistant" and m.get("tool_calls")),+ None,+ )+ reviewer_kind = (+ _REVIEWER_KINDS.get(getattr(self, "reviewer_choice", "session"), "session_model")+ if self.reviewer is not None else None+ )+ try:+ attended = bool(self.is_attended and self.is_attended())+ except Exception: # noqa: BLE001 - a broken getter must not break the turn+ attended = False+ self._obs.begin_turn(+ kind=kind,+ trigger=trigger,+ attrs={+ "approval_mode": self.permissions.mode.value,+ "attended": attended,+ "reviewer": {+ "active": self._reviewer_active() if self.reviewer is not None else False,+ "shadow": bool(self.reviewer_shadow),+ "kind": reviewer_kind,+ "model": getattr(self.reviewer, "model", None) if reviewer_kind == "session_model" else None,+ },+ },+ stamp={},+ first_message_id=first,+ wait_for_first_message=kind == "retry",+ )++ def _observe_event(self, event: Event) -> None:+ """Ids on a UI event, a call that never ran as a zero-length tool step, and the turn's+ end (E1). The app ignores fields it does not know, so older screens keep working."""+ obs = self._obs+ event.data.setdefault("turn_id", obs.turn_id)+ event.data.setdefault("ts_ms", time.time_ns() // 1_000_000)+ if event.type in (EventType.TOOL_STARTED, EventType.TOOL_FINISHED):+ call_id = str(event.data.get("tool_call_id") or "")+ if event.type is EventType.TOOL_FINISHED and call_id:+ mid = next(+ (m.get("mid") for m in reversed(self.messages)+ if m.get("role") == "tool" and m.get("tool_call_id") == call_id),+ None,+ )+ obs.tool_finished(+ event.data, message_id=mid, call_index=self._call_index.get(call_id),+ attrs={"risk_class": self._risk_class(str(event.data.get("name") or ""))},+ )+ step_id = obs.step_for_call(call_id)+ if step_id:+ event.data.setdefault("step_id", step_id)+ if event.type is EventType.TURN_END:+ obs.end_turn(str(event.data.get("status") or "completed"))+ elif event.type is EventType.INTERRUPTED:+ obs.end_turn("interrupted")+ elif event.type is EventType.ERROR:+ obs.end_turn("error")++ def _risk_class(self, tool_name: str) -> Optional[str]:+ try:+ spec = self.registry.get(tool_name)+ return _classify_risk(+ tool_name, spec.metadata if spec else None, self.permissions.risk_overrides+ ).value+ except Exception: # noqa: BLE001 - a fact, never a reason to fail a call+ return None+def _add_message(self, message: dict[str, Any], *, at: Optional[int] = None) -> None:"""Append (or, with `at`, insert) one message into the history, stamping its `mid`.@@ -503,6 +637,9 @@ class TurnEngine:self.messages.append(message)else:self.messages.insert(at, message)+ obs = getattr(self, "_obs", None)+ if obs is not None:+ obs.note_message(message)def _timed_result(self, tool_call, result):message = _tool_result_message(tool_call, result)@@ -517,6 +654,7 @@ class TurnEngine:self._steering.append((text, source, activity))# -- main loop --------------------------------------------------------------+ @_records_turn("run")@_summarizes_turnasync def run(self,@@ -654,6 +792,7 @@ class TurnEngine:notice.update({k: v for k, v in fields.items() if v is not None})self._add_message(notice)+ @_records_turn("retry")@_summarizes_turnasync def retry(self) -> AsyncIterator[Event]:"""Re-run the model loop after a provider error — no new user message; the failed@@ -693,6 +832,7 @@ class TurnEngine:info = workspace.describe()yield Event(EventType.SANDBOX_READY, {k: info.get(k) for k in ("provider", "enforcement", "reason", "sandbox") if info.get(k) is not None})+ @_records_turn("resume")@_summarizes_turnasync def resume(self) -> AsyncIterator[Event]:"""Continue a turn that was suspended at a prompt and persisted — durable resume after a@@ -1269,6 +1409,16 @@ class TurnEngine:# sequential because approval cards are interactive and must reach the human one# at a time, in call order.self._shell_cleared_in_batch = False+ # Each call's position in the model's list for this reply (canonical step order).+ ids = {tc.id for tc in tool_calls}+ listed = next(+ (m["tool_calls"] for m in reversed(self.messages)+ if m.get("role") == "assistant" and m.get("tool_calls")+ and ids & {tc.get("id") for tc in m["tool_calls"]}),+ [],+ )+ order = [tc.get("id") for tc in listed] or [tc.id for tc in tool_calls]+ self._call_index = {cid: i for i, cid in enumerate(order) if cid}await self._preconsult_reviewer(tool_calls)cleared: list[ToolCall] = []for tool_call in tool_calls:@@ -1342,7 +1492,7 @@ class TurnEngine:if concurrent:for tool_call in concurrent:- yield Event(EventType.TOOL_STARTED, {"name": tool_call.name})+ yield Event(EventType.TOOL_STARTED, {"name": tool_call.name, "tool_call_id": tool_call.id})self._audit(tool_call, stage="started")outcomes = await asyncio.gather(*[asyncio.to_thread(self._execute_sync, tc) for tc in concurrent]@@ -1354,7 +1504,7 @@ class TurnEngine:if self._cancel.is_set():yield self._interrupted_tool(tool_call)continue- yield Event(EventType.TOOL_STARTED, {"name": tool_call.name})+ yield Event(EventType.TOOL_STARTED, {"name": tool_call.name, "tool_call_id": tool_call.id})self._audit(tool_call, stage="started")result, status = await asyncio.to_thread(self._execute_sync, tool_call)yield self._record_result(tool_call, result, status)@@ -1387,7 +1537,7 @@ class TurnEngine:self._audit(tool_call, stage="finished", status="error", reason=reason)return Event(EventType.TOOL_FINISHED,- {"name": tool_call.name, "status": "error", "reason": reason},+ {"name": tool_call.name, "tool_call_id": tool_call.id, "status": "error", "reason": reason},)def _interrupted_tool(self, tool_call: ToolCall) -> Event:@@ -2296,7 +2446,34 @@ class TurnEngine:return "read"def _execute_sync(self, tool_call: ToolCall) -> tuple[Any, str]:- """Execute one authorized call (runs in a worker thread)."""+ """Execute one authorized call (runs in a worker thread). Recorded as a tool step+ from here to the RAW result, before `_record_result` shortens it (E1)."""+ step_id = self._obs.start_step(+ kind="tool", name=tool_call.name, call_id=tool_call.id,+ call_index=self._call_index.get(tool_call.id),+ )+ result, status = self._execute_call(tool_call)+ if step_id is not None:+ self._obs.end_step(+ step_id, status=status, attrs=self._tool_step_attrs(tool_call, result),+ wait_for_tool_message=True,+ )+ return result, status++ def _tool_step_attrs(self, tool_call: ToolCall, result: Any) -> dict[str, Any]:+ attrs: dict[str, Any] = {"risk_class": self._risk_class(tool_call.name)}+ try:+ attrs["result_bytes"] = len(+ result.encode("utf-8") if isinstance(result, str)+ else json.dumps(result, default=str, ensure_ascii=False).encode("utf-8")+ )+ except Exception: # noqa: BLE001+ pass+ if isinstance(result, dict) and isinstance(result.get("exit_code"), int):+ attrs["exit_code"] = result["exit_code"]+ return attrs++ def _execute_call(self, tool_call: ToolCall) -> tuple[Any, str]:started, clock = time.time(), time.monotonic()try:delegated = self._authorized_delegates.pop(tool_call.id, None)@@ -2584,7 +2761,7 @@ class TurnEngine:self._audit(tool_call, stage="finished", status=status, result=result, result_preview=_preview(result))yield Event(EventType.TOOL_FINISHED,- {"name": tool_call.name, "status": status, "result_preview": _preview(result)},+ {"name": tool_call.name, "tool_call_id": tool_call.id, "status": status, "result_preview": _preview(result)},)async def _handle_team_proposal(self, tool_call: ToolCall) -> AsyncIterator[Event]:@@ -2721,6 +2898,7 @@ class TurnEngine:EventType.TOOL_FINISHED,{"name": tool_call.name,+ "tool_call_id": tool_call.id,"status": status,"result_preview": _preview(result),},@@ -2915,6 +3093,7 @@ class TurnEngine:EventType.TOOL_FINISHED,{"name": tool_call.name,+ "tool_call_id": tool_call.id,"status": status,"result_preview": _preview(result),},@@ -3010,6 +3189,7 @@ class TurnEngine:EventType.TOOL_FINISHED,{"name": tool_call.name,+ "tool_call_id": tool_call.id,"status": status,"result_preview": _preview(result),},@@ -3091,6 +3271,7 @@ class TurnEngine:EventType.TOOL_FINISHED,{"name": tool_call.name,+ "tool_call_id": tool_call.id,"status": status,"result_preview": _preview(result),},diff --git a/coworker/observability/engine_hooks.py b/coworker/observability/engine_hooks.pynew file mode 100644index 00000000..26a2a94e--- /dev/null+++ b/coworker/observability/engine_hooks.py@@ -0,0 +1,229 @@+"""The engine's side of recording (observability E1): builds the recording events a+`TurnEngine` sends and hands them to its `Recorder`.++The engine calls a handful of methods here at its hook points (a turn starts or ends, a tool+starts or ends, a message is added). This module turns those calls into the five events of+recording-format.md section 2, with `v`, `at_ms`, `mono_ms`, and the copy of the step's+identity that every `step_ended` carries. With no recorder every method returns at once.++Two events wait for a fact that arrives later:+- A `retry` turn's `turn_started` waits for the first message the retry adds, whose `mid` it+ carries (`first_message_id`). Until then the turn's other events are held, in order; their+ times were taken when they happened, so holding them changes nothing in the recording.+- A tool step's `step_ended` is built when the tool returns (its time is then) and sent once+ the tool's result message is added, so it can carry that message's `mid`.+"""++from __future__ import annotations++import copy+import threading+from typing import Any, Callable, Optional++from .ids import new_ulid+from .recorder import EVENT_V, Recorder, now_ms++# TOOL_FINISHED status -> tool step status (technical design §5.1).+_TOOL_STATUS = {"ok": "ok", "error": "error", "denied": "denied", "interrupted": "interrupted",+ "skipped": "skipped"}+++class TurnRecording:+ """One per engine. Thread-safe: tool steps start and end in tool threads."""++ def __init__(self, recorder: Optional[Recorder], session_id: Callable[[], str]):+ self.recorder = recorder+ self._session_id = session_id+ self._lock = threading.RLock()+ self.turn_id: Optional[str] = None+ self._turn_session = ""+ self._ended = True+ self._held: Optional[list[dict[str, Any]]] = None # events behind a waiting turn_started+ self._pending_start: Optional[dict[str, Any]] = None+ self._last_mid: Optional[str] = None+ self._ident: dict[str, dict[str, Any]] = {} # step_id -> identity copy+ self._tool_steps: dict[str, str] = {} # call_id -> step_id, for this turn's tools+ self._tool_ends: dict[str, dict[str, Any]] = {} # call_id -> step_ended awaiting its mid++ @property+ def enabled(self) -> bool:+ return self.recorder is not None++ # -- sending ------------------------------------------------------------------------------+ def _base(self, typ: str, turn_id: Optional[str], session_id: str) -> dict[str, Any]:+ at, mono = now_ms()+ return {"type": typ, "v": EVENT_V, "at_ms": at, "mono_ms": mono,+ "session_id": session_id, "turn_id": turn_id or ""}++ def _send(self, ev: dict[str, Any]) -> None:+ if self._held is not None and ev["type"] != "turn_started" and ev.get("turn_id") == self.turn_id:+ self._held.append(ev)+ else:+ self.recorder.emit(ev)++ def _release(self, first_message_id: Optional[str]) -> None:+ """Send the waiting turn_started with its first message id, then what it held."""+ if self._pending_start is None:+ return+ ev, held = self._pending_start, self._held or []+ self._pending_start, self._held = None, None+ ev["first_message_id"] = first_message_id+ self.recorder.emit(ev)+ for e in held:+ self.recorder.emit(e)++ # -- turns ----------------------------------------------------------------------------------+ def begin_turn(self, *, kind: str, trigger: str, attrs: dict[str, Any], stamp: dict[str, Any],+ first_message_id: Optional[str], wait_for_first_message: bool = False) -> str:+ """`turn_started`, now. With `wait_for_first_message` it is held until `note_message`+ sees the turn's first message (a retry) or the turn ends."""+ with self._lock:+ self.turn_id = new_ulid()+ self._turn_session = self._session_id() or ""+ self._ended = False+ self._last_mid = None+ self._tool_steps.clear()+ self._tool_ends.clear()+ ev = self._base("turn_started", self.turn_id, self._turn_session)+ ev.update({"kind": kind, "trigger": trigger, "first_message_id": first_message_id,+ "attrs": copy.deepcopy(attrs), "stamp": copy.deepcopy(stamp)})+ if wait_for_first_message:+ self._pending_start, self._held = ev, []+ else:+ self.recorder.emit(ev)+ return self.turn_id++ def end_turn(self, status: str) -> None:+ with self._lock:+ if self._ended or self.turn_id is None:+ return+ self._ended = True+ self._release(None) # a retry that added no message at all+ for ev in self._tool_ends.values(): # a tool whose result message never came+ self.recorder.emit(ev)+ self._tool_ends.clear()+ ev = self._base("turn_ended", self.turn_id, self._turn_session)+ ev.update({"status": status, "last_message_id": self._last_mid})+ self.recorder.emit(ev)++ @property+ def in_turn(self) -> bool:+ return self.turn_id is not None and not self._ended++ def step_for_call(self, call_id: str) -> Optional[str]:+ """The tool step recorded for a call in this turn, if any."""+ with self._lock:+ return self._tool_steps.get(call_id) or None++ def note_message(self, message: dict[str, Any]) -> None:+ """Every message the engine adds (E21). Tracks the turn's last `mid`, releases a+ waiting `turn_started`, and sends a tool step's end once its result message exists."""+ if not self.enabled:+ return+ mid = message.get("mid")+ with self._lock:+ if not self.in_turn:+ return+ self._last_mid = mid+ if self._pending_start is not None:+ self._release(mid)+ if message.get("role") == "tool":+ ev = self._tool_ends.pop(str(message.get("tool_call_id") or ""), None)+ if ev is not None:+ ev["message_id"] = mid+ self._send(ev)++ # -- steps ----------------------------------------------------------------------------------+ def start_step(self, *, kind: str, name: str, call_id: Optional[str] = None,+ call_index: Optional[int] = None, parent_step_id: Optional[str] = None,+ attrs: Optional[dict[str, Any]] = None,+ content: Optional[dict[str, Any]] = None) -> Optional[str]:+ if not self.enabled:+ return None+ with self._lock:+ if not self.in_turn:+ return None+ step_id = new_ulid()+ ev = self._base("step_started", self.turn_id, self._turn_session)+ ev.update({"step_id": step_id, "parent_step_id": parent_step_id, "kind": kind,+ "name": name, "call_id": call_id, "call_index": call_index,+ "shadow": False, "after_turn": False, "attrs": attrs or {},+ "content": content or {}})+ self._ident[step_id] = {+ "kind": kind, "name": name, "parent_step_id": parent_step_id, "call_id": call_id,+ "call_index": call_index, "shadow": False, "after_turn": False,+ "started_at": self._started_at(ev), "turn_id": self.turn_id,+ "session_id": self._turn_session,+ }+ if kind == "tool" and call_id:+ self._tool_steps[call_id] = step_id+ self._send(ev)+ return step_id++ def _started_at(self, ev: dict[str, Any]) -> int:+ # The recorder anchors a step's start to its turn's start (§7.3); the identity copy+ # uses the wall clock, which is what a recorder without the turn would use.+ return ev["at_ms"]++ def end_step(self, step_id: Optional[str], *, status: str = "ok",+ attrs: Optional[dict[str, Any]] = None, content: Optional[dict[str, Any]] = None,+ message_id: Optional[str] = None, error: Optional[dict[str, Any]] = None,+ wait_for_tool_message: bool = False) -> None:+ if not self.enabled or step_id is None:+ return+ with self._lock:+ idn = self._ident.pop(step_id, None)+ if idn is None:+ return+ ev = self._base("step_ended", idn["turn_id"], idn["session_id"])+ ev.update({"step_id": step_id, "status": status, "message_id": message_id,+ "attrs": attrs or {}, "content": content or {}})+ if error is not None:+ ev["error"] = error+ ev.update({k: idn[k] for k in ("kind", "name", "parent_step_id", "call_id",+ "call_index", "shadow", "after_turn", "started_at")})+ if wait_for_tool_message and idn["call_id"] and not self._ended:+ self._tool_ends[idn["call_id"]] = ev+ else:+ self._send(ev)++ def instant_step(self, *, kind: str, name: str, status: str, call_id: Optional[str] = None,+ call_index: Optional[int] = None, attrs: Optional[dict[str, Any]] = None,+ message_id: Optional[str] = None) -> None:+ """A zero-length step at the moment of a decision: a call that never ran. Its start+ and end carry the same clocks."""+ if not self.enabled:+ return+ with self._lock:+ if not self.in_turn:+ return+ step_id = new_ulid()+ start = self._base("step_started", self.turn_id, self._turn_session)+ start.update({"step_id": step_id, "parent_step_id": None, "kind": kind, "name": name,+ "call_id": call_id, "call_index": call_index, "shadow": False,+ "after_turn": False, "attrs": {}, "content": {}})+ end = {k: start[k] for k in ("v", "at_ms", "mono_ms", "session_id", "turn_id")}+ end.update({"type": "step_ended", "step_id": step_id, "status": status,+ "message_id": message_id, "attrs": attrs or {}, "content": {},+ "kind": kind, "name": name, "parent_step_id": None, "call_id": call_id,+ "call_index": call_index, "shadow": False, "after_turn": False,+ "started_at": start["at_ms"]})+ self._send(start)+ self._send(end)++ def tool_finished(self, data: dict[str, Any], *, message_id: Optional[str],+ call_index: Optional[int], attrs: dict[str, Any]) -> None:+ """A TOOL_FINISHED for a call that never reached `_execute_sync`: its zero-length step."""+ if not self.enabled:+ return+ call_id = str(data.get("tool_call_id") or "")+ with self._lock:+ if not self.in_turn or not call_id or call_id in self._tool_steps:+ return+ self._tool_steps[call_id] = ""+ status = _TOOL_STATUS.get(str(data.get("status") or ""), "error")+ if "superseded_worker_call" in data:+ status = "skipped"+ self.instant_step(kind="tool", name=str(data.get("name") or ""), status=status,+ call_id=call_id, call_index=call_index, attrs=attrs,+ message_id=message_id)diff --git a/coworker/server/app.py b/coworker/server/app.pyindex e2bb9a99..a0c20f98 100644--- a/coworker/server/app.py+++ b/coworker/server/app.py@@ -3228,10 +3228,11 @@ def create_app(manager: SessionManager) -> FastAPI:)returnevents = (- engine.retry()+ engine.retry(trigger="user")if retryelse engine.run(content, display=display,- activity=manager.prepare_activity(session_id, "user activity"))+ activity=manager.prepare_activity(session_id, "user activity"),+ trigger="user"))async for event in events:data = event.datadiff --git a/coworker/server/manager.py b/coworker/server/manager.pyindex 907bfc26..de48be02 100644--- a/coworker/server/manager.py+++ b/coworker/server/manager.py@@ -1006,6 +1006,8 @@ class SessionManager:auto_approve_shadow=self.auto_approve_shadow(),auto_approve_reviewer=self.auto_approve_reviewer(),)+ # Observability (E1): this engine's turns and steps go to the server's recorder.+ engine.recorder = self.recorderengine.delegated_approval = lambda args: self.worker_review_context(session_id, args)engine.reviewer_context = lambda: self.approval_context(session_id)engine.reviewer_owner_history = lambda: self.approval_owner_history(session_id, engine)@@ -2004,7 +2006,7 @@ class SessionManager:returnself.mark_running(item.session_id)try:- async for _event in engine.resume():+ async for _event in engine.resume(trigger="user"):passself.save(item.session_id, engine)finally:@@ -6960,6 +6962,8 @@ class SessionManager:[d] if (d := self.persona_skill_scope(task.agent)[0]) is not None else None),)+ # Observability (E1): this engine's turns and steps go to the server's recorder.+ engine.recorder = self.recorderself._seed_task_permissions(engine, task)return engine@@ -7231,7 +7235,15 @@ class SessionManager:# semantics intact while grouping check-ins in the transcript.source = self._board_source(team, message, rows=activity["board_rows"])source["board"]["check_in"] = True- async for event in engine.run(message, source=source, activity=activity):+ # Who started the turn (observability §4): board activity and team timers wake a+ # team; a session's own timer is a schedule; anything else came from a channel.+ if board_delivery:+ trigger = "team_wake"+ elif wake is not None:+ trigger = "team_wake" if self.teams.for_lead_session(session_id) is not None else "schedule"+ else:+ trigger = "channel"+ async for event in engine.run(message, source=source, activity=activity, trigger=trigger):# Stream every event to any socket viewing this session, so a background turn# (channel delivery, self-wake, durable resume) is seen live — not just on reselect.await self.broadcast_session(@@ -7806,7 +7818,7 @@ class SessionManager:"scheduled tasks.\n\n"f"{task.instructions}")- async for _event in engine.run(opening):+ async for _event in engine.run(opening, trigger="schedule"):passrun.result_text = _last_assistant_text(engine.messages)run.artifacts = _recent_files(task.workspace, since=run.started_at)diff --git a/coworker/tui/app.py b/coworker/tui/app.pyindex 4f7d069c..65c8997c 100644--- a/coworker/tui/app.py+++ b/coworker/tui/app.py@@ -156,7 +156,7 @@ class CoworkerApp(App):async def run_turn(self, text: str) -> None:assert self.engine is not Nonetry:- async for event in self.engine.run(text):+ async for event in self.engine.run(text, trigger="user"):self._render_event(event)except Exception as exc: # pragma: no cover - surfaced to the userself._write(f"[red]error:[/red] {exc}")diff --git a/tests/test_automation.py b/tests/test_automation.pyindex 3a013493..438e95fd 100644--- a/tests/test_automation.py+++ b/tests/test_automation.py@@ -493,7 +493,7 @@ async def test_cancelled_scheduled_run_is_not_left_running(tmp_path, monkeypatchclass BlockingEngine:messages = []- async def run(self, _opening):+ async def run(self, _opening, **_kw):started.set()await asyncio.Event().wait()yield@@ -573,7 +573,7 @@ async def test_cancelled_completion_notification_preserves_success(class CompletingEngine:messages = [{"role": "assistant", "content": "done"}]- async def run(self, _opening):+ async def run(self, _opening, **_kw):if False:yielddiff --git a/tests/test_observability_engine.py b/tests/test_observability_engine.pynew file mode 100644index 00000000..1975d92d--- /dev/null+++ b/tests/test_observability_engine.py@@ -0,0 +1,187 @@+"""Observability E1: a real engine turn, recorded.++The engine sends `turn_started` / `turn_ended` around run, retry and resume, a tool step for+every call that runs (from `_execute_sync` to the raw result), a zero-length tool step for a+call that never ran, and ids on its UI events. The fold rules are checked by the golden test;+this checks that the engine sends the right events.+"""++from __future__ import annotations++import asyncio++import aisuite as ai++from coworker.engine import TurnEngine+from coworker.events import EventType+from coworker.observability import Recorder+from coworker.permissions import PermissionEngine+from coworker.providers import AssistantTurn, ModelCapabilities, ProviderClient, ToolCall+from coworker.tools import ToolRegistry+++class ScriptedProvider(ProviderClient):+ def __init__(self, turns):+ self._turns = list(turns)++ def complete(self, *, model, messages, tools=None, **settings):+ turn = self._turns.pop(0)+ if isinstance(turn, Exception):+ raise turn+ return turn++ def capabilities(self, model):+ return ModelCapabilities()+++def _text(text):+ return AssistantTurn(text=text, finish_reason="stop")+++def _tools(*calls):+ return AssistantTurn(+ tool_calls=[ToolCall(id=cid, name=name, arguments=args) for cid, name, args in calls],+ finish_reason="tool_calls",+ )+++def _engine(tmp_path, turns):+ registry = ToolRegistry()+ registry.register_all(ai.toolkits.files(root=str(tmp_path), allow_write=True))+ engine = TurnEngine(+ provider=ScriptedProvider(turns),+ registry=registry,+ permissions=PermissionEngine(workspace_root=tmp_path),+ model="gpt-5.5",+ )+ engine.audit_context = {"session_id": "sess_1"}+ rec = Recorder()+ events: list[dict] = []+ rec.add_listener(events.append)+ engine.recorder = rec+ return engine, rec, events+++def _drain(gen):+ async def go():+ return [ev async for ev in gen]++ return asyncio.run(go())+++def test_a_turn_with_tools_is_recorded(tmp_path):+ (tmp_path / "a.txt").write_text("hello", encoding="utf-8")+ (tmp_path / "b.txt").write_text("world", encoding="utf-8")+ engine, rec, _ = _engine(tmp_path, [+ _tools(("c1", "read_file", {"path": "a.txt"}), ("c2", "read_file", {"path": "b.txt"})),+ _text("done"),+ ])+ ui = _drain(engine.run("read both", trigger="user"))+ d = rec.as_dict()++ [turn] = d["turns"]+ user = next(m for m in engine.messages if m["role"] == "user")+ assert (turn["kind"], turn["trigger"], turn["status"], turn["session_id"]) == (+ "run", "user", "completed", "sess_1")+ assert turn["first_message_id"] == user["mid"]+ assert turn["last_message_id"] == engine.messages[-1]["mid"]+ assert turn["approval_mode"] == engine.permissions.mode.value+ assert turn["reviewer"] == {"active": False, "shadow": False, "kind": None, "model": None}+ assert turn["duration_ms"] is not None++ tools = [s for s in d["steps"] if s["kind"] == "tool"]+ assert [(s["call_id"], s["call_index"], s["status"]) for s in tools] == [+ ("c1", 0, "ok"), ("c2", 1, "ok")]+ by_call = {m.get("tool_call_id"): m["mid"] for m in engine.messages if m["role"] == "tool"}+ for s in tools:+ assert s["message_id"] == by_call[s["call_id"]]+ assert s["attrs"]["risk_class"] == "read"+ assert s["attrs"]["result_bytes"] > 0+ assert s["duration_ms"] is not None and s["turn_id"] == turn["turn_id"]++ # UI events carry the turn id, a time, and a tool's call and step ids.+ assert all(e.data.get("turn_id") == turn["turn_id"] for e in ui)+ started = [e for e in ui if e.type is EventType.TOOL_STARTED]+ finished = [e for e in ui if e.type is EventType.TOOL_FINISHED]+ assert {e.data["tool_call_id"] for e in started} == {"c1", "c2"}+ assert {e.data["step_id"] for e in finished} == {s["step_id"] for s in tools}+ assert all(isinstance(e.data["ts_ms"], int) for e in ui)+++def test_every_step_ended_carries_its_identity(tmp_path):+ (tmp_path / "a.txt").write_text("hello", encoding="utf-8")+ engine, _, events = _engine(tmp_path, [_tools(("c1", "read_file", {"path": "a.txt"})), _text("ok")])+ _drain(engine.run("go"))+ ends = [e for e in events if e["type"] == "step_ended"]+ assert ends+ for e in ends:+ assert {"v", "at_ms", "mono_ms", "kind", "name", "call_id", "started_at"} <= set(e)+ assert [e["type"] for e in events][0] == "turn_started"+ assert [e["type"] for e in events][-1] == "turn_ended"+++def test_a_call_that_never_ran_is_a_zero_length_step(tmp_path):+ engine, rec, _ = _engine(tmp_path, [+ _tools(("c1", "read_file", {"_raw": "{not json"})),+ _text("sorry"),+ ])+ _drain(engine.run("go"))+ [step] = [s for s in rec.as_dict()["steps"] if s["kind"] == "tool"]+ assert (step["call_id"], step["status"], step["duration_ms"]) == ("c1", "error", 0)+ tool_msg = next(m for m in engine.messages if m.get("tool_call_id") == "c1")+ assert step["message_id"] == tool_msg["mid"]+++def test_a_failed_model_call_ends_the_turn_as_error_and_retry_is_its_own_turn(tmp_path):+ engine, rec, _ = _engine(tmp_path, [RuntimeError("provider down"), _text("back")])+ _drain(engine.run("hello", trigger="user"))+ _drain(engine.retry(trigger="user"))+ first, second = rec.as_dict()["turns"]+ assert (first["kind"], first["status"]) == ("run", "error")+ assert (second["kind"], second["status"], second["seq"]) == ("retry", "completed", 2)+ # A retry opens with the first message it adds: the model's reply.+ reply = engine.messages[-1]+ assert second["first_message_id"] == reply["mid"] == second["last_message_id"]+++def test_an_abandoned_turn_is_closed_as_unknown(tmp_path):+ (tmp_path / "a.txt").write_text("hello", encoding="utf-8")+ engine, rec, _ = _engine(tmp_path, [_tools(("c1", "read_file", {"path": "a.txt"})), _text("ok")])++ async def go():+ gen = engine.run("go")+ async for ev in gen:+ if ev.type is EventType.TURN_START:+ break+ await gen.aclose() # the surface stopped reading++ asyncio.run(go())+ [turn] = rec.as_dict()["turns"]+ assert turn["status"] == "unknown" and turn["duration_ms"] is not None+++def test_no_recorder_means_no_ids_on_events(tmp_path):+ engine, _, _ = _engine(tmp_path, [_text("hi")])+ engine.recorder = None+ ui = _drain(engine.run("hello", trigger="user"))+ assert all("turn_id" not in e.data for e in ui)+++def test_a_turn_through_the_server_lands_in_recordings_db(tmp_path):+ from fastapi.testclient import TestClient++ from coworker.server import SessionManager, create_app++ manager = SessionManager(workspace=tmp_path, provider=ScriptedProvider([_text("hi there")]))+ client = TestClient(create_app(manager))+ with client.websocket_connect("/ws/session/obs1") as ws:+ assert ws.receive_json()["type"] == "ready"+ ws.send_json({"type": "user_message", "text": "hello"})+ while ws.receive_json()["type"] != "turn_done":+ pass+ [turn] = manager.recordings_store.load_recording("obs1")["turns"]+ assert (turn["kind"], turn["trigger"], turn["status"]) == ("run", "user", "completed")+ assert manager.recordings_store.path.parent == manager.audit_store.db_path.parent++ manager.delete_session("obs1")+ assert manager.recordings_store.load_recording("obs1") == {"turns": [], "steps": []}