Skip to content
Python

feat(observability): the engine records turns and tool steps (OPE-244)

Devika Vermaa08e856a5f5e
diff --git a/coworker/engine.py b/coworker/engine.py
index 6effd282..d5d6ec4a 100644
--- a/coworker/engine.py
+++ b/coworker/engine.py
@@ -33,6 +33,7 @@ from . import toolchain as _toolchain
from . import toolresult
from . import unattended as _attendance
from .events import Event, EventType
+from .observability.engine_hooks import TurnRecording
from .observability.ids import new_ulid
# §8.4 retry guard: the reviewer pauses for the rest of the turn after this many denials
@@ -191,6 +192,51 @@ def _summarizes_turn(gen_fn: Callable[..., AsyncIterator[Event]]) -> Callable[..
return wrapper
+# The engine's reviewer choice -> the recorded reviewer kind (recording format 1.0).
+_REVIEWER_KINDS = {"session": "session_model", "openai-decisions": "openai_decisions", "jev": "typesafe_jev"}
+
+
+def _records_turn(kind: str) -> Callable[[Callable[..., AsyncIterator[Event]]], Callable[..., AsyncIterator[Event]]]:
+ """Record a turn entry point (run / retry / resume) for observability (E1).
+
+ `turn_started` goes out when the entry point emits TURN_START, `turn_ended` at its
+ TURN_END, INTERRUPTED or ERROR, and, for the paths that end with none of these (a
+ resume stopped mid-tools, a generator the surface abandoned), in the `finally`: then the
+ status is `interrupted` if Stop was pressed, else `unknown`. The caller says who started
+ the turn with `trigger=` (user, channel, schedule, team_wake; `unknown` if it does not
+ say). While recording, UI events also carry `turn_id`, `ts_ms`, and a tool's `step_id`."""
+
+ def deco(gen_fn: Callable[..., AsyncIterator[Event]]) -> Callable[..., AsyncIterator[Event]]:
+ @functools.wraps(gen_fn)
+ async def wrapper(self: "TurnEngine", *args: Any, trigger: str = "unknown", **kwargs: Any) -> AsyncIterator[Event]:
+ inner = gen_fn(self, *args, **kwargs)
+ obs = getattr(self, "_obs", None)
+ if obs is None or not obs.enabled:
+ try:
+ async for event in inner:
+ yield event
+ finally:
+ await inner.aclose()
+ return
+ started = False
+ try:
+ async for event in inner:
+ if event.type is EventType.TURN_START and not started:
+ started = True
+ self._begin_recorded_turn(kind, trigger)
+ if started and obs.in_turn:
+ self._observe_event(event)
+ yield event
+ finally:
+ await inner.aclose()
+ if started:
+ obs.end_turn("interrupted" if self._cancel.is_set() else "unknown")
+
+ return wrapper
+
+ return deco
+
+
def _action_failed(action: dict[str, Any]) -> bool:
"""A listed action that changed nothing: the call errored, the command exited
non-zero, or it timed out. A background start is neither done nor failed."""
@@ -435,6 +481,10 @@ class TurnEngine:
self._step = 0
self._last_context_tokens: Optional[int] = None
self.audit_context: dict[str, Any] = {}
+ # Observability (E1): the turn and step events this engine sends to its recorder.
+ # Off until a recorder is set (`engine.recorder = ...`), like the other surface hooks.
+ self._obs = TurnRecording(None, lambda: str(self.audit_context.get("session_id") or ""))
+ self._call_index: dict[str, int] = {}
if instructions and not (
self.messages and self.messages[0].get("role") == "system"
):
@@ -491,6 +541,90 @@ class TurnEngine:
finally:
self._tool_timings.setdefault(tool_call.id, {}).update(waited_started=started, waited_ms=(time.monotonic() - clock) * 1000)
+ @property
+ def recorder(self) -> Any:
+ return self._obs.recorder
+
+ @recorder.setter
+ def recorder(self, value: Any) -> None:
+ self._obs.recorder = value
+
+ def _begin_recorded_turn(self, kind: str, trigger: str) -> None:
+ """`turn_started` (E1, technical design §4). For `run` the turn opens with your
+ message; for `resume`, with the assistant message whose calls are resumed; a `retry`
+ opens with the first message it adds, so its start waits for that."""
+ first: Optional[str] = None
+ if kind == "run":
+ first = next((m.get("mid") for m in reversed(self.messages) if m.get("role") == "user"), None)
+ elif kind == "resume":
+ first = next(
+ (m.get("mid") for m in reversed(self.messages)
+ if m.get("role") == "assistant" and m.get("tool_calls")),
+ None,
+ )
+ reviewer_kind = (
+ _REVIEWER_KINDS.get(getattr(self, "reviewer_choice", "session"), "session_model")
+ if self.reviewer is not None else None
+ )
+ try:
+ attended = bool(self.is_attended and self.is_attended())
+ except Exception: # noqa: BLE001 - a broken getter must not break the turn
+ attended = False
+ self._obs.begin_turn(
+ kind=kind,
+ trigger=trigger,
+ attrs={
+ "approval_mode": self.permissions.mode.value,
+ "attended": attended,
+ "reviewer": {
+ "active": self._reviewer_active() if self.reviewer is not None else False,
+ "shadow": bool(self.reviewer_shadow),
+ "kind": reviewer_kind,
+ "model": getattr(self.reviewer, "model", None) if reviewer_kind == "session_model" else None,
+ },
+ },
+ stamp={},
+ first_message_id=first,
+ wait_for_first_message=kind == "retry",
+ )
+
+ def _observe_event(self, event: Event) -> None:
+ """Ids on a UI event, a call that never ran as a zero-length tool step, and the turn's
+ end (E1). The app ignores fields it does not know, so older screens keep working."""
+ obs = self._obs
+ event.data.setdefault("turn_id", obs.turn_id)
+ event.data.setdefault("ts_ms", time.time_ns() // 1_000_000)
+ if event.type in (EventType.TOOL_STARTED, EventType.TOOL_FINISHED):
+ call_id = str(event.data.get("tool_call_id") or "")
+ if event.type is EventType.TOOL_FINISHED and call_id:
+ mid = next(
+ (m.get("mid") for m in reversed(self.messages)
+ if m.get("role") == "tool" and m.get("tool_call_id") == call_id),
+ None,
+ )
+ obs.tool_finished(
+ event.data, message_id=mid, call_index=self._call_index.get(call_id),
+ attrs={"risk_class": self._risk_class(str(event.data.get("name") or ""))},
+ )
+ step_id = obs.step_for_call(call_id)
+ if step_id:
+ event.data.setdefault("step_id", step_id)
+ if event.type is EventType.TURN_END:
+ obs.end_turn(str(event.data.get("status") or "completed"))
+ elif event.type is EventType.INTERRUPTED:
+ obs.end_turn("interrupted")
+ elif event.type is EventType.ERROR:
+ obs.end_turn("error")
+
+ def _risk_class(self, tool_name: str) -> Optional[str]:
+ try:
+ spec = self.registry.get(tool_name)
+ return _classify_risk(
+ tool_name, spec.metadata if spec else None, self.permissions.risk_overrides
+ ).value
+ except Exception: # noqa: BLE001 - a fact, never a reason to fail a call
+ return None
+
def _add_message(self, message: dict[str, Any], *, at: Optional[int] = None) -> None:
"""Append (or, with `at`, insert) one message into the history, stamping its `mid`.
@@ -503,6 +637,9 @@ class TurnEngine:
self.messages.append(message)
else:
self.messages.insert(at, message)
+ obs = getattr(self, "_obs", None)
+ if obs is not None:
+ obs.note_message(message)
def _timed_result(self, tool_call, result):
message = _tool_result_message(tool_call, result)
@@ -517,6 +654,7 @@ class TurnEngine:
self._steering.append((text, source, activity))
# -- main loop --------------------------------------------------------------
+ @_records_turn("run")
@_summarizes_turn
async def run(
self,
@@ -654,6 +792,7 @@ class TurnEngine:
notice.update({k: v for k, v in fields.items() if v is not None})
self._add_message(notice)
+ @_records_turn("retry")
@_summarizes_turn
async def retry(self) -> AsyncIterator[Event]:
"""Re-run the model loop after a provider error — no new user message; the failed
@@ -693,6 +832,7 @@ class TurnEngine:
info = workspace.describe()
yield Event(EventType.SANDBOX_READY, {k: info.get(k) for k in ("provider", "enforcement", "reason", "sandbox") if info.get(k) is not None})
+ @_records_turn("resume")
@_summarizes_turn
async def resume(self) -> AsyncIterator[Event]:
"""Continue a turn that was suspended at a prompt and persisted — durable resume after a
@@ -1269,6 +1409,16 @@ class TurnEngine:
# sequential because approval cards are interactive and must reach the human one
# at a time, in call order.
self._shell_cleared_in_batch = False
+ # Each call's position in the model's list for this reply (canonical step order).
+ ids = {tc.id for tc in tool_calls}
+ listed = next(
+ (m["tool_calls"] for m in reversed(self.messages)
+ if m.get("role") == "assistant" and m.get("tool_calls")
+ and ids & {tc.get("id") for tc in m["tool_calls"]}),
+ [],
+ )
+ order = [tc.get("id") for tc in listed] or [tc.id for tc in tool_calls]
+ self._call_index = {cid: i for i, cid in enumerate(order) if cid}
await self._preconsult_reviewer(tool_calls)
cleared: list[ToolCall] = []
for tool_call in tool_calls:
@@ -1342,7 +1492,7 @@ class TurnEngine:
if concurrent:
for tool_call in concurrent:
- yield Event(EventType.TOOL_STARTED, {"name": tool_call.name})
+ yield Event(EventType.TOOL_STARTED, {"name": tool_call.name, "tool_call_id": tool_call.id})
self._audit(tool_call, stage="started")
outcomes = await asyncio.gather(
*[asyncio.to_thread(self._execute_sync, tc) for tc in concurrent]
@@ -1354,7 +1504,7 @@ class TurnEngine:
if self._cancel.is_set():
yield self._interrupted_tool(tool_call)
continue
- yield Event(EventType.TOOL_STARTED, {"name": tool_call.name})
+ yield Event(EventType.TOOL_STARTED, {"name": tool_call.name, "tool_call_id": tool_call.id})
self._audit(tool_call, stage="started")
result, status = await asyncio.to_thread(self._execute_sync, tool_call)
yield self._record_result(tool_call, result, status)
@@ -1387,7 +1537,7 @@ class TurnEngine:
self._audit(tool_call, stage="finished", status="error", reason=reason)
return Event(
EventType.TOOL_FINISHED,
- {"name": tool_call.name, "status": "error", "reason": reason},
+ {"name": tool_call.name, "tool_call_id": tool_call.id, "status": "error", "reason": reason},
)
def _interrupted_tool(self, tool_call: ToolCall) -> Event:
@@ -2296,7 +2446,34 @@ class TurnEngine:
return "read"
def _execute_sync(self, tool_call: ToolCall) -> tuple[Any, str]:
- """Execute one authorized call (runs in a worker thread)."""
+ """Execute one authorized call (runs in a worker thread). Recorded as a tool step
+ from here to the RAW result, before `_record_result` shortens it (E1)."""
+ step_id = self._obs.start_step(
+ kind="tool", name=tool_call.name, call_id=tool_call.id,
+ call_index=self._call_index.get(tool_call.id),
+ )
+ result, status = self._execute_call(tool_call)
+ if step_id is not None:
+ self._obs.end_step(
+ step_id, status=status, attrs=self._tool_step_attrs(tool_call, result),
+ wait_for_tool_message=True,
+ )
+ return result, status
+
+ def _tool_step_attrs(self, tool_call: ToolCall, result: Any) -> dict[str, Any]:
+ attrs: dict[str, Any] = {"risk_class": self._risk_class(tool_call.name)}
+ try:
+ attrs["result_bytes"] = len(
+ result.encode("utf-8") if isinstance(result, str)
+ else json.dumps(result, default=str, ensure_ascii=False).encode("utf-8")
+ )
+ except Exception: # noqa: BLE001
+ pass
+ if isinstance(result, dict) and isinstance(result.get("exit_code"), int):
+ attrs["exit_code"] = result["exit_code"]
+ return attrs
+
+ def _execute_call(self, tool_call: ToolCall) -> tuple[Any, str]:
started, clock = time.time(), time.monotonic()
try:
delegated = self._authorized_delegates.pop(tool_call.id, None)
@@ -2584,7 +2761,7 @@ class TurnEngine:
self._audit(tool_call, stage="finished", status=status, result=result, result_preview=_preview(result))
yield Event(
EventType.TOOL_FINISHED,
- {"name": tool_call.name, "status": status, "result_preview": _preview(result)},
+ {"name": tool_call.name, "tool_call_id": tool_call.id, "status": status, "result_preview": _preview(result)},
)
async def _handle_team_proposal(self, tool_call: ToolCall) -> AsyncIterator[Event]:
@@ -2721,6 +2898,7 @@ class TurnEngine:
EventType.TOOL_FINISHED,
{
"name": tool_call.name,
+ "tool_call_id": tool_call.id,
"status": status,
"result_preview": _preview(result),
},
@@ -2915,6 +3093,7 @@ class TurnEngine:
EventType.TOOL_FINISHED,
{
"name": tool_call.name,
+ "tool_call_id": tool_call.id,
"status": status,
"result_preview": _preview(result),
},
@@ -3010,6 +3189,7 @@ class TurnEngine:
EventType.TOOL_FINISHED,
{
"name": tool_call.name,
+ "tool_call_id": tool_call.id,
"status": status,
"result_preview": _preview(result),
},
@@ -3091,6 +3271,7 @@ class TurnEngine:
EventType.TOOL_FINISHED,
{
"name": tool_call.name,
+ "tool_call_id": tool_call.id,
"status": status,
"result_preview": _preview(result),
},
diff --git a/coworker/observability/engine_hooks.py b/coworker/observability/engine_hooks.py
new file mode 100644
index 00000000..26a2a94e
--- /dev/null
+++ b/coworker/observability/engine_hooks.py
@@ -0,0 +1,229 @@
+"""The engine's side of recording (observability E1): builds the recording events a
+`TurnEngine` sends and hands them to its `Recorder`.
+
+The engine calls a handful of methods here at its hook points (a turn starts or ends, a tool
+starts or ends, a message is added). This module turns those calls into the five events of
+recording-format.md section 2, with `v`, `at_ms`, `mono_ms`, and the copy of the step's
+identity that every `step_ended` carries. With no recorder every method returns at once.
+
+Two events wait for a fact that arrives later:
+- A `retry` turn's `turn_started` waits for the first message the retry adds, whose `mid` it
+ carries (`first_message_id`). Until then the turn's other events are held, in order; their
+ times were taken when they happened, so holding them changes nothing in the recording.
+- A tool step's `step_ended` is built when the tool returns (its time is then) and sent once
+ the tool's result message is added, so it can carry that message's `mid`.
+"""
+
+from __future__ import annotations
+
+import copy
+import threading
+from typing import Any, Callable, Optional
+
+from .ids import new_ulid
+from .recorder import EVENT_V, Recorder, now_ms
+
+# TOOL_FINISHED status -> tool step status (technical design §5.1).
+_TOOL_STATUS = {"ok": "ok", "error": "error", "denied": "denied", "interrupted": "interrupted",
+ "skipped": "skipped"}
+
+
+class TurnRecording:
+ """One per engine. Thread-safe: tool steps start and end in tool threads."""
+
+ def __init__(self, recorder: Optional[Recorder], session_id: Callable[[], str]):
+ self.recorder = recorder
+ self._session_id = session_id
+ self._lock = threading.RLock()
+ self.turn_id: Optional[str] = None
+ self._turn_session = ""
+ self._ended = True
+ self._held: Optional[list[dict[str, Any]]] = None # events behind a waiting turn_started
+ self._pending_start: Optional[dict[str, Any]] = None
+ self._last_mid: Optional[str] = None
+ self._ident: dict[str, dict[str, Any]] = {} # step_id -> identity copy
+ self._tool_steps: dict[str, str] = {} # call_id -> step_id, for this turn's tools
+ self._tool_ends: dict[str, dict[str, Any]] = {} # call_id -> step_ended awaiting its mid
+
+ @property
+ def enabled(self) -> bool:
+ return self.recorder is not None
+
+ # -- sending ------------------------------------------------------------------------------
+ def _base(self, typ: str, turn_id: Optional[str], session_id: str) -> dict[str, Any]:
+ at, mono = now_ms()
+ return {"type": typ, "v": EVENT_V, "at_ms": at, "mono_ms": mono,
+ "session_id": session_id, "turn_id": turn_id or ""}
+
+ def _send(self, ev: dict[str, Any]) -> None:
+ if self._held is not None and ev["type"] != "turn_started" and ev.get("turn_id") == self.turn_id:
+ self._held.append(ev)
+ else:
+ self.recorder.emit(ev)
+
+ def _release(self, first_message_id: Optional[str]) -> None:
+ """Send the waiting turn_started with its first message id, then what it held."""
+ if self._pending_start is None:
+ return
+ ev, held = self._pending_start, self._held or []
+ self._pending_start, self._held = None, None
+ ev["first_message_id"] = first_message_id
+ self.recorder.emit(ev)
+ for e in held:
+ self.recorder.emit(e)
+
+ # -- turns ----------------------------------------------------------------------------------
+ def begin_turn(self, *, kind: str, trigger: str, attrs: dict[str, Any], stamp: dict[str, Any],
+ first_message_id: Optional[str], wait_for_first_message: bool = False) -> str:
+ """`turn_started`, now. With `wait_for_first_message` it is held until `note_message`
+ sees the turn's first message (a retry) or the turn ends."""
+ with self._lock:
+ self.turn_id = new_ulid()
+ self._turn_session = self._session_id() or ""
+ self._ended = False
+ self._last_mid = None
+ self._tool_steps.clear()
+ self._tool_ends.clear()
+ ev = self._base("turn_started", self.turn_id, self._turn_session)
+ ev.update({"kind": kind, "trigger": trigger, "first_message_id": first_message_id,
+ "attrs": copy.deepcopy(attrs), "stamp": copy.deepcopy(stamp)})
+ if wait_for_first_message:
+ self._pending_start, self._held = ev, []
+ else:
+ self.recorder.emit(ev)
+ return self.turn_id
+
+ def end_turn(self, status: str) -> None:
+ with self._lock:
+ if self._ended or self.turn_id is None:
+ return
+ self._ended = True
+ self._release(None) # a retry that added no message at all
+ for ev in self._tool_ends.values(): # a tool whose result message never came
+ self.recorder.emit(ev)
+ self._tool_ends.clear()
+ ev = self._base("turn_ended", self.turn_id, self._turn_session)
+ ev.update({"status": status, "last_message_id": self._last_mid})
+ self.recorder.emit(ev)
+
+ @property
+ def in_turn(self) -> bool:
+ return self.turn_id is not None and not self._ended
+
+ def step_for_call(self, call_id: str) -> Optional[str]:
+ """The tool step recorded for a call in this turn, if any."""
+ with self._lock:
+ return self._tool_steps.get(call_id) or None
+
+ def note_message(self, message: dict[str, Any]) -> None:
+ """Every message the engine adds (E21). Tracks the turn's last `mid`, releases a
+ waiting `turn_started`, and sends a tool step's end once its result message exists."""
+ if not self.enabled:
+ return
+ mid = message.get("mid")
+ with self._lock:
+ if not self.in_turn:
+ return
+ self._last_mid = mid
+ if self._pending_start is not None:
+ self._release(mid)
+ if message.get("role") == "tool":
+ ev = self._tool_ends.pop(str(message.get("tool_call_id") or ""), None)
+ if ev is not None:
+ ev["message_id"] = mid
+ self._send(ev)
+
+ # -- steps ----------------------------------------------------------------------------------
+ def start_step(self, *, kind: str, name: str, call_id: Optional[str] = None,
+ call_index: Optional[int] = None, parent_step_id: Optional[str] = None,
+ attrs: Optional[dict[str, Any]] = None,
+ content: Optional[dict[str, Any]] = None) -> Optional[str]:
+ if not self.enabled:
+ return None
+ with self._lock:
+ if not self.in_turn:
+ return None
+ step_id = new_ulid()
+ ev = self._base("step_started", self.turn_id, self._turn_session)
+ ev.update({"step_id": step_id, "parent_step_id": parent_step_id, "kind": kind,
+ "name": name, "call_id": call_id, "call_index": call_index,
+ "shadow": False, "after_turn": False, "attrs": attrs or {},
+ "content": content or {}})
+ self._ident[step_id] = {
+ "kind": kind, "name": name, "parent_step_id": parent_step_id, "call_id": call_id,
+ "call_index": call_index, "shadow": False, "after_turn": False,
+ "started_at": self._started_at(ev), "turn_id": self.turn_id,
+ "session_id": self._turn_session,
+ }
+ if kind == "tool" and call_id:
+ self._tool_steps[call_id] = step_id
+ self._send(ev)
+ return step_id
+
+ def _started_at(self, ev: dict[str, Any]) -> int:
+ # The recorder anchors a step's start to its turn's start (§7.3); the identity copy
+ # uses the wall clock, which is what a recorder without the turn would use.
+ return ev["at_ms"]
+
+ def end_step(self, step_id: Optional[str], *, status: str = "ok",
+ attrs: Optional[dict[str, Any]] = None, content: Optional[dict[str, Any]] = None,
+ message_id: Optional[str] = None, error: Optional[dict[str, Any]] = None,
+ wait_for_tool_message: bool = False) -> None:
+ if not self.enabled or step_id is None:
+ return
+ with self._lock:
+ idn = self._ident.pop(step_id, None)
+ if idn is None:
+ return
+ ev = self._base("step_ended", idn["turn_id"], idn["session_id"])
+ ev.update({"step_id": step_id, "status": status, "message_id": message_id,
+ "attrs": attrs or {}, "content": content or {}})
+ if error is not None:
+ ev["error"] = error
+ ev.update({k: idn[k] for k in ("kind", "name", "parent_step_id", "call_id",
+ "call_index", "shadow", "after_turn", "started_at")})
+ if wait_for_tool_message and idn["call_id"] and not self._ended:
+ self._tool_ends[idn["call_id"]] = ev
+ else:
+ self._send(ev)
+
+ def instant_step(self, *, kind: str, name: str, status: str, call_id: Optional[str] = None,
+ call_index: Optional[int] = None, attrs: Optional[dict[str, Any]] = None,
+ message_id: Optional[str] = None) -> None:
+ """A zero-length step at the moment of a decision: a call that never ran. Its start
+ and end carry the same clocks."""
+ if not self.enabled:
+ return
+ with self._lock:
+ if not self.in_turn:
+ return
+ step_id = new_ulid()
+ start = self._base("step_started", self.turn_id, self._turn_session)
+ start.update({"step_id": step_id, "parent_step_id": None, "kind": kind, "name": name,
+ "call_id": call_id, "call_index": call_index, "shadow": False,
+ "after_turn": False, "attrs": {}, "content": {}})
+ end = {k: start[k] for k in ("v", "at_ms", "mono_ms", "session_id", "turn_id")}
+ end.update({"type": "step_ended", "step_id": step_id, "status": status,
+ "message_id": message_id, "attrs": attrs or {}, "content": {},
+ "kind": kind, "name": name, "parent_step_id": None, "call_id": call_id,
+ "call_index": call_index, "shadow": False, "after_turn": False,
+ "started_at": start["at_ms"]})
+ self._send(start)
+ self._send(end)
+
+ def tool_finished(self, data: dict[str, Any], *, message_id: Optional[str],
+ call_index: Optional[int], attrs: dict[str, Any]) -> None:
+ """A TOOL_FINISHED for a call that never reached `_execute_sync`: its zero-length step."""
+ if not self.enabled:
+ return
+ call_id = str(data.get("tool_call_id") or "")
+ with self._lock:
+ if not self.in_turn or not call_id or call_id in self._tool_steps:
+ return
+ self._tool_steps[call_id] = ""
+ status = _TOOL_STATUS.get(str(data.get("status") or ""), "error")
+ if "superseded_worker_call" in data:
+ status = "skipped"
+ self.instant_step(kind="tool", name=str(data.get("name") or ""), status=status,
+ call_id=call_id, call_index=call_index, attrs=attrs,
+ message_id=message_id)
diff --git a/coworker/server/app.py b/coworker/server/app.py
index e2bb9a99..a0c20f98 100644
--- a/coworker/server/app.py
+++ b/coworker/server/app.py
@@ -3228,10 +3228,11 @@ def create_app(manager: SessionManager) -> FastAPI:
)
return
events = (
- engine.retry()
+ engine.retry(trigger="user")
if retry
else engine.run(content, display=display,
- activity=manager.prepare_activity(session_id, "user activity"))
+ activity=manager.prepare_activity(session_id, "user activity"),
+ trigger="user")
)
async for event in events:
data = event.data
diff --git a/coworker/server/manager.py b/coworker/server/manager.py
index 907bfc26..de48be02 100644
--- a/coworker/server/manager.py
+++ b/coworker/server/manager.py
@@ -1006,6 +1006,8 @@ class SessionManager:
auto_approve_shadow=self.auto_approve_shadow(),
auto_approve_reviewer=self.auto_approve_reviewer(),
)
+ # Observability (E1): this engine's turns and steps go to the server's recorder.
+ engine.recorder = self.recorder
engine.delegated_approval = lambda args: self.worker_review_context(session_id, args)
engine.reviewer_context = lambda: self.approval_context(session_id)
engine.reviewer_owner_history = lambda: self.approval_owner_history(session_id, engine)
@@ -2004,7 +2006,7 @@ class SessionManager:
return
self.mark_running(item.session_id)
try:
- async for _event in engine.resume():
+ async for _event in engine.resume(trigger="user"):
pass
self.save(item.session_id, engine)
finally:
@@ -6960,6 +6962,8 @@ class SessionManager:
[d] if (d := self.persona_skill_scope(task.agent)[0]) is not None else None
),
)
+ # Observability (E1): this engine's turns and steps go to the server's recorder.
+ engine.recorder = self.recorder
self._seed_task_permissions(engine, task)
return engine
@@ -7231,7 +7235,15 @@ class SessionManager:
# semantics intact while grouping check-ins in the transcript.
source = self._board_source(team, message, rows=activity["board_rows"])
source["board"]["check_in"] = True
- async for event in engine.run(message, source=source, activity=activity):
+ # Who started the turn (observability §4): board activity and team timers wake a
+ # team; a session's own timer is a schedule; anything else came from a channel.
+ if board_delivery:
+ trigger = "team_wake"
+ elif wake is not None:
+ trigger = "team_wake" if self.teams.for_lead_session(session_id) is not None else "schedule"
+ else:
+ trigger = "channel"
+ async for event in engine.run(message, source=source, activity=activity, trigger=trigger):
# Stream every event to any socket viewing this session, so a background turn
# (channel delivery, self-wake, durable resume) is seen live — not just on reselect.
await self.broadcast_session(
@@ -7806,7 +7818,7 @@ class SessionManager:
"scheduled tasks.\n\n"
f"{task.instructions}"
)
- async for _event in engine.run(opening):
+ async for _event in engine.run(opening, trigger="schedule"):
pass
run.result_text = _last_assistant_text(engine.messages)
run.artifacts = _recent_files(task.workspace, since=run.started_at)
diff --git a/coworker/tui/app.py b/coworker/tui/app.py
index 4f7d069c..65c8997c 100644
--- a/coworker/tui/app.py
+++ b/coworker/tui/app.py
@@ -156,7 +156,7 @@ class CoworkerApp(App):
async def run_turn(self, text: str) -> None:
assert self.engine is not None
try:
- async for event in self.engine.run(text):
+ async for event in self.engine.run(text, trigger="user"):
self._render_event(event)
except Exception as exc: # pragma: no cover - surfaced to the user
self._write(f"[red]error:[/red] {exc}")
diff --git a/tests/test_automation.py b/tests/test_automation.py
index 3a013493..438e95fd 100644
--- a/tests/test_automation.py
+++ b/tests/test_automation.py
@@ -493,7 +493,7 @@ async def test_cancelled_scheduled_run_is_not_left_running(tmp_path, monkeypatch
class BlockingEngine:
messages = []
- async def run(self, _opening):
+ async def run(self, _opening, **_kw):
started.set()
await asyncio.Event().wait()
yield
@@ -573,7 +573,7 @@ async def test_cancelled_completion_notification_preserves_success(
class CompletingEngine:
messages = [{"role": "assistant", "content": "done"}]
- async def run(self, _opening):
+ async def run(self, _opening, **_kw):
if False:
yield
diff --git a/tests/test_observability_engine.py b/tests/test_observability_engine.py
new file mode 100644
index 00000000..1975d92d
--- /dev/null
+++ b/tests/test_observability_engine.py
@@ -0,0 +1,187 @@
+"""Observability E1: a real engine turn, recorded.
+
+The engine sends `turn_started` / `turn_ended` around run, retry and resume, a tool step for
+every call that runs (from `_execute_sync` to the raw result), a zero-length tool step for a
+call that never ran, and ids on its UI events. The fold rules are checked by the golden test;
+this checks that the engine sends the right events.
+"""
+
+from __future__ import annotations
+
+import asyncio
+
+import aisuite as ai
+
+from coworker.engine import TurnEngine
+from coworker.events import EventType
+from coworker.observability import Recorder
+from coworker.permissions import PermissionEngine
+from coworker.providers import AssistantTurn, ModelCapabilities, ProviderClient, ToolCall
+from coworker.tools import ToolRegistry
+
+
+class ScriptedProvider(ProviderClient):
+ def __init__(self, turns):
+ self._turns = list(turns)
+
+ def complete(self, *, model, messages, tools=None, **settings):
+ turn = self._turns.pop(0)
+ if isinstance(turn, Exception):
+ raise turn
+ return turn
+
+ def capabilities(self, model):
+ return ModelCapabilities()
+
+
+def _text(text):
+ return AssistantTurn(text=text, finish_reason="stop")
+
+
+def _tools(*calls):
+ return AssistantTurn(
+ tool_calls=[ToolCall(id=cid, name=name, arguments=args) for cid, name, args in calls],
+ finish_reason="tool_calls",
+ )
+
+
+def _engine(tmp_path, turns):
+ registry = ToolRegistry()
+ registry.register_all(ai.toolkits.files(root=str(tmp_path), allow_write=True))
+ engine = TurnEngine(
+ provider=ScriptedProvider(turns),
+ registry=registry,
+ permissions=PermissionEngine(workspace_root=tmp_path),
+ model="gpt-5.5",
+ )
+ engine.audit_context = {"session_id": "sess_1"}
+ rec = Recorder()
+ events: list[dict] = []
+ rec.add_listener(events.append)
+ engine.recorder = rec
+ return engine, rec, events
+
+
+def _drain(gen):
+ async def go():
+ return [ev async for ev in gen]
+
+ return asyncio.run(go())
+
+
+def test_a_turn_with_tools_is_recorded(tmp_path):
+ (tmp_path / "a.txt").write_text("hello", encoding="utf-8")
+ (tmp_path / "b.txt").write_text("world", encoding="utf-8")
+ engine, rec, _ = _engine(tmp_path, [
+ _tools(("c1", "read_file", {"path": "a.txt"}), ("c2", "read_file", {"path": "b.txt"})),
+ _text("done"),
+ ])
+ ui = _drain(engine.run("read both", trigger="user"))
+ d = rec.as_dict()
+
+ [turn] = d["turns"]
+ user = next(m for m in engine.messages if m["role"] == "user")
+ assert (turn["kind"], turn["trigger"], turn["status"], turn["session_id"]) == (
+ "run", "user", "completed", "sess_1")
+ assert turn["first_message_id"] == user["mid"]
+ assert turn["last_message_id"] == engine.messages[-1]["mid"]
+ assert turn["approval_mode"] == engine.permissions.mode.value
+ assert turn["reviewer"] == {"active": False, "shadow": False, "kind": None, "model": None}
+ assert turn["duration_ms"] is not None
+
+ tools = [s for s in d["steps"] if s["kind"] == "tool"]
+ assert [(s["call_id"], s["call_index"], s["status"]) for s in tools] == [
+ ("c1", 0, "ok"), ("c2", 1, "ok")]
+ by_call = {m.get("tool_call_id"): m["mid"] for m in engine.messages if m["role"] == "tool"}
+ for s in tools:
+ assert s["message_id"] == by_call[s["call_id"]]
+ assert s["attrs"]["risk_class"] == "read"
+ assert s["attrs"]["result_bytes"] > 0
+ assert s["duration_ms"] is not None and s["turn_id"] == turn["turn_id"]
+
+ # UI events carry the turn id, a time, and a tool's call and step ids.
+ assert all(e.data.get("turn_id") == turn["turn_id"] for e in ui)
+ started = [e for e in ui if e.type is EventType.TOOL_STARTED]
+ finished = [e for e in ui if e.type is EventType.TOOL_FINISHED]
+ assert {e.data["tool_call_id"] for e in started} == {"c1", "c2"}
+ assert {e.data["step_id"] for e in finished} == {s["step_id"] for s in tools}
+ assert all(isinstance(e.data["ts_ms"], int) for e in ui)
+
+
+def test_every_step_ended_carries_its_identity(tmp_path):
+ (tmp_path / "a.txt").write_text("hello", encoding="utf-8")
+ engine, _, events = _engine(tmp_path, [_tools(("c1", "read_file", {"path": "a.txt"})), _text("ok")])
+ _drain(engine.run("go"))
+ ends = [e for e in events if e["type"] == "step_ended"]
+ assert ends
+ for e in ends:
+ assert {"v", "at_ms", "mono_ms", "kind", "name", "call_id", "started_at"} <= set(e)
+ assert [e["type"] for e in events][0] == "turn_started"
+ assert [e["type"] for e in events][-1] == "turn_ended"
+
+
+def test_a_call_that_never_ran_is_a_zero_length_step(tmp_path):
+ engine, rec, _ = _engine(tmp_path, [
+ _tools(("c1", "read_file", {"_raw": "{not json"})),
+ _text("sorry"),
+ ])
+ _drain(engine.run("go"))
+ [step] = [s for s in rec.as_dict()["steps"] if s["kind"] == "tool"]
+ assert (step["call_id"], step["status"], step["duration_ms"]) == ("c1", "error", 0)
+ tool_msg = next(m for m in engine.messages if m.get("tool_call_id") == "c1")
+ assert step["message_id"] == tool_msg["mid"]
+
+
+def test_a_failed_model_call_ends_the_turn_as_error_and_retry_is_its_own_turn(tmp_path):
+ engine, rec, _ = _engine(tmp_path, [RuntimeError("provider down"), _text("back")])
+ _drain(engine.run("hello", trigger="user"))
+ _drain(engine.retry(trigger="user"))
+ first, second = rec.as_dict()["turns"]
+ assert (first["kind"], first["status"]) == ("run", "error")
+ assert (second["kind"], second["status"], second["seq"]) == ("retry", "completed", 2)
+ # A retry opens with the first message it adds: the model's reply.
+ reply = engine.messages[-1]
+ assert second["first_message_id"] == reply["mid"] == second["last_message_id"]
+
+
+def test_an_abandoned_turn_is_closed_as_unknown(tmp_path):
+ (tmp_path / "a.txt").write_text("hello", encoding="utf-8")
+ engine, rec, _ = _engine(tmp_path, [_tools(("c1", "read_file", {"path": "a.txt"})), _text("ok")])
+
+ async def go():
+ gen = engine.run("go")
+ async for ev in gen:
+ if ev.type is EventType.TURN_START:
+ break
+ await gen.aclose() # the surface stopped reading
+
+ asyncio.run(go())
+ [turn] = rec.as_dict()["turns"]
+ assert turn["status"] == "unknown" and turn["duration_ms"] is not None
+
+
+def test_no_recorder_means_no_ids_on_events(tmp_path):
+ engine, _, _ = _engine(tmp_path, [_text("hi")])
+ engine.recorder = None
+ ui = _drain(engine.run("hello", trigger="user"))
+ assert all("turn_id" not in e.data for e in ui)
+
+
+def test_a_turn_through_the_server_lands_in_recordings_db(tmp_path):
+ from fastapi.testclient import TestClient
+
+ from coworker.server import SessionManager, create_app
+
+ manager = SessionManager(workspace=tmp_path, provider=ScriptedProvider([_text("hi there")]))
+ client = TestClient(create_app(manager))
+ with client.websocket_connect("/ws/session/obs1") as ws:
+ assert ws.receive_json()["type"] == "ready"
+ ws.send_json({"type": "user_message", "text": "hello"})
+ while ws.receive_json()["type"] != "turn_done":
+ pass
+ [turn] = manager.recordings_store.load_recording("obs1")["turns"]
+ assert (turn["kind"], turn["trigger"], turn["status"]) == ("run", "user", "completed")
+ assert manager.recordings_store.path.parent == manager.audit_store.db_path.parent
+
+ manager.delete_session("obs1")
+ assert manager.recordings_store.load_recording("obs1") == {"turns": [], "steps": []}