Python
Merge origin/main into the observability branch
Devika Verma8783c340fd24
diff --cc coworker/engine.pyindex dc952f41,a27243cd..74195132--- a/coworker/engine.py+++ b/coworker/engine.py@@@ -650,321 -483,12 +650,321 @@@ class TurnEnginefinally:cancel_wait.cancel()- async def _wait_tool(self, tool_call, coro, interrupted):+ async def _wait_tool(self, tool_call, coro, interrupted, kind: Optional[str] = None):- started, clock = time.time(), time.monotonic()+ started, clock = time.time(), time.perf_counter()+ # Observability E20: a wait step from the moment the card is shown to the answer, so+ # a slow answer is never mistaken for a slow agent (§5.4).+ kind = kind or _WAIT_KINDS.get(tool_call.name, "approval")try:- return await self._interruptible(coro, interrupted)+ attended = bool(self.is_attended and self.is_attended())+ except Exception: # noqa: BLE001 - a broken getter must not break the wait+ attended = False+ step_id = self._obs.start_step(+ kind="wait", name=kind, call_id=tool_call.id,+ attrs={"for_call_id": tool_call.id, "kind": kind, "attended": attended,+ "where": "session" if attended else "inbox",+ "inbox_at": None if attended else int(started * 1000)},+ )+ self._obs.track_wait(tool_call.id, step_id)+ result: Any = interrupted+ failed = False+ try:+ result = await self._interruptible(coro, interrupted)+ return result+ except BaseException:+ failed = True+ raisefinally:- self._tool_timings.setdefault(tool_call.id, {}).update(waited_started=started, waited_ms=(time.monotonic() - clock) * 1000)+ self._tool_timings.setdefault(tool_call.id, {}).update(waited_started=started, waited_ms=(time.perf_counter() - clock) * 1000)+ self._obs.end_wait(tool_call.id)+ if step_id is not None:+ who = ""+ if self.resolver_of is not None:+ try:+ who = str(self.resolver_of(tool_call.id) or "")+ except Exception: # noqa: BLE001+ who = ""+ self._obs.end_step(+ step_id, status="error" if failed else "ok",+ attrs={"outcome": "interrupted" if failed else _wait_outcome(+ result, interrupted, kind, stopped=self._cancel.is_set())},+ content={"resolved_by": who} if who else {},+ )++ def note_wait_where(self, call_id: Optional[str], where: str) -> None:+ """Where a waiting call's card is shown, `session` or `inbox` (observability §5.4)."""+ if call_id:+ self._card_where[call_id] = where+ self._obs.wait_where(call_id, where)++ @property+ def recorder(self) -> Any:+ return self._obs.recorder++ @recorder.setter+ def recorder(self, value: Any) -> None:+ self._obs.recorder = value++ def _begin_recorded_turn(self, kind: str, trigger: str,+ woken_by: Optional[dict[str, Any]] = None) -> None:+ """`turn_started` (E1, technical design §4). For `run` the turn opens with your+ message; for `resume`, with the assistant message whose calls are resumed; a `retry`+ opens with the first message it adds, so its start waits for that."""+ first: Optional[str] = None+ if kind == "run":+ first = next((m.get("mid") for m in reversed(self.messages) if m.get("role") == "user"), None)+ elif kind == "resume":+ first = next(+ (m.get("mid") for m in reversed(self.messages)+ if m.get("role") == "assistant" and m.get("tool_calls")),+ None,+ )+ attended, reviewer = self._attended_now(), self._reviewer_facts()+ stamp = self._turn_stamp()+ turn_id = self._obs.begin_turn(+ kind=kind,+ trigger=trigger,+ attrs={+ "approval_mode": self.permissions.mode.value,+ "attended": attended,+ "reviewer": reviewer,+ "max_iterations": self.max_iterations,+ "woken_by": woken_by,+ },+ stamp=stamp,+ first_message_id=first,+ wait_for_first_message=kind == "retry",+ )+ # What `stamp_changed` compares against for the rest of the turn (V2-27).+ self._stamp_seen = {"turn": turn_id, "attended": attended, "reviewer": reviewer,+ "sandbox": stamp.get("sandbox")}++ def _attended_now(self) -> bool:+ try:+ return bool(self.is_attended and self.is_attended())+ except Exception: # noqa: BLE001 - a broken getter must not break the turn+ return False++ def _reviewer_facts(self) -> dict[str, Any]:+ """The turn's `reviewer` (observability §4): as at turn start, or now."""+ reviewer_kind = (+ _REVIEWER_KINDS.get(getattr(self, "reviewer_choice", "session"), "session_model")+ if self.reviewer is not None else None+ )+ return {+ "active": self._reviewer_active() if self.reviewer is not None else False,+ "shadow": bool(self.reviewer_shadow),+ "kind": reviewer_kind,+ "model": getattr(self.reviewer, "model", None) if reviewer_kind == "session_model" else None,+ }++ def _sandbox_stamp(self) -> Optional[dict[str, Any]]:+ """The stamp's sandbox group now, with `expected` (format 1.2) when it is known. The+ turn's stamp and the mid-turn comparison both use this, so they cannot differ by it."""+ sandbox = _stamps.sandbox_stamp(getattr(self, "sandbox_workspace", None))+ if sandbox and self.sandbox_expected is not None:+ sandbox["expected"] = bool(self.sandbox_expected)+ return sandbox++ def _note_stamp(self) -> None:+ """A `stamp_changed` notice when attendance, the reviewer or the sandbox differ from+ the turn's start or the last notice (observability §9, V2-27). The Unattended toggle+ and the reviewer settings apply mid-turn, and the first turn makes the sandbox after+ its stamp was taken, so the runner version arrives here. Only the keys that changed,+ with their new values."""+ obs = self._obs+ seen = getattr(self, "_stamp_seen", None) or {}+ if (not obs.enabled or not obs.in_turn or not getattr(obs, "records_context", True)+ or seen.get("turn") != obs.turn_id):+ return+ changed: dict[str, Any] = {}+ attended, reviewer = self._attended_now(), self._reviewer_facts()+ if attended != seen.get("attended"):+ changed["attended"] = seen["attended"] = attended+ if reviewer != seen.get("reviewer"):+ changed["reviewer"] = seen["reviewer"] = reviewer+ try:+ sandbox = self._sandbox_stamp()+ except Exception: # noqa: BLE001 - a stamp never breaks a turn+ sandbox = seen.get("sandbox")+ if sandbox and sandbox != seen.get("sandbox"):+ changed["stamp"] = {"sandbox": sandbox}+ seen["sandbox"] = sandbox+ if changed:+ obs.instant_step(kind="notice", name="stamp_changed", status="ok", attrs=changed)++ def _turn_stamp(self) -> dict[str, Any]:+ """Facts saved with the turn (observability §9): machine, model, sandbox, coworker."""+ stamp: dict[str, Any] = {}+ try:+ stamp["machine"] = _stamps.machine_stamp()+ stamp["model"] = _stamps.model_stamp(self.model)+ sandbox = self._sandbox_stamp()+ if sandbox:+ stamp["sandbox"] = sandbox+ if self.persona_fp:+ stamp["persona_fp"] = self.persona_fp+ except Exception: # noqa: BLE001 - a stamp never breaks a turn+ logger.exception("observability: could not stamp the turn")+ return stamp++ def _sandbox_note_added(self) -> bool:+ """Whether this turn's context holds the sandbox's own note: the agent's context+ adds `sandbox_workspace.context()` when it returns text. A workspace with no+ sandbox has no such text, so no note (fix list F-1: it was true whenever the+ workspace could describe itself, which the no-sandbox one can)."""+ context = getattr(getattr(self, "sandbox_workspace", None), "context", None)+ if not callable(context):+ return False+ try:+ return bool(context())+ except Exception: # noqa: BLE001 - a recording never breaks a turn+ return False++ def _note_context(self) -> None:+ """The context step at the turn's first model call, and a `context_changed` notice+ when the per-turn block changes later in the turn (observability E5, §5.6)."""+ obs = self._obs+ if not obs.enabled or not obs.in_turn or not getattr(obs, "records_context", True):+ return+ block = self._last_context_block or ""+ block_hash = _short_hash(block) if block else None+ # The menu this turn's block offered (set by the agent's context provider, which+ # leaves out skills disabled for the session); else every skill the loader found.+ offered = getattr(self, "skills_offered", None)+ skills = sorted(offered if offered is not None+ else getattr(getattr(self, "skill_loader", None), "_skills", None) or {})+ per_turn = {+ "mode_note": self.permissions.mode.value if self.permissions.mode in (Mode.PLAN, Mode.DISCUSS) else None,+ "folders": len(getattr(self.permissions, "roots", None) or []),+ "sandbox_note": self._sandbox_note_added(),+ "skills_menu": len(skills),+ "skills_hash": _short_hash(",".join(skills)) if skills else None,+ }+ if self._context_turn != obs.turn_id:+ self._context_turn, self._turn_context_hash = obs.turn_id, block_hash+ system = self.messages[0].get("content") if self.messages and self.messages[0].get("role") == "system" else None+ attrs: dict[str, Any] = {+ "session_prompt_hash": _short_hash(system) if isinstance(system, str) else None,+ "per_turn": per_turn,+ "size_chars": len(block),+ "block_hash": block_hash,+ }+ if self.context_facts and not getattr(self, "_context_session_told", False):+ attrs["session_start"] = dict(self.context_facts)+ self._context_session_told = True+ obs.instant_step(kind="context", name="context", status="ok", attrs=attrs)+ elif block_hash != self._turn_context_hash:+ self._turn_context_hash = block_hash+ obs.instant_step(kind="notice", name="context_changed", status="ok",+ attrs={"per_turn": per_turn, "size_chars": len(block), "block_hash": block_hash})++ def _observe_event(self, event: Event) -> None:+ """Ids on a UI event, a call that never ran as a zero-length tool step, and the turn's+ end (E1). The app ignores fields it does not know, so older screens keep working."""+ obs = self._obs+ event.data.setdefault("turn_id", obs.turn_id)+ event.data.setdefault("ts_ms", time.time_ns() // 1_000_000)+ if event.type is EventType.TOOL_PROPOSED and event.data.get("tool_call_id"):+ self._call_args[str(event.data["tool_call_id"])] = event.data.get("arguments")+ if event.type in (EventType.TOOL_STARTED, EventType.TOOL_FINISHED):+ call_id = str(event.data.get("tool_call_id") or "")+ if event.type is EventType.TOOL_FINISHED and call_id:+ mid = next(+ (m.get("mid") for m in reversed(self.messages)+ if m.get("role") == "tool" and m.get("tool_call_id") == call_id),+ None,+ )+ attrs: dict[str, Any] = {"risk_class": self._risk_class(str(event.data.get("name") or ""))}+ path = self._decisions.pop(call_id, None)+ if path is not None and obs.step_for_call(call_id) is None:+ attrs["decision"] = path.finish(ran=event.data.get("status") == "ok")+ elif path is None:+ # An interactive tool (question, plan, folder…) skips the permission+ # flow: no stage decided it (recording-format: origin null).+ attrs["decision"] = {"path": [], "origin": None}+ data = event.data+ created = (data.get("display") or {}).get("team_created") if data.get("name") == "propose_team" else None+ if created and created.get("team_id"):+ # The team it created and its worker sessions: ids only (1.2, OB-7).+ attrs["team"] = {"team_id": str(created["team_id"]), "worker_session_ids": [+ str(w.get("session_id")) for w in created.get("workers") or [] if w.get("session_id")]}+ if obs.step_for_call(call_id) is None:+ attrs.update(self._call_facts(+ str(data.get("name") or ""), self._call_args.get(call_id), ran=False,+ status=str(data.get("status") or ""), reason=str(data.get("reason") or ""),+ ))+ answered_by_stop = path is not None and (path.stage("resolved") or {}).get("outcome") == "interrupted"+ if (data.get("status") == "denied" and self._cancel.is_set()+ and (data.get("name") in _WAIT_KINDS or answered_by_stop)):+ # A Stop while a card waited (approval, question, plan, folder) is not a+ # refusal: the handler answers "denied", the step says interrupted (§5.3).+ data = {**data, "status": "interrupted"}+ obs.tool_finished(+ data, message_id=mid, call_index=self._call_index.get(call_id),+ attrs=attrs,+ )+ step_id = obs.step_for_call(call_id)+ if step_id:+ event.data.setdefault("step_id", step_id)+ if event.type is EventType.TURN_END:+ status = str(event.data.get("status") or "completed")+ if status in ("max_iterations_exceeded", "truncated"):+ obs.instant_step(kind="notice", name="step_limit" if status == "max_iterations_exceeded" else "truncated",+ status="ok", attrs={})+ self._end_recorded_turn(status)+ elif event.type is EventType.INTERRUPTED:+ self._end_recorded_turn("interrupted")+ elif event.type is EventType.ERROR:+ self._end_recorded_turn("error")++ def _end_recorded_turn(self, status: str) -> None:+ """`turn_ended` with this moment's time. If shadow checks are still running in the+ background, it is sent once they end (at most 30 s later), so their steps are not+ closed as interrupted while they run; the turn's recorded end stays this moment."""+ pending = self._obs.close_turn(status)+ if pending is None:+ return+ running = [t for t in self._shadow_tasks if not t.done()]+ if not running:+ self._obs.emit_turn_ended(pending)+ return++ async def after_shadows() -> None:+ try:+ await asyncio.wait_for(asyncio.gather(*running, return_exceptions=True), 30)+ except Exception: # noqa: BLE001 - a slow shadow check must not lose the turn's end+ pass+ self._obs.emit_turn_ended(pending)++ task = asyncio.ensure_future(after_shadows())+ self._turn_end_tasks.add(task)+ task.add_done_callback(self._turn_end_tasks.discard)++ def _risk_class(self, tool_name: str) -> Optional[str]:+ try:+ spec = self.registry.get(tool_name)+ return _classify_risk(+ tool_name, spec.metadata if spec else None, self.permissions.risk_overrides+ ).value+ except Exception: # noqa: BLE001 - a fact, never a reason to fail a call+ return None++ def _add_message(self, message: dict[str, Any], *, at: Optional[int] = None) -> None:+ """Append (or, with `at`, insert) one message into the history, stamping its `mid`.++ `mid` is a ULID that never changes, so a recorded step can point at the message it+ produced even after a stub is inserted mid-history or the file is rewritten+ (observability E21). It is a sidecar like `ts`: stripped before any provider sees it.+ """+ message.setdefault("mid", new_ulid())+ if at is None:+ self.messages.append(message)+ else:+ self.messages.insert(at, message)+ obs = getattr(self, "_obs", None)+ if obs is not None:+ obs.note_message(message)def _timed_result(self, tool_call, result):message = _tool_result_message(tool_call, result)@@@ -2950,124 -2287,8 +2955,124 @@@return "read"def _execute_sync(self, tool_call: ToolCall) -> tuple[Any, str]:- """Execute one authorized call (runs in a worker thread)."""+ """Execute one authorized call (runs in a worker thread). Recorded as a tool step+ from here to the RAW result, before `_record_result` shortens it (E1)."""+ step_id = self._obs.start_step(+ kind="tool", name=tool_call.name, call_id=tool_call.id,+ call_index=self._call_index.get(tool_call.id),+ )+ # A tool that starts its own work (the explore helper, E6) records it under this step.+ outer = set_tool_run(ToolRun(step_id=step_id, turn_id=self._obs.turn_id,+ view=self.provider, recording=self._obs))+ try:+ result, status = self._execute_call(tool_call)+ finally:+ set_tool_run(outer)+ blocked = result.pop("_blocked", None) if isinstance(result, dict) else None+ if step_id is not None:+ attrs = self._tool_step_attrs(tool_call, {**result, "_blocked": blocked} if blocked else result, status)+ step_status = status+ if self._cancel.is_set() and isinstance(result, dict) and result.get("error"):+ # Stop killed it (a shell command answers "interrupted by user"): §7.4.+ step_status = "interrupted"+ self._obs.end_step(+ step_id, status=step_status, attrs=attrs,+ content={"blocked": blocked} if blocked else {},+ wait_for_tool_message=True,+ )+ return result, status++ def _tool_step_attrs(self, tool_call: ToolCall, result: Any, status: str = "ok") -> dict[str, Any]:+ attrs: dict[str, Any] = {"risk_class": self._risk_class(tool_call.name)}+ path = self._decisions.pop(tool_call.id, None)+ if path is not None:+ attrs["decision"] = path.finish(ran=True)+ try:+ attrs["result_bytes"] = len(+ result.encode("utf-8") if isinstance(result, str)+ else json.dumps(result, default=str, ensure_ascii=False).encode("utf-8")+ )+ except Exception: # noqa: BLE001+ pass+ if isinstance(result, dict) and isinstance(result.get("exit_code"), int):+ attrs["exit_code"] = result["exit_code"]+ attrs["sandbox"] = self._sandbox_facts(result)+ attrs.update(self._call_facts(tool_call.name, tool_call.arguments, ran=True,+ status=status, result=result, risk=attrs["risk_class"]))+ return attrs++ def _call_facts(self, name: str, arguments: Any, *, ran: bool, status: str, result: Any = None,+ reason: str = "", risk: Optional[str] = None) -> dict[str, Any]:+ """The tool fields of recording format 1.2 that the Issues checks read (phase-2+ design §4): ids and flags only, never text. Facts never break a call."""+ out: dict[str, Any] = {}+ try:+ mangled = isinstance(arguments, dict) and "_raw" in arguments+ out["args_hash"] = None if mangled or arguments is None else _args_hash(arguments)+ spec = self.registry.get(name)+ out["repeat_ok"] = bool(getattr(getattr(spec, "metadata", None), "repeat_ok", False))+ out["result_error"] = bool(ran and status == "ok" and isinstance(result, dict) and result.get("error"))+ if status != "error":+ out["error_kind"] = None+ elif ran:+ # Arguments the tool does not take, or a required one missing, fail at the call+ # itself: the model's mistake, not the tool's (found by the labelling judge).+ out["error_kind"] = "raised" if _args_fit(spec, arguments) else "bad_arguments"+ elif mangled:+ out["error_kind"] = "bad_arguments"+ elif spec is None or reason.startswith("unknown tool"):+ out["error_kind"] = "unknown_tool"+ else:+ out["error_kind"] = "raised"+ if not ran:+ out["reaches_network"] = None+ else:+ command = arguments.get("command") if isinstance(arguments, dict) and name == "run_shell" else None+ out["reaches_network"] = bool(+ (risk or self._risk_class(name)) == "egress"+ or (isinstance(command, str) and review_facts.shell_reaches_network(command))+ )+ if name == "todo_write" and isinstance(arguments, dict):+ entries = arguments.get("todos", arguments.get("items")) or []+ items = []+ for entry in entries if isinstance(entries, list) else []:+ text = str(entry.get("content", "")) if isinstance(entry, dict) else str(entry)+ st = entry.get("status", "pending") if isinstance(entry, dict) else "pending"+ st = "done" if st in ("done", "completed") else st if st in ("pending", "in_progress") else "pending"+ items.append({"h": _text_hash(text), "s": st})+ out["plan"] = {"items": items}+ except Exception: # noqa: BLE001 - recording never breaks a call+ logger.exception("observability: could not record a tool call's facts")+ return out++ @staticmethod+ def _sandbox_notice_id(text: Any) -> Optional[str]:+ """`restart` or `regrant` for the sandbox notice a command's result carries, else None."""+ if not isinstance(text, str) or not text:+ return None+ from .sandbox.workspace import REGRANT_NOTICE, RESTART_NOTICE++ return "restart" if text == RESTART_NOTICE else "regrant" if text == REGRANT_NOTICE else None++ def _sandbox_facts(self, result: Any) -> Optional[dict[str, Any]]:+ """Per-call sandbox facts (observability E13, §9.3): whether this sandbox can report+ blocked sites now, and how many connections it refused during the call. None when+ the call did not run in a sandbox."""+ ws = getattr(self, "sandbox_workspace", None)+ if ws is None or not hasattr(ws, "reports_blocked"):+ return None+ blocked = result.get("_blocked") if isinstance(result, dict) else None+ told = result.get("sandbox_notice") if isinstance(result, dict) else None+ return {+ "blocked_known": bool(getattr(ws, "reports_blocked", False)),+ "blocked_count": sum(int(b.get("count") or 0) for b in blocked or []),+ # The restart the agent was told of with this result (OB-13): the same short id+ # as the `sandbox_restarted` notice's `how`.+ "notice": self._sandbox_notice_id(told),+ }++ def _execute_call(self, tool_call: ToolCall) -> tuple[Any, str]:- started, clock = time.time(), time.monotonic()+ started, clock = time.time(), time.perf_counter()try:delegated = self._authorized_delegates.pop(tool_call.id, None)if delegated and delegated != self._delegated_context(tool_call):diff --cc coworker/server/manager.pyindex 05951730,fad31df5..ff0552f0--- a/coworker/server/manager.py+++ b/coworker/server/manager.py@@@ -8523,31 -8195,49 +8526,51 @@@ class SessionManagerfallback stays; the small-talk sentinel leaves auto_title unset so the turn-2retry can run."""try:- turn = await asyncio.to_thread(- tag_provider(engine.provider, purpose="title", turn=turn_id).complete,- model=engine.model,- messages=[- {"role": "system", "content": self._AUTOTITLE_PROMPT},- {- "role": "user",- "content": "\n\n".join(openers)- + (- f"\n\n[the assistant's first reply]\n{assistant}"- if assistant- else ""- ),- },- ],- temperature=0.2,- # Reasoning-routed models spend hidden tokens BEFORE emitting text; a- # tight cap plus default effort yields an empty completion and a silent- # no-op. Effort "none" means "no thinking": every provider's effort- # mapper handles it (OpenAI-vocabulary endpoints get it on the wire,- # Anthropic omits the field; see effort.NO_EFFORT), and 64 leaves- # headroom either way.- max_tokens=64,- reasoning_effort="none",- )+ messages = [+ {"role": "system", "content": self._AUTOTITLE_PROMPT},+ {+ "role": "user",+ "content": "\n\n".join(openers)+ + (+ f"\n\n[the assistant's first reply]\n{assistant}"+ if assistant+ else ""+ ),+ },+ ]++ # Tagged so the recording files these calls under "title" (observability).++ titler = tag_provider(engine.provider, purpose="title", turn=turn_id)+ try:+ turn = await asyncio.to_thread(- engine.provider.complete,++ titler.complete,+ model=engine.model,+ messages=messages,+ temperature=0.2,+ # Reasoning-routed models spend hidden tokens BEFORE emitting text; a+ # tight cap plus default effort yields an empty completion and a silent+ # no-op. Effort "none" means "no thinking": every provider's effort+ # mapper handles it (OpenAI-vocabulary endpoints get it on the wire,+ # Anthropic omits the field; see effort.NO_EFFORT), and 64 leaves+ # headroom either way.+ max_tokens=64,+ reasoning_effort="none",+ )+ except Exception as exc:+ # OPE-201: some endpoints cannot turn reasoning off and refuse "none"+ # (OpenRouter GLM 5.3: 400 "Reasoning is mandatory for this endpoint and+ # cannot be disabled"). Ask once more with the model's own effort, and+ # room for the reasoning it spends before the few words of the title.+ text = str(exc).lower()+ if "reasoning" not in text and "effort" not in text:+ raise+ logger.info("autotitle for %s: retrying without reasoning effort (%s)", session_id, exc)+ turn = await asyncio.to_thread(- engine.provider.complete,++ titler.complete,+ model=engine.model,+ messages=messages,+ temperature=0.2,+ max_tokens=self._AUTOTITLE_REASONING_MAX_TOKENS,+ )raw = (getattr(turn, "text", None) or "").strip()# Sanitize: surrounding quotes off, whitespace collapsed, capped at 60.title = " ".join(raw.strip("\"'“”‘’`").split())