Python
fix(observability): fixes from the second in-app check (OPE-244)
Devika Vermacc15eff4f74c
diff --git a/coworker/agent.py b/coworker/agent.pyindex 12a27a8f..21602c21 100644--- a/coworker/agent.py+++ b/coworker/agent.py@@ -705,6 +705,10 @@ def build_engine(skills_ctx = skill_catalog_text(skill_loader, allowed=allowed)if skills_ctx:parts.append(skills_ctx)+ if eng is not None:+ # The menu the model is offered this turn, for the recording (observability §5.6).+ eng.skills_offered = [c["name"] for c in skill_loader.catalog()+ if allowed is None or c["name"] in allowed]# Disable countermand (§3): instructions already loaded into this conversation keep# steering the model even after the skill is turned off/deleted — history can't be# un-read. So a loaded-but-no-longer-available skill gets an explicit stop note,diff --git a/coworker/engine.py b/coworker/engine.pyindex b7984c47..191fb2e9 100644--- a/coworker/engine.py+++ b/coworker/engine.py@@ -742,7 +742,11 @@ class TurnEngine:returnblock = self._last_context_block or ""block_hash = _short_hash(block) if block else None- skills = sorted(getattr(getattr(self, "skill_loader", None), "_skills", None) or {})+ # The menu this turn's block offered (set by the agent's context provider, which+ # leaves out skills disabled for the session); else every skill the loader found.+ offered = getattr(self, "skills_offered", None)+ skills = sorted(offered if offered is not None+ else getattr(getattr(self, "skill_loader", None), "_skills", None) or {})per_turn = {"mode_note": self.permissions.mode.value if self.permissions.mode in (Mode.PLAN, Mode.DISCUSS) else None,"folders": len(getattr(self.permissions, "roots", None) or []),@@ -786,8 +790,19 @@ class TurnEngine:path = self._decisions.pop(call_id, None)if path is not None and obs.step_for_call(call_id) is None:attrs["decision"] = path.finish(ran=event.data.get("status") == "ok")+ elif path is None:+ # An interactive tool (question, plan, folder…) skips the permission+ # flow: no stage decided it (recording-format: origin null).+ attrs["decision"] = {"path": [], "origin": None}+ data = event.data+ answered_by_stop = path is not None and (path.stage("resolved") or {}).get("outcome") == "interrupted"+ if (data.get("status") == "denied" and self._cancel.is_set()+ and (data.get("name") in _WAIT_KINDS or answered_by_stop)):+ # A Stop while a card waited (approval, question, plan, folder) is not a+ # refusal: the handler answers "denied", the step says interrupted (§5.3).+ data = {**data, "status": "interrupted"}obs.tool_finished(- event.data, message_id=mid, call_index=self._call_index.get(call_id),+ data, message_id=mid, call_index=self._call_index.get(call_id),attrs=attrs,)step_id = obs.step_for_call(call_id)@@ -2093,8 +2108,18 @@ class TurnEngine:attrs["breaker"] = {"denials_in_row": denials, "tripped": denials >= _REVIEWER_TRIP}self._review_steps[tool_call.id] = step_idreason = getattr(verdict, "reason", "") or ""- self._obs.end_step(step_id, status="error" if failed else "ok", attrs=attrs,- content={"reason": reason} if reason else {})+ content: dict[str, Any] = {"reason": reason} if reason else {}+ # A decision model's answers: each question's probability of "yes" (§5.5, §5.9).+ # There is no single score or cut-off (several questions, each with its own+ # threshold), so those stay null.+ probabilities = getattr(verdict, "probabilities", None)+ if isinstance(probabilities, dict) and probabilities:+ content["questions"] = [+ {"name": name, "answer": "yes" if float(p) >= 0.5 else "no", "score": round(float(p), 4)}+ for name, p in probabilities.items()+ if not str(name).startswith("_") and isinstance(p, (int, float))+ ]+ self._obs.end_step(step_id, status="error" if failed else "ok", attrs=attrs, content=content)return verdictdef _review_context_changed(self, call_id: str) -> None:@@ -2316,6 +2341,13 @@ class TurnEngine:allowed=allowed, needs_user=decision.needs_user, human_only=decision.human_only,reason=reason, tier=decision.tier if allowed else None,)+ if not allowed and decision.needs_user:+ try:+ # A session grant covered it, but Auto-Approve hands it to the reviewer (§8.3).+ if self.permissions.session_grant_suspended(tool_call.name, tool_call.arguments, metadata):+ path.add("grant", "suspended_in_auto_approve")+ except Exception: # noqa: BLE001 - a recording fact never breaks a call+ pass# OPE-233: an Auto-Approve gate tier let this call run without the reviewer —# recorded, never invisible (transcript chip + audit row), like every other@@ -2802,8 +2834,12 @@ class TurnEngine:blocked = result.pop("_blocked", None) if isinstance(result, dict) else Noneif step_id is not None:attrs = self._tool_step_attrs(tool_call, {**result, "_blocked": blocked} if blocked else result)+ step_status = status+ if self._cancel.is_set() and isinstance(result, dict) and result.get("error"):+ # Stop killed it (a shell command answers "interrupted by user"): §7.4.+ step_status = "interrupted"self._obs.end_step(- step_id, status=status, attrs=attrs,+ step_id, status=step_status, attrs=attrs,content={"blocked": blocked} if blocked else {},wait_for_tool_message=True,)diff --git a/coworker/observability/calculator.py b/coworker/observability/calculator.pyindex 433e40c3..f6334c69 100644--- a/coworker/observability/calculator.py+++ b/coworker/observability/calculator.py@@ -134,9 +134,17 @@ def buckets(recording: dict[str, Any], turn: dict[str, Any], now: Optional[int]to the first bucket in the order waiting, tools, model, auto-approver, summary; Other isthe rest, shown when it is 1% of the total or more (M-R5)."""start = turn["started_at"]- end_of_turn = (start + turn["duration_ms"] if turn.get("duration_ms") is not None- else (now if now is not None else start))- steps = [s for s in recording.get("steps") or [] if s.get("turn_id") == turn["turn_id"]]+ # Steps found open when the app started have no time (status unknown): not counted (TD-12).+ steps = [s for s in recording.get("steps") or []+ if s.get("turn_id") == turn["turn_id"] and s.get("status") != "unknown"]+ if turn.get("duration_ms") is not None:+ end_of_turn = start + turn["duration_ms"]+ elif turn.get("status") == "unknown":+ # A turn found open at start: it ran at least until its last known step end.+ end_of_turn = max([start] + [s["started_at"] + s["duration_ms"] for s in steps+ if s.get("duration_ms") is not None])+ else:+ end_of_turn = now if now is not None else start# A step still open is counted up to the turn's end (or now, for a running turn; TD-12).spans = [(s, s["started_at"], s["started_at"] + s["duration_ms"] if s.get("duration_ms") is not Noneelse end_of_turn) for s in steps]diff --git a/coworker/observability/provider_view.py b/coworker/observability/provider_view.pyindex 7b429cd8..abdc78f4 100644--- a/coworker/observability/provider_view.py+++ b/coworker/observability/provider_view.py@@ -195,10 +195,22 @@ def _provider_title(provider: str) -> Optional[str]:def _context_window(model: str) -> Optional[int]:+ """The window the engine compacts against (`TurnEngine._compaction_config`): the+ catalogue, else the user's saved size, else an Ollama model's `num_ctx`. A local+ server's window is left out: finding it may mean a request to that server."""try:from ..providers.matrix import model_context_windows- return model_context_windows().get(model)+ window = model_context_windows().get(model)+ if not window:+ from .. import model_config++ window = model_config.context_size_for(model)+ if not window:+ from ..providers.ollama_context import context_window_for++ window = context_window_for(model)+ return int(window) if window else Noneexcept Exception: # noqa: BLE001return Nonediff --git a/coworker/observability/recorder.py b/coworker/observability/recorder.pyindex 271b11a5..d90dda4c 100644--- a/coworker/observability/recorder.py+++ b/coworker/observability/recorder.py@@ -124,6 +124,24 @@ class Step:return {k: copy.deepcopy(getattr(self, k)) for k in STEP_FIELDS}+def fill_unended(kind: str, attrs: dict[str, Any]) -> dict[str, Any]:+ """A step closed without its own end (at its turn's end, or found open at start, §7.4)+ still has every field its kind requires: what only the end would have told is "not+ known" (recording-format null rules)."""+ if kind == "model":+ attrs.setdefault("finish_reason", None)+ attrs.setdefault("ttft_ms", None)+ attrs.setdefault("usage", {"reported": False})+ elif kind == "tool":+ attrs.setdefault("decision", {"path": [], "origin": None})+ elif kind == "auto_approver":+ attrs.setdefault("result", None)+ elif kind == "summary":+ for key in ("tokens_before", "model_step_id"):+ attrs.setdefault(key, None)+ return attrs++def _merge_top(into: dict[str, Any], new: Optional[dict[str, Any]]) -> None:"""step_updated / step_ended merge: each top-level key replaces the same key."""for k, v in (new or {}).items():@@ -264,6 +282,7 @@ class Recorder:and not s.after_turn):s.duration_ms = max(0, turn.ended_at - s.started_at)s.status, s.closed_by_turn = "interrupted", True+ fill_unended(s.kind, s.attrs)self._assign_seq()self._write_turn(turn)for s in self.steps.values():diff --git a/coworker/observability/store.py b/coworker/observability/store.pyindex ae19c55e..16290ad4 100644--- a/coworker/observability/store.py+++ b/coworker/observability/store.py@@ -265,6 +265,55 @@ class RecordingStore:).fetchone()return None if row is None else zlib.decompress(row["data"])+ # -- found open at start (TD-12) ------------------------------------------------------------+ def close_open_turns(self) -> int:+ """Close the turns and steps a previous run of the app left open (it crashed, was+ killed or restarted mid-turn): status `unknown` with no duration, the turn's steps+ numbered in canonical order, and a `recording_gap` notice at the turn's last known+ moment. Only the process that owns the data folder may call this, at start: a+ running turn of another process would look the same. Returns the turns closed."""+ from .ids import new_ulid+ from .recorder import KIND_ORDER, fill_unended++ with self._lock:+ self.flush()+ open_turns = self._conn.execute(+ "SELECT turn_id, session_id, started_at FROM turns WHERE status = 'running' OR status IS NULL"+ ).fetchall()+ if not open_turns:+ return 0+ with self._conn:+ for t in open_turns:+ tid = t["turn_id"]+ self._conn.execute(+ "UPDATE turns SET status = 'unknown', duration_ms = NULL WHERE turn_id = ?", (tid,))+ opened = self._conn.execute(+ "SELECT step_id, kind, attrs FROM steps "+ "WHERE turn_id = ? AND (status = 'running' OR status IS NULL)", (tid,)).fetchall()+ self._conn.executemany(+ "UPDATE steps SET status = 'unknown', duration_ms = NULL, attrs = ? WHERE step_id = ?",+ [(json.dumps(fill_unended(o["kind"], json.loads(o["attrs"] or "{}"))), o["step_id"])+ for o in opened])+ steps = self._conn.execute(+ "SELECT step_id, kind, name, started_at, duration_ms, call_index, call_id, shadow "+ "FROM steps WHERE turn_id = ?", (tid,)).fetchall()+ # The last moment known: a step's end, or the start of one still open.+ last = max([t["started_at"] or 0] + [s["started_at"] + (s["duration_ms"] or 0) for s in steps+ if s["started_at"] is not None])+ gap = {"step_id": new_ulid(), "kind": "notice", "name": "recording_gap", "started_at": last,+ "duration_ms": 0, "call_index": None, "call_id": None, "shadow": 0}+ self._conn.execute(+ "INSERT INTO steps (step_id, turn_id, session_id, kind, name, started_at, duration_ms, "+ "status, shadow, after_turn, attrs, content) VALUES (?, ?, ?, 'notice', 'recording_gap', ?, 0, "+ "'ok', 0, 0, '{}', '{}')", (gap["step_id"], tid, t["session_id"], last))+ ordered = sorted([dict(s) for s in steps] + [gap], key=lambda s: (+ s["started_at"] or 0, 0 if s["duration_ms"] == 0 else 1, KIND_ORDER.get(s["kind"], 9),+ s["call_index"] or 0, s["name"] or "", s["call_id"] or "", bool(s["shadow"])))+ self._conn.executemany("UPDATE steps SET seq = ? WHERE step_id = ?",+ [(i, s["step_id"]) for i, s in enumerate(ordered, 1)])+ log.info("recordings: closed %d turn(s) left open by an earlier run", len(open_turns))+ return len(open_turns)+# -- deleting -----------------------------------------------------------------------------def delete_session(self, session_id: str) -> None:"""The session-delete cascade (§11): its turns, steps and the pieces only they used."""diff --git a/coworker/permissions.py b/coworker/permissions.pyindex 99a13402..417edcf1 100644--- a/coworker/permissions.py+++ b/coworker/permissions.py@@ -755,6 +755,29 @@ class PermissionEngine:out.append((Path(p).expanduser().resolve(), w))return out+ def session_grant_suspended(+ self, tool_name: str, arguments: dict[str, Any], metadata: Any = None+ ) -> bool:+ """Whether a session or run grant covers this call but Auto-Approve set it aside+ (§1.5: in-flow clicks never skip the reviewer). The same checks `evaluate` makes+ when grants are honoured; read by the observability decision path."""+ if self.mode is not Mode.AUTO_APPROVE:+ return False+ arguments = arguments or {}+ risk = classify(tool_name, metadata, self.risk_overrides)+ is_connector = getattr(metadata, "category", "") == "connector"+ if risk is RiskClass.EXEC:+ command = str(arguments.get("command", ""))+ return bool(command) and (+ command in self.session_allow_commands+ or (self.session_readonly and self._readonly_grant_covers(command))+ )+ if risk is RiskClass.EGRESS:+ url = str(arguments.get("url", ""))+ if self._domain_allowed(url) and not self._domain_allowed(url, include_session=False):+ return True+ return (tool_name in self.session_allow_tools and not is_connector) or tool_name in self.run_allow_tools+def evaluate(self, tool_name: str, arguments: dict[str, Any], metadata: Any = None) -> Decision:diff --git a/coworker/remote/joiner.py b/coworker/remote/joiner.pyindex 3bec75c6..fa7c423b 100644--- a/coworker/remote/joiner.py+++ b/coworker/remote/joiner.py@@ -536,6 +536,7 @@ def _build_local_app(state: Path):os.environ.pop("COWORKER_API_TOKEN", None)cfg = load_config()manager = SessionManager(data_dir=state, model=cfg.model, mode=Mode(cfg.mode))+ manager.recordings_store.close_open_turns() # the engine lock is held (observability TD-12)return create_app(manager)diff --git a/coworker/server/run.py b/coworker/server/run.pyindex 16ad182a..b765fbc3 100644--- a/coworker/server/run.py+++ b/coworker/server/run.py@@ -99,13 +99,18 @@ def _watch_parent_windows(parent: int) -> None:threading.Thread(target=watch, daemon=True).start()-def build_app(workspace: str | None, model: str, mode: str):+def build_app(workspace: str | None, model: str, mode: str, *, owns_state: bool = False):manager = SessionManager(workspace=Path(workspace).expanduser().resolve() if workspace else None,data_dir=state_dir(),model=model,mode=Mode(mode),)+ if owns_state:+ # Turns an earlier run left open (crash, restart mid-turn) are closed as unknown+ # (observability TD-12). Only with the engine lock: a second server on the same+ # folder may have turns running right now.+ manager.recordings_store.close_open_turns()return create_app(manager)@@ -222,7 +227,7 @@ def main(argv=None) -> None:_exit_when_orphaned()_warn_if_state_shared()- app = build_app(args.cwd, args.model, args.mode)+ app = build_app(args.cwd, args.model, args.mode, owns_state=_ENGINE_LOCK is not None)uvicorn.run(app, host=args.host, port=args.port, ws_max_size=_WS_MAX_FRAME_BYTES)diff --git a/tests/test_observability_model_steps.py b/tests/test_observability_model_steps.pyindex 02bc2764..8a3cf761 100644--- a/tests/test_observability_model_steps.py+++ b/tests/test_observability_model_steps.py@@ -286,3 +286,20 @@ def test_a_streamed_model_step_is_closed_before_the_engine_sees_the_reply():[step] = [s for s in rec.as_dict()["steps"] if s["kind"] == "model"]assert step["status"] == "ok" and step["attrs"]["finish_reason"] == "stop"assert step["attrs"]["ttft_ms"] is not None+++def test_the_context_window_is_the_one_the_engine_compacts_against(monkeypatch):+ # Round-2 in-app check: an Ollama model is not in the catalogue, so its steps had no+ # window and the Usage tab hid "Context now". The step records the window the engine+ # uses: the catalogue, else the user's saved size, else Ollama's num_ctx.+ from coworker import model_config+ from coworker.observability.provider_view import _context_window+ from coworker.providers import ollama_context++ monkeypatch.setattr("coworker.providers.matrix.model_context_windows", lambda: {"openai:model-a": 400_000})+ monkeypatch.setattr(model_config, "context_size_for", lambda m: 64_000 if m == "openrouter:model-b" else None)+ monkeypatch.setitem(ollama_context._resolved, "model-c", 40_960)+ assert _context_window("openai:model-a") == 400_000+ assert _context_window("openrouter:model-b") == 64_000+ assert _context_window("ollama:model-c") == 40_960+ assert _context_window("together:model-d") is Nonediff --git a/tests/test_observability_reviewer_helper.py b/tests/test_observability_reviewer_helper.pyindex 0b26126d..2f901920 100644--- a/tests/test_observability_reviewer_helper.py+++ b/tests/test_observability_reviewer_helper.py@@ -242,4 +242,30 @@ def test_a_decision_reviewer_check_has_a_model_step_with_usage(tmp_path):assert inner["attrs"]["provider_meta"][0]["wire"] == "typesafe_jev"assert inner["attrs"]["provider_facts"]["provider_request_id"] == "req_jev"assert check["attrs"]["usage_reported"] is True+ # Each question's answer (round-2 in-app check: these were not recorded).+ questions = {q["name"]: q for q in check["content"]["questions"]}+ assert questions["requested"] == {"name": "requested", "answer": "yes", "score": 0.97}+ assert all(set(q) == {"name", "answer", "score"} for q in questions.values())+ assert check["attrs"].get("score") is None and check["attrs"].get("cutoff") is Noneassert "test-key-not-real" not in json.dumps(steps)+++def test_a_session_grant_set_aside_in_auto_approve_is_in_the_path(tmp_path):+ # Technical design §8.3 worked example (S1-05, call C): a command allowed for the+ # session is still judged in Auto-approve; the path says the grant was set aside.+ engine, rec = _engine(tmp_path, [_call("c1", "run_shell", {"command": "make"}), AssistantTurn(text="ok")],+ [json.dumps({"verdict": "unsure", "reason": "r"})])+ engine.permissions.session_allow_commands.add("make")+ _run(engine)+ [tool] = [s for s in rec.as_dict()["steps"] if s["kind"] == "tool"]+ stages = [(p["stage"], p["outcome"]) for p in tool["attrs"]["decision"]["path"]]+ assert stages[:3] == [("permission", "needs_user"), ("grant", "suspended_in_auto_approve"), ("auto_approver", "unsure")]+ assert tool["attrs"]["decision"]["origin"] == "user"+++def test_no_grant_stage_without_a_matching_grant(tmp_path):+ engine, rec = _engine(tmp_path, [_call("c1", "run_shell", {"command": "make"}), AssistantTurn(text="ok")],+ [json.dumps({"verdict": "allow", "reason": "r"})])+ _run(engine)+ [tool] = [s for s in rec.as_dict()["steps"] if s["kind"] == "tool"]+ assert "grant" not in [p["stage"] for p in tool["attrs"]["decision"]["path"]]diff --git a/tests/test_observability_schema.py b/tests/test_observability_schema.pyindex b35c59e6..4d53715b 100644--- a/tests/test_observability_schema.py+++ b/tests/test_observability_schema.py@@ -118,3 +118,51 @@ def test_context_notices_stamps_and_sandbox_facts(tmp_path):engine, rec = stamps._engine(tmp_path, [RuntimeError("boom")])stamps._run(engine)_check(rec.as_dict())+++def test_interactive_tools_and_steps_closed_by_a_stop_fit_the_schema(tmp_path):+ # Round-2 in-app check: interactive tools (plan, question, folder) had no `decision`, and+ # a step closed at its turn's end kept none of its kind's end fields.+ import asyncio as _asyncio+ from coworker.permissions import Mode++ engine, rec = waits._engine(tmp_path, [waits._call("p1", "propose_plan", {"plan": "do it"}),+ AssistantTurn(text="x")], mode=Mode.PLAN)++ async def never(*a):+ await _asyncio.sleep(30)++ engine.plan_approver = never++ async def go():+ async def stop_soon():+ await _asyncio.sleep(0.3)+ engine.request_interrupt()+ task = _asyncio.create_task(stop_soon())+ [ev async for ev in engine.run("plan", trigger="user")]+ await task++ _asyncio.run(go())+ [plan] = [s for s in rec.as_dict()["steps"] if s["name"] == "propose_plan"]+ assert plan["attrs"]["decision"] == {"path": [], "origin": None}+ _check(rec.as_dict())+++def test_a_model_step_still_open_at_the_turns_end_fits_the_schema():+ from coworker.observability import Recorder++ rec = Recorder()+ base = {"v": 1, "session_id": "s", "turn_id": "t"}+ rec.emit({**base, "type": "turn_started", "at_ms": 1000, "mono_ms": 0, "kind": "run", "trigger": "user",+ "first_message_id": "m1", "stamp": {},+ "attrs": {"approval_mode": "interactive", "attended": True,+ "reviewer": {"active": False, "shadow": False, "kind": None, "model": None}}})+ rec.emit({**base, "type": "step_started", "step_id": "x", "at_ms": 1010, "mono_ms": 10, "kind": "model",+ "name": "model", "parent_step_id": None, "call_id": None, "call_index": None, "shadow": False,+ "after_turn": False, "content": {},+ "attrs": {"purpose": "title", "provider": "openai", "provider_title": "OpenAI", "model": "openai:model-a",+ "stream": False, "context_window": None, "tools_offered": 0, "tools_hash": None}})+ rec.emit({**base, "type": "turn_ended", "at_ms": 1100, "mono_ms": 100, "status": "completed", "last_message_id": "m2"})+ [step] = rec.as_dict()["steps"]+ assert step["status"] == "interrupted" and step["attrs"]["usage"] == {"reported": False}+ _check(rec.as_dict())diff --git a/tests/test_observability_stamps_context.py b/tests/test_observability_stamps_context.pyindex 41f0b46a..72d06638 100644--- a/tests/test_observability_stamps_context.py+++ b/tests/test_observability_stamps_context.py@@ -186,3 +186,28 @@ def test_network_note_keeps_the_counts(tmp_path, monkeypatch):assert note["blocked_sites"] == ["semgrep.dev:443"]assert note["_blocked"] == [{"host": "semgrep.dev", "port": 443, "count": 2, "first_at": 100000,"late": False, "uncertain": True}]+++def test_the_skills_menu_is_the_one_offered_this_turn(tmp_path):+ # Round-2 in-app check: a skill disabled mid-session left the menu, but the context step+ # still counted every skill on disk. The step counts what the turn's block offered.+ from coworker.agent import build_engine+ from coworker.agents.chat import chat_agent++ skills = tmp_path / "skills"+ for name in ("alpha", "beta"):+ (skills / name).mkdir(parents=True)+ (skills / name / "SKILL.md").write_text(+ f"---\nname: {name}\ndescription: The {name} skill.\n---\nDo {name}.\n", encoding="utf-8")+ offered = {"alpha", "beta"}+ (tmp_path / "w").mkdir()+ engine = build_engine(agent=chat_agent(), model="openai:model-a", workspace=tmp_path / "w",+ provider=_Scripted([AssistantTurn(text="a"), AssistantTurn(text="b")]),+ extra_skill_dirs=[skills], skill_filter=lambda: set(offered))+ engine.recorder = rec = Recorder()+ _run(engine, "one")+ offered.discard("beta") # turned off in Settings between the two messages+ _run(engine, "two")+ menus = [s["attrs"]["per_turn"] for s in rec.as_dict()["steps"] if s["kind"] == "context"]+ assert [m["skills_menu"] for m in menus] == [2, 1]+ assert menus[0]["skills_hash"] != menus[1]["skills_hash"]diff --git a/tests/test_observability_store.py b/tests/test_observability_store.pyindex eaad44c0..68ad30e5 100644--- a/tests/test_observability_store.py+++ b/tests/test_observability_store.py@@ -283,3 +283,63 @@ def test_every_store_shares_one_writer_thread(tmp_path):finally:for s in stores:s.close()+++def test_turns_left_open_by_a_crash_are_closed_as_unknown_at_start(tmp_path):+ # TD-12 (round-2 in-app check: a restart while an approval card waited left the turn+ # "running" for good). At start, an open turn and its open steps become `unknown` with+ # no duration, every step gets its place, and a recording_gap notice marks the gap.+ import jsonschema, json+ from pathlib import Path+ from coworker.observability import calculator++ path = tmp_path / "recordings.db"+ s1 = RecordingStore(path, background=False)+ rec = Recorder(store=s1)+ _turn(rec, "t1", at=1000, mono=0)+ _model_step(rec, "m1", "t1", at=1010, mono=10, end=60) # 50 ms on the step clock+ rec.emit({"v": 1, "type": "step_started", "session_id": "sess_A", "turn_id": "t1", "step_id": "w1",+ "at_ms": 1070, "mono_ms": 70, "kind": "wait", "name": "approval", "parent_step_id": None,+ "call_id": "c1", "call_index": 0, "shadow": False, "after_turn": False,+ "attrs": {"for_call_id": "c1", "kind": "approval", "where": "session", "attended": True}, "content": {}})+ s1.flush()+ s1.close() # the app is killed here++ s2 = RecordingStore(path, background=False)+ try:+ assert s2.close_open_turns() == 1+ assert s2.close_open_turns() == 0 # nothing left open+ r = s2.load_recording("sess_A")+ [turn] = r["turns"]+ assert (turn["status"], turn["duration_ms"]) == ("unknown", None)+ by = {s["name"]: s for s in r["steps"]}+ assert (by["approval"]["status"], by["approval"]["duration_ms"]) == ("unknown", None)+ assert by["model"]["status"] == "ok"+ assert by["recording_gap"]["kind"] == "notice" and by["recording_gap"]["started_at"] == 1070 # the open wait's start+ # zero-length first at the same moment (recording-format §4)+ assert [s["name"] for s in r["steps"]] == ["model", "recording_gap", "approval"]+ assert [s["seq"] for s in r["steps"]] == [1, 2, 3]+ schema = json.loads((Path(__file__).parent / "corpora/observability/recording-format.schema.json").read_text("utf-8"))+ errors = jsonschema.validators.validator_for(schema)(schema).iter_errors(r)+ # The helper's model step is deliberately minimal; the turn, the closed wait and the+ # notice must fit the schema exactly.+ model_at = [s["name"] for s in r["steps"]].index("model")+ assert not [e for e in errors if list(e.absolute_path)[:2] != ["steps", model_at]]+ b = calculator.buckets(r, turn) # the unknown wait is not counted; the turn ran until it started+ assert (b["total"], b["model"], b["waiting"]) == (70, 50, 0)+ finally:+ s2.close()+++def test_a_running_turn_is_not_closed_without_being_asked(tmp_path):+ # Only the process that owns the folder closes open turns; opening the store never does.+ path = tmp_path / "recordings.db"+ s1 = RecordingStore(path, background=False)+ _turn(Recorder(store=s1), "t1")+ s1.flush()+ s2 = RecordingStore(path, background=False)+ try:+ assert s2.load_recording("sess_A")["turns"][0]["status"] == "running"+ finally:+ s1.close()+ s2.close()diff --git a/tests/test_observability_waits.py b/tests/test_observability_waits.pyindex c997ab5b..37d1b04f 100644--- a/tests/test_observability_waits.py+++ b/tests/test_observability_waits.py@@ -208,3 +208,75 @@ def test_no_summary_step_when_nothing_was_summarised(tmp_path, monkeypatch):assert asyncio.run(go()) is Noneassert _steps(rec, "summary") == []+++def test_a_stop_while_a_plan_card_waits_is_interrupted_not_refused(tmp_path):+ # Round-2 in-app check: the plan handler answers "denied" when the turn is stopped, so+ # the propose_plan step read as refused. A Stop is recorded as interrupted.+ engine, rec = _engine(tmp_path, [_call("p1", "propose_plan", {"plan": "do it"}), AssistantTurn(text="x")],+ mode=Mode.PLAN)++ async def never(args, call_id):+ await asyncio.sleep(30)++ engine.plan_approver = never++ async def go():+ async def stop_soon():+ await asyncio.sleep(0.3)+ engine.request_interrupt()+ task = asyncio.create_task(stop_soon())+ [ev async for ev in engine.run("plan", trigger="user")]+ await task++ asyncio.run(go())+ [wait] = _steps(rec, "wait")+ [plan] = [s for s in _steps(rec, "tool") if s["name"] == "propose_plan"]+ assert wait["attrs"]["outcome"] == "interrupted"+ assert plan["status"] == "interrupted"+++def test_a_stop_while_an_approval_card_waits_is_interrupted_not_refused(tmp_path):+ # Round-2 in-app check (F3): the wait and the path said interrupted, the tool step denied.+ engine, rec = _engine(tmp_path, [_call("c1", "run_shell", {"command": "make"}), AssistantTurn(text="x")])++ async def never(request):+ await asyncio.sleep(30)++ engine.approver = never++ async def go():+ async def stop_soon():+ await asyncio.sleep(0.3)+ engine.request_interrupt()+ task = asyncio.create_task(stop_soon())+ [ev async for ev in engine.run("go", trigger="user")]+ await task++ asyncio.run(go())+ [tool] = _steps(rec, "tool")+ assert tool["status"] == "interrupted"+ assert tool["attrs"]["decision"]["path"][-1] == {"stage": "resolved", "outcome": "interrupted"}+++def test_a_tool_killed_by_stop_is_interrupted(tmp_path):+ # Round-2 in-app check (F2): a shell command killed by Stop answers "interrupted by+ # user" and the step said ok.+ engine, rec = _engine(tmp_path, [_call("c1", "run_shell", {"command": "sleep 40"}), AssistantTurn(text="x")],+ mode=Mode.BYPASS_APPROVALS)++ def run_shell(command: str) -> dict:+ """Run a command.++ Args:+ command: the command line+ """+ engine.request_interrupt() # the Stop arrives while it runs; it is killed+ return {"command": command, "exit_code": None, "error": "interrupted by user"}++ registry = ToolRegistry()+ registry.register(run_shell, metadata=_Meta())+ engine.registry = registry+ _run(engine)+ [tool] = _steps(rec, "tool")+ assert tool["status"] == "interrupted"