Skip to content
Python

fix(observability): fixes from the second in-app check (OPE-244)

Devika Vermacc15eff4f74c
diff --git a/coworker/agent.py b/coworker/agent.py
index 12a27a8f..21602c21 100644
--- a/coworker/agent.py
+++ b/coworker/agent.py
@@ -705,6 +705,10 @@ def build_engine(
skills_ctx = skill_catalog_text(skill_loader, allowed=allowed)
if skills_ctx:
parts.append(skills_ctx)
+ if eng is not None:
+ # The menu the model is offered this turn, for the recording (observability §5.6).
+ eng.skills_offered = [c["name"] for c in skill_loader.catalog()
+ if allowed is None or c["name"] in allowed]
# Disable countermand (§3): instructions already loaded into this conversation keep
# steering the model even after the skill is turned off/deleted — history can't be
# un-read. So a loaded-but-no-longer-available skill gets an explicit stop note,
diff --git a/coworker/engine.py b/coworker/engine.py
index b7984c47..191fb2e9 100644
--- a/coworker/engine.py
+++ b/coworker/engine.py
@@ -742,7 +742,11 @@ class TurnEngine:
return
block = self._last_context_block or ""
block_hash = _short_hash(block) if block else None
- skills = sorted(getattr(getattr(self, "skill_loader", None), "_skills", None) or {})
+ # The menu this turn's block offered (set by the agent's context provider, which
+ # leaves out skills disabled for the session); else every skill the loader found.
+ offered = getattr(self, "skills_offered", None)
+ skills = sorted(offered if offered is not None
+ else getattr(getattr(self, "skill_loader", None), "_skills", None) or {})
per_turn = {
"mode_note": self.permissions.mode.value if self.permissions.mode in (Mode.PLAN, Mode.DISCUSS) else None,
"folders": len(getattr(self.permissions, "roots", None) or []),
@@ -786,8 +790,19 @@ class TurnEngine:
path = self._decisions.pop(call_id, None)
if path is not None and obs.step_for_call(call_id) is None:
attrs["decision"] = path.finish(ran=event.data.get("status") == "ok")
+ elif path is None:
+ # An interactive tool (question, plan, folder…) skips the permission
+ # flow: no stage decided it (recording-format: origin null).
+ attrs["decision"] = {"path": [], "origin": None}
+ data = event.data
+ answered_by_stop = path is not None and (path.stage("resolved") or {}).get("outcome") == "interrupted"
+ if (data.get("status") == "denied" and self._cancel.is_set()
+ and (data.get("name") in _WAIT_KINDS or answered_by_stop)):
+ # A Stop while a card waited (approval, question, plan, folder) is not a
+ # refusal: the handler answers "denied", the step says interrupted (§5.3).
+ data = {**data, "status": "interrupted"}
obs.tool_finished(
- event.data, message_id=mid, call_index=self._call_index.get(call_id),
+ data, message_id=mid, call_index=self._call_index.get(call_id),
attrs=attrs,
)
step_id = obs.step_for_call(call_id)
@@ -2093,8 +2108,18 @@ class TurnEngine:
attrs["breaker"] = {"denials_in_row": denials, "tripped": denials >= _REVIEWER_TRIP}
self._review_steps[tool_call.id] = step_id
reason = getattr(verdict, "reason", "") or ""
- self._obs.end_step(step_id, status="error" if failed else "ok", attrs=attrs,
- content={"reason": reason} if reason else {})
+ content: dict[str, Any] = {"reason": reason} if reason else {}
+ # A decision model's answers: each question's probability of "yes" (§5.5, §5.9).
+ # There is no single score or cut-off (several questions, each with its own
+ # threshold), so those stay null.
+ probabilities = getattr(verdict, "probabilities", None)
+ if isinstance(probabilities, dict) and probabilities:
+ content["questions"] = [
+ {"name": name, "answer": "yes" if float(p) >= 0.5 else "no", "score": round(float(p), 4)}
+ for name, p in probabilities.items()
+ if not str(name).startswith("_") and isinstance(p, (int, float))
+ ]
+ self._obs.end_step(step_id, status="error" if failed else "ok", attrs=attrs, content=content)
return verdict
def _review_context_changed(self, call_id: str) -> None:
@@ -2316,6 +2341,13 @@ class TurnEngine:
allowed=allowed, needs_user=decision.needs_user, human_only=decision.human_only,
reason=reason, tier=decision.tier if allowed else None,
)
+ if not allowed and decision.needs_user:
+ try:
+ # A session grant covered it, but Auto-Approve hands it to the reviewer (§8.3).
+ if self.permissions.session_grant_suspended(tool_call.name, tool_call.arguments, metadata):
+ path.add("grant", "suspended_in_auto_approve")
+ except Exception: # noqa: BLE001 - a recording fact never breaks a call
+ pass
# OPE-233: an Auto-Approve gate tier let this call run without the reviewer —
# recorded, never invisible (transcript chip + audit row), like every other
@@ -2802,8 +2834,12 @@ class TurnEngine:
blocked = result.pop("_blocked", None) if isinstance(result, dict) else None
if step_id is not None:
attrs = self._tool_step_attrs(tool_call, {**result, "_blocked": blocked} if blocked else result)
+ step_status = status
+ if self._cancel.is_set() and isinstance(result, dict) and result.get("error"):
+ # Stop killed it (a shell command answers "interrupted by user"): §7.4.
+ step_status = "interrupted"
self._obs.end_step(
- step_id, status=status, attrs=attrs,
+ step_id, status=step_status, attrs=attrs,
content={"blocked": blocked} if blocked else {},
wait_for_tool_message=True,
)
diff --git a/coworker/observability/calculator.py b/coworker/observability/calculator.py
index 433e40c3..f6334c69 100644
--- a/coworker/observability/calculator.py
+++ b/coworker/observability/calculator.py
@@ -134,9 +134,17 @@ def buckets(recording: dict[str, Any], turn: dict[str, Any], now: Optional[int]
to the first bucket in the order waiting, tools, model, auto-approver, summary; Other is
the rest, shown when it is 1% of the total or more (M-R5)."""
start = turn["started_at"]
- end_of_turn = (start + turn["duration_ms"] if turn.get("duration_ms") is not None
- else (now if now is not None else start))
- steps = [s for s in recording.get("steps") or [] if s.get("turn_id") == turn["turn_id"]]
+ # Steps found open when the app started have no time (status unknown): not counted (TD-12).
+ steps = [s for s in recording.get("steps") or []
+ if s.get("turn_id") == turn["turn_id"] and s.get("status") != "unknown"]
+ if turn.get("duration_ms") is not None:
+ end_of_turn = start + turn["duration_ms"]
+ elif turn.get("status") == "unknown":
+ # A turn found open at start: it ran at least until its last known step end.
+ end_of_turn = max([start] + [s["started_at"] + s["duration_ms"] for s in steps
+ if s.get("duration_ms") is not None])
+ else:
+ end_of_turn = now if now is not None else start
# A step still open is counted up to the turn's end (or now, for a running turn; TD-12).
spans = [(s, s["started_at"], s["started_at"] + s["duration_ms"] if s.get("duration_ms") is not None
else end_of_turn) for s in steps]
diff --git a/coworker/observability/provider_view.py b/coworker/observability/provider_view.py
index 7b429cd8..abdc78f4 100644
--- a/coworker/observability/provider_view.py
+++ b/coworker/observability/provider_view.py
@@ -195,10 +195,22 @@ def _provider_title(provider: str) -> Optional[str]:
def _context_window(model: str) -> Optional[int]:
+ """The window the engine compacts against (`TurnEngine._compaction_config`): the
+ catalogue, else the user's saved size, else an Ollama model's `num_ctx`. A local
+ server's window is left out: finding it may mean a request to that server."""
try:
from ..providers.matrix import model_context_windows
- return model_context_windows().get(model)
+ window = model_context_windows().get(model)
+ if not window:
+ from .. import model_config
+
+ window = model_config.context_size_for(model)
+ if not window:
+ from ..providers.ollama_context import context_window_for
+
+ window = context_window_for(model)
+ return int(window) if window else None
except Exception: # noqa: BLE001
return None
diff --git a/coworker/observability/recorder.py b/coworker/observability/recorder.py
index 271b11a5..d90dda4c 100644
--- a/coworker/observability/recorder.py
+++ b/coworker/observability/recorder.py
@@ -124,6 +124,24 @@ class Step:
return {k: copy.deepcopy(getattr(self, k)) for k in STEP_FIELDS}
+def fill_unended(kind: str, attrs: dict[str, Any]) -> dict[str, Any]:
+ """A step closed without its own end (at its turn's end, or found open at start, §7.4)
+ still has every field its kind requires: what only the end would have told is "not
+ known" (recording-format null rules)."""
+ if kind == "model":
+ attrs.setdefault("finish_reason", None)
+ attrs.setdefault("ttft_ms", None)
+ attrs.setdefault("usage", {"reported": False})
+ elif kind == "tool":
+ attrs.setdefault("decision", {"path": [], "origin": None})
+ elif kind == "auto_approver":
+ attrs.setdefault("result", None)
+ elif kind == "summary":
+ for key in ("tokens_before", "model_step_id"):
+ attrs.setdefault(key, None)
+ return attrs
+
+
def _merge_top(into: dict[str, Any], new: Optional[dict[str, Any]]) -> None:
"""step_updated / step_ended merge: each top-level key replaces the same key."""
for k, v in (new or {}).items():
@@ -264,6 +282,7 @@ class Recorder:
and not s.after_turn):
s.duration_ms = max(0, turn.ended_at - s.started_at)
s.status, s.closed_by_turn = "interrupted", True
+ fill_unended(s.kind, s.attrs)
self._assign_seq()
self._write_turn(turn)
for s in self.steps.values():
diff --git a/coworker/observability/store.py b/coworker/observability/store.py
index ae19c55e..16290ad4 100644
--- a/coworker/observability/store.py
+++ b/coworker/observability/store.py
@@ -265,6 +265,55 @@ class RecordingStore:
).fetchone()
return None if row is None else zlib.decompress(row["data"])
+ # -- found open at start (TD-12) ------------------------------------------------------------
+ def close_open_turns(self) -> int:
+ """Close the turns and steps a previous run of the app left open (it crashed, was
+ killed or restarted mid-turn): status `unknown` with no duration, the turn's steps
+ numbered in canonical order, and a `recording_gap` notice at the turn's last known
+ moment. Only the process that owns the data folder may call this, at start: a
+ running turn of another process would look the same. Returns the turns closed."""
+ from .ids import new_ulid
+ from .recorder import KIND_ORDER, fill_unended
+
+ with self._lock:
+ self.flush()
+ open_turns = self._conn.execute(
+ "SELECT turn_id, session_id, started_at FROM turns WHERE status = 'running' OR status IS NULL"
+ ).fetchall()
+ if not open_turns:
+ return 0
+ with self._conn:
+ for t in open_turns:
+ tid = t["turn_id"]
+ self._conn.execute(
+ "UPDATE turns SET status = 'unknown', duration_ms = NULL WHERE turn_id = ?", (tid,))
+ opened = self._conn.execute(
+ "SELECT step_id, kind, attrs FROM steps "
+ "WHERE turn_id = ? AND (status = 'running' OR status IS NULL)", (tid,)).fetchall()
+ self._conn.executemany(
+ "UPDATE steps SET status = 'unknown', duration_ms = NULL, attrs = ? WHERE step_id = ?",
+ [(json.dumps(fill_unended(o["kind"], json.loads(o["attrs"] or "{}"))), o["step_id"])
+ for o in opened])
+ steps = self._conn.execute(
+ "SELECT step_id, kind, name, started_at, duration_ms, call_index, call_id, shadow "
+ "FROM steps WHERE turn_id = ?", (tid,)).fetchall()
+ # The last moment known: a step's end, or the start of one still open.
+ last = max([t["started_at"] or 0] + [s["started_at"] + (s["duration_ms"] or 0) for s in steps
+ if s["started_at"] is not None])
+ gap = {"step_id": new_ulid(), "kind": "notice", "name": "recording_gap", "started_at": last,
+ "duration_ms": 0, "call_index": None, "call_id": None, "shadow": 0}
+ self._conn.execute(
+ "INSERT INTO steps (step_id, turn_id, session_id, kind, name, started_at, duration_ms, "
+ "status, shadow, after_turn, attrs, content) VALUES (?, ?, ?, 'notice', 'recording_gap', ?, 0, "
+ "'ok', 0, 0, '{}', '{}')", (gap["step_id"], tid, t["session_id"], last))
+ ordered = sorted([dict(s) for s in steps] + [gap], key=lambda s: (
+ s["started_at"] or 0, 0 if s["duration_ms"] == 0 else 1, KIND_ORDER.get(s["kind"], 9),
+ s["call_index"] or 0, s["name"] or "", s["call_id"] or "", bool(s["shadow"])))
+ self._conn.executemany("UPDATE steps SET seq = ? WHERE step_id = ?",
+ [(i, s["step_id"]) for i, s in enumerate(ordered, 1)])
+ log.info("recordings: closed %d turn(s) left open by an earlier run", len(open_turns))
+ return len(open_turns)
+
# -- deleting -----------------------------------------------------------------------------
def delete_session(self, session_id: str) -> None:
"""The session-delete cascade (§11): its turns, steps and the pieces only they used."""
diff --git a/coworker/permissions.py b/coworker/permissions.py
index 99a13402..417edcf1 100644
--- a/coworker/permissions.py
+++ b/coworker/permissions.py
@@ -755,6 +755,29 @@ class PermissionEngine:
out.append((Path(p).expanduser().resolve(), w))
return out
+ def session_grant_suspended(
+ self, tool_name: str, arguments: dict[str, Any], metadata: Any = None
+ ) -> bool:
+ """Whether a session or run grant covers this call but Auto-Approve set it aside
+ (§1.5: in-flow clicks never skip the reviewer). The same checks `evaluate` makes
+ when grants are honoured; read by the observability decision path."""
+ if self.mode is not Mode.AUTO_APPROVE:
+ return False
+ arguments = arguments or {}
+ risk = classify(tool_name, metadata, self.risk_overrides)
+ is_connector = getattr(metadata, "category", "") == "connector"
+ if risk is RiskClass.EXEC:
+ command = str(arguments.get("command", ""))
+ return bool(command) and (
+ command in self.session_allow_commands
+ or (self.session_readonly and self._readonly_grant_covers(command))
+ )
+ if risk is RiskClass.EGRESS:
+ url = str(arguments.get("url", ""))
+ if self._domain_allowed(url) and not self._domain_allowed(url, include_session=False):
+ return True
+ return (tool_name in self.session_allow_tools and not is_connector) or tool_name in self.run_allow_tools
+
def evaluate(
self, tool_name: str, arguments: dict[str, Any], metadata: Any = None
) -> Decision:
diff --git a/coworker/remote/joiner.py b/coworker/remote/joiner.py
index 3bec75c6..fa7c423b 100644
--- a/coworker/remote/joiner.py
+++ b/coworker/remote/joiner.py
@@ -536,6 +536,7 @@ def _build_local_app(state: Path):
os.environ.pop("COWORKER_API_TOKEN", None)
cfg = load_config()
manager = SessionManager(data_dir=state, model=cfg.model, mode=Mode(cfg.mode))
+ manager.recordings_store.close_open_turns() # the engine lock is held (observability TD-12)
return create_app(manager)
diff --git a/coworker/server/run.py b/coworker/server/run.py
index 16ad182a..b765fbc3 100644
--- a/coworker/server/run.py
+++ b/coworker/server/run.py
@@ -99,13 +99,18 @@ def _watch_parent_windows(parent: int) -> None:
threading.Thread(target=watch, daemon=True).start()
-def build_app(workspace: str | None, model: str, mode: str):
+def build_app(workspace: str | None, model: str, mode: str, *, owns_state: bool = False):
manager = SessionManager(
workspace=Path(workspace).expanduser().resolve() if workspace else None,
data_dir=state_dir(),
model=model,
mode=Mode(mode),
)
+ if owns_state:
+ # Turns an earlier run left open (crash, restart mid-turn) are closed as unknown
+ # (observability TD-12). Only with the engine lock: a second server on the same
+ # folder may have turns running right now.
+ manager.recordings_store.close_open_turns()
return create_app(manager)
@@ -222,7 +227,7 @@ def main(argv=None) -> None:
_exit_when_orphaned()
_warn_if_state_shared()
- app = build_app(args.cwd, args.model, args.mode)
+ app = build_app(args.cwd, args.model, args.mode, owns_state=_ENGINE_LOCK is not None)
uvicorn.run(
app, host=args.host, port=args.port, ws_max_size=_WS_MAX_FRAME_BYTES
)
diff --git a/tests/test_observability_model_steps.py b/tests/test_observability_model_steps.py
index 02bc2764..8a3cf761 100644
--- a/tests/test_observability_model_steps.py
+++ b/tests/test_observability_model_steps.py
@@ -286,3 +286,20 @@ def test_a_streamed_model_step_is_closed_before_the_engine_sees_the_reply():
[step] = [s for s in rec.as_dict()["steps"] if s["kind"] == "model"]
assert step["status"] == "ok" and step["attrs"]["finish_reason"] == "stop"
assert step["attrs"]["ttft_ms"] is not None
+
+
+def test_the_context_window_is_the_one_the_engine_compacts_against(monkeypatch):
+ # Round-2 in-app check: an Ollama model is not in the catalogue, so its steps had no
+ # window and the Usage tab hid "Context now". The step records the window the engine
+ # uses: the catalogue, else the user's saved size, else Ollama's num_ctx.
+ from coworker import model_config
+ from coworker.observability.provider_view import _context_window
+ from coworker.providers import ollama_context
+
+ monkeypatch.setattr("coworker.providers.matrix.model_context_windows", lambda: {"openai:model-a": 400_000})
+ monkeypatch.setattr(model_config, "context_size_for", lambda m: 64_000 if m == "openrouter:model-b" else None)
+ monkeypatch.setitem(ollama_context._resolved, "model-c", 40_960)
+ assert _context_window("openai:model-a") == 400_000
+ assert _context_window("openrouter:model-b") == 64_000
+ assert _context_window("ollama:model-c") == 40_960
+ assert _context_window("together:model-d") is None
diff --git a/tests/test_observability_reviewer_helper.py b/tests/test_observability_reviewer_helper.py
index 0b26126d..2f901920 100644
--- a/tests/test_observability_reviewer_helper.py
+++ b/tests/test_observability_reviewer_helper.py
@@ -242,4 +242,30 @@ def test_a_decision_reviewer_check_has_a_model_step_with_usage(tmp_path):
assert inner["attrs"]["provider_meta"][0]["wire"] == "typesafe_jev"
assert inner["attrs"]["provider_facts"]["provider_request_id"] == "req_jev"
assert check["attrs"]["usage_reported"] is True
+ # Each question's answer (round-2 in-app check: these were not recorded).
+ questions = {q["name"]: q for q in check["content"]["questions"]}
+ assert questions["requested"] == {"name": "requested", "answer": "yes", "score": 0.97}
+ assert all(set(q) == {"name", "answer", "score"} for q in questions.values())
+ assert check["attrs"].get("score") is None and check["attrs"].get("cutoff") is None
assert "test-key-not-real" not in json.dumps(steps)
+
+
+def test_a_session_grant_set_aside_in_auto_approve_is_in_the_path(tmp_path):
+ # Technical design §8.3 worked example (S1-05, call C): a command allowed for the
+ # session is still judged in Auto-approve; the path says the grant was set aside.
+ engine, rec = _engine(tmp_path, [_call("c1", "run_shell", {"command": "make"}), AssistantTurn(text="ok")],
+ [json.dumps({"verdict": "unsure", "reason": "r"})])
+ engine.permissions.session_allow_commands.add("make")
+ _run(engine)
+ [tool] = [s for s in rec.as_dict()["steps"] if s["kind"] == "tool"]
+ stages = [(p["stage"], p["outcome"]) for p in tool["attrs"]["decision"]["path"]]
+ assert stages[:3] == [("permission", "needs_user"), ("grant", "suspended_in_auto_approve"), ("auto_approver", "unsure")]
+ assert tool["attrs"]["decision"]["origin"] == "user"
+
+
+def test_no_grant_stage_without_a_matching_grant(tmp_path):
+ engine, rec = _engine(tmp_path, [_call("c1", "run_shell", {"command": "make"}), AssistantTurn(text="ok")],
+ [json.dumps({"verdict": "allow", "reason": "r"})])
+ _run(engine)
+ [tool] = [s for s in rec.as_dict()["steps"] if s["kind"] == "tool"]
+ assert "grant" not in [p["stage"] for p in tool["attrs"]["decision"]["path"]]
diff --git a/tests/test_observability_schema.py b/tests/test_observability_schema.py
index b35c59e6..4d53715b 100644
--- a/tests/test_observability_schema.py
+++ b/tests/test_observability_schema.py
@@ -118,3 +118,51 @@ def test_context_notices_stamps_and_sandbox_facts(tmp_path):
engine, rec = stamps._engine(tmp_path, [RuntimeError("boom")])
stamps._run(engine)
_check(rec.as_dict())
+
+
+def test_interactive_tools_and_steps_closed_by_a_stop_fit_the_schema(tmp_path):
+ # Round-2 in-app check: interactive tools (plan, question, folder) had no `decision`, and
+ # a step closed at its turn's end kept none of its kind's end fields.
+ import asyncio as _asyncio
+ from coworker.permissions import Mode
+
+ engine, rec = waits._engine(tmp_path, [waits._call("p1", "propose_plan", {"plan": "do it"}),
+ AssistantTurn(text="x")], mode=Mode.PLAN)
+
+ async def never(*a):
+ await _asyncio.sleep(30)
+
+ engine.plan_approver = never
+
+ async def go():
+ async def stop_soon():
+ await _asyncio.sleep(0.3)
+ engine.request_interrupt()
+ task = _asyncio.create_task(stop_soon())
+ [ev async for ev in engine.run("plan", trigger="user")]
+ await task
+
+ _asyncio.run(go())
+ [plan] = [s for s in rec.as_dict()["steps"] if s["name"] == "propose_plan"]
+ assert plan["attrs"]["decision"] == {"path": [], "origin": None}
+ _check(rec.as_dict())
+
+
+def test_a_model_step_still_open_at_the_turns_end_fits_the_schema():
+ from coworker.observability import Recorder
+
+ rec = Recorder()
+ base = {"v": 1, "session_id": "s", "turn_id": "t"}
+ rec.emit({**base, "type": "turn_started", "at_ms": 1000, "mono_ms": 0, "kind": "run", "trigger": "user",
+ "first_message_id": "m1", "stamp": {},
+ "attrs": {"approval_mode": "interactive", "attended": True,
+ "reviewer": {"active": False, "shadow": False, "kind": None, "model": None}}})
+ rec.emit({**base, "type": "step_started", "step_id": "x", "at_ms": 1010, "mono_ms": 10, "kind": "model",
+ "name": "model", "parent_step_id": None, "call_id": None, "call_index": None, "shadow": False,
+ "after_turn": False, "content": {},
+ "attrs": {"purpose": "title", "provider": "openai", "provider_title": "OpenAI", "model": "openai:model-a",
+ "stream": False, "context_window": None, "tools_offered": 0, "tools_hash": None}})
+ rec.emit({**base, "type": "turn_ended", "at_ms": 1100, "mono_ms": 100, "status": "completed", "last_message_id": "m2"})
+ [step] = rec.as_dict()["steps"]
+ assert step["status"] == "interrupted" and step["attrs"]["usage"] == {"reported": False}
+ _check(rec.as_dict())
diff --git a/tests/test_observability_stamps_context.py b/tests/test_observability_stamps_context.py
index 41f0b46a..72d06638 100644
--- a/tests/test_observability_stamps_context.py
+++ b/tests/test_observability_stamps_context.py
@@ -186,3 +186,28 @@ def test_network_note_keeps_the_counts(tmp_path, monkeypatch):
assert note["blocked_sites"] == ["semgrep.dev:443"]
assert note["_blocked"] == [{"host": "semgrep.dev", "port": 443, "count": 2, "first_at": 100000,
"late": False, "uncertain": True}]
+
+
+def test_the_skills_menu_is_the_one_offered_this_turn(tmp_path):
+ # Round-2 in-app check: a skill disabled mid-session left the menu, but the context step
+ # still counted every skill on disk. The step counts what the turn's block offered.
+ from coworker.agent import build_engine
+ from coworker.agents.chat import chat_agent
+
+ skills = tmp_path / "skills"
+ for name in ("alpha", "beta"):
+ (skills / name).mkdir(parents=True)
+ (skills / name / "SKILL.md").write_text(
+ f"---\nname: {name}\ndescription: The {name} skill.\n---\nDo {name}.\n", encoding="utf-8")
+ offered = {"alpha", "beta"}
+ (tmp_path / "w").mkdir()
+ engine = build_engine(agent=chat_agent(), model="openai:model-a", workspace=tmp_path / "w",
+ provider=_Scripted([AssistantTurn(text="a"), AssistantTurn(text="b")]),
+ extra_skill_dirs=[skills], skill_filter=lambda: set(offered))
+ engine.recorder = rec = Recorder()
+ _run(engine, "one")
+ offered.discard("beta") # turned off in Settings between the two messages
+ _run(engine, "two")
+ menus = [s["attrs"]["per_turn"] for s in rec.as_dict()["steps"] if s["kind"] == "context"]
+ assert [m["skills_menu"] for m in menus] == [2, 1]
+ assert menus[0]["skills_hash"] != menus[1]["skills_hash"]
diff --git a/tests/test_observability_store.py b/tests/test_observability_store.py
index eaad44c0..68ad30e5 100644
--- a/tests/test_observability_store.py
+++ b/tests/test_observability_store.py
@@ -283,3 +283,63 @@ def test_every_store_shares_one_writer_thread(tmp_path):
finally:
for s in stores:
s.close()
+
+
+def test_turns_left_open_by_a_crash_are_closed_as_unknown_at_start(tmp_path):
+ # TD-12 (round-2 in-app check: a restart while an approval card waited left the turn
+ # "running" for good). At start, an open turn and its open steps become `unknown` with
+ # no duration, every step gets its place, and a recording_gap notice marks the gap.
+ import jsonschema, json
+ from pathlib import Path
+ from coworker.observability import calculator
+
+ path = tmp_path / "recordings.db"
+ s1 = RecordingStore(path, background=False)
+ rec = Recorder(store=s1)
+ _turn(rec, "t1", at=1000, mono=0)
+ _model_step(rec, "m1", "t1", at=1010, mono=10, end=60) # 50 ms on the step clock
+ rec.emit({"v": 1, "type": "step_started", "session_id": "sess_A", "turn_id": "t1", "step_id": "w1",
+ "at_ms": 1070, "mono_ms": 70, "kind": "wait", "name": "approval", "parent_step_id": None,
+ "call_id": "c1", "call_index": 0, "shadow": False, "after_turn": False,
+ "attrs": {"for_call_id": "c1", "kind": "approval", "where": "session", "attended": True}, "content": {}})
+ s1.flush()
+ s1.close() # the app is killed here
+
+ s2 = RecordingStore(path, background=False)
+ try:
+ assert s2.close_open_turns() == 1
+ assert s2.close_open_turns() == 0 # nothing left open
+ r = s2.load_recording("sess_A")
+ [turn] = r["turns"]
+ assert (turn["status"], turn["duration_ms"]) == ("unknown", None)
+ by = {s["name"]: s for s in r["steps"]}
+ assert (by["approval"]["status"], by["approval"]["duration_ms"]) == ("unknown", None)
+ assert by["model"]["status"] == "ok"
+ assert by["recording_gap"]["kind"] == "notice" and by["recording_gap"]["started_at"] == 1070 # the open wait's start
+ # zero-length first at the same moment (recording-format §4)
+ assert [s["name"] for s in r["steps"]] == ["model", "recording_gap", "approval"]
+ assert [s["seq"] for s in r["steps"]] == [1, 2, 3]
+ schema = json.loads((Path(__file__).parent / "corpora/observability/recording-format.schema.json").read_text("utf-8"))
+ errors = jsonschema.validators.validator_for(schema)(schema).iter_errors(r)
+ # The helper's model step is deliberately minimal; the turn, the closed wait and the
+ # notice must fit the schema exactly.
+ model_at = [s["name"] for s in r["steps"]].index("model")
+ assert not [e for e in errors if list(e.absolute_path)[:2] != ["steps", model_at]]
+ b = calculator.buckets(r, turn) # the unknown wait is not counted; the turn ran until it started
+ assert (b["total"], b["model"], b["waiting"]) == (70, 50, 0)
+ finally:
+ s2.close()
+
+
+def test_a_running_turn_is_not_closed_without_being_asked(tmp_path):
+ # Only the process that owns the folder closes open turns; opening the store never does.
+ path = tmp_path / "recordings.db"
+ s1 = RecordingStore(path, background=False)
+ _turn(Recorder(store=s1), "t1")
+ s1.flush()
+ s2 = RecordingStore(path, background=False)
+ try:
+ assert s2.load_recording("sess_A")["turns"][0]["status"] == "running"
+ finally:
+ s1.close()
+ s2.close()
diff --git a/tests/test_observability_waits.py b/tests/test_observability_waits.py
index c997ab5b..37d1b04f 100644
--- a/tests/test_observability_waits.py
+++ b/tests/test_observability_waits.py
@@ -208,3 +208,75 @@ def test_no_summary_step_when_nothing_was_summarised(tmp_path, monkeypatch):
assert asyncio.run(go()) is None
assert _steps(rec, "summary") == []
+
+
+def test_a_stop_while_a_plan_card_waits_is_interrupted_not_refused(tmp_path):
+ # Round-2 in-app check: the plan handler answers "denied" when the turn is stopped, so
+ # the propose_plan step read as refused. A Stop is recorded as interrupted.
+ engine, rec = _engine(tmp_path, [_call("p1", "propose_plan", {"plan": "do it"}), AssistantTurn(text="x")],
+ mode=Mode.PLAN)
+
+ async def never(args, call_id):
+ await asyncio.sleep(30)
+
+ engine.plan_approver = never
+
+ async def go():
+ async def stop_soon():
+ await asyncio.sleep(0.3)
+ engine.request_interrupt()
+ task = asyncio.create_task(stop_soon())
+ [ev async for ev in engine.run("plan", trigger="user")]
+ await task
+
+ asyncio.run(go())
+ [wait] = _steps(rec, "wait")
+ [plan] = [s for s in _steps(rec, "tool") if s["name"] == "propose_plan"]
+ assert wait["attrs"]["outcome"] == "interrupted"
+ assert plan["status"] == "interrupted"
+
+
+def test_a_stop_while_an_approval_card_waits_is_interrupted_not_refused(tmp_path):
+ # Round-2 in-app check (F3): the wait and the path said interrupted, the tool step denied.
+ engine, rec = _engine(tmp_path, [_call("c1", "run_shell", {"command": "make"}), AssistantTurn(text="x")])
+
+ async def never(request):
+ await asyncio.sleep(30)
+
+ engine.approver = never
+
+ async def go():
+ async def stop_soon():
+ await asyncio.sleep(0.3)
+ engine.request_interrupt()
+ task = asyncio.create_task(stop_soon())
+ [ev async for ev in engine.run("go", trigger="user")]
+ await task
+
+ asyncio.run(go())
+ [tool] = _steps(rec, "tool")
+ assert tool["status"] == "interrupted"
+ assert tool["attrs"]["decision"]["path"][-1] == {"stage": "resolved", "outcome": "interrupted"}
+
+
+def test_a_tool_killed_by_stop_is_interrupted(tmp_path):
+ # Round-2 in-app check (F2): a shell command killed by Stop answers "interrupted by
+ # user" and the step said ok.
+ engine, rec = _engine(tmp_path, [_call("c1", "run_shell", {"command": "sleep 40"}), AssistantTurn(text="x")],
+ mode=Mode.BYPASS_APPROVALS)
+
+ def run_shell(command: str) -> dict:
+ """Run a command.
+
+ Args:
+ command: the command line
+ """
+ engine.request_interrupt() # the Stop arrives while it runs; it is killed
+ return {"command": command, "exit_code": None, "error": "interrupted by user"}
+
+ registry = ToolRegistry()
+ registry.register(run_shell, metadata=_Meta())
+ engine.registry = registry
+ _run(engine)
+ [tool] = _steps(rec, "tool")
+ assert tool["status"] == "interrupted"