Python
feat(observability): gaps in the timeline (OB-14)
Devika Verma21cdb8c1a090
diff --git a/coworker/observability/engine_hooks.py b/coworker/observability/engine_hooks.pyindex ef796952..ca044980 100644--- a/coworker/observability/engine_hooks.py+++ b/coworker/observability/engine_hooks.py@@ -53,6 +53,13 @@ class TurnRecording:self._reserved: dict[str, dict[str, Any]] = {} # summary steps written at their end (E4)self._tokens_after_for: Optional[str] = None # a summary waiting for the next main callself._child_usage: dict[str, bool] = {} # parent step -> its model call reported usage+ # Conditions on now (OB-14): name -> (since ms, attrs). They outlive turns; each turn+ # gets its own gap notice, cut at its start and end.+ self._conditions: dict[str, tuple[int, dict[str, Any]]] = {}+ self._turn_started_ms = self._turn_started_mono = 0+ from . import gaps++ gaps.register(self) # the recorder may be set later; conditions are kept meanwhile@propertydef enabled(self) -> bool:@@ -97,6 +104,7 @@ class TurnRecording:self._reply_end = Noneself._waits.clear()ev = self._base("turn_started", self.turn_id, self._turn_session)+ self._turn_started_ms, self._turn_started_mono = ev["at_ms"], ev["mono_ms"]ev.update({"kind": kind, "trigger": trigger, "first_message_id": first_message_id,"attrs": copy.deepcopy(attrs), "stamp": copy.deepcopy(stamp)})if wait_for_first_message:@@ -124,7 +132,12 @@ class TurnRecording:self.recorder.emit(ev)self._tool_ends.clear()self._flush_reply(None)- ev = self._base("turn_ended", self.turn_id, self._turn_session)+ end = self._base("turn_ended", self.turn_id, self._turn_session)+ # Still on: cut at the turn's end, on the turn's own clock (its length is monotonic).+ cut = self._turn_started_ms + (end["mono_ms"] - self._turn_started_mono)+ for name, (since, attrs) in list(self._conditions.items()):+ self._write_gap(name, since, cut, attrs)+ ev = endev.update({"status": status, "last_message_id": self._last_mid})return ev@@ -135,6 +148,34 @@ class TurnRecording:def in_turn(self) -> bool:return self.turn_id is not None and not self._ended+ # -- gaps (OB-14) ---------------------------------------------------------------------------+ def condition(self, name: str, on: bool, *, at_ms: Optional[int] = None,+ attrs: Optional[dict[str, Any]] = None) -> None:+ """A gap condition starts (`on`) or ends. When it ends during a turn, the turn gets a+ notice lasting from the later of its start and the turn's start, until now."""+ at = at_ms if at_ms is not None else now_ms()[0]+ with self._lock:+ if on:+ self._conditions.setdefault(name, (at, dict(attrs or {})))+ return+ held = self._conditions.pop(name, None)+ if held is None or not self.in_turn:+ return+ since, had = held+ self._write_gap(name, since, at, {**had, **(attrs or {})})++ def _write_gap(self, name: str, since: int, until: int, attrs: dict[str, Any]) -> None:+ """One gap notice in the current turn, whole, from its end (under the lock)."""+ start = max(since, self._turn_started_ms)+ if self.turn_id is None or until <= start:+ return+ ev = self._base("step_ended", self.turn_id, self._turn_session)+ ev.update({"at_ms": until, "step_id": new_ulid(), "status": "ok", "message_id": None,+ "attrs": copy.deepcopy(attrs), "content": {}, "kind": "notice", "name": name,+ "parent_step_id": None, "call_id": None, "call_index": None, "shadow": False,+ "after_turn": False, "started_at": start, "gap": True})+ self._send(ev)+def step_for_call(self, call_id: str) -> Optional[str]:"""The tool step recorded for a call in this turn, if any."""with self._lock:diff --git a/coworker/observability/export.py b/coworker/observability/export.pyindex 48d3c2d5..c1ca1662 100644--- a/coworker/observability/export.py+++ b/coworker/observability/export.py@@ -28,7 +28,7 @@ from typing import Any, Callable, Optionalfrom urllib.parse import urlparsefrom . import export_spans as spans_mod-from .export_spans import Masker, build_turn_spans, encode_otlp, resource_attrs, switches_from+from .export_spans import Masker, _is_gap, build_turn_spans, encode_otlp, resource_attrs, switches_fromlog = logging.getLogger(__name__)@@ -376,7 +376,8 @@ class Exporter:ready = {s["step_id"] for s in stepsif s.get("status") not in (None, "running") and s["step_id"] not in marked}include_root = turn_id not in marked- late_notices = [s["step_id"] for s in steps if s.get("kind") == "notice" and s["step_id"] in ready]+ late_notices = [s["step_id"] for s in steps if s.get("kind") == "notice" and s["step_id"] in ready+ and not _is_gap(s)] # a gap is a span of its own (1.2)if not include_root and late_notices:# A notice is an event on the turn's span, which has gone already.self.store.mark_export(dest, [(n, turn_id) for n in late_notices], "skipped")diff --git a/coworker/observability/export_spans.py b/coworker/observability/export_spans.pyindex aeb45223..81d73633 100644--- a/coworker/observability/export_spans.py+++ b/coworker/observability/export_spans.py@@ -456,6 +456,13 @@ def turn_end_ms(turn: dict[str, Any], steps: list[dict[str, Any]]) -> int:return max([turn.get("started_at") or 0] + ends)+def _is_gap(step: dict[str, Any]) -> bool:+ """A gap notice with a duration: its own span, not an event on the turn (1.2)."""+ from .gaps import GAP_NOTICES++ return step.get("kind") == "notice" and step.get("name") in GAP_NOTICES and bool(step.get("duration_ms"))++def build_turn_spans(turn: dict[str, Any],steps: list[dict[str, Any]],@@ -518,7 +525,7 @@ def build_turn_spans(first_user_text = last_reply_text = Nonefor s in sorted(steps, key=lambda s: (depth(s), s.get("seq") or 0)):- if s.get("kind") == "notice" or (only is not None and s["step_id"] not in only):+ if (s.get("kind") == "notice" and not _is_gap(s)) or (only is not None and s["step_id"] not in only):continuea = s.get("attrs") or {}attrs: dict[str, Any] = dict(common, **{@@ -652,6 +659,11 @@ def build_turn_spans(for k in ("size_chars", "session_prompt_hash", "block_hash"):if a.get(k) is not None:attrs[f"openworker.context.{k}"] = a[k]+ elif s["kind"] == "notice": # a gap with a duration (1.2, tracing-spec §9)+ name = f"openworker.gap.{s.get('name')}"+ for k in ("reason", "skipped_ms"):+ if a.get(k) is not None:+ attrs[f"openworker.gap.{k}"] = a[k]elif s["kind"] == "sandbox":name = "openworker.sandbox.setup"for k in ("provider", "trigger", "failed_stage"):@@ -731,9 +743,9 @@ def build_turn_spans(root_attrs["langfuse.trace.output"] = root_attrs["langfuse.observation.output"] = masker.clean(last_reply_text)events = [{"name": f"openworker.notice.{s.get('name')}", "at": s.get("started_at") or turn["started_at"],"attrs": {"openworker.step.id": s["step_id"]}}- for s in sorted(steps, key=lambda s: s.get("seq") or 0) if s.get("kind") == "notice"]+ for s in sorted(steps, key=lambda s: s.get("seq") or 0) if s.get("kind") == "notice" and not _is_gap(s)]spans.insert(0, {- "_mark": tid, "_notices": [s["step_id"] for s in steps if s.get("kind") == "notice"],+ "_mark": tid, "_notices": [s["step_id"] for s in steps if s.get("kind") == "notice" and not _is_gap(s)],"trace_id": trace, "span_id": root_id, "parent_span_id": None,"name": "invoke_agent coworker", "kind": SPAN_INTERNAL, "start_ms": turn["started_at"],"end_ms": end_ms, "attrs": root_attrs, "events": events,diff --git a/coworker/observability/gaps.py b/coworker/observability/gaps.pynew file mode 100644index 00000000..5db83da3--- /dev/null+++ b/coworker/observability/gaps.py@@ -0,0 +1,112 @@+"""Gaps in the timeline (observability OB-14, format 1.2): stretches when the machine was+offline from its controller, or nobody was watching the session. Each is a notice with a+duration in every turn running at the time, cut at the turn's start and end+(phase-2 design §11.3).++A condition is "on" from a moment until it ends. `nobody_watching` belongs to one session+(its engine's recording); `controller_offline` to the whole machine, so it goes to every+recording here, including those of engines made while it is on."""++from __future__ import annotations++import json+import logging+import threading+import time+import weakref+from pathlib import Path+from typing import Any, Optional++log = logging.getLogger(__name__)++GAP_NOTICES = ("controller_offline", "nobody_watching", "clock_jump")++_lock = threading.Lock()+_recordings: "weakref.WeakSet[Any]" = weakref.WeakSet()+_machine: dict[str, tuple[int, dict[str, Any]]] = {} # name -> (since ms, attrs)+++def now_ms() -> int:+ return int(time.time() * 1000)+++def register(recording: Any) -> None:+ """A new engine's recording hears the machine's conditions, and any already on."""+ with _lock:+ _recordings.add(recording)+ ongoing = dict(_machine)+ for name, (since, attrs) in ongoing.items():+ recording.condition(name, True, at_ms=since, attrs=attrs)+++def machine_condition(name: str, on: bool, *, at_ms: Optional[int] = None, attrs: Optional[dict[str, Any]] = None) -> None:+ """A machine-wide condition starts or ends, for every recording on this machine."""+ at = at_ms if at_ms is not None else now_ms()+ with _lock:+ if on:+ if name in _machine:+ return # already on: the first moment stands+ _machine[name] = (at, dict(attrs or {}))+ else:+ if name not in _machine and not attrs:+ return+ since, held = _machine.pop(name, (at, {}))+ attrs = {**held, **(attrs or {})}+ targets = list(_recordings)+ for rec in targets:+ try:+ rec.condition(name, on, at_ms=at, attrs=attrs)+ except Exception: # noqa: BLE001 - recording never breaks the link+ log.exception("observability: could not note %s", name)+++def machine_on(name: str) -> bool:+ with _lock:+ return name in _machine+++# -- the controller link (E15b-1) ----------------------------------------------------------------+# When the link is lost, the moment is also saved in the state folder, so a restart before+# it comes back still writes the gap into the turns that are running when it does.++_LOST_FILE = "controller-link-lost.json"+++def link_lost(state: Optional[Path], reason: str) -> None:+ at = now_ms()+ if state is not None:+ path = Path(state) / _LOST_FILE+ try:+ if not path.exists():+ path.write_text(json.dumps({"lost_at": at, "reason": reason}), encoding="utf-8")+ except OSError:+ log.warning("observability: could not save when the controller link was lost")+ machine_condition("controller_offline", True, at_ms=at, attrs={"reason": reason})+++def link_back(state: Optional[Path]) -> None:+ saved: dict[str, Any] = {}+ if state is not None:+ path = Path(state) / _LOST_FILE+ try:+ saved = json.loads(path.read_text(encoding="utf-8"))+ path.unlink()+ except FileNotFoundError:+ pass+ except (OSError, ValueError):+ log.warning("observability: could not read when the controller link was lost")+ if saved and not machine_on("controller_offline"):+ # Lost before a restart: the condition starts at the saved moment.+ machine_condition("controller_offline", True, at_ms=int(saved.get("lost_at") or now_ms()),+ attrs={"reason": str(saved.get("reason") or "failed")})+ machine_condition("controller_offline", False)+++def link_reason(exc: Optional[BaseException]) -> str:+ """`closed` (the controller closed it), `ping_timeout` or `failed`."""+ if exc is None:+ return "closed"+ text = str(exc).lower()+ if "ping" in text and ("timeout" in text or "timed out" in text):+ return "ping_timeout"+ return "failed"diff --git a/coworker/observability/recorder.py b/coworker/observability/recorder.pyindex 4ebcf775..9152b701 100644--- a/coworker/observability/recorder.py+++ b/coworker/observability/recorder.py@@ -55,6 +55,9 @@ TURN_FIELDS = ("incomplete", "approval_mode", "attended", "reviewer", "stamp", "first_message_id","last_message_id", "schema_version",)+# A wall clock ahead of the turn's monotonic clock by more than this: the machine slept (OB-14).+CLOCK_JUMP_MS = 5000+# Format 1.2 turn fields: written only when known, so a 1.1 recording reads back unchanged.TURN_FIELDS_12 = ("max_iterations", "woken_by")STEP_FIELDS = (@@ -90,6 +93,10 @@ class Turn:woken_by: Optional[dict[str, Any]] = Nonemono_start: Optional[int] = field(default=None, repr=False) # not part of the recordmono_end: Optional[int] = field(default=None, repr=False)+ # Time the monotonic clock did not count (the machine slept; OB-14 clock_jump): added+ # to every later moment, so step times follow the wall clock again (tracing-spec §7, 1.2).+ shift_ms: int = field(default=0, repr=False)+ last_ms: Optional[int] = field(default=None, repr=False) # the latest moment placed@propertydef ended_at(self) -> Optional[int]:@@ -285,8 +292,9 @@ class Recorder:if turn is None or turn.mono_end is not None:returnturn.status, turn.last_message_id = ev["status"], ev.get("last_message_id")+ self._anchor(turn.turn_id, ev["at_ms"], ev["mono_ms"]) # a sleep just before the endturn.mono_end = ev["mono_ms"]- turn.duration_ms = max(0, ev["mono_ms"] - turn.mono_start)+ turn.duration_ms = max(0, ev["mono_ms"] - turn.mono_start + turn.shift_ms)# §7.4: anything of this turn still open is closed at the turn's end.for s in self.steps.values():if (s.turn_id == turn.turn_id and s.duration_ms is None and s.status == "running"@@ -316,11 +324,26 @@ class Recorder:log.exception("recording: the Issues checks failed for a turn")def _anchor(self, turn_id: Optional[str], at_ms: int, mono_ms: int) -> int:- """§7.3: a step's start is its turn's start plus the monotonic time since then."""+ """§7.3: a step's start is its turn's start plus the monotonic time since then. When+ the wall clock has run ahead of that by more than 5 s, the machine slept: a+ `clock_jump` notice covers the skipped time, and later moments follow the wall+ clock again (1.2, TD2-14)."""t = self.turns.get(turn_id) if turn_id else Noneif t is None or t.mono_start is None:return at_ms- return t.started_at + (mono_ms - t.mono_start)+ placed = t.started_at + (mono_ms - t.mono_start) + t.shift_ms+ skipped = at_ms - placed+ if skipped > CLOCK_JUMP_MS and t.mono_end is None:+ last = t.last_ms if t.last_ms is not None else t.started_at+ jump = Step(step_id=new_ulid(), turn_id=t.turn_id, session_id=t.session_id, kind="notice",+ name="clock_jump", started_at=last, duration_ms=skipped, status="ok",+ attrs={"skipped_ms": skipped})+ self.steps[jump.step_id] = jump+ self._write_step(jump)+ t.shift_ms += skipped+ placed = at_ms+ t.last_ms = max(t.last_ms or 0, placed)+ return placeddef _step_started(self, ev: dict[str, Any]) -> None:if ev["step_id"] in self.steps:@@ -360,6 +383,9 @@ class Recorder:returnelse:s.duration_ms = max(0, ev["mono_ms"] - s.mono_start)+ turn = self.turns.get(s.turn_id) if s.turn_id else None+ if turn is not None:+ turn.last_ms = max(turn.last_ms or 0, s.started_at + s.duration_ms)s.status = ev.get("status", "ok")if "message_id" in ev:s.message_id = ev["message_id"]diff --git a/coworker/observability/signals.py b/coworker/observability/signals.pyindex 495bb809..8c499ef6 100644--- a/coworker/observability/signals.py+++ b/coworker/observability/signals.py@@ -399,17 +399,36 @@ def c_ran_without_sandbox(turn, top, all_steps, p):f"this turn ran {len(acts)} action(s) without it.")], ""+def _overlap(start, end, spans):+ """How much of [start, end) the spans cover, each moment once."""+ total, cur = 0, start+ for a, b in sorted(spans):+ a, b = max(a, cur), min(b, end)+ if b > a:+ total += b - a+ cur = b+ return total++def c_waited_unwatched(turn, top, all_steps, p):+ # 0.4: besides the time after the card reached the Inbox, the wait's overlap with the+ # turn's `nobody_watching` gaps (OB-14) counts: a card left in the open session while+ # the desktop was closed.+ unwatched = [(g["started_at"], g["started_at"] + g["duration_ms"]) for g in all_steps+ if g["kind"] == "notice" and g.get("name") == "nobody_watching" and g.get("duration_ms")]found = []for s in top:a = s["attrs"]- if s["kind"] != "wait" or a.get("kind") not in ("approval", "question", "plan") or a.get("where") != "inbox":+ if s["kind"] != "wait" or a.get("kind") not in ("approval", "question", "plan"):continue- start = a.get("inbox_at") or s.get("started_at") or 0- end = (s.get("started_at") or 0) + (s.get("duration_ms") or 0)- if end - start >= p["min_wait_ms"]:- found.append(issue("waited_unwatched", s, f"waited {round((end - start) / 60000)} min in the "- f"Inbox for an answer while nobody was at the app."))+ began = s.get("started_at") or 0+ end = began + (s.get("duration_ms") or 0)+ alone = end - (a.get("inbox_at") or began) if a.get("where") == "inbox" else 0+ alone = max(alone, _overlap(began, end, unwatched))+ if alone >= p["min_wait_ms"]:+ where = "in the Inbox " if a.get("where") == "inbox" else ""+ found.append(issue("waited_unwatched", s, f"waited {round(alone / 60000)} min {where}for an answer "+ f"while nobody was at the app."))return ("fired" if found else "fine"), found, "Nothing waited long for an answer while nobody was watching"diff --git a/coworker/observability/signals.yaml b/coworker/observability/signals.yamlindex a67567a1..901ef175 100644--- a/coworker/observability/signals.yaml+++ b/coworker/observability/signals.yaml@@ -37,8 +37,9 @@# golden the scenarios that test it (phase-2/scenarios.md)# source where the check was asked for-version: "0.3"-draft: 2026-11-02+version: "0.4"+draft: 2026-11-03+# 0.4: waited_unwatched also counts the wait's overlap with nobody_watching gaps (OB-14).# 0.3: ran_without_sandbox runs only where a sandbox is expected (found in the first live run).# 0.2: reviewer_allowed_send (TD2-20), sandbox_failed, the mock-review wording (M2-R1 to M2-R9),# waited_unwatched from inbox_at (V2-25). Goes with recording format 1.2.@@ -405,18 +406,19 @@ checks:severity: warnapplies: "always"fires: >-- A wait step of kind `approval`, `question` or `plan` with where = `inbox` that- waited at least `min_wait_ms` after it reached the Inbox: wait end minus `inbox_at`- (new; the step's start when it began in the Inbox). A card reaches the Inbox when- the turn started unattended or when the last viewer left the session- (`inbox.py` promote_to_inbox), so where = `inbox` is the recorded fact for "nobody- was watching". The wait's `attended` is not used: it is the Unattended toggle, not- whether a desktop is connected (V2-25). Q27.+ A wait step of kind `approval`, `question` or `plan` that waited at least+ `min_wait_ms` while nobody was watching. That time is the longer of two: for+ where = `inbox`, wait end minus `inbox_at` (the step's start when it began in the+ Inbox); and the wait's overlap with the turn's `nobody_watching` notices (1.2,+ OB-14), which also covers a card left in the open session while the desktop was+ closed. A card reaches the Inbox when the turn started unattended or when the last+ viewer left the session (`inbox.py` promote_to_inbox). The wait's `attended` is not+ used: it is the Unattended toggle, not whether a desktop is connected (V2-25). Q27.points_at: "the wait step"params:min_wait_ms: 900000- says: "{name} waited {wait} in the Inbox for an answer while nobody was at the app."+ says: "{name} waited {wait} in the Inbox for an answer while nobody was at the app. (Without \"in the Inbox\" when the card stayed in the session.)"fine: "Nothing waited long for an answer while nobody was watching"- needs: [wait.inbox_at]+ needs: [wait.inbox_at, notice.nobody_watching]golden: [S2-17, S2-17b, S2-17c]source: "across-features.md OB-14, Q27; validation V2-25"diff --git a/coworker/observability/store.py b/coworker/observability/store.pyindex 412a2348..63c9f40c 100644--- a/coworker/observability/store.py+++ b/coworker/observability/store.py@@ -198,6 +198,7 @@ class RecordingStore:self._pending_issues: dict[str, tuple[str, int]] = {} # turn_id -> (json, seen)self._pending_steps: dict[str, dict[str, Any]] = {}self._pending_lock = threading.Lock()+ self._owner = False # this process called close_open_turns: it owns the data folderself._closed = Falseself._last_sweep = 0.0self._background = background@@ -274,6 +275,26 @@ class RecordingStore:self.flush()self._closed = Trueself._conn.close()+ if self._owner:+ # A clean stop: a turn found open at the next start was cut by the app being+ # stopped, not by a crash (recording_gap cause, OB-14).+ self._marker("running").unlink(missing_ok=True)+ try:+ self._marker("stopped").write_text(str(int(time.time() * 1000)), encoding="utf-8")+ except OSError:+ pass++ def _marker(self, name: str) -> Path:+ return self.path.with_name(self.path.name + f".{name}")++ def _last_stop(self) -> str:+ """How the previous run of the app ended: `stopped` (it closed this store), `restart`+ (it was running and never closed it: a crash or a kill) or `unknown` (no word)."""+ if self._marker("stopped").exists():+ return "stopped"+ if self._marker("running").exists():+ return "restart"+ return "unknown"# -- exchange pieces (TD-17) --------------------------------------------------------------def put_piece(self, data: bytes) -> str:@@ -306,6 +327,13 @@ class RecordingStore:with self._lock:self.flush()+ cause, until = self._last_stop(), int(time.time() * 1000)+ self._owner = True+ self._marker("stopped").unlink(missing_ok=True)+ try:+ self._marker("running").write_text(str(until), encoding="utf-8")+ except OSError:+ passopen_turns = self._conn.execute("SELECT turn_id, session_id, started_at FROM turns WHERE status = 'running' OR status IS NULL").fetchall()@@ -331,10 +359,12 @@ class RecordingStore:if s["started_at"] is not None])gap = {"step_id": new_ulid(), "kind": "notice", "name": "recording_gap", "started_at": last,"duration_ms": 0, "call_index": None, "call_id": None, "shadow": 0}+ # 1.2: why the turn was found open, and the start that found it (OB-14).self._conn.execute("INSERT INTO steps (step_id, turn_id, session_id, kind, name, started_at, duration_ms, ""status, shadow, after_turn, attrs, content) VALUES (?, ?, ?, 'notice', 'recording_gap', ?, 0, "- "'ok', 0, 0, '{}', '{}')", (gap["step_id"], tid, t["session_id"], last))+ "'ok', 0, 0, ?, '{}')",+ (gap["step_id"], tid, t["session_id"], last, json.dumps({"cause": cause, "until": until})))ordered = sorted([dict(s) for s in steps] + [gap], key=lambda s: (s["started_at"] or 0, 0 if s["duration_ms"] == 0 else 1, KIND_ORDER.get(s["kind"], 9),s["call_index"] or 0, s["name"] or "", s["call_id"] or "", bool(s["shadow"])))diff --git a/coworker/remote/joiner.py b/coworker/remote/joiner.pyindex 6cb9d417..47f074ec 100644--- a/coworker/remote/joiner.py+++ b/coworker/remote/joiner.py@@ -475,6 +475,8 @@ async def run_joined("""import websockets+ from ..observability import gaps+identity = load_or_create(state)if app is None:app = _build_local_app(state)@@ -498,6 +500,8 @@ async def run_joined(welcome = await _handshake(ws, identity, name, token)token = None # single-use by design; reconnects sign the challengebackoff = _BACKOFF_START+ # The link is back: running turns get the offline stretch (OB-14).+ gaps.link_back(state)log(f"[openworker] joined {controller} as "f"'{welcome.get('name')}' (machine {welcome.get('machine_id')})"@@ -515,12 +519,15 @@ async def run_joined(ws, dispatcher, app, identity, exporter=exporter, state=state)log("[openworker] controller closed the connection")+ gaps.link_lost(state, "closed")except JoinRejected as exc:if exc.reason in _FATAL_REJECTS:raiselog(f"[openworker] rejected ({exc.reason}); retrying")+ gaps.link_lost(state, "failed")except (OSError, websockets.WebSocketException) as exc:log(f"[openworker] connection failed: {exc}")+ gaps.link_lost(state, gaps.link_reason(exc))if once:returnlog(f"[openworker] reconnecting in {backoff:.0f}s")diff --git a/coworker/server/manager.py b/coworker/server/manager.pyindex d010cd3b..923b40c3 100644--- a/coworker/server/manager.py+++ b/coworker/server/manager.py@@ -1056,6 +1056,8 @@ class SessionManager:engine.compaction_state = CompactionState.from_dict(record.compaction)engine.compaction_settings = self.compaction_settingsself._engines[session_id] = engine+ if not self._session_clients.get(session_id):+ self._note_watching(session_id, False) # built with no window open (a schedule, a wake)self.sync_worker_mode(session_id, engine) # §11.6: approvals follow the leadif is_new_session:self._emit_session_created(session_id, agent_name)@@ -6274,7 +6276,10 @@ class SessionManager:self.unregister_event_client(cb)def register_session_client(self, session_id: str, send_cb: Any) -> None:+ first = not self._session_clients.get(session_id)self._session_clients.setdefault(session_id, set()).add(send_cb)+ if first:+ self._note_watching(session_id, True)def unregister_session_client(self, session_id: str, send_cb: Any) -> None:clients = self._session_clients.get(session_id)@@ -6282,6 +6287,19 @@ class SessionManager:clients.discard(send_cb)if not clients:self._session_clients.pop(session_id, None)+ self._note_watching(session_id, False)++ def _note_watching(self, session_id: str, watching: bool) -> None:+ """The session's viewers went from none to one or from one to none (OB-14): while+ none, its turns get a `nobody_watching` gap. A local window and a desktop reaching+ this machine through the controller open the same socket, so both count."""+ recording = getattr(self._engines.get(session_id), "_obs", None)+ condition = getattr(recording, "condition", None)+ if condition is not None:+ try:+ condition("nobody_watching", not watching)+ except Exception: # noqa: BLE001 - a recording never breaks a socket+ logger.exception("observability: could not note who is watching %s", session_id)# -- session actor (spec §Fleet under the org) --------------------------------def note_session_actor(self, session_id: str, actor: str) -> None:diff --git a/surfaces/gui/src/components/StepsPanel.test.tsx b/surfaces/gui/src/components/StepsPanel.test.tsxindex 20848b4e..9848b047 100644--- a/surfaces/gui/src/components/StepsPanel.test.tsx+++ b/surfaces/gui/src/components/StepsPanel.test.tsx@@ -317,6 +317,49 @@ describe("Sandbox setup (OB-13)", () => {});});+describe("Gaps (OB-14)", () => {+ const gapTurn = { ...turn, duration_ms: 600_000, buckets: { ...turn.buckets, total: 600_000 } };+ const gaps = [+ step({ step_id: "m", kind: "model", name: "model", duration_ms: 5000, attrs: { purpose: "main" } }),+ step({ step_id: "z", kind: "notice", name: "clock_jump", started_at: 6000, duration_ms: 23_000, attrs: { skipped_ms: 23_000 } }),+ step({ step_id: "o", kind: "notice", name: "controller_offline", started_at: 30_000, duration_ms: 420_000, attrs: { reason: "ping_timeout" } }),+ step({ step_id: "n", kind: "notice", name: "nobody_watching", started_at: 40_000, duration_ms: 300_000, attrs: {} }),+ step({ step_id: "r", kind: "notice", name: "recording_gap", started_at: 500_000, duration_ms: 0, attrs: { cause: "restart", until: 740_000 } }),+ ];++ it("draws each gap as a band with its times, and totals offline and unwatched time", async () => {+ vi.mocked(getRecordedTurns).mockResolvedValue([gapTurn] as any);+ vi.mocked(getTurnSteps).mockResolvedValue(gaps as any);+ render(<StepsPanel sessionId="s1" onClose={() => {}} />);+ await waitFor(() => expect(screen.getAllByTestId("steps-row")).toHaveLength(5));+ const rows = screen.getAllByTestId("steps-row");+ const text = rows.map((r) => r.textContent || "");+ expect(text[1]).toContain("Machine asleep");+ expect(text[1]).toContain("+23s");+ expect(text[2]).toContain("Machine offline");+ expect(text[2]).toMatch(/\d{1,2}:\d{2}.*–.*\d{1,2}:\d{2}/); // from–until, on the wall clock+ expect(text[3]).toContain("Nobody watching");+ expect(text[4]).toContain("App restarted");+ expect(text[4]).toContain("4m not recorded");+ expect(rows[2].className).toContain("k-gap");+ expect(rows[4].className).not.toContain("k-gap"); // a point, not a band+ const tiles = screen.getByTestId("steps-tiles").textContent || "";+ expect(tiles).toContain("Machine offline");+ expect(tiles).toContain("Nobody watching");+ });++ it("says why the machine was offline in the details", async () => {+ vi.mocked(getRecordedTurns).mockResolvedValue([gapTurn] as any);+ vi.mocked(getTurnSteps).mockResolvedValue(gaps as any);+ vi.mocked(getStepDetail).mockResolvedValue(gaps[2] as any);+ render(<StepsPanel sessionId="s1" onClose={() => {}} />);+ await waitFor(() => expect(screen.getAllByTestId("steps-row")).toHaveLength(5));+ fireEvent.click(screen.getAllByTestId("steps-row")[2]);+ await waitFor(() => expect(screen.getByTestId("steps-details").textContent).toContain("the controller stopped answering"));+ expect(screen.getByTestId("steps-details").textContent).toContain("7m");+ });+});+describe("formatDuration", () => {it("reads like the mock", () => {expect(formatDuration(100)).toBe("0.1s");diff --git a/surfaces/gui/src/components/StepsPanel.tsx b/surfaces/gui/src/components/StepsPanel.tsxindex fe007518..238ee0d5 100644--- a/surfaces/gui/src/components/StepsPanel.tsx+++ b/surfaces/gui/src/components/StepsPanel.tsx@@ -129,6 +129,7 @@ export function stepLabel(step: RecordedStep, t: T): string {return t("steps.row.sandbox_setup");case "notice":if (step.name === "sandbox_restarted" && a.how === "regrant") return t("steps.notice.sandbox_regranted");+ if (step.name === "recording_gap" && (a.cause === "restart" || a.cause === "stopped")) return t(`steps.notice.recording_gap_${a.cause}`);return t(`steps.notice.${step.name}`, { defaultValue: step.name });default:return step.name;@@ -167,6 +168,12 @@ export function stepFlags(step: RecordedStep, t: T): Flag[] {flags.push({ cls: "err", text: t("steps.flag.sandbox_failed", { stage: t(`steps.sbx_stage.${stage}`, { defaultValue: stage }).toLowerCase() }) });} else if (step.kind === "notice" && step.name === "sandbox_restarted") {if (a.reason) flags.push({ cls: "info", text: t(`steps.flag.restart_${a.reason}`, { defaultValue: a.reason }) });+ } else if (step.kind === "notice" && (step.name === "controller_offline" || step.name === "nobody_watching")) {+ if (step.duration_ms) flags.push({ cls: "info", text: `${clockTime(step.started_at)}–${clockTime(step.started_at + step.duration_ms)}` });+ } else if (step.kind === "notice" && step.name === "clock_jump") {+ flags.push({ cls: "info", text: t("steps.flag.gap_skipped", { t: formatDuration(a.skipped_ms ?? step.duration_ms) }) });+ } else if (step.kind === "notice" && step.name === "recording_gap" && typeof a.until === "number") {+ flags.push({ cls: "warn", text: t("steps.flag.not_recorded", { t: formatDuration(Math.max(0, a.until - step.started_at)) }) });} else if (["error", "denied", "interrupted", "skipped", "unknown"].includes(step.status)) {flags.push({ cls: "err", text: t(`steps.status.${step.status}`) });}@@ -198,12 +205,31 @@ const KIND_CLASS: Record<string, string> = {sandbox: "sbx",};-/** The class of a row: a sandbox restart notice is drawn in the sandbox colour (mock ob13). */+const GAPS = ["controller_offline", "nobody_watching", "clock_jump"];++/** A gap notice with a duration (OB-14): drawn as a dashed band, not a point. */+function isGap(step: RecordedStep): boolean {+ return step.kind === "notice" && GAPS.includes(step.name) && !!step.duration_ms;+}++/** The class of a row: a sandbox restart notice is drawn in the sandbox colour (mock ob13),+ * a gap as a dashed band (mock ob14). */function kindClass(step: RecordedStep): string {if (step.kind === "notice" && step.name === "sandbox_restarted") return "sbx";+ if (isGap(step)) return "gap";return KIND_CLASS[step.kind] || "ctx";}+/** A moment as the clock on the wall shows it, "14:02". */+function clockTime(ms: number): string {+ return new Date(ms).toLocaleTimeString([], { hour: "2-digit", minute: "2-digit" });+}++/** How long the turn spent with the machine offline, or with nobody watching (mock ob14). */+function gapTotal(steps: RecordedStep[], name: string): number {+ return steps.filter((s) => s.kind === "notice" && s.name === name).reduce((n, s) => n + (s.duration_ms || 0), 0);+}+type SandboxStage = { name: string; started_at: number; duration_ms: number | null; status: string; uncertain?: boolean };/** A sandbox setup step's stages as rows under it, in the order run (OB-13). An image pull@@ -379,6 +405,12 @@ export function StepsPanel({<Tile key={b} value={formatDuration(turn.buckets[b])} label={t(`steps.bucket.${b}`)} />))}{turn.buckets.other_shown && <Tile value={formatDuration(turn.buckets.other)} label={t("steps.bucket.other")} />}+ {gapTotal(steps, "controller_offline") > 0 && (+ <Tile value={formatDuration(gapTotal(steps, "controller_offline"))} label={t("steps.gap.offline_total")} />+ )}+ {gapTotal(steps, "nobody_watching") > 0 && (+ <Tile value={formatDuration(gapTotal(steps, "nobody_watching"))} label={t("steps.gap.nobody_total")} />+ )}</div>{steps.some((s) => s.status === "unknown") && (<p className="steps-line" data-testid="steps-gap">@@ -871,6 +903,16 @@ function Inspector({ d, turn, steps, sessionId, t }: { d: any; turn: RecordedTur[t("steps.sbx.reason"), t(`steps.sbx.reason_${a.reason}`, { defaultValue: a.reason || "—" })],[t("steps.sbx.how"), t(`steps.sbx.how_${a.how}`, { defaultValue: a.how || "—" })]);}+ if (isGap(d) || (d.kind === "notice" && d.name === "recording_gap")) {+ const until = d.name === "recording_gap" ? a.until : d.started_at + (d.duration_ms || 0);+ rows.splice(0, rows.length,+ [t("steps.gap.from"), clockTime(d.started_at)],+ [t("steps.gap.until"), typeof until === "number" ? clockTime(until) : "—"]);+ if (d.name === "clock_jump") rows.push([t("steps.gap.skipped"), formatDuration(a.skipped_ms ?? d.duration_ms)]);+ else rows.push([t("steps.detail.duration"), formatDuration(typeof until === "number" ? until - d.started_at : d.duration_ms)]);+ if (a.reason) rows.push([t("steps.gap.why"), t(`steps.gap.reason_${a.reason}`, { defaultValue: a.reason })]);+ if (a.cause) rows.push([t("steps.gap.why"), t(`steps.gap.cause_${a.cause}`, { defaultValue: a.cause })]);+ }const sbxStages: SandboxStage[] = d.kind === "sandbox" && Array.isArray(a.stages) ? a.stages : [];const changed = d.kind === "notice" && d.name === "stamp_changed" ? stampChanges(a, t) : [];const showsContext = d.kind === "context" || (d.kind === "notice" && d.name === "context_changed");@@ -895,6 +937,7 @@ function Inspector({ d, turn, steps, sessionId, t }: { d: any; turn: RecordedTur] as [string, React.ReactNode])} /></>)}+ {d.kind === "notice" && d.name === "clock_jump" && <p className="steps-line">{t("steps.gap.asleep_note")}</p>}{d.kind === "sandbox" && a.error?.text && (<><Sec label={t("steps.sbx.error_text")} />diff --git a/surfaces/gui/src/locales/en.json b/surfaces/gui/src/locales/en.jsonindex 986cd766..92f8e017 100644--- a/surfaces/gui/src/locales/en.json+++ b/surfaces/gui/src/locales/en.json@@ -3037,6 +3037,21 @@}},"steps": {+ "gap": {+ "offline_total": "Machine offline",+ "nobody_total": "Nobody watching",+ "from": "From",+ "until": "Until",+ "why": "Why",+ "reason_closed": "the controller closed the link",+ "reason_failed": "the link to the controller failed",+ "reason_ping_timeout": "the controller stopped answering",+ "cause_restart": "the app ended without closing (a crash or a kill) and started again",+ "cause_stopped": "the app was closed while this turn ran",+ "cause_unknown": "not known",+ "skipped": "Time the machine slept",+ "asleep_note": "Steps after this row are placed by the wall clock again."+ },"sbx_stage": {"preparing": "Preparing","download_image": "Download image",@@ -3266,7 +3281,9 @@"auto_approved": "auto-approved","sandbox_failed": "failed at {{stage}}","restart_folders_changed": "folders",- "restart_runner_restarted": "runner came back"+ "restart_runner_restarted": "runner came back",+ "gap_skipped": "+{{t}}",+ "not_recorded": "{{t}} not recorded"},"purpose": {"main": "main",@@ -3313,7 +3330,12 @@"summary_failed": "Summary failed","recording_gap": "Recording gap","sandbox_restarted": "Sandbox restarted",- "sandbox_regranted": "Sandbox folders updated"+ "sandbox_regranted": "Sandbox folders updated",+ "controller_offline": "Machine offline",+ "nobody_watching": "Nobody watching",+ "clock_jump": "Machine asleep",+ "recording_gap_restart": "App restarted",+ "recording_gap_stopped": "App was closed"},"status": {"ok": "ok",diff --git a/surfaces/gui/src/locales/zh.json b/surfaces/gui/src/locales/zh.jsonindex a078aa21..d9257def 100644--- a/surfaces/gui/src/locales/zh.json+++ b/surfaces/gui/src/locales/zh.json@@ -2993,6 +2993,21 @@}},"steps": {+ "gap": {+ "offline_total": "机器离线",+ "nobody_total": "无人查看",+ "from": "开始",+ "until": "结束",+ "why": "原因",+ "reason_closed": "控制端关闭了连接",+ "reason_failed": "与控制端的连接失败",+ "reason_ping_timeout": "控制端不再响应",+ "cause_restart": "应用未正常关闭(崩溃或被终止)后再次启动",+ "cause_stopped": "本回合运行时应用被关闭",+ "cause_unknown": "未知",+ "skipped": "机器休眠的时间",+ "asleep_note": "此行之后的步骤重新按系统时钟排列。"+ },"sbx_stage": {"preparing": "准备","download_image": "下载镜像",@@ -3217,7 +3232,9 @@"auto_approved": "已自动批准","sandbox_failed": "在{{stage}}失败","restart_folders_changed": "文件夹",- "restart_runner_restarted": "运行器已重启"+ "restart_runner_restarted": "运行器已重启",+ "gap_skipped": "+{{t}}",+ "not_recorded": "{{t}} 未记录"},"purpose": {"main": "主要",@@ -3264,7 +3281,12 @@"summary_failed": "摘要失败","recording_gap": "记录缺口","sandbox_restarted": "沙箱已重启",- "sandbox_regranted": "沙箱文件夹已更新"+ "sandbox_regranted": "沙箱文件夹已更新",+ "controller_offline": "机器离线",+ "nobody_watching": "无人查看",+ "clock_jump": "机器休眠",+ "recording_gap_restart": "应用已重启",+ "recording_gap_stopped": "应用已关闭"},"status": {"ok": "正常",diff --git a/surfaces/gui/src/styles.css b/surfaces/gui/src/styles.cssindex 077738c3..65bef3ba 100644--- a/surfaces/gui/src/styles.css+++ b/surfaces/gui/src/styles.css@@ -2487,6 +2487,8 @@ html[data-theme="dark"] { --bar-model:#4c8dff; --bar-tool:#3fb8ad; --bar-wait:#3.k-ctx .steps-kind, .steps-kind.k-ctx, .k-ctx .steps-bar { background:var(--bar-ctx); }.k-sbx .steps-kind, .steps-kind.k-sbx, .k-sbx .steps-bar { background:var(--bar-sbx); }.steps-stage { cursor:default; }+.k-gap .steps-kind, .steps-kind.k-gap { background:transparent; outline:1px dashed var(--faint); }+.k-gap .steps-bar { background:repeating-linear-gradient(90deg, var(--line-strong) 0 2px, transparent 2px 5px); border:1px dashed var(--faint); top:1px; height:10px; }.steps-stage .steps-bar { opacity:.75; }.k-wait .steps-kind, .steps-kind.k-wait { background:var(--bar-wait); outline:1px dashed var(--faint); }.steps-track { position:relative; height:12px; background:repeating-linear-gradient(90deg, transparent 0 24.6%, var(--line) 24.6% 25%); }diff --git a/tests/corpora/observability/golden-phase2/README.md b/tests/corpora/observability/golden-phase2/README.mdindex 499de2fc..02bf4c44 100644--- a/tests/corpora/observability/golden-phase2/README.md+++ b/tests/corpora/observability/golden-phase2/README.md@@ -1,6 +1,6 @@# Golden test, Phase 2-*Written 2026-11-02 with [recording-format.md](../../recording-format.md) 1.2 and [signals.yaml](../../signals.yaml) 0.3. Made by `phase-2/lab/make_golden2.py`, checked by `phase-2/lab/check_golden2.py`.*+*Written 2026-11-02 with [recording-format.md](../../recording-format.md) 1.2 and [signals.yaml](../../signals.yaml) 0.3; rebuilt 2026-11-03 for signals 0.4 (S2-17d added). Made by `phase-2/lab/make_golden2.py`, checked by `phase-2/lab/check_golden2.py`.*## What it is@@ -14,7 +14,7 @@| `vectors/hashes.json` | `args_hash` cases, each with its value, its RFC 8785 canonical JSON and its 6-hex hash; plan item hashes || `vectors/masking.json` | Rule R4 cases: secrets masked (patterns and stored values), and the 8 KB cut on a character boundary |-There are **38 scenarios**, S2-01 to S2-19 with their clean twins. Each is described in words in [phase-2/scenarios.md](../../phase-2/scenarios.md).+There are **39 scenarios**, S2-01 to S2-19 with their clean twins. Each is described in words in [phase-2/scenarios.md](../../phase-2/scenarios.md).## Built, not captureddiff --git a/tests/corpora/observability/golden-phase2/S2-01/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-01/expected-issues.jsonindex 1969fdfc..359a9f69 100644--- a/tests/corpora/observability/golden-phase2/S2-01/expected-issues.json+++ b/tests/corpora/observability/golden-phase2/S2-01/expected-issues.json@@ -1,5 +1,5 @@{- "signals_version": "0.3",+ "signals_version": "0.4","warn": 1,"fired": {"repeated_step": [diff --git a/tests/corpora/observability/golden-phase2/S2-01c/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-01c/expected-issues.jsonindex 49ff0804..bc994a3a 100644--- a/tests/corpora/observability/golden-phase2/S2-01c/expected-issues.json+++ b/tests/corpora/observability/golden-phase2/S2-01c/expected-issues.json@@ -1,5 +1,5 @@{- "signals_version": "0.3",+ "signals_version": "0.4","warn": 0,"fired": {},"fine": [diff --git a/tests/corpora/observability/golden-phase2/S2-02/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-02/expected-issues.jsonindex fe56fae4..579901cb 100644--- a/tests/corpora/observability/golden-phase2/S2-02/expected-issues.json+++ b/tests/corpora/observability/golden-phase2/S2-02/expected-issues.json@@ -1,5 +1,5 @@{- "signals_version": "0.3",+ "signals_version": "0.4","warn": 1,"fired": {"off_plan": [diff --git a/tests/corpora/observability/golden-phase2/S2-02b/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-02b/expected-issues.jsonindex 52a7068a..739d59d6 100644--- a/tests/corpora/observability/golden-phase2/S2-02b/expected-issues.json+++ b/tests/corpora/observability/golden-phase2/S2-02b/expected-issues.json@@ -1,5 +1,5 @@{- "signals_version": "0.3",+ "signals_version": "0.4","warn": 1,"fired": {"off_plan": [diff --git a/tests/corpora/observability/golden-phase2/S2-02c/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-02c/expected-issues.jsonindex 11d75035..5061dd2d 100644--- a/tests/corpora/observability/golden-phase2/S2-02c/expected-issues.json+++ b/tests/corpora/observability/golden-phase2/S2-02c/expected-issues.json@@ -1,5 +1,5 @@{- "signals_version": "0.3",+ "signals_version": "0.4","warn": 0,"fired": {},"fine": [diff --git a/tests/corpora/observability/golden-phase2/S2-02n/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-02n/expected-issues.jsonindex 49ff0804..bc994a3a 100644--- a/tests/corpora/observability/golden-phase2/S2-02n/expected-issues.json+++ b/tests/corpora/observability/golden-phase2/S2-02n/expected-issues.json@@ -1,5 +1,5 @@{- "signals_version": "0.3",+ "signals_version": "0.4","warn": 0,"fired": {},"fine": [diff --git a/tests/corpora/observability/golden-phase2/S2-03/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-03/expected-issues.jsonindex 058d86ad..53a46fa6 100644--- a/tests/corpora/observability/golden-phase2/S2-03/expected-issues.json+++ b/tests/corpora/observability/golden-phase2/S2-03/expected-issues.json@@ -1,5 +1,5 @@{- "signals_version": "0.3",+ "signals_version": "0.4","warn": 1,"fired": {"refused": [diff --git a/tests/corpora/observability/golden-phase2/S2-03b/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-03b/expected-issues.jsonindex 49ff0804..bc994a3a 100644--- a/tests/corpora/observability/golden-phase2/S2-03b/expected-issues.json+++ b/tests/corpora/observability/golden-phase2/S2-03b/expected-issues.json@@ -1,5 +1,5 @@{- "signals_version": "0.3",+ "signals_version": "0.4","warn": 0,"fired": {},"fine": [diff --git a/tests/corpora/observability/golden-phase2/S2-04/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-04/expected-issues.jsonindex fa30041d..fdb534a5 100644--- a/tests/corpora/observability/golden-phase2/S2-04/expected-issues.json+++ b/tests/corpora/observability/golden-phase2/S2-04/expected-issues.json@@ -1,5 +1,5 @@{- "signals_version": "0.3",+ "signals_version": "0.4","warn": 0,"fired": {"overridden": [diff --git a/tests/corpora/observability/golden-phase2/S2-04n/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-04n/expected-issues.jsonindex 49ff0804..bc994a3a 100644--- a/tests/corpora/observability/golden-phase2/S2-04n/expected-issues.json+++ b/tests/corpora/observability/golden-phase2/S2-04n/expected-issues.json@@ -1,5 +1,5 @@{- "signals_version": "0.3",+ "signals_version": "0.4","warn": 0,"fired": {},"fine": [diff --git a/tests/corpora/observability/golden-phase2/S2-05/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-05/expected-issues.jsonindex 25ba330d..da408400 100644--- a/tests/corpora/observability/golden-phase2/S2-05/expected-issues.json+++ b/tests/corpora/observability/golden-phase2/S2-05/expected-issues.json@@ -1,5 +1,5 @@{- "signals_version": "0.3",+ "signals_version": "0.4","warn": 1,"fired": {"reviewer_could_not_run": [diff --git a/tests/corpora/observability/golden-phase2/S2-05c/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-05c/expected-issues.jsonindex b142c8d2..4042fe25 100644--- a/tests/corpora/observability/golden-phase2/S2-05c/expected-issues.json+++ b/tests/corpora/observability/golden-phase2/S2-05c/expected-issues.json@@ -1,5 +1,5 @@{- "signals_version": "0.3",+ "signals_version": "0.4","warn": 0,"fired": {},"fine": [diff --git a/tests/corpora/observability/golden-phase2/S2-06/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-06/expected-issues.jsonindex 0bc057dc..e525f777 100644--- a/tests/corpora/observability/golden-phase2/S2-06/expected-issues.json+++ b/tests/corpora/observability/golden-phase2/S2-06/expected-issues.json@@ -1,5 +1,5 @@{- "signals_version": "0.3",+ "signals_version": "0.4","warn": 1,"fired": {"reviewer_paused": [diff --git a/tests/corpora/observability/golden-phase2/S2-07/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-07/expected-issues.jsonindex 64d81e9e..2fe645b2 100644--- a/tests/corpora/observability/golden-phase2/S2-07/expected-issues.json+++ b/tests/corpora/observability/golden-phase2/S2-07/expected-issues.json@@ -1,5 +1,5 @@{- "signals_version": "0.3",+ "signals_version": "0.4","warn": 0,"fired": {"shadow_disagreed": [diff --git a/tests/corpora/observability/golden-phase2/S2-08/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-08/expected-issues.jsonindex 271fef7b..513b11c2 100644--- a/tests/corpora/observability/golden-phase2/S2-08/expected-issues.json+++ b/tests/corpora/observability/golden-phase2/S2-08/expected-issues.json@@ -1,5 +1,5 @@{- "signals_version": "0.3",+ "signals_version": "0.4","warn": 1,"fired": {"tool_failed": [diff --git a/tests/corpora/observability/golden-phase2/S2-08b/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-08b/expected-issues.jsonindex 271fef7b..513b11c2 100644--- a/tests/corpora/observability/golden-phase2/S2-08b/expected-issues.json+++ b/tests/corpora/observability/golden-phase2/S2-08b/expected-issues.json@@ -1,5 +1,5 @@{- "signals_version": "0.3",+ "signals_version": "0.4","warn": 1,"fired": {"tool_failed": [diff --git a/tests/corpora/observability/golden-phase2/S2-08c/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-08c/expected-issues.jsonindex 8d3822b3..2598d4db 100644--- a/tests/corpora/observability/golden-phase2/S2-08c/expected-issues.json+++ b/tests/corpora/observability/golden-phase2/S2-08c/expected-issues.json@@ -1,5 +1,5 @@{- "signals_version": "0.3",+ "signals_version": "0.4","warn": 1,"fired": {"made_up_tool": [diff --git a/tests/corpora/observability/golden-phase2/S2-09/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-09/expected-issues.jsonindex 7c7c4519..7e9020eb 100644--- a/tests/corpora/observability/golden-phase2/S2-09/expected-issues.json+++ b/tests/corpora/observability/golden-phase2/S2-09/expected-issues.json@@ -1,5 +1,5 @@{- "signals_version": "0.3",+ "signals_version": "0.4","warn": 1,"fired": {"model_failed": [diff --git a/tests/corpora/observability/golden-phase2/S2-09c/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-09c/expected-issues.jsonindex 49ff0804..bc994a3a 100644--- a/tests/corpora/observability/golden-phase2/S2-09c/expected-issues.json+++ b/tests/corpora/observability/golden-phase2/S2-09c/expected-issues.json@@ -1,5 +1,5 @@{- "signals_version": "0.3",+ "signals_version": "0.4","warn": 0,"fired": {},"fine": [diff --git a/tests/corpora/observability/golden-phase2/S2-10/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-10/expected-issues.jsonindex bf5dc859..2a7940af 100644--- a/tests/corpora/observability/golden-phase2/S2-10/expected-issues.json+++ b/tests/corpora/observability/golden-phase2/S2-10/expected-issues.json@@ -1,5 +1,5 @@{- "signals_version": "0.3",+ "signals_version": "0.4","warn": 1,"fired": {"retried": [diff --git a/tests/corpora/observability/golden-phase2/S2-10b/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-10b/expected-issues.jsonindex 49ff0804..bc994a3a 100644--- a/tests/corpora/observability/golden-phase2/S2-10b/expected-issues.json+++ b/tests/corpora/observability/golden-phase2/S2-10b/expected-issues.json@@ -1,5 +1,5 @@{- "signals_version": "0.3",+ "signals_version": "0.4","warn": 0,"fired": {},"fine": [diff --git a/tests/corpora/observability/golden-phase2/S2-11/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-11/expected-issues.jsonindex 0436ff42..0ed0fd33 100644--- a/tests/corpora/observability/golden-phase2/S2-11/expected-issues.json+++ b/tests/corpora/observability/golden-phase2/S2-11/expected-issues.json@@ -1,5 +1,5 @@{- "signals_version": "0.3",+ "signals_version": "0.4","warn": 1,"fired": {"cut_off": [diff --git a/tests/corpora/observability/golden-phase2/S2-12/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-12/expected-issues.jsonindex 0ccfcaba..cf9ac100 100644--- a/tests/corpora/observability/golden-phase2/S2-12/expected-issues.json+++ b/tests/corpora/observability/golden-phase2/S2-12/expected-issues.json@@ -1,5 +1,5 @@{- "signals_version": "0.3",+ "signals_version": "0.4","warn": 1,"fired": {"step_limit": [diff --git a/tests/corpora/observability/golden-phase2/S2-13/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-13/expected-issues.jsonindex 60324950..934ff556 100644--- a/tests/corpora/observability/golden-phase2/S2-13/expected-issues.json+++ b/tests/corpora/observability/golden-phase2/S2-13/expected-issues.json@@ -1,5 +1,5 @@{- "signals_version": "0.3",+ "signals_version": "0.4","warn": 2,"fired": {"made_up_tool": [diff --git a/tests/corpora/observability/golden-phase2/S2-14/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-14/expected-issues.jsonindex 2407cc96..dde419fa 100644--- a/tests/corpora/observability/golden-phase2/S2-14/expected-issues.json+++ b/tests/corpora/observability/golden-phase2/S2-14/expected-issues.json@@ -1,5 +1,5 @@{- "signals_version": "0.3",+ "signals_version": "0.4","warn": 1,"fired": {"context_full": [diff --git a/tests/corpora/observability/golden-phase2/S2-14b/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-14b/expected-issues.jsonindex 49ff0804..bc994a3a 100644--- a/tests/corpora/observability/golden-phase2/S2-14b/expected-issues.json+++ b/tests/corpora/observability/golden-phase2/S2-14b/expected-issues.json@@ -1,5 +1,5 @@{- "signals_version": "0.3",+ "signals_version": "0.4","warn": 0,"fired": {},"fine": [diff --git a/tests/corpora/observability/golden-phase2/S2-14d/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-14d/expected-issues.jsonindex d0ab7d0a..8fcfa96b 100644--- a/tests/corpora/observability/golden-phase2/S2-14d/expected-issues.json+++ b/tests/corpora/observability/golden-phase2/S2-14d/expected-issues.json@@ -1,5 +1,5 @@{- "signals_version": "0.3",+ "signals_version": "0.4","warn": 1,"fired": {"context_full": [diff --git a/tests/corpora/observability/golden-phase2/S2-15/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-15/expected-issues.jsonindex b1634142..140ad354 100644--- a/tests/corpora/observability/golden-phase2/S2-15/expected-issues.json+++ b/tests/corpora/observability/golden-phase2/S2-15/expected-issues.json@@ -1,5 +1,5 @@{- "signals_version": "0.3",+ "signals_version": "0.4","warn": 1,"fired": {"blocked_by_sandbox": [diff --git a/tests/corpora/observability/golden-phase2/S2-15c/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-15c/expected-issues.jsonindex b4d8c189..bf937979 100644--- a/tests/corpora/observability/golden-phase2/S2-15c/expected-issues.json+++ b/tests/corpora/observability/golden-phase2/S2-15c/expected-issues.json@@ -1,5 +1,5 @@{- "signals_version": "0.3",+ "signals_version": "0.4","warn": 0,"fired": {},"fine": [diff --git a/tests/corpora/observability/golden-phase2/S2-16/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-16/expected-issues.jsonindex cea4e470..03d5f06d 100644--- a/tests/corpora/observability/golden-phase2/S2-16/expected-issues.json+++ b/tests/corpora/observability/golden-phase2/S2-16/expected-issues.json@@ -1,5 +1,5 @@{- "signals_version": "0.3",+ "signals_version": "0.4","warn": 1,"fired": {"ran_without_sandbox": [diff --git a/tests/corpora/observability/golden-phase2/S2-16c/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-16c/expected-issues.jsonindex c6e0bfe7..9665d26d 100644--- a/tests/corpora/observability/golden-phase2/S2-16c/expected-issues.json+++ b/tests/corpora/observability/golden-phase2/S2-16c/expected-issues.json@@ -1,5 +1,5 @@{- "signals_version": "0.3",+ "signals_version": "0.4","warn": 0,"fired": {},"fine": [diff --git a/tests/corpora/observability/golden-phase2/S2-17/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-17/expected-issues.jsonindex a3eb80ff..e41d83a0 100644--- a/tests/corpora/observability/golden-phase2/S2-17/expected-issues.json+++ b/tests/corpora/observability/golden-phase2/S2-17/expected-issues.json@@ -1,5 +1,5 @@{- "signals_version": "0.3",+ "signals_version": "0.4","warn": 1,"fired": {"waited_unwatched": [diff --git a/tests/corpora/observability/golden-phase2/S2-17b/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-17b/expected-issues.jsonindex 49ff0804..bc994a3a 100644--- a/tests/corpora/observability/golden-phase2/S2-17b/expected-issues.json+++ b/tests/corpora/observability/golden-phase2/S2-17b/expected-issues.json@@ -1,5 +1,5 @@{- "signals_version": "0.3",+ "signals_version": "0.4","warn": 0,"fired": {},"fine": [diff --git a/tests/corpora/observability/golden-phase2/S2-17c/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-17c/expected-issues.jsonindex 49ff0804..bc994a3a 100644--- a/tests/corpora/observability/golden-phase2/S2-17c/expected-issues.json+++ b/tests/corpora/observability/golden-phase2/S2-17c/expected-issues.json@@ -1,5 +1,5 @@{- "signals_version": "0.3",+ "signals_version": "0.4","warn": 0,"fired": {},"fine": [diff --git a/tests/corpora/observability/golden-phase2/S2-17d/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-17d/expected-issues.jsonnew file mode 100644index 00000000..e41d83a0--- /dev/null+++ b/tests/corpora/observability/golden-phase2/S2-17d/expected-issues.json@@ -0,0 +1,31 @@+{+ "signals_version": "0.4",+ "warn": 1,+ "fired": {+ "waited_unwatched": [+ 2+ ]+ },+ "fine": [+ "context_full",+ "cut_off",+ "made_up_tool",+ "model_failed",+ "refused",+ "repeated_step",+ "retried",+ "step_limit",+ "tool_failed"+ ],+ "not_checked": [+ "blocked_by_sandbox",+ "off_plan",+ "overridden",+ "ran_without_sandbox",+ "reviewer_allowed_send",+ "reviewer_could_not_run",+ "reviewer_paused",+ "sandbox_failed",+ "shadow_disagreed"+ ]+}diff --git a/tests/corpora/observability/golden-phase2/S2-17d/recording.json b/tests/corpora/observability/golden-phase2/S2-17d/recording.jsonnew file mode 100644index 00000000..2cc4eae2--- /dev/null+++ b/tests/corpora/observability/golden-phase2/S2-17d/recording.json@@ -0,0 +1,190 @@+{+ "turns": [+ {+ "turn_id": "01JP2TRN000000000000000000",+ "session_id": "golden-s2-17d",+ "seq": 1,+ "status": "completed",+ "approval_mode": "interactive",+ "attended": true,+ "reviewer": {+ "active": false,+ "shadow": false,+ "kind": null,+ "model": null+ },+ "stamp": {+ "sandbox": {+ "provider": "direct",+ "enforcement": "none"+ }+ },+ "max_iterations": 50,+ "kind": "run",+ "trigger": "user",+ "started_at": 1793600000000,+ "incomplete": false,+ "first_message_id": "01JP2MSG000000000000000000",+ "last_message_id": "01JP2MSG000000000000000000",+ "schema_version": 1,+ "duration_ms": 1200021+ }+ ],+ "steps": [+ {+ "step_id": "01JP2STP000000000000000000",+ "turn_id": "01JP2TRN000000000000000000",+ "session_id": "golden-s2-17d",+ "parent_step_id": null,+ "seq": 1,+ "kind": "model",+ "name": "model",+ "started_at": 1793600000000,+ "duration_ms": 10,+ "status": "ok",+ "call_id": null,+ "call_index": null,+ "message_id": "01JP2MSG000000000000000010",+ "shadow": false,+ "after_turn": false,+ "attrs": {+ "purpose": "main",+ "provider": "testprov",+ "provider_title": "Test provider",+ "model": "testprov:model-a",+ "stream": true,+ "finish_reason": "stop",+ "ttft_ms": 5,+ "context_window": 100000,+ "usage": {+ "reported": true,+ "input_fresh": 1000,+ "cache_read": 0,+ "cache_write": null,+ "output": 100,+ "reasoning": null+ },+ "tools_offered": 3,+ "max_output": 4096+ },+ "content": {}+ },+ {+ "step_id": "01JP2STP000000000000000000",+ "turn_id": "01JP2TRN000000000000000000",+ "session_id": "golden-s2-17d",+ "parent_step_id": null,+ "seq": 2,+ "kind": "wait",+ "name": "approval",+ "started_at": 1793600000020,+ "duration_ms": 1200000,+ "status": "ok",+ "call_id": "call_3",+ "call_index": null,+ "message_id": null,+ "shadow": false,+ "after_turn": false,+ "attrs": {+ "for_call_id": "call_3",+ "kind": "approval",+ "where": "session",+ "attended": true,+ "outcome": "approved",+ "inbox_at": null+ },+ "content": {}+ },+ {+ "step_id": "01JP2STP000000000000000000",+ "turn_id": "01JP2TRN000000000000000000",+ "session_id": "golden-s2-17d",+ "parent_step_id": null,+ "seq": 3,+ "kind": "notice",+ "name": "nobody_watching",+ "started_at": 1793600060020,+ "duration_ms": 1080000,+ "status": "ok",+ "call_id": null,+ "call_index": null,+ "message_id": null,+ "shadow": false,+ "after_turn": false,+ "attrs": {},+ "content": {}+ },+ {+ "step_id": "01JP2STP000000000000000000",+ "turn_id": "01JP2TRN000000000000000000",+ "session_id": "golden-s2-17d",+ "parent_step_id": null,+ "seq": 4,+ "kind": "tool",+ "name": "run_shell",+ "started_at": 1793601140021,+ "duration_ms": 10,+ "status": "ok",+ "call_id": "call_4",+ "call_index": 0,+ "message_id": "01JP2MSG000000000000000020",+ "shadow": false,+ "after_turn": false,+ "attrs": {+ "risk_class": "exec",+ "decision": {+ "path": [],+ "origin": "low_risk"+ },+ "args_hash": "215ddd",+ "repeat_ok": false,+ "result_error": false,+ "error_kind": null,+ "connector": null,+ "result_bytes": 120,+ "exit_code": null,+ "sandbox": null,+ "reaches_network": false+ },+ "content": {}+ },+ {+ "step_id": "01JP2STP000000000000000000",+ "turn_id": "01JP2TRN000000000000000000",+ "session_id": "golden-s2-17d",+ "parent_step_id": null,+ "seq": 5,+ "kind": "model",+ "name": "model",+ "started_at": 1793601140032,+ "duration_ms": 10,+ "status": "ok",+ "call_id": null,+ "call_index": null,+ "message_id": "01JP2MSG000000000000000010",+ "shadow": false,+ "after_turn": false,+ "attrs": {+ "purpose": "main",+ "provider": "testprov",+ "provider_title": "Test provider",+ "model": "testprov:model-a",+ "stream": true,+ "finish_reason": "stop",+ "ttft_ms": 5,+ "context_window": 100000,+ "usage": {+ "reported": true,+ "input_fresh": 1000,+ "cache_read": 0,+ "cache_write": null,+ "output": 100,+ "reasoning": null+ },+ "tools_offered": 3,+ "max_output": 4096+ },+ "content": {}+ }+ ]+}diff --git a/tests/corpora/observability/golden-phase2/S2-18/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-18/expected-issues.jsonindex 541d78da..afb49b23 100644--- a/tests/corpora/observability/golden-phase2/S2-18/expected-issues.json+++ b/tests/corpora/observability/golden-phase2/S2-18/expected-issues.json@@ -1,5 +1,5 @@{- "signals_version": "0.3",+ "signals_version": "0.4","warn": 1,"fired": {"sandbox_failed": [diff --git a/tests/corpora/observability/golden-phase2/S2-18c/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-18c/expected-issues.jsonindex ab3defd8..6c2d41c1 100644--- a/tests/corpora/observability/golden-phase2/S2-18c/expected-issues.json+++ b/tests/corpora/observability/golden-phase2/S2-18c/expected-issues.json@@ -1,5 +1,5 @@{- "signals_version": "0.3",+ "signals_version": "0.4","warn": 0,"fired": {},"fine": [diff --git a/tests/corpora/observability/golden-phase2/S2-19/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-19/expected-issues.jsonindex 34c71169..ef073414 100644--- a/tests/corpora/observability/golden-phase2/S2-19/expected-issues.json+++ b/tests/corpora/observability/golden-phase2/S2-19/expected-issues.json@@ -1,5 +1,5 @@{- "signals_version": "0.3",+ "signals_version": "0.4","warn": 0,"fired": {"reviewer_allowed_send": [diff --git a/tests/corpora/observability/golden-phase2/S2-19c/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-19c/expected-issues.jsonindex b142c8d2..4042fe25 100644--- a/tests/corpora/observability/golden-phase2/S2-19c/expected-issues.json+++ b/tests/corpora/observability/golden-phase2/S2-19c/expected-issues.json@@ -1,5 +1,5 @@{- "signals_version": "0.3",+ "signals_version": "0.4","warn": 0,"fired": {},"fine": [diff --git a/tests/corpora/observability/golden-phase2/index.json b/tests/corpora/observability/golden-phase2/index.jsonindex 8feb4310..56e98666 100644--- a/tests/corpora/observability/golden-phase2/index.json+++ b/tests/corpora/observability/golden-phase2/index.json@@ -1,6 +1,6 @@{"format": "1.2",- "signals_version": "0.3",+ "signals_version": "0.4","built": true,"scenarios": [{@@ -316,6 +316,15 @@"should": "stay quiet","also_fires": [],"fired": []+ },+ {+ "id": "S2-17d",+ "check": "waited_unwatched",+ "should": "fire",+ "also_fires": [],+ "fired": [+ "waited_unwatched"+ ]}]}diff --git a/tests/test_observability_gaps.py b/tests/test_observability_gaps.pynew file mode 100644index 00000000..416bc6df--- /dev/null+++ b/tests/test_observability_gaps.py@@ -0,0 +1,301 @@+"""OB-14: gaps in the timeline (phase-2 design §11, recording format 1.2): the machine+offline from its controller, nobody watching the session, the machine asleep, and the+app restarted mid-turn. Each is a notice; the first three have a duration."""++from __future__ import annotations++import json+from pathlib import Path+from types import SimpleNamespace++import jsonschema+import pytest++from coworker.observability import Recorder, RecordingStore, gaps, signals+from coworker.observability.engine_hooks import TurnRecording+from coworker.observability.export_spans import Masker, build_turn_spans+from coworker.server.manager import SessionManager++SCHEMA = json.loads((Path(__file__).parent / "corpora/observability/recording-format.schema.json").read_text("utf-8"))+++def _check(recording):+ validator = jsonschema.validators.validator_for(SCHEMA)(SCHEMA)+ errors = sorted(validator.iter_errors(recording), key=lambda e: list(e.absolute_path))+ assert not errors, "\n".join(f"{list(e.absolute_path)}: {e.message[:300]}" for e in errors[:8])++[email protected](autouse=True)+def _no_machine_conditions():+ gaps._machine.clear()+ yield+ gaps._machine.clear()+++def _recording(session="sess_g"):+ rec = Recorder()+ obs = TurnRecording(rec, lambda: session)+ return rec, obs+++def _begin(obs):+ return obs.begin_turn(kind="run", trigger="user", attrs={+ "approval_mode": "ask", "attended": True,+ "reviewer": {"active": False, "shadow": False, "kind": None, "model": None}}, stamp={}, first_message_id=None)+++def _gaps(rec, name):+ return [s for s in rec.as_dict()["steps"] if s["kind"] == "notice" and s["name"] == name]+++# -- nobody watching (E15b-2) ------------------------------------------------------------------+++def test_nobody_watching_mid_turn_is_a_band_from_leaving_to_coming_back():+ rec, obs = _recording()+ _begin(obs)+ start = obs._turn_started_ms+ obs.condition("nobody_watching", True, at_ms=start + 1000)+ obs.condition("nobody_watching", False, at_ms=start + 61_000)+ obs.end_turn("completed")+ [gap] = _gaps(rec, "nobody_watching")+ assert (gap["started_at"], gap["duration_ms"], gap["status"], gap["attrs"]) == (start + 1000, 60_000, "ok", {})+ assert gap["parent_step_id"] is None+++def test_a_gap_still_on_is_cut_at_each_turns_start_and_end():+ rec, obs = _recording()+ obs.condition("nobody_watching", True, at_ms=1) # left long before the turn: a schedule ran it+ import time++ t1 = _begin(obs)+ time.sleep(0.01)+ obs.end_turn("completed")+ t2 = _begin(obs)+ time.sleep(0.01)+ obs.end_turn("completed")+ one, two = _gaps(rec, "nobody_watching")+ assert one["turn_id"] == t1 and two["turn_id"] == t2+ turns = {t["turn_id"]: t for t in rec.as_dict()["turns"]}+ assert one["started_at"] == turns[t1]["started_at"] # not before its turn+ assert one["started_at"] + one["duration_ms"] <= turns[t1]["started_at"] + turns[t1]["duration_ms"] + 1+ _check(rec.as_dict())+++def test_a_condition_outside_any_turn_writes_nothing():+ rec, obs = _recording()+ obs.condition("nobody_watching", True)+ obs.condition("nobody_watching", False)+ assert rec.as_dict()["steps"] == []+++def test_only_the_crossings_between_none_and_one_viewer_count():+ told = []+ fake = SimpleNamespace(_session_clients={}, _engines={})+ fake._note_watching = lambda sid, watching: told.append((sid, watching))+ register = SessionManager.register_session_client.__get__(fake)+ unregister = SessionManager.unregister_session_client.__get__(fake)+ a, b = object(), object()+ register("s1", a)+ register("s1", b) # a second window: no row+ unregister("s1", a)+ unregister("s1", b)+ assert told == [("s1", True), ("s1", False)]+++def test_the_manager_tells_the_engines_recording():+ rec, obs = _recording("s1")+ fake = SimpleNamespace(_engines={"s1": SimpleNamespace(_obs=obs)})+ note = SessionManager._note_watching.__get__(fake)+ _begin(obs)+ note("s1", False)+ note("s1", True)+ obs.end_turn("completed")+ assert len(_gaps(rec, "nobody_watching")) <= 1 # a band only if any time passed+ note("missing", False) # no engine: nothing, no error+++# -- the controller link (E15b-1) --------------------------------------------------------------+++def test_machine_offline_reaches_every_running_turn(tmp_path):+ rec1, obs1 = _recording("a")+ rec2, obs2 = _recording("b")+ _begin(obs1)+ _begin(obs2)+ start = max(obs1._turn_started_ms, obs2._turn_started_ms)+ gaps.machine_condition("controller_offline", True, at_ms=start + 10, attrs={"reason": "ping_timeout"})+ gaps.machine_condition("controller_offline", False, at_ms=start + 420_010)+ obs1.end_turn("completed")+ obs2.end_turn("completed")+ for rec in (rec1, rec2):+ [gap] = _gaps(rec, "controller_offline")+ assert gap["duration_ms"] == 420_000 and gap["attrs"] == {"reason": "ping_timeout"}+ _check(rec.as_dict())+++def test_an_engine_made_while_offline_hears_it():+ gaps.machine_condition("controller_offline", True, at_ms=5, attrs={"reason": "failed"})+ rec, obs = _recording()+ _begin(obs)+ gaps.machine_condition("controller_offline", False, at_ms=obs._turn_started_ms + 1_000)+ [gap] = _gaps(rec, "controller_offline")+ assert gap["started_at"] == obs._turn_started_ms and gap["attrs"] == {"reason": "failed"}+++def test_the_lost_moment_survives_a_restart(tmp_path):+ gaps.link_lost(tmp_path, "closed")+ saved = json.loads((tmp_path / "controller-link-lost.json").read_text("utf-8"))+ assert saved["reason"] == "closed"+ gaps._machine.clear() # the app restarts: the memory is gone, the file is not+ rec, obs = _recording()+ _begin(obs)+ import time+ time.sleep(0.01) # the link comes back a moment after the turn started+ gaps.link_back(tmp_path)+ [gap] = _gaps(rec, "controller_offline")+ assert gap["attrs"] == {"reason": "closed"}+ assert not (tmp_path / "controller-link-lost.json").exists()+ gaps.link_back(tmp_path) # a welcome with nothing lost: nothing+ assert len(_gaps(rec, "controller_offline")) == 1+++def test_link_reasons():+ assert gaps.link_reason(None) == "closed"+ assert gaps.link_reason(TimeoutError("keepalive ping timeout")) == "ping_timeout"+ assert gaps.link_reason(OSError("connection refused")) == "failed"+++# -- the machine asleep (E15b-3) ---------------------------------------------------------------+++def _ev(typ, **kw):+ return {"v": 1, "type": typ, "session_id": "sess_c", "turn_id": "t1", **kw}+++def _step(step_id, at, mono, end_at, end_mono):+ return [+ _ev("step_started", step_id=step_id, at_ms=at, mono_ms=mono, kind="tool", name="look",+ parent_step_id=None, call_id=step_id, call_index=0, shadow=False, after_turn=False,+ attrs={}, content={}),+ _ev("step_ended", step_id=step_id, at_ms=end_at, mono_ms=end_mono, status="ok", message_id=None,+ attrs={}, content={}, kind="tool", name="look", parent_step_id=None, call_id=step_id,+ call_index=0, shadow=False, after_turn=False, started_at=at),+ ]+++def test_a_sleep_mid_turn_is_a_clock_jump_and_later_steps_follow_the_wall_clock():+ rec = Recorder()+ rec.emit(_ev("turn_started", at_ms=1_000, mono_ms=0, kind="run", trigger="user", first_message_id=None,+ attrs={}, stamp={}))+ for ev in _step("a", 1_100, 100, 1_200, 200):+ rec.emit(ev)+ # The machine slept 60 s: the wall clock moved on, the monotonic clock did not.+ for ev in _step("b", 61_300, 300, 61_400, 400):+ rec.emit(ev)+ rec.emit(_ev("turn_ended", at_ms=61_500, mono_ms=500, status="completed", last_message_id=None))+ d = rec.as_dict()+ by = {s["name"]: s for s in d["steps"]}+ jump = by["clock_jump"]+ assert jump["started_at"] == 1_200 and jump["duration_ms"] == 60_000+ assert jump["attrs"] == {"skipped_ms": 60_000}+ assert by["look"]["started_at"] in (1_100, 61_300)+ starts = sorted(s["started_at"] for s in d["steps"] if s["kind"] == "tool")+ assert starts == [1_100, 61_300] # the second step is placed by the wall clock again+ assert d["turns"][0]["duration_ms"] == 60_500 # the turn's length counts the sleep+ assert [s["name"] for s in d["steps"]] == ["look", "clock_jump", "look"]+++def test_small_drift_is_not_a_jump():+ rec = Recorder()+ rec.emit(_ev("turn_started", at_ms=1_000, mono_ms=0, kind="run", trigger="user", first_message_id=None,+ attrs={}, stamp={}))+ for ev in _step("a", 4_000, 100, 4_100, 200): # 2.9 s ahead: within the 5 s allowed+ rec.emit(ev)+ assert not [s for s in rec.as_dict()["steps"] if s["name"] == "clock_jump"]+++# -- the app restarted (E15b-4) ----------------------------------------------------------------+++def _open_turn(path):+ store = RecordingStore(path, background=False)+ rec = Recorder(store=store)+ rec.emit(_ev("turn_started", at_ms=1_000, mono_ms=0, kind="run", trigger="user", first_message_id=None,+ attrs={}, stamp={}))+ store.flush()+ return store+++def _gap_attrs(path):+ s = RecordingStore(path, background=False)+ try:+ assert s.close_open_turns() == 1+ [gap] = [x for x in s.load_recording("sess_c")["steps"] if x["name"] == "recording_gap"]+ return gap["attrs"]+ finally:+ s.close()+++def test_a_turn_cut_by_a_crash_says_restart(tmp_path):+ path = tmp_path / "recordings.db"+ first = RecordingStore(path, background=False)+ first.close_open_turns() # this run owns the folder+ first._conn.close() # killed: never closed properly+ s = _open_turn(path)+ s._conn.close()+ attrs = _gap_attrs(path)+ assert attrs["cause"] == "restart" and isinstance(attrs["until"], int)+++def test_a_turn_cut_by_closing_the_app_says_stopped(tmp_path):+ path = tmp_path / "recordings.db"+ first = RecordingStore(path, background=False)+ first.close_open_turns()+ rec = Recorder(store=first)+ rec.emit(_ev("turn_started", at_ms=1_000, mono_ms=0, kind="run", trigger="user", first_message_id=None,+ attrs={}, stamp={}))+ first.close() # a clean stop while the turn ran+ assert _gap_attrs(path)["cause"] == "stopped"+++def test_with_no_word_of_the_last_run_the_cause_is_unknown(tmp_path):+ path = tmp_path / "recordings.db"+ s = _open_turn(path)+ s.close() # not the owner: leaves no word+ assert _gap_attrs(path)["cause"] == "unknown"+++# -- what reads the gaps -----------------------------------------------------------------------+++def test_a_gap_is_its_own_span_and_other_notices_stay_events():+ rec, obs = _recording()+ _begin(obs)+ start = obs._turn_started_ms+ obs.condition("nobody_watching", True, at_ms=start)+ obs.condition("nobody_watching", False, at_ms=start + 5_000)+ obs.instant_step(kind="notice", name="interrupted", status="ok")+ obs.end_turn("interrupted")+ d = rec.as_dict()+ turn = d["turns"][0]+ spans = build_turn_spans(turn, d["steps"], sw={}, dialect="otel", get_piece=lambda k: None, masker=Masker())+ [gap] = [s for s in spans if s["name"] == "openworker.gap.nobody_watching"]+ assert gap["end_ms"] - gap["start_ms"] == 5_000+ [root] = [s for s in spans if s["name"] == "invoke_agent coworker"]+ assert [e["name"] for e in root["events"]] == ["openworker.notice.interrupted"]+++def test_a_card_left_in_the_session_while_nobody_watched_is_an_issue():+ turn = {"turn_id": "t1", "started_at": 0, "duration_ms": 1_300_000, "status": "completed", "stamp": {},+ "reviewer": None, "attended": True}+ wait = {"step_id": "w", "kind": "wait", "name": "approval", "status": "ok", "parent_step_id": None,+ "shadow": False, "after_turn": False, "started_at": 0, "duration_ms": 1_200_000, "seq": 1,+ "attrs": {"kind": "approval", "where": "session", "outcome": "approved"}}+ gap = {"step_id": "g", "kind": "notice", "name": "nobody_watching", "status": "ok", "parent_step_id": None,+ "shadow": False, "after_turn": False, "started_at": 100_000, "duration_ms": 1_000_000, "seq": 2, "attrs": {}}+ found = signals.check_turn(turn, [wait, gap])+ [i] = [x for x in found["issues"] if x["check"] == "waited_unwatched"]+ assert "17 min for an answer" in i["text"] and "Inbox" not in i["text"]+ quiet = signals.check_turn(turn, [wait])+ assert not [x for x in quiet["issues"] if x["check"] == "waited_unwatched"]