Skip to content
Python

feat(observability): gaps in the timeline (OB-14)

Devika Verma21cdb8c1a090
diff --git a/coworker/observability/engine_hooks.py b/coworker/observability/engine_hooks.py
index ef796952..ca044980 100644
--- a/coworker/observability/engine_hooks.py
+++ b/coworker/observability/engine_hooks.py
@@ -53,6 +53,13 @@ class TurnRecording:
self._reserved: dict[str, dict[str, Any]] = {} # summary steps written at their end (E4)
self._tokens_after_for: Optional[str] = None # a summary waiting for the next main call
self._child_usage: dict[str, bool] = {} # parent step -> its model call reported usage
+ # Conditions on now (OB-14): name -> (since ms, attrs). They outlive turns; each turn
+ # gets its own gap notice, cut at its start and end.
+ self._conditions: dict[str, tuple[int, dict[str, Any]]] = {}
+ self._turn_started_ms = self._turn_started_mono = 0
+ from . import gaps
+
+ gaps.register(self) # the recorder may be set later; conditions are kept meanwhile
@property
def enabled(self) -> bool:
@@ -97,6 +104,7 @@ class TurnRecording:
self._reply_end = None
self._waits.clear()
ev = self._base("turn_started", self.turn_id, self._turn_session)
+ self._turn_started_ms, self._turn_started_mono = ev["at_ms"], ev["mono_ms"]
ev.update({"kind": kind, "trigger": trigger, "first_message_id": first_message_id,
"attrs": copy.deepcopy(attrs), "stamp": copy.deepcopy(stamp)})
if wait_for_first_message:
@@ -124,7 +132,12 @@ class TurnRecording:
self.recorder.emit(ev)
self._tool_ends.clear()
self._flush_reply(None)
- ev = self._base("turn_ended", self.turn_id, self._turn_session)
+ end = self._base("turn_ended", self.turn_id, self._turn_session)
+ # Still on: cut at the turn's end, on the turn's own clock (its length is monotonic).
+ cut = self._turn_started_ms + (end["mono_ms"] - self._turn_started_mono)
+ for name, (since, attrs) in list(self._conditions.items()):
+ self._write_gap(name, since, cut, attrs)
+ ev = end
ev.update({"status": status, "last_message_id": self._last_mid})
return ev
@@ -135,6 +148,34 @@ class TurnRecording:
def in_turn(self) -> bool:
return self.turn_id is not None and not self._ended
+ # -- gaps (OB-14) ---------------------------------------------------------------------------
+ def condition(self, name: str, on: bool, *, at_ms: Optional[int] = None,
+ attrs: Optional[dict[str, Any]] = None) -> None:
+ """A gap condition starts (`on`) or ends. When it ends during a turn, the turn gets a
+ notice lasting from the later of its start and the turn's start, until now."""
+ at = at_ms if at_ms is not None else now_ms()[0]
+ with self._lock:
+ if on:
+ self._conditions.setdefault(name, (at, dict(attrs or {})))
+ return
+ held = self._conditions.pop(name, None)
+ if held is None or not self.in_turn:
+ return
+ since, had = held
+ self._write_gap(name, since, at, {**had, **(attrs or {})})
+
+ def _write_gap(self, name: str, since: int, until: int, attrs: dict[str, Any]) -> None:
+ """One gap notice in the current turn, whole, from its end (under the lock)."""
+ start = max(since, self._turn_started_ms)
+ if self.turn_id is None or until <= start:
+ return
+ ev = self._base("step_ended", self.turn_id, self._turn_session)
+ ev.update({"at_ms": until, "step_id": new_ulid(), "status": "ok", "message_id": None,
+ "attrs": copy.deepcopy(attrs), "content": {}, "kind": "notice", "name": name,
+ "parent_step_id": None, "call_id": None, "call_index": None, "shadow": False,
+ "after_turn": False, "started_at": start, "gap": True})
+ self._send(ev)
+
def step_for_call(self, call_id: str) -> Optional[str]:
"""The tool step recorded for a call in this turn, if any."""
with self._lock:
diff --git a/coworker/observability/export.py b/coworker/observability/export.py
index 48d3c2d5..c1ca1662 100644
--- a/coworker/observability/export.py
+++ b/coworker/observability/export.py
@@ -28,7 +28,7 @@ from typing import Any, Callable, Optional
from urllib.parse import urlparse
from . import export_spans as spans_mod
-from .export_spans import Masker, build_turn_spans, encode_otlp, resource_attrs, switches_from
+from .export_spans import Masker, _is_gap, build_turn_spans, encode_otlp, resource_attrs, switches_from
log = logging.getLogger(__name__)
@@ -376,7 +376,8 @@ class Exporter:
ready = {s["step_id"] for s in steps
if s.get("status") not in (None, "running") and s["step_id"] not in marked}
include_root = turn_id not in marked
- late_notices = [s["step_id"] for s in steps if s.get("kind") == "notice" and s["step_id"] in ready]
+ late_notices = [s["step_id"] for s in steps if s.get("kind") == "notice" and s["step_id"] in ready
+ and not _is_gap(s)] # a gap is a span of its own (1.2)
if not include_root and late_notices:
# A notice is an event on the turn's span, which has gone already.
self.store.mark_export(dest, [(n, turn_id) for n in late_notices], "skipped")
diff --git a/coworker/observability/export_spans.py b/coworker/observability/export_spans.py
index aeb45223..81d73633 100644
--- a/coworker/observability/export_spans.py
+++ b/coworker/observability/export_spans.py
@@ -456,6 +456,13 @@ def turn_end_ms(turn: dict[str, Any], steps: list[dict[str, Any]]) -> int:
return max([turn.get("started_at") or 0] + ends)
+def _is_gap(step: dict[str, Any]) -> bool:
+ """A gap notice with a duration: its own span, not an event on the turn (1.2)."""
+ from .gaps import GAP_NOTICES
+
+ return step.get("kind") == "notice" and step.get("name") in GAP_NOTICES and bool(step.get("duration_ms"))
+
+
def build_turn_spans(
turn: dict[str, Any],
steps: list[dict[str, Any]],
@@ -518,7 +525,7 @@ def build_turn_spans(
first_user_text = last_reply_text = None
for s in sorted(steps, key=lambda s: (depth(s), s.get("seq") or 0)):
- if s.get("kind") == "notice" or (only is not None and s["step_id"] not in only):
+ if (s.get("kind") == "notice" and not _is_gap(s)) or (only is not None and s["step_id"] not in only):
continue
a = s.get("attrs") or {}
attrs: dict[str, Any] = dict(common, **{
@@ -652,6 +659,11 @@ def build_turn_spans(
for k in ("size_chars", "session_prompt_hash", "block_hash"):
if a.get(k) is not None:
attrs[f"openworker.context.{k}"] = a[k]
+ elif s["kind"] == "notice": # a gap with a duration (1.2, tracing-spec §9)
+ name = f"openworker.gap.{s.get('name')}"
+ for k in ("reason", "skipped_ms"):
+ if a.get(k) is not None:
+ attrs[f"openworker.gap.{k}"] = a[k]
elif s["kind"] == "sandbox":
name = "openworker.sandbox.setup"
for k in ("provider", "trigger", "failed_stage"):
@@ -731,9 +743,9 @@ def build_turn_spans(
root_attrs["langfuse.trace.output"] = root_attrs["langfuse.observation.output"] = masker.clean(last_reply_text)
events = [{"name": f"openworker.notice.{s.get('name')}", "at": s.get("started_at") or turn["started_at"],
"attrs": {"openworker.step.id": s["step_id"]}}
- for s in sorted(steps, key=lambda s: s.get("seq") or 0) if s.get("kind") == "notice"]
+ for s in sorted(steps, key=lambda s: s.get("seq") or 0) if s.get("kind") == "notice" and not _is_gap(s)]
spans.insert(0, {
- "_mark": tid, "_notices": [s["step_id"] for s in steps if s.get("kind") == "notice"],
+ "_mark": tid, "_notices": [s["step_id"] for s in steps if s.get("kind") == "notice" and not _is_gap(s)],
"trace_id": trace, "span_id": root_id, "parent_span_id": None,
"name": "invoke_agent coworker", "kind": SPAN_INTERNAL, "start_ms": turn["started_at"],
"end_ms": end_ms, "attrs": root_attrs, "events": events,
diff --git a/coworker/observability/gaps.py b/coworker/observability/gaps.py
new file mode 100644
index 00000000..5db83da3
--- /dev/null
+++ b/coworker/observability/gaps.py
@@ -0,0 +1,112 @@
+"""Gaps in the timeline (observability OB-14, format 1.2): stretches when the machine was
+offline from its controller, or nobody was watching the session. Each is a notice with a
+duration in every turn running at the time, cut at the turn's start and end
+(phase-2 design §11.3).
+
+A condition is "on" from a moment until it ends. `nobody_watching` belongs to one session
+(its engine's recording); `controller_offline` to the whole machine, so it goes to every
+recording here, including those of engines made while it is on."""
+
+from __future__ import annotations
+
+import json
+import logging
+import threading
+import time
+import weakref
+from pathlib import Path
+from typing import Any, Optional
+
+log = logging.getLogger(__name__)
+
+GAP_NOTICES = ("controller_offline", "nobody_watching", "clock_jump")
+
+_lock = threading.Lock()
+_recordings: "weakref.WeakSet[Any]" = weakref.WeakSet()
+_machine: dict[str, tuple[int, dict[str, Any]]] = {} # name -> (since ms, attrs)
+
+
+def now_ms() -> int:
+ return int(time.time() * 1000)
+
+
+def register(recording: Any) -> None:
+ """A new engine's recording hears the machine's conditions, and any already on."""
+ with _lock:
+ _recordings.add(recording)
+ ongoing = dict(_machine)
+ for name, (since, attrs) in ongoing.items():
+ recording.condition(name, True, at_ms=since, attrs=attrs)
+
+
+def machine_condition(name: str, on: bool, *, at_ms: Optional[int] = None, attrs: Optional[dict[str, Any]] = None) -> None:
+ """A machine-wide condition starts or ends, for every recording on this machine."""
+ at = at_ms if at_ms is not None else now_ms()
+ with _lock:
+ if on:
+ if name in _machine:
+ return # already on: the first moment stands
+ _machine[name] = (at, dict(attrs or {}))
+ else:
+ if name not in _machine and not attrs:
+ return
+ since, held = _machine.pop(name, (at, {}))
+ attrs = {**held, **(attrs or {})}
+ targets = list(_recordings)
+ for rec in targets:
+ try:
+ rec.condition(name, on, at_ms=at, attrs=attrs)
+ except Exception: # noqa: BLE001 - recording never breaks the link
+ log.exception("observability: could not note %s", name)
+
+
+def machine_on(name: str) -> bool:
+ with _lock:
+ return name in _machine
+
+
+# -- the controller link (E15b-1) ----------------------------------------------------------------
+# When the link is lost, the moment is also saved in the state folder, so a restart before
+# it comes back still writes the gap into the turns that are running when it does.
+
+_LOST_FILE = "controller-link-lost.json"
+
+
+def link_lost(state: Optional[Path], reason: str) -> None:
+ at = now_ms()
+ if state is not None:
+ path = Path(state) / _LOST_FILE
+ try:
+ if not path.exists():
+ path.write_text(json.dumps({"lost_at": at, "reason": reason}), encoding="utf-8")
+ except OSError:
+ log.warning("observability: could not save when the controller link was lost")
+ machine_condition("controller_offline", True, at_ms=at, attrs={"reason": reason})
+
+
+def link_back(state: Optional[Path]) -> None:
+ saved: dict[str, Any] = {}
+ if state is not None:
+ path = Path(state) / _LOST_FILE
+ try:
+ saved = json.loads(path.read_text(encoding="utf-8"))
+ path.unlink()
+ except FileNotFoundError:
+ pass
+ except (OSError, ValueError):
+ log.warning("observability: could not read when the controller link was lost")
+ if saved and not machine_on("controller_offline"):
+ # Lost before a restart: the condition starts at the saved moment.
+ machine_condition("controller_offline", True, at_ms=int(saved.get("lost_at") or now_ms()),
+ attrs={"reason": str(saved.get("reason") or "failed")})
+ machine_condition("controller_offline", False)
+
+
+def link_reason(exc: Optional[BaseException]) -> str:
+ """`closed` (the controller closed it), `ping_timeout` or `failed`."""
+ if exc is None:
+ return "closed"
+ text = str(exc).lower()
+ if "ping" in text and ("timeout" in text or "timed out" in text):
+ return "ping_timeout"
+ return "failed"
diff --git a/coworker/observability/recorder.py b/coworker/observability/recorder.py
index 4ebcf775..9152b701 100644
--- a/coworker/observability/recorder.py
+++ b/coworker/observability/recorder.py
@@ -55,6 +55,9 @@ TURN_FIELDS = (
"incomplete", "approval_mode", "attended", "reviewer", "stamp", "first_message_id",
"last_message_id", "schema_version",
)
+# A wall clock ahead of the turn's monotonic clock by more than this: the machine slept (OB-14).
+CLOCK_JUMP_MS = 5000
+
# Format 1.2 turn fields: written only when known, so a 1.1 recording reads back unchanged.
TURN_FIELDS_12 = ("max_iterations", "woken_by")
STEP_FIELDS = (
@@ -90,6 +93,10 @@ class Turn:
woken_by: Optional[dict[str, Any]] = None
mono_start: Optional[int] = field(default=None, repr=False) # not part of the record
mono_end: Optional[int] = field(default=None, repr=False)
+ # Time the monotonic clock did not count (the machine slept; OB-14 clock_jump): added
+ # to every later moment, so step times follow the wall clock again (tracing-spec §7, 1.2).
+ shift_ms: int = field(default=0, repr=False)
+ last_ms: Optional[int] = field(default=None, repr=False) # the latest moment placed
@property
def ended_at(self) -> Optional[int]:
@@ -285,8 +292,9 @@ class Recorder:
if turn is None or turn.mono_end is not None:
return
turn.status, turn.last_message_id = ev["status"], ev.get("last_message_id")
+ self._anchor(turn.turn_id, ev["at_ms"], ev["mono_ms"]) # a sleep just before the end
turn.mono_end = ev["mono_ms"]
- turn.duration_ms = max(0, ev["mono_ms"] - turn.mono_start)
+ turn.duration_ms = max(0, ev["mono_ms"] - turn.mono_start + turn.shift_ms)
# §7.4: anything of this turn still open is closed at the turn's end.
for s in self.steps.values():
if (s.turn_id == turn.turn_id and s.duration_ms is None and s.status == "running"
@@ -316,11 +324,26 @@ class Recorder:
log.exception("recording: the Issues checks failed for a turn")
def _anchor(self, turn_id: Optional[str], at_ms: int, mono_ms: int) -> int:
- """§7.3: a step's start is its turn's start plus the monotonic time since then."""
+ """§7.3: a step's start is its turn's start plus the monotonic time since then. When
+ the wall clock has run ahead of that by more than 5 s, the machine slept: a
+ `clock_jump` notice covers the skipped time, and later moments follow the wall
+ clock again (1.2, TD2-14)."""
t = self.turns.get(turn_id) if turn_id else None
if t is None or t.mono_start is None:
return at_ms
- return t.started_at + (mono_ms - t.mono_start)
+ placed = t.started_at + (mono_ms - t.mono_start) + t.shift_ms
+ skipped = at_ms - placed
+ if skipped > CLOCK_JUMP_MS and t.mono_end is None:
+ last = t.last_ms if t.last_ms is not None else t.started_at
+ jump = Step(step_id=new_ulid(), turn_id=t.turn_id, session_id=t.session_id, kind="notice",
+ name="clock_jump", started_at=last, duration_ms=skipped, status="ok",
+ attrs={"skipped_ms": skipped})
+ self.steps[jump.step_id] = jump
+ self._write_step(jump)
+ t.shift_ms += skipped
+ placed = at_ms
+ t.last_ms = max(t.last_ms or 0, placed)
+ return placed
def _step_started(self, ev: dict[str, Any]) -> None:
if ev["step_id"] in self.steps:
@@ -360,6 +383,9 @@ class Recorder:
return
else:
s.duration_ms = max(0, ev["mono_ms"] - s.mono_start)
+ turn = self.turns.get(s.turn_id) if s.turn_id else None
+ if turn is not None:
+ turn.last_ms = max(turn.last_ms or 0, s.started_at + s.duration_ms)
s.status = ev.get("status", "ok")
if "message_id" in ev:
s.message_id = ev["message_id"]
diff --git a/coworker/observability/signals.py b/coworker/observability/signals.py
index 495bb809..8c499ef6 100644
--- a/coworker/observability/signals.py
+++ b/coworker/observability/signals.py
@@ -399,17 +399,36 @@ def c_ran_without_sandbox(turn, top, all_steps, p):
f"this turn ran {len(acts)} action(s) without it.")], ""
+def _overlap(start, end, spans):
+ """How much of [start, end) the spans cover, each moment once."""
+ total, cur = 0, start
+ for a, b in sorted(spans):
+ a, b = max(a, cur), min(b, end)
+ if b > a:
+ total += b - a
+ cur = b
+ return total
+
+
def c_waited_unwatched(turn, top, all_steps, p):
+ # 0.4: besides the time after the card reached the Inbox, the wait's overlap with the
+ # turn's `nobody_watching` gaps (OB-14) counts: a card left in the open session while
+ # the desktop was closed.
+ unwatched = [(g["started_at"], g["started_at"] + g["duration_ms"]) for g in all_steps
+ if g["kind"] == "notice" and g.get("name") == "nobody_watching" and g.get("duration_ms")]
found = []
for s in top:
a = s["attrs"]
- if s["kind"] != "wait" or a.get("kind") not in ("approval", "question", "plan") or a.get("where") != "inbox":
+ if s["kind"] != "wait" or a.get("kind") not in ("approval", "question", "plan"):
continue
- start = a.get("inbox_at") or s.get("started_at") or 0
- end = (s.get("started_at") or 0) + (s.get("duration_ms") or 0)
- if end - start >= p["min_wait_ms"]:
- found.append(issue("waited_unwatched", s, f"waited {round((end - start) / 60000)} min in the "
- f"Inbox for an answer while nobody was at the app."))
+ began = s.get("started_at") or 0
+ end = began + (s.get("duration_ms") or 0)
+ alone = end - (a.get("inbox_at") or began) if a.get("where") == "inbox" else 0
+ alone = max(alone, _overlap(began, end, unwatched))
+ if alone >= p["min_wait_ms"]:
+ where = "in the Inbox " if a.get("where") == "inbox" else ""
+ found.append(issue("waited_unwatched", s, f"waited {round(alone / 60000)} min {where}for an answer "
+ f"while nobody was at the app."))
return ("fired" if found else "fine"), found, "Nothing waited long for an answer while nobody was watching"
diff --git a/coworker/observability/signals.yaml b/coworker/observability/signals.yaml
index a67567a1..901ef175 100644
--- a/coworker/observability/signals.yaml
+++ b/coworker/observability/signals.yaml
@@ -37,8 +37,9 @@
# golden the scenarios that test it (phase-2/scenarios.md)
# source where the check was asked for
-version: "0.3"
-draft: 2026-11-02
+version: "0.4"
+draft: 2026-11-03
+# 0.4: waited_unwatched also counts the wait's overlap with nobody_watching gaps (OB-14).
# 0.3: ran_without_sandbox runs only where a sandbox is expected (found in the first live run).
# 0.2: reviewer_allowed_send (TD2-20), sandbox_failed, the mock-review wording (M2-R1 to M2-R9),
# waited_unwatched from inbox_at (V2-25). Goes with recording format 1.2.
@@ -405,18 +406,19 @@ checks:
severity: warn
applies: "always"
fires: >-
- A wait step of kind `approval`, `question` or `plan` with where = `inbox` that
- waited at least `min_wait_ms` after it reached the Inbox: wait end minus `inbox_at`
- (new; the step's start when it began in the Inbox). A card reaches the Inbox when
- the turn started unattended or when the last viewer left the session
- (`inbox.py` promote_to_inbox), so where = `inbox` is the recorded fact for "nobody
- was watching". The wait's `attended` is not used: it is the Unattended toggle, not
- whether a desktop is connected (V2-25). Q27.
+ A wait step of kind `approval`, `question` or `plan` that waited at least
+ `min_wait_ms` while nobody was watching. That time is the longer of two: for
+ where = `inbox`, wait end minus `inbox_at` (the step's start when it began in the
+ Inbox); and the wait's overlap with the turn's `nobody_watching` notices (1.2,
+ OB-14), which also covers a card left in the open session while the desktop was
+ closed. A card reaches the Inbox when the turn started unattended or when the last
+ viewer left the session (`inbox.py` promote_to_inbox). The wait's `attended` is not
+ used: it is the Unattended toggle, not whether a desktop is connected (V2-25). Q27.
points_at: "the wait step"
params:
min_wait_ms: 900000
- says: "{name} waited {wait} in the Inbox for an answer while nobody was at the app."
+ says: "{name} waited {wait} in the Inbox for an answer while nobody was at the app. (Without \"in the Inbox\" when the card stayed in the session.)"
fine: "Nothing waited long for an answer while nobody was watching"
- needs: [wait.inbox_at]
+ needs: [wait.inbox_at, notice.nobody_watching]
golden: [S2-17, S2-17b, S2-17c]
source: "across-features.md OB-14, Q27; validation V2-25"
diff --git a/coworker/observability/store.py b/coworker/observability/store.py
index 412a2348..63c9f40c 100644
--- a/coworker/observability/store.py
+++ b/coworker/observability/store.py
@@ -198,6 +198,7 @@ class RecordingStore:
self._pending_issues: dict[str, tuple[str, int]] = {} # turn_id -> (json, seen)
self._pending_steps: dict[str, dict[str, Any]] = {}
self._pending_lock = threading.Lock()
+ self._owner = False # this process called close_open_turns: it owns the data folder
self._closed = False
self._last_sweep = 0.0
self._background = background
@@ -274,6 +275,26 @@ class RecordingStore:
self.flush()
self._closed = True
self._conn.close()
+ if self._owner:
+ # A clean stop: a turn found open at the next start was cut by the app being
+ # stopped, not by a crash (recording_gap cause, OB-14).
+ self._marker("running").unlink(missing_ok=True)
+ try:
+ self._marker("stopped").write_text(str(int(time.time() * 1000)), encoding="utf-8")
+ except OSError:
+ pass
+
+ def _marker(self, name: str) -> Path:
+ return self.path.with_name(self.path.name + f".{name}")
+
+ def _last_stop(self) -> str:
+ """How the previous run of the app ended: `stopped` (it closed this store), `restart`
+ (it was running and never closed it: a crash or a kill) or `unknown` (no word)."""
+ if self._marker("stopped").exists():
+ return "stopped"
+ if self._marker("running").exists():
+ return "restart"
+ return "unknown"
# -- exchange pieces (TD-17) --------------------------------------------------------------
def put_piece(self, data: bytes) -> str:
@@ -306,6 +327,13 @@ class RecordingStore:
with self._lock:
self.flush()
+ cause, until = self._last_stop(), int(time.time() * 1000)
+ self._owner = True
+ self._marker("stopped").unlink(missing_ok=True)
+ try:
+ self._marker("running").write_text(str(until), encoding="utf-8")
+ except OSError:
+ pass
open_turns = self._conn.execute(
"SELECT turn_id, session_id, started_at FROM turns WHERE status = 'running' OR status IS NULL"
).fetchall()
@@ -331,10 +359,12 @@ class RecordingStore:
if s["started_at"] is not None])
gap = {"step_id": new_ulid(), "kind": "notice", "name": "recording_gap", "started_at": last,
"duration_ms": 0, "call_index": None, "call_id": None, "shadow": 0}
+ # 1.2: why the turn was found open, and the start that found it (OB-14).
self._conn.execute(
"INSERT INTO steps (step_id, turn_id, session_id, kind, name, started_at, duration_ms, "
"status, shadow, after_turn, attrs, content) VALUES (?, ?, ?, 'notice', 'recording_gap', ?, 0, "
- "'ok', 0, 0, '{}', '{}')", (gap["step_id"], tid, t["session_id"], last))
+ "'ok', 0, 0, ?, '{}')",
+ (gap["step_id"], tid, t["session_id"], last, json.dumps({"cause": cause, "until": until})))
ordered = sorted([dict(s) for s in steps] + [gap], key=lambda s: (
s["started_at"] or 0, 0 if s["duration_ms"] == 0 else 1, KIND_ORDER.get(s["kind"], 9),
s["call_index"] or 0, s["name"] or "", s["call_id"] or "", bool(s["shadow"])))
diff --git a/coworker/remote/joiner.py b/coworker/remote/joiner.py
index 6cb9d417..47f074ec 100644
--- a/coworker/remote/joiner.py
+++ b/coworker/remote/joiner.py
@@ -475,6 +475,8 @@ async def run_joined(
"""
import websockets
+ from ..observability import gaps
+
identity = load_or_create(state)
if app is None:
app = _build_local_app(state)
@@ -498,6 +500,8 @@ async def run_joined(
welcome = await _handshake(ws, identity, name, token)
token = None # single-use by design; reconnects sign the challenge
backoff = _BACKOFF_START
+ # The link is back: running turns get the offline stretch (OB-14).
+ gaps.link_back(state)
log(
f"[openworker] joined {controller} as "
f"'{welcome.get('name')}' (machine {welcome.get('machine_id')})"
@@ -515,12 +519,15 @@ async def run_joined(
ws, dispatcher, app, identity, exporter=exporter, state=state
)
log("[openworker] controller closed the connection")
+ gaps.link_lost(state, "closed")
except JoinRejected as exc:
if exc.reason in _FATAL_REJECTS:
raise
log(f"[openworker] rejected ({exc.reason}); retrying")
+ gaps.link_lost(state, "failed")
except (OSError, websockets.WebSocketException) as exc:
log(f"[openworker] connection failed: {exc}")
+ gaps.link_lost(state, gaps.link_reason(exc))
if once:
return
log(f"[openworker] reconnecting in {backoff:.0f}s")
diff --git a/coworker/server/manager.py b/coworker/server/manager.py
index d010cd3b..923b40c3 100644
--- a/coworker/server/manager.py
+++ b/coworker/server/manager.py
@@ -1056,6 +1056,8 @@ class SessionManager:
engine.compaction_state = CompactionState.from_dict(record.compaction)
engine.compaction_settings = self.compaction_settings
self._engines[session_id] = engine
+ if not self._session_clients.get(session_id):
+ self._note_watching(session_id, False) # built with no window open (a schedule, a wake)
self.sync_worker_mode(session_id, engine) # §11.6: approvals follow the lead
if is_new_session:
self._emit_session_created(session_id, agent_name)
@@ -6274,7 +6276,10 @@ class SessionManager:
self.unregister_event_client(cb)
def register_session_client(self, session_id: str, send_cb: Any) -> None:
+ first = not self._session_clients.get(session_id)
self._session_clients.setdefault(session_id, set()).add(send_cb)
+ if first:
+ self._note_watching(session_id, True)
def unregister_session_client(self, session_id: str, send_cb: Any) -> None:
clients = self._session_clients.get(session_id)
@@ -6282,6 +6287,19 @@ class SessionManager:
clients.discard(send_cb)
if not clients:
self._session_clients.pop(session_id, None)
+ self._note_watching(session_id, False)
+
+ def _note_watching(self, session_id: str, watching: bool) -> None:
+ """The session's viewers went from none to one or from one to none (OB-14): while
+ none, its turns get a `nobody_watching` gap. A local window and a desktop reaching
+ this machine through the controller open the same socket, so both count."""
+ recording = getattr(self._engines.get(session_id), "_obs", None)
+ condition = getattr(recording, "condition", None)
+ if condition is not None:
+ try:
+ condition("nobody_watching", not watching)
+ except Exception: # noqa: BLE001 - a recording never breaks a socket
+ logger.exception("observability: could not note who is watching %s", session_id)
# -- session actor (spec §Fleet under the org) --------------------------------
def note_session_actor(self, session_id: str, actor: str) -> None:
diff --git a/surfaces/gui/src/components/StepsPanel.test.tsx b/surfaces/gui/src/components/StepsPanel.test.tsx
index 20848b4e..9848b047 100644
--- a/surfaces/gui/src/components/StepsPanel.test.tsx
+++ b/surfaces/gui/src/components/StepsPanel.test.tsx
@@ -317,6 +317,49 @@ describe("Sandbox setup (OB-13)", () => {
});
});
+describe("Gaps (OB-14)", () => {
+ const gapTurn = { ...turn, duration_ms: 600_000, buckets: { ...turn.buckets, total: 600_000 } };
+ const gaps = [
+ step({ step_id: "m", kind: "model", name: "model", duration_ms: 5000, attrs: { purpose: "main" } }),
+ step({ step_id: "z", kind: "notice", name: "clock_jump", started_at: 6000, duration_ms: 23_000, attrs: { skipped_ms: 23_000 } }),
+ step({ step_id: "o", kind: "notice", name: "controller_offline", started_at: 30_000, duration_ms: 420_000, attrs: { reason: "ping_timeout" } }),
+ step({ step_id: "n", kind: "notice", name: "nobody_watching", started_at: 40_000, duration_ms: 300_000, attrs: {} }),
+ step({ step_id: "r", kind: "notice", name: "recording_gap", started_at: 500_000, duration_ms: 0, attrs: { cause: "restart", until: 740_000 } }),
+ ];
+
+ it("draws each gap as a band with its times, and totals offline and unwatched time", async () => {
+ vi.mocked(getRecordedTurns).mockResolvedValue([gapTurn] as any);
+ vi.mocked(getTurnSteps).mockResolvedValue(gaps as any);
+ render(<StepsPanel sessionId="s1" onClose={() => {}} />);
+ await waitFor(() => expect(screen.getAllByTestId("steps-row")).toHaveLength(5));
+ const rows = screen.getAllByTestId("steps-row");
+ const text = rows.map((r) => r.textContent || "");
+ expect(text[1]).toContain("Machine asleep");
+ expect(text[1]).toContain("+23s");
+ expect(text[2]).toContain("Machine offline");
+ expect(text[2]).toMatch(/\d{1,2}:\d{2}.*–.*\d{1,2}:\d{2}/); // from–until, on the wall clock
+ expect(text[3]).toContain("Nobody watching");
+ expect(text[4]).toContain("App restarted");
+ expect(text[4]).toContain("4m not recorded");
+ expect(rows[2].className).toContain("k-gap");
+ expect(rows[4].className).not.toContain("k-gap"); // a point, not a band
+ const tiles = screen.getByTestId("steps-tiles").textContent || "";
+ expect(tiles).toContain("Machine offline");
+ expect(tiles).toContain("Nobody watching");
+ });
+
+ it("says why the machine was offline in the details", async () => {
+ vi.mocked(getRecordedTurns).mockResolvedValue([gapTurn] as any);
+ vi.mocked(getTurnSteps).mockResolvedValue(gaps as any);
+ vi.mocked(getStepDetail).mockResolvedValue(gaps[2] as any);
+ render(<StepsPanel sessionId="s1" onClose={() => {}} />);
+ await waitFor(() => expect(screen.getAllByTestId("steps-row")).toHaveLength(5));
+ fireEvent.click(screen.getAllByTestId("steps-row")[2]);
+ await waitFor(() => expect(screen.getByTestId("steps-details").textContent).toContain("the controller stopped answering"));
+ expect(screen.getByTestId("steps-details").textContent).toContain("7m");
+ });
+});
+
describe("formatDuration", () => {
it("reads like the mock", () => {
expect(formatDuration(100)).toBe("0.1s");
diff --git a/surfaces/gui/src/components/StepsPanel.tsx b/surfaces/gui/src/components/StepsPanel.tsx
index fe007518..238ee0d5 100644
--- a/surfaces/gui/src/components/StepsPanel.tsx
+++ b/surfaces/gui/src/components/StepsPanel.tsx
@@ -129,6 +129,7 @@ export function stepLabel(step: RecordedStep, t: T): string {
return t("steps.row.sandbox_setup");
case "notice":
if (step.name === "sandbox_restarted" && a.how === "regrant") return t("steps.notice.sandbox_regranted");
+ if (step.name === "recording_gap" && (a.cause === "restart" || a.cause === "stopped")) return t(`steps.notice.recording_gap_${a.cause}`);
return t(`steps.notice.${step.name}`, { defaultValue: step.name });
default:
return step.name;
@@ -167,6 +168,12 @@ export function stepFlags(step: RecordedStep, t: T): Flag[] {
flags.push({ cls: "err", text: t("steps.flag.sandbox_failed", { stage: t(`steps.sbx_stage.${stage}`, { defaultValue: stage }).toLowerCase() }) });
} else if (step.kind === "notice" && step.name === "sandbox_restarted") {
if (a.reason) flags.push({ cls: "info", text: t(`steps.flag.restart_${a.reason}`, { defaultValue: a.reason }) });
+ } else if (step.kind === "notice" && (step.name === "controller_offline" || step.name === "nobody_watching")) {
+ if (step.duration_ms) flags.push({ cls: "info", text: `${clockTime(step.started_at)}–${clockTime(step.started_at + step.duration_ms)}` });
+ } else if (step.kind === "notice" && step.name === "clock_jump") {
+ flags.push({ cls: "info", text: t("steps.flag.gap_skipped", { t: formatDuration(a.skipped_ms ?? step.duration_ms) }) });
+ } else if (step.kind === "notice" && step.name === "recording_gap" && typeof a.until === "number") {
+ flags.push({ cls: "warn", text: t("steps.flag.not_recorded", { t: formatDuration(Math.max(0, a.until - step.started_at)) }) });
} else if (["error", "denied", "interrupted", "skipped", "unknown"].includes(step.status)) {
flags.push({ cls: "err", text: t(`steps.status.${step.status}`) });
}
@@ -198,12 +205,31 @@ const KIND_CLASS: Record<string, string> = {
sandbox: "sbx",
};
-/** The class of a row: a sandbox restart notice is drawn in the sandbox colour (mock ob13). */
+const GAPS = ["controller_offline", "nobody_watching", "clock_jump"];
+
+/** A gap notice with a duration (OB-14): drawn as a dashed band, not a point. */
+function isGap(step: RecordedStep): boolean {
+ return step.kind === "notice" && GAPS.includes(step.name) && !!step.duration_ms;
+}
+
+/** The class of a row: a sandbox restart notice is drawn in the sandbox colour (mock ob13),
+ * a gap as a dashed band (mock ob14). */
function kindClass(step: RecordedStep): string {
if (step.kind === "notice" && step.name === "sandbox_restarted") return "sbx";
+ if (isGap(step)) return "gap";
return KIND_CLASS[step.kind] || "ctx";
}
+/** A moment as the clock on the wall shows it, "14:02". */
+function clockTime(ms: number): string {
+ return new Date(ms).toLocaleTimeString([], { hour: "2-digit", minute: "2-digit" });
+}
+
+/** How long the turn spent with the machine offline, or with nobody watching (mock ob14). */
+function gapTotal(steps: RecordedStep[], name: string): number {
+ return steps.filter((s) => s.kind === "notice" && s.name === name).reduce((n, s) => n + (s.duration_ms || 0), 0);
+}
+
type SandboxStage = { name: string; started_at: number; duration_ms: number | null; status: string; uncertain?: boolean };
/** A sandbox setup step's stages as rows under it, in the order run (OB-13). An image pull
@@ -379,6 +405,12 @@ export function StepsPanel({
<Tile key={b} value={formatDuration(turn.buckets[b])} label={t(`steps.bucket.${b}`)} />
))}
{turn.buckets.other_shown && <Tile value={formatDuration(turn.buckets.other)} label={t("steps.bucket.other")} />}
+ {gapTotal(steps, "controller_offline") > 0 && (
+ <Tile value={formatDuration(gapTotal(steps, "controller_offline"))} label={t("steps.gap.offline_total")} />
+ )}
+ {gapTotal(steps, "nobody_watching") > 0 && (
+ <Tile value={formatDuration(gapTotal(steps, "nobody_watching"))} label={t("steps.gap.nobody_total")} />
+ )}
</div>
{steps.some((s) => s.status === "unknown") && (
<p className="steps-line" data-testid="steps-gap">
@@ -871,6 +903,16 @@ function Inspector({ d, turn, steps, sessionId, t }: { d: any; turn: RecordedTur
[t("steps.sbx.reason"), t(`steps.sbx.reason_${a.reason}`, { defaultValue: a.reason || "—" })],
[t("steps.sbx.how"), t(`steps.sbx.how_${a.how}`, { defaultValue: a.how || "—" })]);
}
+ if (isGap(d) || (d.kind === "notice" && d.name === "recording_gap")) {
+ const until = d.name === "recording_gap" ? a.until : d.started_at + (d.duration_ms || 0);
+ rows.splice(0, rows.length,
+ [t("steps.gap.from"), clockTime(d.started_at)],
+ [t("steps.gap.until"), typeof until === "number" ? clockTime(until) : "—"]);
+ if (d.name === "clock_jump") rows.push([t("steps.gap.skipped"), formatDuration(a.skipped_ms ?? d.duration_ms)]);
+ else rows.push([t("steps.detail.duration"), formatDuration(typeof until === "number" ? until - d.started_at : d.duration_ms)]);
+ if (a.reason) rows.push([t("steps.gap.why"), t(`steps.gap.reason_${a.reason}`, { defaultValue: a.reason })]);
+ if (a.cause) rows.push([t("steps.gap.why"), t(`steps.gap.cause_${a.cause}`, { defaultValue: a.cause })]);
+ }
const sbxStages: SandboxStage[] = d.kind === "sandbox" && Array.isArray(a.stages) ? a.stages : [];
const changed = d.kind === "notice" && d.name === "stamp_changed" ? stampChanges(a, t) : [];
const showsContext = d.kind === "context" || (d.kind === "notice" && d.name === "context_changed");
@@ -895,6 +937,7 @@ function Inspector({ d, turn, steps, sessionId, t }: { d: any; turn: RecordedTur
] as [string, React.ReactNode])} />
</>
)}
+ {d.kind === "notice" && d.name === "clock_jump" && <p className="steps-line">{t("steps.gap.asleep_note")}</p>}
{d.kind === "sandbox" && a.error?.text && (
<>
<Sec label={t("steps.sbx.error_text")} />
diff --git a/surfaces/gui/src/locales/en.json b/surfaces/gui/src/locales/en.json
index 986cd766..92f8e017 100644
--- a/surfaces/gui/src/locales/en.json
+++ b/surfaces/gui/src/locales/en.json
@@ -3037,6 +3037,21 @@
}
},
"steps": {
+ "gap": {
+ "offline_total": "Machine offline",
+ "nobody_total": "Nobody watching",
+ "from": "From",
+ "until": "Until",
+ "why": "Why",
+ "reason_closed": "the controller closed the link",
+ "reason_failed": "the link to the controller failed",
+ "reason_ping_timeout": "the controller stopped answering",
+ "cause_restart": "the app ended without closing (a crash or a kill) and started again",
+ "cause_stopped": "the app was closed while this turn ran",
+ "cause_unknown": "not known",
+ "skipped": "Time the machine slept",
+ "asleep_note": "Steps after this row are placed by the wall clock again."
+ },
"sbx_stage": {
"preparing": "Preparing",
"download_image": "Download image",
@@ -3266,7 +3281,9 @@
"auto_approved": "auto-approved",
"sandbox_failed": "failed at {{stage}}",
"restart_folders_changed": "folders",
- "restart_runner_restarted": "runner came back"
+ "restart_runner_restarted": "runner came back",
+ "gap_skipped": "+{{t}}",
+ "not_recorded": "{{t}} not recorded"
},
"purpose": {
"main": "main",
@@ -3313,7 +3330,12 @@
"summary_failed": "Summary failed",
"recording_gap": "Recording gap",
"sandbox_restarted": "Sandbox restarted",
- "sandbox_regranted": "Sandbox folders updated"
+ "sandbox_regranted": "Sandbox folders updated",
+ "controller_offline": "Machine offline",
+ "nobody_watching": "Nobody watching",
+ "clock_jump": "Machine asleep",
+ "recording_gap_restart": "App restarted",
+ "recording_gap_stopped": "App was closed"
},
"status": {
"ok": "ok",
diff --git a/surfaces/gui/src/locales/zh.json b/surfaces/gui/src/locales/zh.json
index a078aa21..d9257def 100644
--- a/surfaces/gui/src/locales/zh.json
+++ b/surfaces/gui/src/locales/zh.json
@@ -2993,6 +2993,21 @@
}
},
"steps": {
+ "gap": {
+ "offline_total": "机器离线",
+ "nobody_total": "无人查看",
+ "from": "开始",
+ "until": "结束",
+ "why": "原因",
+ "reason_closed": "控制端关闭了连接",
+ "reason_failed": "与控制端的连接失败",
+ "reason_ping_timeout": "控制端不再响应",
+ "cause_restart": "应用未正常关闭(崩溃或被终止)后再次启动",
+ "cause_stopped": "本回合运行时应用被关闭",
+ "cause_unknown": "未知",
+ "skipped": "机器休眠的时间",
+ "asleep_note": "此行之后的步骤重新按系统时钟排列。"
+ },
"sbx_stage": {
"preparing": "准备",
"download_image": "下载镜像",
@@ -3217,7 +3232,9 @@
"auto_approved": "已自动批准",
"sandbox_failed": "在{{stage}}失败",
"restart_folders_changed": "文件夹",
- "restart_runner_restarted": "运行器已重启"
+ "restart_runner_restarted": "运行器已重启",
+ "gap_skipped": "+{{t}}",
+ "not_recorded": "{{t}} 未记录"
},
"purpose": {
"main": "主要",
@@ -3264,7 +3281,12 @@
"summary_failed": "摘要失败",
"recording_gap": "记录缺口",
"sandbox_restarted": "沙箱已重启",
- "sandbox_regranted": "沙箱文件夹已更新"
+ "sandbox_regranted": "沙箱文件夹已更新",
+ "controller_offline": "机器离线",
+ "nobody_watching": "无人查看",
+ "clock_jump": "机器休眠",
+ "recording_gap_restart": "应用已重启",
+ "recording_gap_stopped": "应用已关闭"
},
"status": {
"ok": "正常",
diff --git a/surfaces/gui/src/styles.css b/surfaces/gui/src/styles.css
index 077738c3..65bef3ba 100644
--- a/surfaces/gui/src/styles.css
+++ b/surfaces/gui/src/styles.css
@@ -2487,6 +2487,8 @@ html[data-theme="dark"] { --bar-model:#4c8dff; --bar-tool:#3fb8ad; --bar-wait:#3
.k-ctx .steps-kind, .steps-kind.k-ctx, .k-ctx .steps-bar { background:var(--bar-ctx); }
.k-sbx .steps-kind, .steps-kind.k-sbx, .k-sbx .steps-bar { background:var(--bar-sbx); }
.steps-stage { cursor:default; }
+.k-gap .steps-kind, .steps-kind.k-gap { background:transparent; outline:1px dashed var(--faint); }
+.k-gap .steps-bar { background:repeating-linear-gradient(90deg, var(--line-strong) 0 2px, transparent 2px 5px); border:1px dashed var(--faint); top:1px; height:10px; }
.steps-stage .steps-bar { opacity:.75; }
.k-wait .steps-kind, .steps-kind.k-wait { background:var(--bar-wait); outline:1px dashed var(--faint); }
.steps-track { position:relative; height:12px; background:repeating-linear-gradient(90deg, transparent 0 24.6%, var(--line) 24.6% 25%); }
diff --git a/tests/corpora/observability/golden-phase2/README.md b/tests/corpora/observability/golden-phase2/README.md
index 499de2fc..02bf4c44 100644
--- a/tests/corpora/observability/golden-phase2/README.md
+++ b/tests/corpora/observability/golden-phase2/README.md
@@ -1,6 +1,6 @@
# Golden test, Phase 2
-*Written 2026-11-02 with [recording-format.md](../../recording-format.md) 1.2 and [signals.yaml](../../signals.yaml) 0.3. Made by `phase-2/lab/make_golden2.py`, checked by `phase-2/lab/check_golden2.py`.*
+*Written 2026-11-02 with [recording-format.md](../../recording-format.md) 1.2 and [signals.yaml](../../signals.yaml) 0.3; rebuilt 2026-11-03 for signals 0.4 (S2-17d added). Made by `phase-2/lab/make_golden2.py`, checked by `phase-2/lab/check_golden2.py`.*
## What it is
@@ -14,7 +14,7 @@
| `vectors/hashes.json` | `args_hash` cases, each with its value, its RFC 8785 canonical JSON and its 6-hex hash; plan item hashes |
| `vectors/masking.json` | Rule R4 cases: secrets masked (patterns and stored values), and the 8 KB cut on a character boundary |
-There are **38 scenarios**, S2-01 to S2-19 with their clean twins. Each is described in words in [phase-2/scenarios.md](../../phase-2/scenarios.md).
+There are **39 scenarios**, S2-01 to S2-19 with their clean twins. Each is described in words in [phase-2/scenarios.md](../../phase-2/scenarios.md).
## Built, not captured
diff --git a/tests/corpora/observability/golden-phase2/S2-01/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-01/expected-issues.json
index 1969fdfc..359a9f69 100644
--- a/tests/corpora/observability/golden-phase2/S2-01/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-01/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.3",
+ "signals_version": "0.4",
"warn": 1,
"fired": {
"repeated_step": [
diff --git a/tests/corpora/observability/golden-phase2/S2-01c/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-01c/expected-issues.json
index 49ff0804..bc994a3a 100644
--- a/tests/corpora/observability/golden-phase2/S2-01c/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-01c/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.3",
+ "signals_version": "0.4",
"warn": 0,
"fired": {},
"fine": [
diff --git a/tests/corpora/observability/golden-phase2/S2-02/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-02/expected-issues.json
index fe56fae4..579901cb 100644
--- a/tests/corpora/observability/golden-phase2/S2-02/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-02/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.3",
+ "signals_version": "0.4",
"warn": 1,
"fired": {
"off_plan": [
diff --git a/tests/corpora/observability/golden-phase2/S2-02b/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-02b/expected-issues.json
index 52a7068a..739d59d6 100644
--- a/tests/corpora/observability/golden-phase2/S2-02b/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-02b/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.3",
+ "signals_version": "0.4",
"warn": 1,
"fired": {
"off_plan": [
diff --git a/tests/corpora/observability/golden-phase2/S2-02c/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-02c/expected-issues.json
index 11d75035..5061dd2d 100644
--- a/tests/corpora/observability/golden-phase2/S2-02c/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-02c/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.3",
+ "signals_version": "0.4",
"warn": 0,
"fired": {},
"fine": [
diff --git a/tests/corpora/observability/golden-phase2/S2-02n/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-02n/expected-issues.json
index 49ff0804..bc994a3a 100644
--- a/tests/corpora/observability/golden-phase2/S2-02n/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-02n/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.3",
+ "signals_version": "0.4",
"warn": 0,
"fired": {},
"fine": [
diff --git a/tests/corpora/observability/golden-phase2/S2-03/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-03/expected-issues.json
index 058d86ad..53a46fa6 100644
--- a/tests/corpora/observability/golden-phase2/S2-03/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-03/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.3",
+ "signals_version": "0.4",
"warn": 1,
"fired": {
"refused": [
diff --git a/tests/corpora/observability/golden-phase2/S2-03b/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-03b/expected-issues.json
index 49ff0804..bc994a3a 100644
--- a/tests/corpora/observability/golden-phase2/S2-03b/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-03b/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.3",
+ "signals_version": "0.4",
"warn": 0,
"fired": {},
"fine": [
diff --git a/tests/corpora/observability/golden-phase2/S2-04/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-04/expected-issues.json
index fa30041d..fdb534a5 100644
--- a/tests/corpora/observability/golden-phase2/S2-04/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-04/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.3",
+ "signals_version": "0.4",
"warn": 0,
"fired": {
"overridden": [
diff --git a/tests/corpora/observability/golden-phase2/S2-04n/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-04n/expected-issues.json
index 49ff0804..bc994a3a 100644
--- a/tests/corpora/observability/golden-phase2/S2-04n/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-04n/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.3",
+ "signals_version": "0.4",
"warn": 0,
"fired": {},
"fine": [
diff --git a/tests/corpora/observability/golden-phase2/S2-05/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-05/expected-issues.json
index 25ba330d..da408400 100644
--- a/tests/corpora/observability/golden-phase2/S2-05/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-05/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.3",
+ "signals_version": "0.4",
"warn": 1,
"fired": {
"reviewer_could_not_run": [
diff --git a/tests/corpora/observability/golden-phase2/S2-05c/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-05c/expected-issues.json
index b142c8d2..4042fe25 100644
--- a/tests/corpora/observability/golden-phase2/S2-05c/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-05c/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.3",
+ "signals_version": "0.4",
"warn": 0,
"fired": {},
"fine": [
diff --git a/tests/corpora/observability/golden-phase2/S2-06/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-06/expected-issues.json
index 0bc057dc..e525f777 100644
--- a/tests/corpora/observability/golden-phase2/S2-06/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-06/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.3",
+ "signals_version": "0.4",
"warn": 1,
"fired": {
"reviewer_paused": [
diff --git a/tests/corpora/observability/golden-phase2/S2-07/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-07/expected-issues.json
index 64d81e9e..2fe645b2 100644
--- a/tests/corpora/observability/golden-phase2/S2-07/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-07/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.3",
+ "signals_version": "0.4",
"warn": 0,
"fired": {
"shadow_disagreed": [
diff --git a/tests/corpora/observability/golden-phase2/S2-08/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-08/expected-issues.json
index 271fef7b..513b11c2 100644
--- a/tests/corpora/observability/golden-phase2/S2-08/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-08/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.3",
+ "signals_version": "0.4",
"warn": 1,
"fired": {
"tool_failed": [
diff --git a/tests/corpora/observability/golden-phase2/S2-08b/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-08b/expected-issues.json
index 271fef7b..513b11c2 100644
--- a/tests/corpora/observability/golden-phase2/S2-08b/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-08b/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.3",
+ "signals_version": "0.4",
"warn": 1,
"fired": {
"tool_failed": [
diff --git a/tests/corpora/observability/golden-phase2/S2-08c/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-08c/expected-issues.json
index 8d3822b3..2598d4db 100644
--- a/tests/corpora/observability/golden-phase2/S2-08c/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-08c/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.3",
+ "signals_version": "0.4",
"warn": 1,
"fired": {
"made_up_tool": [
diff --git a/tests/corpora/observability/golden-phase2/S2-09/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-09/expected-issues.json
index 7c7c4519..7e9020eb 100644
--- a/tests/corpora/observability/golden-phase2/S2-09/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-09/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.3",
+ "signals_version": "0.4",
"warn": 1,
"fired": {
"model_failed": [
diff --git a/tests/corpora/observability/golden-phase2/S2-09c/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-09c/expected-issues.json
index 49ff0804..bc994a3a 100644
--- a/tests/corpora/observability/golden-phase2/S2-09c/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-09c/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.3",
+ "signals_version": "0.4",
"warn": 0,
"fired": {},
"fine": [
diff --git a/tests/corpora/observability/golden-phase2/S2-10/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-10/expected-issues.json
index bf5dc859..2a7940af 100644
--- a/tests/corpora/observability/golden-phase2/S2-10/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-10/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.3",
+ "signals_version": "0.4",
"warn": 1,
"fired": {
"retried": [
diff --git a/tests/corpora/observability/golden-phase2/S2-10b/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-10b/expected-issues.json
index 49ff0804..bc994a3a 100644
--- a/tests/corpora/observability/golden-phase2/S2-10b/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-10b/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.3",
+ "signals_version": "0.4",
"warn": 0,
"fired": {},
"fine": [
diff --git a/tests/corpora/observability/golden-phase2/S2-11/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-11/expected-issues.json
index 0436ff42..0ed0fd33 100644
--- a/tests/corpora/observability/golden-phase2/S2-11/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-11/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.3",
+ "signals_version": "0.4",
"warn": 1,
"fired": {
"cut_off": [
diff --git a/tests/corpora/observability/golden-phase2/S2-12/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-12/expected-issues.json
index 0ccfcaba..cf9ac100 100644
--- a/tests/corpora/observability/golden-phase2/S2-12/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-12/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.3",
+ "signals_version": "0.4",
"warn": 1,
"fired": {
"step_limit": [
diff --git a/tests/corpora/observability/golden-phase2/S2-13/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-13/expected-issues.json
index 60324950..934ff556 100644
--- a/tests/corpora/observability/golden-phase2/S2-13/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-13/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.3",
+ "signals_version": "0.4",
"warn": 2,
"fired": {
"made_up_tool": [
diff --git a/tests/corpora/observability/golden-phase2/S2-14/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-14/expected-issues.json
index 2407cc96..dde419fa 100644
--- a/tests/corpora/observability/golden-phase2/S2-14/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-14/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.3",
+ "signals_version": "0.4",
"warn": 1,
"fired": {
"context_full": [
diff --git a/tests/corpora/observability/golden-phase2/S2-14b/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-14b/expected-issues.json
index 49ff0804..bc994a3a 100644
--- a/tests/corpora/observability/golden-phase2/S2-14b/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-14b/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.3",
+ "signals_version": "0.4",
"warn": 0,
"fired": {},
"fine": [
diff --git a/tests/corpora/observability/golden-phase2/S2-14d/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-14d/expected-issues.json
index d0ab7d0a..8fcfa96b 100644
--- a/tests/corpora/observability/golden-phase2/S2-14d/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-14d/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.3",
+ "signals_version": "0.4",
"warn": 1,
"fired": {
"context_full": [
diff --git a/tests/corpora/observability/golden-phase2/S2-15/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-15/expected-issues.json
index b1634142..140ad354 100644
--- a/tests/corpora/observability/golden-phase2/S2-15/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-15/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.3",
+ "signals_version": "0.4",
"warn": 1,
"fired": {
"blocked_by_sandbox": [
diff --git a/tests/corpora/observability/golden-phase2/S2-15c/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-15c/expected-issues.json
index b4d8c189..bf937979 100644
--- a/tests/corpora/observability/golden-phase2/S2-15c/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-15c/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.3",
+ "signals_version": "0.4",
"warn": 0,
"fired": {},
"fine": [
diff --git a/tests/corpora/observability/golden-phase2/S2-16/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-16/expected-issues.json
index cea4e470..03d5f06d 100644
--- a/tests/corpora/observability/golden-phase2/S2-16/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-16/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.3",
+ "signals_version": "0.4",
"warn": 1,
"fired": {
"ran_without_sandbox": [
diff --git a/tests/corpora/observability/golden-phase2/S2-16c/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-16c/expected-issues.json
index c6e0bfe7..9665d26d 100644
--- a/tests/corpora/observability/golden-phase2/S2-16c/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-16c/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.3",
+ "signals_version": "0.4",
"warn": 0,
"fired": {},
"fine": [
diff --git a/tests/corpora/observability/golden-phase2/S2-17/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-17/expected-issues.json
index a3eb80ff..e41d83a0 100644
--- a/tests/corpora/observability/golden-phase2/S2-17/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-17/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.3",
+ "signals_version": "0.4",
"warn": 1,
"fired": {
"waited_unwatched": [
diff --git a/tests/corpora/observability/golden-phase2/S2-17b/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-17b/expected-issues.json
index 49ff0804..bc994a3a 100644
--- a/tests/corpora/observability/golden-phase2/S2-17b/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-17b/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.3",
+ "signals_version": "0.4",
"warn": 0,
"fired": {},
"fine": [
diff --git a/tests/corpora/observability/golden-phase2/S2-17c/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-17c/expected-issues.json
index 49ff0804..bc994a3a 100644
--- a/tests/corpora/observability/golden-phase2/S2-17c/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-17c/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.3",
+ "signals_version": "0.4",
"warn": 0,
"fired": {},
"fine": [
diff --git a/tests/corpora/observability/golden-phase2/S2-17d/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-17d/expected-issues.json
new file mode 100644
index 00000000..e41d83a0
--- /dev/null
+++ b/tests/corpora/observability/golden-phase2/S2-17d/expected-issues.json
@@ -0,0 +1,31 @@
+{
+ "signals_version": "0.4",
+ "warn": 1,
+ "fired": {
+ "waited_unwatched": [
+ 2
+ ]
+ },
+ "fine": [
+ "context_full",
+ "cut_off",
+ "made_up_tool",
+ "model_failed",
+ "refused",
+ "repeated_step",
+ "retried",
+ "step_limit",
+ "tool_failed"
+ ],
+ "not_checked": [
+ "blocked_by_sandbox",
+ "off_plan",
+ "overridden",
+ "ran_without_sandbox",
+ "reviewer_allowed_send",
+ "reviewer_could_not_run",
+ "reviewer_paused",
+ "sandbox_failed",
+ "shadow_disagreed"
+ ]
+}
diff --git a/tests/corpora/observability/golden-phase2/S2-17d/recording.json b/tests/corpora/observability/golden-phase2/S2-17d/recording.json
new file mode 100644
index 00000000..2cc4eae2
--- /dev/null
+++ b/tests/corpora/observability/golden-phase2/S2-17d/recording.json
@@ -0,0 +1,190 @@
+{
+ "turns": [
+ {
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-17d",
+ "seq": 1,
+ "status": "completed",
+ "approval_mode": "interactive",
+ "attended": true,
+ "reviewer": {
+ "active": false,
+ "shadow": false,
+ "kind": null,
+ "model": null
+ },
+ "stamp": {
+ "sandbox": {
+ "provider": "direct",
+ "enforcement": "none"
+ }
+ },
+ "max_iterations": 50,
+ "kind": "run",
+ "trigger": "user",
+ "started_at": 1793600000000,
+ "incomplete": false,
+ "first_message_id": "01JP2MSG000000000000000000",
+ "last_message_id": "01JP2MSG000000000000000000",
+ "schema_version": 1,
+ "duration_ms": 1200021
+ }
+ ],
+ "steps": [
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-17d",
+ "parent_step_id": null,
+ "seq": 1,
+ "kind": "model",
+ "name": "model",
+ "started_at": 1793600000000,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": null,
+ "call_index": null,
+ "message_id": "01JP2MSG000000000000000010",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "purpose": "main",
+ "provider": "testprov",
+ "provider_title": "Test provider",
+ "model": "testprov:model-a",
+ "stream": true,
+ "finish_reason": "stop",
+ "ttft_ms": 5,
+ "context_window": 100000,
+ "usage": {
+ "reported": true,
+ "input_fresh": 1000,
+ "cache_read": 0,
+ "cache_write": null,
+ "output": 100,
+ "reasoning": null
+ },
+ "tools_offered": 3,
+ "max_output": 4096
+ },
+ "content": {}
+ },
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-17d",
+ "parent_step_id": null,
+ "seq": 2,
+ "kind": "wait",
+ "name": "approval",
+ "started_at": 1793600000020,
+ "duration_ms": 1200000,
+ "status": "ok",
+ "call_id": "call_3",
+ "call_index": null,
+ "message_id": null,
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "for_call_id": "call_3",
+ "kind": "approval",
+ "where": "session",
+ "attended": true,
+ "outcome": "approved",
+ "inbox_at": null
+ },
+ "content": {}
+ },
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-17d",
+ "parent_step_id": null,
+ "seq": 3,
+ "kind": "notice",
+ "name": "nobody_watching",
+ "started_at": 1793600060020,
+ "duration_ms": 1080000,
+ "status": "ok",
+ "call_id": null,
+ "call_index": null,
+ "message_id": null,
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {},
+ "content": {}
+ },
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-17d",
+ "parent_step_id": null,
+ "seq": 4,
+ "kind": "tool",
+ "name": "run_shell",
+ "started_at": 1793601140021,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": "call_4",
+ "call_index": 0,
+ "message_id": "01JP2MSG000000000000000020",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "risk_class": "exec",
+ "decision": {
+ "path": [],
+ "origin": "low_risk"
+ },
+ "args_hash": "215ddd",
+ "repeat_ok": false,
+ "result_error": false,
+ "error_kind": null,
+ "connector": null,
+ "result_bytes": 120,
+ "exit_code": null,
+ "sandbox": null,
+ "reaches_network": false
+ },
+ "content": {}
+ },
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-17d",
+ "parent_step_id": null,
+ "seq": 5,
+ "kind": "model",
+ "name": "model",
+ "started_at": 1793601140032,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": null,
+ "call_index": null,
+ "message_id": "01JP2MSG000000000000000010",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "purpose": "main",
+ "provider": "testprov",
+ "provider_title": "Test provider",
+ "model": "testprov:model-a",
+ "stream": true,
+ "finish_reason": "stop",
+ "ttft_ms": 5,
+ "context_window": 100000,
+ "usage": {
+ "reported": true,
+ "input_fresh": 1000,
+ "cache_read": 0,
+ "cache_write": null,
+ "output": 100,
+ "reasoning": null
+ },
+ "tools_offered": 3,
+ "max_output": 4096
+ },
+ "content": {}
+ }
+ ]
+}
diff --git a/tests/corpora/observability/golden-phase2/S2-18/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-18/expected-issues.json
index 541d78da..afb49b23 100644
--- a/tests/corpora/observability/golden-phase2/S2-18/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-18/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.3",
+ "signals_version": "0.4",
"warn": 1,
"fired": {
"sandbox_failed": [
diff --git a/tests/corpora/observability/golden-phase2/S2-18c/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-18c/expected-issues.json
index ab3defd8..6c2d41c1 100644
--- a/tests/corpora/observability/golden-phase2/S2-18c/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-18c/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.3",
+ "signals_version": "0.4",
"warn": 0,
"fired": {},
"fine": [
diff --git a/tests/corpora/observability/golden-phase2/S2-19/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-19/expected-issues.json
index 34c71169..ef073414 100644
--- a/tests/corpora/observability/golden-phase2/S2-19/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-19/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.3",
+ "signals_version": "0.4",
"warn": 0,
"fired": {
"reviewer_allowed_send": [
diff --git a/tests/corpora/observability/golden-phase2/S2-19c/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-19c/expected-issues.json
index b142c8d2..4042fe25 100644
--- a/tests/corpora/observability/golden-phase2/S2-19c/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-19c/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.3",
+ "signals_version": "0.4",
"warn": 0,
"fired": {},
"fine": [
diff --git a/tests/corpora/observability/golden-phase2/index.json b/tests/corpora/observability/golden-phase2/index.json
index 8feb4310..56e98666 100644
--- a/tests/corpora/observability/golden-phase2/index.json
+++ b/tests/corpora/observability/golden-phase2/index.json
@@ -1,6 +1,6 @@
{
"format": "1.2",
- "signals_version": "0.3",
+ "signals_version": "0.4",
"built": true,
"scenarios": [
{
@@ -316,6 +316,15 @@
"should": "stay quiet",
"also_fires": [],
"fired": []
+ },
+ {
+ "id": "S2-17d",
+ "check": "waited_unwatched",
+ "should": "fire",
+ "also_fires": [],
+ "fired": [
+ "waited_unwatched"
+ ]
}
]
}
diff --git a/tests/test_observability_gaps.py b/tests/test_observability_gaps.py
new file mode 100644
index 00000000..416bc6df
--- /dev/null
+++ b/tests/test_observability_gaps.py
@@ -0,0 +1,301 @@
+"""OB-14: gaps in the timeline (phase-2 design §11, recording format 1.2): the machine
+offline from its controller, nobody watching the session, the machine asleep, and the
+app restarted mid-turn. Each is a notice; the first three have a duration."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from types import SimpleNamespace
+
+import jsonschema
+import pytest
+
+from coworker.observability import Recorder, RecordingStore, gaps, signals
+from coworker.observability.engine_hooks import TurnRecording
+from coworker.observability.export_spans import Masker, build_turn_spans
+from coworker.server.manager import SessionManager
+
+SCHEMA = json.loads((Path(__file__).parent / "corpora/observability/recording-format.schema.json").read_text("utf-8"))
+
+
+def _check(recording):
+ validator = jsonschema.validators.validator_for(SCHEMA)(SCHEMA)
+ errors = sorted(validator.iter_errors(recording), key=lambda e: list(e.absolute_path))
+ assert not errors, "\n".join(f"{list(e.absolute_path)}: {e.message[:300]}" for e in errors[:8])
+
+
[email protected](autouse=True)
+def _no_machine_conditions():
+ gaps._machine.clear()
+ yield
+ gaps._machine.clear()
+
+
+def _recording(session="sess_g"):
+ rec = Recorder()
+ obs = TurnRecording(rec, lambda: session)
+ return rec, obs
+
+
+def _begin(obs):
+ return obs.begin_turn(kind="run", trigger="user", attrs={
+ "approval_mode": "ask", "attended": True,
+ "reviewer": {"active": False, "shadow": False, "kind": None, "model": None}}, stamp={}, first_message_id=None)
+
+
+def _gaps(rec, name):
+ return [s for s in rec.as_dict()["steps"] if s["kind"] == "notice" and s["name"] == name]
+
+
+# -- nobody watching (E15b-2) ------------------------------------------------------------------
+
+
+def test_nobody_watching_mid_turn_is_a_band_from_leaving_to_coming_back():
+ rec, obs = _recording()
+ _begin(obs)
+ start = obs._turn_started_ms
+ obs.condition("nobody_watching", True, at_ms=start + 1000)
+ obs.condition("nobody_watching", False, at_ms=start + 61_000)
+ obs.end_turn("completed")
+ [gap] = _gaps(rec, "nobody_watching")
+ assert (gap["started_at"], gap["duration_ms"], gap["status"], gap["attrs"]) == (start + 1000, 60_000, "ok", {})
+ assert gap["parent_step_id"] is None
+
+
+def test_a_gap_still_on_is_cut_at_each_turns_start_and_end():
+ rec, obs = _recording()
+ obs.condition("nobody_watching", True, at_ms=1) # left long before the turn: a schedule ran it
+ import time
+
+ t1 = _begin(obs)
+ time.sleep(0.01)
+ obs.end_turn("completed")
+ t2 = _begin(obs)
+ time.sleep(0.01)
+ obs.end_turn("completed")
+ one, two = _gaps(rec, "nobody_watching")
+ assert one["turn_id"] == t1 and two["turn_id"] == t2
+ turns = {t["turn_id"]: t for t in rec.as_dict()["turns"]}
+ assert one["started_at"] == turns[t1]["started_at"] # not before its turn
+ assert one["started_at"] + one["duration_ms"] <= turns[t1]["started_at"] + turns[t1]["duration_ms"] + 1
+ _check(rec.as_dict())
+
+
+def test_a_condition_outside_any_turn_writes_nothing():
+ rec, obs = _recording()
+ obs.condition("nobody_watching", True)
+ obs.condition("nobody_watching", False)
+ assert rec.as_dict()["steps"] == []
+
+
+def test_only_the_crossings_between_none_and_one_viewer_count():
+ told = []
+ fake = SimpleNamespace(_session_clients={}, _engines={})
+ fake._note_watching = lambda sid, watching: told.append((sid, watching))
+ register = SessionManager.register_session_client.__get__(fake)
+ unregister = SessionManager.unregister_session_client.__get__(fake)
+ a, b = object(), object()
+ register("s1", a)
+ register("s1", b) # a second window: no row
+ unregister("s1", a)
+ unregister("s1", b)
+ assert told == [("s1", True), ("s1", False)]
+
+
+def test_the_manager_tells_the_engines_recording():
+ rec, obs = _recording("s1")
+ fake = SimpleNamespace(_engines={"s1": SimpleNamespace(_obs=obs)})
+ note = SessionManager._note_watching.__get__(fake)
+ _begin(obs)
+ note("s1", False)
+ note("s1", True)
+ obs.end_turn("completed")
+ assert len(_gaps(rec, "nobody_watching")) <= 1 # a band only if any time passed
+ note("missing", False) # no engine: nothing, no error
+
+
+# -- the controller link (E15b-1) --------------------------------------------------------------
+
+
+def test_machine_offline_reaches_every_running_turn(tmp_path):
+ rec1, obs1 = _recording("a")
+ rec2, obs2 = _recording("b")
+ _begin(obs1)
+ _begin(obs2)
+ start = max(obs1._turn_started_ms, obs2._turn_started_ms)
+ gaps.machine_condition("controller_offline", True, at_ms=start + 10, attrs={"reason": "ping_timeout"})
+ gaps.machine_condition("controller_offline", False, at_ms=start + 420_010)
+ obs1.end_turn("completed")
+ obs2.end_turn("completed")
+ for rec in (rec1, rec2):
+ [gap] = _gaps(rec, "controller_offline")
+ assert gap["duration_ms"] == 420_000 and gap["attrs"] == {"reason": "ping_timeout"}
+ _check(rec.as_dict())
+
+
+def test_an_engine_made_while_offline_hears_it():
+ gaps.machine_condition("controller_offline", True, at_ms=5, attrs={"reason": "failed"})
+ rec, obs = _recording()
+ _begin(obs)
+ gaps.machine_condition("controller_offline", False, at_ms=obs._turn_started_ms + 1_000)
+ [gap] = _gaps(rec, "controller_offline")
+ assert gap["started_at"] == obs._turn_started_ms and gap["attrs"] == {"reason": "failed"}
+
+
+def test_the_lost_moment_survives_a_restart(tmp_path):
+ gaps.link_lost(tmp_path, "closed")
+ saved = json.loads((tmp_path / "controller-link-lost.json").read_text("utf-8"))
+ assert saved["reason"] == "closed"
+ gaps._machine.clear() # the app restarts: the memory is gone, the file is not
+ rec, obs = _recording()
+ _begin(obs)
+ import time
+ time.sleep(0.01) # the link comes back a moment after the turn started
+ gaps.link_back(tmp_path)
+ [gap] = _gaps(rec, "controller_offline")
+ assert gap["attrs"] == {"reason": "closed"}
+ assert not (tmp_path / "controller-link-lost.json").exists()
+ gaps.link_back(tmp_path) # a welcome with nothing lost: nothing
+ assert len(_gaps(rec, "controller_offline")) == 1
+
+
+def test_link_reasons():
+ assert gaps.link_reason(None) == "closed"
+ assert gaps.link_reason(TimeoutError("keepalive ping timeout")) == "ping_timeout"
+ assert gaps.link_reason(OSError("connection refused")) == "failed"
+
+
+# -- the machine asleep (E15b-3) ---------------------------------------------------------------
+
+
+def _ev(typ, **kw):
+ return {"v": 1, "type": typ, "session_id": "sess_c", "turn_id": "t1", **kw}
+
+
+def _step(step_id, at, mono, end_at, end_mono):
+ return [
+ _ev("step_started", step_id=step_id, at_ms=at, mono_ms=mono, kind="tool", name="look",
+ parent_step_id=None, call_id=step_id, call_index=0, shadow=False, after_turn=False,
+ attrs={}, content={}),
+ _ev("step_ended", step_id=step_id, at_ms=end_at, mono_ms=end_mono, status="ok", message_id=None,
+ attrs={}, content={}, kind="tool", name="look", parent_step_id=None, call_id=step_id,
+ call_index=0, shadow=False, after_turn=False, started_at=at),
+ ]
+
+
+def test_a_sleep_mid_turn_is_a_clock_jump_and_later_steps_follow_the_wall_clock():
+ rec = Recorder()
+ rec.emit(_ev("turn_started", at_ms=1_000, mono_ms=0, kind="run", trigger="user", first_message_id=None,
+ attrs={}, stamp={}))
+ for ev in _step("a", 1_100, 100, 1_200, 200):
+ rec.emit(ev)
+ # The machine slept 60 s: the wall clock moved on, the monotonic clock did not.
+ for ev in _step("b", 61_300, 300, 61_400, 400):
+ rec.emit(ev)
+ rec.emit(_ev("turn_ended", at_ms=61_500, mono_ms=500, status="completed", last_message_id=None))
+ d = rec.as_dict()
+ by = {s["name"]: s for s in d["steps"]}
+ jump = by["clock_jump"]
+ assert jump["started_at"] == 1_200 and jump["duration_ms"] == 60_000
+ assert jump["attrs"] == {"skipped_ms": 60_000}
+ assert by["look"]["started_at"] in (1_100, 61_300)
+ starts = sorted(s["started_at"] for s in d["steps"] if s["kind"] == "tool")
+ assert starts == [1_100, 61_300] # the second step is placed by the wall clock again
+ assert d["turns"][0]["duration_ms"] == 60_500 # the turn's length counts the sleep
+ assert [s["name"] for s in d["steps"]] == ["look", "clock_jump", "look"]
+
+
+def test_small_drift_is_not_a_jump():
+ rec = Recorder()
+ rec.emit(_ev("turn_started", at_ms=1_000, mono_ms=0, kind="run", trigger="user", first_message_id=None,
+ attrs={}, stamp={}))
+ for ev in _step("a", 4_000, 100, 4_100, 200): # 2.9 s ahead: within the 5 s allowed
+ rec.emit(ev)
+ assert not [s for s in rec.as_dict()["steps"] if s["name"] == "clock_jump"]
+
+
+# -- the app restarted (E15b-4) ----------------------------------------------------------------
+
+
+def _open_turn(path):
+ store = RecordingStore(path, background=False)
+ rec = Recorder(store=store)
+ rec.emit(_ev("turn_started", at_ms=1_000, mono_ms=0, kind="run", trigger="user", first_message_id=None,
+ attrs={}, stamp={}))
+ store.flush()
+ return store
+
+
+def _gap_attrs(path):
+ s = RecordingStore(path, background=False)
+ try:
+ assert s.close_open_turns() == 1
+ [gap] = [x for x in s.load_recording("sess_c")["steps"] if x["name"] == "recording_gap"]
+ return gap["attrs"]
+ finally:
+ s.close()
+
+
+def test_a_turn_cut_by_a_crash_says_restart(tmp_path):
+ path = tmp_path / "recordings.db"
+ first = RecordingStore(path, background=False)
+ first.close_open_turns() # this run owns the folder
+ first._conn.close() # killed: never closed properly
+ s = _open_turn(path)
+ s._conn.close()
+ attrs = _gap_attrs(path)
+ assert attrs["cause"] == "restart" and isinstance(attrs["until"], int)
+
+
+def test_a_turn_cut_by_closing_the_app_says_stopped(tmp_path):
+ path = tmp_path / "recordings.db"
+ first = RecordingStore(path, background=False)
+ first.close_open_turns()
+ rec = Recorder(store=first)
+ rec.emit(_ev("turn_started", at_ms=1_000, mono_ms=0, kind="run", trigger="user", first_message_id=None,
+ attrs={}, stamp={}))
+ first.close() # a clean stop while the turn ran
+ assert _gap_attrs(path)["cause"] == "stopped"
+
+
+def test_with_no_word_of_the_last_run_the_cause_is_unknown(tmp_path):
+ path = tmp_path / "recordings.db"
+ s = _open_turn(path)
+ s.close() # not the owner: leaves no word
+ assert _gap_attrs(path)["cause"] == "unknown"
+
+
+# -- what reads the gaps -----------------------------------------------------------------------
+
+
+def test_a_gap_is_its_own_span_and_other_notices_stay_events():
+ rec, obs = _recording()
+ _begin(obs)
+ start = obs._turn_started_ms
+ obs.condition("nobody_watching", True, at_ms=start)
+ obs.condition("nobody_watching", False, at_ms=start + 5_000)
+ obs.instant_step(kind="notice", name="interrupted", status="ok")
+ obs.end_turn("interrupted")
+ d = rec.as_dict()
+ turn = d["turns"][0]
+ spans = build_turn_spans(turn, d["steps"], sw={}, dialect="otel", get_piece=lambda k: None, masker=Masker())
+ [gap] = [s for s in spans if s["name"] == "openworker.gap.nobody_watching"]
+ assert gap["end_ms"] - gap["start_ms"] == 5_000
+ [root] = [s for s in spans if s["name"] == "invoke_agent coworker"]
+ assert [e["name"] for e in root["events"]] == ["openworker.notice.interrupted"]
+
+
+def test_a_card_left_in_the_session_while_nobody_watched_is_an_issue():
+ turn = {"turn_id": "t1", "started_at": 0, "duration_ms": 1_300_000, "status": "completed", "stamp": {},
+ "reviewer": None, "attended": True}
+ wait = {"step_id": "w", "kind": "wait", "name": "approval", "status": "ok", "parent_step_id": None,
+ "shadow": False, "after_turn": False, "started_at": 0, "duration_ms": 1_200_000, "seq": 1,
+ "attrs": {"kind": "approval", "where": "session", "outcome": "approved"}}
+ gap = {"step_id": "g", "kind": "notice", "name": "nobody_watching", "status": "ok", "parent_step_id": None,
+ "shadow": False, "after_turn": False, "started_at": 100_000, "duration_ms": 1_000_000, "seq": 2, "attrs": {}}
+ found = signals.check_turn(turn, [wait, gap])
+ [i] = [x for x in found["issues"] if x["check"] == "waited_unwatched"]
+ assert "17 min for an answer" in i["text"] and "Inbox" not in i["text"]
+ quiet = signals.check_turn(turn, [wait])
+ assert not [x for x in quiet["issues"] if x["check"] == "waited_unwatched"]