Skip to content
Python
12.2 KB
Raw Blame
"""OB-14: gaps in the timeline (phase-2 design §11, recording format 1.2): the machine
offline from its controller, nobody watching the session, the machine asleep, and the
app restarted mid-turn. Each is a notice; the first three have a duration."""

from __future__ import annotations

import json
from pathlib import Path
from types import SimpleNamespace

import jsonschema
import pytest

from coworker.observability import Recorder, RecordingStore, gaps, signals
from coworker.observability.engine_hooks import TurnRecording
from coworker.observability.export_spans import Masker, build_turn_spans
from coworker.server.manager import SessionManager

SCHEMA = json.loads((Path(__file__).parent / "corpora/observability/recording-format.schema.json").read_text("utf-8"))


def _check(recording):
    validator = jsonschema.validators.validator_for(SCHEMA)(SCHEMA)
    errors = sorted(validator.iter_errors(recording), key=lambda e: list(e.absolute_path))
    assert not errors, "\n".join(f"{list(e.absolute_path)}: {e.message[:300]}" for e in errors[:8])


@pytest.fixture(autouse=True)
def _no_machine_conditions():
    gaps._machine.clear()
    yield
    gaps._machine.clear()


def _recording(session="sess_g"):
    rec = Recorder()
    obs = TurnRecording(rec, lambda: session)
    return rec, obs


def _begin(obs):
    return obs.begin_turn(kind="run", trigger="user", attrs={
        "approval_mode": "ask", "attended": True,
        "reviewer": {"active": False, "shadow": False, "kind": None, "model": None}}, stamp={}, first_message_id=None)


def _gaps(rec, name):
    return [s for s in rec.as_dict()["steps"] if s["kind"] == "notice" and s["name"] == name]


# -- nobody watching (E15b-2) ------------------------------------------------------------------


def test_nobody_watching_mid_turn_is_a_band_from_leaving_to_coming_back():
    rec, obs = _recording()
    _begin(obs)
    start = obs._turn_started_ms
    obs.condition("nobody_watching", True, at_ms=start + 1000)
    obs.condition("nobody_watching", False, at_ms=start + 61_000)
    obs.end_turn("completed")
    [gap] = _gaps(rec, "nobody_watching")
    assert (gap["started_at"], gap["duration_ms"], gap["status"], gap["attrs"]) == (start + 1000, 60_000, "ok", {})
    assert gap["parent_step_id"] is None


def test_a_gap_still_on_is_cut_at_each_turns_start_and_end():
    rec, obs = _recording()
    obs.condition("nobody_watching", True, at_ms=1)  # left long before the turn: a schedule ran it
    import time

    t1 = _begin(obs)
    time.sleep(0.01)
    obs.end_turn("completed")
    t2 = _begin(obs)
    time.sleep(0.01)
    obs.end_turn("completed")
    one, two = _gaps(rec, "nobody_watching")
    assert one["turn_id"] == t1 and two["turn_id"] == t2
    turns = {t["turn_id"]: t for t in rec.as_dict()["turns"]}
    assert one["started_at"] == turns[t1]["started_at"]  # not before its turn
    assert one["started_at"] + one["duration_ms"] <= turns[t1]["started_at"] + turns[t1]["duration_ms"] + 1
    _check(rec.as_dict())


def test_a_condition_outside_any_turn_writes_nothing():
    rec, obs = _recording()
    obs.condition("nobody_watching", True)
    obs.condition("nobody_watching", False)
    assert rec.as_dict()["steps"] == []


def test_only_the_crossings_between_none_and_one_viewer_count():
    told = []
    fake = SimpleNamespace(_session_clients={}, _engines={})
    fake._note_watching = lambda sid, watching: told.append((sid, watching))
    register = SessionManager.register_session_client.__get__(fake)
    unregister = SessionManager.unregister_session_client.__get__(fake)
    a, b = object(), object()
    register("s1", a)
    register("s1", b)  # a second window: no row
    unregister("s1", a)
    unregister("s1", b)
    assert told == [("s1", True), ("s1", False)]


def test_the_manager_tells_the_engines_recording():
    rec, obs = _recording("s1")
    fake = SimpleNamespace(_engines={"s1": SimpleNamespace(_obs=obs)})
    note = SessionManager._note_watching.__get__(fake)
    _begin(obs)
    note("s1", False)
    note("s1", True)
    obs.end_turn("completed")
    assert len(_gaps(rec, "nobody_watching")) <= 1  # a band only if any time passed
    note("missing", False)  # no engine: nothing, no error


# -- the controller link (E15b-1) --------------------------------------------------------------


def test_machine_offline_reaches_every_running_turn(tmp_path):
    rec1, obs1 = _recording("a")
    rec2, obs2 = _recording("b")
    _begin(obs1)
    _begin(obs2)
    start = max(obs1._turn_started_ms, obs2._turn_started_ms)
    gaps.machine_condition("controller_offline", True, at_ms=start + 10, attrs={"reason": "ping_timeout"})
    gaps.machine_condition("controller_offline", False, at_ms=start + 420_010)
    obs1.end_turn("completed")
    obs2.end_turn("completed")
    for rec in (rec1, rec2):
        [gap] = _gaps(rec, "controller_offline")
        assert gap["duration_ms"] == 420_000 and gap["attrs"] == {"reason": "ping_timeout"}
        _check(rec.as_dict())


def test_an_engine_made_while_offline_hears_it():
    gaps.machine_condition("controller_offline", True, at_ms=5, attrs={"reason": "failed"})
    rec, obs = _recording()
    _begin(obs)
    gaps.machine_condition("controller_offline", False, at_ms=obs._turn_started_ms + 1_000)
    [gap] = _gaps(rec, "controller_offline")
    assert gap["started_at"] == obs._turn_started_ms and gap["attrs"] == {"reason": "failed"}


def test_the_lost_moment_survives_a_restart(tmp_path):
    gaps.link_lost(tmp_path, "closed")
    saved = json.loads((tmp_path / "controller-link-lost.json").read_text("utf-8"))
    assert saved["reason"] == "closed"
    gaps._machine.clear()  # the app restarts: the memory is gone, the file is not
    rec, obs = _recording()
    _begin(obs)
    import time
    time.sleep(0.01)  # the link comes back a moment after the turn started
    gaps.link_back(tmp_path)
    [gap] = _gaps(rec, "controller_offline")
    assert gap["attrs"] == {"reason": "closed"}
    assert not (tmp_path / "controller-link-lost.json").exists()
    gaps.link_back(tmp_path)  # a welcome with nothing lost: nothing
    assert len(_gaps(rec, "controller_offline")) == 1


def test_link_reasons():
    assert gaps.link_reason(None) == "closed"
    assert gaps.link_reason(TimeoutError("keepalive ping timeout")) == "ping_timeout"
    assert gaps.link_reason(OSError("connection refused")) == "failed"


# -- the machine asleep (E15b-3) ---------------------------------------------------------------


def _ev(typ, **kw):
    return {"v": 1, "type": typ, "session_id": "sess_c", "turn_id": "t1", **kw}


def _step(step_id, at, mono, end_at, end_mono):
    return [
        _ev("step_started", step_id=step_id, at_ms=at, mono_ms=mono, kind="tool", name="look",
            parent_step_id=None, call_id=step_id, call_index=0, shadow=False, after_turn=False,
            attrs={}, content={}),
        _ev("step_ended", step_id=step_id, at_ms=end_at, mono_ms=end_mono, status="ok", message_id=None,
            attrs={}, content={}, kind="tool", name="look", parent_step_id=None, call_id=step_id,
            call_index=0, shadow=False, after_turn=False, started_at=at),
    ]


def test_a_sleep_mid_turn_is_a_clock_jump_and_later_steps_follow_the_wall_clock():
    rec = Recorder()
    rec.emit(_ev("turn_started", at_ms=1_000, mono_ms=0, kind="run", trigger="user", first_message_id=None,
                 attrs={}, stamp={}))
    for ev in _step("a", 1_100, 100, 1_200, 200):
        rec.emit(ev)
    # The machine slept 60 s: the wall clock moved on, the monotonic clock did not.
    for ev in _step("b", 61_300, 300, 61_400, 400):
        rec.emit(ev)
    rec.emit(_ev("turn_ended", at_ms=61_500, mono_ms=500, status="completed", last_message_id=None))
    d = rec.as_dict()
    by = {s["name"]: s for s in d["steps"]}
    jump = by["clock_jump"]
    assert jump["started_at"] == 1_200 and jump["duration_ms"] == 60_000
    assert jump["attrs"] == {"skipped_ms": 60_000}
    assert by["look"]["started_at"] in (1_100, 61_300)
    starts = sorted(s["started_at"] for s in d["steps"] if s["kind"] == "tool")
    assert starts == [1_100, 61_300]  # the second step is placed by the wall clock again
    assert d["turns"][0]["duration_ms"] == 60_500  # the turn's length counts the sleep
    assert [s["name"] for s in d["steps"]] == ["look", "clock_jump", "look"]


def test_small_drift_is_not_a_jump():
    rec = Recorder()
    rec.emit(_ev("turn_started", at_ms=1_000, mono_ms=0, kind="run", trigger="user", first_message_id=None,
                 attrs={}, stamp={}))
    for ev in _step("a", 4_000, 100, 4_100, 200):  # 2.9 s ahead: within the 5 s allowed
        rec.emit(ev)
    assert not [s for s in rec.as_dict()["steps"] if s["name"] == "clock_jump"]


# -- the app restarted (E15b-4) ----------------------------------------------------------------


def _open_turn(path):
    store = RecordingStore(path, background=False)
    rec = Recorder(store=store)
    rec.emit(_ev("turn_started", at_ms=1_000, mono_ms=0, kind="run", trigger="user", first_message_id=None,
                 attrs={}, stamp={}))
    store.flush()
    return store


def _gap_attrs(path):
    s = RecordingStore(path, background=False)
    try:
        assert s.close_open_turns() == 1
        [gap] = [x for x in s.load_recording("sess_c")["steps"] if x["name"] == "recording_gap"]
        return gap["attrs"]
    finally:
        s.close()


def test_a_turn_cut_by_a_crash_says_restart(tmp_path):
    path = tmp_path / "recordings.db"
    first = RecordingStore(path, background=False)
    first.close_open_turns()  # this run owns the folder
    first._conn.close()  # killed: never closed properly
    s = _open_turn(path)
    s._conn.close()
    attrs = _gap_attrs(path)
    assert attrs["cause"] == "restart" and isinstance(attrs["until"], int)


def test_a_turn_cut_by_closing_the_app_says_stopped(tmp_path):
    path = tmp_path / "recordings.db"
    first = RecordingStore(path, background=False)
    first.close_open_turns()
    rec = Recorder(store=first)
    rec.emit(_ev("turn_started", at_ms=1_000, mono_ms=0, kind="run", trigger="user", first_message_id=None,
                 attrs={}, stamp={}))
    first.close()  # a clean stop while the turn ran
    assert _gap_attrs(path)["cause"] == "stopped"


def test_with_no_word_of_the_last_run_the_cause_is_unknown(tmp_path):
    path = tmp_path / "recordings.db"
    s = _open_turn(path)
    s.close()  # not the owner: leaves no word
    assert _gap_attrs(path)["cause"] == "unknown"


# -- what reads the gaps -----------------------------------------------------------------------


def test_a_gap_is_its_own_span_and_other_notices_stay_events():
    rec, obs = _recording()
    _begin(obs)
    start = obs._turn_started_ms
    obs.condition("nobody_watching", True, at_ms=start)
    obs.condition("nobody_watching", False, at_ms=start + 5_000)
    obs.instant_step(kind="notice", name="interrupted", status="ok")
    obs.end_turn("interrupted")
    d = rec.as_dict()
    turn = d["turns"][0]
    spans = build_turn_spans(turn, d["steps"], sw={}, dialect="otel", get_piece=lambda k: None, masker=Masker())
    [gap] = [s for s in spans if s["name"] == "openworker.gap.nobody_watching"]
    assert gap["end_ms"] - gap["start_ms"] == 5_000
    [root] = [s for s in spans if s["name"] == "invoke_agent coworker"]
    assert [e["name"] for e in root["events"]] == ["openworker.notice.interrupted"]


def test_a_card_left_in_the_session_while_nobody_watched_is_an_issue():
    turn = {"turn_id": "t1", "started_at": 0, "duration_ms": 1_300_000, "status": "completed", "stamp": {},
            "reviewer": None, "attended": True}
    wait = {"step_id": "w", "kind": "wait", "name": "approval", "status": "ok", "parent_step_id": None,
            "shadow": False, "after_turn": False, "started_at": 0, "duration_ms": 1_200_000, "seq": 1,
            "attrs": {"kind": "approval", "where": "session", "outcome": "approved"}}
    gap = {"step_id": "g", "kind": "notice", "name": "nobody_watching", "status": "ok", "parent_step_id": None,
           "shadow": False, "after_turn": False, "started_at": 100_000, "duration_ms": 1_000_000, "seq": 2, "attrs": {}}
    found = signals.check_turn(turn, [wait, gap])
    [i] = [x for x in found["issues"] if x["check"] == "waited_unwatched"]
    assert "17 min for an answer" in i["text"] and "Inbox" not in i["text"]
    quiet = signals.check_turn(turn, [wait])
    assert not [x for x in quiet["issues"] if x["check"] == "waited_unwatched"]