Skip to content
Python

fix(observability): Phase 3 review before merging (P3B-2 to P3B-7)

Devika Verma847aca2f8495
diff --git a/coworker/observability/export.py b/coworker/observability/export.py
index fc2e94b6..068148de 100644
--- a/coworker/observability/export.py
+++ b/coworker/observability/export.py
@@ -383,8 +383,9 @@ class Exporter:
rows = getattr(self.store, "feedback_since", None)
if rows is None:
return 0
- todo = [fb for fb in rows(cfg.since_ms, limit=100)
- if self._feedback_mark(fb) not in self.store.export_marked(dest, fb["turn_id"])]
+ # The unsent ones are picked in the query, not after it: filtering after a LIMIT left the
+ # queue stuck on the first 100 once those had gone (P3B-2).
+ todo = rows(cfg.since_ms, limit=100, unsent_for=dest)
sent = 0
for fb in todo:
mark = [(self._feedback_mark(fb), fb["turn_id"])]
diff --git a/coworker/observability/scopes.py b/coworker/observability/scopes.py
index c782bae2..944222d5 100644
--- a/coworker/observability/scopes.py
+++ b/coworker/observability/scopes.py
@@ -169,6 +169,12 @@ def same_approval_twice(cards: list[dict]) -> list[dict]:
"""Task check over the team's approval cards in order: the same tool and args_hash asked
again after an earlier card for it was approved. `cards`: [{tool, args_hash, session,
outcome}]."""
+ return [f for f, _card in same_approval_repeats(cards)]
+
+
+def same_approval_repeats(cards: list[dict]) -> list[tuple[dict, dict]]:
+ """`same_approval_twice` with the card that asked again: (finding, repeat card). Both the
+ Issues tab and TeamView's event line point at that card, the second ask (P3B-4)."""
approved: dict[tuple, dict] = {}
fired = []
for c in cards:
@@ -176,7 +182,7 @@ def same_approval_twice(cards: list[dict]) -> list[dict]:
continue # arguments that never parsed match nothing (build: not in the prototype)
k = (c["tool"], c["args_hash"])
if k in approved:
- fired.append({"tool": c["tool"], "first_by": approved[k]["session"], "again_by": c["session"]})
+ fired.append(({"tool": c["tool"], "first_by": approved[k]["session"], "again_by": c["session"]}, c))
if c.get("outcome") == "approved":
approved.setdefault(k, c)
return fired
@@ -321,14 +327,11 @@ def failed_in_a_row_issue(runs: list[dict], last_step: Optional[dict] = None) ->
def same_approval_issues(cards: list[dict]) -> list[dict]:
- out = []
- for i, f in enumerate(same_approval_twice(cards)):
- step = next((c.get("step") for c in cards
- if c["tool"] == f["tool"] and c["session"] == f["again_by"] and c.get("step")), None)
- out.append(_issue("same_approval_twice",
- f"Same approval asked twice: {f['tool']} ({f['first_by']}, then {f['again_by']}).",
- step, **f))
- return out
+ """Each repeat as an issue on the card that asked again (P3B-4), not the first one."""
+ return [_issue("same_approval_twice",
+ f"Same approval asked twice: {f['tool']} ({f['first_by']}, then {f['again_by']}).",
+ card.get("step"), **f)
+ for f, card in same_approval_repeats(cards)]
def reviewed_before_prereq_issues(events: list[dict], links: Iterable[tuple[Any, Any]]) -> list[dict]:
diff --git a/coworker/observability/store.py b/coworker/observability/store.py
index ad2ad0b1..308423ae 100644
--- a/coworker/observability/store.py
+++ b/coworker/observability/store.py
@@ -236,6 +236,9 @@ class RecordingStore:
self._pending_steps: dict[str, dict[str, Any]] = {}
self._pending_refresh: set[str] = set() # turns whose ledger rows to work out again
self._pending_lock = threading.Lock()
+ # How many writes touched each session since start (P3B-5): a team view keys its
+ # cache on its members' numbers, so it is worked out again only when one changed.
+ self._generation: dict[str, int] = {}
self._owner = False # this process called close_open_turns: it owns the data folder
self._closed = False
self._last_sweep = 0.0
@@ -292,11 +295,14 @@ class RecordingStore:
"INSERT OR IGNORE INTO step_pieces (step_id, sha256) VALUES (?, ?)",
[(s["step_id"], r) for r in refs],
)
- # Phase 3: the ledger rows of turns that ended or changed, from what was
- # just written (the turn, its steps and its Issues result).
- self._refresh_usage(refresh)
except Exception:
log.exception("recordings: write failed (%d turns, %d steps)", len(turns), len(steps))
+ self._touch_sessions({t.get("session_id") for t in turns.values()} | {x.get("session_id") for x in steps.values()})
+ # Phase 3: the ledger rows of turns that ended or changed, from what was just
+ # written (the turn, its steps and its Issues result). After the recording's own
+ # transaction, one turn at a time: a ledger error loses that turn's rows only,
+ # never the recorded steps (P3B-6).
+ self._refresh_usage(refresh)
def _tick(self) -> None:
"""One turn of the shared writer: write what is pending, and sweep once a day."""
@@ -418,9 +424,23 @@ class RecordingStore:
return len(open_turns)
# -- deleting -----------------------------------------------------------------------------
+ def _touch_sessions(self, session_ids: Iterable[Optional[str]]) -> None:
+ with self._pending_lock:
+ for sid in session_ids:
+ if sid:
+ self._generation[sid] = self._generation.get(sid, 0) + 1
+
+ def generation(self, session_ids: Iterable[str]) -> tuple[int, ...]:
+ """How many writes have touched each of these sessions since start (P3B-5). Writes
+ still pending are written first, so an unchanged answer means nothing new."""
+ self.flush()
+ with self._pending_lock:
+ return tuple(self._generation.get(sid, 0) for sid in session_ids)
+
def delete_session(self, session_id: str) -> None:
"""The session-delete cascade (§11): its turns, steps and the pieces only they used."""
self.flush() # the ledger rows of its turns are worked out before the steps go
+ self._touch_sessions({session_id})
with self._lock, self._conn:
with self._pending_lock:
self._pending_turns = {
@@ -723,41 +743,51 @@ class RecordingStore:
_writer.wake()
def _refresh_usage(self, turn_ids: Iterable[str]) -> None:
- """Inside the writer's transaction: replace each finished turn's ledger rows."""
- from .rollups import LEDGER_COLUMNS, ledger_rows
-
+ """With the connection lock held: replace each finished turn's ledger rows, each turn in
+ a transaction of its own. A turn whose rows cannot be worked out keeps its old rows and
+ is logged; the others go on (P3B-6)."""
now = int(time.time() * 1000)
for tid in turn_ids:
- t = self._conn.execute("SELECT * FROM turns WHERE turn_id = ?", (tid,)).fetchone()
- if t is None or t["status"] in (None, "running"):
- continue
- steps = [_step_record(r) for r in self._conn.execute(
- "SELECT * FROM steps WHERE turn_id = ? AND kind IN ('model', 'wait', 'auto_approver', 'tool')",
- (tid,))]
try:
- issues = json.loads(t["issues"]) if t["issues"] else None
- except ValueError:
- issues = None
- prior = self._conn.execute(
- "SELECT rating, session_deleted, persona_id FROM turn_usage WHERE turn_id = ? AND first = 1",
- (tid,)).fetchone()
- fb = self._conn.execute("SELECT rating FROM feedback WHERE turn_id = ?", (tid,)).fetchone()
- persona = prior["persona_id"] if prior is not None else None
- if persona is None and self.persona_of is not None:
- try:
- persona = self.persona_of(t["session_id"])
- except Exception: # noqa: BLE001 - a stand-in never breaks the write
- persona = None
- rating = fb["rating"] if fb is not None else (prior["rating"] if prior is not None else None)
- rows = ledger_rows(_turn_record(t), steps, issues, persona_id=persona, rating=rating, now_ms=now)
- if prior is not None and prior["session_deleted"]:
- for r in rows:
- r["session_deleted"] = 1
- self._conn.execute("DELETE FROM turn_usage WHERE turn_id = ?", (tid,))
- self._conn.executemany(
- f"INSERT INTO turn_usage ({', '.join(LEDGER_COLUMNS)}) "
- f"VALUES ({', '.join('?' * len(LEDGER_COLUMNS))})",
- [tuple(r.get(c) for c in LEDGER_COLUMNS) for r in rows])
+ with self._conn:
+ self._refresh_turn_usage(tid, now)
+ except Exception:
+ log.exception("recordings: usage ledger not updated for turn %s", tid)
+
+ def _refresh_turn_usage(self, tid: str, now: int) -> None:
+ from .rollups import LEDGER_COLUMNS, ledger_rows
+
+ t = self._conn.execute("SELECT * FROM turns WHERE turn_id = ?", (tid,)).fetchone()
+ if t is None or t["status"] in (None, "running"):
+ return
+ self._touch_sessions({t["session_id"]}) # its Issues result or ledger rows changed
+ steps = [_step_record(r) for r in self._conn.execute(
+ "SELECT * FROM steps WHERE turn_id = ? AND kind IN ('model', 'wait', 'auto_approver', 'tool')",
+ (tid,))]
+ try:
+ issues = json.loads(t["issues"]) if t["issues"] else None
+ except ValueError:
+ issues = None
+ prior = self._conn.execute(
+ "SELECT rating, session_deleted, persona_id FROM turn_usage WHERE turn_id = ? AND first = 1",
+ (tid,)).fetchone()
+ fb = self._conn.execute("SELECT rating FROM feedback WHERE turn_id = ?", (tid,)).fetchone()
+ persona = prior["persona_id"] if prior is not None else None
+ if persona is None and self.persona_of is not None:
+ try:
+ persona = self.persona_of(t["session_id"])
+ except Exception: # noqa: BLE001 - a stand-in never breaks the write
+ persona = None
+ rating = fb["rating"] if fb is not None else (prior["rating"] if prior is not None else None)
+ rows = ledger_rows(_turn_record(t), steps, issues, persona_id=persona, rating=rating, now_ms=now)
+ if prior is not None and prior["session_deleted"]:
+ for r in rows:
+ r["session_deleted"] = 1
+ self._conn.execute("DELETE FROM turn_usage WHERE turn_id = ?", (tid,))
+ self._conn.executemany(
+ f"INSERT INTO turn_usage ({', '.join(LEDGER_COLUMNS)}) "
+ f"VALUES ({', '.join('?' * len(LEDGER_COLUMNS))})",
+ [tuple(r.get(c) for c in LEDGER_COLUMNS) for r in rows])
def sweep_ledger(self, *, now_ms: Optional[int] = None) -> int:
"""Ledger rows older than 90 days go (TD3-5), whatever "Keep for" says."""
@@ -919,6 +949,34 @@ class RecordingStore:
"wake": json.loads(r["wake"]) if r["wake"] else None,
"stamp": json.loads(r["stamp"] or "{}")} for r in rows]
+ def last_turn_automation(self, session_id: str) -> Optional[dict[str, Any]]:
+ """The session's latest turn's `turn.automation` (1.3), or None when that turn was not
+ an automation run's own (a follow-up typed later, or no turn yet)."""
+ self.flush()
+ with self._lock:
+ r = self._conn.execute("SELECT automation FROM turns WHERE session_id = ? "
+ "ORDER BY started_at DESC, seq DESC LIMIT 1", (session_id,)).fetchone()
+ if r is None or not r["automation"]:
+ return None
+ try:
+ return json.loads(r["automation"])
+ except ValueError:
+ return None
+
+ def last_tool_step(self, session_id: str, name: str, *, before_ms: Optional[int] = None) -> Optional[dict[str, Any]]:
+ """The session's last tool step of this name started at or before `before_ms` (any
+ time when None): {step_id, seq, turn_id}. What a hand-off's `transition` step is found
+ by, without loading the whole recording (P3B-5)."""
+ self.flush()
+ sql = "SELECT step_id, seq, turn_id FROM steps WHERE session_id = ? AND kind = 'tool' AND name = ?"
+ args: list[Any] = [session_id, name]
+ if before_ms is not None:
+ sql += " AND COALESCE(started_at, 0) <= ?"
+ args.append(before_ms)
+ with self._lock:
+ r = self._conn.execute(sql + " ORDER BY started_at DESC, seq DESC LIMIT 1", args).fetchone()
+ return None if r is None else {"step_id": r["step_id"], "seq": r["seq"], "turn_id": r["turn_id"]}
+
def team_approval_cards(self, session_ids: list[str]) -> list[dict[str, Any]]:
"""The members' approval cards in order: each wait step of kind approval with its
tool step's name and args_hash (`same_approval_twice`, signals 0.6)."""
@@ -932,14 +990,24 @@ class RecordingStore:
"JOIN turns t ON t.turn_id = s.turn_id "
f"WHERE s.kind = 'wait' AND s.session_id IN ({marks}) ORDER BY s.started_at, s.step_id",
list(session_ids)).fetchall()
- tools = {(r["session_id"], r["call_id"]): r for r in self._conn.execute(
- "SELECT session_id, call_id, name, attrs FROM steps "
- f"WHERE kind = 'tool' AND session_id IN ({marks})", list(session_ids)).fetchall()}
+ approvals = []
+ for w in waits:
+ a = json.loads(w["attrs"] or "{}")
+ if a.get("kind") == "approval" and a.get("for_call_id"):
+ approvals.append((w, a))
+ # Only the tool steps the cards ask about, by (session, call id): not every tool
+ # step of every member (P3B-5).
+ tools = {}
+ wanted = sorted({(w["session_id"], a["for_call_id"]) for w, a in approvals})
+ for i in range(0, len(wanted), 400):
+ part = wanted[i:i + 400]
+ where = " OR ".join("(session_id = ? AND call_id = ?)" for _ in part)
+ for r in self._conn.execute(
+ f"SELECT session_id, call_id, name, attrs FROM steps WHERE kind = 'tool' AND ({where})",
+ [v for pair in part for v in pair]).fetchall():
+ tools[(r["session_id"], r["call_id"])] = r
out = []
- for w in waits:
- a = json.loads(w["attrs"] or "{}")
- if a.get("kind") != "approval":
- continue
+ for w, a in approvals:
tool = tools.get((w["session_id"], a.get("for_call_id")))
if tool is None:
continue
@@ -1019,13 +1087,22 @@ class RecordingStore:
(session_id,)).fetchall()
return [_feedback_record(r) for r in rows]
- def feedback_since(self, since_ms: int, limit: int = 100) -> list[dict[str, Any]]:
+ def feedback_since(self, since_ms: int, limit: int = 100, *,
+ unsent_for: Optional[str] = None) -> list[dict[str, Any]]:
"""Ratings given or changed at or after `since_ms`, oldest change first: the
- exporter's queue (E27b)."""
+ exporter's queue (E27b). With `unsent_for`, only the versions with no export mark for
+ that destination yet (mark `feedback|<turn_id>|<changed_at or rated_at>`), so the
+ queue keeps moving past the first `limit` once those have gone (P3B-2)."""
+ sql = "SELECT * FROM feedback f WHERE COALESCE(f.changed_at, f.rated_at) >= ?"
+ args: list[Any] = [since_ms]
+ if unsent_for is not None:
+ sql += (" AND NOT EXISTS (SELECT 1 FROM export_marks m WHERE m.destination = ? AND m.turn_id = f.turn_id"
+ " AND m.item_id = 'feedback|' || f.turn_id || '|' || COALESCE(f.changed_at, f.rated_at))")
+ args.append(unsent_for)
+ sql += " ORDER BY COALESCE(f.changed_at, f.rated_at) LIMIT ?"
+ args.append(limit)
with self._lock:
- rows = self._conn.execute(
- "SELECT * FROM feedback WHERE COALESCE(changed_at, rated_at) >= ? "
- "ORDER BY COALESCE(changed_at, rated_at) LIMIT ?", (since_ms, limit)).fetchall()
+ rows = self._conn.execute(sql, args).fetchall()
return [_feedback_record(r) for r in rows]
# -- coworker versions (Phase 3, E25c) -------------------------------------------------------
diff --git a/coworker/observability/usage_api.py b/coworker/observability/usage_api.py
index 45030fa2..96b2e5b5 100644
--- a/coworker/observability/usage_api.py
+++ b/coworker/observability/usage_api.py
@@ -265,12 +265,14 @@ def team_wakes(store: Any, session_ids: Iterable[str], *, start: int = 0, end: i
def team_events(store: Any, *, lead_session: str, worker_sessions: dict[str, str],
- board_events: list[dict[str, Any]], links: list[tuple[int, int]], limit: int = 50) -> list[dict[str, Any]]:
+ board_events: list[dict[str, Any]], links: list[tuple[int, int]], limit: int = 50,
+ turns: Optional[list[dict[str, Any]]] = None) -> list[dict[str, Any]]:
""""Team events that used to be silent" (§11.4), newest first: cap hits, long deferrals,
watchdog nudges, lead polling, the same approval asked twice and an item reviewed before
- its prerequisite. Each names the session and turn whose Steps it opens."""
+ its prerequisite. Each names the session and turn whose Steps it opens. `turns`: the
+ members' turns when the caller has read them already (P3B-5)."""
sessions = {lead_session: "lead", **{sid: actor for actor, sid in worker_sessions.items()}}
- rows = store.team_turns(list(sessions))
+ rows = store.team_turns(list(sessions)) if turns is None else turns
events: list[dict[str, Any]] = []
for t in rows:
wake = t.get("wake") or {}
@@ -290,10 +292,9 @@ def team_events(store: Any, *, lead_session: str, worker_sessions: dict[str, str
events.append({"session_id": lead_session, "turn_id": t["turn_id"], "seq": t["seq"], "at": t["started_at"],
"who": "lead", "kind": "lead_polling", "severity": "warn", "count": f["count"]})
cards = store.team_approval_cards(list(sessions))
- for f in scopes.same_approval_twice(cards):
- c = next((c for c in reversed(cards) if c["tool"] == f["tool"] and c["session"] == f["again_by"]), None)
- events.append({"session_id": f["again_by"], "turn_id": (c or {}).get("turn_id"), "seq": (c or {}).get("seq"),
- "at": (c or {}).get("at"), "who": sessions.get(f["again_by"], "?"), "kind": "same_approval_twice",
+ for f, c in scopes.same_approval_repeats(cards): # the repeat card, as the Issues tab (P3B-4)
+ events.append({"session_id": f["again_by"], "turn_id": c.get("turn_id"), "seq": c.get("seq"),
+ "at": c.get("at"), "who": sessions.get(f["again_by"], "?"), "kind": "same_approval_twice",
"severity": "warn", "tool": f["tool"], "first_by": sessions.get(f["first_by"], f["first_by"]),
"again_by": sessions.get(f["again_by"], f["again_by"])})
transitions = [{"seq": e["seq"], "item": e.get("item_id"), "to": (e.get("payload") or {}).get("to"),
diff --git a/coworker/remote/acceptor.py b/coworker/remote/acceptor.py
index 145ec4ce..0b0313e4 100644
--- a/coworker/remote/acceptor.py
+++ b/coworker/remote/acceptor.py
@@ -814,10 +814,11 @@ def mount_acceptor(
)
return {"ok": True, "revoked": names}
- def _cached_rollup(machine_id: str, grouping: str):
- """An offline machine's last rollup, labelled with when it was fetched (TD3-6). The
+ def _cached_rollup(machine_id: str, query: str):
+ """An offline machine's last answer to this same rollup request, labelled with when it
+ was fetched (TD3-6). Only the same period, grouping and coworker count (P3B-3). The
desktop never adds it to the live totals without that label."""
- found = registry.last_rollup(machine_id, grouping)
+ found = registry.last_rollup(machine_id, query)
if found is None:
return None
try:
@@ -841,9 +842,8 @@ def mount_acceptor(
_touch(machine_id)
live = await _live_or_wake(machine_id)
rollup = request.method == "GET" and path == "v1/observability/rollup"
- grouping = request.query_params.get("group") or "all"
if live is None:
- cached = _cached_rollup(machine_id, grouping) if rollup else None
+ cached = _cached_rollup(machine_id, request.url.query) if rollup else None
return cached or JSONResponse({"error": "machine offline"}, status_code=503)
target = "/" + path
if request.url.query:
@@ -863,12 +863,12 @@ def mount_acceptor(
except asyncio.TimeoutError:
return JSONResponse({"error": "machine timed out"}, status_code=504)
except ConnectionError:
- cached = _cached_rollup(machine_id, grouping) if rollup else None
+ cached = _cached_rollup(machine_id, request.url.query) if rollup else None
return cached or JSONResponse({"error": "machine disconnected"}, status_code=503)
- # Phase 3 (E28): each machine's last Usage rollup, per grouping, for when it is off.
+ # Phase 3 (E28): each machine's last answer per Usage request, for when it is off.
if rollup and status == 200:
try:
- registry.save_rollup(machine_id, grouping, request.url.query, payload.decode("utf-8"))
+ registry.save_rollup(machine_id, request.url.query, payload.decode("utf-8"))
except Exception:
pass
# Tier-2 snapshot: every successful unfiltered session-list fetch doubles
diff --git a/coworker/remote/registry.py b/coworker/remote/registry.py
index facc1e1d..0b9e2414 100644
--- a/coworker/remote/registry.py
+++ b/coworker/remote/registry.py
@@ -48,24 +48,27 @@ class MachinesRegistry:
)
"""
)
- # Tier-2 snapshot (remote-home-design.md data path): the last-known
- # session-LIST METADATA per machine — what keeps an offline machine's
- # sessions greyed-but-visible instead of vanished.
# Observability Phase 3 (E28, TD3-6): the last Usage rollup each machine answered,
- # per grouping, so an offline machine still shows its numbers on the Usage page,
- # labelled "as of". Counts only (the ledger is content-free).
+ # per request (period, grouping, coworker, ...), so an offline machine still shows
+ # its numbers on the Usage page, labelled "as of", and only for the question that
+ # was asked (P3B-3). Counts only (the ledger is content-free). The first build kept
+ # one row per grouping, whatever the period; that table is dropped.
+ self._conn.execute("DROP TABLE IF EXISTS machine_rollups")
self._conn.execute(
"""
- CREATE TABLE IF NOT EXISTS machine_rollups (
+ CREATE TABLE IF NOT EXISTS machine_rollup_answers (
machine_id TEXT NOT NULL,
- grouping TEXT NOT NULL,
+ query_key TEXT NOT NULL,
query TEXT NOT NULL,
payload TEXT NOT NULL,
updated_at REAL NOT NULL,
- PRIMARY KEY (machine_id, grouping)
+ PRIMARY KEY (machine_id, query_key)
)
"""
)
+ # Tier-2 snapshot (remote-home-design.md data path): the last-known
+ # session-LIST METADATA per machine — what keeps an offline machine's
+ # sessions greyed-but-visible instead of vanished.
self._conn.execute(
"""
CREATE TABLE IF NOT EXISTS machine_sessions (
@@ -289,22 +292,43 @@ class MachinesRegistry:
).fetchall()
return [dict(r) for r in rows]
- def save_rollup(self, machine_id: str, grouping: str, query: str, payload: str) -> None:
+ ROLLUPS_KEPT = 50 # per machine: the few periods and coworkers a person looks at
+
+ @staticmethod
+ def rollup_key(query: str) -> str:
+ """A rollup request's cache key: its query parameters, sorted, so the same question
+ asked with its parameters in another order finds the same answer."""
+ from urllib.parse import parse_qsl, urlencode
+
+ return urlencode(sorted(parse_qsl(query or "", keep_blank_values=True)))
+
+ def save_rollup(self, machine_id: str, query: str, payload: str) -> None:
+ """Keep a machine's answer to one rollup request (P3B-3: the whole request is the key,
+ not just its grouping). Only the newest `ROLLUPS_KEPT` per machine stay."""
with self._lock:
self._conn.execute(
- "INSERT INTO machine_rollups (machine_id, grouping, query, payload, updated_at)"
- " VALUES (?, ?, ?, ?, ?) ON CONFLICT(machine_id, grouping) DO UPDATE SET"
+ "INSERT INTO machine_rollup_answers (machine_id, query_key, query, payload, updated_at)"
+ " VALUES (?, ?, ?, ?, ?) ON CONFLICT(machine_id, query_key) DO UPDATE SET"
" query = excluded.query, payload = excluded.payload, updated_at = excluded.updated_at",
- (machine_id, grouping, query, payload, time.time()),
+ (machine_id, self.rollup_key(query), query, payload, time.time()),
+ )
+ self._conn.execute(
+ "DELETE FROM machine_rollup_answers WHERE machine_id = ? AND query_key NOT IN ("
+ " SELECT query_key FROM machine_rollup_answers WHERE machine_id = ?"
+ " ORDER BY updated_at DESC LIMIT ?)",
+ (machine_id, machine_id, self.ROLLUPS_KEPT),
)
self._conn.commit()
- def last_rollup(self, machine_id: str, grouping: str) -> Optional[dict]:
- """The machine's last Usage rollup for this grouping: {payload, query, updated_at}."""
+ def last_rollup(self, machine_id: str, query: str) -> Optional[dict]:
+ """The machine's last answer to this same rollup request: {payload, query, updated_at},
+ or None when that request was never answered (another period or coworker is not a
+ stand-in)."""
with self._lock:
row = self._conn.execute(
- "SELECT payload, query, updated_at FROM machine_rollups WHERE machine_id = ? AND grouping = ?",
- (machine_id, grouping),
+ "SELECT payload, query, updated_at FROM machine_rollup_answers"
+ " WHERE machine_id = ? AND query_key = ?",
+ (machine_id, self.rollup_key(query)),
).fetchone()
return dict(row) if row else None
diff --git a/coworker/server/manager.py b/coworker/server/manager.py
index 7f59a089..f7723c1c 100644
--- a/coworker/server/manager.py
+++ b/coworker/server/manager.py
@@ -454,6 +454,11 @@ class SessionManager:
self._scope_pool = concurrent.futures.ThreadPoolExecutor(max_workers=1, thread_name_prefix="obs-scopes")
self.recorder.add_listener(self._scopes_on_turn_end)
+ # P3B-5: a team's Phase 3 numbers and task checks, kept until a member's recording,
+ # the board or the journal changes (team_id -> what they were worked out from, result).
+ self._team_obs_cache: dict[str, tuple[Any, Any]] = {}
+ self._team_checked: dict[str, Any] = {}
+ self._team_obs_lock = threading.Lock()
# No explicit provider injected → route by the model's `provider:` prefix (OpenAI default,
# Ollama, …). Tests inject a provider directly and bypass the router. The same router is
# shared by every engine and the `/v1/chat/completions` proxy.
@@ -2057,9 +2062,20 @@ class SessionManager:
current = self.inbox.get(item.id) if getattr(item, "id", None) else None
by_raw = str(getattr(current or item, "resolved_by", "") or "")
by = "lead" if by_raw.startswith(("lead", "agent")) else "user"
+ # P3B-7 (TD3-23): the resume finishes the turn that asked, so when that turn was an
+ # automation run's own, the resumed turn is the run's too: its tokens and health count
+ # in the run's row. The card is the latest turn's (the turn stopped at it); a follow-up
+ # typed into the run's session later carries no `automation`, and neither does its resume.
+ automation = None
+ if item.session_id.startswith("__run__"):
+ try:
+ automation = self.recordings_store.last_turn_automation(item.session_id)
+ except Exception: # noqa: BLE001 - a recording fact never blocks the resume
+ logger.exception("could not tell whether %s resumes a run's turn", item.session_id)
try:
async for _event in engine.resume(trigger="user", team=self._team_stamp(item.session_id), wake={
- "reason": "answer", "by": by, "board_seq": self._board_seq(item.session_id)}):
+ "reason": "answer", "by": by, "board_seq": self._board_seq(item.session_id)},
+ automation=automation):
pass
self.save(item.session_id, engine)
finally:
@@ -2748,29 +2764,10 @@ class SessionManager:
out = make_summary(self, team, time.time())
# Phase 3 (OB-17, E19c): why each team turn started, from the ledger of every member
# session (its total equals their pills), and the team events that used to be silent.
+ # Worked out again only when something they read has changed (P3B-5).
try:
- members = {w.actor: w.session_id for w in team.workers}
- sids = [team.lead_session, *members.values()]
- out["wakes"] = usage_api.team_wakes(self.recordings_store, sids)
- from ..teams.summary import all_events
-
- board = list(all_events(self.team_store, team.space))
- links = []
- for item in self.team_store.list_items(team.space, self._user_actor()):
- for link in item.get("links") or []:
- if link.get("kind") == "blocked_by" and link.get("item") is not None:
- links.append((int(link["item"]), int(item["id"])))
- out["events"] = usage_api.team_events(self.recordings_store, lead_session=team.lead_session,
- worker_sessions=members, board_events=board, links=links)
- out["wake_explained"] = self._wake_explained(sids)
- # OB-16: "Coverage for this task" on each item a coworker that declares coverage
- # worked on, with its findings' evidence links.
- cov = {}
- for item in self.team_store.list_items(team.space, self._user_actor()):
- c = self.item_coverage(team, item, board)
- if c is not None:
- cov[str(item["id"])] = c
- out["coverage"] = cov
+ out.update(self._team_obs(team))
+ cov = out["coverage"]
by_session = {team.lead_session: team.lead_actor, **{w.session_id: w.actor for w in team.workers}}
codes = {name: list(sc.get("findings_exit") or []) for name, sc in
((recording_signals.signals().get("scanners") or {}).get("list") or {}).items()}
@@ -2802,9 +2799,64 @@ class SessionManager:
out.setdefault("events", [])
return out
- def _wake_explained(self, session_ids: list[str]) -> dict[str, int]:
+ def _team_inputs(self, team) -> dict[str, Any]:
+ """What a team's Phase 3 numbers and task checks are worked out from, read once: the
+ board, its items, and a mark of everything they depend on (P3B-5). The mark is the
+ members' write counts, the board's last event, the journal's last entry and the items
+ themselves; when it is unchanged, so is the answer."""
+ from ..teams.summary import all_events
+
+ members = {w.actor: w.session_id for w in team.workers}
+ sids = [team.lead_session, *members.values()]
+ board = list(all_events(self.team_store, team.space))
+ items = self.team_store.list_items(team.space, self._user_actor())
+ js = self.journal_store
+ with js._lock:
+ row = js._conn.execute("SELECT MAX(seq) AS m FROM journal_entries WHERE space = ?",
+ (team.space,)).fetchone()
+ mark = (tuple(sids), self.recordings_store.generation(sids), len(board),
+ board[-1]["seq"] if board else None, row["m"] if row else None,
+ json.dumps(items, sort_keys=True, default=str))
+ return {"members": members, "sids": sids, "board": board, "items": items, "mark": mark}
+
+ def _team_obs(self, team) -> dict[str, Any]:
+ """A team's wakes, events, wake gate and per-item coverage, kept until its inputs
+ change: a refresh of an idle team rereads only the board and the counts (P3B-5)."""
+ inputs = self._team_inputs(team)
+ with self._team_obs_lock:
+ hit = self._team_obs_cache.get(team.team_id)
+ if hit is not None and hit[0] == inputs["mark"]:
+ return hit[1]
+ members, sids, board, items = inputs["members"], inputs["sids"], inputs["board"], inputs["items"]
+ turns = self.recordings_store.team_turns(sids)
+ links = []
+ for item in items:
+ for link in item.get("links") or []:
+ if link.get("kind") == "blocked_by" and link.get("item") is not None:
+ links.append((int(link["item"]), int(item["id"])))
+ res: dict[str, Any] = {
+ "wakes": usage_api.team_wakes(self.recordings_store, sids),
+ "events": usage_api.team_events(self.recordings_store, lead_session=team.lead_session,
+ worker_sessions=members, board_events=board, links=links,
+ turns=turns),
+ "wake_explained": self._wake_explained(sids, turns),
+ }
+ # OB-16: "Coverage for this task" on each item a coworker that declares coverage
+ # worked on, with its findings' evidence links.
+ cov = {}
+ for item in items:
+ c = self.item_coverage(team, item, board, turns=turns)
+ if c is not None:
+ cov[str(item["id"])] = c
+ res["coverage"] = cov
+ with self._team_obs_lock:
+ self._team_obs_cache[team.team_id] = (inputs["mark"], res)
+ return res
+
+ def _wake_explained(self, session_ids: list[str],
+ rows: Optional[list[dict[str, Any]]] = None) -> dict[str, int]:
"""The 95% gate on a real team (V3-08): team turns, and those whose wake is known."""
- rows = self.recordings_store.team_turns(session_ids)
+ rows = self.recordings_store.team_turns(session_ids) if rows is None else rows
team_rows = [t for t in rows if (t.get("stamp") or {}).get("team")]
known = sum(1 for t in team_rows if (t.get("wake") or {}).get("reason") not in (None, "unknown"))
return {"turns": len(team_rows), "explained": known}
@@ -6805,10 +6857,12 @@ class SessionManager:
found.setdefault(i.get("turn_id") or last, []).append(i)
self._store_scope_issues([t["turn_id"] for t in done], found, ["scan_missing", "scanner_failed"])
- def item_coverage(self, team, item: dict[str, Any], events: list[dict[str, Any]]) -> Optional[dict[str, Any]]:
+ def item_coverage(self, team, item: dict[str, Any], events: list[dict[str, Any]], *,
+ turns: Optional[list[dict[str, Any]]] = None) -> Optional[dict[str, Any]]:
""""Coverage for this task" (OB-16, §10): a board item of a worker that declares
coverage, its scans across the item's sessions, the hand-off's report, its findings
- with their evidence, and the fix it verifies. None when nobody on it promised a scan."""
+ with their evidence, and the fix it verifies. None when nobody on it promised a scan.
+ `turns`: the members' turns, when the caller has read them already (P3B-5)."""
by_actor = {w.actor: w.session_id for w in team.workers}
by_actor[team.lead_actor] = team.lead_session
mine = [e for e in events if e.get("item_id") == item["id"]]
@@ -6819,7 +6873,9 @@ class SessionManager:
if not sessions:
return None
promised: list[str] = []
- for t in self.recordings_store.team_turns(sessions):
+ mine_turns = ([t for t in turns if t["session_id"] in sessions] if turns is not None
+ else self.recordings_store.team_turns(sessions))
+ for t in mine_turns:
for a in (t.get("stamp") or {}).get("persona_coverage") or []:
if a not in promised:
promised.append(a)
@@ -6885,13 +6941,9 @@ class SessionManager:
def _handoff_step(self, session_id: str, at_ms: Optional[int]) -> Optional[dict[str, Any]]:
"""The `transition` tool step that handed the item off: the last one of the session at
or before the board event's time."""
- rec = self.recordings_store.load_recording(session_id)
- steps = [s for s in rec.get("steps") or [] if s.get("kind") == "tool" and s.get("name") == "transition"
- and (at_ms is None or (s.get("started_at") or 0) <= at_ms + 1000)]
- if not steps:
- return None
- s = steps[-1]
- return {"step_id": s["step_id"], "seq": s.get("seq"), "turn_id": s.get("turn_id")}
+ # One indexed read, not the whole recording (P3B-5).
+ return self.recordings_store.last_tool_step(session_id, "transition",
+ before_ms=None if at_ms is None else at_ms + 1000)
def _scope_team_tasks(self, team_id: str) -> None:
"""A team's task checks: coverage of each handed-off item of a worker that declares
@@ -6899,13 +6951,15 @@ class SessionManager:
team = self.teams.get(team_id)
if team is None:
return
- from ..teams.summary import all_events
-
- events = list(all_events(self.team_store, team.space))
- sids = [team.lead_session, *(w.session_id for w in team.workers)]
+ inputs = self._team_inputs(team)
+ with self._team_obs_lock:
+ if self._team_checked.get(team_id) == inputs["mark"]:
+ return # nothing the checks read has changed since the last run (P3B-5)
+ events, sids, items = inputs["board"], inputs["sids"], inputs["items"]
+ turns = self.recordings_store.team_turns(sids)
found: dict[str, list] = {}
- for item in self.team_store.list_items(team.space, self._user_actor()):
- cov = self.item_coverage(team, item, events)
+ for item in items:
+ cov = self.item_coverage(team, item, events, turns=turns)
if cov is None or not cov["handed_off"]:
continue
for i in cov["issues"]:
@@ -6916,7 +6970,7 @@ class SessionManager:
if i.get("turn_id"):
found.setdefault(i["turn_id"], []).append(i)
links = []
- for item in self.team_store.list_items(team.space, self._user_actor()):
+ for item in items:
for link in item.get("links") or []:
if link.get("kind") == "blocked_by":
links.append((int(link["item"]), int(item["id"])))
@@ -6931,8 +6985,12 @@ class SessionManager:
if step and step.get("turn_id"):
i.update({"step_id": step["step_id"], "step_seq": step["seq"], "turn_id": step["turn_id"]})
found.setdefault(step["turn_id"], []).append(i)
- turn_ids = [t["turn_id"] for t in self.recordings_store.team_turns(sids)]
+ turn_ids = [t["turn_id"] for t in turns]
self._store_scope_issues(turn_ids, found, list(_obs_scopes.TASK_CHECKS))
+ # Storing the issues bumps the members' write counts: the mark to compare next time is
+ # the one after this run's own writes.
+ with self._team_obs_lock:
+ self._team_checked[team_id] = self._team_inputs(team)["mark"]
# -- observability Phase 3: rollups over many turns (E24b, E24c, E26b, E27a) ---------------
def usage_names(self) -> dict[str, str]:
diff --git a/tests/test_observability_phase3_review_fixes.py b/tests/test_observability_phase3_review_fixes.py
new file mode 100644
index 00000000..5827a1b7
--- /dev/null
+++ b/tests/test_observability_phase3_review_fixes.py
@@ -0,0 +1,221 @@
+"""Phase 3 code review before merging (4 Nov), one test per fix: the rating export keeps moving
+past 100 (P3B-2), the same approval asked twice points at the repeat in both views (P3B-4), a
+team's numbers are read once and kept until something changes (P3B-5), a ledger error keeps the
+recorded steps (P3B-6), and a resumed automation turn is the run's (P3B-7). P3B-3 (the offline
+machine's cached Usage answer) is in test_remote_home.py."""
+
+from __future__ import annotations
+
+import asyncio
+import json
+from types import SimpleNamespace
+
+from coworker.observability import RecordingStore, rollups, scopes, signals, usage_api
+from coworker.observability import export as ex
+from coworker.server.manager import SessionManager
+from coworker.sessions import SessionRecord
+from coworker.teams.model import Actor, Role
+from coworker.teams.registry import TeamWorker
+
+
+def _turn(tid, sid, seq, at, *, automation=None, status="completed"):
+ t = {"turn_id": tid, "session_id": sid, "seq": seq, "kind": "run", "trigger": "user", "started_at": at,
+ "duration_ms": 1000, "status": status, "incomplete": False, "approval_mode": "interactive",
+ "attended": True, "reviewer": None, "stamp": {"machine": {"kind": "this_machine", "id": None}},
+ "first_message_id": None, "last_message_id": None, "schema_version": 1, "wake": {"reason": "user"}}
+ if automation:
+ t["automation"] = automation
+ return t
+
+
+def _step(tid, sid, n, at, *, kind="model", name="model", call_id=None, attrs=None):
+ return {"step_id": f"{tid}-s{n}", "turn_id": tid, "session_id": sid, "parent_step_id": None, "seq": n,
+ "kind": kind, "name": name, "started_at": at, "duration_ms": 100, "status": "ok",
+ "call_id": call_id, "call_index": None, "message_id": None, "shadow": False, "after_turn": False,
+ "attrs": attrs if attrs is not None else
+ {"purpose": "main", "model": "model-x",
+ "usage": {"reported": True, "input_fresh": 10, "cache_read": 0, "cache_write": 0, "output": 1}},
+ "content": {}}
+
+
+def _write(store, turns, steps):
+ for t in turns:
+ store.write_turn(t)
+ for s in steps:
+ store.write_step(s)
+ store.flush()
+
+
+# -- P3B-2 ----------------------------------------------------------------------------------------
+def test_ratings_keep_going_out_after_the_first_100(tmp_path):
+ store = RecordingStore(tmp_path / "recordings.db", background=False)
+ for i in range(130):
+ store.put_feedback({"turn_id": f"T{i:03d}", "session_id": "s", "rating": "up", "reasons": [],
+ "note": None, "auto_approved": []})
+ sent = []
+
+ def post(url, body, headers, timeout):
+ sent.append(json.loads(body)["traceId"])
+ return 200, ""
+
+ cfg = ex.ExportConfig(destination=ex.Destination(kind="langfuse", host="http://lf.test"), headers={},
+ include={}, since_ms=0)
+ exp = ex.Exporter(store, cfg, post=post, secret_values=lambda: [])
+ exp.flush_once()
+ assert len(sent) == 100 # one batch
+ exp.flush_once()
+ assert len(sent) == 130 and len(set(sent)) == 130 # the next batch is the other 30, not the first 100 again
+ exp.flush_once()
+ assert len(sent) == 130 # nothing left
+ # a rating changed after all that still goes out
+ store.put_feedback({"turn_id": "T000", "session_id": "s", "rating": "down", "reasons": [],
+ "note": None, "auto_approved": []})
+ exp.flush_once()
+ assert len(sent) == 131
+ store.close()
+
+
+# -- P3B-4 ----------------------------------------------------------------------------------------
+def _approval(store, tid, sid, seq, at, call_id, args_hash, outcome):
+ """A turn with one tool step and its approval card (wait step)."""
+ _write(store, [_turn(tid, sid, seq, at)], [
+ _step(tid, sid, 1, at, kind="tool", name="run_shell", call_id=call_id, attrs={"args_hash": args_hash}),
+ _step(tid, sid, 2, at + 1, kind="wait", name="approval",
+ attrs={"kind": "approval", "for_call_id": call_id, "outcome": outcome}),
+ ])
+
+
+def test_the_same_approval_twice_points_at_the_repeat_in_both_views(tmp_path):
+ store = RecordingStore(tmp_path / "recordings.db", background=False)
+ _approval(store, "L1", "lead-s", 1, 1_000, "c1", "h-same", "approved")
+ # maya's first card is for the same tool with other arguments: not a repeat
+ _approval(store, "M1", "maya-s", 1, 2_000, "c2", "h-other", "denied")
+ # her second card asks again what the lead already approved: the repeat
+ _approval(store, "M2", "maya-s", 2, 3_000, "c3", "h-same", "approved")
+ cards = store.team_approval_cards(["lead-s", "maya-s"])
+ [issue] = scopes.same_approval_issues(cards)
+ assert issue["turn_id"] == "M2" and issue["step_id"] == "M2-s2"
+ events = usage_api.team_events(store, lead_session="lead-s", worker_sessions={"maya": "maya-s"},
+ board_events=[], links=[])
+ [ev] = [e for e in events if e["kind"] == "same_approval_twice"]
+ assert ev["turn_id"] == issue["turn_id"] == "M2"
+ # the golden check's own answer is unchanged
+ assert scopes.same_approval_twice(cards) == [{"tool": "run_shell", "first_by": "lead-s", "again_by": "maya-s"}]
+ store.close()
+
+
+# -- P3B-6 ----------------------------------------------------------------------------------------
+def test_a_ledger_error_keeps_the_recorded_steps(tmp_path, monkeypatch):
+ store = RecordingStore(tmp_path / "recordings.db", background=False)
+ real = rollups.ledger_rows
+
+ def ledger_rows(turn, *args, **kwargs):
+ if turn["turn_id"] == "BAD":
+ raise RuntimeError("ledger bug")
+ return real(turn, *args, **kwargs)
+
+ monkeypatch.setattr(rollups, "ledger_rows", ledger_rows)
+ good, bad = _turn("GOOD", "s1", 1, 1_000), _turn("BAD", "s2", 1, 1_000)
+ for t in (good, bad):
+ store.write_turn(t)
+ store.write_step(_step(t["turn_id"], t["session_id"], 1, 1_000))
+ store.write_issues(t["turn_id"], {"signals_version": signals.version(), "warn": 0, "info": 0,
+ "issues": [], "fine": []}, seen=True)
+ store.flush() # one batch with both turns: the ledger error must not throw it away
+ for sid, tid in (("s1", "GOOD"), ("s2", "BAD")):
+ rec = store.load_recording(sid)
+ assert [t["turn_id"] for t in rec["turns"]] == [tid]
+ assert [s["step_id"] for s in rec["steps"]] == [f"{tid}-s1"]
+ # the other turn's ledger rows were still worked out
+ assert [t["turn_id"] for t in store.usage_turns(0, 2**62)] == ["GOOD"]
+ store.close()
+
+
+# -- P3B-5 ----------------------------------------------------------------------------------------
+def test_a_team_summary_reads_once_and_works_out_again_only_on_a_change(tmp_path, monkeypatch):
+ monkeypatch.setenv("COWORKER_STATE_DIR", str(tmp_path / "state"))
+ manager = SessionManager(workspace=tmp_path / "repo", data_dir=tmp_path / "data")
+ manager.session_store.save(SessionRecord(session_id="lead-session", workspace=str(tmp_path / "repo"),
+ model="sample", mode="interactive", agent="swe-lead"))
+ team = manager.teams.create(space=manager._board_space("lead-session"), lead_session="lead-session",
+ lead_actor="lead",
+ workers=[TeamWorker(actor="sam", persona="swe-worker", session_id="worker-session")])
+ store = manager.recordings_store
+ _write(store, [_turn("W1", "worker-session", 1, 1_000)],
+ [_step("W1", "worker-session", 1, 1_000),
+ _step("W1", "worker-session", 2, 1_100, kind="tool", name="transition", call_id="t1", attrs={})])
+
+ reads = {"team_turns": 0, "load_recording": 0}
+ for name in reads:
+ real = getattr(store, name)
+
+ def counted(*a, _real=real, _name=name, **k):
+ reads[_name] += 1
+ return _real(*a, **k)
+
+ monkeypatch.setattr(store, name, counted)
+ # the hand-off step is one indexed read, not the whole recording
+ assert manager._handoff_step("worker-session", 2_000) == {"step_id": "W1-s2", "seq": 2, "turn_id": "W1"}
+ assert manager._handoff_step("worker-session", 0) is None # before the step
+ assert reads["load_recording"] == 0
+
+ first = manager.team_summary(team.team_id)
+ assert first["wake_explained"] == {"turns": 0, "explained": 0}
+ after_first = reads["team_turns"]
+ assert after_first == 1 # once for the whole summary, not once per item and per gate
+ again = manager.team_summary(team.team_id)
+ assert reads["team_turns"] == after_first # nothing changed: the kept answer
+ assert again["wakes"] == first["wakes"] and again["events"] == first["events"]
+ # a member's new turn is a change
+ _write(store, [_turn("W2", "worker-session", 2, 5_000)], [_step("W2", "worker-session", 1, 5_000)])
+ manager.team_summary(team.team_id)
+ assert reads["team_turns"] == after_first + 1
+ # so is a board change
+ manager.team_store.create_item(team.space, Actor(id="lead", role=Role.LEAD), title="New", criteria="Done")
+ manager.team_summary(team.team_id)
+ assert reads["team_turns"] == after_first + 2
+ # the task checks skip a rerun when nothing changed since the last one
+ manager._scope_team_tasks(team.team_id)
+ n = reads["team_turns"]
+ manager._scope_team_tasks(team.team_id)
+ assert reads["team_turns"] == n
+ manager.observability.close()
+
+
+# -- P3B-7 ----------------------------------------------------------------------------------------
+class _Resumable:
+ def __init__(self):
+ self.kwargs = None
+
+ async def resume(self, **kwargs):
+ self.kwargs = kwargs
+ if False:
+ yield None
+
+
+def _resume(manager, session_id):
+ engine = _Resumable()
+ manager.get_engine = lambda sid, *a, **k: engine
+ manager.save = lambda *a, **k: None
+ asyncio.run(manager._durable_resume(SimpleNamespace(id=None, session_id=session_id, tool_call_id="c1",
+ resolved_by="")))
+ return engine.kwargs
+
+
+def test_a_resumed_automation_turn_is_tagged_with_its_run(tmp_path, monkeypatch):
+ monkeypatch.setenv("COWORKER_STATE_DIR", str(tmp_path / "state"))
+ manager = SessionManager(workspace=tmp_path / "repo", data_dir=tmp_path / "data")
+ store = manager.recordings_store
+ run = {"task_id": "task-1", "run_id": "run-1", "trigger": "schedule"}
+ # the run's own turn stopped at an approval; the app restarted; the card is answered
+ _write(store, [_turn("R1", "__run__run-1", 1, 1_000, automation=run, status="unknown")],
+ [_step("R1", "__run__run-1", 1, 1_000)])
+ got = _resume(manager, "__run__run-1")
+ assert got["automation"] == run and got["wake"]["reason"] == "answer"
+ # a follow-up typed into the run's session later is not the run's, nor is its resume (TD3-23)
+ _write(store, [_turn("R2", "__run__run-1", 2, 9_000, status="unknown")], [_step("R2", "__run__run-1", 1, 9_000)])
+ assert _resume(manager, "__run__run-1")["automation"] is None
+ # an ordinary session's resume carries none
+ _write(store, [_turn("S1", "plain", 1, 1_000, automation=run)], [_step("S1", "plain", 1, 1_000)])
+ assert _resume(manager, "plain")["automation"] is None
+ manager.observability.close()
diff --git a/tests/test_remote_home.py b/tests/test_remote_home.py
index 21458814..82210179 100644
--- a/tests/test_remote_home.py
+++ b/tests/test_remote_home.py
@@ -2503,7 +2503,7 @@ async def test_browser_sealed_managed_grant_lands_on_the_box(tmp_path):
async def test_offline_machine_serves_its_last_usage_rollup_as_of(tmp_path):
"""Observability Phase 3 (E28, TD3-6): the desktop keeps each machine's last Usage rollup
- per grouping; while the machine is offline the proxy answers with it, labelled `cached`
+ per request; while the machine is offline the proxy answers the same request with it, labelled `cached`
with the time it was fetched, never as live numbers."""
async with _Server(_controller(tmp_path)) as server:
join_url = await _arm(server.base)
@@ -2540,6 +2540,18 @@ async def test_offline_machine_serves_its_last_usage_rollup_as_of(tmp_path):
none = await client.get(f"{server.base}/v1/machines/{mid}/p/v1/observability/rollup",
params={"group": "day"})
assert none.status_code == 503
+ # P3B-3: the same grouping for another period or another coworker is another
+ # question; the copy for "from 0" never stands in for it
+ other_period = await client.get(f"{server.base}/v1/machines/{mid}/p/v1/observability/rollup",
+ params={"group": "all", "from": 1000})
+ assert other_period.status_code == 503
+ other_coworker = await client.get(f"{server.base}/v1/machines/{mid}/p/v1/observability/rollup",
+ params={"group": "all", "from": 0, "persona_id": "p-x"})
+ assert other_coworker.status_code == 503
+ # the same question with its parameters in another order still finds its copy
+ reordered = await client.get(f"{server.base}/v1/machines/{mid}/p/v1/observability/rollup",
+ params={"from": 0, "group": "all"})
+ assert reordered.status_code == 200 and reordered.json()["cached"]["as_of"] > 0
async def test_a_joined_machine_stamps_its_turns_with_the_controllers_id(tmp_path, monkeypatch):