Python
fix(observability): Phase 1 and 2 problems found by the Phase 3 design
Devika Verma7610424642aa
diff --git a/coworker/observability/calculator.py b/coworker/observability/calculator.pyindex af55ae5f..674a7f09 100644--- a/coworker/observability/calculator.py+++ b/coworker/observability/calculator.py@@ -58,7 +58,7 @@ def usage(recording: dict[str, Any], session_id: Optional[str] = None,unreported_ids.append(s.get("step_id"))continuetotal += t- p = attrs.get("purpose", "main")+ p = attrs.get("purpose") or "unattributed" # the format's word for a call with no purpose (V3-19)by_purpose[p] = by_purpose.get(p, 0) + tkey = attrs.get("model", "?") + ("" if p == "main" else f" ({p})")by_model[key] = by_model.get(key, 0) + tdiff --git a/coworker/observability/export.py b/coworker/observability/export.pyindex f87318df..0ac5f0b4 100644--- a/coworker/observability/export.py+++ b/coworker/observability/export.py@@ -640,10 +640,13 @@ class ObservabilityExport:_put_profile(self.secrets, {k: v for k, v in held.items() if v})# -- status (§10.5) ----------------------------------------------------------------------------------- def status(self) -> dict[str, Any]:+ def status(self, *, day_start_ms: Optional[int] = None) -> dict[str, Any]:+ """The status card. `day_start_ms`: when the viewer's day began (their midnight, sent+ by the desktop, Phase 3 TD3-2); without it, this machine's own midnight."""dest, _ = self.effective()ex = self.exporter- midnight = time.mktime(time.localtime(self.clock())[:3] + (0, 0, 0, 0, 0, -1)) * 1000+ midnight = day_start_ms if isinstance(day_start_ms, int) and day_start_ms > 0 else \+ time.mktime(time.localtime(self.clock())[:3] + (0, 0, 0, 0, 0, -1)) * 1000try:outside = self.store.outside_calls(since_ms=int(midnight))except Exception: # noqa: BLE001 - the status never fails on this linediff --git a/coworker/observability/read_api.py b/coworker/observability/read_api.pyindex 8fdd0b45..911ab026 100644--- a/coworker/observability/read_api.py+++ b/coworker/observability/read_api.py@@ -167,7 +167,7 @@ def usage(recording: dict[str, Any], session_id: str) -> dict[str, Any]:u = a.get("usage") or {}if s.get("kind") != "model" or not u.get("reported") or a.get("error") or a.get("purpose") == "proxy":continue- p = a.get("purpose", "main")+ p = a.get("purpose") or "unattributed"key = a.get("model", "?") + ("" if p == "main" else f" ({p})")row = parts.setdefault(key, {"input": 0, "cache": 0, "output": 0})for target in (row, totals):diff --git a/coworker/observability/store.py b/coworker/observability/store.pyindex bd1a30a2..c43d4418 100644--- a/coworker/observability/store.py+++ b/coworker/observability/store.py@@ -154,7 +154,10 @@ def _step_row(s: dict[str, Any]) -> tuple:# A call the provider gave no counts for is `usage: {"reported": false}` (§5.2).reported: Optional[int] = int(bool(usage) and usage.get("reported", True) is not False)else:- reported = _bool(attrs.get("usage_reported"))+ # The column is the model call's own usage. An auto-approver check's `usage_reported`+ # (whether its nested call reported) stays in attrs: in the column it read as a call+ # of 0 tokens to any sum over it (Phase 3 V3-06).+ reported = Nonefacts = attrs.get("provider_facts") or {}error = attrs.get("error") if isinstance(attrs.get("error"), dict) else {}return (@@ -617,6 +620,26 @@ class RecordingStore:continuereturn out+ def sessions_needing_check(self, version: str) -> list[str]:+ """Sessions with a finished turn whose Issues result is missing or was made by+ another `signals.yaml` version (P3-4): the ones the start-up re-check works on."""+ self.flush()+ with self._lock:+ rows = self._conn.execute(+ "SELECT session_id, issues FROM turns WHERE status IS NOT NULL AND status != 'running'"+ ).fetchall()+ out: dict[str, None] = {}+ for r in rows:+ if r["session_id"] in out:+ continue+ try:+ have = json.loads(r["issues"]).get("signals_version") if r["issues"] else None+ except (ValueError, AttributeError):+ have = None+ if have != version:+ out[r["session_id"]] = None+ return list(out)+def mark_issues_seen(self, session_id: str) -> None:self.flush()with self._lock, self._conn:diff --git a/coworker/remote/joiner.py b/coworker/remote/joiner.pyindex 47f074ec..ac854125 100644--- a/coworker/remote/joiner.py+++ b/coworker/remote/joiner.py@@ -554,6 +554,7 @@ def _build_local_app(state: Path):cfg = load_config()manager = SessionManager(data_dir=state, model=cfg.model, mode=Mode(cfg.mode))manager.recordings_store.close_open_turns() # the engine lock is held (observability TD-12)+ manager.start_background_checks() # stale manual runs, Issues re-check (P3-3, P3-4)return create_app(manager)diff --git a/coworker/server/app.py b/coworker/server/app.pyindex 9d98bd4c..a26ca162 100644--- a/coworker/server/app.py+++ b/coworker/server/app.py@@ -876,8 +876,9 @@ def create_app(manager: SessionManager) -> FastAPI:return await asyncio.to_thread(manager.observability.send_test)@app.get("/v1/observability/status")- def observability_status() -> dict[str, Any]:- return manager.observability.status()+ def observability_status(day_start_ms: Optional[int] = None) -> dict[str, Any]:+ # "Today" is the viewer's day: the desktop sends when it began (Phase 3 TD3-2, V3-18).+ return manager.observability.status(day_start_ms=day_start_ms)@app.patch("/v1/sessions/{session_id}")def session_patch(session_id: str, body: dict) -> dict[str, Any]:diff --git a/coworker/server/manager.py b/coworker/server/manager.pyindex 5f86cbf4..05951730 100644--- a/coworker/server/manager.py+++ b/coworker/server/manager.py@@ -6565,6 +6565,9 @@ class SessionManager:if message.get("type") in {"turn_start", "iteration_end", "permission_required", "turn_done"}:self.persist_session(session_id)self.reconcile_activity_receipts(session_id)+ if message.get("type") == "turn_done" and session_id.startswith("__run__"):+ # A manual run is done when its turn is, whether or not a window is open (P3-3).+ self._finish_manual_run(session_id)data = message.get("data") or {}if message.get("type") == "team_proposed":message = {**message, "data": {**data, **self.team_card_extras(session_id, data.get("members") or [])}}@@ -8271,6 +8274,69 @@ class SessionManager:),}+ def _finish_manual_run(self, session_id: str) -> None:+ """Close the manual run whose session this is, once its turn ended (P3-3). The desktop+ used to be the only caller of `finalize_manual_run`, so a run whose window closed first+ stayed "running" for good. Calling it twice is harmless."""+ try:+ run = self.task_store.find_run(session_id[len("__run__"):])+ if run is not None and run.trigger == "manual" and run.status == "running":+ self.finalize_manual_run(run.task_id, run.run_id)+ except Exception: # noqa: BLE001 - closing the run must never break the turn's events+ logger.exception("could not close manual run of %s", session_id)++ def finish_stale_manual_runs(self) -> int:+ """At start: close the manual runs an earlier run of the app left "running" (P3-3).+ One whose turn finished is closed as usual; one whose turn the app's stop cut short+ (closed as `unknown` at start) is marked failed. A run with no turn yet is left as is.+ Returns how many were closed."""+ closed = 0+ for task in self.task_store.list():+ for run in self.task_store.runs(task.id):+ if run.trigger != "manual" or run.status != "running":+ continue+ turns = [t for t in (self.recordings_store.load_recording(run.session_id).get("turns") or [])+ if t.get("status") not in (None, "running")]+ if not turns:+ continue+ if turns[-1].get("status") == "unknown":+ run.status, run.error, run.finished_at = "error", "The app stopped before the run finished", _epoch()+ self.task_store.add_run(run)+ task.last_run, task.last_status = run.finished_at, "error"+ task.run_count += 1+ self.task_store.save(task)+ else:+ self.finalize_manual_run(task.id, run.run_id)+ closed += 1+ return closed++ def recheck_issues(self) -> int:+ """Work out the Issues result of every finished turn that has none, or one from an+ older `signals.yaml`, so the sidebar's warnings are right before anyone opens the+ session (P3-4, Phase 3 TD3-8). A result worked out here is stored as seen: an old turn+ never lights the dot. Returns how many sessions were checked."""+ sids = self.recordings_store.sessions_needing_check(recording_signals.version())+ for sid in sids:+ try:+ self._recording_issues(sid, self.recordings_store.load_recording(sid))+ except Exception: # noqa: BLE001 - one bad recording must not stop the rest+ logger.exception("could not re-check issues of %s", sid)+ return len(sids)++ def start_background_checks(self) -> threading.Thread:+ """At start, after the open turns are closed: close stale manual runs and re-check+ Issues, in a background thread so the app is usable at once (P3-3, P3-4)."""+ def work() -> None:+ for job in (self.finish_stale_manual_runs, self.recheck_issues):+ try:+ job()+ except Exception: # noqa: BLE001+ logger.exception("background check %s failed", job.__name__)++ t = threading.Thread(target=work, name="observability-recheck", daemon=True)+ t.start()+ return t+def finalize_manual_run(self, task_id: str, run_id: str) -> dict[str, Any]:"""Mark a manual run complete once its first turn finished (the WS already saved thesession). Pulls result text + artifacts from the persisted transcript/workspace.diff --git a/coworker/server/run.py b/coworker/server/run.pyindex 6741b7fb..2b2010e8 100644--- a/coworker/server/run.py+++ b/coworker/server/run.py@@ -111,6 +111,7 @@ def build_app(workspace: str | None, model: str, mode: str):# (observability TD-12). Only with the engine lock: a second server on the same# folder may have turns running right now.manager.recordings_store.close_open_turns()+ manager.start_background_checks() # stale manual runs, Issues re-check (P3-3, P3-4)return create_app(manager)diff --git a/surfaces/gui/src/api.ts b/surfaces/gui/src/api.tsindex 5e24a0fa..4e37b7d4 100644--- a/surfaces/gui/src/api.ts+++ b/surfaces/gui/src/api.ts@@ -2407,7 +2407,10 @@ export async function setObservabilitySettings(}export async function getObservabilityStatus(machineId?: string | null): Promise<ObservabilityStatus> {- const res = await fetch(`${engineBase(machineId)}/v1/observability/status`);+ // "Today" is your day, even for a machine in another time zone (Phase 3 TD3-2).+ const midnight = new Date();+ midnight.setHours(0, 0, 0, 0);+ const res = await fetch(`${engineBase(machineId)}/v1/observability/status?day_start_ms=${midnight.getTime()}`);return res.json();}diff --git a/surfaces/gui/src/components/RightRail.tsx b/surfaces/gui/src/components/RightRail.tsxindex 1154ec50..19d40767 100644--- a/surfaces/gui/src/components/RightRail.tsx+++ b/surfaces/gui/src/components/RightRail.tsx@@ -361,7 +361,7 @@ export function RightRail({{/* The team panel: who's working, on what, and the way into their sessions —the altitude-3 escape hatch, moved here from the sidebar (RECENT keeps ONEentry per team: the lead). */}- {teamMembers.length > 0 && <TeamRail key={sessionId} members={teamMembers} summary={teamSummary}+ {teamMembers.length > 0 && <TeamRail key={sessionId} leadSessionId={sessionId} members={teamMembers} summary={teamSummary}open={open.team} onToggle={() => setOpen({ ...open, team: !open.team })}usage={teamUsage} machine={teamMachine} chatEnabled={teamChatEnabled} unread={teamChatUnread}onChat={onOpenTeamChat} onWorker={onOpenWorker} onTeam={onOpenTeamView} onWorkers={onOpenWorkers} />}diff --git a/surfaces/gui/src/components/StepsPanel.tsx b/surfaces/gui/src/components/StepsPanel.tsxindex 94a8797d..90e485f5 100644--- a/surfaces/gui/src/components/StepsPanel.tsx+++ b/surfaces/gui/src/components/StepsPanel.tsx@@ -113,7 +113,8 @@ export function stepLabel(step: RecordedStep, t: T): string {case "context":return t("steps.row.context");case "model": {- const purpose = a.purpose && a.purpose !== "main" ? ` · ${t(`steps.purpose.${a.purpose}`, { defaultValue: a.purpose })}` : "";+ const p = a.purpose || "unattributed"; // no purpose: the format's "unattributed" (V3-19)+ const purpose = p !== "main" ? ` · ${t(`steps.purpose.${p}`, { defaultValue: p })}` : "";return t("steps.row.model") + purpose;}case "wait":@@ -267,7 +268,7 @@ function H(t: T, key: string): string {/** The hover help for a step's kind: the row's hover text and its details heading. */function kindHelp(step: { kind: string; name?: string; shadow?: boolean; attrs?: any }, t: T): string {const a = step.attrs || {};- const key = step.kind === "model" ? `kind.model_${a.purpose || "main"}`+ const key = step.kind === "model" ? `kind.model_${a.purpose || "unattributed"}`: step.kind === "auto_approver" ? (step.shadow ? "kind.shadow" : "kind.auto_approver"): step.kind === "summary" ? (step.name === "trim" ? "kind.trim" : "kind.summary"): step.kind === "notice" ? `notice.${step.name}`@@ -983,9 +984,9 @@ function Inspector({ d, turn, steps, sessionId, t }: { d: any; turn: RecordedTur);if (d.kind === "model") {- const mains = steps.filter((s) => s.kind === "model" && (s.attrs?.purpose || "main") === (a.purpose || "main") && !s.parent_step_id);+ const mains = steps.filter((s) => s.kind === "model" && (s.attrs?.purpose || "unattributed") === (a.purpose || "unattributed") && !s.parent_step_id);const n = mains.findIndex((s) => s.step_id === d.step_id) + 1;- const title = (a.purpose || "main") === "main" && n > 0+ const title = a.purpose === "main" && n > 0? t("steps.insp.model_call", { n, total: mains.length }): stepLabel(d, t);const attempts = exchanges.filter((x) => !x.auxiliary).length;@@ -1022,7 +1023,7 @@ function Inspector({ d, turn, steps, sessionId, t }: { d: any; turn: RecordedTur<TokenBar u={u} t={t} /></>)}- {(a.purpose || "main") === "main" && ctxStep && (+ {a.purpose === "main" && ctxStep && (<><Sec label={t("steps.insp.context_added")} help={H(t, "sec.context_added")} /><ContextAdded ctx={ctxStep.attrs} t={t} />diff --git a/surfaces/gui/src/components/TeamRail.test.tsx b/surfaces/gui/src/components/TeamRail.test.tsxindex fab7c839..cf45263d 100644--- a/surfaces/gui/src/components/TeamRail.test.tsx+++ b/surfaces/gui/src/components/TeamRail.test.tsx@@ -1,9 +1,15 @@-import { cleanup, fireEvent, render, screen } from "@testing-library/react";+import { cleanup, fireEvent, render, screen, waitFor } from "@testing-library/react";import { afterEach, expect, it, vi } from "vitest";import { TeamRail } from "./TeamRail";import { sampleTeam } from "../gallery/states/team-view";import type { SessionInfo } from "../types";afterEach(cleanup);+// The team's tokens come from the sessions' recordings when they have one (P3-1).+const recorded: Record<string, any> = {};+vi.mock("../api", async (orig) => ({+ ...(await orig<typeof import("../api")>()),+ getRecordedUsage: vi.fn(async (id: string) => recorded[id] ?? { recorded: false }),+}));const summary = sampleTeam(100);const members = summary.workers.map((w) =>@@ -60,3 +66,14 @@ it("bounds large rosters, hands exact filters to Workers and preserves empty seaexpect(screen.queryAllByTestId(/^team-row-/)).toHaveLength(0);expect(screen.getByRole("status").textContent).toBe("0 matching workers");});++it("counts the team's tokens from the recordings, helpers included (P3-1)", async () => {+ const two = members.slice(0, 2);+ recorded["lead"] = { recorded: true, by_model_parts: { "model-a": { input: 1000, cache: 0, output: 100 }, "model-a (title)": { input: 400, cache: 0, output: 500 } } };+ recorded[two[0].session_id] = { recorded: true, by_model_parts: { "model-a": { input: 1000, cache: 0, output: 0 } } };+ const transcript = { byModel: { "model-a": { input: 10, output: 1, cache_read: 0, cache_write: 0 } }, context: 0 } as any;+ render(+ <TeamRail leadSessionId="lead" members={two} usage={transcript} chatEnabled onChat={vi.fn()} onTeam={vi.fn()} open onToggle={vi.fn()} />,+ );+ await waitFor(() => expect(screen.getByTestId("team-usage").textContent).toContain("3k"));+});diff --git a/surfaces/gui/src/components/TeamRail.tsx b/surfaces/gui/src/components/TeamRail.tsxindex 5ab91c97..a7a78b5f 100644--- a/surfaces/gui/src/components/TeamRail.tsx+++ b/surfaces/gui/src/components/TeamRail.tsx@@ -1,4 +1,4 @@-import { useState } from "react";+import { useEffect, useState } from "react";import { useTranslation } from "react-i18next";import type { SessionInfo, SessionUsage } from "../types";import type { TeamSummary } from "../teamView";@@ -8,7 +8,8 @@ import {type RosterEntry,type WorkerFilter,} from "../teamRoster";-import { formatTokens, totalTokens } from "../usage";+import { formatTokens, totalTokens, usageFromRecordings } from "../usage";+import { getRecordedUsage } from "../api";import { Icon } from "./Icon";export function WorkerSearch({@@ -36,7 +37,8 @@ export function WorkerSearch({export function TeamRail({members,summary,- usage,+ usage: transcriptUsage,+ leadSessionId,machine,chatEnabled,unread = 0,@@ -50,6 +52,8 @@ export function TeamRail({members: SessionInfo[];summary?: TeamSummary | null;usage?: SessionUsage;+ // The lead's own session: its recording is added to the members' (P3-1).+ leadSessionId?: string;machine?: string;chatEnabled: boolean;unread?: number;@@ -60,6 +64,21 @@ export function TeamRail({open: boolean;onToggle: () => void;}) {+ // The team's tokens as the Steps pill counts them, from each session's recording; the+ // transcript's count (main calls only) stays the fallback (P3-1, Phase 3 V3-01).+ const [recorded, setRecorded] = useState<SessionUsage | null>(null);+ const ids = [leadSessionId, ...members.map((m) => m.session_id)].filter(Boolean) as string[];+ const idsKey = ids.join(",");+ useEffect(() => {+ let live = true;+ Promise.all(ids.map((id) => getRecordedUsage(id).catch(() => null)))+ .then((recs) => live && setRecorded(usageFromRecordings(recs)))+ .catch(() => {});+ return () => {+ live = false;+ };+ }, [idsKey, transcriptUsage]); // eslint-disable-line react-hooks/exhaustive-deps+ const usage = recorded ?? transcriptUsage;const { t } = useTranslation();const [chatNotice, setChatNotice] = useState(false),[query, setQuery] = useState("");diff --git a/surfaces/gui/src/usage.test.ts b/surfaces/gui/src/usage.test.tsindex 040ec51e..b6e573ec 100644--- a/surfaces/gui/src/usage.test.ts+++ b/surfaces/gui/src/usage.test.ts@@ -1,5 +1,5 @@import { describe, expect, it } from "vitest";-import { addTurnUsage, emptyUsage, formatTokens, teamUsage, totalTokens, usageFromMessages } from "./usage";+import { addTurnUsage, emptyUsage, formatTokens, teamUsage, totalTokens, usageFromMessages, usageFromRecordings } from "./usage";import type { TurnUsage } from "./types";const turn = (over: Record<string, unknown> = {}) => ({@@ -95,3 +95,20 @@ describe("teamUsage", () => {expect(t.context).toBe(lead.context); // the lead's window, not a sum});});++describe("usageFromRecordings (P3-1)", () => {+ it("adds every recorded call of the team, helpers included", () => {+ const u = usageFromRecordings([+ { recorded: true, by_model_parts: { "model-a": { input: 1000, cache: 400, output: 100 }, "model-a (title)": { input: 50, cache: 0, output: 5 } } },+ { recorded: true, by_model_parts: { "model-b": { input: 200, cache: 0, output: 20 } } },+ null,+ ])!;+ // input already holds the cached part, so it is not added twice+ expect(totalTokens(u)).toBe(1000 + 100 + 50 + 5 + 200 + 20);+ expect(Object.keys(u.byModel).sort()).toEqual(["model-a", "model-b"]);+ });+ it("is null when no session was recorded, so the transcript count stays", () => {+ expect(usageFromRecordings([{ recorded: false }, null])).toBeNull();+ expect(usageFromRecordings([])).toBeNull();+ });+});diff --git a/surfaces/gui/src/usage.ts b/surfaces/gui/src/usage.tsindex 51267a0e..80d2bdcb 100644--- a/surfaces/gui/src/usage.ts+++ b/surfaces/gui/src/usage.ts@@ -54,6 +54,22 @@ export function usageFromMessages(messages: ConversationMessage[]): SessionUsagereturn acc;}+/** A team's tokens from its sessions' recordings (P3-1): every call the Steps pill counts,+ * titles, helpers and auto-approver checks included, which the transcript's per-message+ * counts leave out. `input` here is the call's whole input, so the total adds up the same.+ * Null when none of the sessions was recorded. */+export function usageFromRecordings(recs: any[]): SessionUsage | null {+ const used = (recs || []).filter((r) => r && r.recorded);+ if (!used.length) return null;+ let acc = emptyUsage();+ for (const r of used) {+ for (const [key, p] of Object.entries((r.by_model_parts || {}) as Record<string, any>)) {+ acc = addTurnUsage(acc, { model: key.split(" (")[0], input: p.input, output: p.output, cache_read: 0, cache_write: 0 });+ }+ }+ return { byModel: acc.byModel, context: 0 };+}+/** All tokens consumed this session, across models and directions (chip headline). */export function totalTokens(u: SessionUsage): number {return Object.values(u.byModel).reduce(diff --git a/tests/test_observability_p3_fixes.py b/tests/test_observability_p3_fixes.pynew file mode 100644index 00000000..316d616f--- /dev/null+++ b/tests/test_observability_p3_fixes.py@@ -0,0 +1,141 @@+"""Phase 1 and 2 problems found by the Phase 3 design (observability/FIX-LIST.md, P3-n):+P3-3 a manual run is closed when its turn ends, and at start; P3-4 Issues are re-checked at+start; P3-5 `usage_reported` is a model step's column only; P3-6 "today" is the viewer's day;+P3-7 a model call with no purpose is "unattributed"."""++from __future__ import annotations++import importlib++from coworker.observability import calculator, read_api+from coworker.observability.store import _step_row+from coworker.providers import AssistantTurn, ModelCapabilities, ProviderClient++auto = importlib.import_module("test_automation")+++class _Scripted(ProviderClient):+ def __init__(self, n=4):+ self._turns = [AssistantTurn(text=f"Done {i}.", finish_reason="stop") for i in range(n)]++ def complete(self, *, model, messages, tools=None, **settings):+ return self._turns.pop(0)++ def capabilities(self, model):+ return ModelCapabilities()+++def _manager(tmp_path, monkeypatch):+ from coworker.server.manager import SessionManager++ monkeypatch.setenv("COWORKER_STATE_DIR", str(tmp_path / "state"))+ ws = tmp_path / "ws"+ ws.mkdir(exist_ok=True)+ return SessionManager(data_dir=tmp_path / "data", provider=_Scripted()), ws+++async def _manual_turn(manager, ws):+ task = auto._task(workspace=str(ws), agent="cowork")+ manager.task_store.save(task)+ prep = manager.prepare_manual_run(task.id)+ engine = manager.get_engine(prep["session_id"], workspace=str(ws), agent="cowork")+ async for _ in engine.run(prep["prompt"]):+ pass+ return task, prep+++async def test_a_manual_run_is_closed_when_its_turn_ends(tmp_path, monkeypatch):+ manager, ws = _manager(tmp_path, monkeypatch)+ task, prep = await _manual_turn(manager, ws)+ assert manager.task_store.runs(task.id)[0].status == "running"+ # No window asks for it: the turn's end is enough.+ await manager.broadcast_session(prep["session_id"], {"type": "turn_done", "data": {}})+ run = manager.task_store.runs(task.id)[0]+ assert run.status == "ok" and run.result_text == "Done 0."+ assert manager.task_store.get(task.id).run_count == 1+ # The desktop's own finalize, arriving later, changes nothing.+ manager.finalize_manual_run(task.id, prep["run_id"])+ assert manager.task_store.get(task.id).run_count == 1+++async def test_stale_manual_runs_are_closed_at_start(tmp_path, monkeypatch):+ manager, ws = _manager(tmp_path, monkeypatch)+ done, _ = await _manual_turn(manager, ws)+ cut, cut_prep = await _manual_turn(manager, ws)+ waiting = auto._task(workspace=str(ws), agent="cowork")+ manager.task_store.save(waiting)+ manager.prepare_manual_run(waiting.id) # opened, never run+ store = manager.recordings_store+ store.flush()+ with store._lock, store._conn: # the app stopped mid-turn: start closed it as unknown+ store._conn.execute("UPDATE turns SET status = 'unknown' WHERE session_id = ?", (cut_prep["session_id"],))++ assert manager.finish_stale_manual_runs() == 2+ assert manager.task_store.runs(done.id)[0].status == "ok"+ failed = manager.task_store.runs(cut.id)[0]+ assert failed.status == "error" and failed.error+ assert manager.task_store.get(cut.id).last_status == "error"+ assert manager.task_store.runs(waiting.id)[0].status == "running"+ assert manager.finish_stale_manual_runs() == 0+++async def test_issues_are_rechecked_at_start(tmp_path, monkeypatch):+ from coworker.observability import signals++ manager, ws = _manager(tmp_path, monkeypatch)+ engine = manager.get_engine("s1", workspace=str(ws))+ async for _ in engine.run("hello"):+ pass+ store = manager.recordings_store+ store.flush()+ version = signals.version()+ with store._lock, store._conn: # a turn that ended before checks existed, or an older version+ store._conn.execute("UPDATE turns SET issues = NULL WHERE session_id = 's1'")+ assert store.sessions_needing_check(version) == ["s1"]++ assert manager.recheck_issues() == 1+ store.flush()+ [result] = store.read_issues("s1").values()+ assert result["signals_version"] == version+ assert result["seen"] is True # an old turn's result never lights the sidebar dot+ assert store.sessions_needing_check(version) == []+ assert manager.recheck_issues() == 0+++async def test_background_checks_run_in_a_thread(tmp_path, monkeypatch):+ manager, ws = _manager(tmp_path, monkeypatch)+ task, _ = await _manual_turn(manager, ws)+ t = manager.start_background_checks()+ t.join(timeout=10)+ assert not t.is_alive()+ assert manager.task_store.runs(task.id)[0].status == "ok"+++def test_usage_reported_is_a_model_steps_column_only():+ cols = __import__("coworker.observability.store", fromlist=["_STEP_COLUMNS"])._STEP_COLUMNS+ at = cols.index("usage_reported")+ model = {"step_id": "m", "kind": "model", "attrs": {"usage": {"reported": True, "input_fresh": 5, "output": 1}}}+ check = {"step_id": "a", "kind": "auto_approver", "attrs": {"usage_reported": False}}+ assert _step_row(model)[at] == 1+ assert _step_row(check)[at] is None # it read as a call of 0 tokens to any sum over the column+++def test_a_model_call_with_no_purpose_is_unattributed():+ u = {"reported": True, "input_fresh": 10, "cache_read": 0, "cache_write": 0, "output": 2}+ rec = {"turns": [{"turn_id": "t", "status": "ok"}],+ "steps": [{"step_id": "m", "turn_id": "t", "kind": "model", "status": "ok",+ "attrs": {"model": "model-a", "usage": u}}]}+ assert calculator.usage(rec)["by_purpose"] == {"unattributed": 12}+ assert list(read_api.usage(rec, "s")["by_model_parts"]) == ["model-a (unattributed)"]+++def test_today_is_the_viewers_day(tmp_path, monkeypatch):+ manager, _ = _manager(tmp_path, monkeypatch)+ obs = manager.observability+ seen = []+ monkeypatch.setattr(obs.store, "outside_calls", lambda since_ms: seen.append(since_ms) or {"calls": 0, "tokens": 0})+ obs.status(day_start_ms=1_700_000_000_000)+ obs.status() # no viewer's day: this machine's midnight+ obs.status(day_start_ms=-5) # nonsense: the same+ assert seen[0] == 1_700_000_000_000+ assert seen[1] == seen[2] != seen[0]