Skip to content
Python

fix(observability): a turn on a new event loop kept its model call; Phase 3 open items

Devika Vermaa5ffefe6bf36
diff --git a/coworker/engine.py b/coworker/engine.py
index a9ba7e69..557beb35 100644
--- a/coworker/engine.py
+++ b/coworker/engine.py
@@ -642,6 +642,21 @@ class TurnEngine:
self._interrupt_hooks: list[Callable[[], None]] = list(interrupt_hooks or [])
# -- external controls ------------------------------------------------------
+ def _reset_cancel(self) -> None:
+ """A new turn starts not stopped. The Stop flag is an asyncio.Event, which belongs to
+ the event loop that first waited on it; a turn running on another loop (each test
+ client socket has its own) could not wait on the old one: the wait failed at once,
+ the stream was read as stopped, and the turn ended with an empty answer while its
+ model call went on outside it. So a turn on a new loop gets a new flag."""
+ try:
+ loop = asyncio.get_running_loop()
+ except RuntimeError:
+ loop = None
+ if loop is not None and getattr(self._cancel, "_loop", None) not in (None, loop):
+ self._cancel = asyncio.Event()
+ else:
+ self._cancel.clear()
+
def request_interrupt(self) -> None:
"""Stop the turn as soon as possible, from ANY state: mid-stream (the producer
thread drops the stream between chunks), mid-tool (interrupt hooks kill the
@@ -1050,7 +1065,7 @@ class TurnEngine:
if activity:
message["_activity"] = activity
self._add_message(message)
- self._cancel.clear()
+ self._reset_cancel()
if not self._dangerous_warned:
if self.permissions.mode is Mode.DANGEROUSLY_BYPASS_APPROVALS:
# Once per engine: the name is the warning, and the transcript keeps it.
@@ -1171,7 +1186,7 @@ class TurnEngine:
is the intended recovery path (owner-hit 2026-07-23)."""
if not self._tail_is_retriable_error():
return
- self._cancel.clear()
+ self._reset_cancel()
yield Event(EventType.TURN_START, {"input": ""})
async for event in self._start_sandbox():
yield event
@@ -1244,7 +1259,7 @@ class TurnEngine:
pending = self._unanswered_trailing_tool_calls()
if not pending:
return
- self._cancel.clear()
+ self._reset_cancel()
self._yield_for_wake = False
yield Event(EventType.TURN_START, {"input": "(resumed)"})
async for event in self._handle_tool_calls(pending):
diff --git a/coworker/observability/stamps.py b/coworker/observability/stamps.py
index adf27712..2c2a9f29 100644
--- a/coworker/observability/stamps.py
+++ b/coworker/observability/stamps.py
@@ -38,6 +38,18 @@ def app_version_or_unknown() -> str:
return "unknown" if (not v or v == "0.0.0") else v
+def note_joined(controller: str, machine_id: str, name: Optional[str] = None) -> None:
+ """The controller welcomed this machine: from now on its turns are stamped with the id the
+ controller knows it by (`machines.id`), whatever was read before. So the Usage page's
+ machine rows match the controller's list (Phase 3 V3-17), even for a process that stamped
+ turns before it joined, or one whose state folder is not the default."""
+ global _machine
+ stamp = machine_stamp()
+ with _machine_lock:
+ _machine = {**stamp, "kind": machine_kind(controller), "id": machine_id or None,
+ "name": name or stamp.get("name")}
+
+
def machine_stamp() -> dict[str, Any]:
"""Where this server runs: read once per process."""
global _machine
diff --git a/coworker/observability/store.py b/coworker/observability/store.py
index 5ad83db3..ad2ad0b1 100644
--- a/coworker/observability/store.py
+++ b/coworker/observability/store.py
@@ -864,7 +864,7 @@ class RecordingStore:
row = self._conn.execute("SELECT MIN(started_at) AS m FROM turn_usage").fetchone()
return row["m"]
- def approver_steps(self, start: int, end: int) -> list[dict[str, Any]]:
+ def approver_steps(self, start: int, end: int, *, persona_id: Optional[str] = None) -> list[dict[str, Any]]:
"""The auto-approver checks, and the tool steps a person let through anyway, of turns
that started in the period (the Auto-approve strip, section 7.3). Steps, not the ledger:
the strip reads Phase 1 fields that only steps hold."""
@@ -873,8 +873,11 @@ class RecordingStore:
rows = self._conn.execute(
"SELECT s.kind, s.shadow, s.duration_ms, s.attrs, s.parent_step_id FROM steps s "
"JOIN turns t ON t.turn_id = s.turn_id WHERE t.started_at >= ? AND t.started_at < ? "
- "AND (s.kind = 'auto_approver' OR (s.kind = 'tool' AND s.attrs LIKE '%allow_anyway%'))",
- (start, end)).fetchall()
+ "AND (s.kind = 'auto_approver' OR (s.kind = 'tool' AND s.attrs LIKE '%allow_anyway%'))"
+ # one coworker: by the ledger's coworker id, which stands in for 1.2 turns
+ + (" AND EXISTS (SELECT 1 FROM turn_usage u WHERE u.turn_id = t.turn_id AND u.persona_id = ?)"
+ if persona_id else ""),
+ (start, end, persona_id) if persona_id else (start, end)).fetchall()
return [{"kind": r["kind"], "shadow": bool(r["shadow"]), "duration_ms": r["duration_ms"],
"parent_step_id": r["parent_step_id"], "attrs": json.loads(r["attrs"] or "{}")} for r in rows]
diff --git a/coworker/observability/usage_api.py b/coworker/observability/usage_api.py
index 81ca544c..45030fa2 100644
--- a/coworker/observability/usage_api.py
+++ b/coworker/observability/usage_api.py
@@ -64,11 +64,13 @@ def rollup(store: Any, *, start: int, end: int, group: list[str], days: Optional
def heaviest(store: Any, *, start: int, end: int, limit: int = 20,
- title_of: Optional[Callable[[str], Optional[str]]] = None) -> dict[str, Any]:
+ title_of: Optional[Callable[[str], Optional[str]]] = None,
+ persona_id: Optional[str] = None) -> dict[str, Any]:
"""`GET /v1/observability/turns?order=tokens`: the heaviest turns of the period. Each names
its turn (`seq`, M3-R2), says when it was found open after a crash (M3-R3), and lists its
warning checks. A deleted session's turn has no title and no link."""
- turns = sorted(store.usage_turns(start, end), key=lambda t: (-t["total"], t["started_at"]))[: max(1, min(limit, 100))]
+ where = {"persona_id": persona_id} if persona_id else None
+ turns = sorted(store.usage_turns(start, end, where=where), key=lambda t: (-t["total"], t["started_at"]))[: max(1, min(limit, 100))]
seqs = store.turn_seqs([t["turn_id"] for t in turns]) if turns else {}
out = []
for t in turns:
@@ -91,12 +93,12 @@ def heaviest(store: Any, *, start: int, end: int, limit: int = 20,
ASKED = ("unsure", "context_changed", "could_not_run")
-def approver(store: Any, *, start: int, end: int) -> dict[str, Any]:
+def approver(store: Any, *, start: int, end: int, persona_id: Optional[str] = None) -> dict[str, Any]:
"""`GET /v1/observability/approver`: the Auto-approve strip (§7.3). Live checks by result
(asked you includes could-not-run: the card falls back to asking), overridden tool
steps, the median check time, and the checks' tokens with shadow checks included, as
the pill counts them. One line per reviewer kind used in the period."""
- steps = store.approver_steps(start, end)
+ steps = store.approver_steps(start, end, persona_id=persona_id)
live = [s for s in steps if s["kind"] == "auto_approver" and not s["shadow"] and s["parent_step_id"] is None]
shadow = [s for s in steps if s["kind"] == "auto_approver" and s["shadow"]]
overridden = sum(1 for s in steps if s["kind"] == "tool" and any(
@@ -112,7 +114,8 @@ def approver(store: Any, *, start: int, end: int) -> dict[str, Any]:
by_kind: dict[str, list] = {}
for s in live:
by_kind.setdefault((s["attrs"].get("reviewer") or {}).get("kind") or "unknown", []).append(s)
- tokens = sum(g["total"] for g in store.usage_rollup(["purpose"], start, end)
+ tokens = sum(g["total"] for g in store.usage_rollup(["purpose"], start, end,
+ where={"persona_id": persona_id} if persona_id else None)
if g["key"]["purpose"] == "auto_approver")
return {**counts(live), "overridden": overridden, "tokens": tokens, "shadow_checks": len(shadow),
"by_reviewer": [{"kind": k, **counts(v)} for k, v in sorted(by_kind.items())],
diff --git a/coworker/remote/joiner.py b/coworker/remote/joiner.py
index ac854125..7c0d70c0 100644
--- a/coworker/remote/joiner.py
+++ b/coworker/remote/joiner.py
@@ -509,6 +509,13 @@ async def run_joined(
if on_welcome is not None:
on_welcome(welcome)
on_welcome = None
+ # Turns are stamped with the id the controller knows this machine by.
+ try:
+ from ..observability.stamps import note_joined
+
+ note_joined(controller, str(welcome.get("machine_id") or ""), welcome.get("name"))
+ except Exception: # noqa: BLE001 - a stamp never breaks the link
+ pass
if exporter is None:
exporter = _build_exporter(state, app, welcome)
# The welcome may carry the org policy (hosted controllers);
diff --git a/coworker/server/app.py b/coworker/server/app.py
index af65c9d8..23cfcee1 100644
--- a/coworker/server/app.py
+++ b/coworker/server/app.py
@@ -908,22 +908,23 @@ def create_app(manager: SessionManager) -> FastAPI:
return JSONResponse({"error": str(exc)}, status_code=400)
@app.get("/v1/observability/turns")
- def observability_heaviest(request: Request, to: Optional[int] = None, order: str = "tokens", limit: int = 20):
+ def observability_heaviest(request: Request, to: Optional[int] = None, order: str = "tokens", limit: int = 20,
+ persona_id: Optional[str] = None):
if order != "tokens":
return JSONResponse({"error": "order must be tokens"}, status_code=400)
try:
start, end, _ = _period(request.query_params.get("from"), to, None)
except (ValueError, TypeError) as exc:
return JSONResponse({"error": str(exc)}, status_code=400)
- return manager.usage_heaviest(start=start, end=end, limit=limit)
+ return manager.usage_heaviest(start=start, end=end, limit=limit, persona_id=persona_id)
@app.get("/v1/observability/approver")
- def observability_approver(request: Request, to: Optional[int] = None):
+ def observability_approver(request: Request, to: Optional[int] = None, persona_id: Optional[str] = None):
try:
start, end, _ = _period(request.query_params.get("from"), to, None)
except (ValueError, TypeError) as exc:
return JSONResponse({"error": str(exc)}, status_code=400)
- return manager.usage_approver(start=start, end=end)
+ return manager.usage_approver(start=start, end=end, persona_id=persona_id)
@app.get("/v1/personas/{persona_id}/doing")
def persona_doing(persona_id: str, days: int = 30, to: Optional[int] = None) -> dict[str, Any]:
diff --git a/coworker/server/manager.py b/coworker/server/manager.py
index dea6ed2d..7f59a089 100644
--- a/coworker/server/manager.py
+++ b/coworker/server/manager.py
@@ -2772,6 +2772,8 @@ class SessionManager:
cov[str(item["id"])] = c
out["coverage"] = cov
by_session = {team.lead_session: team.lead_actor, **{w.session_id: w.actor for w in team.workers}}
+ codes = {name: list(sc.get("findings_exit") or []) for name, sc in
+ ((recording_signals.signals().get("scanners") or {}).get("list") or {}).items()}
for it in out.get("items") or []:
c = cov.get(str(it.get("id")))
if c is None:
@@ -2790,7 +2792,10 @@ class SessionManager:
it["findings"] = [{"id": f["seq"], "severity": f.get("severity"), "evidence": f.get("evidence") or [],
"step_call_id": f.get("step_call_id"), "session_id": f.get("session_id"),
"author": f.get("author"), "fixed_by": fix.get("fixed_by"),
- "verified_by": fix.get("verified_by")} for f in c["findings"]]
+ "verified_by": fix.get("verified_by"),
+ # each scanner's "found something" exit codes (signals.yaml),
+ # so the card words a non-zero exit right (M3-R9)
+ "findings_exit": codes} for f in c["findings"]]
except Exception: # noqa: BLE001 - the split never hides the rest of the summary
logger.exception("could not work out team %s wakes", team_id)
out.setdefault("wakes", [])
@@ -6955,14 +6960,15 @@ class SessionManager:
out["automations"] = tasks
return out
- def usage_heaviest(self, *, start: int, end: int, limit: int = 20) -> dict[str, Any]:
+ def usage_heaviest(self, *, start: int, end: int, limit: int = 20,
+ persona_id: Optional[str] = None) -> dict[str, Any]:
out = usage_api.heaviest(self.recordings_store, start=start, end=end, limit=limit,
- title_of=self.session_store.display_title)
+ title_of=self.session_store.display_title, persona_id=persona_id)
out["names"] = self.usage_names()
return out
- def usage_approver(self, *, start: int, end: int) -> dict[str, Any]:
- return usage_api.approver(self.recordings_store, start=start, end=end)
+ def usage_approver(self, *, start: int, end: int, persona_id: Optional[str] = None) -> dict[str, Any]:
+ return usage_api.approver(self.recordings_store, start=start, end=end, persona_id=persona_id)
def current_persona_fp(self, persona_id: str) -> Optional[str]:
"""The coworker's fingerprint now, worked out as the session build does (E16)."""
diff --git a/surfaces/gui/src/api.ts b/surfaces/gui/src/api.ts
index ebe4c67a..39a91eeb 100644
--- a/surfaces/gui/src/api.ts
+++ b/surfaces/gui/src/api.ts
@@ -3064,6 +3064,9 @@ export interface Automation {
// without asking. `entry` is the raw record entry — the revoke handle; `target` is
// null for legacy name-only entries.
always_allowed: { entry: string; tool: string; target: string | null }[];
+ // Phase 3 (E26c): a joined machine's automation, listed beside this machine's.
+ machine_id?: string | null;
+ machine_name?: string;
}
export interface AutomationRun {
@@ -3099,9 +3102,43 @@ export interface FailedInARow {
turn_id?: string | null;
}
+// Phase 3 (E26c, V3-16): automations live on the machine that runs them. The list joins this
+// machine's with each joined machine's (through the controller's proxy), and every
+// automation call goes to the machine its task came from, as sessions do.
+const automationMachines = new Map<string, string>();
+const automationBase = (id: string): string => engineBase(automationMachines.get(id) ?? null);
+export function machineOfAutomation(id: string): string | null {
+ return automationMachines.get(id) ?? null;
+}
+
+async function machineAutomations(m: Machine): Promise<Automation[]> {
+ const ctl = new AbortController();
+ const timer = setTimeout(() => ctl.abort(), 4000); // an offline machine must not hold the list
+ try {
+ const res = await fetch(`${engineBase(m.id)}/v1/automations`, { signal: ctl.signal });
+ if (!res.ok) return [];
+ const tasks: Automation[] = (await res.json()).tasks ?? [];
+ return tasks.map((t) => ({ ...t, machine_id: m.id, machine_name: m.name }));
+ } catch {
+ return [];
+ } finally {
+ clearTimeout(timer);
+ }
+}
+
export async function getAutomations(): Promise<Automation[]> {
const res = await fetch(`${httpBase()}/v1/automations`);
- return (await res.json()).tasks ?? [];
+ const local: Automation[] = (await res.json()).tasks ?? [];
+ let machines: Machine[] = [];
+ try {
+ machines = ((await getMachines()).machines || []).filter((m) => m.connected);
+ } catch {
+ machines = [];
+ }
+ const remote = (await Promise.all(machines.map(machineAutomations))).flat();
+ for (const t of local) automationMachines.delete(t.id);
+ for (const t of remote) if (t.machine_id) automationMachines.set(t.id, t.machine_id);
+ return [...local, ...remote];
}
// Fired after any automation mutation the sidebar should reflect immediately
@@ -3144,7 +3181,7 @@ export function connectEvents(
/** Advance the automation's seen mark — clears its unseen-runs badge (UX-023). */
export async function markAutomationSeen(id: string): Promise<{ ok: boolean }> {
- const res = await fetch(`${httpBase()}/v1/automations/${id}/seen`, { method: "POST" });
+ const res = await fetch(`${automationBase(id)}/v1/automations/${id}/seen`, { method: "POST" });
return res.json();
}
@@ -3167,12 +3204,17 @@ export async function createAutomation(payload: {
}
export async function getAutomation(id: string): Promise<{ task: Automation; runs: AutomationRun[]; failed_in_a_row?: FailedInARow | null }> {
- const res = await fetch(`${httpBase()}/v1/automations/${encodeURIComponent(id)}`);
- return res.json();
+ const res = await fetch(`${automationBase(id)}/v1/automations/${encodeURIComponent(id)}`);
+ const body = await res.json();
+ // A remote automation's run sessions live on its machine: route them there.
+ const mid = automationMachines.get(id) ?? null;
+ for (const r of body.runs || []) if (r.session_id) registerSessionMachine(r.session_id, mid);
+ if (body.task && mid) body.task = { ...body.task, machine_id: mid };
+ return body;
}
export async function updateAutomation(id: string, changes: Record<string, any>) {
- const res = await fetch(`${httpBase()}/v1/automations/${encodeURIComponent(id)}`, {
+ const res = await fetch(`${automationBase(id)}/v1/automations/${encodeURIComponent(id)}`, {
method: "PATCH",
headers: { "Content-Type": "application/json" },
body: JSON.stringify(changes),
@@ -3181,7 +3223,7 @@ export async function updateAutomation(id: string, changes: Record<string, any>)
}
export async function deleteAutomation(id: string) {
- const res = await fetch(`${httpBase()}/v1/automations/${encodeURIComponent(id)}`, { method: "DELETE" });
+ const res = await fetch(`${automationBase(id)}/v1/automations/${encodeURIComponent(id)}`, { method: "DELETE" });
return res.json();
}
@@ -3197,14 +3239,16 @@ export interface PreparedRun {
/** Prepare a live manual run: returns the session to open + the opening prompt to send. */
export async function runAutomation(id: string): Promise<PreparedRun> {
- const res = await fetch(`${httpBase()}/v1/automations/${encodeURIComponent(id)}/run`, { method: "POST" });
- return res.json();
+ const res = await fetch(`${automationBase(id)}/v1/automations/${encodeURIComponent(id)}/run`, { method: "POST" });
+ const body = await res.json();
+ if (body?.session_id) registerSessionMachine(body.session_id, automationMachines.get(id) ?? null);
+ return body;
}
/** Mark a manual run complete after its first turn finished. */
export async function finalizeAutomationRun(id: string, runId: string) {
const res = await fetch(
- `${httpBase()}/v1/automations/${encodeURIComponent(id)}/runs/${encodeURIComponent(runId)}/finalize`,
+ `${automationBase(id)}/v1/automations/${encodeURIComponent(id)}/runs/${encodeURIComponent(runId)}/finalize`,
{ method: "POST" },
);
return res.json();
@@ -4668,8 +4712,10 @@ export interface HeavyTurn {
task_id?: string | null;
}
-export async function getHeaviestTurns(from: number, to: number, machineId?: string | null, limit = 20): Promise<{ turns: HeavyTurn[]; names?: Record<string, string> }> {
+export async function getHeaviestTurns(from: number, to: number, machineId?: string | null, limit = 20,
+ personaId?: string | null): Promise<{ turns: HeavyTurn[]; names?: Record<string, string> }> {
const p = new URLSearchParams({ from: String(from), to: String(to), order: "tokens", limit: String(limit) });
+ if (personaId) p.set("persona_id", personaId);
const res = await fetch(`${engineBase(machineId)}/v1/observability/turns?${p}`);
if (!res.ok) throw new Error(`HTTP ${res.status}`);
return res.json();
@@ -4691,8 +4737,10 @@ export interface ApproverStrip extends ApproverCounts {
shown: boolean;
}
-export async function getApproverStrip(from: number, to: number, machineId?: string | null): Promise<ApproverStrip> {
- const res = await fetch(`${engineBase(machineId)}/v1/observability/approver?from=${from}&to=${to}`);
+export async function getApproverStrip(from: number, to: number, machineId?: string | null,
+ personaId?: string | null): Promise<ApproverStrip> {
+ const who = personaId ? `&persona_id=${encodeURIComponent(personaId)}` : "";
+ const res = await fetch(`${engineBase(machineId)}/v1/observability/approver?from=${from}&to=${to}${who}`);
if (!res.ok) throw new Error(`HTTP ${res.status}`);
return res.json();
}
diff --git a/surfaces/gui/src/automationMachines.test.ts b/surfaces/gui/src/automationMachines.test.ts
new file mode 100644
index 00000000..913c39fc
--- /dev/null
+++ b/surfaces/gui/src/automationMachines.test.ts
@@ -0,0 +1,43 @@
+// Phase 3 (E26c, V3-16): a joined machine's automations are listed beside this machine's,
+// and every call for one goes to the machine it came from, through the controller's proxy.
+import { afterEach, describe, expect, it, vi } from "vitest";
+import { getAutomation, getAutomations, machineOfAutomation, machineOfSession, runAutomation } from "./api";
+
+afterEach(() => vi.unstubAllGlobals());
+
+const task = (id: string) => ({ id, title: id, instructions: "", schedule: "", workspace: "", agent: "cowork",
+ enabled: true, next_run: null, last_run: null, last_status: null, run_count: 0, notify_on_completion: false,
+ always_allowed: [] });
+
+describe("automations on joined machines", () => {
+ it("lists remote automations with their machine and routes their calls there", async () => {
+ const urls: string[] = [];
+ vi.stubGlobal("fetch", vi.fn(async (u: string) => {
+ urls.push(String(u));
+ const url = String(u);
+ let body: any;
+ if (url.endsWith("/v1/machines")) {
+ body = { machines: [{ id: "m1", name: "build-box-1", connected: true }, { id: "m2", name: "off", connected: false }] };
+ } else if (url.includes("/v1/machines/m1/p/v1/automations/task-r/run")) {
+ body = { ok: true, run_id: "run-9", session_id: "__run__run-9", workspace: "", agent: "cowork", prompt: "go" };
+ } else if (url.includes("/v1/machines/m1/p/v1/automations/task-r")) {
+ body = { task: task("task-r"), runs: [{ run_id: "run-1", session_id: "__run__run-1" }] };
+ } else if (url.includes("/v1/machines/m1/p/v1/automations")) {
+ body = { tasks: [task("task-r")] };
+ } else {
+ body = { tasks: [task("task-l")] };
+ }
+ return { ok: true, json: async () => body } as any;
+ }));
+ const tasks = await getAutomations();
+ expect(tasks.map((t) => [t.id, t.machine_name ?? null])).toEqual([["task-l", null], ["task-r", "build-box-1"]]);
+ expect(urls.some((u) => u.includes("/machines/m2/"))).toBe(false); // offline: not asked
+ expect(machineOfAutomation("task-r")).toBe("m1");
+ expect(machineOfAutomation("task-l")).toBeNull();
+ await getAutomation("task-r");
+ expect(urls[urls.length - 1]).toContain("/v1/machines/m1/p/v1/automations/task-r");
+ expect(machineOfSession("__run__run-1")).toBe("m1");
+ await runAutomation("task-r");
+ expect(machineOfSession("__run__run-9")).toBe("m1");
+ });
+});
diff --git a/surfaces/gui/src/components/ScheduledView.tsx b/surfaces/gui/src/components/ScheduledView.tsx
index ee03d458..fde619bb 100644
--- a/surfaces/gui/src/components/ScheduledView.tsx
+++ b/surfaces/gui/src/components/ScheduledView.tsx
@@ -177,7 +177,14 @@ export function ScheduledView({ onOpenRun, onRunNow, initialOpenId, onOpenRunSte
onClick={() => setOpenId(task.id)}
>
<div className="flex items-center justify-between gap-2.5 mb-1">
- <span className="text-ui font-semibold truncate">{task.title}</span>
+ <span className="text-ui font-semibold truncate">
+ {task.title}
+ {task.machine_name && (
+ <span className="text-meta text-muted font-normal" data-testid="automation-machine">
+ {" · "}{t("p3.runs.on_machine", { name: task.machine_name })}
+ </span>
+ )}
+ </span>
<button
className="sched-card-del"
title={t("automations.delete_title")}
diff --git a/surfaces/gui/src/components/TeamPhase3.test.tsx b/surfaces/gui/src/components/TeamPhase3.test.tsx
index 5b579ca4..82bbd5b8 100644
--- a/surfaces/gui/src/components/TeamPhase3.test.tsx
+++ b/surfaces/gui/src/components/TeamPhase3.test.tsx
@@ -3,7 +3,7 @@ import { afterEach, describe, expect, it, vi } from "vitest";
import { cleanup, fireEvent, render, screen } from "@testing-library/react";
import i18n from "i18next";
import type { TeamWake } from "../teamView";
-import { CoverageBlock, eventText, exitText, TeamEvents, WakeSplit } from "./TeamPhase3";
+import { CoverageBlock, eventText, exitText, foundSomething, TeamEvents, WakeSplit } from "./TeamPhase3";
afterEach(cleanup);
const t = i18n.t.bind(i18n) as any;
@@ -63,3 +63,15 @@ describe("CoverageBlock", () => {
expect(text).toContain("no secret problems were found");
});
});
+
+describe("foundSomething", () => {
+ it("reads each scanner's own codes: gitleaks exit 1 found something, trivy exit 1 failed (M3-R9)", () => {
+ const codes = { gitleaks: [1], "npm-audit": [1], trivy: [] };
+ expect(foundSomething("gitleaks", 1, codes)).toBe(true);
+ expect(foundSomething("trivy", 1, codes)).toBe(false);
+ expect(foundSomething("gitleaks", 126, codes)).toBe(false);
+ expect(foundSomething("gitleaks", 0, codes)).toBe(false);
+ expect(foundSomething(undefined, 1, codes)).toBe(false);
+ expect(foundSomething("gitleaks", 1, undefined)).toBe(false);
+ });
+});
diff --git a/surfaces/gui/src/components/TeamPhase3.tsx b/surfaces/gui/src/components/TeamPhase3.tsx
index bd95d0f6..a9eb4914 100644
--- a/surfaces/gui/src/components/TeamPhase3.tsx
+++ b/surfaces/gui/src/components/TeamPhase3.tsx
@@ -113,6 +113,14 @@ export function TeamEvents({ events, t, onOpenSteps, onOpenBoard }: {
// -- OB-16 ---------------------------------------------------------------------------------------
+/** Whether a scanner's non-zero exit means "found something": its own codes from signals.yaml
+ * (`findings_exit`), as the engine sends them. With no codes for it, it is a failure. */
+export function foundSomething(scanner: string | undefined, code: number | null | undefined,
+ codes?: Record<string, number[]>): boolean {
+ if (!scanner || code == null || code === 0) return false;
+ return (codes?.[scanner] || []).includes(code);
+}
+
/** "exit 1 (found something)" when the scanner's code means it found something (M3-R9). */
export function exitText(s: { exit_code: number | null; found_something?: boolean }, t: T): string {
if (s.exit_code == null) return t("p3.cov.no_exit");
@@ -186,7 +194,7 @@ export function EvidenceBlock({ finding, t, onOpenStep }: { finding: TaskFinding
<div className="flex-1 min-w-0">
<b className="font-medium">{t("p3.ev2.found_by", { turn: turn.seq, step: step.seq ?? "?" })}</b>
<div className="font-mono text-muted">
- {[sc || step.name, a.exit_code != null ? exitText({ exit_code: a.exit_code, found_something: a.exit_code === 1 && !!sc }, t) : null,
+ {[sc || step.name, a.exit_code != null ? exitText({ exit_code: a.exit_code, found_something: foundSomething(sc, a.exit_code, finding.findings_exit) }, t) : null,
a.result_sha256 ? `output sha256:${String(a.result_sha256).slice(0, 4)}…${String(a.result_sha256).slice(-2)}` : null,
step.duration_ms != null ? formatDuration(step.duration_ms) : null].filter(Boolean).join(" · ")}
</div>
diff --git a/surfaces/gui/src/components/UsagePage.test.tsx b/surfaces/gui/src/components/UsagePage.test.tsx
index c81d8b01..c490328c 100644
--- a/surfaces/gui/src/components/UsagePage.test.tsx
+++ b/surfaces/gui/src/components/UsagePage.test.tsx
@@ -2,7 +2,7 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import { cleanup, fireEvent, render, screen } from "@testing-library/react";
import i18n from "i18next";
-import { ApproverStripView, coworkerTable, UsageBody, type UsageData } from "./UsagePage";
+import { ApproverStripView, coworkerChoices, coworkerTable, loadUsage, UsageBody, type UsageData } from "./UsagePage";
import type { UsageRollup } from "../api";
afterEach(cleanup);
@@ -99,3 +99,31 @@ describe("ApproverStripView", () => {
expect(text).not.toContain("Safety reviewer");
});
});
+
+describe("one coworker", () => {
+ it("lists each coworker of the period once, by name", () => {
+ const r = rollup([group({ persona_id: "sec", task_id: null }, 5), group({ persona_id: "sec", task_id: "t1" }, 1),
+ group({ persona_id: "cowork", task_id: null }, 3), group({ persona_id: null, task_id: null }, 1)],
+ { names: { sec: "Security Coworker", cowork: "OpenWorker" } });
+ expect(coworkerChoices(r)).toEqual([{ id: "cowork", name: "OpenWorker" }, { id: "sec", name: "Security Coworker" }]);
+ });
+
+ it("narrows every call but the coworker list to the picked coworker", async () => {
+ const urls: string[] = [];
+ const fetchMock = vi.fn(async (u: string) => {
+ urls.push(String(u));
+ const body = String(u).includes("/machines") ? { machines: [] } : String(u).includes("/turns") ? { turns: [] }
+ : String(u).includes("/approver") ? { shown: false } : { groups: [], no_turn: { calls: 0, total: 0 }, not_checked: 0, kept_from: null };
+ return { ok: true, json: async () => body } as any;
+ });
+ vi.stubGlobal("fetch", fetchMock);
+ await loadUsage("7d", new Date(2026, 10, 4, 12), "sec");
+ vi.unstubAllGlobals();
+ const usage = urls.filter((u) => u.includes("/observability/"));
+ const narrowed = usage.filter((u) => u.includes("persona_id=sec"));
+ const list = usage.filter((u) => u.includes("group=persona_id%2Ctask_id"));
+ expect(list).toHaveLength(1);
+ expect(list[0]).not.toContain("persona_id=sec");
+ expect(narrowed).toHaveLength(usage.length - 1);
+ });
+});
diff --git a/surfaces/gui/src/components/UsagePage.tsx b/surfaces/gui/src/components/UsagePage.tsx
index 7d385afb..a84e9f7b 100644
--- a/surfaces/gui/src/components/UsagePage.tsx
+++ b/surfaces/gui/src/components/UsagePage.tsx
@@ -52,23 +52,37 @@ export interface UsageData {
machines: MachineRow[];
}
+/** The coworkers of the period for the picker: each id once, by its name (M3-R1), A to Z. */
+export function coworkerChoices(r: UsageRollup | null | undefined): { id: string; name: string }[] {
+ const names = r?.names || {};
+ const ids = new Set<string>();
+ for (const g of r?.groups || []) {
+ const id = g.key.persona_id as string | null;
+ if (id) ids.add(id);
+ }
+ return [...ids].map((id) => ({ id, name: names[id] || id })).sort((a, b) => a.name.localeCompare(b.name));
+}
+
const sumAll = (r: UsageRollup | null | undefined): RollupGroup | null =>
r && r.groups.length ? r.groups[0] : null;
/** Everything the page shows, for one period: one call per grouping, each to the machine
* that ran the work. Remote machines answer through the controller's proxy. */
-export async function loadUsage(period: Period, now = new Date()): Promise<UsageData> {
+export async function loadUsage(period: Period, now = new Date(), personaId: string | null = null): Promise<UsageData> {
const days = localDays(period, now);
const from = days[0][1];
const to = days[days.length - 1][2];
const prevDays = previousDays(days);
+ // One coworker, when picked: every number on the page narrows to its turns. The By
+ // coworker table keeps listing all of them, so the picker always has its choices.
+ const who = personaId ? { persona_id: personaId } : {};
const [total, prev, dayPurpose, coworkers, heavy, approver, fleet] = await Promise.all([
- getUsageRollup({ from, to, group: "all" }),
- getUsageRollup({ from: prevDays[0][1], to: prevDays[prevDays.length - 1][2], group: "all" }).catch(() => null),
- getUsageRollup({ from, to, days, group: "day,purpose" }),
+ getUsageRollup({ from, to, group: "all", ...who }),
+ getUsageRollup({ from: prevDays[0][1], to: prevDays[prevDays.length - 1][2], group: "all", ...who }).catch(() => null),
+ getUsageRollup({ from, to, days, group: "day,purpose", ...who }),
getUsageRollup({ from, to, group: "persona_id,task_id" }),
- getHeaviestTurns(from, to).then((r) => r.turns).catch(() => [] as HeavyTurn[]),
- getApproverStrip(from, to).catch(() => null),
+ getHeaviestTurns(from, to, null, 20, personaId).then((r) => r.turns).catch(() => [] as HeavyTurn[]),
+ getApproverStrip(from, to, null, personaId).catch(() => null),
getMachines().then((r) => r.machines || []).catch(() => [] as Machine[]),
]);
const here = sumAll(total);
@@ -78,7 +92,7 @@ export async function loadUsage(period: Period, now = new Date()): Promise<Usage
const remote = await Promise.all(
fleet.map(async (m): Promise<MachineRow> => {
try {
- const r = await getUsageRollup({ from, to, group: "all" }, m.id);
+ const r = await getUsageRollup({ from, to, group: "all", ...who }, m.id);
const g = sumAll(r);
return {
id: m.id, name: m.name, total: g?.total ?? 0, at_least: (g?.unreported ?? 0) > 0,
@@ -95,19 +109,21 @@ export async function loadUsage(period: Period, now = new Date()): Promise<Usage
export function UsagePage({ onOpenTurn }: { onOpenTurn?: (sessionId: string, turnId: string) => void }) {
const { t } = useTranslation();
const [period, setPeriod] = useState<Period>("7d");
+ const [persona, setPersona] = useState<string | null>(null);
const [data, setData] = useState<UsageData | null>(null);
const [error, setError] = useState(false);
useEffect(() => {
let live = true;
setError(false);
- loadUsage(period)
+ loadUsage(period, new Date(), persona)
.then((d) => live && setData(d))
.catch(() => live && setError(true));
return () => {
live = false;
};
- }, [period]);
+ }, [period, persona]);
+ const choices = data ? coworkerChoices(data.coworkers) : [];
return (
<div data-testid="usage-page">
@@ -119,6 +135,19 @@ export function UsagePage({ onOpenTurn }: { onOpenTurn?: (sessionId: string, tur
</button>
))}
</div>
+ <select
+ className="p3-select"
+ data-testid="usage-coworker"
+ aria-label={t("p3.usage.coworker_filter")}
+ value={persona || ""}
+ onChange={(e) => setPersona(e.target.value || null)}
+ >
+ <option value="">{t("p3.usage.all_coworkers")}</option>
+ {choices.map((c) => (
+ <option key={c.id} value={c.id}>{c.name}</option>
+ ))}
+ {persona && !choices.some((c) => c.id === persona) && <option value={persona}>{persona}</option>}
+ </select>
</div>
{error ? (
<div className={CARD + " p-4 text-ui text-muted"}>{t("p3.usage.load_error")}</div>
diff --git a/surfaces/gui/src/locales/en.json b/surfaces/gui/src/locales/en.json
index 5a008180..f6914485 100644
--- a/surfaces/gui/src/locales/en.json
+++ b/surfaces/gui/src/locales/en.json
@@ -3908,6 +3908,8 @@
},
"p3": {
"usage": {
+ "all_coworkers": "All coworkers",
+ "coworker_filter": "Show one coworker",
"tab_log": "Log",
"tab_usage": "Usage",
"sub": "Tokens your coworkers used, by day, coworker and machine. Waiting for you is never counted as work.",
@@ -4131,6 +4133,7 @@
"self_verified": "Verified by {{who}}, the worker that made the fix"
},
"runs": {
+ "on_machine": "on {{name}}",
"failed_in_a_row_one": "Failed {{count}} run in a row.",
"failed_in_a_row_other": "Failed {{count}} runs in a row.",
"last_error": "Last error: {{error}}",
diff --git a/surfaces/gui/src/locales/zh.json b/surfaces/gui/src/locales/zh.json
index fd8533d6..7ec1611a 100644
--- a/surfaces/gui/src/locales/zh.json
+++ b/surfaces/gui/src/locales/zh.json
@@ -3851,6 +3851,8 @@
},
"p3": {
"usage": {
+ "all_coworkers": "所有同事",
+ "coworker_filter": "只看一个同事",
"tab_log": "日志",
"tab_usage": "用量",
"sub": "同事们用了多少令牌,按天、同事和机器分。等你回复的时间从不算作工作。",
@@ -4065,6 +4067,7 @@
"self_verified": "由 {{who}} 验证,也就是修复者本人"
},
"runs": {
+ "on_machine": "在 {{name}} 上",
"failed_in_a_row_other": "连续 {{count}} 次运行失败。",
"last_error": "最近的错误:{{error}}",
"unknown_error": "未知",
diff --git a/surfaces/gui/src/styles.css b/surfaces/gui/src/styles.css
index 5e589a4a..d5ceb078 100644
--- a/surfaces/gui/src/styles.css
+++ b/surfaces/gui/src/styles.css
@@ -2609,6 +2609,7 @@ html[data-theme="dark"] { --bar-model:#4c8dff; --bar-tool:#3fb8ad; --bar-wait:#3
.p3-tabs button, .p3-seg button { padding:4px 12px; border-radius:6px; font-size:var(--fs-ui); color:var(--muted); background:none; border:0; cursor:pointer; }
.p3-tabs button[aria-selected="true"], .p3-seg button[aria-pressed="true"] { background:var(--paper); color:var(--ink); box-shadow:0 0 0 1px var(--line); }
.p3-seg { display:inline-flex; gap:2px; padding:2px; border:1px solid var(--line); border-radius:8px; background:var(--panel); }
+.p3-select { padding:4px 8px; border:1px solid var(--line); border-radius:8px; background:var(--panel); color:var(--ink); font-size:var(--fs-ui); }
.p3-tiles { display:grid; grid-template-columns:repeat(4, minmax(0,1fr)); gap:10px; margin-bottom:12px; }
@media (max-width: 760px) { .p3-tiles { grid-template-columns:repeat(2, minmax(0,1fr)); } .p3-two { grid-template-columns:1fr; } }
.p3-kpi { border:1px solid var(--line); border-radius:12px; background:var(--panel); padding:10px 12px; display:flex; flex-direction:column; gap:2px; min-width:0; }
diff --git a/surfaces/gui/src/teamView.ts b/surfaces/gui/src/teamView.ts
index 5e1a6167..f18b67f7 100644
--- a/surfaces/gui/src/teamView.ts
+++ b/surfaces/gui/src/teamView.ts
@@ -141,6 +141,8 @@ export interface TaskFinding {
step_call_id?: string | null;
fixed_by?: string | null;
verified_by?: string | null;
+ // each scanner's "found something" exit codes (signals.yaml `scanners`)
+ findings_exit?: Record<string, number[]>;
}
export const tokenTotal = (u: TokenKinds) =>
u.input + u.output + u.cache_read + u.cache_write;
diff --git a/tests/test_observability_phase3_engine.py b/tests/test_observability_phase3_engine.py
index 103d74b8..55a6815a 100644
--- a/tests/test_observability_phase3_engine.py
+++ b/tests/test_observability_phase3_engine.py
@@ -371,3 +371,21 @@ def test_the_wake_cap_event_wakes_nobody_and_deferrals_are_recorded(tmp_path):
assert wake["board_seq"] == caps[0]["seq"]
assert manager._team_stamp("w-s") == {"team_id": team.team_id, "role": "worker"}
manager.observability.close()
+
+
+def test_a_turn_on_a_new_event_loop_is_not_read_as_stopped(tmp_path):
+ """Two turns of one engine on two event loops (each test-client socket has its own): the
+ second turn's answer and its model call stay in that turn (gate 1)."""
+ engine, rec = _engine(tmp_path, [AssistantTurn(text="one", usage=TokenUsage(input=10, output=2)),
+ AssistantTurn(text="two", usage=TokenUsage(input=10, output=3))])
+
+ async def go(text):
+ return [ev async for ev in engine.run(text, trigger="user")]
+
+ asyncio.run(go("first"))
+ asyncio.run(go("second"))
+ d = rec.as_dict()
+ assert [m.get("content") for m in engine.messages if m.get("role") == "assistant"] == ["one", "two"]
+ models = [s for s in d["steps"] if s["kind"] == "model"]
+ assert len(models) == 2 and all(s["turn_id"] for s in models)
+ assert {s["turn_id"] for s in models} == {t["turn_id"] for t in d["turns"]}
diff --git a/tests/test_observability_phase3_server.py b/tests/test_observability_phase3_server.py
index 86080020..d2465240 100644
--- a/tests/test_observability_phase3_server.py
+++ b/tests/test_observability_phase3_server.py
@@ -59,14 +59,18 @@ def test_a_user_turn_records_wake_and_persona_and_validates_as_1_3(tmp_path):
def test_the_ledger_equals_the_pill_and_the_endpoints_add_up(tmp_path):
- # One turn per session: two turns sent back to back over the test client sometimes record
- # the second turn's end before its model call (seen on the base branch too; a Phase 1
- # finding in the build notes), which would leave that call outside every turn.
+ # Two turns back to back in one session: each test-client socket runs on its own event
+ # loop, which once left the second turn's model call outside its turn (gate 1 broke).
manager, client = _app(tmp_path)
- _turn(client, "s1")
+ for _ in range(3):
+ _turn(client, "s1")
+ _turn(client, "s1", "again")
_turn(client, "s2")
_turn(client, "s3")
pills = {sid: client.get(f"/v1/sessions/{sid}/usage").json()["pill"] for sid in ("s1", "s2", "s3")}
+ stray = manager.recordings_store._conn.execute(
+ "SELECT COUNT(*) FROM steps WHERE kind = 'model' AND turn_id IS NULL").fetchone()[0]
+ assert stray == 0
t0 = min(t["started_at"] for t in manager.recordings_store.usage_turns(0, 2**62)) - 1000
days = _days(t0, 2)
r = client.get("/v1/observability/rollup", params={"from": t0, "to": t0 + 2 * DAY, "days": days,
@@ -75,14 +79,14 @@ def test_the_ledger_equals_the_pill_and_the_endpoints_add_up(tmp_path):
assert by_session == pills
total = client.get("/v1/observability/rollup", params={"from": t0, "to": t0 + 2 * DAY}).json()
assert total["groups"][0]["total"] == sum(pills.values())
- assert total["groups"][0]["turns"] == 3 and total["not_checked"] == 0
+ assert total["groups"][0]["turns"] == 8 and total["not_checked"] == 0
assert total["kept_from"] is not None and total["no_turn"] == {"calls": 0, "total": 0}
# day x purpose adds up to the day, and the days to the total (gate 1)
dp = client.get("/v1/observability/rollup", params={"days": days, "group": "day,purpose"}).json()
assert sum(g["total"] for g in dp["groups"]) == sum(pills.values())
heavy = client.get("/v1/observability/turns", params={"from": t0, "to": t0 + 2 * DAY}).json()["turns"]
assert [t["total"] for t in heavy] == sorted((t["total"] for t in heavy), reverse=True)
- assert {t["seq"] for t in heavy} == {1} and len(heavy) == 3
+ assert sorted(t["seq"] for t in heavy if t["session_id"] == "s1") == [1, 2, 3, 4, 5, 6]
assert client.get("/v1/observability/approver", params={"from": t0, "to": t0 + DAY}).json()["shown"] is False
assert client.get("/v1/observability/rollup", params={"group": "day"}).status_code == 400
@@ -138,3 +142,16 @@ def test_the_coworker_page_rows_split_by_fingerprint(tmp_path):
# the version was kept with its name and files
versions = manager.recordings_store.persona_versions(pid)
assert versions and versions[0]["fp"] == row["fp"] and "manifest.md" in versions[0]["files"]
+
+
+def test_the_usage_endpoints_narrow_to_one_coworker(tmp_path):
+ manager, client = _app(tmp_path)
+ _turn(client, "s1")
+ pid = manager.session_store.agent_of("s1")
+ params = {"from": 0}
+ mine = client.get("/v1/observability/rollup", params={**params, "persona_id": pid}).json()
+ other = client.get("/v1/observability/rollup", params={**params, "persona_id": "nobody"}).json()
+ assert mine["groups"][0]["turns"] == 1 and other["groups"] == []
+ assert len(client.get("/v1/observability/turns", params={**params, "persona_id": pid}).json()["turns"]) == 1
+ assert client.get("/v1/observability/turns", params={**params, "persona_id": "nobody"}).json()["turns"] == []
+ assert client.get("/v1/observability/approver", params={**params, "persona_id": "nobody"}).json()["checks"] == 0
diff --git a/tests/test_remote_home.py b/tests/test_remote_home.py
index e9e42241..21458814 100644
--- a/tests/test_remote_home.py
+++ b/tests/test_remote_home.py
@@ -2540,3 +2540,29 @@ async def test_offline_machine_serves_its_last_usage_rollup_as_of(tmp_path):
none = await client.get(f"{server.base}/v1/machines/{mid}/p/v1/observability/rollup",
params={"group": "day"})
assert none.status_code == 503
+
+
+async def test_a_joined_machine_stamps_its_turns_with_the_controllers_id(tmp_path, monkeypatch):
+ """Phase 3 V3-17: the machine id on a joined machine's turns (`stamp.machine.id`) is the id
+ the controller lists it under, so the Usage page's machine rows line up with Machines."""
+ from coworker.observability import stamps
+
+ monkeypatch.setattr(stamps, "_machine", {"kind": "this_machine", "id": None, "name": "box", "app_version": "1"})
+ async with _Server(_controller(tmp_path)) as server:
+ join_url = await _arm(server.base)
+ _, token = parse_join_url(join_url)
+ state = tmp_path / "id-state"
+ state.mkdir()
+ task = asyncio.create_task(
+ run_joined(state=state, controller=server.base, name="id-box", token=token,
+ app=_box_app(tmp_path, "id-data"), once=True, log=lambda *_: None)
+ )
+ try:
+ machines = await _wait_connected(server.base)
+ stamp = stamps.machine_stamp()
+ assert stamp["id"] == machines[0]["id"]
+ assert stamp["kind"] == "own_machine"
+ finally:
+ task.cancel()
+ with pytest.raises(asyncio.CancelledError):
+ await task