Python
feat(observability): Phase 3 evidence links, verifies link, coverage, stored scope checks, rollup cache
Devika Verma36e5a1502c1a
diff --git a/coworker/engine.py b/coworker/engine.pyindex 44f3c597..a9ba7e69 100644--- a/coworker/engine.py+++ b/coworker/engine.py@@ -2999,7 +2999,7 @@ class TurnEngine:)# A tool that starts its own work (the explore helper, E6) records it under this step.outer = set_tool_run(ToolRun(step_id=step_id, turn_id=self._obs.turn_id,- view=self.provider, recording=self._obs))+ view=self.provider, recording=self._obs, call_id=tool_call.id))try:result, status = self._execute_call(tool_call)finally:diff --git a/coworker/observability/engine_hooks.py b/coworker/observability/engine_hooks.pyindex 8ba8a63a..4b90c785 100644--- a/coworker/observability/engine_hooks.py+++ b/coworker/observability/engine_hooks.py@@ -453,8 +453,11 @@ class ToolRun:provider view and recording. Published for the tool's thread by `_execute_sync`."""def __init__(self, *, step_id: Optional[str], turn_id: Optional[str], view: Any,- recording: "TurnRecording") -> None:+ recording: "TurnRecording", call_id: Optional[str] = None) -> None:self.step_id, self.turn_id, self.view, self.recording = step_id, turn_id, view, recording+ # 1.3 (E17a): the tool call's id, which exists whether or not recording is on: the+ # link a journal entry keeps to the step that wrote it.+ self.call_id = call_iddef current_tool_run() -> Optional[ToolRun]:diff --git a/coworker/observability/store.py b/coworker/observability/store.pyindex fc5f28ea..5ad83db3 100644--- a/coworker/observability/store.py+++ b/coworker/observability/store.py@@ -946,6 +946,31 @@ class RecordingStore:"step": {"step_id": w["step_id"], "turn_id": w["turn_id"]}})return out+ def scan_steps(self, session_ids: Any, *, since_ms: int = 0) -> list[dict[str, Any]]:+ """Tool steps whose `scanner.action` is `scan` (1.3), of one or more sessions, oldest+ first: what a finding's evidence and the coverage checks read."""+ ids = [session_ids] if isinstance(session_ids, str) else list(session_ids)+ if not ids:+ return []+ self.flush()+ with self._lock:+ rows = self._conn.execute(+ "SELECT s.step_id, s.turn_id, s.session_id, s.call_id, s.seq, s.started_at, s.status, s.attrs, "+ "t.seq AS turn_seq FROM steps s LEFT JOIN turns t ON t.turn_id = s.turn_id "+ f"WHERE s.kind = 'tool' AND s.session_id IN ({', '.join('?' * len(ids))}) AND s.started_at >= ? "+ "AND s.attrs LIKE '%\"scanner\": {%' ORDER BY s.started_at, s.step_id", [*ids, since_ms]).fetchall()+ out = []+ for r in rows:+ a = json.loads(r["attrs"] or "{}")+ sc = a.get("scanner") or {}+ if sc.get("action") != "scan":+ continue+ out.append({"step_id": r["step_id"], "turn_id": r["turn_id"], "session_id": r["session_id"],+ "call_id": r["call_id"], "seq": r["seq"], "turn_seq": r["turn_seq"],+ "started_at": r["started_at"], "status": r["status"], "scanner": sc,+ "exit_code": a.get("exit_code"), "result_sha256": a.get("result_sha256")})+ return out+def read_turn_issues(self, turn_id: str) -> Optional[dict[str, Any]]:"""One turn's stored Issues result with `seen`, or None."""self.flush()diff --git a/coworker/personas/builtin/appsec-worker/manifest.md b/coworker/personas/builtin/appsec-worker/manifest.mdindex befc462b..cffba982 100644--- a/coworker/personas/builtin/appsec-worker/manifest.md+++ b/coworker/personas/builtin/appsec-worker/manifest.md@@ -11,6 +11,7 @@ team: workertools: [code_files, git, search, shell, todo]connectors: [github]skills: [semgrep-review, security-fix-pr]+coverage: [code]models: [anthropic:claude-opus-4-8, openai:gpt-5.6-sol]default_permission_mode: interactivedescription: An application-security coworker that works team-style — it takes assigned code-review items from a security lead, drives scanners (semgrep), triages findings in context, fixes what matters, and hands off through review with evidence.diff --git a/coworker/personas/builtin/cloud-posture/manifest.md b/coworker/personas/builtin/cloud-posture/manifest.mdindex 5f668f9f..7cf5d2ee 100644--- a/coworker/personas/builtin/cloud-posture/manifest.md+++ b/coworker/personas/builtin/cloud-posture/manifest.md@@ -10,6 +10,7 @@ version: "1"tools: [code_files, git, search, shell, todo]connectors: [github]skills: [iac-scan, aws-posture]+coverage: [iac]models: [anthropic:claude-opus-4-8, openai:gpt-5.6-sol]default_permission_mode: interactivedescription: An infrastructure-security reviewer for teams without a cloud security team. Scans Terraform and cloud configuration with open-source tools (trivy, checkov), reads your live cloud posture strictly read-only, and fixes what matters in the IaC — never by clicking around a console.diff --git a/coworker/personas/builtin/dep-audit/manifest.md b/coworker/personas/builtin/dep-audit/manifest.mdindex ea2f12c4..f2a8054f 100644--- a/coworker/personas/builtin/dep-audit/manifest.md+++ b/coworker/personas/builtin/dep-audit/manifest.md@@ -10,6 +10,7 @@ version: "1"tools: [code_files, git, search, shell, todo]connectors: [github]skills: [dependency-audit, safe-upgrade-pr]+coverage: [deps]models: [anthropic:claude-opus-4-8, openai:gpt-5.6-sol]default_permission_mode: interactivedescription: A dependency auditor for teams without a security team. Runs open-source vulnerability scanners (osv-scanner, npm audit, pip-audit, trivy) across your lockfiles, separates exploitable from theoretical, and ships minimal, test-verified upgrade PRs.diff --git a/coworker/personas/builtin/posture-worker/manifest.md b/coworker/personas/builtin/posture-worker/manifest.mdindex 961a328a..7d14f615 100644--- a/coworker/personas/builtin/posture-worker/manifest.md+++ b/coworker/personas/builtin/posture-worker/manifest.md@@ -11,6 +11,7 @@ team: workertools: [code_files, git, search, shell, todo]connectors: [github]skills: [iac-scan, aws-posture]+coverage: [iac]models: [anthropic:claude-opus-4-8, openai:gpt-5.6-sol]default_permission_mode: interactivedescription: An infrastructure-security coworker that works team-style — it takes assigned posture items from a security lead, scans Terraform and cloud configuration (trivy, checkov; cloud strictly read-only), fixes in the IaC, and hands off through review with evidence.diff --git a/coworker/personas/builtin/secrets-worker/manifest.md b/coworker/personas/builtin/secrets-worker/manifest.mdindex e4aec261..7b86c94b 100644--- a/coworker/personas/builtin/secrets-worker/manifest.md+++ b/coworker/personas/builtin/secrets-worker/manifest.md@@ -10,6 +10,7 @@ version: "1"team: workertools: [code_files, git, search, shell, todo]skills: [secret-scan]+coverage: [secrets]models: [anthropic:claude-opus-4-8, openai:gpt-5.6-sol]default_permission_mode: interactivedescription: A secret-hunting coworker that works team-style — it takes assigned items from a security lead, sweeps working trees and full git history for leaked credentials (gitleaks + manual history reads), verifies what's live, and hands off through review with evidence.diff --git a/coworker/personas/builtin/security/manifest.md b/coworker/personas/builtin/security/manifest.mdindex c33d8b84..5f6bb5c0 100644--- a/coworker/personas/builtin/security/manifest.md+++ b/coworker/personas/builtin/security/manifest.md@@ -10,6 +10,7 @@ version: "1"tools: [code_files, git, search, shell, todo]connectors: [github]skills: [semgrep-review, secret-scan, security-fix-pr]+coverage: [code, secrets]models: [anthropic:claude-opus-4-8, openai:gpt-5.6-sol]default_permission_mode: interactivedescription: A code-security reviewer for teams without a security team. Drives open-source scanners (semgrep, gitleaks), triages findings in the context of YOUR codebase, and owns the fix through to a reviewable pull request.diff --git a/coworker/remote/acceptor.py b/coworker/remote/acceptor.pyindex fe45d81f..145ec4ce 100644--- a/coworker/remote/acceptor.py+++ b/coworker/remote/acceptor.py@@ -814,6 +814,19 @@ def mount_acceptor()return {"ok": True, "revoked": names}+ def _cached_rollup(machine_id: str, grouping: str):+ """An offline machine's last rollup, labelled with when it was fetched (TD3-6). The+ desktop never adds it to the live totals without that label."""+ found = registry.last_rollup(machine_id, grouping)+ if found is None:+ return None+ try:+ body = json.loads(found["payload"])+ except ValueError:+ return None+ body["cached"] = {"as_of": int(found["updated_at"] * 1000), "query": found["query"]}+ return JSONResponse(body, status_code=200)+@app.api_route("/v1/machines/{machine_id}/p/{path:path}",methods=["GET", "POST", "PUT", "PATCH", "DELETE"],@@ -827,8 +840,11 @@ def mount_acceptor(return JSONResponse({"error": "unknown machine"}, status_code=404)_touch(machine_id)live = await _live_or_wake(machine_id)+ rollup = request.method == "GET" and path == "v1/observability/rollup"+ grouping = request.query_params.get("group") or "all"if live is None:- return JSONResponse({"error": "machine offline"}, status_code=503)+ cached = _cached_rollup(machine_id, grouping) if rollup else None+ return cached or JSONResponse({"error": "machine offline"}, status_code=503)target = "/" + pathif request.url.query:target += "?" + request.url.query@@ -847,7 +863,14 @@ def mount_acceptor(except asyncio.TimeoutError:return JSONResponse({"error": "machine timed out"}, status_code=504)except ConnectionError:- return JSONResponse({"error": "machine disconnected"}, status_code=503)+ cached = _cached_rollup(machine_id, grouping) if rollup else None+ return cached or JSONResponse({"error": "machine disconnected"}, status_code=503)+ # Phase 3 (E28): each machine's last Usage rollup, per grouping, for when it is off.+ if rollup and status == 200:+ try:+ registry.save_rollup(machine_id, grouping, request.url.query, payload.decode("utf-8"))+ except Exception:+ pass# Tier-2 snapshot: every successful unfiltered session-list fetch doubles# as the offline fallback (greyed, never vanished). Metadata only.if (diff --git a/coworker/remote/registry.py b/coworker/remote/registry.pyindex 2613296b..facc1e1d 100644--- a/coworker/remote/registry.py+++ b/coworker/remote/registry.py@@ -51,6 +51,21 @@ class MachinesRegistry:# Tier-2 snapshot (remote-home-design.md data path): the last-known# session-LIST METADATA per machine — what keeps an offline machine's# sessions greyed-but-visible instead of vanished.+ # Observability Phase 3 (E28, TD3-6): the last Usage rollup each machine answered,+ # per grouping, so an offline machine still shows its numbers on the Usage page,+ # labelled "as of". Counts only (the ledger is content-free).+ self._conn.execute(+ """+ CREATE TABLE IF NOT EXISTS machine_rollups (+ machine_id TEXT NOT NULL,+ grouping TEXT NOT NULL,+ query TEXT NOT NULL,+ payload TEXT NOT NULL,+ updated_at REAL NOT NULL,+ PRIMARY KEY (machine_id, grouping)+ )+ """+ )self._conn.execute("""CREATE TABLE IF NOT EXISTS machine_sessions (@@ -274,6 +289,25 @@ class MachinesRegistry:).fetchall()return [dict(r) for r in rows]+ def save_rollup(self, machine_id: str, grouping: str, query: str, payload: str) -> None:+ with self._lock:+ self._conn.execute(+ "INSERT INTO machine_rollups (machine_id, grouping, query, payload, updated_at)"+ " VALUES (?, ?, ?, ?, ?) ON CONFLICT(machine_id, grouping) DO UPDATE SET"+ " query = excluded.query, payload = excluded.payload, updated_at = excluded.updated_at",+ (machine_id, grouping, query, payload, time.time()),+ )+ self._conn.commit()++ def last_rollup(self, machine_id: str, grouping: str) -> Optional[dict]:+ """The machine's last Usage rollup for this grouping: {payload, query, updated_at}."""+ with self._lock:+ row = self._conn.execute(+ "SELECT payload, query, updated_at FROM machine_rollups WHERE machine_id = ? AND grouping = ?",+ (machine_id, grouping),+ ).fetchone()+ return dict(row) if row else None+def save_sessions_snapshot(self, machine_id: str, payload: str) -> None:with self._lock:self._conn.execute(diff --git a/coworker/server/manager.py b/coworker/server/manager.pyindex 0b451c19..ffb00a44 100644--- a/coworker/server/manager.py+++ b/coworker/server/manager.py@@ -21,7 +21,7 @@ import sysimport timeimport uuidfrom pathlib import Path-from typing import Any, Optional+from typing import Any, Iterable, Optionalfrom urllib.parse import urlsplitfrom .. import model_config as _model_config@@ -49,6 +49,7 @@ from ..observability import signals as recording_signalsfrom ..observability import rollups as recording_rollupsfrom ..observability import stamps as _obs_stampsfrom ..observability import usage_api+from ..observability import scopes as _obs_scopesfrom ..observability.export import ObservabilityExportfrom ..observability.engine_hooks import note_teamfrom ..observability.provider_view import RecordingProvider, tag as tag_provider@@ -448,6 +449,11 @@ class SessionManager:)self.observability.apply_policy(load_policy(self.secrets.path.parent))self.recorder.add_listener(self._export_on_turn_end)+ # Phase 3: the series and task checks, off the turn's own thread, one at a time.+ import concurrent.futures++ self._scope_pool = concurrent.futures.ThreadPoolExecutor(max_workers=1, thread_name_prefix="obs-scopes")+ self.recorder.add_listener(self._scopes_on_turn_end)# No explicit provider injected → route by the model's `provider:` prefix (OpenAI default,# Ollama, …). Tests inject a provider directly and bypass the router. The same router is# shared by every engine and the `/v1/chat/completions` proxy.@@ -2757,6 +2763,14 @@ class SessionManager:out["events"] = usage_api.team_events(self.recordings_store, lead_session=team.lead_session,worker_sessions=members, board_events=board, links=links)out["wake_explained"] = self._wake_explained(sids)+ # OB-16: "Coverage for this task" on each item a coworker that declares coverage+ # worked on, with its findings' evidence links.+ cov = {}+ for item in self.team_store.list_items(team.space, self._user_actor()):+ c = self.item_coverage(team, item, board)+ if c is not None:+ cov[str(item["id"])] = c+ out["coverage"] = covexcept Exception: # noqa: BLE001 - the split never hides the rest of the summarylogger.exception("could not work out team %s wakes", team_id)out.setdefault("wakes", [])@@ -2948,7 +2962,9 @@ class SessionManager:team_id=team_identity,worker_session=worker_session,) + journal_tools(- self.journal_store, actor=actor, space=space+ self.journal_store, actor=actor, space=space,+ scan_evidence=lambda item, _sid=session_id, _space=space, _actor=actor.id:+ self._scan_evidence(_sid, _space, _actor, item),)from ..teams.artifacts import artifact_tools@@ -6614,6 +6630,14 @@ class SessionManager:out[t["turn_id"]] = havecontinueresult = recording_signals.check_turn(t, steps_of.get(t["turn_id"], []))+ if have is not None:+ # Series and task results are worked out at turn end over more than this+ # turn; a re-check of the turn alone keeps them (Phase 3).+ kept = [i for i in have.get("issues") or [] if i.get("check") in self.SCOPE_CHECKS]+ if kept:+ result["issues"] += kept+ result["warn"] = sum(1 for i in result["issues"] if i.get("severity") == "warn")+ result["info"] = sum(1 for i in result["issues"] if i.get("severity") == "info")seen = True if have is None else bool(have.get("seen"))self.recordings_store.write_issues(t["turn_id"], result, seen=seen)out[t["turn_id"]] = {**result, "seen": seen}@@ -6635,6 +6659,256 @@ class SessionManager:def recording_usage(self, session_id: str) -> dict[str, Any]:return recording_api.usage(self.recordings_store.load_recording(session_id), session_id)+ # -- observability Phase 3: series and task checks (TD3-17, E17e, E18d, E26d) --------------+ # They are pure functions (`observability.scopes`) of more than one turn. At each turn's+ # end the series or task the turn belongs to is worked out again, and every result is+ # stored with the turn whose step it points at, as Phase 2's are, replacing what that+ # series or task said before. So the amber dot and the Issues tab carry them.+ SCOPE_CHECKS = frozenset(_obs_scopes.SERIES_CHECKS + _obs_scopes.TASK_CHECKS)++ def _scopes_on_turn_end(self, event: dict[str, Any]) -> None:+ if event.get("type") != "turn_ended":+ return+ tid = event.get("turn_id")+ sid = event.get("session_id")+ if not tid or not sid:+ return+ try:+ self._scope_pool.submit(self.run_scope_checks, sid, tid)+ except RuntimeError:+ pass # shutting down++ def run_scope_checks(self, session_id: str, turn_id: str) -> None:+ """The series and task checks for the series or task this turn belongs to."""+ try:+ store = self.recordings_store+ store.flush()+ loaded = store.load_turn(turn_id)+ if loaded is None:+ return+ turn, _steps = loaded+ stamp = turn.get("stamp") or {}+ team = stamp.get("team") or {}+ if team.get("role") == "lead":+ self._scope_lead_polling(session_id)+ if team.get("team_id"):+ self._scope_team_tasks(team["team_id"])+ if turn.get("automation"):+ self._scope_failed_in_a_row(turn["automation"]["task_id"])+ if stamp.get("persona_coverage") and not team:+ self._scope_solo_coverage(session_id, list(stamp["persona_coverage"]))+ except Exception: # noqa: BLE001 - a check never breaks anything else+ logger.exception("series and task checks failed for %s", turn_id)++ def _store_scope_issues(self, turn_ids: Iterable[str], found: dict[str, list[dict[str, Any]]],+ checks: Iterable[str]) -> None:+ """Replace, on each of `turn_ids`, the issues of `checks` with what `found` says."""+ checks = set(checks)+ store = self.recordings_store+ for tid in dict.fromkeys(list(turn_ids) + list(found)):+ have = store.read_turn_issues(tid)+ if have is None:+ continue # never checked yet: the re-check will store it with these+ seen = bool(have.pop("seen", False))+ keep = [i for i in have.get("issues") or [] if i.get("check") not in checks]+ new = found.get(tid, [])+ old = [i for i in have.get("issues") or [] if i.get("check") in checks]+ if [(i["check"], i.get("text")) for i in old] == [(i["check"], i.get("text")) for i in new]:+ continue+ result = {**have, "issues": keep + [{k: v for k, v in i.items() if k != "turn_id"} for i in new]}+ result["warn"] = sum(1 for i in result["issues"] if i.get("severity") == "warn")+ result["info"] = sum(1 for i in result["issues"] if i.get("severity") == "info")+ fine = [f for f in result.get("fine") or [] if f.get("check") not in checks]+ result["fine"] = fine+ added = {(i["check"], i.get("text")) for i in new} - {(i["check"], i.get("text")) for i in old}+ store.write_issues(tid, result, seen=seen and not added)++ def _first_step(self, turn_id: str) -> Optional[dict[str, Any]]:+ loaded = self.recordings_store.load_turn(turn_id)+ if loaded is None:+ return None+ top = [s for s in loaded[1] if s.get("parent_step_id") is None and not s.get("shadow")]+ return {**top[0], "turn_id": turn_id} if top else {"step_id": None, "seq": None, "turn_id": turn_id}++ def _last_step(self, turn_id: str) -> Optional[dict[str, Any]]:+ loaded = self.recordings_store.load_turn(turn_id)+ if loaded is None:+ return None+ top = [s for s in loaded[1] if s.get("parent_step_id") is None and not s.get("shadow")+ and not s.get("after_turn")]+ return {**top[-1], "turn_id": turn_id} if top else {"step_id": None, "seq": None, "turn_id": turn_id}++ def _scope_lead_polling(self, lead_session: str) -> None:+ rows = self.recordings_store.team_turns([lead_session])+ series = [{"id": t["turn_id"], "wake": t.get("wake")} for t in rows if t["status"] not in (None, "running")]+ found: dict[str, list] = {}+ for i in _obs_scopes.lead_polling_issues(series, {t["id"]: self._first_step(t["id"]) for t in series[-60:]}):+ found.setdefault(i["turn_id"] or i["facts"]["to"], []).append(i)+ self._store_scope_issues([t["id"] for t in series], found, ["lead_polling"])++ def _scope_failed_in_a_row(self, task_id: str) -> None:+ detail = self.get_automation(task_id)+ banner = detail.get("failed_in_a_row")+ runs = detail.get("runs") or []+ turn_ids = [r["first_turn_id"] for r in runs if r.get("first_turn_id")]+ found: dict[str, list] = {}+ if banner and banner.get("turn_id"):+ series = [{"health": "failed" if r["health"] in ("failed", "did_not_finish") else r["health"],+ "error": r.get("error") or (r.get("check_titles") or [None])[0]}+ for r in sorted(runs, key=lambda x: x.get("started_at") or 0)]+ issue = _obs_scopes.failed_in_a_row_issue(series, self._first_step(banner["turn_id"]))+ if issue is not None:+ found[banner["turn_id"]] = [issue]+ self._store_scope_issues(turn_ids, found, ["failed_in_a_row"])++ def _scope_solo_coverage(self, session_id: str, promised: list[str]) -> None:+ """A coworker working alone: its task is the session. Only the scan checks apply (no+ board, no report, no journal), on the session's last turn."""+ rows = self.recordings_store.team_turns([session_id])+ done = [t for t in rows if t["status"] not in (None, "running")]+ if not done:+ return+ last = done[-1]["turn_id"]+ scans = self.recordings_store.scan_steps(session_id)+ task = {"promised": promised, "steps": [{"id": s["step_id"], "scanner": s["scanner"],+ "exit_code": s["exit_code"]} for s in scans]}+ by_id = {s["step_id"]: s for s in scans}+ issues = [i for i in _obs_scopes.coverage_issues(task, handoff=self._last_step(last), steps_by_id=by_id)+ if i["check"] in ("scan_missing", "scanner_failed")]+ found: dict[str, list] = {}+ for i in issues:+ found.setdefault(i.get("turn_id") or last, []).append(i)+ self._store_scope_issues([t["turn_id"] for t in done], found, ["scan_missing", "scanner_failed"])++ def item_coverage(self, team, item: dict[str, Any], events: list[dict[str, Any]]) -> Optional[dict[str, Any]]:+ """"Coverage for this task" (OB-16, §10): a board item of a worker that declares+ coverage, its scans across the item's sessions, the hand-off's report, its findings+ with their evidence, and the fix it verifies. None when nobody on it promised a scan."""+ by_actor = {w.actor: w.session_id for w in team.workers}+ by_actor[team.lead_actor] = team.lead_session+ mine = [e for e in events if e.get("item_id") == item["id"]]+ assignees = [str((e.get("payload") or {}).get("assignee") or "") for e in mine if e["kind"] == "item_assigned"]+ sessions = [by_actor[a] for a in dict.fromkeys(assignees) if a in by_actor]+ if item.get("assignee") in by_actor and by_actor[item["assignee"]] not in sessions:+ sessions.append(by_actor[item["assignee"]])+ if not sessions:+ return None+ promised: list[str] = []+ for t in self.recordings_store.team_turns(sessions):+ for a in (t.get("stamp") or {}).get("persona_coverage") or []:+ if a not in promised:+ promised.append(a)+ if not promised:+ return None+ since = min([usage_api._ts_ms(e.get("ts")) or 0 for e in mine if e["kind"] == "item_assigned"] or [0])+ scans = self.recordings_store.scan_steps(sessions, since_ms=since)+ handoffs = [e for e in mine if e["kind"] == "item_transitioned" and (e.get("payload") or {}).get("to") in ("review", "done")]+ report = next(((e.get("payload") or {}).get("coverage") for e in reversed(handoffs)+ if (e.get("payload") or {}).get("coverage")), None)+ findings = self._item_findings(item["id"], team.space)+ fix = None+ verifies = [l for l in item.get("links") or [] if l.get("kind") == "verifies"]+ if verifies:+ target = verifies[0]["item"]+ fixed_by = next((e["actor"] for e in reversed(events) if e.get("item_id") == target+ and e["kind"] == "item_transitioned" and (e.get("payload") or {}).get("to") == "review"), None)+ verified_by = next((e["actor"] for e in reversed(handoffs)), item.get("assignee"))+ fix = {"item": target, "fixed_by": fixed_by, "verified_by": verified_by}+ task = {"promised": promised, "report": report, "fix": fix,+ "findings": [{"id": f["seq"], "severity": f.get("severity"), "evidence": f.get("evidence")} for f in findings],+ "steps": [{"id": s["step_id"], "scanner": s["scanner"], "exit_code": s["exit_code"]} for s in scans]}+ handoff = None+ if handoffs:+ h = handoffs[-1]+ sid = by_actor.get(h["actor"])+ handoff = self._handoff_step(sid, usage_api._ts_ms(h.get("ts"))) if sid else None+ issues = _obs_scopes.coverage_issues(task, handoff=handoff, steps_by_id={s["step_id"]: s for s in scans})+ fine = _obs_scopes.coverage_fine(task)+ fired = {i["check"] for i in issues}+ lst = (recording_signals.signals().get("scanners") or {}).get("list") or {}+ return {+ "item": item["id"], "promised": promised, "report": report, "handed_off": bool(handoffs),+ "scans": [{**s, "area": (lst.get(s["scanner"]["name"]) or {}).get("area"),+ "found_something": s["exit_code"] not in (0, None)+ and s["exit_code"] in ((lst.get(s["scanner"]["name"]) or {}).get("findings_exit") or []),+ "failed": s["exit_code"] not in (0, None)+ and s["exit_code"] not in ((lst.get(s["scanner"]["name"]) or {}).get("findings_exit") or [])}+ for s in scans],+ "findings": findings, "fix": fix, "issues": issues,+ "fine": [{"check": c, "text": t} for c, t in fine.items() if c not in fired],+ }++ def _item_findings(self, item_id: int, space: str) -> list[dict[str, Any]]:+ """The journal findings about an item, with the step that wrote each and its evidence+ links (E17a). Read-only, by item and space; the hash chain is not touched."""+ js = self.journal_store+ with js._lock:+ rows = js._conn.execute(+ "SELECT * FROM journal_entries WHERE item_id = ? AND space = ? AND kind = 'finding' ORDER BY seq",+ (item_id, space)).fetchall()+ from ..teams.journal import _row_to_entry++ out = []+ for r in rows:+ e = _row_to_entry(r)+ out.append({"seq": e["seq"], "case": e.get("case_id"), "author": e.get("author"),+ "session_id": e.get("session_id"), "severity": e.get("severity"),+ "entities": e.get("entities") or [], "step_call_id": e.get("step_call_id"),+ "evidence": e.get("evidence"), "ts": e.get("ts")})+ return out++ def _handoff_step(self, session_id: str, at_ms: Optional[int]) -> Optional[dict[str, Any]]:+ """The `transition` tool step that handed the item off: the last one of the session at+ or before the board event's time."""+ rec = self.recordings_store.load_recording(session_id)+ steps = [s for s in rec.get("steps") or [] if s.get("kind") == "tool" and s.get("name") == "transition"+ and (at_ms is None or (s.get("started_at") or 0) <= at_ms + 1000)]+ if not steps:+ return None+ s = steps[-1]+ return {"step_id": s["step_id"], "seq": s.get("seq"), "turn_id": s.get("turn_id")}++ def _scope_team_tasks(self, team_id: str) -> None:+ """A team's task checks: coverage of each handed-off item of a worker that declares+ coverage, the same approval asked twice, and items reviewed before a prerequisite."""+ team = self.teams.get(team_id)+ if team is None:+ return+ from ..teams.summary import all_events++ events = list(all_events(self.team_store, team.space))+ sids = [team.lead_session, *(w.session_id for w in team.workers)]+ found: dict[str, list] = {}+ for item in self.team_store.list_items(team.space, self._user_actor()):+ cov = self.item_coverage(team, item, events)+ if cov is None or not cov["handed_off"]:+ continue+ for i in cov["issues"]:+ if i.get("turn_id"):+ found.setdefault(i["turn_id"], []).append(i)+ cards = self.recordings_store.team_approval_cards(sids)+ for i in _obs_scopes.same_approval_issues(cards):+ if i.get("turn_id"):+ found.setdefault(i["turn_id"], []).append(i)+ links = []+ for item in self.team_store.list_items(team.space, self._user_actor()):+ for link in item.get("links") or []:+ if link.get("kind") == "blocked_by":+ links.append((int(link["item"]), int(item["id"])))+ transitions = [{"seq": e["seq"], "item": e.get("item_id"), "to": (e.get("payload") or {}).get("to"),+ "actor": e.get("actor"), "ts": e.get("ts")} for e in events if e["kind"] == "item_transitioned"]+ by_actor = {w.actor: w.session_id for w in team.workers}+ by_actor[team.lead_actor] = team.lead_session+ for i in _obs_scopes.reviewed_before_prereq_issues(transitions, links):+ e = next((x for x in transitions if x["seq"] == i["facts"]["at_seq"]), None)+ sid = by_actor.get(e["actor"]) if e else None+ step = self._handoff_step(sid, usage_api._ts_ms(e["ts"])) if sid else None+ if step and step.get("turn_id"):+ i.update({"step_id": step["step_id"], "step_seq": step["seq"], "turn_id": step["turn_id"]})+ found.setdefault(step["turn_id"], []).append(i)+ turn_ids = [t["turn_id"] for t in self.recordings_store.team_turns(sids)]+ self._store_scope_issues(turn_ids, found, list(_obs_scopes.TASK_CHECKS))+# -- observability Phase 3: rollups over many turns (E24b, E24c, E26b, E27a) ---------------def usage_names(self) -> dict[str, str]:"""Coworker names by id, for the Usage page: the installed coworkers, else the name@@ -8598,6 +8872,25 @@ class SessionManager:),}+ def _scan_evidence(self, session_id: str, space: str, actor_id: str, item: Optional[int]) -> list[dict[str, str]]:+ """A finding's default evidence (E17a, TD3-15): the successful-or-findings scan steps of+ the same session, run since the item was assigned to this actor (or, with no item, in+ the session), as `[{session_id, call_id}]`. Built differently from the design: every+ scan since assignment, not only those whose area matches the finding's entities+ (entities are free text; matching them to an area would be a guess)."""+ since = 0+ if item is not None and space:+ try:+ for e in self.team_store.events(space, item_id=int(item)):+ if e["kind"] == "item_assigned" and (e.get("payload") or {}).get("assignee") == actor_id:+ since = usage_api._ts_ms(e.get("ts")) or since+ except Exception: # noqa: BLE001+ since = 0+ out = []+ for s in self.recordings_store.scan_steps(session_id, since_ms=since):+ out.append({"session_id": session_id, "call_id": s["call_id"]})+ return out+def _note_persona_version(self, engine: TurnEngine) -> None:"""Keep this coworker version the first time its fingerprint is seen (E25c): name,file list and file copies, for OB-15's "What changed" and "Show diff". Local only."""@@ -8676,11 +8969,29 @@ class SessionManager:logger.exception("could not re-check issues of %s", sid)return len(sids)+ def backfill_usage(self, batch: int = 200) -> int:+ """Ledger rows for finished turns recorded before Phase 3 (TD3-8): worked out from their+ steps, in batches, so the Usage page's numbers reach back as far as the recordings do.+ Returns how many turns were filled."""+ done, last = 0, None+ while True:+ ids = self.recordings_store.turns_without_usage(limit=batch)+ if not ids or ids == last:+ return done # nothing left, or a batch that cannot be written (logged)+ last = ids+ for tid in ids:+ self.recordings_store.refresh_usage(tid)+ self.recordings_store.flush()+ done += len(ids)+ if len(ids) < batch:+ return done+ time.sleep(0.05)+def start_background_checks(self) -> threading.Thread:"""At start, after the open turns are closed: close stale manual runs and re-checkIssues, in a background thread so the app is usable at once (P3-3, P3-4)."""def work() -> None:- for job in (self.finish_stale_manual_runs, self.recheck_issues):+ for job in (self.finish_stale_manual_runs, self.recheck_issues, self.backfill_usage):try:job()except Exception: # noqa: BLE001diff --git a/coworker/teams/journal.py b/coworker/teams/journal.pyindex 1cdc180d..70101313 100644--- a/coworker/teams/journal.py+++ b/coworker/teams/journal.py@@ -98,6 +98,14 @@ class JournalStore:created_ts TEXT NOT NULL);""")+ # Observability Phase 3 (E17a, TD3-15): the tool call that wrote an entry, and the+ # calls that are its evidence ([{session_id, call_id}]). Pointers the app adds, so+ # outside the hash chain, like persona, model and session_id (FO-137 would hash them).+ for column in ("step_call_id TEXT", "evidence TEXT"):+ try:+ self._conn.execute(f"ALTER TABLE journal_entries ADD COLUMN {column}")+ except sqlite3.OperationalError:+ pass # already thereself._conn.commit()# ---------------------------------------------------------------------- verbs@@ -114,6 +122,9 @@ class JournalStore:entities: Optional[list[str]] = None,refs: Optional[list[str]] = None,taint: bool = False,+ severity: Optional[str] = None,+ step_call_id: Optional[str] = None,+ evidence: Optional[list[dict[str, str]]] = None,) -> dict[str, Any]:if not (case or "").strip():raise BoardError("case is required")@@ -145,6 +156,9 @@ class JournalStore:"body": body,"entities": sorted(set(entities or [])),"refs": [str(ref) for ref in refs or []],+ # A finding's severity is the coworker's own claim: in the chain.+ # Only written when given, so older entries hash as before.+ **({"severity": severity} if severity else {}),}),"taint": 1 if taint else 0,@@ -156,8 +170,9 @@ class JournalStore:"""INSERT INTO journal_entries(ts, case_id, kind, actor, actor_role, persona, model,- session_id, space, item_id, payload, taint, prev_hash, hash)- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)+ session_id, space, item_id, payload, taint, prev_hash, hash,+ step_call_id, evidence)+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",(ts,@@ -174,6 +189,8 @@ class JournalStore:record["taint"],prev,record["hash"],+ step_call_id,+ json.dumps(list(evidence or [])) if evidence is not None else None,),)if not exists:@@ -193,7 +210,8 @@ class JournalStore:except Exception:self._conn.rollback()raise- return {**record, "seq": cursor.lastrowid}+ return {**record, "seq": cursor.lastrowid, "step_call_id": step_call_id,+ "evidence": list(evidence) if evidence is not None else None}def read(self,@@ -428,6 +446,13 @@ def _row_to_entry(row: sqlite3.Row) -> dict[str, Any]:entry["body"] = payload.get("body")entry["entities"] = payload.get("entities") or []entry["refs"] = payload.get("refs") or []+ entry["severity"] = payload.get("severity")+ # 1.3 (E17a): pointers outside the chain; None on entries written before them.+ raw = entry.get("evidence")+ try:+ entry["evidence"] = json.loads(raw) if raw else None+ except (TypeError, json.JSONDecodeError):+ entry["evidence"] = Noneentry["author"] = entry.pop("actor")entry["role"] = entry.pop("actor_role")entry["item"] = entry.pop("item_id")diff --git a/coworker/teams/model.py b/coworker/teams/model.pyindex 3f89a34b..a5f640f0 100644--- a/coworker/teams/model.py+++ b/coworker/teams/model.py@@ -61,8 +61,11 @@ class Actor:session_id: str = ""-LINK_KINDS = ("parent", "blocks") # link(src, "parent", dst): dst is src's parent+LINK_KINDS = ("parent", "blocks", "verifies") # link(src, "parent", dst): dst is src's parent# link(src, "blocks", dst): src blocks dst+ # link(src, "verifies", dst): src re-checks dst's fix+ # (observability Phase 3, E17d: who verified a fix is a+ # board fact, not a guess from proposal metadata)# `note` is any observation — the journal is not only for investigations. `raw` is# a capture (log excerpt, command output); reads skip raw unless asked, and largediff --git a/coworker/teams/store.py b/coworker/teams/store.pyindex 7be434fd..6fca76f2 100644--- a/coworker/teams/store.py+++ b/coworker/teams/store.py@@ -574,9 +574,11 @@ class TeamStore:description: str = "",parent: Optional[int] = None,case: Optional[str] = None,+ verifies: Optional[int] = None,) -> dict[str, Any]:"""New item, `open` and unassigned. Acceptance criteria are load-bearing —- required.+ required. `verifies` names the item whose fix this one re-checks (a `verifies`+ link, observability Phase 3 E17d).Workers may create too — a bug spotted in passing, a follow-up — becausefiling is harmless: nothing runs until the item is ASSIGNED, and assign@@ -619,6 +621,12 @@ class TeamStore:"case": case,},)+ if verifies is not None:+ self._item(space, int(verifies))+ if int(verifies) == item_id:+ raise BoardError("an item cannot verify itself")+ event = self.append_event(space, ITEM_LINKED, actor, item_id=item_id,+ payload={"src": item_id, "kind": "verifies", "dst": int(verifies)})if self.journal is not None and case:self.journal.ensure_case(case, actor.id)return self.get_item(space, item_id, actor=actor, seq=event["seq"])@@ -871,8 +879,10 @@ class TeamStore:comment: str = "",refs: Optional[list[str]] = None,taint: bool = False,+ coverage: Optional[dict[str, Any]] = None,) -> dict[str, Any]:target = ItemState(to)+ report = _coverage_report(coverage) if coverage else Nonewith self._lock:item = self._item(space, item_id)current = ItemState(item["state"])@@ -896,6 +906,9 @@ class TeamStore:"to": target.value,"comment": comment,"refs": list(refs or []),+ # Observability Phase 3 (E17c): the hand-off's structured coverage+ # report, the coworker's own claim; only written when given.+ **({"coverage": report} if report else {}),},taint=taint,)@@ -1323,7 +1336,7 @@ class TeamStore:if row["src"] == item_id:out.append({"kind": row["kind"], "item": row["dst"]})else:- inverse = "child" if row["kind"] == "parent" else "blocked_by"+ inverse = {"parent": "child", "verifies": "verified_by"}.get(row["kind"], "blocked_by")out.append({"kind": inverse, "item": row["src"]})return out@@ -1514,3 +1527,28 @@ def text_page(text: str, offset: int, max_chars: int) -> dict:end = min(len(text), offset + max_chars)return {"text": text[offset:end], "offset": offset, "next_offset": end,"total_chars": len(text), "has_more": end < len(text)}+++COVERAGE_AREAS = ("code", "secrets", "deps", "iac")+++def _coverage_report(raw: Any) -> dict[str, dict[str, Any]]:+ """A hand-off's coverage report, `{area: {ran, findings, reason?}}` (observability Phase 3,+ E17c): areas from the fixed list, `ran` a flag, `findings` a count, `reason` a short text+ (why a scan could not run). Anything else is refused, so the checks read only facts."""+ if not isinstance(raw, dict):+ raise BoardError("coverage must be an object of {area: {ran, findings, reason}}")+ out: dict[str, dict[str, Any]] = {}+ for area, value in raw.items():+ if area not in COVERAGE_AREAS:+ raise BoardError(f"coverage area must be one of {list(COVERAGE_AREAS)}, not {area!r}")+ if not isinstance(value, dict) or not isinstance(value.get("ran"), bool):+ raise BoardError(f"coverage[{area!r}] needs ran: true or false")+ findings = value.get("findings", 0)+ if isinstance(findings, bool) or not isinstance(findings, int) or findings < 0:+ raise BoardError(f"coverage[{area!r}].findings must be a count")+ entry: dict[str, Any] = {"ran": value["ran"], "findings": findings}+ if value.get("reason"):+ entry["reason"] = str(value["reason"])[:300]+ out[area] = entry+ return outdiff --git a/coworker/teams/tools.py b/coworker/teams/tools.pyindex efddc3ed..ad7046c4 100644--- a/coworker/teams/tools.py+++ b/coworker/teams/tools.py@@ -73,7 +73,8 @@ _CREATE_ITEM_SCHEMA = {" assigned). `criteria` is the acceptance criteria — what gets verified"" before the item can be done; required. `parent` links it under"" another item; `case` names its journal case (children inherit the"- " parent's case by default)."+ " parent's case by default). `verifies` names the item whose fix this"+ " one re-checks: give it to a different worker than the one who fixed it."),"parameters": {"type": "object",@@ -83,6 +84,7 @@ _CREATE_ITEM_SCHEMA = {"description": {"type": "string"},"parent": {"type": "integer"},"case": {"type": "string"},+ "verifies": {"type": "integer"},},"required": ["title", "criteria"],},@@ -126,6 +128,7 @@ def board_tools(description: str = "",parent: Optional[int] = None,case: str = "",+ verifies: Optional[int] = None,) -> dict:"""Create a work item (open, unassigned — work starts when it isassigned). `criteria` is the acceptance criteria — what gets verified@@ -141,6 +144,7 @@ def board_tools(description=description,parent=parent,case=case or None,+ verifies=verifies,)if "error" not in result:on_change()@@ -202,12 +206,16 @@ def board_tools("authority": "intent_not_access_grants"}def transition(- item: int, to: str, comment: str = "", refs: Optional[list] = None+ item: int, to: str, comment: str = "", refs: Optional[list] = None,+ coverage: Optional[dict] = None,) -> dict:"""Move a work item to a new state. Workers move their own item toin_progress, blocked, or review (attach the blocker or a hand-off summaryas `comment`, and artifact pointers — branch, report, session — as- `refs`); done requires review verification first."""+ `refs`); done requires review verification first. When handing off a+ security task, `coverage` reports each area you were to scan:+ {"secrets": {"ran": false, "findings": 0, "reason": "gitleaks not installed"}},+ areas code, secrets, deps, iac. Say ran: false when a scan could not run."""recorded(item)return receipt(_call(store.transition,@@ -218,6 +226,7 @@ def board_tools(comment=comment,refs=[str(ref) for ref in refs or []],taint=taint(),+ coverage=coverage,))def comment(item: int, body: str, refs: Optional[list] = None, needs_attention: bool = False) -> dict:@@ -292,7 +301,12 @@ def journal_tools(actor: Actor,space: str = "",taint: Callable[[], bool] = lambda: False,+ scan_evidence: Callable[[Optional[int]], list] = lambda item: [],) -> list:+ """`scan_evidence(item)` gives the default evidence of a finding (observability Phase 3,+ TD3-15): the scan steps of this session since the item was assigned, as+ `[{session_id, call_id}]`. It never changes what is written, only the pointer kept."""+def journal_append(case: str,body: str,@@ -300,13 +314,34 @@ def journal_tools(item: Optional[int] = None,entities: Optional[list] = None,refs: Optional[list] = None,+ severity: str = "",+ evidence: Optional[list] = None,) -> dict:"""Append an entry to a journal case as you work: kind is finding,evidence, decision, note (any observation), or raw (a capture like a logexcerpt — for large captures, save the full output to a file and journalan excerpt that references it). `entities` are the concrete things it isabout (file paths, resource names, CVE ids) — they power later recall;- `refs` are pointers (file:line, commit, url)."""+ `refs` are pointers (file:line, commit, url). For a finding, `severity` is+ critical, high, medium, low or info, and `evidence` lists the tool call ids+ whose output shows it (the scan that found it); when you leave it out, the+ scans you ran on this task are linked."""+ from ..observability.engine_hooks import current_tool_run++ run = current_tool_run()+ here = getattr(run, "call_id", None)+ sev = (severity or "").strip().lower() or None+ if sev is not None and sev not in ("critical", "high", "medium", "low", "info"):+ return {"error": "severity must be critical, high, medium, low or info"}+ links = None+ if kind == "finding":+ if evidence:+ links = [{"session_id": actor.session_id or "", "call_id": str(c)} for c in evidence if c]+ else:+ try:+ links = list(scan_evidence(item))+ except Exception: # noqa: BLE001 - a pointer never blocks the entry+ links = []return _call(journal.append,actor,@@ -318,6 +353,9 @@ def journal_tools(entities=[str(entity) for entity in entities or []],refs=[str(ref) for ref in refs or []],taint=taint(),+ severity=sev,+ step_call_id=here,+ evidence=links,)def journal_read(diff --git a/tests/test_observability_phase3_engine.py b/tests/test_observability_phase3_engine.pynew file mode 100644index 00000000..a0909bd8--- /dev/null+++ b/tests/test_observability_phase3_engine.py@@ -0,0 +1,341 @@+"""Phase 3 engine items, one test each: the scanner facts on a real shell step (E17b), the+1.3 turn facts an entry point passes (E18a, E18b, E26a, E25a), the wake reason of a team+delivery (TD3-18), the journal's evidence links (E17a), the `verifies` link (E17d), the+hand-off coverage report (E17c), stored series and task results, the ledger's 90 days and+the feedback export (E27b)."""++from __future__ import annotations++import asyncio+import hashlib+import json+from pathlib import Path++import jsonschema+import pytest++from coworker.engine import TurnEngine+from coworker.observability import Recorder, RecordingStore, scopes, signals+from coworker.observability import export as ex+from coworker.observability import export_spans as spans+from coworker.permissions import Mode, PermissionEngine+from coworker.providers import AssistantTurn, ModelCapabilities, ProviderClient, ToolCall+from coworker.providers.base import TokenUsage+from coworker.server.manager import SessionManager+from coworker.teams.journal import JournalStore+from coworker.teams.model import Actor, Role+from coworker.teams.store import BoardError, TeamStore+from coworker.tools import ToolRegistry++SCHEMA = json.loads((Path(__file__).parent / "corpora/observability/recording-format.schema.json").read_text("utf-8"))+++class _Scripted(ProviderClient):+ def __init__(self, turns):+ self._turns = list(turns)++ def complete(self, *, model, messages, tools=None, **settings):+ return self._turns.pop(0)++ def capabilities(self, model):+ return ModelCapabilities()+++class _Meta:+ category: str = ""+ risk_level: str = "low"+ requires_approval: bool = False+++def run_shell(command: str) -> dict:+ """Run a shell command.++ Args:+ command: the command+ """+ return {"exit_code": 1 if "gitleaks" in command else 0, "stdout": "found 2 leaks"}+++def _call(cid, name, args):+ return AssistantTurn(tool_calls=[ToolCall(id=cid, name=name, arguments=args)], finish_reason="tool_calls",+ usage=TokenUsage(input=10, output=2))+++def _engine(tmp_path, turns):+ registry = ToolRegistry()+ registry.register(run_shell, metadata=_Meta())+ engine = TurnEngine(+ provider=_Scripted(turns), registry=registry,+ permissions=PermissionEngine(workspace_root=tmp_path, mode=Mode.BYPASS_APPROVALS),+ model="openai:model-x", instructions="You are a test coworker.", max_iterations=12,+ )+ engine.audit_context = {"session_id": "sess_p3"}+ engine.persona_id, engine.persona_fp, engine.persona_coverage = "security", "a1b2c3", ["secrets"]+ rec = Recorder()+ engine.recorder = rec+ return engine, rec+++def test_a_shell_step_records_its_scanner_and_result_hash(tmp_path):+ engine, rec = _engine(tmp_path, [+ _call("c1", "run_shell", {"command": "npx gitleaks detect --source ."}),+ _call("c2", "run_shell", {"command": "grep -r semgrep docs"}),+ AssistantTurn(text="done", usage=TokenUsage(input=10, output=2)),+ ])++ async def go():+ return [ev async for ev in engine.run("scan", trigger="team_wake", team={"team_id": "t1", "role": "worker"},+ wake={"reason": "assigned", "item_ids": [7], "board_seq": 41})]++ asyncio.run(go())+ d = rec.as_dict()+ tools = {s["call_id"]: s["attrs"] for s in d["steps"] if s["kind"] == "tool"}+ assert tools["c1"]["scanner"] == {"name": "gitleaks", "action": "scan"} and tools["c1"]["exit_code"] == 1+ assert tools["c2"]["scanner"] is None+ raw = json.dumps({"exit_code": 1, "stdout": "found 2 leaks"}, ensure_ascii=False).encode("utf-8")+ assert tools["c1"]["result_sha256"] == hashlib.sha256(raw).hexdigest()+ turn = d["turns"][0]+ assert turn["stamp"]["persona_id"] == "security" and turn["stamp"]["persona_coverage"] == ["secrets"]+ assert turn["stamp"]["team"] == {"team_id": "t1", "role": "worker"}+ assert turn["wake"] == {"reason": "assigned", "kinds": [], "item_ids": ["7"], "board_seq": 41,+ "batched": None, "deferred": None, "by": None}+ errs = list(jsonschema.Draft202012Validator({**SCHEMA, "$ref": "#/$defs/recording"}).iter_errors(d))+ assert not errs, errs[0].message+++def test_an_automation_turn_carries_its_run_and_a_bad_wake_reason_reads_unknown(tmp_path):+ engine, rec = _engine(tmp_path, [AssistantTurn(text="brief", usage=TokenUsage(input=10, output=2))])++ async def go():+ return [ev async for ev in engine.run("go", trigger="schedule", wake={"reason": "bogus"},+ automation={"task_id": "task-1", "run_id": "run-1", "trigger": "catch_up"})]++ asyncio.run(go())+ turn = rec.as_dict()["turns"][0]+ assert turn["automation"] == {"task_id": "task-1", "run_id": "run-1", "trigger": "catch_up"}+ assert turn["wake"]["reason"] == "unknown" and turn["trigger"] == "schedule"++[email protected]("rows,lead,want", [+ ([{"kind": "moved", "to": "review", "item": 3}, {"kind": "assigned", "item": 4}], True, "review"),+ ([{"kind": "moved", "to": "blocked", "item": 3}, {"kind": "waiting", "item": 4}], True, "blocker"),+ ([{"kind": "chat"}, {"kind": "chat"}], False, "mention"),+ ([{"kind": "moved", "to": "done", "item": 3}], False, "prereq_done"),+ ([{"kind": "moved", "to": "done", "item": 3}], True, "board"),+ ([{"kind": "claimed", "item": 5}, {"kind": "chat"}], True, "assigned"),+])+def test_one_wake_reason_by_fixed_priority(rows, lead, want):+ reason, kinds, items = SessionManager._wake_from_rows(rows, is_lead=lead)+ assert reason == want+ assert kinds == list(dict.fromkeys(r["kind"] for r in rows))+++def _team(tmp_path):+ journal = JournalStore(tmp_path / "journal.db")+ store = TeamStore(tmp_path / "board.db", journal=journal)+ lead = Actor(id="lead:1", role=Role.LEAD, persona="devsecops-lead", session_id="lead-s")+ worker = Actor(id="appsec:1", role=Role.WORKER, persona="appsec-worker", session_id="w-s")+ return store, journal, lead, worker+++def test_verifies_is_a_board_link_both_ways(tmp_path):+ store, _j, lead, _w = _team(tmp_path)+ fix = store.create_item("space", lead, title="Fix SQLi", criteria="query is parameterised")+ check = store.create_item("space", lead, title="Verify the fix", criteria="rerun semgrep", verifies=fix["id"])+ assert {"kind": "verifies", "item": fix["id"]} in check["links"]+ again = store.get_item("space", fix["id"], actor=lead)+ assert {"kind": "verified_by", "item": check["id"]} in again["links"]+ with pytest.raises(BoardError):+ store.create_item("space", lead, title="x", criteria="y", verifies=999)+++def test_a_hand_off_carries_a_checked_coverage_report(tmp_path):+ store, _j, lead, worker = _team(tmp_path)+ item = store.create_item("space", lead, title="Scan", criteria="all areas")+ store.assign("space", lead, item["id"], worker.id)+ store.transition("space", worker, item["id"], "in_progress")+ store.transition("space", worker, item["id"], "review", comment="done",+ coverage={"secrets": {"ran": False, "findings": 0, "reason": "gitleaks missing"},+ "code": {"ran": True, "findings": 3}})+ ev = [e for e in store.events("space", item_id=item["id"]) if e["kind"] == "item_transitioned"][-1]+ assert ev["payload"]["coverage"] == {"secrets": {"ran": False, "findings": 0, "reason": "gitleaks missing"},+ "code": {"ran": True, "findings": 3}}+ for bad in ({"network": {"ran": True}}, {"code": {"ran": "yes"}}, {"code": {"ran": True, "findings": -1}}):+ with pytest.raises(BoardError):+ store.transition("space", worker, item["id"], "in_progress", coverage=bad)+++def test_journal_evidence_is_outside_the_hash_chain(tmp_path):+ _s, journal, lead, worker = _team(tmp_path)+ plain = journal.append(worker, "case-1", "an old-style note")+ found = journal.append(worker, "case-1", "SQLi in login", kind="finding", severity="high",+ step_call_id="call_9", evidence=[{"session_id": "w-s", "call_id": "call_4"}])+ assert found["evidence"] == [{"session_id": "w-s", "call_id": "call_4"}] and found["step_call_id"] == "call_9"+ rows = journal.read(worker, "case-1")+ assert rows[0]["evidence"] is None and rows[0]["severity"] is None+ assert rows[1]["evidence"] == [{"session_id": "w-s", "call_id": "call_4"}] and rows[1]["severity"] == "high"+ # changing the pointer does not touch the chain+ with journal._lock:+ journal._conn.execute("UPDATE journal_entries SET evidence = '[]' WHERE seq = ?", (found["seq"],))+ assert journal.read(worker, "case-1")[1]["hash"] == found["hash"]+ assert plain["hash"] != found["hash"]+++def test_journal_append_links_the_sessions_scans_by_default(tmp_path):+ from coworker.teams.tools import journal_tools++ _s, journal, _lead, worker = _team(tmp_path)+ tools = {t.__name__: t for t in journal_tools(+ journal, actor=worker, space="space",+ scan_evidence=lambda item: [{"session_id": "w-s", "call_id": "call_2"}])}+ out = tools["journal_append"]("case-2", "a leaked key", kind="finding", severity="critical")+ assert out["evidence"] == [{"session_id": "w-s", "call_id": "call_2"}]+ named = tools["journal_append"]("case-2", "another", kind="finding", evidence=["call_7"])+ assert named["evidence"] == [{"session_id": "w-s", "call_id": "call_7"}]+ note = tools["journal_append"]("case-2", "just a note")+ assert note["evidence"] is None+ assert "error" in tools["journal_append"]("case-2", "x", kind="finding", severity="severe")+++def _write(store, turns, steps, issues=None):+ for t in turns:+ store.write_turn(t)+ for s in steps:+ store.write_step(s)+ for t in turns:+ store.write_issues(t["turn_id"], (issues or {}).get(t["turn_id"]) or+ {"signals_version": signals.version(), "warn": 0, "info": 0, "issues": [], "fine": []}, seen=True)+ store.flush()+++def _turn(tid, sid, seq, at, wake, team=None, status="completed"):+ stamp = {"machine": {"kind": "this_machine", "id": None}, "persona_id": "devsecops-lead"}+ if team:+ stamp["team"] = team+ return {"turn_id": tid, "session_id": sid, "seq": seq, "kind": "run", "trigger": "team_wake", "started_at": at,+ "duration_ms": 1000, "status": status, "incomplete": False, "approval_mode": "interactive",+ "attended": True, "reviewer": None, "stamp": stamp, "first_message_id": None,+ "last_message_id": None, "schema_version": 1, "wake": wake}+++def _step(tid, sid, n, at):+ return {"step_id": f"{tid}-s{n}", "turn_id": tid, "session_id": sid, "parent_step_id": None, "seq": n,+ "kind": "model", "name": "model", "started_at": at, "duration_ms": 100, "status": "ok",+ "call_id": None, "call_index": None, "message_id": None, "shadow": False, "after_turn": False,+ "attrs": {"purpose": "main", "model": "model-x", "usage": {"reported": True, "input_fresh": 10,+ "cache_read": 0, "cache_write": 0, "output": 1}},+ "content": {}}+++def test_lead_polling_is_stored_once_on_the_last_turn_of_the_run(tmp_path):+ manager = SessionManager(workspace=tmp_path, provider=_Scripted([]))+ store = manager.recordings_store+ team = {"team_id": "t1", "role": "lead"}+ turns = [_turn(f"L{i}", "lead-s", i, 1_000_000 + i * 60_000, {"reason": "timer", "board_seq": 5}, team)+ for i in range(1, 5)]+ _write(store, turns, [_step(t["turn_id"], "lead-s", 1, t["started_at"]) for t in turns])+ manager._scope_lead_polling("lead-s")+ got = {tid: [i["check"] for i in (store.read_turn_issues(tid) or {}).get("issues", [])] for tid in ("L3", "L4")}+ assert got == {"L3": [], "L4": ["lead_polling"]}+ issue = store.read_turn_issues("L4")+ assert issue["warn"] == 1 and issue["seen"] is False+ assert issue["issues"][0]["text"] == "The lead woke on its timer 4 times with no board change."+ # the ledger follows the stored result+ [t4] = [t for t in store.usage_turns(0, 2**62) if t["turn_id"] == "L4"]+ assert t4["warn"] == 1 and "lead_polling" in t4["warn_checks"]+ # a re-check of the turn alone keeps the series result+ rec = store.load_recording("lead-s")+ store.write_issues("L4", {**issue, "signals_version": "old"})+ store.flush()+ out = manager._recording_issues("lead-s", rec)+ assert [i["check"] for i in out["L4"]["issues"]] == ["lead_polling"]+ manager.observability.close()+++def test_the_ledger_is_kept_90_days(tmp_path):+ store = RecordingStore(tmp_path / "recordings.db", background=False)+ now = 200 * 24 * 3600 * 1000+ old = _turn("old", "s", 1, now - 91 * 24 * 3600 * 1000, {"reason": "user"})+ new = _turn("new", "s", 2, now - 40 * 24 * 3600 * 1000, {"reason": "user"})+ _write(store, [old, new], [_step("old", "s", 1, old["started_at"]), _step("new", "s", 1, new["started_at"])])+ store.sweep(now_ms=now) # recordings: 30 days+ assert store.load_recording("s")["turns"] == []+ store.sweep_ledger(now_ms=now) # the ledger: 90 days+ assert [t["turn_id"] for t in store.usage_turns(0, 2**62)] == ["new"]+ store.close()+++def test_ratings_go_out_as_a_langfuse_score_and_an_otlp_span(tmp_path):+ store = RecordingStore(tmp_path / "recordings.db", background=False)+ t = _turn("T1", "s", 1, 5_000, {"reason": "user"})+ _write(store, [t], [_step("T1", "s", 1, 5_000)])+ store.put_feedback({"turn_id": "T1", "session_id": "s", "rating": "down", "reasons": ["did_unasked"],+ "note": "used my key sk-PLANT", "auto_approved": []})+ posts = []++ def post(url, body, headers, timeout):+ posts.append((url, json.loads(body)))+ return 200, ""++ for kind, prompts in (("langfuse", False), ("otlp", True)):+ posts.clear()+ dest = ex.Destination(kind="langfuse", host="http://lf.test") if kind == "langfuse" else \+ ex.Destination(kind="otlp", url="http://otel.test/v1/traces")+ cfg = ex.ExportConfig(destination=dest, headers={}, include={"messages": prompts}, since_ms=0)+ exp = ex.Exporter(store, cfg, post=post, secret_values=lambda: ["sk-PLANT"])+ exp.flush_once()+ fb = [p for p in posts if "scores" in p[0] or "feedback" in json.dumps(p[1])]+ assert len(fb) == 1+ if kind == "langfuse":+ body = fb[0][1]+ assert fb[0][0] == "http://lf.test/api/public/scores"+ assert body["name"] == "user_feedback" and body["value"] == -1+ assert body["id"] == spans.feedback_id("T1") and body["traceId"] == spans.trace_id("T1")+ assert body["metadata"] == {"reasons": ["did_unasked"]} and "comment" not in body+ else:+ span = fb[0][1]["resourceSpans"][0]["scopeSpans"][0]["spans"][0]+ attrs = {a["key"]: a["value"] for a in span["attributes"]}+ assert span["name"] == "openworker.feedback" and span["parentSpanId"] == spans.root_span_id("T1")+ assert attrs["openworker.feedback.rating"] == {"stringValue": "down"}+ assert "PLANT" not in json.dumps(attrs) and "openworker.feedback.note" in attrs+ # sent once; a changed rating goes again, under the same fixed id+ posts.clear()+ exp.flush_once()+ assert not [p for p in posts if "scores" in p[0] or "feedback" in json.dumps(p[1])]+ store.close()+++def test_the_turn_span_names_the_coworker_and_keeps_agent_name(tmp_path):+ t = _turn("T1", "s", 1, 5_000, {"reason": "review"}, {"team_id": "t1", "role": "lead"})+ t["stamp"]["persona_fp"] = "a1b2c3"+ t["automation"] = {"task_id": "task-1", "run_id": "run-1", "trigger": "manual"}+ built = spans.build_turn_spans(t, [_step("T1", "s", 1, 5_000)], sw=spans.switches_from({}), dialect="semconv",+ get_piece=lambda k: None, masker=spans.Masker(), include_root=True,+ only=None, session_title=None)+ root = built[0]["attrs"]+ assert root["gen_ai.agent.name"] == "coworker" # TD3-25: the 1.0 value stays+ assert root["openworker.persona.id"] == "devsecops-lead" and root["openworker.persona.fp"] == "a1b2c3"+ assert root["openworker.wake.reason"] == "review" and root["openworker.team.role"] == "lead"+ assert root["openworker.automation.run_id"] == "run-1" and root["openworker.automation.trigger"] == "manual"+++def test_run_health_counts_a_stuck_run_as_failed():+ turn = {"status": "completed", "warn_checks": [], "ended_at": 1_000}+ assert scopes.run_health({"status": "running"}, turn, now_ms=1_000 + 600_001) == "failed"+ assert scopes.run_health({"status": "running"}, turn, now_ms=1_000 + 60_000) == "ok"+ assert scopes.run_health({"status": "ok"}, {**turn, "warn_checks": ["model_failed"]}) == "failed"+ assert scopes.run_health({"status": "ok"}, None) == "unknown"+++def test_turns_recorded_before_the_ledger_are_filled_in(tmp_path):+ manager = SessionManager(workspace=tmp_path, provider=_Scripted([]))+ store = manager.recordings_store+ t = _turn("T0", "s-old", 1, 1_000, {"reason": "user"})+ _write(store, [t], [_step("T0", "s-old", 1, 1_000)])+ with store._lock, store._conn:+ store._conn.execute("DELETE FROM turn_usage") # as a Phase 2 store has none+ assert store.turns_without_usage() == ["T0"]+ assert manager.backfill_usage() == 1+ [row] = store.usage_turns(0, 2**62)+ assert row["total"] == 11 and store.turns_without_usage() == []+ manager.observability.close()diff --git a/tests/test_remote_home.py b/tests/test_remote_home.pyindex c302ca39..e9e42241 100644--- a/tests/test_remote_home.py+++ b/tests/test_remote_home.py@@ -2499,3 +2499,44 @@ async def test_browser_sealed_managed_grant_lands_on_the_box(tmp_path):task.cancel()with pytest.raises(asyncio.CancelledError):await task+++async def test_offline_machine_serves_its_last_usage_rollup_as_of(tmp_path):+ """Observability Phase 3 (E28, TD3-6): the desktop keeps each machine's last Usage rollup+ per grouping; while the machine is offline the proxy answers with it, labelled `cached`+ with the time it was fetched, never as live numbers."""+ async with _Server(_controller(tmp_path)) as server:+ join_url = await _arm(server.base)+ _, token = parse_join_url(join_url)+ state = tmp_path / "rollup-state"+ state.mkdir()+ box_manager = SessionManager(data_dir=tmp_path / "rollup-data")+ task = asyncio.create_task(+ run_joined(state=state, controller=server.base, name="usage-box", token=token,+ app=create_app(box_manager), once=True, log=lambda *_: None)+ )+ machines = await _wait_connected(server.base)+ mid = machines[0]["id"]+ async with httpx.AsyncClient() as client:+ live = await client.get(f"{server.base}/v1/machines/{mid}/p/v1/observability/rollup",+ params={"group": "all", "from": 0})+ assert live.status_code == 200 and "cached" not in live.json()+ task.cancel()+ with pytest.raises(asyncio.CancelledError):+ await task+ deadline = asyncio.get_running_loop().time() + 5+ while True:+ listing = await client.get(f"{server.base}/v1/machines")+ if not any(m["connected"] for m in listing.json()["machines"]):+ break+ assert asyncio.get_running_loop().time() < deadline+ await asyncio.sleep(0.05)+ off = await client.get(f"{server.base}/v1/machines/{mid}/p/v1/observability/rollup",+ params={"group": "all", "from": 0})+ assert off.status_code == 200+ body = off.json()+ assert body["cached"]["as_of"] > 0 and body["groups"] == live.json()["groups"]+ # another grouping was never fetched: no copy, the machine is just offline+ none = await client.get(f"{server.base}/v1/machines/{mid}/p/v1/observability/rollup",+ params={"group": "day"})+ assert none.status_code == 503