Skip to content
Python

feat(observability): Phase 3 evidence links, verifies link, coverage, stored scope checks, rollup cache

Devika Verma36e5a1502c1a
diff --git a/coworker/engine.py b/coworker/engine.py
index 44f3c597..a9ba7e69 100644
--- a/coworker/engine.py
+++ b/coworker/engine.py
@@ -2999,7 +2999,7 @@ class TurnEngine:
)
# A tool that starts its own work (the explore helper, E6) records it under this step.
outer = set_tool_run(ToolRun(step_id=step_id, turn_id=self._obs.turn_id,
- view=self.provider, recording=self._obs))
+ view=self.provider, recording=self._obs, call_id=tool_call.id))
try:
result, status = self._execute_call(tool_call)
finally:
diff --git a/coworker/observability/engine_hooks.py b/coworker/observability/engine_hooks.py
index 8ba8a63a..4b90c785 100644
--- a/coworker/observability/engine_hooks.py
+++ b/coworker/observability/engine_hooks.py
@@ -453,8 +453,11 @@ class ToolRun:
provider view and recording. Published for the tool's thread by `_execute_sync`."""
def __init__(self, *, step_id: Optional[str], turn_id: Optional[str], view: Any,
- recording: "TurnRecording") -> None:
+ recording: "TurnRecording", call_id: Optional[str] = None) -> None:
self.step_id, self.turn_id, self.view, self.recording = step_id, turn_id, view, recording
+ # 1.3 (E17a): the tool call's id, which exists whether or not recording is on: the
+ # link a journal entry keeps to the step that wrote it.
+ self.call_id = call_id
def current_tool_run() -> Optional[ToolRun]:
diff --git a/coworker/observability/store.py b/coworker/observability/store.py
index fc5f28ea..5ad83db3 100644
--- a/coworker/observability/store.py
+++ b/coworker/observability/store.py
@@ -946,6 +946,31 @@ class RecordingStore:
"step": {"step_id": w["step_id"], "turn_id": w["turn_id"]}})
return out
+ def scan_steps(self, session_ids: Any, *, since_ms: int = 0) -> list[dict[str, Any]]:
+ """Tool steps whose `scanner.action` is `scan` (1.3), of one or more sessions, oldest
+ first: what a finding's evidence and the coverage checks read."""
+ ids = [session_ids] if isinstance(session_ids, str) else list(session_ids)
+ if not ids:
+ return []
+ self.flush()
+ with self._lock:
+ rows = self._conn.execute(
+ "SELECT s.step_id, s.turn_id, s.session_id, s.call_id, s.seq, s.started_at, s.status, s.attrs, "
+ "t.seq AS turn_seq FROM steps s LEFT JOIN turns t ON t.turn_id = s.turn_id "
+ f"WHERE s.kind = 'tool' AND s.session_id IN ({', '.join('?' * len(ids))}) AND s.started_at >= ? "
+ "AND s.attrs LIKE '%\"scanner\": {%' ORDER BY s.started_at, s.step_id", [*ids, since_ms]).fetchall()
+ out = []
+ for r in rows:
+ a = json.loads(r["attrs"] or "{}")
+ sc = a.get("scanner") or {}
+ if sc.get("action") != "scan":
+ continue
+ out.append({"step_id": r["step_id"], "turn_id": r["turn_id"], "session_id": r["session_id"],
+ "call_id": r["call_id"], "seq": r["seq"], "turn_seq": r["turn_seq"],
+ "started_at": r["started_at"], "status": r["status"], "scanner": sc,
+ "exit_code": a.get("exit_code"), "result_sha256": a.get("result_sha256")})
+ return out
+
def read_turn_issues(self, turn_id: str) -> Optional[dict[str, Any]]:
"""One turn's stored Issues result with `seen`, or None."""
self.flush()
diff --git a/coworker/personas/builtin/appsec-worker/manifest.md b/coworker/personas/builtin/appsec-worker/manifest.md
index befc462b..cffba982 100644
--- a/coworker/personas/builtin/appsec-worker/manifest.md
+++ b/coworker/personas/builtin/appsec-worker/manifest.md
@@ -11,6 +11,7 @@ team: worker
tools: [code_files, git, search, shell, todo]
connectors: [github]
skills: [semgrep-review, security-fix-pr]
+coverage: [code]
models: [anthropic:claude-opus-4-8, openai:gpt-5.6-sol]
default_permission_mode: interactive
description: An application-security coworker that works team-style — it takes assigned code-review items from a security lead, drives scanners (semgrep), triages findings in context, fixes what matters, and hands off through review with evidence.
diff --git a/coworker/personas/builtin/cloud-posture/manifest.md b/coworker/personas/builtin/cloud-posture/manifest.md
index 5f668f9f..7cf5d2ee 100644
--- a/coworker/personas/builtin/cloud-posture/manifest.md
+++ b/coworker/personas/builtin/cloud-posture/manifest.md
@@ -10,6 +10,7 @@ version: "1"
tools: [code_files, git, search, shell, todo]
connectors: [github]
skills: [iac-scan, aws-posture]
+coverage: [iac]
models: [anthropic:claude-opus-4-8, openai:gpt-5.6-sol]
default_permission_mode: interactive
description: An infrastructure-security reviewer for teams without a cloud security team. Scans Terraform and cloud configuration with open-source tools (trivy, checkov), reads your live cloud posture strictly read-only, and fixes what matters in the IaC — never by clicking around a console.
diff --git a/coworker/personas/builtin/dep-audit/manifest.md b/coworker/personas/builtin/dep-audit/manifest.md
index ea2f12c4..f2a8054f 100644
--- a/coworker/personas/builtin/dep-audit/manifest.md
+++ b/coworker/personas/builtin/dep-audit/manifest.md
@@ -10,6 +10,7 @@ version: "1"
tools: [code_files, git, search, shell, todo]
connectors: [github]
skills: [dependency-audit, safe-upgrade-pr]
+coverage: [deps]
models: [anthropic:claude-opus-4-8, openai:gpt-5.6-sol]
default_permission_mode: interactive
description: A dependency auditor for teams without a security team. Runs open-source vulnerability scanners (osv-scanner, npm audit, pip-audit, trivy) across your lockfiles, separates exploitable from theoretical, and ships minimal, test-verified upgrade PRs.
diff --git a/coworker/personas/builtin/posture-worker/manifest.md b/coworker/personas/builtin/posture-worker/manifest.md
index 961a328a..7d14f615 100644
--- a/coworker/personas/builtin/posture-worker/manifest.md
+++ b/coworker/personas/builtin/posture-worker/manifest.md
@@ -11,6 +11,7 @@ team: worker
tools: [code_files, git, search, shell, todo]
connectors: [github]
skills: [iac-scan, aws-posture]
+coverage: [iac]
models: [anthropic:claude-opus-4-8, openai:gpt-5.6-sol]
default_permission_mode: interactive
description: An infrastructure-security coworker that works team-style — it takes assigned posture items from a security lead, scans Terraform and cloud configuration (trivy, checkov; cloud strictly read-only), fixes in the IaC, and hands off through review with evidence.
diff --git a/coworker/personas/builtin/secrets-worker/manifest.md b/coworker/personas/builtin/secrets-worker/manifest.md
index e4aec261..7b86c94b 100644
--- a/coworker/personas/builtin/secrets-worker/manifest.md
+++ b/coworker/personas/builtin/secrets-worker/manifest.md
@@ -10,6 +10,7 @@ version: "1"
team: worker
tools: [code_files, git, search, shell, todo]
skills: [secret-scan]
+coverage: [secrets]
models: [anthropic:claude-opus-4-8, openai:gpt-5.6-sol]
default_permission_mode: interactive
description: A secret-hunting coworker that works team-style — it takes assigned items from a security lead, sweeps working trees and full git history for leaked credentials (gitleaks + manual history reads), verifies what's live, and hands off through review with evidence.
diff --git a/coworker/personas/builtin/security/manifest.md b/coworker/personas/builtin/security/manifest.md
index c33d8b84..5f6bb5c0 100644
--- a/coworker/personas/builtin/security/manifest.md
+++ b/coworker/personas/builtin/security/manifest.md
@@ -10,6 +10,7 @@ version: "1"
tools: [code_files, git, search, shell, todo]
connectors: [github]
skills: [semgrep-review, secret-scan, security-fix-pr]
+coverage: [code, secrets]
models: [anthropic:claude-opus-4-8, openai:gpt-5.6-sol]
default_permission_mode: interactive
description: A code-security reviewer for teams without a security team. Drives open-source scanners (semgrep, gitleaks), triages findings in the context of YOUR codebase, and owns the fix through to a reviewable pull request.
diff --git a/coworker/remote/acceptor.py b/coworker/remote/acceptor.py
index fe45d81f..145ec4ce 100644
--- a/coworker/remote/acceptor.py
+++ b/coworker/remote/acceptor.py
@@ -814,6 +814,19 @@ def mount_acceptor(
)
return {"ok": True, "revoked": names}
+ def _cached_rollup(machine_id: str, grouping: str):
+ """An offline machine's last rollup, labelled with when it was fetched (TD3-6). The
+ desktop never adds it to the live totals without that label."""
+ found = registry.last_rollup(machine_id, grouping)
+ if found is None:
+ return None
+ try:
+ body = json.loads(found["payload"])
+ except ValueError:
+ return None
+ body["cached"] = {"as_of": int(found["updated_at"] * 1000), "query": found["query"]}
+ return JSONResponse(body, status_code=200)
+
@app.api_route(
"/v1/machines/{machine_id}/p/{path:path}",
methods=["GET", "POST", "PUT", "PATCH", "DELETE"],
@@ -827,8 +840,11 @@ def mount_acceptor(
return JSONResponse({"error": "unknown machine"}, status_code=404)
_touch(machine_id)
live = await _live_or_wake(machine_id)
+ rollup = request.method == "GET" and path == "v1/observability/rollup"
+ grouping = request.query_params.get("group") or "all"
if live is None:
- return JSONResponse({"error": "machine offline"}, status_code=503)
+ cached = _cached_rollup(machine_id, grouping) if rollup else None
+ return cached or JSONResponse({"error": "machine offline"}, status_code=503)
target = "/" + path
if request.url.query:
target += "?" + request.url.query
@@ -847,7 +863,14 @@ def mount_acceptor(
except asyncio.TimeoutError:
return JSONResponse({"error": "machine timed out"}, status_code=504)
except ConnectionError:
- return JSONResponse({"error": "machine disconnected"}, status_code=503)
+ cached = _cached_rollup(machine_id, grouping) if rollup else None
+ return cached or JSONResponse({"error": "machine disconnected"}, status_code=503)
+ # Phase 3 (E28): each machine's last Usage rollup, per grouping, for when it is off.
+ if rollup and status == 200:
+ try:
+ registry.save_rollup(machine_id, grouping, request.url.query, payload.decode("utf-8"))
+ except Exception:
+ pass
# Tier-2 snapshot: every successful unfiltered session-list fetch doubles
# as the offline fallback (greyed, never vanished). Metadata only.
if (
diff --git a/coworker/remote/registry.py b/coworker/remote/registry.py
index 2613296b..facc1e1d 100644
--- a/coworker/remote/registry.py
+++ b/coworker/remote/registry.py
@@ -51,6 +51,21 @@ class MachinesRegistry:
# Tier-2 snapshot (remote-home-design.md data path): the last-known
# session-LIST METADATA per machine — what keeps an offline machine's
# sessions greyed-but-visible instead of vanished.
+ # Observability Phase 3 (E28, TD3-6): the last Usage rollup each machine answered,
+ # per grouping, so an offline machine still shows its numbers on the Usage page,
+ # labelled "as of". Counts only (the ledger is content-free).
+ self._conn.execute(
+ """
+ CREATE TABLE IF NOT EXISTS machine_rollups (
+ machine_id TEXT NOT NULL,
+ grouping TEXT NOT NULL,
+ query TEXT NOT NULL,
+ payload TEXT NOT NULL,
+ updated_at REAL NOT NULL,
+ PRIMARY KEY (machine_id, grouping)
+ )
+ """
+ )
self._conn.execute(
"""
CREATE TABLE IF NOT EXISTS machine_sessions (
@@ -274,6 +289,25 @@ class MachinesRegistry:
).fetchall()
return [dict(r) for r in rows]
+ def save_rollup(self, machine_id: str, grouping: str, query: str, payload: str) -> None:
+ with self._lock:
+ self._conn.execute(
+ "INSERT INTO machine_rollups (machine_id, grouping, query, payload, updated_at)"
+ " VALUES (?, ?, ?, ?, ?) ON CONFLICT(machine_id, grouping) DO UPDATE SET"
+ " query = excluded.query, payload = excluded.payload, updated_at = excluded.updated_at",
+ (machine_id, grouping, query, payload, time.time()),
+ )
+ self._conn.commit()
+
+ def last_rollup(self, machine_id: str, grouping: str) -> Optional[dict]:
+ """The machine's last Usage rollup for this grouping: {payload, query, updated_at}."""
+ with self._lock:
+ row = self._conn.execute(
+ "SELECT payload, query, updated_at FROM machine_rollups WHERE machine_id = ? AND grouping = ?",
+ (machine_id, grouping),
+ ).fetchone()
+ return dict(row) if row else None
+
def save_sessions_snapshot(self, machine_id: str, payload: str) -> None:
with self._lock:
self._conn.execute(
diff --git a/coworker/server/manager.py b/coworker/server/manager.py
index 0b451c19..ffb00a44 100644
--- a/coworker/server/manager.py
+++ b/coworker/server/manager.py
@@ -21,7 +21,7 @@ import sys
import time
import uuid
from pathlib import Path
-from typing import Any, Optional
+from typing import Any, Iterable, Optional
from urllib.parse import urlsplit
from .. import model_config as _model_config
@@ -49,6 +49,7 @@ from ..observability import signals as recording_signals
from ..observability import rollups as recording_rollups
from ..observability import stamps as _obs_stamps
from ..observability import usage_api
+from ..observability import scopes as _obs_scopes
from ..observability.export import ObservabilityExport
from ..observability.engine_hooks import note_team
from ..observability.provider_view import RecordingProvider, tag as tag_provider
@@ -448,6 +449,11 @@ class SessionManager:
)
self.observability.apply_policy(load_policy(self.secrets.path.parent))
self.recorder.add_listener(self._export_on_turn_end)
+ # Phase 3: the series and task checks, off the turn's own thread, one at a time.
+ import concurrent.futures
+
+ self._scope_pool = concurrent.futures.ThreadPoolExecutor(max_workers=1, thread_name_prefix="obs-scopes")
+ self.recorder.add_listener(self._scopes_on_turn_end)
# No explicit provider injected → route by the model's `provider:` prefix (OpenAI default,
# Ollama, …). Tests inject a provider directly and bypass the router. The same router is
# shared by every engine and the `/v1/chat/completions` proxy.
@@ -2757,6 +2763,14 @@ class SessionManager:
out["events"] = usage_api.team_events(self.recordings_store, lead_session=team.lead_session,
worker_sessions=members, board_events=board, links=links)
out["wake_explained"] = self._wake_explained(sids)
+ # OB-16: "Coverage for this task" on each item a coworker that declares coverage
+ # worked on, with its findings' evidence links.
+ cov = {}
+ for item in self.team_store.list_items(team.space, self._user_actor()):
+ c = self.item_coverage(team, item, board)
+ if c is not None:
+ cov[str(item["id"])] = c
+ out["coverage"] = cov
except Exception: # noqa: BLE001 - the split never hides the rest of the summary
logger.exception("could not work out team %s wakes", team_id)
out.setdefault("wakes", [])
@@ -2948,7 +2962,9 @@ class SessionManager:
team_id=team_identity,
worker_session=worker_session,
) + journal_tools(
- self.journal_store, actor=actor, space=space
+ self.journal_store, actor=actor, space=space,
+ scan_evidence=lambda item, _sid=session_id, _space=space, _actor=actor.id:
+ self._scan_evidence(_sid, _space, _actor, item),
)
from ..teams.artifacts import artifact_tools
@@ -6614,6 +6630,14 @@ class SessionManager:
out[t["turn_id"]] = have
continue
result = recording_signals.check_turn(t, steps_of.get(t["turn_id"], []))
+ if have is not None:
+ # Series and task results are worked out at turn end over more than this
+ # turn; a re-check of the turn alone keeps them (Phase 3).
+ kept = [i for i in have.get("issues") or [] if i.get("check") in self.SCOPE_CHECKS]
+ if kept:
+ result["issues"] += kept
+ result["warn"] = sum(1 for i in result["issues"] if i.get("severity") == "warn")
+ result["info"] = sum(1 for i in result["issues"] if i.get("severity") == "info")
seen = True if have is None else bool(have.get("seen"))
self.recordings_store.write_issues(t["turn_id"], result, seen=seen)
out[t["turn_id"]] = {**result, "seen": seen}
@@ -6635,6 +6659,256 @@ class SessionManager:
def recording_usage(self, session_id: str) -> dict[str, Any]:
return recording_api.usage(self.recordings_store.load_recording(session_id), session_id)
+ # -- observability Phase 3: series and task checks (TD3-17, E17e, E18d, E26d) --------------
+ # They are pure functions (`observability.scopes`) of more than one turn. At each turn's
+ # end the series or task the turn belongs to is worked out again, and every result is
+ # stored with the turn whose step it points at, as Phase 2's are, replacing what that
+ # series or task said before. So the amber dot and the Issues tab carry them.
+ SCOPE_CHECKS = frozenset(_obs_scopes.SERIES_CHECKS + _obs_scopes.TASK_CHECKS)
+
+ def _scopes_on_turn_end(self, event: dict[str, Any]) -> None:
+ if event.get("type") != "turn_ended":
+ return
+ tid = event.get("turn_id")
+ sid = event.get("session_id")
+ if not tid or not sid:
+ return
+ try:
+ self._scope_pool.submit(self.run_scope_checks, sid, tid)
+ except RuntimeError:
+ pass # shutting down
+
+ def run_scope_checks(self, session_id: str, turn_id: str) -> None:
+ """The series and task checks for the series or task this turn belongs to."""
+ try:
+ store = self.recordings_store
+ store.flush()
+ loaded = store.load_turn(turn_id)
+ if loaded is None:
+ return
+ turn, _steps = loaded
+ stamp = turn.get("stamp") or {}
+ team = stamp.get("team") or {}
+ if team.get("role") == "lead":
+ self._scope_lead_polling(session_id)
+ if team.get("team_id"):
+ self._scope_team_tasks(team["team_id"])
+ if turn.get("automation"):
+ self._scope_failed_in_a_row(turn["automation"]["task_id"])
+ if stamp.get("persona_coverage") and not team:
+ self._scope_solo_coverage(session_id, list(stamp["persona_coverage"]))
+ except Exception: # noqa: BLE001 - a check never breaks anything else
+ logger.exception("series and task checks failed for %s", turn_id)
+
+ def _store_scope_issues(self, turn_ids: Iterable[str], found: dict[str, list[dict[str, Any]]],
+ checks: Iterable[str]) -> None:
+ """Replace, on each of `turn_ids`, the issues of `checks` with what `found` says."""
+ checks = set(checks)
+ store = self.recordings_store
+ for tid in dict.fromkeys(list(turn_ids) + list(found)):
+ have = store.read_turn_issues(tid)
+ if have is None:
+ continue # never checked yet: the re-check will store it with these
+ seen = bool(have.pop("seen", False))
+ keep = [i for i in have.get("issues") or [] if i.get("check") not in checks]
+ new = found.get(tid, [])
+ old = [i for i in have.get("issues") or [] if i.get("check") in checks]
+ if [(i["check"], i.get("text")) for i in old] == [(i["check"], i.get("text")) for i in new]:
+ continue
+ result = {**have, "issues": keep + [{k: v for k, v in i.items() if k != "turn_id"} for i in new]}
+ result["warn"] = sum(1 for i in result["issues"] if i.get("severity") == "warn")
+ result["info"] = sum(1 for i in result["issues"] if i.get("severity") == "info")
+ fine = [f for f in result.get("fine") or [] if f.get("check") not in checks]
+ result["fine"] = fine
+ added = {(i["check"], i.get("text")) for i in new} - {(i["check"], i.get("text")) for i in old}
+ store.write_issues(tid, result, seen=seen and not added)
+
+ def _first_step(self, turn_id: str) -> Optional[dict[str, Any]]:
+ loaded = self.recordings_store.load_turn(turn_id)
+ if loaded is None:
+ return None
+ top = [s for s in loaded[1] if s.get("parent_step_id") is None and not s.get("shadow")]
+ return {**top[0], "turn_id": turn_id} if top else {"step_id": None, "seq": None, "turn_id": turn_id}
+
+ def _last_step(self, turn_id: str) -> Optional[dict[str, Any]]:
+ loaded = self.recordings_store.load_turn(turn_id)
+ if loaded is None:
+ return None
+ top = [s for s in loaded[1] if s.get("parent_step_id") is None and not s.get("shadow")
+ and not s.get("after_turn")]
+ return {**top[-1], "turn_id": turn_id} if top else {"step_id": None, "seq": None, "turn_id": turn_id}
+
+ def _scope_lead_polling(self, lead_session: str) -> None:
+ rows = self.recordings_store.team_turns([lead_session])
+ series = [{"id": t["turn_id"], "wake": t.get("wake")} for t in rows if t["status"] not in (None, "running")]
+ found: dict[str, list] = {}
+ for i in _obs_scopes.lead_polling_issues(series, {t["id"]: self._first_step(t["id"]) for t in series[-60:]}):
+ found.setdefault(i["turn_id"] or i["facts"]["to"], []).append(i)
+ self._store_scope_issues([t["id"] for t in series], found, ["lead_polling"])
+
+ def _scope_failed_in_a_row(self, task_id: str) -> None:
+ detail = self.get_automation(task_id)
+ banner = detail.get("failed_in_a_row")
+ runs = detail.get("runs") or []
+ turn_ids = [r["first_turn_id"] for r in runs if r.get("first_turn_id")]
+ found: dict[str, list] = {}
+ if banner and banner.get("turn_id"):
+ series = [{"health": "failed" if r["health"] in ("failed", "did_not_finish") else r["health"],
+ "error": r.get("error") or (r.get("check_titles") or [None])[0]}
+ for r in sorted(runs, key=lambda x: x.get("started_at") or 0)]
+ issue = _obs_scopes.failed_in_a_row_issue(series, self._first_step(banner["turn_id"]))
+ if issue is not None:
+ found[banner["turn_id"]] = [issue]
+ self._store_scope_issues(turn_ids, found, ["failed_in_a_row"])
+
+ def _scope_solo_coverage(self, session_id: str, promised: list[str]) -> None:
+ """A coworker working alone: its task is the session. Only the scan checks apply (no
+ board, no report, no journal), on the session's last turn."""
+ rows = self.recordings_store.team_turns([session_id])
+ done = [t for t in rows if t["status"] not in (None, "running")]
+ if not done:
+ return
+ last = done[-1]["turn_id"]
+ scans = self.recordings_store.scan_steps(session_id)
+ task = {"promised": promised, "steps": [{"id": s["step_id"], "scanner": s["scanner"],
+ "exit_code": s["exit_code"]} for s in scans]}
+ by_id = {s["step_id"]: s for s in scans}
+ issues = [i for i in _obs_scopes.coverage_issues(task, handoff=self._last_step(last), steps_by_id=by_id)
+ if i["check"] in ("scan_missing", "scanner_failed")]
+ found: dict[str, list] = {}
+ for i in issues:
+ found.setdefault(i.get("turn_id") or last, []).append(i)
+ self._store_scope_issues([t["turn_id"] for t in done], found, ["scan_missing", "scanner_failed"])
+
+ def item_coverage(self, team, item: dict[str, Any], events: list[dict[str, Any]]) -> Optional[dict[str, Any]]:
+ """"Coverage for this task" (OB-16, §10): a board item of a worker that declares
+ coverage, its scans across the item's sessions, the hand-off's report, its findings
+ with their evidence, and the fix it verifies. None when nobody on it promised a scan."""
+ by_actor = {w.actor: w.session_id for w in team.workers}
+ by_actor[team.lead_actor] = team.lead_session
+ mine = [e for e in events if e.get("item_id") == item["id"]]
+ assignees = [str((e.get("payload") or {}).get("assignee") or "") for e in mine if e["kind"] == "item_assigned"]
+ sessions = [by_actor[a] for a in dict.fromkeys(assignees) if a in by_actor]
+ if item.get("assignee") in by_actor and by_actor[item["assignee"]] not in sessions:
+ sessions.append(by_actor[item["assignee"]])
+ if not sessions:
+ return None
+ promised: list[str] = []
+ for t in self.recordings_store.team_turns(sessions):
+ for a in (t.get("stamp") or {}).get("persona_coverage") or []:
+ if a not in promised:
+ promised.append(a)
+ if not promised:
+ return None
+ since = min([usage_api._ts_ms(e.get("ts")) or 0 for e in mine if e["kind"] == "item_assigned"] or [0])
+ scans = self.recordings_store.scan_steps(sessions, since_ms=since)
+ handoffs = [e for e in mine if e["kind"] == "item_transitioned" and (e.get("payload") or {}).get("to") in ("review", "done")]
+ report = next(((e.get("payload") or {}).get("coverage") for e in reversed(handoffs)
+ if (e.get("payload") or {}).get("coverage")), None)
+ findings = self._item_findings(item["id"], team.space)
+ fix = None
+ verifies = [l for l in item.get("links") or [] if l.get("kind") == "verifies"]
+ if verifies:
+ target = verifies[0]["item"]
+ fixed_by = next((e["actor"] for e in reversed(events) if e.get("item_id") == target
+ and e["kind"] == "item_transitioned" and (e.get("payload") or {}).get("to") == "review"), None)
+ verified_by = next((e["actor"] for e in reversed(handoffs)), item.get("assignee"))
+ fix = {"item": target, "fixed_by": fixed_by, "verified_by": verified_by}
+ task = {"promised": promised, "report": report, "fix": fix,
+ "findings": [{"id": f["seq"], "severity": f.get("severity"), "evidence": f.get("evidence")} for f in findings],
+ "steps": [{"id": s["step_id"], "scanner": s["scanner"], "exit_code": s["exit_code"]} for s in scans]}
+ handoff = None
+ if handoffs:
+ h = handoffs[-1]
+ sid = by_actor.get(h["actor"])
+ handoff = self._handoff_step(sid, usage_api._ts_ms(h.get("ts"))) if sid else None
+ issues = _obs_scopes.coverage_issues(task, handoff=handoff, steps_by_id={s["step_id"]: s for s in scans})
+ fine = _obs_scopes.coverage_fine(task)
+ fired = {i["check"] for i in issues}
+ lst = (recording_signals.signals().get("scanners") or {}).get("list") or {}
+ return {
+ "item": item["id"], "promised": promised, "report": report, "handed_off": bool(handoffs),
+ "scans": [{**s, "area": (lst.get(s["scanner"]["name"]) or {}).get("area"),
+ "found_something": s["exit_code"] not in (0, None)
+ and s["exit_code"] in ((lst.get(s["scanner"]["name"]) or {}).get("findings_exit") or []),
+ "failed": s["exit_code"] not in (0, None)
+ and s["exit_code"] not in ((lst.get(s["scanner"]["name"]) or {}).get("findings_exit") or [])}
+ for s in scans],
+ "findings": findings, "fix": fix, "issues": issues,
+ "fine": [{"check": c, "text": t} for c, t in fine.items() if c not in fired],
+ }
+
+ def _item_findings(self, item_id: int, space: str) -> list[dict[str, Any]]:
+ """The journal findings about an item, with the step that wrote each and its evidence
+ links (E17a). Read-only, by item and space; the hash chain is not touched."""
+ js = self.journal_store
+ with js._lock:
+ rows = js._conn.execute(
+ "SELECT * FROM journal_entries WHERE item_id = ? AND space = ? AND kind = 'finding' ORDER BY seq",
+ (item_id, space)).fetchall()
+ from ..teams.journal import _row_to_entry
+
+ out = []
+ for r in rows:
+ e = _row_to_entry(r)
+ out.append({"seq": e["seq"], "case": e.get("case_id"), "author": e.get("author"),
+ "session_id": e.get("session_id"), "severity": e.get("severity"),
+ "entities": e.get("entities") or [], "step_call_id": e.get("step_call_id"),
+ "evidence": e.get("evidence"), "ts": e.get("ts")})
+ return out
+
+ def _handoff_step(self, session_id: str, at_ms: Optional[int]) -> Optional[dict[str, Any]]:
+ """The `transition` tool step that handed the item off: the last one of the session at
+ or before the board event's time."""
+ rec = self.recordings_store.load_recording(session_id)
+ steps = [s for s in rec.get("steps") or [] if s.get("kind") == "tool" and s.get("name") == "transition"
+ and (at_ms is None or (s.get("started_at") or 0) <= at_ms + 1000)]
+ if not steps:
+ return None
+ s = steps[-1]
+ return {"step_id": s["step_id"], "seq": s.get("seq"), "turn_id": s.get("turn_id")}
+
+ def _scope_team_tasks(self, team_id: str) -> None:
+ """A team's task checks: coverage of each handed-off item of a worker that declares
+ coverage, the same approval asked twice, and items reviewed before a prerequisite."""
+ team = self.teams.get(team_id)
+ if team is None:
+ return
+ from ..teams.summary import all_events
+
+ events = list(all_events(self.team_store, team.space))
+ sids = [team.lead_session, *(w.session_id for w in team.workers)]
+ found: dict[str, list] = {}
+ for item in self.team_store.list_items(team.space, self._user_actor()):
+ cov = self.item_coverage(team, item, events)
+ if cov is None or not cov["handed_off"]:
+ continue
+ for i in cov["issues"]:
+ if i.get("turn_id"):
+ found.setdefault(i["turn_id"], []).append(i)
+ cards = self.recordings_store.team_approval_cards(sids)
+ for i in _obs_scopes.same_approval_issues(cards):
+ if i.get("turn_id"):
+ found.setdefault(i["turn_id"], []).append(i)
+ links = []
+ for item in self.team_store.list_items(team.space, self._user_actor()):
+ for link in item.get("links") or []:
+ if link.get("kind") == "blocked_by":
+ links.append((int(link["item"]), int(item["id"])))
+ transitions = [{"seq": e["seq"], "item": e.get("item_id"), "to": (e.get("payload") or {}).get("to"),
+ "actor": e.get("actor"), "ts": e.get("ts")} for e in events if e["kind"] == "item_transitioned"]
+ by_actor = {w.actor: w.session_id for w in team.workers}
+ by_actor[team.lead_actor] = team.lead_session
+ for i in _obs_scopes.reviewed_before_prereq_issues(transitions, links):
+ e = next((x for x in transitions if x["seq"] == i["facts"]["at_seq"]), None)
+ sid = by_actor.get(e["actor"]) if e else None
+ step = self._handoff_step(sid, usage_api._ts_ms(e["ts"])) if sid else None
+ if step and step.get("turn_id"):
+ i.update({"step_id": step["step_id"], "step_seq": step["seq"], "turn_id": step["turn_id"]})
+ found.setdefault(step["turn_id"], []).append(i)
+ turn_ids = [t["turn_id"] for t in self.recordings_store.team_turns(sids)]
+ self._store_scope_issues(turn_ids, found, list(_obs_scopes.TASK_CHECKS))
+
# -- observability Phase 3: rollups over many turns (E24b, E24c, E26b, E27a) ---------------
def usage_names(self) -> dict[str, str]:
"""Coworker names by id, for the Usage page: the installed coworkers, else the name
@@ -8598,6 +8872,25 @@ class SessionManager:
),
}
+ def _scan_evidence(self, session_id: str, space: str, actor_id: str, item: Optional[int]) -> list[dict[str, str]]:
+ """A finding's default evidence (E17a, TD3-15): the successful-or-findings scan steps of
+ the same session, run since the item was assigned to this actor (or, with no item, in
+ the session), as `[{session_id, call_id}]`. Built differently from the design: every
+ scan since assignment, not only those whose area matches the finding's entities
+ (entities are free text; matching them to an area would be a guess)."""
+ since = 0
+ if item is not None and space:
+ try:
+ for e in self.team_store.events(space, item_id=int(item)):
+ if e["kind"] == "item_assigned" and (e.get("payload") or {}).get("assignee") == actor_id:
+ since = usage_api._ts_ms(e.get("ts")) or since
+ except Exception: # noqa: BLE001
+ since = 0
+ out = []
+ for s in self.recordings_store.scan_steps(session_id, since_ms=since):
+ out.append({"session_id": session_id, "call_id": s["call_id"]})
+ return out
+
def _note_persona_version(self, engine: TurnEngine) -> None:
"""Keep this coworker version the first time its fingerprint is seen (E25c): name,
file list and file copies, for OB-15's "What changed" and "Show diff". Local only."""
@@ -8676,11 +8969,29 @@ class SessionManager:
logger.exception("could not re-check issues of %s", sid)
return len(sids)
+ def backfill_usage(self, batch: int = 200) -> int:
+ """Ledger rows for finished turns recorded before Phase 3 (TD3-8): worked out from their
+ steps, in batches, so the Usage page's numbers reach back as far as the recordings do.
+ Returns how many turns were filled."""
+ done, last = 0, None
+ while True:
+ ids = self.recordings_store.turns_without_usage(limit=batch)
+ if not ids or ids == last:
+ return done # nothing left, or a batch that cannot be written (logged)
+ last = ids
+ for tid in ids:
+ self.recordings_store.refresh_usage(tid)
+ self.recordings_store.flush()
+ done += len(ids)
+ if len(ids) < batch:
+ return done
+ time.sleep(0.05)
+
def start_background_checks(self) -> threading.Thread:
"""At start, after the open turns are closed: close stale manual runs and re-check
Issues, in a background thread so the app is usable at once (P3-3, P3-4)."""
def work() -> None:
- for job in (self.finish_stale_manual_runs, self.recheck_issues):
+ for job in (self.finish_stale_manual_runs, self.recheck_issues, self.backfill_usage):
try:
job()
except Exception: # noqa: BLE001
diff --git a/coworker/teams/journal.py b/coworker/teams/journal.py
index 1cdc180d..70101313 100644
--- a/coworker/teams/journal.py
+++ b/coworker/teams/journal.py
@@ -98,6 +98,14 @@ class JournalStore:
created_ts TEXT NOT NULL
);
""")
+ # Observability Phase 3 (E17a, TD3-15): the tool call that wrote an entry, and the
+ # calls that are its evidence ([{session_id, call_id}]). Pointers the app adds, so
+ # outside the hash chain, like persona, model and session_id (FO-137 would hash them).
+ for column in ("step_call_id TEXT", "evidence TEXT"):
+ try:
+ self._conn.execute(f"ALTER TABLE journal_entries ADD COLUMN {column}")
+ except sqlite3.OperationalError:
+ pass # already there
self._conn.commit()
# ---------------------------------------------------------------------- verbs
@@ -114,6 +122,9 @@ class JournalStore:
entities: Optional[list[str]] = None,
refs: Optional[list[str]] = None,
taint: bool = False,
+ severity: Optional[str] = None,
+ step_call_id: Optional[str] = None,
+ evidence: Optional[list[dict[str, str]]] = None,
) -> dict[str, Any]:
if not (case or "").strip():
raise BoardError("case is required")
@@ -145,6 +156,9 @@ class JournalStore:
"body": body,
"entities": sorted(set(entities or [])),
"refs": [str(ref) for ref in refs or []],
+ # A finding's severity is the coworker's own claim: in the chain.
+ # Only written when given, so older entries hash as before.
+ **({"severity": severity} if severity else {}),
}
),
"taint": 1 if taint else 0,
@@ -156,8 +170,9 @@ class JournalStore:
"""
INSERT INTO journal_entries
(ts, case_id, kind, actor, actor_role, persona, model,
- session_id, space, item_id, payload, taint, prev_hash, hash)
- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+ session_id, space, item_id, payload, taint, prev_hash, hash,
+ step_call_id, evidence)
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""",
(
ts,
@@ -174,6 +189,8 @@ class JournalStore:
record["taint"],
prev,
record["hash"],
+ step_call_id,
+ json.dumps(list(evidence or [])) if evidence is not None else None,
),
)
if not exists:
@@ -193,7 +210,8 @@ class JournalStore:
except Exception:
self._conn.rollback()
raise
- return {**record, "seq": cursor.lastrowid}
+ return {**record, "seq": cursor.lastrowid, "step_call_id": step_call_id,
+ "evidence": list(evidence) if evidence is not None else None}
def read(
self,
@@ -428,6 +446,13 @@ def _row_to_entry(row: sqlite3.Row) -> dict[str, Any]:
entry["body"] = payload.get("body")
entry["entities"] = payload.get("entities") or []
entry["refs"] = payload.get("refs") or []
+ entry["severity"] = payload.get("severity")
+ # 1.3 (E17a): pointers outside the chain; None on entries written before them.
+ raw = entry.get("evidence")
+ try:
+ entry["evidence"] = json.loads(raw) if raw else None
+ except (TypeError, json.JSONDecodeError):
+ entry["evidence"] = None
entry["author"] = entry.pop("actor")
entry["role"] = entry.pop("actor_role")
entry["item"] = entry.pop("item_id")
diff --git a/coworker/teams/model.py b/coworker/teams/model.py
index 3f89a34b..a5f640f0 100644
--- a/coworker/teams/model.py
+++ b/coworker/teams/model.py
@@ -61,8 +61,11 @@ class Actor:
session_id: str = ""
-LINK_KINDS = ("parent", "blocks") # link(src, "parent", dst): dst is src's parent
+LINK_KINDS = ("parent", "blocks", "verifies") # link(src, "parent", dst): dst is src's parent
# link(src, "blocks", dst): src blocks dst
+ # link(src, "verifies", dst): src re-checks dst's fix
+ # (observability Phase 3, E17d: who verified a fix is a
+ # board fact, not a guess from proposal metadata)
# `note` is any observation — the journal is not only for investigations. `raw` is
# a capture (log excerpt, command output); reads skip raw unless asked, and large
diff --git a/coworker/teams/store.py b/coworker/teams/store.py
index 7be434fd..6fca76f2 100644
--- a/coworker/teams/store.py
+++ b/coworker/teams/store.py
@@ -574,9 +574,11 @@ class TeamStore:
description: str = "",
parent: Optional[int] = None,
case: Optional[str] = None,
+ verifies: Optional[int] = None,
) -> dict[str, Any]:
"""New item, `open` and unassigned. Acceptance criteria are load-bearing —
- required.
+ required. `verifies` names the item whose fix this one re-checks (a `verifies`
+ link, observability Phase 3 E17d).
Workers may create too — a bug spotted in passing, a follow-up — because
filing is harmless: nothing runs until the item is ASSIGNED, and assign
@@ -619,6 +621,12 @@ class TeamStore:
"case": case,
},
)
+ if verifies is not None:
+ self._item(space, int(verifies))
+ if int(verifies) == item_id:
+ raise BoardError("an item cannot verify itself")
+ event = self.append_event(space, ITEM_LINKED, actor, item_id=item_id,
+ payload={"src": item_id, "kind": "verifies", "dst": int(verifies)})
if self.journal is not None and case:
self.journal.ensure_case(case, actor.id)
return self.get_item(space, item_id, actor=actor, seq=event["seq"])
@@ -871,8 +879,10 @@ class TeamStore:
comment: str = "",
refs: Optional[list[str]] = None,
taint: bool = False,
+ coverage: Optional[dict[str, Any]] = None,
) -> dict[str, Any]:
target = ItemState(to)
+ report = _coverage_report(coverage) if coverage else None
with self._lock:
item = self._item(space, item_id)
current = ItemState(item["state"])
@@ -896,6 +906,9 @@ class TeamStore:
"to": target.value,
"comment": comment,
"refs": list(refs or []),
+ # Observability Phase 3 (E17c): the hand-off's structured coverage
+ # report, the coworker's own claim; only written when given.
+ **({"coverage": report} if report else {}),
},
taint=taint,
)
@@ -1323,7 +1336,7 @@ class TeamStore:
if row["src"] == item_id:
out.append({"kind": row["kind"], "item": row["dst"]})
else:
- inverse = "child" if row["kind"] == "parent" else "blocked_by"
+ inverse = {"parent": "child", "verifies": "verified_by"}.get(row["kind"], "blocked_by")
out.append({"kind": inverse, "item": row["src"]})
return out
@@ -1514,3 +1527,28 @@ def text_page(text: str, offset: int, max_chars: int) -> dict:
end = min(len(text), offset + max_chars)
return {"text": text[offset:end], "offset": offset, "next_offset": end,
"total_chars": len(text), "has_more": end < len(text)}
+
+
+COVERAGE_AREAS = ("code", "secrets", "deps", "iac")
+
+
+def _coverage_report(raw: Any) -> dict[str, dict[str, Any]]:
+ """A hand-off's coverage report, `{area: {ran, findings, reason?}}` (observability Phase 3,
+ E17c): areas from the fixed list, `ran` a flag, `findings` a count, `reason` a short text
+ (why a scan could not run). Anything else is refused, so the checks read only facts."""
+ if not isinstance(raw, dict):
+ raise BoardError("coverage must be an object of {area: {ran, findings, reason}}")
+ out: dict[str, dict[str, Any]] = {}
+ for area, value in raw.items():
+ if area not in COVERAGE_AREAS:
+ raise BoardError(f"coverage area must be one of {list(COVERAGE_AREAS)}, not {area!r}")
+ if not isinstance(value, dict) or not isinstance(value.get("ran"), bool):
+ raise BoardError(f"coverage[{area!r}] needs ran: true or false")
+ findings = value.get("findings", 0)
+ if isinstance(findings, bool) or not isinstance(findings, int) or findings < 0:
+ raise BoardError(f"coverage[{area!r}].findings must be a count")
+ entry: dict[str, Any] = {"ran": value["ran"], "findings": findings}
+ if value.get("reason"):
+ entry["reason"] = str(value["reason"])[:300]
+ out[area] = entry
+ return out
diff --git a/coworker/teams/tools.py b/coworker/teams/tools.py
index efddc3ed..ad7046c4 100644
--- a/coworker/teams/tools.py
+++ b/coworker/teams/tools.py
@@ -73,7 +73,8 @@ _CREATE_ITEM_SCHEMA = {
" assigned). `criteria` is the acceptance criteria — what gets verified"
" before the item can be done; required. `parent` links it under"
" another item; `case` names its journal case (children inherit the"
- " parent's case by default)."
+ " parent's case by default). `verifies` names the item whose fix this"
+ " one re-checks: give it to a different worker than the one who fixed it."
),
"parameters": {
"type": "object",
@@ -83,6 +84,7 @@ _CREATE_ITEM_SCHEMA = {
"description": {"type": "string"},
"parent": {"type": "integer"},
"case": {"type": "string"},
+ "verifies": {"type": "integer"},
},
"required": ["title", "criteria"],
},
@@ -126,6 +128,7 @@ def board_tools(
description: str = "",
parent: Optional[int] = None,
case: str = "",
+ verifies: Optional[int] = None,
) -> dict:
"""Create a work item (open, unassigned — work starts when it is
assigned). `criteria` is the acceptance criteria — what gets verified
@@ -141,6 +144,7 @@ def board_tools(
description=description,
parent=parent,
case=case or None,
+ verifies=verifies,
)
if "error" not in result:
on_change()
@@ -202,12 +206,16 @@ def board_tools(
"authority": "intent_not_access_grants"}
def transition(
- item: int, to: str, comment: str = "", refs: Optional[list] = None
+ item: int, to: str, comment: str = "", refs: Optional[list] = None,
+ coverage: Optional[dict] = None,
) -> dict:
"""Move a work item to a new state. Workers move their own item to
in_progress, blocked, or review (attach the blocker or a hand-off summary
as `comment`, and artifact pointers — branch, report, session — as
- `refs`); done requires review verification first."""
+ `refs`); done requires review verification first. When handing off a
+ security task, `coverage` reports each area you were to scan:
+ {"secrets": {"ran": false, "findings": 0, "reason": "gitleaks not installed"}},
+ areas code, secrets, deps, iac. Say ran: false when a scan could not run."""
recorded(item)
return receipt(_call(
store.transition,
@@ -218,6 +226,7 @@ def board_tools(
comment=comment,
refs=[str(ref) for ref in refs or []],
taint=taint(),
+ coverage=coverage,
))
def comment(item: int, body: str, refs: Optional[list] = None, needs_attention: bool = False) -> dict:
@@ -292,7 +301,12 @@ def journal_tools(
actor: Actor,
space: str = "",
taint: Callable[[], bool] = lambda: False,
+ scan_evidence: Callable[[Optional[int]], list] = lambda item: [],
) -> list:
+ """`scan_evidence(item)` gives the default evidence of a finding (observability Phase 3,
+ TD3-15): the scan steps of this session since the item was assigned, as
+ `[{session_id, call_id}]`. It never changes what is written, only the pointer kept."""
+
def journal_append(
case: str,
body: str,
@@ -300,13 +314,34 @@ def journal_tools(
item: Optional[int] = None,
entities: Optional[list] = None,
refs: Optional[list] = None,
+ severity: str = "",
+ evidence: Optional[list] = None,
) -> dict:
"""Append an entry to a journal case as you work: kind is finding,
evidence, decision, note (any observation), or raw (a capture like a log
excerpt — for large captures, save the full output to a file and journal
an excerpt that references it). `entities` are the concrete things it is
about (file paths, resource names, CVE ids) — they power later recall;
- `refs` are pointers (file:line, commit, url)."""
+ `refs` are pointers (file:line, commit, url). For a finding, `severity` is
+ critical, high, medium, low or info, and `evidence` lists the tool call ids
+ whose output shows it (the scan that found it); when you leave it out, the
+ scans you ran on this task are linked."""
+ from ..observability.engine_hooks import current_tool_run
+
+ run = current_tool_run()
+ here = getattr(run, "call_id", None)
+ sev = (severity or "").strip().lower() or None
+ if sev is not None and sev not in ("critical", "high", "medium", "low", "info"):
+ return {"error": "severity must be critical, high, medium, low or info"}
+ links = None
+ if kind == "finding":
+ if evidence:
+ links = [{"session_id": actor.session_id or "", "call_id": str(c)} for c in evidence if c]
+ else:
+ try:
+ links = list(scan_evidence(item))
+ except Exception: # noqa: BLE001 - a pointer never blocks the entry
+ links = []
return _call(
journal.append,
actor,
@@ -318,6 +353,9 @@ def journal_tools(
entities=[str(entity) for entity in entities or []],
refs=[str(ref) for ref in refs or []],
taint=taint(),
+ severity=sev,
+ step_call_id=here,
+ evidence=links,
)
def journal_read(
diff --git a/tests/test_observability_phase3_engine.py b/tests/test_observability_phase3_engine.py
new file mode 100644
index 00000000..a0909bd8
--- /dev/null
+++ b/tests/test_observability_phase3_engine.py
@@ -0,0 +1,341 @@
+"""Phase 3 engine items, one test each: the scanner facts on a real shell step (E17b), the
+1.3 turn facts an entry point passes (E18a, E18b, E26a, E25a), the wake reason of a team
+delivery (TD3-18), the journal's evidence links (E17a), the `verifies` link (E17d), the
+hand-off coverage report (E17c), stored series and task results, the ledger's 90 days and
+the feedback export (E27b)."""
+
+from __future__ import annotations
+
+import asyncio
+import hashlib
+import json
+from pathlib import Path
+
+import jsonschema
+import pytest
+
+from coworker.engine import TurnEngine
+from coworker.observability import Recorder, RecordingStore, scopes, signals
+from coworker.observability import export as ex
+from coworker.observability import export_spans as spans
+from coworker.permissions import Mode, PermissionEngine
+from coworker.providers import AssistantTurn, ModelCapabilities, ProviderClient, ToolCall
+from coworker.providers.base import TokenUsage
+from coworker.server.manager import SessionManager
+from coworker.teams.journal import JournalStore
+from coworker.teams.model import Actor, Role
+from coworker.teams.store import BoardError, TeamStore
+from coworker.tools import ToolRegistry
+
+SCHEMA = json.loads((Path(__file__).parent / "corpora/observability/recording-format.schema.json").read_text("utf-8"))
+
+
+class _Scripted(ProviderClient):
+ def __init__(self, turns):
+ self._turns = list(turns)
+
+ def complete(self, *, model, messages, tools=None, **settings):
+ return self._turns.pop(0)
+
+ def capabilities(self, model):
+ return ModelCapabilities()
+
+
+class _Meta:
+ category: str = ""
+ risk_level: str = "low"
+ requires_approval: bool = False
+
+
+def run_shell(command: str) -> dict:
+ """Run a shell command.
+
+ Args:
+ command: the command
+ """
+ return {"exit_code": 1 if "gitleaks" in command else 0, "stdout": "found 2 leaks"}
+
+
+def _call(cid, name, args):
+ return AssistantTurn(tool_calls=[ToolCall(id=cid, name=name, arguments=args)], finish_reason="tool_calls",
+ usage=TokenUsage(input=10, output=2))
+
+
+def _engine(tmp_path, turns):
+ registry = ToolRegistry()
+ registry.register(run_shell, metadata=_Meta())
+ engine = TurnEngine(
+ provider=_Scripted(turns), registry=registry,
+ permissions=PermissionEngine(workspace_root=tmp_path, mode=Mode.BYPASS_APPROVALS),
+ model="openai:model-x", instructions="You are a test coworker.", max_iterations=12,
+ )
+ engine.audit_context = {"session_id": "sess_p3"}
+ engine.persona_id, engine.persona_fp, engine.persona_coverage = "security", "a1b2c3", ["secrets"]
+ rec = Recorder()
+ engine.recorder = rec
+ return engine, rec
+
+
+def test_a_shell_step_records_its_scanner_and_result_hash(tmp_path):
+ engine, rec = _engine(tmp_path, [
+ _call("c1", "run_shell", {"command": "npx gitleaks detect --source ."}),
+ _call("c2", "run_shell", {"command": "grep -r semgrep docs"}),
+ AssistantTurn(text="done", usage=TokenUsage(input=10, output=2)),
+ ])
+
+ async def go():
+ return [ev async for ev in engine.run("scan", trigger="team_wake", team={"team_id": "t1", "role": "worker"},
+ wake={"reason": "assigned", "item_ids": [7], "board_seq": 41})]
+
+ asyncio.run(go())
+ d = rec.as_dict()
+ tools = {s["call_id"]: s["attrs"] for s in d["steps"] if s["kind"] == "tool"}
+ assert tools["c1"]["scanner"] == {"name": "gitleaks", "action": "scan"} and tools["c1"]["exit_code"] == 1
+ assert tools["c2"]["scanner"] is None
+ raw = json.dumps({"exit_code": 1, "stdout": "found 2 leaks"}, ensure_ascii=False).encode("utf-8")
+ assert tools["c1"]["result_sha256"] == hashlib.sha256(raw).hexdigest()
+ turn = d["turns"][0]
+ assert turn["stamp"]["persona_id"] == "security" and turn["stamp"]["persona_coverage"] == ["secrets"]
+ assert turn["stamp"]["team"] == {"team_id": "t1", "role": "worker"}
+ assert turn["wake"] == {"reason": "assigned", "kinds": [], "item_ids": ["7"], "board_seq": 41,
+ "batched": None, "deferred": None, "by": None}
+ errs = list(jsonschema.Draft202012Validator({**SCHEMA, "$ref": "#/$defs/recording"}).iter_errors(d))
+ assert not errs, errs[0].message
+
+
+def test_an_automation_turn_carries_its_run_and_a_bad_wake_reason_reads_unknown(tmp_path):
+ engine, rec = _engine(tmp_path, [AssistantTurn(text="brief", usage=TokenUsage(input=10, output=2))])
+
+ async def go():
+ return [ev async for ev in engine.run("go", trigger="schedule", wake={"reason": "bogus"},
+ automation={"task_id": "task-1", "run_id": "run-1", "trigger": "catch_up"})]
+
+ asyncio.run(go())
+ turn = rec.as_dict()["turns"][0]
+ assert turn["automation"] == {"task_id": "task-1", "run_id": "run-1", "trigger": "catch_up"}
+ assert turn["wake"]["reason"] == "unknown" and turn["trigger"] == "schedule"
+
+
[email protected]("rows,lead,want", [
+ ([{"kind": "moved", "to": "review", "item": 3}, {"kind": "assigned", "item": 4}], True, "review"),
+ ([{"kind": "moved", "to": "blocked", "item": 3}, {"kind": "waiting", "item": 4}], True, "blocker"),
+ ([{"kind": "chat"}, {"kind": "chat"}], False, "mention"),
+ ([{"kind": "moved", "to": "done", "item": 3}], False, "prereq_done"),
+ ([{"kind": "moved", "to": "done", "item": 3}], True, "board"),
+ ([{"kind": "claimed", "item": 5}, {"kind": "chat"}], True, "assigned"),
+])
+def test_one_wake_reason_by_fixed_priority(rows, lead, want):
+ reason, kinds, items = SessionManager._wake_from_rows(rows, is_lead=lead)
+ assert reason == want
+ assert kinds == list(dict.fromkeys(r["kind"] for r in rows))
+
+
+def _team(tmp_path):
+ journal = JournalStore(tmp_path / "journal.db")
+ store = TeamStore(tmp_path / "board.db", journal=journal)
+ lead = Actor(id="lead:1", role=Role.LEAD, persona="devsecops-lead", session_id="lead-s")
+ worker = Actor(id="appsec:1", role=Role.WORKER, persona="appsec-worker", session_id="w-s")
+ return store, journal, lead, worker
+
+
+def test_verifies_is_a_board_link_both_ways(tmp_path):
+ store, _j, lead, _w = _team(tmp_path)
+ fix = store.create_item("space", lead, title="Fix SQLi", criteria="query is parameterised")
+ check = store.create_item("space", lead, title="Verify the fix", criteria="rerun semgrep", verifies=fix["id"])
+ assert {"kind": "verifies", "item": fix["id"]} in check["links"]
+ again = store.get_item("space", fix["id"], actor=lead)
+ assert {"kind": "verified_by", "item": check["id"]} in again["links"]
+ with pytest.raises(BoardError):
+ store.create_item("space", lead, title="x", criteria="y", verifies=999)
+
+
+def test_a_hand_off_carries_a_checked_coverage_report(tmp_path):
+ store, _j, lead, worker = _team(tmp_path)
+ item = store.create_item("space", lead, title="Scan", criteria="all areas")
+ store.assign("space", lead, item["id"], worker.id)
+ store.transition("space", worker, item["id"], "in_progress")
+ store.transition("space", worker, item["id"], "review", comment="done",
+ coverage={"secrets": {"ran": False, "findings": 0, "reason": "gitleaks missing"},
+ "code": {"ran": True, "findings": 3}})
+ ev = [e for e in store.events("space", item_id=item["id"]) if e["kind"] == "item_transitioned"][-1]
+ assert ev["payload"]["coverage"] == {"secrets": {"ran": False, "findings": 0, "reason": "gitleaks missing"},
+ "code": {"ran": True, "findings": 3}}
+ for bad in ({"network": {"ran": True}}, {"code": {"ran": "yes"}}, {"code": {"ran": True, "findings": -1}}):
+ with pytest.raises(BoardError):
+ store.transition("space", worker, item["id"], "in_progress", coverage=bad)
+
+
+def test_journal_evidence_is_outside_the_hash_chain(tmp_path):
+ _s, journal, lead, worker = _team(tmp_path)
+ plain = journal.append(worker, "case-1", "an old-style note")
+ found = journal.append(worker, "case-1", "SQLi in login", kind="finding", severity="high",
+ step_call_id="call_9", evidence=[{"session_id": "w-s", "call_id": "call_4"}])
+ assert found["evidence"] == [{"session_id": "w-s", "call_id": "call_4"}] and found["step_call_id"] == "call_9"
+ rows = journal.read(worker, "case-1")
+ assert rows[0]["evidence"] is None and rows[0]["severity"] is None
+ assert rows[1]["evidence"] == [{"session_id": "w-s", "call_id": "call_4"}] and rows[1]["severity"] == "high"
+ # changing the pointer does not touch the chain
+ with journal._lock:
+ journal._conn.execute("UPDATE journal_entries SET evidence = '[]' WHERE seq = ?", (found["seq"],))
+ assert journal.read(worker, "case-1")[1]["hash"] == found["hash"]
+ assert plain["hash"] != found["hash"]
+
+
+def test_journal_append_links_the_sessions_scans_by_default(tmp_path):
+ from coworker.teams.tools import journal_tools
+
+ _s, journal, _lead, worker = _team(tmp_path)
+ tools = {t.__name__: t for t in journal_tools(
+ journal, actor=worker, space="space",
+ scan_evidence=lambda item: [{"session_id": "w-s", "call_id": "call_2"}])}
+ out = tools["journal_append"]("case-2", "a leaked key", kind="finding", severity="critical")
+ assert out["evidence"] == [{"session_id": "w-s", "call_id": "call_2"}]
+ named = tools["journal_append"]("case-2", "another", kind="finding", evidence=["call_7"])
+ assert named["evidence"] == [{"session_id": "w-s", "call_id": "call_7"}]
+ note = tools["journal_append"]("case-2", "just a note")
+ assert note["evidence"] is None
+ assert "error" in tools["journal_append"]("case-2", "x", kind="finding", severity="severe")
+
+
+def _write(store, turns, steps, issues=None):
+ for t in turns:
+ store.write_turn(t)
+ for s in steps:
+ store.write_step(s)
+ for t in turns:
+ store.write_issues(t["turn_id"], (issues or {}).get(t["turn_id"]) or
+ {"signals_version": signals.version(), "warn": 0, "info": 0, "issues": [], "fine": []}, seen=True)
+ store.flush()
+
+
+def _turn(tid, sid, seq, at, wake, team=None, status="completed"):
+ stamp = {"machine": {"kind": "this_machine", "id": None}, "persona_id": "devsecops-lead"}
+ if team:
+ stamp["team"] = team
+ return {"turn_id": tid, "session_id": sid, "seq": seq, "kind": "run", "trigger": "team_wake", "started_at": at,
+ "duration_ms": 1000, "status": status, "incomplete": False, "approval_mode": "interactive",
+ "attended": True, "reviewer": None, "stamp": stamp, "first_message_id": None,
+ "last_message_id": None, "schema_version": 1, "wake": wake}
+
+
+def _step(tid, sid, n, at):
+ return {"step_id": f"{tid}-s{n}", "turn_id": tid, "session_id": sid, "parent_step_id": None, "seq": n,
+ "kind": "model", "name": "model", "started_at": at, "duration_ms": 100, "status": "ok",
+ "call_id": None, "call_index": None, "message_id": None, "shadow": False, "after_turn": False,
+ "attrs": {"purpose": "main", "model": "model-x", "usage": {"reported": True, "input_fresh": 10,
+ "cache_read": 0, "cache_write": 0, "output": 1}},
+ "content": {}}
+
+
+def test_lead_polling_is_stored_once_on_the_last_turn_of_the_run(tmp_path):
+ manager = SessionManager(workspace=tmp_path, provider=_Scripted([]))
+ store = manager.recordings_store
+ team = {"team_id": "t1", "role": "lead"}
+ turns = [_turn(f"L{i}", "lead-s", i, 1_000_000 + i * 60_000, {"reason": "timer", "board_seq": 5}, team)
+ for i in range(1, 5)]
+ _write(store, turns, [_step(t["turn_id"], "lead-s", 1, t["started_at"]) for t in turns])
+ manager._scope_lead_polling("lead-s")
+ got = {tid: [i["check"] for i in (store.read_turn_issues(tid) or {}).get("issues", [])] for tid in ("L3", "L4")}
+ assert got == {"L3": [], "L4": ["lead_polling"]}
+ issue = store.read_turn_issues("L4")
+ assert issue["warn"] == 1 and issue["seen"] is False
+ assert issue["issues"][0]["text"] == "The lead woke on its timer 4 times with no board change."
+ # the ledger follows the stored result
+ [t4] = [t for t in store.usage_turns(0, 2**62) if t["turn_id"] == "L4"]
+ assert t4["warn"] == 1 and "lead_polling" in t4["warn_checks"]
+ # a re-check of the turn alone keeps the series result
+ rec = store.load_recording("lead-s")
+ store.write_issues("L4", {**issue, "signals_version": "old"})
+ store.flush()
+ out = manager._recording_issues("lead-s", rec)
+ assert [i["check"] for i in out["L4"]["issues"]] == ["lead_polling"]
+ manager.observability.close()
+
+
+def test_the_ledger_is_kept_90_days(tmp_path):
+ store = RecordingStore(tmp_path / "recordings.db", background=False)
+ now = 200 * 24 * 3600 * 1000
+ old = _turn("old", "s", 1, now - 91 * 24 * 3600 * 1000, {"reason": "user"})
+ new = _turn("new", "s", 2, now - 40 * 24 * 3600 * 1000, {"reason": "user"})
+ _write(store, [old, new], [_step("old", "s", 1, old["started_at"]), _step("new", "s", 1, new["started_at"])])
+ store.sweep(now_ms=now) # recordings: 30 days
+ assert store.load_recording("s")["turns"] == []
+ store.sweep_ledger(now_ms=now) # the ledger: 90 days
+ assert [t["turn_id"] for t in store.usage_turns(0, 2**62)] == ["new"]
+ store.close()
+
+
+def test_ratings_go_out_as_a_langfuse_score_and_an_otlp_span(tmp_path):
+ store = RecordingStore(tmp_path / "recordings.db", background=False)
+ t = _turn("T1", "s", 1, 5_000, {"reason": "user"})
+ _write(store, [t], [_step("T1", "s", 1, 5_000)])
+ store.put_feedback({"turn_id": "T1", "session_id": "s", "rating": "down", "reasons": ["did_unasked"],
+ "note": "used my key sk-PLANT", "auto_approved": []})
+ posts = []
+
+ def post(url, body, headers, timeout):
+ posts.append((url, json.loads(body)))
+ return 200, ""
+
+ for kind, prompts in (("langfuse", False), ("otlp", True)):
+ posts.clear()
+ dest = ex.Destination(kind="langfuse", host="http://lf.test") if kind == "langfuse" else \
+ ex.Destination(kind="otlp", url="http://otel.test/v1/traces")
+ cfg = ex.ExportConfig(destination=dest, headers={}, include={"messages": prompts}, since_ms=0)
+ exp = ex.Exporter(store, cfg, post=post, secret_values=lambda: ["sk-PLANT"])
+ exp.flush_once()
+ fb = [p for p in posts if "scores" in p[0] or "feedback" in json.dumps(p[1])]
+ assert len(fb) == 1
+ if kind == "langfuse":
+ body = fb[0][1]
+ assert fb[0][0] == "http://lf.test/api/public/scores"
+ assert body["name"] == "user_feedback" and body["value"] == -1
+ assert body["id"] == spans.feedback_id("T1") and body["traceId"] == spans.trace_id("T1")
+ assert body["metadata"] == {"reasons": ["did_unasked"]} and "comment" not in body
+ else:
+ span = fb[0][1]["resourceSpans"][0]["scopeSpans"][0]["spans"][0]
+ attrs = {a["key"]: a["value"] for a in span["attributes"]}
+ assert span["name"] == "openworker.feedback" and span["parentSpanId"] == spans.root_span_id("T1")
+ assert attrs["openworker.feedback.rating"] == {"stringValue": "down"}
+ assert "PLANT" not in json.dumps(attrs) and "openworker.feedback.note" in attrs
+ # sent once; a changed rating goes again, under the same fixed id
+ posts.clear()
+ exp.flush_once()
+ assert not [p for p in posts if "scores" in p[0] or "feedback" in json.dumps(p[1])]
+ store.close()
+
+
+def test_the_turn_span_names_the_coworker_and_keeps_agent_name(tmp_path):
+ t = _turn("T1", "s", 1, 5_000, {"reason": "review"}, {"team_id": "t1", "role": "lead"})
+ t["stamp"]["persona_fp"] = "a1b2c3"
+ t["automation"] = {"task_id": "task-1", "run_id": "run-1", "trigger": "manual"}
+ built = spans.build_turn_spans(t, [_step("T1", "s", 1, 5_000)], sw=spans.switches_from({}), dialect="semconv",
+ get_piece=lambda k: None, masker=spans.Masker(), include_root=True,
+ only=None, session_title=None)
+ root = built[0]["attrs"]
+ assert root["gen_ai.agent.name"] == "coworker" # TD3-25: the 1.0 value stays
+ assert root["openworker.persona.id"] == "devsecops-lead" and root["openworker.persona.fp"] == "a1b2c3"
+ assert root["openworker.wake.reason"] == "review" and root["openworker.team.role"] == "lead"
+ assert root["openworker.automation.run_id"] == "run-1" and root["openworker.automation.trigger"] == "manual"
+
+
+def test_run_health_counts_a_stuck_run_as_failed():
+ turn = {"status": "completed", "warn_checks": [], "ended_at": 1_000}
+ assert scopes.run_health({"status": "running"}, turn, now_ms=1_000 + 600_001) == "failed"
+ assert scopes.run_health({"status": "running"}, turn, now_ms=1_000 + 60_000) == "ok"
+ assert scopes.run_health({"status": "ok"}, {**turn, "warn_checks": ["model_failed"]}) == "failed"
+ assert scopes.run_health({"status": "ok"}, None) == "unknown"
+
+
+def test_turns_recorded_before_the_ledger_are_filled_in(tmp_path):
+ manager = SessionManager(workspace=tmp_path, provider=_Scripted([]))
+ store = manager.recordings_store
+ t = _turn("T0", "s-old", 1, 1_000, {"reason": "user"})
+ _write(store, [t], [_step("T0", "s-old", 1, 1_000)])
+ with store._lock, store._conn:
+ store._conn.execute("DELETE FROM turn_usage") # as a Phase 2 store has none
+ assert store.turns_without_usage() == ["T0"]
+ assert manager.backfill_usage() == 1
+ [row] = store.usage_turns(0, 2**62)
+ assert row["total"] == 11 and store.turns_without_usage() == []
+ manager.observability.close()
diff --git a/tests/test_remote_home.py b/tests/test_remote_home.py
index c302ca39..e9e42241 100644
--- a/tests/test_remote_home.py
+++ b/tests/test_remote_home.py
@@ -2499,3 +2499,44 @@ async def test_browser_sealed_managed_grant_lands_on_the_box(tmp_path):
task.cancel()
with pytest.raises(asyncio.CancelledError):
await task
+
+
+async def test_offline_machine_serves_its_last_usage_rollup_as_of(tmp_path):
+ """Observability Phase 3 (E28, TD3-6): the desktop keeps each machine's last Usage rollup
+ per grouping; while the machine is offline the proxy answers with it, labelled `cached`
+ with the time it was fetched, never as live numbers."""
+ async with _Server(_controller(tmp_path)) as server:
+ join_url = await _arm(server.base)
+ _, token = parse_join_url(join_url)
+ state = tmp_path / "rollup-state"
+ state.mkdir()
+ box_manager = SessionManager(data_dir=tmp_path / "rollup-data")
+ task = asyncio.create_task(
+ run_joined(state=state, controller=server.base, name="usage-box", token=token,
+ app=create_app(box_manager), once=True, log=lambda *_: None)
+ )
+ machines = await _wait_connected(server.base)
+ mid = machines[0]["id"]
+ async with httpx.AsyncClient() as client:
+ live = await client.get(f"{server.base}/v1/machines/{mid}/p/v1/observability/rollup",
+ params={"group": "all", "from": 0})
+ assert live.status_code == 200 and "cached" not in live.json()
+ task.cancel()
+ with pytest.raises(asyncio.CancelledError):
+ await task
+ deadline = asyncio.get_running_loop().time() + 5
+ while True:
+ listing = await client.get(f"{server.base}/v1/machines")
+ if not any(m["connected"] for m in listing.json()["machines"]):
+ break
+ assert asyncio.get_running_loop().time() < deadline
+ await asyncio.sleep(0.05)
+ off = await client.get(f"{server.base}/v1/machines/{mid}/p/v1/observability/rollup",
+ params={"group": "all", "from": 0})
+ assert off.status_code == 200
+ body = off.json()
+ assert body["cached"]["as_of"] > 0 and body["groups"] == live.json()["groups"]
+ # another grouping was never fetched: no copy, the machine is just offline
+ none = await client.get(f"{server.base}/v1/machines/{mid}/p/v1/observability/rollup",
+ params={"group": "day"})
+ assert none.status_code == 503