Skip to content
Python

fix(observability): recordings fit the shared schema; test the engine's own recordings against it (OPE-244)

Devika Verma158c4bb004f5
diff --git a/coworker/engine.py b/coworker/engine.py
index ce101c3d..59f29301 100644
--- a/coworker/engine.py
+++ b/coworker/engine.py
@@ -741,12 +741,13 @@ class TurnEngine:
return
block = self._last_context_block or ""
block_hash = _short_hash(block) if block else None
+ skills = sorted(getattr(getattr(self, "skill_loader", None), "_skills", None) or {})
per_turn = {
"mode_note": self.permissions.mode.value if self.permissions.mode in (Mode.PLAN, Mode.DISCUSS) else None,
- "folders": len(getattr(self, "roots", None) or []) if isinstance(getattr(self, "roots", None), (list, tuple)) else None,
+ "folders": len(getattr(self.permissions, "roots", None) or []),
"sandbox_note": getattr(getattr(self, "sandbox_workspace", None), "describe", None) is not None,
- "skills_menu": None,
- "skills_hash": None,
+ "skills_menu": len(skills),
+ "skills_hash": _short_hash(",".join(skills)) if skills else None,
}
if self._context_turn != obs.turn_id:
self._context_turn, self._turn_context_hash = obs.turn_id, block_hash
diff --git a/coworker/observability/engine_hooks.py b/coworker/observability/engine_hooks.py
index 9c9cbcad..28fc261e 100644
--- a/coworker/observability/engine_hooks.py
+++ b/coworker/observability/engine_hooks.py
@@ -60,8 +60,9 @@ class TurnRecording:
# -- sending ------------------------------------------------------------------------------
def _base(self, typ: str, turn_id: Optional[str], session_id: str) -> dict[str, Any]:
at, mono = now_ms()
+ # A step with no session (a proxy call) or no turn records null, as the format says.
return {"type": typ, "v": EVENT_V, "at_ms": at, "mono_ms": mono,
- "session_id": session_id, "turn_id": turn_id or ""}
+ "session_id": session_id or None, "turn_id": turn_id or None}
def _send(self, ev: dict[str, Any]) -> None:
if self._held is not None and ev["type"] != "turn_started" and ev.get("turn_id") == self.turn_id:
diff --git a/coworker/sandbox/workspace.py b/coworker/sandbox/workspace.py
index c2a52b9e..61d01d45 100644
--- a/coworker/sandbox/workspace.py
+++ b/coworker/sandbox/workspace.py
@@ -287,7 +287,7 @@ class RunnerWorkspace(Workspace):
# `_blocked` is for the recording's tool step (observability E13, V-04); the engine
# takes it out of the result before the model sees it.
blocked = [
- {"site": entry, "count": n, "first_at": int(first[entry] * 1000),
+ {**_host_port(entry), "count": n, "first_at": int(first[entry] * 1000),
"late": first[entry] < started - 0.5, "uncertain": True}
for entry, n in counts.items()
]
@@ -339,6 +339,14 @@ def _site_name(entry: str) -> str:
return entry[:-4] if entry.endswith(":443") else entry
+def _host_port(entry: str) -> dict[str, Any]:
+ """A blocked-site entry ("host:port", or a bare host) as the recording's {host, port}."""
+ host, sep, port = str(entry).rpartition(":")
+ if sep and port.isdigit() and host:
+ return {"host": host.strip("[]"), "port": int(port)}
+ return {"host": str(entry), "port": 443}
+
+
def network_note_text(counts: dict[str, int], *, can_ask: bool, late: bool = False) -> str:
"""The note itself (owner wording 2026-10-03: open about what the sandbox saw, and
about not knowing whether it is what broke the command). `late`: some of the refusals
diff --git a/tests/corpora/observability/recording-format.schema.json b/tests/corpora/observability/recording-format.schema.json
new file mode 100644
index 00000000..c6466525
--- /dev/null
+++ b/tests/corpora/observability/recording-format.schema.json
@@ -0,0 +1,547 @@
+{
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
+ "$id": "urn:openworker:observability:recording-format:1.0",
+ "title": "OpenWorker observability recording format 1.0",
+ "description": "Turns, steps, recording events and exchange pieces, as defined in recording-format.md (version 1.0, frozen 2026-10-31). Field names are those of phase-1/technical-design.md (Draft 5). Validate a recording file against #/$defs/recording, one recording event against #/$defs/event, one line of an exchange piece store against #/$defs/piece, one live message against #/$defs/live_message. Wherever a field may be null, its description says what null means.",
+ "$ref": "#/$defs/recording",
+ "$defs": {
+
+ "id": {"type": "string", "minLength": 1},
+ "id_or_null": {"type": ["string", "null"], "minLength": 1},
+ "epoch_ms": {"type": "integer", "minimum": 0, "description": "Wall-clock time, epoch milliseconds."},
+ "count": {"type": "integer", "minimum": 0},
+ "count_or_null": {"type": ["integer", "null"], "minimum": 0},
+ "sha256": {"type": "string", "pattern": "^[0-9a-f]{64}$"},
+ "short_hash": {"type": ["string", "null"], "pattern": "^[0-9a-f]{6}$", "description": "First 6 hex characters of a SHA-256. null = nothing to hash."},
+ "step_kind": {"enum": ["context", "model", "tool", "wait", "auto_approver", "summary", "notice"]},
+ "step_status": {"enum": ["running", "ok", "error", "denied", "interrupted", "skipped", "unknown"]},
+ "turn_status": {"enum": ["completed", "max_iterations_exceeded", "truncated", "sleeping", "interrupted", "error", "unknown"]},
+ "turn_kind": {"enum": ["run", "resume", "retry"]},
+ "trigger": {"enum": ["user", "channel", "schedule", "team_wake", "unknown"]},
+ "reviewer_kind": {"enum": ["session_model", "openai_decisions", "typesafe_jev"]},
+ "wire": {"enum": ["anthropic", "openai_responses", "openai_chat", "gemini", "bedrock_converse", "ollama", "openai_decisions", "typesafe_jev"]},
+
+ "recording": {
+ "description": "One recording file: the turns and steps of one or more sessions. Turns are sorted by started_at, then turn_id. Steps are sorted by seq within their turn (the canonical order, recording-format.md section 4); steps of no turn (proxy calls) follow, by started_at.",
+ "type": "object",
+ "required": ["turns", "steps"],
+ "additionalProperties": false,
+ "properties": {
+ "turns": {"type": "array", "items": {"$ref": "#/$defs/turn"}},
+ "steps": {"type": "array", "items": {"$ref": "#/$defs/step"}}
+ }
+ },
+
+ "reviewer_settings": {
+ "description": "Auto-approver settings at turn start (technical design section 4).",
+ "type": "object",
+ "additionalProperties": false,
+ "required": ["active", "shadow", "kind"],
+ "properties": {
+ "active": {"type": "boolean"},
+ "shadow": {"type": "boolean"},
+ "kind": {"anyOf": [{"$ref": "#/$defs/reviewer_kind"}, {"type": "null"}], "description": "null = no reviewer is configured."},
+ "model": {"type": ["string", "null"], "description": "The model used, for kind session_model. null = not a session-model reviewer, or none configured."}
+ }
+ },
+
+ "stamp": {
+ "description": "Machine, model provider, sandbox and coworker facts taken at turn start (technical design section 9): the groups machine, model and sandbox, and the coworker fingerprint persona_fp. Every key is optional; a missing key means it was not stamped.",
+ "type": "object",
+ "additionalProperties": false,
+ "properties": {
+ "machine": {
+ "type": "object",
+ "additionalProperties": false,
+ "properties": {
+ "kind": {"enum": ["this_machine", "own_machine", "cloud"]},
+ "id": {"type": ["string", "null"], "description": "null = the machine has no id (not joined to a controller)."},
+ "name": {"type": "string", "description": "The machine's name, or the host name. Personal data for Phase 2's export rules (P-7)."},
+ "app_version": {"type": "string", "description": "\"unknown\" when only 0.0.0 is available (C6); never 0.0.0."}
+ }
+ },
+ "model": {
+ "type": "object",
+ "additionalProperties": false,
+ "properties": {
+ "id": {"type": "string"},
+ "provider_title": {"type": "string", "description": "The provider card's name in Settings; identifies which key."},
+ "kind": {"enum": ["api_key", "subscription", "local"]}
+ }
+ },
+ "sandbox": {
+ "type": "object",
+ "additionalProperties": false,
+ "properties": {
+ "provider": {"type": "string", "description": "Which sandbox, or direct."},
+ "enforcement": {"enum": ["full", "none"]},
+ "profile": {"enum": ["allowlist", "open", null], "description": "null = no network profile (direct)."},
+ "sites": {"type": ["integer", "null"], "minimum": 0, "description": "null = no allowlist."},
+ "runner_version": {"type": ["string", "null"]},
+ "image": {"type": ["string", "null"]}
+ }
+ },
+ "persona_fp": {"type": "string", "pattern": "^[0-9a-f]{6}$", "description": "Coworker fingerprint: first 6 hex characters of the section 9.4 hash. The session record keeps the same value."}
+ }
+ },
+
+ "turn": {
+ "description": "One turn: the engine working on one request (technical design section 4).",
+ "type": "object",
+ "additionalProperties": false,
+ "required": ["turn_id", "session_id", "seq", "kind", "trigger", "started_at", "duration_ms", "status", "incomplete",
+ "approval_mode", "attended", "reviewer", "stamp", "first_message_id", "last_message_id", "schema_version"],
+ "properties": {
+ "turn_id": {"$ref": "#/$defs/id", "description": "A ULID in the app; any unique string in tests."},
+ "session_id": {"$ref": "#/$defs/id"},
+ "seq": {"type": "integer", "minimum": 1, "description": "1st, 2nd, 3rd... turn of the session."},
+ "kind": {"$ref": "#/$defs/turn_kind"},
+ "trigger": {"$ref": "#/$defs/trigger"},
+ "started_at": {"$ref": "#/$defs/epoch_ms"},
+ "duration_ms": {"type": ["integer", "null"], "minimum": 0, "description": "From a monotonic clock. null = unknown: the turn was found open after a crash (status unknown)."},
+ "ended_at": {"type": ["integer", "null"], "description": "Optional and derived: started_at + duration_ms. null when duration_ms is null. Never read from a clock."},
+ "status": {"$ref": "#/$defs/turn_status"},
+ "incomplete": {"type": "boolean", "description": "True if any recording event of this turn was dropped."},
+ "approval_mode": {"type": "string", "description": "engine.permissions.mode at turn start, for example auto-approve."},
+ "attended": {"type": "boolean"},
+ "reviewer": {"$ref": "#/$defs/reviewer_settings"},
+ "stamp": {"$ref": "#/$defs/stamp"},
+ "first_message_id": {"$ref": "#/$defs/id_or_null", "description": "mid of the message that opened the turn. null = empty: a retry that appended no message, or a message written before Phase 1."},
+ "last_message_id": {"$ref": "#/$defs/id_or_null", "description": "mid of the last message appended before the turn ended. null = no message was appended."},
+ "schema_version": {"const": 1, "description": "The record layout version: 1 for every 1.x version of this format. It goes up only when old readers cannot read the new shape."}
+ }
+ },
+
+ "step": {
+ "description": "One unit of work inside a turn (technical design section 5.1). attrs and content depend on kind.",
+ "type": "object",
+ "additionalProperties": false,
+ "required": ["step_id", "turn_id", "session_id", "parent_step_id", "seq", "kind", "name", "started_at", "duration_ms",
+ "status", "call_id", "call_index", "message_id", "shadow", "after_turn", "attrs", "content"],
+ "properties": {
+ "step_id": {"$ref": "#/$defs/id"},
+ "turn_id": {"$ref": "#/$defs/id_or_null", "description": "null = empty: a proxy call, which belongs to no turn."},
+ "session_id": {"$ref": "#/$defs/id_or_null", "description": "null = empty: a proxy call, which belongs to no session."},
+ "parent_step_id": {"$ref": "#/$defs/id_or_null", "description": "null = not nested. Real nesting only; auto-approver and wait steps link to their tool call by call_id instead (V-09)."},
+ "seq": {"type": "integer", "minimum": 1, "description": "Position in the turn's canonical order; final when the turn ends."},
+ "kind": {"$ref": "#/$defs/step_kind"},
+ "name": {"type": "string", "minLength": 1},
+ "started_at": {"$ref": "#/$defs/epoch_ms", "description": "The turn's started_at plus the monotonic time since the turn started; steps with no turn read the wall clock."},
+ "duration_ms": {"type": ["integer", "null"], "minimum": 0, "description": "From a monotonic clock. null = still running (live only), or status unknown (found open after a crash)."},
+ "ended_at": {"type": ["integer", "null"], "description": "Optional and derived: started_at + duration_ms; null when duration_ms is null."},
+ "status": {"$ref": "#/$defs/step_status", "description": "running only while live."},
+ "call_id": {"$ref": "#/$defs/id_or_null", "description": "The tool_call_id. null = the step is not about a tool call."},
+ "call_index": {"type": ["integer", "null"], "minimum": 0, "description": "The call's position in the model's list of tool calls for that reply. null = not a tool call."},
+ "message_id": {"$ref": "#/$defs/id_or_null", "description": "mid of the transcript message this step produced. null = it produced none."},
+ "shadow": {"type": "boolean", "description": "True only for Shadow evaluation checks and their nested model step."},
+ "after_turn": {"type": "boolean", "description": "True when tagged with a turn that had already ended when the step started."},
+ "attrs": {"type": "object"},
+ "content": {"type": "object", "description": "Text and personal data only (TD-13). {} when there is none."}
+ },
+ "allOf": [
+ {"if": {"properties": {"kind": {"const": "model"}}},
+ "then": {"properties": {"attrs": {"$ref": "#/$defs/model_attrs"}, "content": {"$ref": "#/$defs/model_content"},
+ "status": {"enum": ["running", "ok", "error", "interrupted", "unknown"]}}}},
+ {"if": {"properties": {"kind": {"const": "tool"}}},
+ "then": {"properties": {"attrs": {"$ref": "#/$defs/tool_attrs"}, "content": {"$ref": "#/$defs/tool_content"},
+ "status": {"enum": ["running", "ok", "error", "denied", "interrupted", "skipped", "unknown"]}}}},
+ {"if": {"properties": {"kind": {"const": "wait"}}},
+ "then": {"properties": {"attrs": {"$ref": "#/$defs/wait_attrs"}, "content": {"$ref": "#/$defs/wait_content"},
+ "status": {"enum": ["running", "ok", "error", "interrupted", "unknown"]}}}},
+ {"if": {"properties": {"kind": {"const": "auto_approver"}}},
+ "then": {"properties": {"attrs": {"$ref": "#/$defs/auto_approver_attrs"}, "content": {"$ref": "#/$defs/auto_approver_content"},
+ "status": {"enum": ["running", "ok", "error", "interrupted", "unknown"]}}}},
+ {"if": {"properties": {"kind": {"const": "context"}}},
+ "then": {"properties": {"attrs": {"$ref": "#/$defs/context_attrs"}, "content": {"$ref": "#/$defs/context_content"},
+ "status": {"const": "ok"}}}},
+ {"if": {"properties": {"kind": {"const": "summary"}}},
+ "then": {"properties": {"name": {"enum": ["summary", "trim"]}, "attrs": {"$ref": "#/$defs/summary_attrs"},
+ "content": {"$ref": "#/$defs/no_content"},
+ "status": {"enum": ["running", "ok", "error", "interrupted", "unknown"]}}}},
+ {"if": {"properties": {"kind": {"const": "notice"}}},
+ "then": {"properties": {"name": {"$ref": "#/$defs/notice_name"}, "attrs": {"$ref": "#/$defs/notice_attrs"},
+ "content": {"$ref": "#/$defs/no_content"}, "status": {"const": "ok"}}}}
+ ]
+ },
+
+ "ended_after_turn_ms": {"type": "integer", "minimum": 0, "description": "Set when a step_ended arrived after its turn had already closed the step: how late it came (section 7.4). Absent otherwise."},
+
+ "usage": {
+ "description": "Token counts of one model call (technical design sections 5.2 and 6.2).",
+ "type": "object",
+ "additionalProperties": false,
+ "required": ["reported"],
+ "properties": {
+ "reported": {"type": "boolean", "description": "Whether the provider gave token counts. When false, the four parts are absent or null and the call adds the at-least marker."},
+ "input_fresh": {"type": ["integer", "null"], "minimum": 0, "description": "Fresh input tokens. null only when reported is false."},
+ "cache_read": {"type": ["integer", "null"], "minimum": 0, "description": "Input reused from cache. null = the provider sent no cache-read count (TypeSafe Jev, older Ollama); left out of the cache share; shown as a dash, never 0."},
+ "cache_write": {"type": ["integer", "null"], "minimum": 0, "description": "Input stored in cache. null = the provider sent no cache-write count (V-05, V-13); shown as a dash, never 0."},
+ "output": {"type": ["integer", "null"], "minimum": 0, "description": "Output tokens, including thinking where billed so. null only when reported is false."},
+ "reasoning": {"type": ["integer", "null"], "minimum": 0, "description": "Thinking tokens within output. null = the provider sent none."},
+ "inconsistent": {"type": "array", "items": {"enum": ["reasoning_exceeds_output", "provider_total_differs"]}, "description": "Present only when the provider's numbers are impossible; they are kept as sent (V-19)."}
+ },
+ "if": {"properties": {"reported": {"const": true}}},
+ "then": {"required": ["input_fresh", "cache_read", "cache_write", "output", "reasoning"],
+ "properties": {"input_fresh": {"type": "integer"}, "output": {"type": "integer"}}}
+ },
+
+ "model_error": {
+ "type": "object",
+ "additionalProperties": false,
+ "required": ["type"],
+ "properties": {
+ "type": {"enum": ["context_length", "overloaded", "rate_limited", "auth", "network", "timeout", "provider_error", "cancelled"]},
+ "text": {"type": ["string", "null"], "description": "The root cause's short text, for example 529. null = none."}
+ }
+ },
+
+ "provider_facts": {
+ "description": "Fixed names read from provider_meta when the step ends (section 5.2.1). Every fact: null = the provider did not send it.",
+ "type": "object",
+ "additionalProperties": false,
+ "properties": {
+ "served_model": {"type": ["string", "null"]},
+ "provider_request_id": {"type": ["string", "null"]},
+ "provider_response_id": {"type": ["string", "null"]},
+ "served_by": {"type": ["string", "null"]},
+ "reasoning": {"$ref": "#/$defs/count_or_null"},
+ "cache_read": {"$ref": "#/$defs/count_or_null"},
+ "cache_write": {"$ref": "#/$defs/count_or_null"},
+ "cache_write_by_ttl": {"type": ["object", "null"]},
+ "provider_total": {"$ref": "#/$defs/count_or_null"},
+ "requested": {"type": ["object", "null"]},
+ "applied": {"type": ["object", "null"]},
+ "stop": {"type": ["object", "null"]},
+ "provider_timing": {"type": ["object", "null"]},
+ "rate_limit": {"type": ["object", "null"]},
+ "hosted_tools": {"type": ["object", "null"]},
+ "cost_reported": {"type": ["object", "null"], "description": "Saved, not shown (tokens-only ruling; FO-78)."}
+ }
+ },
+
+ "provider_meta_entry": {
+ "description": "One HTTP response of a model call, as sent, minus content (deny list) and secrets (section 5.2.1).",
+ "type": "object",
+ "required": ["wire", "status", "request", "headers", "event_counts", "events"],
+ "properties": {
+ "wire": {"$ref": "#/$defs/wire", "description": "Which API shape. A helper request such as Ollama's /api/show is on the call's own wire, with auxiliary true."},
+ "status": {"type": "integer"},
+ "request": {"type": "object", "description": "Every request parameter except content; tools_offered as a count."},
+ "headers": {"type": "object", "additionalProperties": {"type": "string"}, "description": "Every response header except secrets."},
+ "event_counts": {"type": "object", "additionalProperties": {"type": "integer", "minimum": 0}},
+ "events": {"type": "object", "description": "Per event kind, every field of those events merged in order, later values winning, content removed."},
+ "auxiliary": {"type": "boolean", "description": "True for a helper request made before the call (M-R9); never counted as an attempt."},
+ "after_call": {"type": ["object", "null"], "description": "The provider's own record fetched after the call (OpenRouter). null = the fetch failed or is turned off."}
+ }
+ },
+
+ "blob_ref": {
+ "type": "object",
+ "additionalProperties": false,
+ "required": ["$blob"],
+ "properties": {"$blob": {"$ref": "#/$defs/sha256"}},
+ "description": "A reference to one piece in the exchange piece store, by its SHA-256."
+ },
+
+ "exchange_entry": {
+ "description": "One HTTP attempt of a model call: the exact request and the exact response, content included, secrets left out (TD-15, TD-17).",
+ "type": "object",
+ "required": ["request", "response"],
+ "properties": {
+ "auxiliary": {"type": "boolean"},
+ "request": {
+ "type": "object",
+ "required": ["method", "url", "headers", "body_sha256"],
+ "properties": {
+ "method": {"type": "string"},
+ "url": {"type": "string"},
+ "headers": {"type": "object", "additionalProperties": {"type": "string"}, "description": "Every request header except secrets (section 6 of recording-format.md)."},
+ "body_sha256": {"$ref": "#/$defs/sha256", "description": "SHA-256 of the exact bytes sent."},
+ "body_size": {"$ref": "#/$defs/count"},
+ "manifest": {"type": "object", "description": "The request JSON with pieces replaced by blob references."},
+ "blob": {"$ref": "#/$defs/sha256", "description": "For a body that is not a JSON object: the whole body as one piece."}
+ }
+ },
+ "response": {
+ "type": ["object", "null"],
+ "description": "null = no response arrived (the connection failed).",
+ "required": ["status", "headers", "body_blob"],
+ "properties": {
+ "status": {"type": "integer"},
+ "headers": {"type": "object", "additionalProperties": {"type": "string"}},
+ "body_blob": {"$ref": "#/$defs/sha256", "description": "The full response body as received (after content decoding), one piece."},
+ "body_size": {"$ref": "#/$defs/count"}
+ }
+ }
+ }
+ },
+
+ "piece": {
+ "description": "One line of an exchange piece store file (exchange-store.jsonl), or one row of exchange_pieces.",
+ "type": "object",
+ "required": ["sha256", "size", "data"],
+ "properties": {
+ "sha256": {"$ref": "#/$defs/sha256", "description": "SHA-256 of the piece's bytes (data encoded as UTF-8)."},
+ "size": {"$ref": "#/$defs/count", "description": "Length of the piece in bytes."},
+ "data": {"type": "string", "description": "The piece, as UTF-8 text."},
+ "first_seen": {"$ref": "#/$defs/epoch_ms", "description": "When the piece was first stored."}
+ }
+ },
+
+ "model_attrs": {
+ "type": "object",
+ "additionalProperties": false,
+ "required": ["purpose", "provider", "provider_title", "model", "stream", "finish_reason", "ttft_ms", "context_window", "usage", "tools_offered"],
+ "properties": {
+ "purpose": {"enum": ["main", "summary", "auto_approver", "helper", "title", "proxy", "unattributed"], "description": "unattributed = a call made with no tags at all (section 3.3)."},
+ "provider": {"type": "string", "description": "Provider id, the prefix of the model id; openai_decisions or typesafe_jev for decision-model checks."},
+ "provider_title": {"type": ["string", "null"], "description": "null = the provider has no Settings card (decision-model services)."},
+ "model": {"type": "string"},
+ "served_by": {"type": ["string", "null"], "description": "Backend host behind a router. null or absent = not a router, or not said."},
+ "provider_facts": {"$ref": "#/$defs/provider_facts"},
+ "provider_meta": {"type": "array", "items": {"$ref": "#/$defs/provider_meta_entry"}, "description": "One entry per HTTP response; the last is the reply."},
+ "stream": {"type": "boolean"},
+ "finish_reason": {"type": ["string", "null"], "description": "null = the call ended without a reply (failed or stopped)."},
+ "ttft_ms": {"type": ["integer", "null"], "minimum": 0, "description": "Time to the first streamed piece of any kind. null = not streamed, or nothing arrived."},
+ "context_window": {"type": ["integer", "null"], "minimum": 0, "description": "From the model catalogue at call time. null = the catalogue has no value."},
+ "usage": {"$ref": "#/$defs/usage"},
+ "tools_offered": {"$ref": "#/$defs/count"},
+ "tools_hash": {"$ref": "#/$defs/short_hash"},
+ "error": {"$ref": "#/$defs/model_error"},
+ "ended_after_turn_ms": {"$ref": "#/$defs/ended_after_turn_ms"}
+ }
+ },
+ "model_content": {
+ "type": "object",
+ "additionalProperties": false,
+ "properties": {"exchange": {"type": "array", "items": {"$ref": "#/$defs/exchange_entry"}}}
+ },
+
+ "decision": {
+ "description": "How a tool call was allowed or refused, stage by stage (section 8.3). A helper's tool step has {helper: true} instead.",
+ "oneOf": [
+ {"type": "object", "additionalProperties": false, "required": ["helper"], "properties": {"helper": {"const": true}}},
+ {"type": "object", "additionalProperties": false, "required": ["path", "origin"],
+ "properties": {
+ "path": {"type": "array", "items": {
+ "type": "object", "additionalProperties": false, "required": ["stage", "outcome"],
+ "properties": {
+ "stage": {"enum": ["permission", "grant", "auto_approver", "card", "resolved"]},
+ "outcome": {"enum": ["allowed", "needs_user", "human_only", "denied",
+ "config_allowlist", "session_grant", "standing_rule", "trusted_rule", "trusted_server", "run_grant",
+ "suspended_in_auto_approve", "allow_anyway", "lead_denial",
+ "allow", "deny", "unsure", "could_not_run", "context_changed",
+ "skipped_human_only", "skipped_unattended", "skipped_paused", "skipped_off",
+ "shown_session", "parked_inbox", "auto_refused",
+ "approved_once", "approved_session", "approved_run", "approved_always", "interrupted", "superseded"]},
+ "detail": {"enum": ["low_risk", "bypass", "cleared_by_mode", "always_tool", "always_command", "always_domain", "always_site", "readonly_session"], "description": "permission allowed: the reason id (low_risk, bypass, cleared_by_mode); resolved approved_session: which approval (always_tool, always_command, always_domain, always_site, readonly_session). Absent when the outcome has none."}
+ }}},
+ "origin": {"enum": ["low_risk", "bypass", "config_allowlist", "session_grant", "standing_rule", "trusted_rule", "trusted_server",
+ "run_grant", "allow_anyway", "lead_denial", "reviewer", "user",
+ "denied_by_rule", "reviewer_denied", "unattended_auto", null],
+ "description": "The single stage that decided. null = no stage decided (the call never reached a decision: bad arguments, unknown tool, superseded, stopped)."}
+ }}
+ ]
+ },
+ "tool_attrs": {
+ "type": "object",
+ "additionalProperties": false,
+ "required": ["decision"],
+ "properties": {
+ "risk_class": {"enum": ["read", "egress", "write_local", "exec", "external", null], "description": "From risk.classify. null = not classified (unknown tool)."},
+ "connector": {"type": ["string", "null"], "description": "null = not a connector tool."},
+ "result_bytes": {"$ref": "#/$defs/count_or_null", "description": "null = the call never ran."},
+ "exit_code": {"type": ["integer", "null"], "description": "Shell commands only. null = not a shell command, or it did not exit."},
+ "decision": {"$ref": "#/$defs/decision"},
+ "sandbox": {
+ "type": ["object", "null"],
+ "description": "null = not sandboxed.",
+ "additionalProperties": false,
+ "properties": {
+ "blocked_known": {"type": "boolean", "description": "When false, blocked sites are shown as unknown, never none."},
+ "blocked_count": {"$ref": "#/$defs/count_or_null", "description": "null = unknown: blocked_known is false, or the sandbox gives no counts (V-04, until E13)."},
+ "notice": {"enum": ["restart", "regrant", null]}
+ }
+ },
+ "ended_after_turn_ms": {"$ref": "#/$defs/ended_after_turn_ms"}
+ }
+ },
+ "tool_content": {
+ "type": "object",
+ "additionalProperties": false,
+ "properties": {
+ "decision_details": {"type": ["object", "null"], "description": "Permission rule text, the allow reason, a standing rule's tool to target."},
+ "blocked": {"type": "array", "items": {
+ "type": "object", "additionalProperties": false,
+ "required": ["host", "port", "count", "first_at", "late", "uncertain"],
+ "properties": {
+ "host": {"type": "string"}, "port": {"type": "integer"},
+ "count": {"type": ["integer", "null"], "minimum": 1, "description": "null = the sandbox did not report it (V-04: today's sandbox gives host names only, until E13)."},
+ "first_at": {"type": ["integer", "null"], "minimum": 0, "description": "Epoch milliseconds. null = not reported (V-04, until E13)."},
+ "late": {"type": ["boolean", "null"], "description": "null = not reported (V-04, until E13)."},
+ "uncertain": {"type": "boolean"}
+ }}}
+ }
+ },
+
+ "wait_attrs": {
+ "type": "object",
+ "additionalProperties": false,
+ "required": ["for_call_id", "kind", "where", "attended"],
+ "properties": {
+ "for_call_id": {"$ref": "#/$defs/id_or_null", "description": "null = empty: the summary's Retry-or-trim question."},
+ "kind": {"enum": ["approval", "question", "plan", "team", "items", "connector", "tool_install", "directory"]},
+ "where": {"enum": ["session", "inbox"]},
+ "attended": {"type": "boolean"},
+ "outcome": {"enum": ["approved", "denied", "answered", "interrupted", null], "description": "null or absent = still waiting (live only)."},
+ "ended_after_turn_ms": {"$ref": "#/$defs/ended_after_turn_ms"}
+ }
+ },
+ "wait_content": {
+ "type": "object",
+ "additionalProperties": false,
+ "properties": {"resolved_by": {"type": ["string", "null"], "description": "Who answered. null = not known."}}
+ },
+
+ "auto_approver_attrs": {
+ "type": "object",
+ "additionalProperties": false,
+ "required": ["reviewer", "mode", "result"],
+ "properties": {
+ "reviewer": {"type": "object", "additionalProperties": false, "required": ["kind", "external"],
+ "properties": {"kind": {"$ref": "#/$defs/reviewer_kind"},
+ "model": {"type": ["string", "null"], "description": "For session_model. null = a decision-model reviewer."},
+ "external": {"type": "boolean"}}},
+ "mode": {"enum": ["live", "shadow"]},
+ "result": {"enum": ["allow", "deny", "unsure", "could_not_run", "context_changed", null], "description": "null = still running (live only)."},
+ "score": {"type": ["number", "null"], "description": "Decision-model reviewers only. null = session-model reviewer."},
+ "cutoff": {"type": ["number", "null"]},
+ "usage_reported": {"type": "boolean"},
+ "breaker": {"type": "object", "additionalProperties": false, "required": ["denials_in_row", "tripped"],
+ "properties": {"denials_in_row": {"$ref": "#/$defs/count"}, "tripped": {"type": "boolean"}}},
+ "matched_user": {"type": ["boolean", "null"], "description": "Shadow checks only. null = cannot be compared (no answer, the check failed, or it said unsure)."},
+ "ended_after_turn_ms": {"$ref": "#/$defs/ended_after_turn_ms"}
+ }
+ },
+ "auto_approver_content": {
+ "type": "object",
+ "additionalProperties": false,
+ "properties": {
+ "reason": {"type": ["string", "null"], "description": "The reviewer's own sentence. null = none given."},
+ "questions": {"type": ["array", "null"], "description": "A decision model's questions, each with its answer and score. null = not a decision-model check."}
+ }
+ },
+
+ "context_attrs": {
+ "type": "object",
+ "additionalProperties": false,
+ "required": ["session_prompt_hash", "per_turn", "size_chars", "block_hash"],
+ "properties": {
+ "session_prompt_hash": {"$ref": "#/$defs/short_hash", "description": "null = the session has no system prompt."},
+ "session_start": {"type": "object", "additionalProperties": false, "description": "First turn only.",
+ "properties": {"memories": {"type": "array"}, "memory_mode": {"enum": ["full", "index"]},
+ "rules": {"$ref": "#/$defs/count"}, "agents_md": {"$ref": "#/$defs/count"}}},
+ "per_turn": {"type": "object", "additionalProperties": false,
+ "properties": {"mode_note": {"enum": ["plan", "discuss", null], "description": "null = no mode note."},
+ "folders": {"$ref": "#/$defs/count"}, "sandbox_note": {"type": "boolean"},
+ "skills_menu": {"$ref": "#/$defs/count"}, "skills_hash": {"$ref": "#/$defs/short_hash"}}},
+ "size_chars": {"$ref": "#/$defs/count"},
+ "block_hash": {"$ref": "#/$defs/short_hash"}
+ }
+ },
+ "context_content": {
+ "type": "object",
+ "additionalProperties": false,
+ "properties": {"agents_md": {"type": "array", "items": {"type": "string"}}}
+ },
+
+ "summary_attrs": {
+ "type": "object",
+ "additionalProperties": false,
+ "required": ["reason", "tokens_before", "model_step_id"],
+ "properties": {
+ "reason": {"enum": ["threshold", "overflow"]},
+ "tokens_before": {"$ref": "#/$defs/count_or_null", "description": "Context tokens of the last main call before. null = no main call reported usage before it."},
+ "tokens_after": {"$ref": "#/$defs/count_or_null", "description": "Filled by step_updated when the first main call after it ends. null or absent = not yet."},
+ "model_step_id": {"$ref": "#/$defs/id_or_null", "description": "The model call inside it. null = empty: a trim."},
+ "ended_after_turn_ms": {"$ref": "#/$defs/ended_after_turn_ms"}
+ }
+ },
+
+ "notice_name": {"enum": ["interrupted", "error", "reviewer_paused", "step_limit", "truncated", "context_changed", "stamp_changed", "summary_failed", "recording_gap"]},
+ "notice_attrs": {
+ "description": "{} for every notice except two, which carry what changed, under names already defined elsewhere, with the new value and only the keys that changed. context_changed: the per-turn context block as it is now, under the context step's names (per_turn, size_chars, block_hash). stamp_changed: attended, reviewer or stamp, under the turn's names (stamp holding only the changed groups).",
+ "type": "object",
+ "additionalProperties": false,
+ "properties": {
+ "per_turn": {"$ref": "#/$defs/context_attrs/properties/per_turn"},
+ "size_chars": {"$ref": "#/$defs/count"},
+ "block_hash": {"$ref": "#/$defs/short_hash"},
+ "attended": {"type": "boolean"},
+ "reviewer": {"$ref": "#/$defs/reviewer_settings"},
+ "stamp": {"$ref": "#/$defs/stamp"}
+ }
+ },
+ "no_content": {"type": "object", "additionalProperties": false, "maxProperties": 0},
+
+ "live_message": {
+ "description": "One live message on the session's WebSocket while a turn runs (technical design section 12): the recording events of the last 100 ms, unchanged.",
+ "type": "object",
+ "additionalProperties": false,
+ "required": ["type", "events"],
+ "properties": {
+ "type": {"const": "step"},
+ "events": {"type": "array", "items": {"$ref": "#/$defs/event"}}
+ }
+ },
+
+ "event": {
+ "description": "One recording event (technical design section 3.2). Common fields first; the rest depend on type.",
+ "type": "object",
+ "required": ["type", "v", "at_ms", "mono_ms", "session_id", "turn_id"],
+ "properties": {
+ "type": {"enum": ["turn_started", "turn_ended", "step_started", "step_updated", "step_ended"]},
+ "v": {"const": 1, "description": "The event contract version."},
+ "at_ms": {"$ref": "#/$defs/epoch_ms"},
+ "mono_ms": {"type": "integer", "description": "A monotonic clock in milliseconds, for durations. Only differences within one process mean anything."},
+ "session_id": {"$ref": "#/$defs/id_or_null", "description": "null = empty: a proxy call."},
+ "turn_id": {"$ref": "#/$defs/id_or_null", "description": "null = empty: no turn."}
+ },
+ "allOf": [
+ {"if": {"properties": {"type": {"const": "turn_started"}}},
+ "then": {"required": ["kind", "trigger", "first_message_id", "attrs", "stamp"],
+ "properties": {"kind": {"$ref": "#/$defs/turn_kind"}, "trigger": {"$ref": "#/$defs/trigger"},
+ "first_message_id": {"$ref": "#/$defs/id_or_null"},
+ "attrs": {"type": "object", "additionalProperties": false, "required": ["approval_mode", "attended", "reviewer"],
+ "properties": {"approval_mode": {"type": "string"}, "attended": {"type": "boolean"},
+ "reviewer": {"$ref": "#/$defs/reviewer_settings"}}},
+ "stamp": {"$ref": "#/$defs/stamp"}}}},
+ {"if": {"properties": {"type": {"const": "turn_ended"}}},
+ "then": {"required": ["status", "last_message_id"],
+ "properties": {"status": {"$ref": "#/$defs/turn_status"}, "last_message_id": {"$ref": "#/$defs/id_or_null"}}}},
+ {"if": {"properties": {"type": {"const": "step_started"}}},
+ "then": {"required": ["step_id", "parent_step_id", "kind", "name", "call_id", "call_index", "shadow", "after_turn", "attrs", "content"],
+ "properties": {"step_id": {"$ref": "#/$defs/id"}, "parent_step_id": {"$ref": "#/$defs/id_or_null"},
+ "kind": {"$ref": "#/$defs/step_kind"}, "name": {"type": "string", "minLength": 1},
+ "call_id": {"$ref": "#/$defs/id_or_null"}, "call_index": {"type": ["integer", "null"], "minimum": 0},
+ "shadow": {"type": "boolean"}, "after_turn": {"type": "boolean"},
+ "attrs": {"type": "object"}, "content": {"type": "object"}}}},
+ {"if": {"properties": {"type": {"const": "step_updated"}}},
+ "then": {"required": ["step_id"], "anyOf": [{"required": ["attrs"]}, {"required": ["content"]}],
+ "properties": {"step_id": {"$ref": "#/$defs/id"}, "attrs": {"type": "object"}, "content": {"type": "object"}}}},
+ {"if": {"properties": {"type": {"const": "step_ended"}}},
+ "then": {"required": ["step_id", "status", "kind", "name", "parent_step_id", "call_id", "started_at"],
+ "properties": {"step_id": {"$ref": "#/$defs/id"}, "status": {"$ref": "#/$defs/step_status"},
+ "message_id": {"$ref": "#/$defs/id_or_null"},
+ "attrs": {"type": "object"}, "content": {"type": "object"},
+ "error": {"anyOf": [{"$ref": "#/$defs/model_error"}, {"type": "null"}], "description": "null or absent = no error. Folded into attrs.error."},
+ "kind": {"$ref": "#/$defs/step_kind"}, "name": {"type": "string", "minLength": 1},
+ "parent_step_id": {"$ref": "#/$defs/id_or_null"}, "call_id": {"$ref": "#/$defs/id_or_null"},
+ "call_index": {"type": ["integer", "null"], "minimum": 0},
+ "shadow": {"type": "boolean"}, "after_turn": {"type": "boolean"},
+ "started_at": {"$ref": "#/$defs/epoch_ms"}}}}
+ ],
+ "unevaluatedProperties": false
+ }
+ }
+}
diff --git a/tests/test_observability_schema.py b/tests/test_observability_schema.py
new file mode 100644
index 00000000..b35c59e6
--- /dev/null
+++ b/tests/test_observability_schema.py
@@ -0,0 +1,120 @@
+"""The engine's own recordings fit the shared schema (recording-format.schema.json).
+
+The golden test checks the recorder against the shared scenarios; this checks the other
+half: that what the real engine records (turns, model, tool, wait, auto-approver, summary,
+context and notice steps, stamps, decision paths, exchanges) has exactly the shape of
+recording format 1.x, the same file the Rust SDK is built against.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import importlib
+import json
+from pathlib import Path
+from types import SimpleNamespace
+
+import jsonschema
+import pytest
+
+from coworker import compaction as _compaction
+from coworker.engine import ApprovalOutcome
+from coworker.providers import AssistantTurn
+from coworker.providers.base import TokenUsage
+
+SCHEMA = json.loads((Path(__file__).parent / "corpora/observability/recording-format.schema.json").read_text("utf-8"))
+waits = importlib.import_module("test_observability_waits")
+reviews = importlib.import_module("test_observability_reviewer_helper")
+stamps = importlib.import_module("test_observability_stamps_context")
+
+
+def _check(recording):
+ validator = jsonschema.validators.validator_for(SCHEMA)(SCHEMA)
+ errors = sorted(validator.iter_errors(recording), key=lambda e: list(e.absolute_path))
+ assert not errors, "\n".join(f"{list(e.absolute_path)}: {e.message[:300]}" for e in errors[:8])
+
+
+def test_approved_card_session_grant_and_low_risk(tmp_path):
+ engine, rec = waits._engine(tmp_path, [
+ waits._call("c1", "run_shell", {"command": "pytest"}),
+ waits._call("c2", "run_shell", {"command": "pytest"}),
+ waits._call("c3", "look", {"path": "x"}),
+ AssistantTurn(text="done", usage=TokenUsage(input=10, output=2)),
+ ], answer=ApprovalOutcome.ALWAYS_COMMAND)
+ waits._run(engine)
+ _check(rec.as_dict())
+
+
+def test_denied_card(tmp_path):
+ engine, rec = waits._engine(tmp_path, [waits._call("c1", "run_shell", {"command": "rm -rf b"}),
+ AssistantTurn(text="ok")], answer=ApprovalOutcome.DENY)
+ waits._run(engine)
+ _check(rec.as_dict())
+
+
+def test_summary_step(tmp_path, monkeypatch):
+ engine, rec = waits._engine(tmp_path, [AssistantTurn(text="sum", usage=TokenUsage(input=9, output=1)),
+ AssistantTurn(text="hi", usage=TokenUsage(input=5, output=1)),
+ AssistantTurn(text="next")])
+
+ def fake(messages, *, provider, model, **kw):
+ provider.complete(model=model, messages=[{"role": "user", "content": "summarize"}])
+ return SimpleNamespace(boundary_index=0, carries_outside=False, transcript_path=None)
+
+ monkeypatch.setattr(_compaction, "build_state", fake)
+ waits._run(engine) # a turn, so the summary has one to belong to
+
+ async def go():
+ await engine._compact_now()
+
+ asyncio.run(go())
+ waits._run(engine, "again")
+ _check(rec.as_dict())
+
+
[email protected]("verdict", ["allow", "deny"])
+def test_live_auto_approver_checks(tmp_path, verdict):
+ engine, rec = reviews._engine(tmp_path, [reviews._call("c1", "run_shell", {"command": "make"}),
+ AssistantTurn(text="ok")],
+ [json.dumps({"verdict": verdict, "reason": "r"})])
+ reviews._run(engine)
+ _check(rec.as_dict())
+
+
+def test_shadow_check_and_could_not_run(tmp_path):
+ engine, rec = reviews._engine(tmp_path, [reviews._call("c1", "run_shell", {"command": "make"}),
+ AssistantTurn(text="ok")],
+ [json.dumps({"verdict": "allow", "reason": "fine"})], shadow=True)
+ reviews._run(engine)
+ _check(rec.as_dict())
+ engine, rec = reviews._engine(tmp_path, [reviews._call("c1", "run_shell", {"command": "make"}),
+ AssistantTurn(text="ok")], [RuntimeError("no key")])
+ reviews._run(engine)
+ _check(rec.as_dict())
+
+
+def test_explore_helper(tmp_path):
+ rec_holder = {}
+ original = reviews.Recorder
+
+ class Keep(original):
+ def __init__(self, *a, **k):
+ super().__init__(*a, **k)
+ rec_holder["rec"] = self
+
+ reviews.Recorder = Keep
+ try:
+ reviews.test_explore_helper_steps_are_nested_under_the_explore_step(tmp_path)
+ finally:
+ reviews.Recorder = original
+ _check(rec_holder["rec"].as_dict())
+
+
+def test_context_notices_stamps_and_sandbox_facts(tmp_path):
+ engine, rec = stamps._engine(tmp_path, [stamps._call("c1", "look", {"path": "x"}), AssistantTurn(text="ok")],
+ blocks=["Folders: /w"])
+ stamps._run(engine)
+ _check(rec.as_dict())
+ engine, rec = stamps._engine(tmp_path, [RuntimeError("boom")])
+ stamps._run(engine)
+ _check(rec.as_dict())
diff --git a/tests/test_observability_stamps_context.py b/tests/test_observability_stamps_context.py
index 3aae7409..41f0b46a 100644
--- a/tests/test_observability_stamps_context.py
+++ b/tests/test_observability_stamps_context.py
@@ -158,7 +158,7 @@ def test_blocked_sites_go_to_the_tool_step_not_the_transcript(tmp_path):
url: the page
"""
return {"exit_code": 1, "blocked_sites": ["semgrep.dev:443"], "network_note": "blocked",
- "_blocked": [{"site": "semgrep.dev:443", "count": 2, "first_at": 1, "late": False,
+ "_blocked": [{"host": "semgrep.dev", "port": 443, "count": 2, "first_at": 1, "late": False,
"uncertain": True}]}
engine, rec = _engine(tmp_path, [_call("c1", "fetch", {"url": "https://semgrep.dev"}), AssistantTurn(text="ok")])
@@ -167,7 +167,7 @@ def test_blocked_sites_go_to_the_tool_step_not_the_transcript(tmp_path):
_run(engine)
[tool] = [s for s in rec.as_dict()["steps"] if s["kind"] == "tool"]
assert tool["attrs"]["sandbox"] == {"blocked_known": True, "blocked_count": 2, "notice": None}
- assert tool["content"]["blocked"][0]["site"] == "semgrep.dev:443"
+ assert tool["content"]["blocked"][0]["host"] == "semgrep.dev"
result = next(m for m in engine.messages if m.get("role") == "tool")
assert "_blocked" not in str(result["content"])
assert rec.as_dict()["turns"][0]["stamp"]["sandbox"] == {"provider": "openshell", "enforcement": "full"}
@@ -184,5 +184,5 @@ def test_network_note_keeps_the_counts(tmp_path, monkeypatch):
monkeypatch.setattr(ws, "_asking", lambda: False, raising=False)
note = ws.network_note(started=100.0, result={"exit_code": 0})
assert note["blocked_sites"] == ["semgrep.dev:443"]
- assert note["_blocked"] == [{"site": "semgrep.dev:443", "count": 2, "first_at": 100000, "late": False,
- "uncertain": True}]
+ assert note["_blocked"] == [{"host": "semgrep.dev", "port": 443, "count": 2, "first_at": 100000,
+ "late": False, "uncertain": True}]