Skip to content
Python

feat(observability): Issues tab, the OB-3 checks and the 1.2 fields they read

Devika Verma5b740201080e
diff --git a/coworker/agent.py b/coworker/agent.py
index 21602c21..831a23d7 100644
--- a/coworker/agent.py
+++ b/coworker/agent.py
@@ -52,7 +52,7 @@ from .web import make_web_fetch_tool, make_web_search_tool
from .workspace_trust import WorkspaceTrustStore
from .sandbox import inside as _openshell
from .sandbox.selection import select as select_sandbox
-from .sandbox.workspace import open_workspace
+from .sandbox.workspace import open_workspace, provider_name as _sandbox_provider_name
from .tools.todo import TodoList
# Appended each turn while discuss mode is active: enforcement-only read-only, with no
@@ -776,6 +776,9 @@ def build_engine(
if executor is not None:
engine.permissions.shell_cwd = engine.current_shell_cwd
engine.sandbox_workspace = sandbox_workspace # type: ignore[attr-defined]
+ # Whether this machine has a sandbox switched on (stamp.sandbox.expected, format 1.2), for
+ # the "ran without the sandbox" check: the setting, whatever this session ended up using.
+ engine.sandbox_expected = _sandbox_provider_name(config.sandbox_provider) != "direct"
# OPE-219: a sandboxed session with "Only the sites you allow" holds its web tools to
# the same list as its commands. They run in this process, outside the sandbox, so the
# permission engine is the wall for them. No sandbox, or "Allow everything": no wall.
diff --git a/coworker/engine.py b/coworker/engine.py
index c94c11b2..28e99932 100644
--- a/coworker/engine.py
+++ b/coworker/engine.py
@@ -37,6 +37,7 @@ from .observability.decision import DecisionPath
from .observability.engine_hooks import ToolRun, TurnRecording, set_tool_run
from .observability.recorder import short_hash as _short_hash
from .observability import stamps as _stamps
+from .observability.canonical import short_hash as _args_hash, text_hash as _text_hash
from .observability.provider_view import tag as _tag_provider, view_for_engine
from .observability.ids import new_ulid
@@ -557,6 +558,11 @@ class TurnEngine:
# it makes is recorded as a model step of the current turn, purpose `main`.
self.provider = view_for_engine(self.provider, self._obs)
self._call_index: dict[str, int] = {}
+ # Each proposed call's arguments, for the 1.2 tool fields of calls that never ran.
+ self._call_args: dict[str, Any] = {}
+ # Whether this machine has a sandbox switched on (stamp.sandbox.expected, 1.2): set
+ # by whoever builds the engine from the machine's setting; None = not known.
+ self.sandbox_expected: Optional[bool] = None
# Who answered a waiting call's card (observability §5.4): set by the server, which
# reads it from the Inbox.
self.resolver_of: Optional[Callable[[str], str]] = None
@@ -639,7 +645,8 @@ class TurnEngine:
step_id = self._obs.start_step(
kind="wait", name=kind, call_id=tool_call.id,
attrs={"for_call_id": tool_call.id, "kind": kind, "attended": attended,
- "where": "session" if attended else "inbox"},
+ "where": "session" if attended else "inbox",
+ "inbox_at": None if attended else int(started * 1000)},
)
self._obs.track_wait(tool_call.id, step_id)
result: Any = interrupted
@@ -703,6 +710,7 @@ class TurnEngine:
"approval_mode": self.permissions.mode.value,
"attended": attended,
"reviewer": reviewer,
+ "max_iterations": self.max_iterations,
},
stamp=stamp,
first_message_id=first,
@@ -731,6 +739,14 @@ class TurnEngine:
"model": getattr(self.reviewer, "model", None) if reviewer_kind == "session_model" else None,
}
+ def _sandbox_stamp(self) -> Optional[dict[str, Any]]:
+ """The stamp's sandbox group now, with `expected` (format 1.2) when it is known. The
+ turn's stamp and the mid-turn comparison both use this, so they cannot differ by it."""
+ sandbox = _stamps.sandbox_stamp(getattr(self, "sandbox_workspace", None))
+ if sandbox and self.sandbox_expected is not None:
+ sandbox["expected"] = bool(self.sandbox_expected)
+ return sandbox
+
def _note_stamp(self) -> None:
"""A `stamp_changed` notice when attendance, the reviewer or the sandbox differ from
the turn's start or the last notice (observability §9, V2-27). The Unattended toggle
@@ -749,7 +765,7 @@ class TurnEngine:
if reviewer != seen.get("reviewer"):
changed["reviewer"] = seen["reviewer"] = reviewer
try:
- sandbox = _stamps.sandbox_stamp(getattr(self, "sandbox_workspace", None))
+ sandbox = self._sandbox_stamp()
except Exception: # noqa: BLE001 - a stamp never breaks a turn
sandbox = seen.get("sandbox")
if sandbox and sandbox != seen.get("sandbox"):
@@ -764,7 +780,7 @@ class TurnEngine:
try:
stamp["machine"] = _stamps.machine_stamp()
stamp["model"] = _stamps.model_stamp(self.model)
- sandbox = _stamps.sandbox_stamp(getattr(self, "sandbox_workspace", None))
+ sandbox = self._sandbox_stamp()
if sandbox:
stamp["sandbox"] = sandbox
if self.persona_fp:
@@ -817,6 +833,8 @@ class TurnEngine:
obs = self._obs
event.data.setdefault("turn_id", obs.turn_id)
event.data.setdefault("ts_ms", time.time_ns() // 1_000_000)
+ if event.type is EventType.TOOL_PROPOSED and event.data.get("tool_call_id"):
+ self._call_args[str(event.data["tool_call_id"])] = event.data.get("arguments")
if event.type in (EventType.TOOL_STARTED, EventType.TOOL_FINISHED):
call_id = str(event.data.get("tool_call_id") or "")
if event.type is EventType.TOOL_FINISHED and call_id:
@@ -834,6 +852,11 @@ class TurnEngine:
# flow: no stage decided it (recording-format: origin null).
attrs["decision"] = {"path": [], "origin": None}
data = event.data
+ if obs.step_for_call(call_id) is None:
+ attrs.update(self._call_facts(
+ str(data.get("name") or ""), self._call_args.get(call_id), ran=False,
+ status=str(data.get("status") or ""), reason=str(data.get("reason") or ""),
+ ))
answered_by_stop = path is not None and (path.stage("resolved") or {}).get("outcome") == "interrupted"
if (data.get("status") == "denied" and self._cancel.is_set()
and (data.get("name") in _WAIT_KINDS or answered_by_stop)):
@@ -2874,7 +2897,7 @@ class TurnEngine:
set_tool_run(outer)
blocked = result.pop("_blocked", None) if isinstance(result, dict) else None
if step_id is not None:
- attrs = self._tool_step_attrs(tool_call, {**result, "_blocked": blocked} if blocked else result)
+ attrs = self._tool_step_attrs(tool_call, {**result, "_blocked": blocked} if blocked else result, status)
step_status = status
if self._cancel.is_set() and isinstance(result, dict) and result.get("error"):
# Stop killed it (a shell command answers "interrupted by user"): §7.4.
@@ -2886,7 +2909,7 @@ class TurnEngine:
)
return result, status
- def _tool_step_attrs(self, tool_call: ToolCall, result: Any) -> dict[str, Any]:
+ def _tool_step_attrs(self, tool_call: ToolCall, result: Any, status: str = "ok") -> dict[str, Any]:
attrs: dict[str, Any] = {"risk_class": self._risk_class(tool_call.name)}
path = self._decisions.pop(tool_call.id, None)
if path is not None:
@@ -2901,8 +2924,52 @@ class TurnEngine:
if isinstance(result, dict) and isinstance(result.get("exit_code"), int):
attrs["exit_code"] = result["exit_code"]
attrs["sandbox"] = self._sandbox_facts(result)
+ attrs.update(self._call_facts(tool_call.name, tool_call.arguments, ran=True,
+ status=status, result=result, risk=attrs["risk_class"]))
return attrs
+ def _call_facts(self, name: str, arguments: Any, *, ran: bool, status: str, result: Any = None,
+ reason: str = "", risk: Optional[str] = None) -> dict[str, Any]:
+ """The tool fields of recording format 1.2 that the Issues checks read (phase-2
+ design §4): ids and flags only, never text. Facts never break a call."""
+ out: dict[str, Any] = {}
+ try:
+ mangled = isinstance(arguments, dict) and "_raw" in arguments
+ out["args_hash"] = None if mangled or arguments is None else _args_hash(arguments)
+ spec = self.registry.get(name)
+ out["repeat_ok"] = bool(getattr(getattr(spec, "metadata", None), "repeat_ok", False))
+ out["result_error"] = bool(ran and status == "ok" and isinstance(result, dict) and result.get("error"))
+ if status != "error":
+ out["error_kind"] = None
+ elif ran:
+ out["error_kind"] = "raised"
+ elif mangled:
+ out["error_kind"] = "bad_arguments"
+ elif spec is None or reason.startswith("unknown tool"):
+ out["error_kind"] = "unknown_tool"
+ else:
+ out["error_kind"] = "raised"
+ if not ran:
+ out["reaches_network"] = None
+ else:
+ command = arguments.get("command") if isinstance(arguments, dict) and name == "run_shell" else None
+ out["reaches_network"] = bool(
+ (risk or self._risk_class(name)) == "egress"
+ or (isinstance(command, str) and review_facts.shell_reaches_network(command))
+ )
+ if name == "todo_write" and isinstance(arguments, dict):
+ entries = arguments.get("todos", arguments.get("items")) or []
+ items = []
+ for entry in entries if isinstance(entries, list) else []:
+ text = str(entry.get("content", "")) if isinstance(entry, dict) else str(entry)
+ st = entry.get("status", "pending") if isinstance(entry, dict) else "pending"
+ st = "done" if st in ("done", "completed") else st if st in ("pending", "in_progress") else "pending"
+ items.append({"h": _text_hash(text), "s": st})
+ out["plan"] = {"items": items}
+ except Exception: # noqa: BLE001 - recording never breaks a call
+ logger.exception("observability: could not record a tool call's facts")
+ return out
+
def _sandbox_facts(self, result: Any) -> Optional[dict[str, Any]]:
"""Per-call sandbox facts (observability E13, §9.3): whether this sandbox can report
blocked sites now, and how many connections it refused during the call. None when
diff --git a/coworker/observability/canonical.py b/coworker/observability/canonical.py
new file mode 100644
index 00000000..8a9d2273
--- /dev/null
+++ b/coworker/observability/canonical.py
@@ -0,0 +1,60 @@
+"""The byte-exact hashing rules shared with the Rust harness (recording format 1.2, §3.11).
+
+`args_hash` is the first 6 hex characters of SHA-256 over the arguments' canonical JSON, and
+canonical JSON is RFC 8785 (JCS): keys sorted by UTF-16 code units, no whitespace, strings with
+only the required escapes, numbers in the ECMAScript form. `json.dumps` is not the rule: it
+writes 1e-07 and 100.0 where JCS writes 1e-7 and 100. A plan item's hash is over its plain
+text. The vectors are ocw-context docs/observability/golden/phase2/vectors/hashes.json.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import math
+from typing import Any
+
+
+def _number(x: Any) -> str:
+ if isinstance(x, bool):
+ return "true" if x else "false"
+ if isinstance(x, int):
+ return str(x)
+ if math.isnan(x) or math.isinf(x):
+ raise ValueError("JSON has no NaN or Infinity")
+ if x == 0:
+ return "0"
+ if x == int(x) and abs(x) < 1e21:
+ return str(int(x))
+ r = repr(x) # shortest round-trip digits, as ECMAScript writes them
+ if "e" in r:
+ mant, exp = r.split("e")
+ sign = "-" if exp.startswith("-") else "+"
+ return f"{mant}e{sign}{int(exp.lstrip('+-'))}"
+ return r
+
+
+def jcs(value: Any) -> str:
+ """RFC 8785 canonical JSON text of `value`."""
+ if value is None:
+ return "null"
+ if isinstance(value, (bool, int, float)):
+ return _number(value)
+ if isinstance(value, str):
+ return json.dumps(value, ensure_ascii=False)
+ if isinstance(value, (list, tuple)):
+ return "[" + ",".join(jcs(v) for v in value) + "]"
+ if isinstance(value, dict):
+ keys = sorted(value, key=lambda k: str(k).encode("utf-16-be"))
+ return "{" + ",".join(json.dumps(str(k), ensure_ascii=False) + ":" + jcs(value[k]) for k in keys) + "}"
+ raise TypeError(f"not JSON: {type(value).__name__}")
+
+
+def short_hash(value: Any) -> str:
+ """6 hex of SHA-256 over the canonical JSON of `value`."""
+ return hashlib.sha256(jcs(value).encode("utf-8")).hexdigest()[:6]
+
+
+def text_hash(text: str) -> str:
+ """6 hex of SHA-256 over the text's own UTF-8 bytes."""
+ return hashlib.sha256(text.encode("utf-8")).hexdigest()[:6]
diff --git a/coworker/observability/engine_hooks.py b/coworker/observability/engine_hooks.py
index 492a727d..ef796952 100644
--- a/coworker/observability/engine_hooks.py
+++ b/coworker/observability/engine_hooks.py
@@ -18,6 +18,7 @@ from __future__ import annotations
import copy
import threading
+import time
from typing import Any, Callable, Optional
from .ids import new_ulid
@@ -242,7 +243,10 @@ class TurnRecording:
return
with self._lock:
step_id = self._waits.get(call_id)
- self.update_step(step_id, attrs={"where": where})
+ attrs: dict[str, Any] = {"where": where}
+ if where == "inbox":
+ attrs["inbox_at"] = int(time.time() * 1000) # when nobody was watching any more (1.2)
+ self.update_step(step_id, attrs=attrs)
# -- summary steps (E4) ---------------------------------------------------------------------
def reserve_step(self) -> Optional[str]:
diff --git a/coworker/observability/provider_view.py b/coworker/observability/provider_view.py
index a2b7ecac..3672d008 100644
--- a/coworker/observability/provider_view.py
+++ b/coworker/observability/provider_view.py
@@ -110,7 +110,7 @@ class ProviderView(ProviderClient):
return result
def complete(self, *, model: str, messages: list[dict[str, Any]], tools: Any = None, **settings: Any):
- call = _ModelCall.begin(self.tags, model=model, tools=tools, streamed=False)
+ call = _ModelCall.begin(self.tags, model=model, tools=tools, streamed=False, settings=settings)
if call is None:
return self.base.inner.complete(model=model, messages=messages, tools=tools, **settings)
try:
@@ -123,7 +123,7 @@ class ProviderView(ProviderClient):
return turn
def stream(self, *, model: str, messages: list[dict[str, Any]], tools: Any = None, **settings: Any) -> Iterator[Any]:
- call = _ModelCall.begin(self.tags, model=model, tools=tools, streamed=True)
+ call = _ModelCall.begin(self.tags, model=model, tools=tools, streamed=True, settings=settings)
if call is None:
yield from self.base.inner.stream(model=model, messages=messages, tools=tools, **settings)
return
@@ -268,6 +268,15 @@ def usage_attrs(usage: Any) -> dict[str, Any]:
}
+def _max_output(settings: Optional[dict[str, Any]]) -> Optional[int]:
+ """The output limit the request asks for (format 1.2): the "cut off" issue says it."""
+ for key in ("max_tokens", "max_output_tokens", "max_completion_tokens"):
+ value = (settings or {}).get(key)
+ if isinstance(value, int) and not isinstance(value, bool) and value > 0:
+ return value
+ return None
+
+
class _ModelCall:
"""The state of one recorded model call, from start to end."""
@@ -278,7 +287,8 @@ class _ModelCall:
self.ttft_ms: Optional[int] = None
@classmethod
- def begin(cls, tags: Tags, *, model: str, tools: Any, streamed: bool) -> Optional["_ModelCall"]:
+ def begin(cls, tags: Tags, *, model: str, tools: Any, streamed: bool,
+ settings: Optional[dict[str, Any]] = None) -> Optional["_ModelCall"]:
sink = tags.sink
if sink is None or not sink.enabled:
return None
@@ -293,6 +303,7 @@ class _ModelCall:
"provider_title": _provider_title(provider), "model": model,
"stream": streamed, "context_window": _context_window(model, catalogue_only=True),
"tools_offered": len(tools or []), "tools_hash": tools_hash(tools),
+ "max_output": _max_output(settings),
},
)
except Exception: # noqa: BLE001 - recording never breaks a call
diff --git a/coworker/observability/recorder.py b/coworker/observability/recorder.py
index d90dda4c..6c765ad9 100644
--- a/coworker/observability/recorder.py
+++ b/coworker/observability/recorder.py
@@ -54,6 +54,8 @@ TURN_FIELDS = (
"incomplete", "approval_mode", "attended", "reviewer", "stamp", "first_message_id",
"last_message_id", "schema_version",
)
+# Format 1.2 turn fields: written only when known, so a 1.1 recording reads back unchanged.
+TURN_FIELDS_12 = ("max_iterations", "woken_by")
STEP_FIELDS = (
"step_id", "turn_id", "session_id", "parent_step_id", "seq", "kind", "name", "started_at",
"duration_ms", "status", "call_id", "call_index", "message_id", "shadow", "after_turn",
@@ -83,6 +85,8 @@ class Turn:
status: str = "running"
incomplete: bool = False
schema_version: int = SCHEMA_VERSION
+ max_iterations: Optional[int] = None
+ woken_by: Optional[dict[str, Any]] = None
mono_start: Optional[int] = field(default=None, repr=False) # not part of the record
mono_end: Optional[int] = field(default=None, repr=False)
@@ -91,7 +95,9 @@ class Turn:
return None if self.duration_ms is None else self.started_at + self.duration_ms
def record(self) -> dict[str, Any]:
- return {k: copy.deepcopy(getattr(self, k)) for k in TURN_FIELDS}
+ out = {k: copy.deepcopy(getattr(self, k)) for k in TURN_FIELDS}
+ out.update({k: copy.deepcopy(getattr(self, k)) for k in TURN_FIELDS_12 if getattr(self, k) is not None})
+ return out
@dataclass
@@ -265,6 +271,7 @@ class Recorder:
attended=a.get("attended"), reviewer=copy.deepcopy(a.get("reviewer")),
stamp=copy.deepcopy(ev.get("stamp") or {}),
first_message_id=ev.get("first_message_id"), mono_start=ev["mono_ms"],
+ max_iterations=a.get("max_iterations"), woken_by=copy.deepcopy(a.get("woken_by")),
)
self.turns[turn.turn_id] = turn
self._write_turn(turn)
@@ -288,8 +295,22 @@ class Recorder:
for s in self.steps.values():
if s.turn_id == turn.turn_id:
self._write_step(s)
+ self._check_turn(turn)
self._forget_old_turns(ev["mono_ms"])
+ def _check_turn(self, turn: "Turn") -> None:
+ """The Issues checks over the turn just ended (OB-3), stored with it as not yet seen.
+ A check never breaks the recording."""
+ if self.store is None:
+ return
+ try:
+ from . import signals
+
+ steps = [s.record() for s in self.steps.values() if s.turn_id == turn.turn_id]
+ self.store.write_issues(turn.turn_id, signals.check_turn(turn.record(), steps))
+ except Exception: # noqa: BLE001
+ log.exception("recording: the Issues checks failed for a turn")
+
def _anchor(self, turn_id: Optional[str], at_ms: int, mono_ms: int) -> int:
"""§7.3: a step's start is its turn's start plus the monotonic time since then."""
t = self.turns.get(turn_id) if turn_id else None
diff --git a/coworker/observability/signals.py b/coworker/observability/signals.py
new file mode 100644
index 00000000..495bb809
--- /dev/null
+++ b/coworker/observability/signals.py
@@ -0,0 +1,455 @@
+"""The Issues checks (OB-3, phase-2 design Part A): fixed rules over one finished turn.
+
+The rules and their numbers are in `signals.yaml`, a copy of the shared file in ocw-context
+(docs/observability/signals.yaml); the Rust harness reads the same file. A check is a pure
+function of the turn record and its steps' `attrs` (recording format 1.2), never `content`, so
+the same recording gives the same issues in both engines and in an export with metadata only
+(TD2-1). The functions are the ones validated on the Phase 1 in-app recordings and the golden
+scenarios (docs/observability/phase-2/lab/check_issues.py), moved here unchanged except where a
+comment says so.
+
+`check_turn` returns `{signals_version, warn, info, issues, fine, not_checked}`. An issue is
+`{check, severity, step_id, step_seq, text, facts}`: `text` is the English reference sentence,
+`facts` what a screen needs to say it in another language (TD2-3).
+"""
+
+from __future__ import annotations
+
+import functools
+from pathlib import Path
+from typing import Any, Optional
+
+import yaml
+
+SIGNALS_FILE = Path(__file__).with_name("signals.yaml")
+
+# The engine's own default when a turn did not record its step limit (before format 1.2).
+DEFAULT_MAX_ITERATIONS: Optional[int] = None
+
+
[email protected]_cache(maxsize=1)
+def signals() -> dict[str, Any]:
+ return yaml.safe_load(SIGNALS_FILE.read_text(encoding="utf-8"))
+
+
+def version() -> str:
+ return str(signals().get("version"))
+
+
+def total_tokens(step: dict) -> int:
+ u = (step.get("attrs") or {}).get("usage") or {}
+ if not u.get("reported"):
+ return 0
+ return sum(u.get(k) or 0 for k in ("input_fresh", "cache_read", "cache_write", "output"))
+
+
+def total_input(step: dict) -> int:
+ u = (step.get("attrs") or {}).get("usage") or {}
+ return sum(u.get(k) or 0 for k in ("input_fresh", "cache_read", "cache_write"))
+
+
+def purpose(step: dict) -> Optional[str]:
+ return (step.get("attrs") or {}).get("purpose")
+
+
+def issue(check: str, step: Optional[dict], text: str, **facts: Any) -> dict:
+ return {"check": check, "step_seq": step["seq"] if step else None,
+ "step_id": step["step_id"] if step else None, "text": text, **facts}
+
+
+# -- the checks --------------------------------------------------------------------------------
+# Each returns (state, issues, fine_text): state is fired | fine | not_checked.
+
+def c_repeated_step(turn, top, all_steps, p):
+ tools = [s for s in top if s["kind"] == "tool"]
+ found, run = [], []
+
+ def close():
+ if len(run) >= p["min_run"]:
+ # The main model step that asked for each call: the last main model step before it.
+ askers = set()
+ for s in run[1:]:
+ prior = [m for m in top if m["kind"] == "model" and purpose(m) == "main"
+ and m["seq"] < s["seq"]]
+ if prior:
+ askers.add(prior[-1]["step_id"])
+ extra = sum(total_tokens(m) for m in top if m["step_id"] in askers)
+ found.append(issue("repeated_step", run[-1],
+ f"{run[0]['name']} was called {len(run)} times in a row with the "
+ f"same arguments. That cost {extra:,} extra tokens.",
+ count=len(run), extra_tokens=extra))
+
+ for s in tools:
+ a = s["attrs"]
+ key = (s["name"], a.get("args_hash"))
+ breaks = s["status"] in ("denied", "skipped") or a.get("repeat_ok") or a.get("args_hash") is None
+ if run and not breaks and key == (run[-1]["name"], run[-1]["attrs"].get("args_hash")):
+ run.append(s)
+ continue
+ close()
+ run = [] if breaks else [s]
+ close()
+ return ("fired" if found else "fine"), found, f"No step repeated {p['min_run']} times in a row"
+
+
+def c_off_plan(turn, top, all_steps, p):
+ writes = [s for s in top if s["kind"] == "tool" and s["name"] == "todo_write"
+ and (s["attrs"].get("plan") or {}).get("items")]
+ if not writes:
+ return "not_checked", [], "No plan to check against"
+ found = []
+ prev: dict[str, str] = {}
+ for w in writes:
+ items = w["attrs"]["plan"]["items"]
+ now = {i["h"] for i in items}
+ vanished = [h for h, st in prev.items() if st != "done" and h not in now]
+ added = [h for h in now if h not in prev]
+ # A new item stands in for a vanished one: the agent reworded it (V2-08).
+ dropped = len(vanished) - len(added) if prev else 0
+ if dropped > 0:
+ found.append(issue("off_plan", w, f"A plan step was dropped before it was done "
+ f"({dropped} of {len(prev)}).", kind="dropped", dropped=dropped))
+ prev = {i["h"]: i["s"] for i in items}
+ # (b) acting after everything was marked done
+ for idx, w in enumerate(writes):
+ items = w["attrs"]["plan"]["items"]
+ if not all(i["s"] == "done" for i in items):
+ continue
+ nxt = writes[idx + 1]["seq"] if idx + 1 < len(writes) else None
+ later_adds = nxt is not None
+ for s in top:
+ if (s["kind"] == "tool" and s["seq"] > w["seq"] and s["name"] != "todo_write"
+ and (nxt is None or s["seq"] < nxt)
+ and s["attrs"].get("risk_class") not in ("read", None)
+ and s["status"] not in ("denied", "skipped")):
+ if not later_adds:
+ found.append(issue("off_plan", s, f"It kept acting after it marked every plan "
+ f"step done: {s['name']}.", kind="after_done"))
+ break
+ last = writes[-1]["attrs"]["plan"]["items"]
+ done = sum(1 for i in last if i["s"] == "done")
+ return ("fired" if found else "fine"), found, \
+ f"No plan step was dropped ({done} of {len(last)} done)"
+
+
+_REFUSED = {
+ "user": "You refused {name}.",
+ "denied_by_rule": "A rule refused {name}.",
+ "reviewer_denied": "The auto-approver refused {name} and you did not allow it.",
+ "unattended_auto": "{name} was refused because nobody was at the app to approve it.",
+}
+
+
+def c_refused(turn, top, all_steps, p):
+ found = []
+ for s in top:
+ origin = (s["attrs"].get("decision") or {}).get("origin") if s["kind"] == "tool" else None
+ # origin null: no stage decided (stopped, superseded), so nobody refused it (V2-07).
+ if s["kind"] == "tool" and s["status"] == "denied" and origin:
+ text = _REFUSED.get(origin, "{name} was refused.").format(name=s["name"])
+ found.append(issue("refused", s, text, by=origin))
+ return ("fired" if found else "fine"), found, "No actions were refused by you, a rule or the auto-approver"
+
+
+def _reviewer_on(turn, live_only=False):
+ r = turn.get("reviewer") or {}
+ return bool(r.get("active")) or (not live_only and bool(r.get("shadow")))
+
+
+def _checks(all_steps, shadow=None):
+ return [s for s in all_steps if s["kind"] == "auto_approver" and (shadow is None or bool(s.get("shadow")) == shadow)]
+
+
+def _call_name(all_steps, call_id):
+ for s in all_steps:
+ if s["kind"] == "tool" and s.get("call_id") == call_id:
+ return s["name"]
+ return "an action"
+
+
+def c_overridden(turn, top, all_steps, p):
+ if not _reviewer_on(turn, live_only=True):
+ return "not_checked", [], ""
+ found = [issue("overridden", s, f"The auto-approver blocked {s['name']} and you used Allow anyway.")
+ for s in top if s["kind"] == "tool"
+ and any(x.get("stage") == "grant" and x.get("outcome") == "allow_anyway"
+ for x in (s["attrs"].get("decision") or {}).get("path", []))]
+ return ("fired" if found else "fine"), found, "You did not override the auto-approver"
+
+
+def c_reviewer_could_not_run(turn, top, all_steps, p):
+ if not _reviewer_on(turn):
+ return "not_checked", [], ""
+ checks = _checks(all_steps)
+ found = [issue("reviewer_could_not_run", s, f"The auto-approver could not check "
+ f"{_call_name(all_steps, s.get('call_id'))}, so it was shown to you instead.")
+ for s in checks if s["attrs"].get("result") == "could_not_run"]
+ live = [s for s in checks if not s.get("shadow")]
+ n = lambda k: f"{k} action" + ("" if k == 1 else "s") # noqa: E731
+ if live: # the verdicts, so "allowed everything" does not read as just "ran" (M2-R1)
+ res = [s["attrs"].get("result") for s in live]
+ fine = (f"Auto-approver checked {n(len(live))}: {res.count('allow')} allowed, "
+ f"{sum(r in ('unsure', 'context_changed', 'could_not_run') for r in res)} asked you, "
+ f"{res.count('deny')} blocked; none failed to run")
+ else: # a shadow-only turn (M2-R5)
+ fine = f"Shadow reviewer checked {n(len(checks))}; none failed to run"
+ return ("fired" if found else "fine"), found, fine
+
+
+def c_reviewer_paused(turn, top, all_steps, p):
+ if not _reviewer_on(turn, live_only=True):
+ return "not_checked", [], ""
+ notices = [s for s in top if s["kind"] == "notice" and s["name"] == "reviewer_paused"]
+ tripped = [s for s in _checks(all_steps) if (s["attrs"].get("breaker") or {}).get("tripped")]
+ at = notices[0] if notices else (tripped[0] if tripped else None)
+ if not at:
+ return "fine", [], "Auto-approver was never paused"
+ denials = max([(s["attrs"].get("breaker") or {}).get("denials_in_row") or 0 for s in tripped] or [0])
+ return "fired", [issue("reviewer_paused", at, f"The auto-approver paused after {denials} "
+ f"refusals in a row. Later actions were shown to you.")], ""
+
+
+def c_reviewer_allowed_send(turn, top, all_steps, p):
+ if not _reviewer_on(turn, live_only=True):
+ return "not_checked", [], ""
+ found = [issue("reviewer_allowed_send", s, f"The auto-approver allowed {s['name']} without asking you. "
+ f"It sends data off the machine.")
+ for s in top if s["kind"] == "tool" and s["status"] == "ok"
+ and (s["attrs"].get("risk_class") == "egress" or s["attrs"].get("reaches_network"))
+ and (s["attrs"].get("decision") or {}).get("origin") == "reviewer"]
+ return ("fired" if found else "fine"), found, "The auto-approver let nothing leave the machine without asking you"
+
+
+def c_shadow_disagreed(turn, top, all_steps, p):
+ if not (turn.get("reviewer") or {}).get("shadow"):
+ return "not_checked", [], ""
+ shadow = _checks(all_steps, shadow=True)
+ compared = [s for s in shadow if s["attrs"].get("matched_user") is not None]
+ found = [issue("shadow_disagreed", s, f"The shadow reviewer would have said "
+ f"{s['attrs'].get('result')} on {_call_name(all_steps, s.get('call_id'))}.")
+ for s in compared if s["attrs"].get("matched_user") is False]
+ return ("fired" if found else "fine"), found, \
+ f"The shadow reviewer agreed with you on {len(compared)} of {len(compared)} actions"
+
+
+def c_tool_failed(turn, top, all_steps, p):
+ tools = [s for s in top if s["kind"] == "tool"]
+ sandboxed = ((turn.get("stamp") or {}).get("sandbox") or {}).get("enforcement") not in (None, "none")
+ found = []
+ for i, s in enumerate(tools):
+ a = s["attrs"]
+ sb = a.get("sandbox") or {}
+ if sandboxed and sb.get("blocked_known") and (sb.get("blocked_count") or 0) > 0:
+ continue # blocked_by_sandbox says it, with the exit code (V2-10)
+ failed = ((s["status"] == "error" and a.get("error_kind") in ("raised", None))
+ or (s["status"] == "ok" and a.get("result_error"))
+ or (a.get("exit_code") not in (None, 0)))
+ if not failed:
+ continue
+ recovered = any(t["name"] == s["name"] and t["status"] == "ok" and not t["attrs"].get("result_error")
+ and t["attrs"].get("exit_code") in (None, 0) for t in tools[i + 1:])
+ exit_part = f" with exit code {a['exit_code']}" if a.get("exit_code") not in (None, 0) else ""
+ rec = f" A later {s['name']} in this turn worked." if recovered else ""
+ found.append(issue("tool_failed", s, f"{s['name']} failed{exit_part}.{rec}",
+ recovered=recovered, how=("exit" if exit_part else
+ "result_error" if a.get("result_error") else "raised")))
+ return ("fired" if found else "fine"), found, "No tool failed"
+
+
+def c_model_failed(turn, top, all_steps, p):
+ # Main calls only: a failed summary is context_full's business (trim, summary_failed),
+ # so it is not reported twice (V2-06).
+ models = [s for s in top if s["kind"] == "model" and purpose(s) == "main"]
+ found = []
+ for i, s in enumerate(models):
+ err = (s["attrs"].get("error") or {}).get("type")
+ if s["status"] != "error" or err in ("context_length", "cancelled"):
+ continue
+ if any(m["status"] == "ok" and purpose(m) == purpose(s) for m in models[i + 1:]):
+ continue
+ tail = " The turn stopped here." if turn["status"] == "error" else ""
+ found.append(issue("model_failed", s, f"The model call failed ({err}).{tail}",
+ purpose=purpose(s), error_type=err))
+ return ("fired" if found else "fine"), found, "No model call failed"
+
+
+def c_retried(turn, top, all_steps, p):
+ # Not the session title: its retry is the engine's own parameter fallback (V2-05).
+ models = [s for s in all_steps if s["kind"] == "model" and not s.get("shadow")
+ and purpose(s) != "title"]
+ models.sort(key=lambda s: s["seq"])
+ found, done = [], set()
+ for s in models:
+ wire = [m for m in (s["attrs"].get("provider_meta") or []) if not m.get("auxiliary")]
+ if s["status"] == "ok" and len(wire) > 1:
+ found.append(issue("retried", s, f"The model call needed {len(wire)} tries ({purpose(s)}).",
+ attempts=len(wire), how="wire", purpose=purpose(s)))
+ done.add(s["step_id"])
+ for i, s in enumerate(models):
+ # A too-long call that was summarised and tried again is context_full's (V2-09).
+ if s["status"] != "error" or (s["attrs"].get("error") or {}).get("type") in ("cancelled", "context_length"):
+ continue
+ later = [m for m in models[i + 1:] if purpose(m) == purpose(s)
+ and m.get("parent_step_id") == s.get("parent_step_id")]
+ ok = next((m for m in later if m["status"] == "ok"), None)
+ if ok and ok["step_id"] not in done:
+ tries = 1 + sum(1 for m in later if m["seq"] <= ok["seq"])
+ found.append(issue("retried", ok, f"The model call needed {tries} tries ({purpose(s)}).",
+ attempts=tries, how="engine", purpose=purpose(s)))
+ done.add(ok["step_id"])
+ return ("fired" if found else "fine"), found, "No model call needed a second try" # M2-R2
+
+
+def c_cut_off(turn, top, all_steps, p):
+ found = [issue("cut_off", s, "The model stopped at its output limit. The answer may be incomplete.")
+ for s in top if s["kind"] == "model" and purpose(s) == "main"
+ and s["attrs"].get("finish_reason") == "length"]
+ if not found and turn["status"] == "truncated":
+ n = next((s for s in top if s["kind"] == "notice" and s["name"] == "truncated"), None)
+ found.append(issue("cut_off", n, "The model stopped at its output limit. The answer may be incomplete."))
+ return ("fired" if found else "fine"), found, "No answers were cut off (length limit)"
+
+
+def c_step_limit(turn, top, all_steps, p):
+ iterations = sum(1 for s in top if s["kind"] == "model" and purpose(s) == "main")
+ cap = turn.get("max_iterations") or DEFAULT_MAX_ITERATIONS
+ n = next((s for s in top if s["kind"] == "notice" and s["name"] == "step_limit"), None)
+ if turn["status"] == "max_iterations_exceeded" or n:
+ return "fired", [issue("step_limit", n, f"The turn stopped after {iterations} steps, the limit.",
+ iterations=iterations)], ""
+ return "fine", [], (f"Stayed under the step limit ({iterations} of {cap})" if cap
+ else f"Stayed under the step limit ({iterations} steps)")
+
+
+def c_made_up_tool(turn, top, all_steps, p):
+ found = []
+ for s in top:
+ k = s["attrs"].get("error_kind") if s["kind"] == "tool" else None
+ if s["kind"] == "tool" and s["status"] == "error" and k in ("unknown_tool", "bad_arguments"):
+ text = (f"The model asked for a tool that does not exist: {s['name']}." if k == "unknown_tool"
+ else f"The model sent arguments for {s['name']} that could not be read.")
+ found.append(issue("made_up_tool", s, text, kind=k))
+ return ("fired" if found else "fine"), found, "No unknown tools or invalid arguments"
+
+
+def c_context_full(turn, top, all_steps, p):
+ found = []
+ summaries = [s for s in all_steps if s["kind"] == "summary"]
+ for s in summaries:
+ if s["name"] == "trim":
+ found.append(issue("context_full", s, "Older messages were cut to make room.", kind="trim"))
+ elif s["attrs"].get("reason") == "overflow":
+ found.append(issue("context_full", s, "The conversation became too long for the model, "
+ "so it was summarised in a hurry.", kind="overflow"))
+ for s in top:
+ if s["kind"] == "notice" and s["name"] == "summary_failed":
+ found.append(issue("context_full", s, "Summarising failed, so the context stayed full.",
+ kind="summary_failed"))
+ max_fill = 0.0
+ for s in top:
+ a = s["attrs"]
+ if s["kind"] == "model" and purpose(s) == "main" and (a.get("usage") or {}).get("reported") \
+ and a.get("context_window"):
+ fill = total_input(s) / a["context_window"]
+ max_fill = max(max_fill, fill)
+ if fill >= p["hard_fill"]:
+ found.append(issue("context_full", s, f"The context was {round(fill * 100)}% full on this call.",
+ kind="hard_fill", fill=round(fill, 3)))
+ normal = sum(1 for s in summaries if s["attrs"].get("reason") == "threshold" and s["name"] != "trim")
+ part = f"; summarised {normal} time(s) at the normal point" if normal else ""
+ return ("fired" if found else "fine"), found, \
+ f"Context had room (at most {max(1, round(max_fill * 100))}% of the window){part}" # M2-R7
+
+
+def c_blocked_by_sandbox(turn, top, all_steps, p):
+ sb = (turn.get("stamp") or {}).get("sandbox") or {}
+ if sb.get("enforcement") in (None, "none"):
+ return "not_checked", [], ""
+ found = [issue("blocked_by_sandbox", s, f"The sandbox blocked {s['attrs']['sandbox']['blocked_count']} "
+ f"network request(s) during {s['name']}"
+ + (f", which exited with code {s['attrs']['exit_code']}." if s["attrs"].get("exit_code") not in (None, 0) else "."))
+ for s in top if s["kind"] == "tool" and (s["attrs"].get("sandbox") or {}).get("blocked_known")
+ and ((s["attrs"].get("sandbox") or {}).get("blocked_count") or 0) > 0]
+ return ("fired" if found else "fine"), found, "The sandbox blocked nothing"
+
+
+def c_sandbox_failed(turn, top, all_steps, p):
+ setups = [s for s in top if s["kind"] == "sandbox" and s["name"] == "setup"]
+ if not setups:
+ return "not_checked", [], ""
+ found = [issue("sandbox_failed", s, f"The sandbox could not start at {s['attrs'].get('failed_stage')}: "
+ f"{(s['attrs'].get('error') or {}).get('type')}.") for s in setups if s["status"] == "error"]
+ s0 = setups[0]
+ return ("fired" if found else "fine"), found, f"The sandbox started ({s0['attrs'].get('provider')}, {round((s0.get('duration_ms') or 0) / 1000)} s)"
+
+
+def c_ran_without_sandbox(turn, top, all_steps, p):
+ sb = (turn.get("stamp") or {}).get("sandbox") or {}
+ if not sb.get("expected"):
+ # No sandbox switched on here (or format 1.1, which cannot tell): the check does not
+ # run, so "Ran inside the sandbox" is never said of a machine with none (signals 0.3).
+ return "not_checked", [], ""
+ if sb.get("provider") != "direct":
+ return "fine", [], "Ran inside the sandbox"
+ acts = [s for s in top if s["kind"] == "tool" and s["status"] == "ok"
+ and s["attrs"].get("risk_class") in ("exec", "egress", "write_local")]
+ if not acts:
+ return "fine", [], "Ran inside the sandbox"
+ return "fired", [issue("ran_without_sandbox", acts[0], f"This machine has a sandbox switched on, but "
+ f"this turn ran {len(acts)} action(s) without it.")], ""
+
+
+def c_waited_unwatched(turn, top, all_steps, p):
+ found = []
+ for s in top:
+ a = s["attrs"]
+ if s["kind"] != "wait" or a.get("kind") not in ("approval", "question", "plan") or a.get("where") != "inbox":
+ continue
+ start = a.get("inbox_at") or s.get("started_at") or 0
+ end = (s.get("started_at") or 0) + (s.get("duration_ms") or 0)
+ if end - start >= p["min_wait_ms"]:
+ found.append(issue("waited_unwatched", s, f"waited {round((end - start) / 60000)} min in the "
+ f"Inbox for an answer while nobody was at the app."))
+ return ("fired" if found else "fine"), found, "Nothing waited long for an answer while nobody was watching"
+
+
+CHECKS = {
+ "repeated_step": c_repeated_step, "off_plan": c_off_plan, "refused": c_refused,
+ "overridden": c_overridden, "reviewer_could_not_run": c_reviewer_could_not_run,
+ "reviewer_paused": c_reviewer_paused, "reviewer_allowed_send": c_reviewer_allowed_send,
+ "shadow_disagreed": c_shadow_disagreed,
+ "tool_failed": c_tool_failed, "model_failed": c_model_failed, "retried": c_retried,
+ "cut_off": c_cut_off, "step_limit": c_step_limit, "made_up_tool": c_made_up_tool,
+ "context_full": c_context_full, "blocked_by_sandbox": c_blocked_by_sandbox, "sandbox_failed": c_sandbox_failed,
+ "ran_without_sandbox": c_ran_without_sandbox, "waited_unwatched": c_waited_unwatched,
+}
+
+
+def check_turn(turn: dict, steps: list[dict], spec: Optional[dict] = None) -> dict:
+ """Every check over one finished turn. Pure: reads the turn and its steps' attrs only."""
+ spec = spec or signals()
+ steps = sorted(steps, key=lambda s: s.get("seq") or 0)
+ usable = [s for s in steps if not s.get("after_turn")]
+ top = [s for s in usable if s.get("parent_step_id") is None and not s.get("shadow")]
+ out: dict[str, Any] = {"signals_version": str(spec.get("version")), "issues": [], "fine": [],
+ "not_checked": []}
+ for cid, check in spec["checks"].items():
+ fn = CHECKS.get(cid)
+ if fn is None:
+ continue # a check this app version does not know yet: left out, never guessed
+ try:
+ state, found, fine = fn(turn, top, usable, check.get("params") or {})
+ except Exception: # noqa: BLE001 - one broken check never hides the others
+ state, found, fine = "not_checked", [], ""
+ for f in found:
+ out["issues"].append({
+ "check": cid, "severity": check["severity"], "step_id": f.get("step_id"),
+ "step_seq": f.get("step_seq"), "text": f.get("text", ""),
+ "facts": {k: v for k, v in f.items() if k not in ("check", "step_id", "step_seq", "text")},
+ })
+ if state == "fired":
+ continue
+ (out["fine"] if state == "fine" else out["not_checked"]).append({"check": cid, "text": fine})
+ out["warn"] = sum(1 for i in out["issues"] if i["severity"] == "warn")
+ out["info"] = sum(1 for i in out["issues"] if i["severity"] == "info")
+ return out
diff --git a/coworker/observability/signals.yaml b/coworker/observability/signals.yaml
new file mode 100644
index 00000000..a67567a1
--- /dev/null
+++ b/coworker/observability/signals.yaml
@@ -0,0 +1,422 @@
+# OpenWorker observability: the Issues checks (signals)
+#
+# What this file is. Every check on the Issues tab (OB-3), in one place: what it looks at,
+# when it fires, its numbers, and the words it shows. Both engines (the desktop's Python
+# engine now, the Rust harness SDK later) read the same file, so the same recording gets
+# the same issues in both. The numbers in a rule ("3 times", "80%") are settings here, not
+# code.
+#
+# Status: DRAFT 0.2 (2026-11-02), Phase 2 design. Not frozen. It freezes at the end of the
+# Phase 2 design (phased-plan.md section 3), after validation on real recordings
+# (phase-2/validation-report.md).
+#
+# How a check works (phase-2/technical-design.md section 2).
+# - A check is a pure function of one finished turn: the turn record and its steps, as
+# recording-format.md defines them, plus the step fields this file adds (`needs`). It
+# reads `attrs` only, never `content`, so the same issues come out of a recording whose
+# text was left out (export with metadata only).
+# - Every check gives one of three answers for a turn:
+# fired one or more issues, each pointing at a step
+# fine it ran and found nothing; the tab lists it under "Checked and fine"
+# not_checked it does not apply to this turn (for example the auto-approver checks in
+# a turn with no reviewer); the tab does not list it
+# - `severity`: `warn` issues count on the tab and turn the session dot amber; `info`
+# issues are listed but count nowhere.
+#
+# How to read an entry.
+# title the issue's name on the tab
+# severity warn | info
+# applies when the check runs at all (else not_checked)
+# fires the exact rule, in plain words, with the fields it reads
+# points_at which step the issue's "Go to step" opens
+# params the numbers, with their defaults
+# says the issue text; {placeholders} are filled from the fields named
+# fine the "Checked and fine" line
+# needs recording fields this check needs that format 1.1 does not have yet
+# (the Phase 2 additions, phase-2/technical-design.md section 4)
+# golden the scenarios that test it (phase-2/scenarios.md)
+# source where the check was asked for
+
+version: "0.3"
+draft: 2026-11-02
+# 0.3: ran_without_sandbox runs only where a sandbox is expected (found in the first live run).
+# 0.2: reviewer_allowed_send (TD2-20), sandbox_failed, the mock-review wording (M2-R1 to M2-R9),
+# waited_unwatched from inbox_at (V2-25). Goes with recording format 1.2.
+
+# Steps the checks never look at, whatever the check:
+# - nested steps (parent_step_id not null) unless a check says otherwise: a helper's own
+# tool calls belong to the helper (OB-7 nests them; their checks come with OB-7)
+# - shadow steps, except in shadow_disagreed
+# - steps with after_turn = true (the session title runs after the answer)
+scope:
+ top_level_only: true
+ skip_shadow: true
+ skip_after_turn: true
+
+checks:
+
+ repeated_step:
+ title: "Repeated the same step"
+ severity: warn
+ applies: "always"
+ fires: >-
+ `min_run` or more tool steps in a row (canonical order, top-level tool steps only)
+ with the same `name` and the same `args_hash`, none of them `repeat_ok`. Steps with
+ status `denied` or `skipped` break a run (a refused call that is asked again is the
+ refused check's business). One issue per run.
+ points_at: "the last step of the run"
+ params:
+ min_run: 3
+ says: "{name} was called {count} times in a row with the same arguments. That cost {extra_tokens} extra tokens."
+ extra_tokens: >-
+ Total tokens (input + output, recording-format.md 3.5) of the main model steps that
+ asked for the 2nd to the last call of the run. A model step that asked for several
+ calls counts once.
+ fine: "No step repeated {min_run} times in a row"
+ needs: [tool.args_hash, tool.repeat_ok]
+ golden: [S2-01, S2-01c]
+ source: "desktop-feature-spec.md OB-3"
+
+ off_plan:
+ title: "Left its plan"
+ severity: warn
+ applies: "the turn has at least one top-level todo_write step with at least one item"
+ fires: >-
+ Either (a) an item disappeared from the plan before it was done: comparing a
+ todo_write's `plan.items` with the previous todo_write in the same turn, the items
+ that were not `done` and whose hash is gone outnumber the new hashes (a new item
+ stands in for a gone one, because the agent may reword an item; V2-08); or (b) the agent kept acting after it said the plan was done:
+ a top-level tool step other than todo_write, with risk_class other than `read`,
+ starts after a todo_write whose items are all `done`, and no later todo_write adds
+ an item. Version 1 matches items by exact hash only (Q6; FO-12 is fuzzy matching).
+ points_at: "(a) the todo_write that dropped the item; (b) the first such tool step"
+ params: {}
+ says:
+ dropped: "A plan step was dropped before it was done ({dropped} of {items})."
+ after_done: "It kept acting after it marked every plan step done: {name}."
+ fine: "No plan step was dropped ({done} of {items} done)"
+ not_checked_says: "No plan to check against"
+ note: >-
+ Per Q6 the tab shows "No plan to check against" in the fine list when there is no
+ plan, instead of hiding the check, so an empty list never reads as "we checked".
+ Items still pending when the turn ends are not an issue: the agent may be waiting
+ for you. They show in the fine line ("3 of 4 steps done").
+ needs: [tool.plan]
+ golden: [S2-02, S2-02b, S2-02c]
+ source: "desktop-feature-spec.md OB-3, Q6"
+
+ refused:
+ title: "An action was refused"
+ severity: warn
+ applies: "always"
+ fires: >-
+ A top-level tool step with status `denied` and a `decision.origin` (who said no:
+ `user`, `denied_by_rule`, `reviewer_denied`, `unattended_auto`). A denied step with
+ origin `null` was stopped or superseded, not refused, so it does not fire (V2-07).
+ One issue per step.
+ points_at: "the denied step"
+ params: {}
+ says:
+ user: "You refused {name}."
+ denied_by_rule: "A rule refused {name}."
+ reviewer_denied: "The auto-approver refused {name} and you did not allow it."
+ unattended_auto: "{name} was refused because nobody was at the app to approve it."
+ fine: "No actions were refused by you, a rule or the auto-approver"
+ needs: []
+ golden: [S2-03, S2-03b]
+ source: "desktop-feature-spec.md OB-3"
+
+ overridden:
+ title: "You allowed a blocked action"
+ severity: info
+ applies: "the turn's reviewer is active (turn.reviewer.active)"
+ fires: "A top-level tool step whose decision.path has a `grant` stage with outcome `allow_anyway`."
+ points_at: "the tool step"
+ params: {}
+ says: "The auto-approver blocked {name} and you used Allow anyway. Either the reviewer or the decision was wrong; worth a look."
+ fine: "You did not override the auto-approver"
+ needs: []
+ golden: [S2-04]
+ source: "desktop-feature-spec.md OB-3"
+
+ reviewer_could_not_run:
+ title: "Auto-approver could not run"
+ severity: warn
+ applies: "turn.reviewer.active or turn.reviewer.shadow"
+ fires: "An auto_approver step (live or shadow) with result `could_not_run`."
+ points_at: "the auto_approver step"
+ params: {}
+ says: "The auto-approver could not check {name}, so it was shown to you instead."
+ fine:
+ live: "Auto-approver checked {checks} action(s): {allowed} allowed, {asked} asked you, {blocked} blocked; none failed to run"
+ shadow: "Shadow reviewer checked {checks} action(s); none failed to run"
+ fine_note: "The verdicts are in the line, so a turn where the reviewer allowed everything does not read as just \"ran\" (mock review M2-R1). A shadow-only turn names the shadow reviewer (M2-R5)."
+ needs: []
+ golden: [S2-05]
+ source: "desktop-feature-spec.md OB-3"
+
+ reviewer_paused:
+ title: "Auto-approver paused"
+ severity: warn
+ applies: "turn.reviewer.active"
+ fires: "A notice step named `reviewer_paused`, or an auto_approver step with breaker.tripped = true."
+ points_at: "the notice, else the first tripped check"
+ params: {}
+ says: "The auto-approver paused after {denials} refusals in a row. Later actions were shown to you."
+ fine: "Auto-approver was never paused"
+ needs: []
+ golden: [S2-06]
+ source: "desktop-feature-spec.md OB-3"
+
+ reviewer_allowed_send:
+ title: "The auto-approver let data leave the machine"
+ severity: info
+ applies: "turn.reviewer.active"
+ fires: >-
+ A top-level tool step that ran (status `ok`), with decision.origin `reviewer`, that
+ reaches the network: risk_class `egress`, or `reaches_network` true (new in 1.2; a
+ shell command that names a network client or a URL, by the engine's own rule,
+ `review_facts.shell_reaches_network`). Shell commands are classed `exec` even when
+ they upload, so risk_class alone missed every real case (B5). The auto-approver, not
+ you or a rule, allowed an action that sends data off the machine. Information, not a warning: the reviewer may be
+ right; this makes the decision visible. Found in the Phase 2 mock review (M2-R1):
+ the reviewer allowed eight uploads of a file to unknown hosts and nothing was
+ flagged. TD2-20.
+ points_at: "the tool step; one line per step, grouped"
+ params: {}
+ says: "The auto-approver allowed {name} without asking you. It sends data off the machine."
+ fine: "The auto-approver let nothing leave the machine without asking you"
+ needs: [tool.reaches_network]
+ golden: [S2-19, S2-19c]
+ source: "phase-2/mock-review/REVIEW.md M2-R1; TD2-20"
+
+ shadow_disagreed:
+ title: "Shadow check disagreed with you"
+ severity: info
+ applies: "turn.reviewer.shadow"
+ fires: "An auto_approver step with shadow = true and matched_user = false."
+ points_at: "the shadow check; the line names the tool call it judged (mock review M2-R6)"
+ params: {}
+ says: "The shadow reviewer would have {result_in_words} {name}; you {user_decision}. You were the decider; this is information."
+ fine: "The shadow reviewer agreed with you on {compared} of {compared} actions"
+ needs: []
+ golden: [S2-07]
+ source: "desktop-feature-spec.md OB-3"
+
+ tool_failed:
+ title: "A tool failed"
+ severity: warn
+ applies: "always"
+ fires: >-
+ A top-level tool step that ran and failed: status `error` with `error_kind` `raised`
+ or `result_error`, or status `ok` with `result_error` = true (a tool that returns an
+ error instead of raising, such as read_file on a missing file), or a non-zero
+ `exit_code`. Unknown tools and bad arguments are made_up_tool, not this check. A
+ step the sandbox blocked requests on (blocked_by_sandbox fires on it) is not
+ reported again here: one issue per step, the more specific one (V2-10).
+ points_at: "the failed step"
+ params: {}
+ says: "{name} failed{exit_part}. {first_error_line} {recovered_part}"
+ first_error_line: "Shown in the app only: the first line of the command's error output (content), so \"exit code 6\" reads as \"Could not resolve host\" (mock review M2-R8)."
+ exit_part: " with exit code {exit_code}"
+ recovered_part:
+ yes: "A later {name} in this turn worked."
+ no: ""
+ fine: "No tool failed"
+ needs: [tool.error_kind, tool.result_error]
+ golden: [S2-08, S2-08b]
+ source: "desktop-feature-spec.md OB-3; build-notes.md (error results)"
+
+ model_failed:
+ title: "A model call failed"
+ severity: warn
+ applies: "always"
+ fires: >-
+ A top-level main model step with status `error`, error.type other than
+ `context_length` and `cancelled`, that is not followed in the same turn by a main
+ model step with status `ok` (a failure that was retried and then worked is the
+ retried check's business). Added in Phase 2 validation (V2-03): a turn that ends
+ because its model call failed matched no check in the spec. A failed summary call
+ is not counted here: context_full reports what it led to, a trim or summary_failed
+ (V2-06). A failed session title is not counted anywhere (FO-97).
+ points_at: "the failed model step"
+ params: {}
+ says: "The model call failed: {error_in_words}. {provider_message} {turn_part}"
+ provider_message: "Shown in the app only: the provider's own error message from the response (content), cut to one line. The error type alone, and error.text (a URL), did not say what went wrong (mock review M2-R3)."
+ turn_part:
+ error: "The turn stopped here."
+ other: ""
+ fine: "No model call failed"
+ needs: []
+ golden: [S2-09]
+ source: "phase-2/validation-report.md V2-03"
+
+ retried:
+ title: "A model call was retried"
+ severity: warn
+ applies: "always"
+ fires: >-
+ Either a top-level model step whose `provider_meta` has more than one entry without
+ `auxiliary` = true (the provider's SDK retried on the wire), or a model step with
+ status `error` followed later in the same parent (the turn, or the same summary
+ step) by a model step of the same purpose with status `ok` (the engine tried
+ again). An error of type `context_length` or `cancelled` does not count: a call
+ that was too long, summarised and tried again is context_full's (V2-09). Nested
+ model steps count here (a summary's retry is nested in its summary
+ step). Session title calls never count: their second try is the engine's own
+ parameter fallback, not provider trouble (V2-05). One issue per call that needed
+ more than one try.
+ points_at: "the model step that finally worked"
+ params: {}
+ says: "The model call needed {attempts} tries ({purpose})."
+ fine: "No model call needed a second try"
+ fine_note: "Not \"worked first time\": that read as a contradiction beside \"A model call failed\" (mock review M2-R2)."
+ needs: []
+ golden: [S2-10, S2-10b]
+ source: "desktop-feature-spec.md OB-3"
+
+ cut_off:
+ title: "An answer was cut off"
+ severity: warn
+ applies: "always"
+ fires: >-
+ A top-level main model step with finish_reason `length` (the engine's normalized
+ value; tracing-spec.md), or turn.status `truncated`.
+ points_at: "the model step, else the `truncated` notice"
+ params: {}
+ says: "The model stopped at its output limit{limit_part}. The answer may be incomplete."
+ limit_part: " ({max_output} tokens); needs `max_output` on model steps (1.2, mock review M2-R9)"
+ fine: "No answers were cut off (length limit)"
+ needs: [model.max_output]
+ golden: [S2-11]
+ source: "desktop-feature-spec.md OB-3"
+
+ step_limit:
+ title: "Hit the step limit"
+ severity: warn
+ applies: "always"
+ fires: "turn.status `max_iterations_exceeded`, or a notice step named `step_limit`."
+ points_at: "the `step_limit` notice"
+ params: {}
+ says: "The turn stopped after {iterations} steps, the limit. The task may not be finished."
+ fine: "Stayed under the step limit ({iterations} of {max_iterations})"
+ iterations: "the number of top-level main model steps in the turn"
+ needs: [turn.max_iterations]
+ golden: [S2-12]
+ source: "desktop-feature-spec.md OB-3"
+
+ made_up_tool:
+ title: "Unknown tool or bad arguments"
+ severity: warn
+ applies: "always"
+ fires: "A top-level tool step with status `error` and error_kind `unknown_tool` or `bad_arguments`."
+ points_at: "the tool step"
+ params: {}
+ says:
+ unknown_tool: "The model asked for a tool that does not exist: {name}."
+ bad_arguments: "The model sent arguments for {name} that could not be read."
+ fine: "No unknown tools or invalid arguments"
+ needs: [tool.error_kind]
+ golden: [S2-13]
+ source: "desktop-feature-spec.md OB-3"
+
+ context_full:
+ title: "Context was full"
+ severity: warn
+ applies: "always"
+ fires: >-
+ Any of: (a) a summary step with reason `overflow` (the provider refused the call as
+ too long, so the summary was forced); (b) a summary step named `trim` (older
+ messages were cut, not summarised); (c) a notice step named `summary_failed`; (d) a
+ top-level main model step whose context fill is at least `hard_fill`. Context fill
+ = total input (recording-format.md 3.5) / attrs.context_window, for a step with
+ usage reported and a window known. A summary at the normal threshold (reason
+ `threshold`) is not an issue: it is the engine working as planned (V2-04).
+ points_at: "the summary or notice, else the model step"
+ params:
+ hard_fill: 0.95
+ titles:
+ overflow: "Context overflowed"
+ trim: "Older messages were cut"
+ summary_failed: "Summary failed"
+ hard_fill: "Context nearly full"
+ says:
+ overflow: "The conversation became too long for the model, so it was summarised in a hurry."
+ trim: "Summarising failed ({summary_error}), so older messages were cut to make room. The agent no longer sees them."
+ summary_failed: "Summarising failed ({summary_error}), so the context stayed full."
+ hard_fill: "The context was {fill_pct}% full on this call."
+ summary_error: "the error type of the summary's last failed model call, in words"
+ fine: "Context had room (at most {max_fill_pct}% of the window){summary_part}"
+ summary_part: "; summarised {summaries} time(s) at the normal point"
+ title_note: "One title per kind: a failed summary followed by Trim is not \"context was full\" (mock review M2-R4)."
+ needs: []
+ golden: [S2-14, S2-14b]
+ source: "desktop-feature-spec.md OB-3; validation V2-04"
+
+ blocked_by_sandbox:
+ title: "Blocked by the sandbox"
+ severity: warn
+ applies: "turn.stamp.sandbox.enforcement is not `none`"
+ fires: "A top-level tool step with sandbox.blocked_known = true and sandbox.blocked_count > 0."
+ points_at: "the tool step"
+ params: {}
+ says: "The sandbox blocked {blocked_count} network request(s) during {name}{exit_part}. If the command needed them, add the site in Settings."
+ exit_part: ", which exited with code {exit_code}"
+ fine: "The sandbox blocked nothing"
+ needs: []
+ golden: [S2-15]
+ source: "across-features.md OB-12"
+
+ sandbox_failed:
+ title: "The sandbox could not start"
+ severity: warn
+ applies: "the turn has a top-level step of kind `sandbox` (new in 1.2, OB-13)"
+ fires: "A `sandbox` step named `setup` with status `error`."
+ points_at: "the setup step"
+ params: {}
+ says: "The sandbox could not start at {failed_stage}: {error_type}."
+ fine: "The sandbox started ({provider}, {setup_s} s)"
+ needs: [step.kind.sandbox]
+ golden: [S2-18, S2-18c]
+ source: "phase-2/technical-design.md section 12 (OB-13)"
+
+ ran_without_sandbox:
+ title: "Ran without the sandbox"
+ severity: warn
+ applies: >-
+ turn.stamp.sandbox.expected is true: this machine has a sandbox switched on. Otherwise
+ the check does not run, and "Checked and fine" does not list it: on a machine with no
+ sandbox set up, "Ran inside the sandbox" would be false (found live, 2026-11-02)
+ fires: >-
+ turn.stamp.sandbox.expected = true and turn.stamp.sandbox.provider = `direct` (the
+ machine has a sandbox switched on but this session ran direct, or the sandbox could
+ not start), and the turn has at least one top-level tool step with risk_class
+ `exec`, `egress` or `write_local` that ran (Q24).
+ points_at: "the first such tool step"
+ params: {}
+ says: "This machine has a sandbox switched on, but this turn ran {count} action(s) without it."
+ fine: "Ran inside the sandbox"
+ needs: [stamp.sandbox.expected]
+ golden: [S2-16]
+ source: "across-features.md OB-12, Q24"
+
+ waited_unwatched:
+ title: "Waited while nobody was watching"
+ severity: warn
+ applies: "always"
+ fires: >-
+ A wait step of kind `approval`, `question` or `plan` with where = `inbox` that
+ waited at least `min_wait_ms` after it reached the Inbox: wait end minus `inbox_at`
+ (new; the step's start when it began in the Inbox). A card reaches the Inbox when
+ the turn started unattended or when the last viewer left the session
+ (`inbox.py` promote_to_inbox), so where = `inbox` is the recorded fact for "nobody
+ was watching". The wait's `attended` is not used: it is the Unattended toggle, not
+ whether a desktop is connected (V2-25). Q27.
+ points_at: "the wait step"
+ params:
+ min_wait_ms: 900000
+ says: "{name} waited {wait} in the Inbox for an answer while nobody was at the app."
+ fine: "Nothing waited long for an answer while nobody was watching"
+ needs: [wait.inbox_at]
+ golden: [S2-17, S2-17b, S2-17c]
+ source: "across-features.md OB-14, Q27; validation V2-25"
diff --git a/coworker/observability/store.py b/coworker/observability/store.py
index 16290ad4..59664091 100644
--- a/coworker/observability/store.py
+++ b/coworker/observability/store.py
@@ -70,7 +70,16 @@ CREATE INDEX IF NOT EXISTS step_pieces_sha ON step_pieces(sha256);
_TURN_COLUMNS = (
"turn_id", "session_id", "seq", "kind", "trigger", "started_at", "duration_ms", "status",
"incomplete", "approval_mode", "attended", "reviewer", "stamp", "first_message_id",
- "last_message_id", "schema_version",
+ "last_message_id", "schema_version", "max_iterations", "woken_by",
+)
+# Columns added after the first release of the store: an existing recordings.db gets them
+# here (CREATE TABLE IF NOT EXISTS does not add columns). `issues` and `issues_seen` hold the
+# Issues checks' result for the turn (phase-2 design §5): derived, not part of the format.
+_ADDED_COLUMNS = (
+ ("turns", "max_iterations", "INTEGER"),
+ ("turns", "woken_by", "TEXT"),
+ ("turns", "issues", "TEXT"),
+ ("turns", "issues_seen", "INTEGER DEFAULT 0"),
)
_STEP_COLUMNS = (
"step_id", "turn_id", "session_id", "parent_step_id", "seq", "kind", "name", "started_at",
@@ -112,7 +121,7 @@ def _turn_row(t: dict[str, Any]) -> tuple:
t.get("started_at"), t.get("duration_ms"), t.get("status"), int(bool(t.get("incomplete"))),
t.get("approval_mode"), _bool(t.get("attended")), _js(t.get("reviewer")),
_js(t.get("stamp") or {}), t.get("first_message_id"), t.get("last_message_id"),
- t.get("schema_version", 1),
+ t.get("schema_version", 1), t.get("max_iterations"), _js(t.get("woken_by")),
)
@@ -172,9 +181,15 @@ class RecordingStore:
self._conn.row_factory = sqlite3.Row
self._conn.execute("PRAGMA journal_mode=WAL")
self._conn.executescript(_SCHEMA)
+ for table, column, kind in _ADDED_COLUMNS:
+ try:
+ self._conn.execute(f"ALTER TABLE {table} ADD COLUMN {column} {kind}")
+ except sqlite3.OperationalError:
+ pass # already there
self._conn.commit()
# Records waiting to be written, by id; the latest state of each replaces the last.
self._pending_turns: dict[str, dict[str, Any]] = {}
+ self._pending_issues: dict[str, tuple[str, int]] = {} # turn_id -> (json, seen)
self._pending_steps: dict[str, dict[str, Any]] = {}
self._pending_lock = threading.Lock()
self._closed = False
@@ -209,12 +224,17 @@ class RecordingStore:
with self._pending_lock:
turns, self._pending_turns = self._pending_turns, {}
steps, self._pending_steps = self._pending_steps, {}
- if not turns and not steps:
+ issues, self._pending_issues = self._pending_issues, {}
+ if not turns and not steps and not issues:
return
try:
with self._conn:
self._conn.executemany(_UPSERT_TURN, [_turn_row(t) for t in turns.values()])
self._conn.executemany(_UPSERT_STEP, [_step_row(s) for s in steps.values()])
+ self._conn.executemany(
+ "UPDATE turns SET issues = ?, issues_seen = ? WHERE turn_id = ?",
+ [(js, seen, tid) for tid, (js, seen) in issues.items()],
+ )
for s in steps.values():
refs = set(_piece_refs((s.get("content") or {}).get("exchange")))
if refs:
@@ -402,6 +422,53 @@ class RecordingStore:
break
time.sleep(SWEEP_PAUSE_S)
+ # -- the Issues result (phase-2 design §5): derived, stored beside the turn ---------------
+ def write_issues(self, turn_id: str, result: dict[str, Any], *, seen: bool = False) -> None:
+ """Store a turn's Issues result. `seen` is false only for a result worked out when the
+ turn ended: one worked out later, for an old turn, never lights the session dot."""
+ with self._pending_lock:
+ self._pending_issues[turn_id] = (json.dumps(result, ensure_ascii=False), int(seen))
+ if self._background:
+ _writer.wake()
+
+ def read_issues(self, session_id: str) -> dict[str, dict[str, Any]]:
+ """The stored results of a session's turns, by turn id, each with `seen`."""
+ self.flush()
+ with self._lock:
+ rows = self._conn.execute(
+ "SELECT turn_id, issues, issues_seen FROM turns WHERE session_id = ? AND issues IS NOT NULL",
+ (session_id,),
+ ).fetchall()
+ out = {}
+ for r in rows:
+ try:
+ out[r["turn_id"]] = {**json.loads(r["issues"]), "seen": bool(r["issues_seen"])}
+ except ValueError:
+ continue
+ return out
+
+ def mark_issues_seen(self, session_id: str) -> None:
+ self.flush()
+ with self._lock, self._conn:
+ self._conn.execute("UPDATE turns SET issues_seen = 1 WHERE session_id = ?", (session_id,))
+
+ def unseen_warnings(self) -> dict[str, int]:
+ """Warnings not seen yet, by session: the sidebar dot (TD2-4)."""
+ self.flush()
+ with self._lock:
+ rows = self._conn.execute(
+ "SELECT session_id, issues FROM turns WHERE issues_seen = 0 AND issues IS NOT NULL"
+ ).fetchall()
+ out: dict[str, int] = {}
+ for r in rows:
+ try:
+ warn = int(json.loads(r["issues"]).get("warn") or 0)
+ except (ValueError, TypeError):
+ continue
+ if warn:
+ out[r["session_id"]] = out.get(r["session_id"], 0) + warn
+ return out
+
# -- reading ------------------------------------------------------------------------------
def turn_count(self, session_id: str) -> int:
with self._lock:
@@ -482,6 +549,13 @@ def _turn_record(r: sqlite3.Row) -> dict[str, Any]:
d["attended"] = None if d["attended"] is None else bool(d["attended"])
d["reviewer"] = None if d["reviewer"] is None else json.loads(d["reviewer"])
d["stamp"] = json.loads(d["stamp"] or "{}")
+ # Format 1.2 fields are left out when not recorded, so a 1.1 turn reads back unchanged.
+ if d.get("max_iterations") is None:
+ d.pop("max_iterations", None)
+ if d.get("woken_by") is None:
+ d.pop("woken_by", None)
+ else:
+ d["woken_by"] = json.loads(d["woken_by"])
return d
diff --git a/coworker/server/app.py b/coworker/server/app.py
index 834fcb59..0d11f0af 100644
--- a/coworker/server/app.py
+++ b/coworker/server/app.py
@@ -843,6 +843,15 @@ def create_app(manager: SessionManager) -> FastAPI:
def session_usage(session_id: str) -> dict[str, Any]:
return manager.recording_usage(session_id)
+ # OB-3: a turn's Issues, and "I have looked" (clears the session's amber dot).
+ @app.get("/v1/sessions/{session_id}/turns/{turn_id}/issues")
+ def session_turn_issues(session_id: str, turn_id: str) -> dict[str, Any]:
+ return manager.recording_turn_issues(session_id, turn_id)
+
+ @app.post("/v1/sessions/{session_id}/issues/seen")
+ def session_issues_seen(session_id: str) -> dict[str, Any]:
+ return manager.mark_recording_issues_seen(session_id)
+
@app.patch("/v1/sessions/{session_id}")
def session_patch(session_id: str, body: dict) -> dict[str, Any]:
body = body or {}
diff --git a/coworker/server/manager.py b/coworker/server/manager.py
index 54be9087..8bba3f7d 100644
--- a/coworker/server/manager.py
+++ b/coworker/server/manager.py
@@ -45,6 +45,7 @@ from ..unattended import UnattendedRegistry
from ..audit import AuditStore
from ..observability import Recorder, RecordingStore
from ..observability import read_api as recording_api
+from ..observability import signals as recording_signals
from ..observability.provider_view import RecordingProvider, tag as tag_provider
from ..config import load_config, workspace_allowed_commands
from ..conversations import ConversationStore, title_from
@@ -6318,7 +6319,49 @@ class SessionManager:
LIVE_STEP_BATCH_S = 0.1
def recording_turns(self, session_id: str) -> dict[str, Any]:
- return {"turns": recording_api.turns(self.recordings_store.load_recording(session_id))}
+ rec = self.recordings_store.load_recording(session_id)
+ issues = self._recording_issues(session_id, rec)
+ turns = recording_api.turns(rec)
+ for t in turns:
+ r = issues.get(t["turn_id"]) or {}
+ t["issues"] = {"warn": int(r.get("warn") or 0), "info": int(r.get("info") or 0)}
+ return {"turns": turns}
+
+ def recording_turn_issues(self, session_id: str, turn_id: str) -> dict[str, Any]:
+ """The Issues tab of one turn (OB-3): its issues, the checks that were fine, the ones
+ not run here, and the signals.yaml version that made them."""
+ rec = self.recordings_store.load_recording(session_id)
+ found = self._recording_issues(session_id, rec).get(turn_id)
+ if found is None:
+ return {"turn_id": turn_id, "error": "no such turn"}
+ return {"turn_id": turn_id, **found}
+
+ def mark_recording_issues_seen(self, session_id: str) -> dict[str, Any]:
+ self.recordings_store.mark_issues_seen(session_id)
+ return {"ok": True}
+
+ def _recording_issues(self, session_id: str, rec: dict[str, Any]) -> dict[str, dict[str, Any]]:
+ """Each finished turn's stored Issues result. A turn with none (it ended before this
+ version), or one made by an older signals.yaml, is checked again now and stored; a
+ result worked out now for an old turn is stored as seen (TD2-2, TD2-4)."""
+ stored = self.recordings_store.read_issues(session_id)
+ current = recording_signals.version()
+ steps_of: dict[str, list[dict[str, Any]]] = {}
+ for s in rec.get("steps") or []:
+ steps_of.setdefault(s.get("turn_id"), []).append(s)
+ out: dict[str, dict[str, Any]] = {}
+ for t in rec.get("turns") or []:
+ if t.get("status") in (None, "running"):
+ continue
+ have = stored.get(t["turn_id"])
+ if have is not None and have.get("signals_version") == current:
+ out[t["turn_id"]] = have
+ continue
+ result = recording_signals.check_turn(t, steps_of.get(t["turn_id"], []))
+ seen = True if have is None else bool(have.get("seen"))
+ self.recordings_store.write_issues(t["turn_id"], result, seen=seen)
+ out[t["turn_id"]] = {**result, "seen": seen}
+ return out
def recording_turn_steps(self, session_id: str, turn_id: str) -> dict[str, Any]:
rec = self.recordings_store.load_recording(session_id)
@@ -8786,6 +8829,10 @@ class SessionManager:
# -- read models ------------------------------------------------------------
def list_sessions(self, workspace: Optional[str] = None) -> list[dict[str, Any]]:
ws = self.resolve_workspace(workspace) if workspace else None
+ try:
+ unseen_issues = self.recordings_store.unseen_warnings()
+ except Exception: # noqa: BLE001 - the dot never breaks the list
+ unseen_issues = {}
return [
{
"session_id": r.session_id,
@@ -8808,6 +8855,8 @@ class SessionManager:
# session → persona → footer Inbox). Liveness = working (in-flight turn) /
# sleeping (a self-wake is pending) / idle — a count-less dot that never bubbles.
"attention": len(self.inbox.pending(session_id=r.session_id)),
+ # OB-3: warnings in turns whose Issues you have not opened (the amber dot).
+ "issues_unseen": unseen_issues.get(r.session_id, 0),
"liveness": self._session_liveness(r.session_id),
# When sleeping: the next timer fire (ISO) — drives the "sleeping
# until…" strip so a scheduled agent never reads as a dead one.
diff --git a/pyproject.toml b/pyproject.toml
index d0cae266..19b9ccb3 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -72,6 +72,7 @@ coworker = [
"personas/builtin/*/manifest.md",
"personas/builtin/*/skills/*/SKILL.md",
"headless/*.yaml",
+ "observability/*.yaml",
"providers/*.json",
]
diff --git a/surfaces/gui/src/api.ts b/surfaces/gui/src/api.ts
index 314a35ce..9b9bc283 100644
--- a/surfaces/gui/src/api.ts
+++ b/surfaces/gui/src/api.ts
@@ -4405,6 +4405,29 @@ export interface RecordedTurn {
approval_mode?: string | null;
attended?: boolean | null;
reviewer?: { active?: boolean; shadow?: boolean; kind?: string | null; model?: string | null } | null;
+ // OB-3: how many issues the turn's checks found (warnings count on the tab).
+ issues?: { warn: number; info: number };
+}
+
+// One turn's Issues (OB-3): what the checks in signals.yaml found, what was fine, what did not
+// apply. `text` is the engine's English sentence; `facts` what it was built from.
+export interface TurnIssue {
+ check: string;
+ severity: "warn" | "info";
+ step_id: string | null;
+ step_seq: number | null;
+ text: string;
+ facts: Record<string, any>;
+}
+export interface TurnIssues {
+ turn_id: string;
+ signals_version?: string;
+ warn: number;
+ info: number;
+ seen?: boolean;
+ issues: TurnIssue[];
+ fine: { check: string; text: string }[];
+ not_checked: { check: string; text: string }[];
}
export interface RecordedStep {
@@ -4446,3 +4469,13 @@ export async function getStepDetail(sessionId: string, stepId: string): Promise<
export async function getRecordedUsage(sessionId: string): Promise<any> {
return recordingGet(sessionId, "/usage");
}
+
+export async function getTurnIssues(sessionId: string, turnId: string): Promise<TurnIssues | null> {
+ const r = await recordingGet(sessionId, `/turns/${encodeURIComponent(turnId)}/issues`);
+ return r && Array.isArray(r.issues) ? (r as TurnIssues) : null;
+}
+
+/** Opening a session's Issues clears its amber ring in the sidebar (TD2-4). */
+export async function markIssuesSeen(sessionId: string): Promise<void> {
+ await fetch(`${sessionApiBase(sessionId)}/v1/sessions/${encodeURIComponent(sessionId)}/issues/seen`, { method: "POST" });
+}
diff --git a/surfaces/gui/src/components/Sidebar.tsx b/surfaces/gui/src/components/Sidebar.tsx
index 7f349b7f..ce5703b7 100644
--- a/surfaces/gui/src/components/Sidebar.tsx
+++ b/surfaces/gui/src/components/Sidebar.tsx
@@ -49,6 +49,20 @@ const surfaceFromPersona = (p: Persona) => ({
// "Needs you": a session parked on a question or an approval. A dot, not a count — a
// session parks on one prompt at a time, so the number was always 1 (owner 2026-09-03).
// Amber, so it never reads as the accent "working" pulse beside it.
+// OB-3: warnings in Steps ▸ Issues not opened yet. A ring, so it never reads as the solid
+// "waiting for you" dot beside it.
+function IssueRing({ n }: { n: number }) {
+ const { t } = useTranslation();
+ if (!n) return null;
+ return (
+ <span
+ className="w-1.5 h-1.5 rounded-full border border-warnInk shrink-0"
+ title={t("steps.issues.unseen", { count: n })}
+ data-testid="issue-ring"
+ />
+ );
+}
+
function AttnBadge({ n }: { n: number }) {
const { t } = useTranslation();
if (!n) return null;
@@ -622,6 +636,7 @@ export function Sidebar(props: Props) {
)}
<OriginIcon s={s} />
<LiveDot state={s.liveness} />
+ <IssueRing n={s.issues_unseen || 0} />
<AttnBadge n={s.attention || 0} />
</span>
{rowActions(s, title)}
@@ -699,6 +714,7 @@ export function Sidebar(props: Props) {
<OriginIcon s={s} />
<ConnectorDot subs={s.subscriptions} />
<LiveDot state={s.liveness} />
+ <IssueRing n={s.issues_unseen || 0} />
<AttnBadge n={s.attention || 0} />
</span>
{rowActions(s, title)}
diff --git a/surfaces/gui/src/components/StepsPanel.test.tsx b/surfaces/gui/src/components/StepsPanel.test.tsx
index 4fd7ed68..e06ba488 100644
--- a/surfaces/gui/src/components/StepsPanel.test.tsx
+++ b/surfaces/gui/src/components/StepsPanel.test.tsx
@@ -1,7 +1,7 @@
// Observability Phase 1 (UX-059 OB-1, OB-2, OB-4): the Steps panel reads the recording.
import { cleanup, fireEvent, render, screen, waitFor } from "@testing-library/react";
import { afterEach, describe, expect, it, vi } from "vitest";
-import { getRecordedTurns, getRecordedUsage, getStepDetail, getTurnSteps } from "../api";
+import { getRecordedTurns, getRecordedUsage, getStepDetail, getTurnIssues, getTurnSteps, markIssuesSeen } from "../api";
import { StepsPanel, formatDuration } from "./StepsPanel";
vi.mock("../api", () => ({
@@ -9,6 +9,8 @@ vi.mock("../api", () => ({
getTurnSteps: vi.fn(),
getStepDetail: vi.fn(),
getRecordedUsage: vi.fn(),
+ getTurnIssues: vi.fn(() => Promise.resolve(null)),
+ markIssuesSeen: vi.fn(() => Promise.resolve()),
}));
const turn = {
@@ -276,3 +278,39 @@ describe("formatDuration", () => {
expect(formatDuration(null)).toBe("—");
});
});
+
+describe("Issues tab (OB-3)", () => {
+ const issues = {
+ turn_id: "t1", signals_version: "0.2", warn: 2, info: 1, seen: false,
+ issues: [
+ { check: "tool_failed", severity: "warn", step_id: "x1", step_seq: 5, text: "run_shell failed with exit code 6.", facts: { how: "exit" } },
+ { check: "tool_failed", severity: "warn", step_id: "x1", step_seq: 5, text: "run_shell failed with exit code 6 again.", facts: { how: "exit" } },
+ { check: "shadow_disagreed", severity: "info", step_id: "a1", step_seq: 3, text: "The shadow reviewer would have said allow.", facts: {} },
+ ],
+ fine: [{ check: "repeated_step", text: "No step repeated 3 times in a row" }],
+ not_checked: [{ check: "off_plan", text: "No plan to check against" }],
+ };
+
+ it("groups issues by check, puts warnings first, folds the fine list, and goes to the step", async () => {
+ vi.mocked(getRecordedTurns).mockResolvedValue([{ ...turn, issues: { warn: 2, info: 1 } }] as any);
+ vi.mocked(getTurnSteps).mockResolvedValue(steps as any);
+ vi.mocked(getTurnIssues).mockResolvedValue(issues as any);
+ vi.mocked(getStepDetail).mockResolvedValue(null);
+ render(<StepsPanel sessionId="s1" refreshKey={0} onClose={() => {}} />);
+ const tab = await screen.findByTestId("steps-tab-issues");
+ expect(tab.textContent).toBe("Issues · 2");
+ fireEvent.click(tab);
+ const groups = await screen.findAllByTestId("steps-issue");
+ expect(groups.map((g) => g.querySelector("b")?.textContent)).toEqual(["A tool failed · 2", "Shadow check disagreed with you"]);
+ expect(groups[1].className).toContain("info");
+ expect(markIssuesSeen).toHaveBeenCalledWith("s1");
+ expect(screen.queryByTestId("steps-fine")?.textContent ?? "").not.toContain("No step repeated");
+ fireEvent.click(screen.getByTestId("steps-fine-toggle"));
+ expect(screen.getByTestId("steps-fine").textContent).toContain("No step repeated 3 times in a row");
+ expect(screen.getByTestId("steps-fine").textContent).toContain("No plan to check against");
+ fireEvent.click(screen.getAllByText("Go to step")[0]);
+ await waitFor(() => expect(screen.getByTestId("steps-timeline")).toBeTruthy());
+ expect(screen.getAllByTestId("steps-row-issue").length).toBe(1);
+ });
+});
+
diff --git a/surfaces/gui/src/components/StepsPanel.tsx b/surfaces/gui/src/components/StepsPanel.tsx
index e116611e..cc1181f4 100644
--- a/surfaces/gui/src/components/StepsPanel.tsx
+++ b/surfaces/gui/src/components/StepsPanel.tsx
@@ -11,9 +11,13 @@ import {
getRecordedTurns,
getRecordedUsage,
getStepDetail,
+ getTurnIssues,
getTurnSteps,
+ markIssuesSeen,
type RecordedStep,
type RecordedTurn,
+ type TurnIssue,
+ type TurnIssues,
} from "../api";
import { formatTokens } from "../usage";
@@ -201,7 +205,8 @@ export function StepsPanel({
const [turns, setTurns] = useState<RecordedTurn[] | null>(null);
const [turnIdx, setTurnIdx] = useState<number | null>(null);
const [steps, setSteps] = useState<RecordedStep[]>([]);
- const [tab, setTab] = useState<"timeline" | "usage">("timeline");
+ const [tab, setTab] = useState<"timeline" | "usage" | "issues">("timeline");
+ const [issues, setIssues] = useState<TurnIssues | null>(null);
const [selected, setSelected] = useState<string | null>(null);
const [error, setError] = useState(false);
@@ -248,6 +253,22 @@ export function StepsPanel({
};
}, [sessionId, turn?.turn_id, refreshKey]);
+ // OB-3: the turn's Issues, for the tab and the row marks; a turn still running has none.
+ useEffect(() => {
+ setIssues(null);
+ if (!turn || turn.status === "running") return;
+ let live = true;
+ getTurnIssues(sessionId, turn.turn_id).then((x) => live && setIssues(x)).catch(() => {});
+ return () => {
+ live = false;
+ };
+ }, [sessionId, turn?.turn_id, turn?.status, refreshKey]);
+ // Opening Issues is "I have looked": the session's ring in the sidebar clears.
+ useEffect(() => {
+ if (tab === "issues") markIssuesSeen(sessionId).catch(() => {});
+ }, [tab, sessionId, turn?.turn_id]);
+ const flagged = useMemo(() => new Set((issues?.issues || []).filter((i) => i.severity === "warn").map((i) => i.step_id)), [issues]);
+
const rows = useMemo(() => nest(steps), [steps]);
const checks = steps.filter((s) => s.kind === "auto_approver" && !s.shadow);
const inspecting = tab === "timeline" && !!selected && !!turn;
@@ -277,6 +298,7 @@ export function StepsPanel({
total={total}
label={stepLabel(step, t)}
selected={selected === step.step_id}
+ issue={flagged.has(step.step_id)}
onSelect={() => setSelected(selected === step.step_id ? null : step.step_id)}
/>
))}
@@ -332,6 +354,9 @@ export function StepsPanel({
<div className="steps-tabs" role="tablist">
<button role="tab" aria-selected={tab === "timeline"} onClick={() => setTab("timeline")}>{t("steps.tab.timeline")}</button>
<button role="tab" aria-selected={tab === "usage"} onClick={() => setTab("usage")}>{t("steps.tab.usage")}</button>
+ <button role="tab" aria-selected={tab === "issues"} onClick={() => setTab("issues")} data-testid="steps-tab-issues">
+ {(turn.issues?.warn ?? issues?.warn) ? t("steps.issues.tab_n", { n: turn.issues?.warn ?? issues?.warn }) : t("steps.issues.tab")}
+ </button>
</div>
)}
</div>
@@ -353,6 +378,20 @@ export function StepsPanel({
</div>
)}
{turn && tab === "usage" && <div className="steps-railbody"><UsageView sessionId={sessionId} refreshKey={refreshKey} /></div>}
+ {turn && tab === "issues" && (
+ <div className="steps-railbody">
+ <IssuesView
+ sessionId={sessionId}
+ turn={turn}
+ issues={issues}
+ steps={steps}
+ onGo={(stepId) => {
+ setTab("timeline");
+ setSelected(stepId);
+ }}
+ />
+ </div>
+ )}
</div>
);
}
@@ -366,8 +405,9 @@ function Tile({ value, label }: { value: string; label: string }) {
);
}
-function StepRow({ step, depth, turn, total, label, selected, onSelect }: {
- step: RecordedStep; depth: number; turn: RecordedTurn; total: number; label: string; selected: boolean; onSelect: () => void;
+function StepRow({ step, depth, turn, total, label, selected, issue, onSelect }: {
+ step: RecordedStep; depth: number; turn: RecordedTurn; total: number; label: string; selected: boolean;
+ issue?: boolean; onSelect: () => void;
}) {
const { t } = useTranslation();
const left = Math.min(100, Math.max(0, ((step.started_at - turn.started_at) / total) * 100));
@@ -385,6 +425,7 @@ function StepRow({ step, depth, turn, total, label, selected, onSelect }: {
{stepFlags(step, t).map((f) => (
<span key={f.text} className={`steps-badge ${f.cls}`}>{f.text}</span>
))}
+ {issue && <span className="steps-badge warn" data-testid="steps-row-issue">{t("steps.issues.badge")}</span>}
</span>
<span className="steps-track">
<span className="steps-bar" style={{ left: `${left}%`, width: width ? `${width}%` : undefined }} />
@@ -394,6 +435,100 @@ function StepRow({ step, depth, turn, total, label, selected, onSelect }: {
);
}
+// -- Issues (OB-3) --------------------------------------------------------------------------------
+
+/** One title per check, and per kind for the context check (mock review M2-R4). */
+function issueTitle(i: { check: string; facts?: Record<string, any> }, t: T): string {
+ const kind = i.check === "context_full" ? i.facts?.kind : null;
+ return kind ? t(`steps.issues.title.context_full_${kind}`, { defaultValue: t("steps.issues.title.context_full") })
+ : t(`steps.issues.title.${i.check}`, { defaultValue: i.check });
+}
+
+/** What the app adds from the step's own content, shown here only (design §6): the command a
+ * failed tool ran and the first line of its output; the call a shadow check judged. */
+function IssueDetail({ sessionId, issue }: { sessionId: string; issue: TurnIssue }) {
+ const [d, setD] = useState<any>(null);
+ useEffect(() => {
+ if (!issue.step_id || !["tool_failed", "shadow_disagreed", "refused", "reviewer_allowed_send", "blocked_by_sandbox"].includes(issue.check)) return;
+ let live = true;
+ getStepDetail(sessionId, issue.step_id).then((x) => live && setD(x)).catch(() => {});
+ return () => {
+ live = false;
+ };
+ }, [sessionId, issue.step_id, issue.check]);
+ if (!d) return null;
+ const args = d.kind === "auto_approver" ? d.for_step?.arguments : d.arguments;
+ const main = args && typeof args === "object" ? (args.command ?? args.path ?? args.url ?? args.query) : null;
+ let first = "";
+ if (issue.check === "tool_failed" && typeof d.message?.content === "string") {
+ try {
+ const r = JSON.parse(d.message.content);
+ first = String(r?.output ?? r?.error ?? "").split("\n").map((l: string) => l.trim()).find(Boolean) || "";
+ } catch {
+ first = "";
+ }
+ }
+ return (
+ <>
+ {main != null && <span className="steps-mono steps-issue-arg"> {String(main).slice(0, 120)}</span>}
+ {first && <span className="steps-issue-out"> {first.slice(0, 140)}</span>}
+ </>
+ );
+}
+
+function IssuesView({ sessionId, turn, issues, steps, onGo }: {
+ sessionId: string; turn: RecordedTurn; issues: TurnIssues | null; steps: RecordedStep[]; onGo: (stepId: string) => void;
+}) {
+ const { t } = useTranslation();
+ const [openFine, setOpenFine] = useState(false);
+ if (turn.status === "running") return <p className="steps-empty">{t("steps.issues.running")}</p>;
+ if (!issues) return <p className="steps-empty">{t("steps.loading")}</p>;
+ // Grouped by check (V2-17), warnings first; context_full by kind (M2-R4).
+ const groups: { key: string; items: TurnIssue[] }[] = [];
+ for (const i of [...issues.issues].sort((a, b) => (a.severity === b.severity ? 0 : a.severity === "warn" ? -1 : 1))) {
+ const key = i.check === "context_full" ? `${i.check}:${i.facts?.kind}` : i.check;
+ const g = groups.find((x) => x.key === key);
+ if (g) g.items.push(i);
+ else groups.push({ key, items: [i] });
+ }
+ const noPlan = issues.not_checked.find((n) => n.check === "off_plan");
+ return (
+ <div className="steps-issues" data-testid="steps-issues">
+ {groups.length === 0 && <p className="steps-issues-none">{t("steps.issues.none")}</p>}
+ {groups.map((g) => {
+ const first = g.items[0];
+ const info = first.severity === "info";
+ return (
+ <div key={g.key} className={"steps-callout " + (info ? "info" : "warn")} data-testid="steps-issue">
+ <b>{issueTitle(first, t)}{g.items.length > 1 ? ` · ${g.items.length}` : ""}</b>
+ {info && <span className="steps-pill steps-info-pill">{t("steps.issues.information")}</span>}
+ {g.items.map((i, n) => (
+ <div key={(i.step_id || "") + n} className="steps-issue-line">
+ {i.step_seq != null && <span className="steps-faint">{t("steps.insp.step", { n: i.step_seq })} </span>}
+ <span>{i.text}</span>
+ <IssueDetail sessionId={sessionId} issue={i} />
+ {i.step_id && steps.some((s) => s.step_id === i.step_id) && (
+ <button className="steps-link" onClick={() => onGo(i.step_id!)}>{t("steps.issues.go")}</button>
+ )}
+ </div>
+ ))}
+ </div>
+ );
+ })}
+ <button className="steps-fine-toggle" onClick={() => setOpenFine(!openFine)} data-testid="steps-fine-toggle">
+ {t("steps.issues.fine", { n: issues.fine.length })} {openFine ? "▾" : "▸"}
+ </button>
+ {openFine && (
+ <div className="steps-fine" data-testid="steps-fine">
+ {issues.fine.map((f) => <div key={f.check}>✓ {f.text}</div>)}
+ {noPlan && <div className="steps-faint">– {t("steps.issues.no_plan")}</div>}
+ </div>
+ )}
+ {!openFine && noPlan && <div className="steps-faint steps-fine">– {t("steps.issues.no_plan")}</div>}
+ </div>
+ );
+}
+
// -- details (the inspector of the mock) ---------------------------------------------------------
/** The messages a model call was given, as "role · text" lines (Chat, Responses or Gemini shape). */
diff --git a/surfaces/gui/src/locales/en.json b/surfaces/gui/src/locales/en.json
index 3fde17f6..17d9d2db 100644
--- a/surfaces/gui/src/locales/en.json
+++ b/surfaces/gui/src/locales/en.json
@@ -3036,6 +3036,44 @@
}
},
"steps": {
+ "issues": {
+ "tab": "Issues",
+ "tab_n": "Issues · {{n}}",
+ "badge": "issue",
+ "go": "Go to step",
+ "none": "No issues in this turn.",
+ "running": "The checks run when the turn ends.",
+ "information": "information",
+ "fine": "Checked and fine · {{n}}",
+ "no_plan": "No plan to check against",
+ "unseen_one": "{{count}} issue in Steps you have not looked at",
+ "unseen_other": "{{count}} issues in Steps you have not looked at",
+ "title": {
+ "repeated_step": "Repeated the same step",
+ "off_plan": "Left its plan",
+ "refused": "An action was refused",
+ "overridden": "You allowed a blocked action",
+ "reviewer_could_not_run": "Auto-approver could not run",
+ "reviewer_paused": "Auto-approver paused",
+ "reviewer_allowed_send": "The auto-approver let data leave the machine",
+ "shadow_disagreed": "Shadow check disagreed with you",
+ "tool_failed": "A tool failed",
+ "model_failed": "A model call failed",
+ "retried": "A model call was retried",
+ "cut_off": "An answer was cut off",
+ "step_limit": "Hit the step limit",
+ "made_up_tool": "Unknown tool or bad arguments",
+ "context_full": "Context was full",
+ "context_full_overflow": "Context overflowed",
+ "context_full_trim": "Older messages were cut",
+ "context_full_summary_failed": "Summary failed",
+ "context_full_hard_fill": "Context nearly full",
+ "blocked_by_sandbox": "Blocked by the sandbox",
+ "sandbox_failed": "The sandbox could not start",
+ "ran_without_sandbox": "Ran without the sandbox",
+ "waited_unwatched": "Waited while nobody was watching"
+ }
+ },
"title": "Steps",
"open": "Steps",
"open_title": "Every step of each turn, and where the time and tokens went",
diff --git a/surfaces/gui/src/locales/zh.json b/surfaces/gui/src/locales/zh.json
index e9ea1241..84111f1c 100644
--- a/surfaces/gui/src/locales/zh.json
+++ b/surfaces/gui/src/locales/zh.json
@@ -2992,6 +2992,43 @@
}
},
"steps": {
+ "issues": {
+ "tab": "问题",
+ "tab_n": "问题 · {{n}}",
+ "badge": "问题",
+ "go": "转到该步骤",
+ "none": "本轮没有问题。",
+ "running": "检查在本轮结束时运行。",
+ "information": "信息",
+ "fine": "已检查,正常 · {{n}}",
+ "no_plan": "没有可对照的计划",
+ "unseen_other": "Steps 中有 {{count}} 个你还没看的问题",
+ "title": {
+ "repeated_step": "重复了同一步骤",
+ "off_plan": "偏离了计划",
+ "refused": "有操作被拒绝",
+ "overridden": "你允许了被拦截的操作",
+ "reviewer_could_not_run": "自动批准无法运行",
+ "reviewer_paused": "自动批准已暂停",
+ "reviewer_allowed_send": "自动批准让数据离开了本机",
+ "shadow_disagreed": "影子检查与你意见不同",
+ "tool_failed": "工具失败",
+ "model_failed": "模型调用失败",
+ "retried": "模型调用重试了",
+ "cut_off": "回答被截断",
+ "step_limit": "达到步骤上限",
+ "made_up_tool": "未知工具或参数错误",
+ "context_full": "上下文已满",
+ "context_full_overflow": "上下文溢出",
+ "context_full_trim": "较早的消息被删减",
+ "context_full_summary_failed": "摘要失败",
+ "context_full_hard_fill": "上下文接近满",
+ "blocked_by_sandbox": "被沙盒拦截",
+ "sandbox_failed": "沙盒无法启动",
+ "ran_without_sandbox": "未在沙盒中运行",
+ "waited_unwatched": "无人值守时等待"
+ }
+ },
"title": "步骤",
"open": "步骤",
"open_title": "每一轮的每个步骤,以及时间和令牌用在了哪里",
diff --git a/surfaces/gui/src/styles.css b/surfaces/gui/src/styles.css
index 4fcb9cb6..7eb407b5 100644
--- a/surfaces/gui/src/styles.css
+++ b/surfaces/gui/src/styles.css
@@ -2566,3 +2566,20 @@ html[data-theme="dark"] { --bar-model:#4c8dff; --bar-tool:#3fb8ad; --bar-wait:#3
.steps-kv { grid-template-columns:minmax(0,1fr); gap:0; }
.steps-kv dd { margin-bottom:4px; }
}
+
+/* OB-3 Issues tab (UX-059 ob3, Phase 2 mock review): grouped callouts, folded "Checked and fine". */
+.steps-issues{padding:10px 14px;display:flex;flex-direction:column;gap:8px}
+.steps-callout{border-radius:8px;padding:8px 10px;font-size:12px;line-height:1.5}
+.steps-callout.warn{background:var(--warn-soft);color:var(--warn-ink)}
+.steps-callout.info{background:var(--accent-soft);color:var(--accent)}
+.steps-callout b{font-weight:600}
+.steps-info-pill{margin-left:6px;font-size:10px;padding:0 5px;background:var(--paper);color:var(--muted)}
+.steps-issue-line{margin-top:4px;overflow-wrap:anywhere}
+.steps-issue-arg{font-size:11px}
+.steps-issue-out{font-style:italic}
+.steps-link{background:none;border:0;padding:0 0 0 6px;color:var(--accent);cursor:pointer;font:inherit}
+.steps-link:hover{text-decoration:underline}
+.steps-issues-none{margin:4px 0;color:var(--muted);font-size:12px}
+.steps-fine-toggle{align-self:flex-start;background:none;border:0;padding:4px 0;color:var(--faint);font-size:12px;font-weight:600;cursor:pointer}
+.steps-fine{font-size:12px;line-height:1.85;color:var(--muted)}
+
diff --git a/surfaces/gui/src/types.ts b/surfaces/gui/src/types.ts
index f44c9c03..bbfc593c 100644
--- a/surfaces/gui/src/types.ts
+++ b/surfaces/gui/src/types.ts
@@ -111,6 +111,8 @@ export interface SessionInfo {
attention?: number;
// working = in-flight turn; sleeping = a self-wake is pending; idle = neither. A count-less dot.
liveness?: "working" | "sleeping" | "idle";
+ // OB-3: warnings in turns whose Issues you have not opened (an amber ring in the sidebar).
+ issues_unseen?: number;
// When sleeping: the next timer fire (ISO) — drives the "sleeping until…" strip.
sleeping_until?: string | null;
// Channels this session listens to (inbound subscriptions).
diff --git a/tests/corpora/observability/README.md b/tests/corpora/observability/README.md
index ddf679fa..b926b6f5 100644
--- a/tests/corpora/observability/README.md
+++ b/tests/corpora/observability/README.md
@@ -12,3 +12,11 @@ Do not edit these files here. A change to the format needs a new version in `ocw
recorder and compares the result with `expected-recording.json`, and computes the screen
numbers (`coworker/observability/calculator.py`) and compares them with
`expected-numbers.json`.
+
+`golden-phase2/` is a copy of `docs/observability/golden/phase2/` in `ocw-context`: the shared
+test of the Issues checks (OB-3), with `signals.yaml` 0.3 and recording format 1.2. Each
+`recording.json` must give exactly its `expected-issues.json` through
+`coworker/observability/signals.py`, and `vectors/hashes.json` pins the argument hash (RFC 8785).
+`tests/test_observability_issues.py` runs it. `coworker/observability/signals.yaml` is a copy of
+the shared `docs/observability/signals.yaml`, and `recording-format.schema.json` here is the 1.2
+schema; the same rule applies to all three: change them in `ocw-context`, then copy them here.
diff --git a/tests/corpora/observability/golden-phase2/README.md b/tests/corpora/observability/golden-phase2/README.md
new file mode 100644
index 00000000..499de2fc
--- /dev/null
+++ b/tests/corpora/observability/golden-phase2/README.md
@@ -0,0 +1,30 @@
+# Golden test, Phase 2
+
+*Written 2026-11-02 with [recording-format.md](../../recording-format.md) 1.2 and [signals.yaml](../../signals.yaml) 0.3. Made by `phase-2/lab/make_golden2.py`, checked by `phase-2/lab/check_golden2.py`.*
+
+## What it is
+
+**Context.** Both engines must find the same issues in the same turn, and must turn the same arguments into the same hash. This folder pins both. Feed each `recording.json` to an engine's Issues checks, and it must give exactly `expected-issues.json`. Feed each vector to its hashing or masking code, and it must give exactly the expected bytes.
+
+| File | What it is |
+|---|---|
+| `<id>/recording.json` | One turn as a format 1.2 recording, `{"turns": [...], "steps": [...]}`. It is valid against [recording-format.schema.json](../../recording-format.schema.json) |
+| `<id>/expected-issues.json` | What the checks must give: `fired` (check, and the `seq` of each step it points at), `fine`, `not_checked`, the warning count, and the `signals.yaml` version |
+| `index.json` | The scenario list: which check each one tests, and whether it should fire, stay quiet or not run |
+| `vectors/hashes.json` | `args_hash` cases, each with its value, its RFC 8785 canonical JSON and its 6-hex hash; plan item hashes |
+| `vectors/masking.json` | Rule R4 cases: secrets masked (patterns and stored values), and the 8 KB cut on a character boundary |
+
+There are **38 scenarios**, S2-01 to S2-19 with their clean twins. Each is described in words in [phase-2/scenarios.md](../../phase-2/scenarios.md).
+
+## Built, not captured
+
+Unlike `golden/phase1/`, these recordings were **built** from the scenarios by hand rules (`phase-2/lab/synthetic.py`), not captured from a scripted run of the engine. The engine does not record the 1.2 fields yet. Once it does (engine change E8a), each scenario is run through the engine in scripted mode and its recording replaces the built one. Its expected issues must not change. If they do, either the engine or the scenario is wrong, and the difference is a finding.
+
+## The byte-exact rules
+
+- **`args_hash`** is the first 6 hex characters of SHA-256 over the arguments' canonical JSON, encoded as UTF-8. Canonical JSON is **RFC 8785 (JCS)**: object keys sorted by their UTF-16 code units, no whitespace, strings with only the required escapes, numbers in the ECMAScript form. Neither Python's `json.dumps` nor Rust's `serde_json` is the rule: they differ on numbers (`1e-07` against `1e-7`, `100.0` against `100`). The `numbers` vector catches that.
+- **A plan item's `h`** is the first 6 hex characters of SHA-256 over the item's own text, encoded as UTF-8. It is not JSON.
+- **Masking and cutting** (export rule R4) work in this order:
+ 1. Mask stored secret values of 8 characters or more, longest first.
+ 2. Mask the patterns in `attributes.yaml` (`export.content_switches.masking`).
+ 3. Cut to 8,192 bytes of UTF-8 without splitting a character, then add `[cut: N more bytes]`.
diff --git a/tests/corpora/observability/golden-phase2/S2-01/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-01/expected-issues.json
new file mode 100644
index 00000000..1969fdfc
--- /dev/null
+++ b/tests/corpora/observability/golden-phase2/S2-01/expected-issues.json
@@ -0,0 +1,31 @@
+{
+ "signals_version": "0.3",
+ "warn": 1,
+ "fired": {
+ "repeated_step": [
+ 6
+ ]
+ },
+ "fine": [
+ "context_full",
+ "cut_off",
+ "made_up_tool",
+ "model_failed",
+ "refused",
+ "retried",
+ "step_limit",
+ "tool_failed",
+ "waited_unwatched"
+ ],
+ "not_checked": [
+ "blocked_by_sandbox",
+ "off_plan",
+ "overridden",
+ "ran_without_sandbox",
+ "reviewer_allowed_send",
+ "reviewer_could_not_run",
+ "reviewer_paused",
+ "sandbox_failed",
+ "shadow_disagreed"
+ ]
+}
diff --git a/tests/corpora/observability/golden-phase2/S2-01/recording.json b/tests/corpora/observability/golden-phase2/S2-01/recording.json
new file mode 100644
index 00000000..07f21868
--- /dev/null
+++ b/tests/corpora/observability/golden-phase2/S2-01/recording.json
@@ -0,0 +1,289 @@
+{
+ "turns": [
+ {
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-01",
+ "seq": 1,
+ "status": "completed",
+ "approval_mode": "interactive",
+ "attended": true,
+ "reviewer": {
+ "active": false,
+ "shadow": false,
+ "kind": null,
+ "model": null
+ },
+ "stamp": {
+ "sandbox": {
+ "provider": "direct",
+ "enforcement": "none"
+ }
+ },
+ "max_iterations": 50,
+ "kind": "run",
+ "trigger": "user",
+ "started_at": 1793600000000,
+ "incomplete": false,
+ "first_message_id": "01JP2MSG000000000000000000",
+ "last_message_id": "01JP2MSG000000000000000000",
+ "schema_version": 1,
+ "duration_ms": 77
+ }
+ ],
+ "steps": [
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-01",
+ "parent_step_id": null,
+ "seq": 1,
+ "kind": "model",
+ "name": "model",
+ "started_at": 1793600000000,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": null,
+ "call_index": null,
+ "message_id": "01JP2MSG000000000000000010",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "purpose": "main",
+ "provider": "testprov",
+ "provider_title": "Test provider",
+ "model": "testprov:model-a",
+ "stream": true,
+ "finish_reason": "stop",
+ "ttft_ms": 5,
+ "context_window": 100000,
+ "usage": {
+ "reported": true,
+ "input_fresh": 1000,
+ "cache_read": 0,
+ "cache_write": null,
+ "output": 100,
+ "reasoning": null
+ },
+ "tools_offered": 3,
+ "max_output": 4096
+ },
+ "content": {}
+ },
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-01",
+ "parent_step_id": null,
+ "seq": 2,
+ "kind": "tool",
+ "name": "read_file",
+ "started_at": 1793600000011,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": "call_2",
+ "call_index": 0,
+ "message_id": "01JP2MSG000000000000000020",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "risk_class": "read",
+ "decision": {
+ "path": [],
+ "origin": "low_risk"
+ },
+ "args_hash": "327e09",
+ "repeat_ok": false,
+ "result_error": false,
+ "error_kind": null,
+ "connector": null,
+ "result_bytes": 120,
+ "exit_code": null,
+ "sandbox": null,
+ "reaches_network": false
+ },
+ "content": {}
+ },
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-01",
+ "parent_step_id": null,
+ "seq": 3,
+ "kind": "model",
+ "name": "model",
+ "started_at": 1793600000022,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": null,
+ "call_index": null,
+ "message_id": "01JP2MSG000000000000000010",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "purpose": "main",
+ "provider": "testprov",
+ "provider_title": "Test provider",
+ "model": "testprov:model-a",
+ "stream": true,
+ "finish_reason": "stop",
+ "ttft_ms": 5,
+ "context_window": 100000,
+ "usage": {
+ "reported": true,
+ "input_fresh": 1000,
+ "cache_read": 0,
+ "cache_write": null,
+ "output": 100,
+ "reasoning": null
+ },
+ "tools_offered": 3,
+ "max_output": 4096
+ },
+ "content": {}
+ },
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-01",
+ "parent_step_id": null,
+ "seq": 4,
+ "kind": "tool",
+ "name": "read_file",
+ "started_at": 1793600000033,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": "call_4",
+ "call_index": 0,
+ "message_id": "01JP2MSG000000000000000020",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "risk_class": "read",
+ "decision": {
+ "path": [],
+ "origin": "low_risk"
+ },
+ "args_hash": "327e09",
+ "repeat_ok": false,
+ "result_error": false,
+ "error_kind": null,
+ "connector": null,
+ "result_bytes": 120,
+ "exit_code": null,
+ "sandbox": null,
+ "reaches_network": false
+ },
+ "content": {}
+ },
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-01",
+ "parent_step_id": null,
+ "seq": 5,
+ "kind": "model",
+ "name": "model",
+ "started_at": 1793600000044,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": null,
+ "call_index": null,
+ "message_id": "01JP2MSG000000000000000010",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "purpose": "main",
+ "provider": "testprov",
+ "provider_title": "Test provider",
+ "model": "testprov:model-a",
+ "stream": true,
+ "finish_reason": "stop",
+ "ttft_ms": 5,
+ "context_window": 100000,
+ "usage": {
+ "reported": true,
+ "input_fresh": 1000,
+ "cache_read": 0,
+ "cache_write": null,
+ "output": 100,
+ "reasoning": null
+ },
+ "tools_offered": 3,
+ "max_output": 4096
+ },
+ "content": {}
+ },
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-01",
+ "parent_step_id": null,
+ "seq": 6,
+ "kind": "tool",
+ "name": "read_file",
+ "started_at": 1793600000055,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": "call_6",
+ "call_index": 0,
+ "message_id": "01JP2MSG000000000000000020",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "risk_class": "read",
+ "decision": {
+ "path": [],
+ "origin": "low_risk"
+ },
+ "args_hash": "327e09",
+ "repeat_ok": false,
+ "result_error": false,
+ "error_kind": null,
+ "connector": null,
+ "result_bytes": 120,
+ "exit_code": null,
+ "sandbox": null,
+ "reaches_network": false
+ },
+ "content": {}
+ },
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-01",
+ "parent_step_id": null,
+ "seq": 7,
+ "kind": "model",
+ "name": "model",
+ "started_at": 1793600000066,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": null,
+ "call_index": null,
+ "message_id": "01JP2MSG000000000000000010",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "purpose": "main",
+ "provider": "testprov",
+ "provider_title": "Test provider",
+ "model": "testprov:model-a",
+ "stream": true,
+ "finish_reason": "stop",
+ "ttft_ms": 5,
+ "context_window": 100000,
+ "usage": {
+ "reported": true,
+ "input_fresh": 1000,
+ "cache_read": 0,
+ "cache_write": null,
+ "output": 100,
+ "reasoning": null
+ },
+ "tools_offered": 3,
+ "max_output": 4096
+ },
+ "content": {}
+ }
+ ]
+}
diff --git a/tests/corpora/observability/golden-phase2/S2-01c/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-01c/expected-issues.json
new file mode 100644
index 00000000..49ff0804
--- /dev/null
+++ b/tests/corpora/observability/golden-phase2/S2-01c/expected-issues.json
@@ -0,0 +1,28 @@
+{
+ "signals_version": "0.3",
+ "warn": 0,
+ "fired": {},
+ "fine": [
+ "context_full",
+ "cut_off",
+ "made_up_tool",
+ "model_failed",
+ "refused",
+ "repeated_step",
+ "retried",
+ "step_limit",
+ "tool_failed",
+ "waited_unwatched"
+ ],
+ "not_checked": [
+ "blocked_by_sandbox",
+ "off_plan",
+ "overridden",
+ "ran_without_sandbox",
+ "reviewer_allowed_send",
+ "reviewer_could_not_run",
+ "reviewer_paused",
+ "sandbox_failed",
+ "shadow_disagreed"
+ ]
+}
diff --git a/tests/corpora/observability/golden-phase2/S2-01c/recording.json b/tests/corpora/observability/golden-phase2/S2-01c/recording.json
new file mode 100644
index 00000000..e9ee6425
--- /dev/null
+++ b/tests/corpora/observability/golden-phase2/S2-01c/recording.json
@@ -0,0 +1,429 @@
+{
+ "turns": [
+ {
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-01c",
+ "seq": 1,
+ "status": "completed",
+ "approval_mode": "interactive",
+ "attended": true,
+ "reviewer": {
+ "active": false,
+ "shadow": false,
+ "kind": null,
+ "model": null
+ },
+ "stamp": {
+ "sandbox": {
+ "provider": "direct",
+ "enforcement": "none"
+ }
+ },
+ "max_iterations": 50,
+ "kind": "run",
+ "trigger": "user",
+ "started_at": 1793600000000,
+ "incomplete": false,
+ "first_message_id": "01JP2MSG000000000000000000",
+ "last_message_id": "01JP2MSG000000000000000000",
+ "schema_version": 1,
+ "duration_ms": 121
+ }
+ ],
+ "steps": [
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-01c",
+ "parent_step_id": null,
+ "seq": 1,
+ "kind": "model",
+ "name": "model",
+ "started_at": 1793600000000,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": null,
+ "call_index": null,
+ "message_id": "01JP2MSG000000000000000010",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "purpose": "main",
+ "provider": "testprov",
+ "provider_title": "Test provider",
+ "model": "testprov:model-a",
+ "stream": true,
+ "finish_reason": "stop",
+ "ttft_ms": 5,
+ "context_window": 100000,
+ "usage": {
+ "reported": true,
+ "input_fresh": 1000,
+ "cache_read": 0,
+ "cache_write": null,
+ "output": 100,
+ "reasoning": null
+ },
+ "tools_offered": 3,
+ "max_output": 4096
+ },
+ "content": {}
+ },
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-01c",
+ "parent_step_id": null,
+ "seq": 2,
+ "kind": "tool",
+ "name": "read_file",
+ "started_at": 1793600000011,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": "call_2",
+ "call_index": 0,
+ "message_id": "01JP2MSG000000000000000020",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "risk_class": "read",
+ "decision": {
+ "path": [],
+ "origin": "low_risk"
+ },
+ "args_hash": "327e09",
+ "repeat_ok": false,
+ "result_error": false,
+ "error_kind": null,
+ "connector": null,
+ "result_bytes": 120,
+ "exit_code": null,
+ "sandbox": null,
+ "reaches_network": false
+ },
+ "content": {}
+ },
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-01c",
+ "parent_step_id": null,
+ "seq": 3,
+ "kind": "model",
+ "name": "model",
+ "started_at": 1793600000022,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": null,
+ "call_index": null,
+ "message_id": "01JP2MSG000000000000000010",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "purpose": "main",
+ "provider": "testprov",
+ "provider_title": "Test provider",
+ "model": "testprov:model-a",
+ "stream": true,
+ "finish_reason": "stop",
+ "ttft_ms": 5,
+ "context_window": 100000,
+ "usage": {
+ "reported": true,
+ "input_fresh": 1000,
+ "cache_read": 0,
+ "cache_write": null,
+ "output": 100,
+ "reasoning": null
+ },
+ "tools_offered": 3,
+ "max_output": 4096
+ },
+ "content": {}
+ },
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-01c",
+ "parent_step_id": null,
+ "seq": 4,
+ "kind": "tool",
+ "name": "read_file",
+ "started_at": 1793600000033,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": "call_4",
+ "call_index": 0,
+ "message_id": "01JP2MSG000000000000000020",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "risk_class": "read",
+ "decision": {
+ "path": [],
+ "origin": "low_risk"
+ },
+ "args_hash": "327e09",
+ "repeat_ok": false,
+ "result_error": false,
+ "error_kind": null,
+ "connector": null,
+ "result_bytes": 120,
+ "exit_code": null,
+ "sandbox": null,
+ "reaches_network": false
+ },
+ "content": {}
+ },
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-01c",
+ "parent_step_id": null,
+ "seq": 5,
+ "kind": "model",
+ "name": "model",
+ "started_at": 1793600000044,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": null,
+ "call_index": null,
+ "message_id": "01JP2MSG000000000000000010",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "purpose": "main",
+ "provider": "testprov",
+ "provider_title": "Test provider",
+ "model": "testprov:model-a",
+ "stream": true,
+ "finish_reason": "stop",
+ "ttft_ms": 5,
+ "context_window": 100000,
+ "usage": {
+ "reported": true,
+ "input_fresh": 1000,
+ "cache_read": 0,
+ "cache_write": null,
+ "output": 100,
+ "reasoning": null
+ },
+ "tools_offered": 3,
+ "max_output": 4096
+ },
+ "content": {}
+ },
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-01c",
+ "parent_step_id": null,
+ "seq": 6,
+ "kind": "tool",
+ "name": "read_file",
+ "started_at": 1793600000055,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": "call_6",
+ "call_index": 0,
+ "message_id": "01JP2MSG000000000000000020",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "risk_class": "read",
+ "decision": {
+ "path": [],
+ "origin": "low_risk"
+ },
+ "args_hash": "920253",
+ "repeat_ok": false,
+ "result_error": false,
+ "error_kind": null,
+ "connector": null,
+ "result_bytes": 120,
+ "exit_code": null,
+ "sandbox": null,
+ "reaches_network": false
+ },
+ "content": {}
+ },
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-01c",
+ "parent_step_id": null,
+ "seq": 7,
+ "kind": "model",
+ "name": "model",
+ "started_at": 1793600000066,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": null,
+ "call_index": null,
+ "message_id": "01JP2MSG000000000000000010",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "purpose": "main",
+ "provider": "testprov",
+ "provider_title": "Test provider",
+ "model": "testprov:model-a",
+ "stream": true,
+ "finish_reason": "stop",
+ "ttft_ms": 5,
+ "context_window": 100000,
+ "usage": {
+ "reported": true,
+ "input_fresh": 1000,
+ "cache_read": 0,
+ "cache_write": null,
+ "output": 100,
+ "reasoning": null
+ },
+ "tools_offered": 3,
+ "max_output": 4096
+ },
+ "content": {}
+ },
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-01c",
+ "parent_step_id": null,
+ "seq": 8,
+ "kind": "tool",
+ "name": "check_job",
+ "started_at": 1793600000077,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": "call_8",
+ "call_index": 0,
+ "message_id": "01JP2MSG000000000000000020",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "risk_class": "read",
+ "decision": {
+ "path": [],
+ "origin": "low_risk"
+ },
+ "args_hash": "327e09",
+ "repeat_ok": true,
+ "result_error": false,
+ "error_kind": null,
+ "connector": null,
+ "result_bytes": 120,
+ "exit_code": null,
+ "sandbox": null,
+ "reaches_network": false
+ },
+ "content": {}
+ },
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-01c",
+ "parent_step_id": null,
+ "seq": 9,
+ "kind": "tool",
+ "name": "check_job",
+ "started_at": 1793600000088,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": "call_9",
+ "call_index": 0,
+ "message_id": "01JP2MSG000000000000000020",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "risk_class": "read",
+ "decision": {
+ "path": [],
+ "origin": "low_risk"
+ },
+ "args_hash": "327e09",
+ "repeat_ok": true,
+ "result_error": false,
+ "error_kind": null,
+ "connector": null,
+ "result_bytes": 120,
+ "exit_code": null,
+ "sandbox": null,
+ "reaches_network": false
+ },
+ "content": {}
+ },
+ {
+ "step_id": "01JP2STP000000000000000001",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-01c",
+ "parent_step_id": null,
+ "seq": 10,
+ "kind": "tool",
+ "name": "check_job",
+ "started_at": 1793600000099,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": "call_10",
+ "call_index": 0,
+ "message_id": "01JP2MSG000000000000000021",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "risk_class": "read",
+ "decision": {
+ "path": [],
+ "origin": "low_risk"
+ },
+ "args_hash": "327e09",
+ "repeat_ok": true,
+ "result_error": false,
+ "error_kind": null,
+ "connector": null,
+ "result_bytes": 120,
+ "exit_code": null,
+ "sandbox": null,
+ "reaches_network": false
+ },
+ "content": {}
+ },
+ {
+ "step_id": "01JP2STP000000000000000001",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-01c",
+ "parent_step_id": null,
+ "seq": 11,
+ "kind": "model",
+ "name": "model",
+ "started_at": 1793600000110,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": null,
+ "call_index": null,
+ "message_id": "01JP2MSG000000000000000011",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "purpose": "main",
+ "provider": "testprov",
+ "provider_title": "Test provider",
+ "model": "testprov:model-a",
+ "stream": true,
+ "finish_reason": "stop",
+ "ttft_ms": 5,
+ "context_window": 100000,
+ "usage": {
+ "reported": true,
+ "input_fresh": 1000,
+ "cache_read": 0,
+ "cache_write": null,
+ "output": 100,
+ "reasoning": null
+ },
+ "tools_offered": 3,
+ "max_output": 4096
+ },
+ "content": {}
+ }
+ ]
+}
diff --git a/tests/corpora/observability/golden-phase2/S2-02/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-02/expected-issues.json
new file mode 100644
index 00000000..fe56fae4
--- /dev/null
+++ b/tests/corpora/observability/golden-phase2/S2-02/expected-issues.json
@@ -0,0 +1,31 @@
+{
+ "signals_version": "0.3",
+ "warn": 1,
+ "fired": {
+ "off_plan": [
+ 4
+ ]
+ },
+ "fine": [
+ "context_full",
+ "cut_off",
+ "made_up_tool",
+ "model_failed",
+ "refused",
+ "repeated_step",
+ "retried",
+ "step_limit",
+ "tool_failed",
+ "waited_unwatched"
+ ],
+ "not_checked": [
+ "blocked_by_sandbox",
+ "overridden",
+ "ran_without_sandbox",
+ "reviewer_allowed_send",
+ "reviewer_could_not_run",
+ "reviewer_paused",
+ "sandbox_failed",
+ "shadow_disagreed"
+ ]
+}
diff --git a/tests/corpora/observability/golden-phase2/S2-02/recording.json b/tests/corpora/observability/golden-phase2/S2-02/recording.json
new file mode 100644
index 00000000..1fab30b2
--- /dev/null
+++ b/tests/corpora/observability/golden-phase2/S2-02/recording.json
@@ -0,0 +1,245 @@
+{
+ "turns": [
+ {
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-02",
+ "seq": 1,
+ "status": "completed",
+ "approval_mode": "interactive",
+ "attended": true,
+ "reviewer": {
+ "active": false,
+ "shadow": false,
+ "kind": null,
+ "model": null
+ },
+ "stamp": {
+ "sandbox": {
+ "provider": "direct",
+ "enforcement": "none"
+ }
+ },
+ "max_iterations": 50,
+ "kind": "run",
+ "trigger": "user",
+ "started_at": 1793600000000,
+ "incomplete": false,
+ "first_message_id": "01JP2MSG000000000000000000",
+ "last_message_id": "01JP2MSG000000000000000000",
+ "schema_version": 1,
+ "duration_ms": 55
+ }
+ ],
+ "steps": [
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-02",
+ "parent_step_id": null,
+ "seq": 1,
+ "kind": "model",
+ "name": "model",
+ "started_at": 1793600000000,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": null,
+ "call_index": null,
+ "message_id": "01JP2MSG000000000000000010",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "purpose": "main",
+ "provider": "testprov",
+ "provider_title": "Test provider",
+ "model": "testprov:model-a",
+ "stream": true,
+ "finish_reason": "stop",
+ "ttft_ms": 5,
+ "context_window": 100000,
+ "usage": {
+ "reported": true,
+ "input_fresh": 1000,
+ "cache_read": 0,
+ "cache_write": null,
+ "output": 100,
+ "reasoning": null
+ },
+ "tools_offered": 3,
+ "max_output": 4096
+ },
+ "content": {}
+ },
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-02",
+ "parent_step_id": null,
+ "seq": 2,
+ "kind": "tool",
+ "name": "todo_write",
+ "started_at": 1793600000011,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": "call_2",
+ "call_index": 0,
+ "message_id": "01JP2MSG000000000000000020",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "risk_class": "read",
+ "decision": {
+ "path": [],
+ "origin": "low_risk"
+ },
+ "args_hash": "63b05b",
+ "repeat_ok": false,
+ "result_error": false,
+ "error_kind": null,
+ "plan": {
+ "items": [
+ {
+ "h": "331634",
+ "s": "in_progress"
+ },
+ {
+ "h": "1c6e6c",
+ "s": "pending"
+ },
+ {
+ "h": "9f86d0",
+ "s": "pending"
+ }
+ ]
+ },
+ "connector": null,
+ "result_bytes": 120,
+ "exit_code": null,
+ "sandbox": null,
+ "reaches_network": false
+ },
+ "content": {}
+ },
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-02",
+ "parent_step_id": null,
+ "seq": 3,
+ "kind": "model",
+ "name": "model",
+ "started_at": 1793600000022,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": null,
+ "call_index": null,
+ "message_id": "01JP2MSG000000000000000010",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "purpose": "main",
+ "provider": "testprov",
+ "provider_title": "Test provider",
+ "model": "testprov:model-a",
+ "stream": true,
+ "finish_reason": "stop",
+ "ttft_ms": 5,
+ "context_window": 100000,
+ "usage": {
+ "reported": true,
+ "input_fresh": 1000,
+ "cache_read": 0,
+ "cache_write": null,
+ "output": 100,
+ "reasoning": null
+ },
+ "tools_offered": 3,
+ "max_output": 4096
+ },
+ "content": {}
+ },
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-02",
+ "parent_step_id": null,
+ "seq": 4,
+ "kind": "tool",
+ "name": "todo_write",
+ "started_at": 1793600000033,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": "call_4",
+ "call_index": 0,
+ "message_id": "01JP2MSG000000000000000020",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "risk_class": "read",
+ "decision": {
+ "path": [],
+ "origin": "low_risk"
+ },
+ "args_hash": "ba305f",
+ "repeat_ok": false,
+ "result_error": false,
+ "error_kind": null,
+ "plan": {
+ "items": [
+ {
+ "h": "331634",
+ "s": "done"
+ },
+ {
+ "h": "9f86d0",
+ "s": "pending"
+ }
+ ]
+ },
+ "connector": null,
+ "result_bytes": 120,
+ "exit_code": null,
+ "sandbox": null,
+ "reaches_network": false
+ },
+ "content": {}
+ },
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-02",
+ "parent_step_id": null,
+ "seq": 5,
+ "kind": "model",
+ "name": "model",
+ "started_at": 1793600000044,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": null,
+ "call_index": null,
+ "message_id": "01JP2MSG000000000000000010",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "purpose": "main",
+ "provider": "testprov",
+ "provider_title": "Test provider",
+ "model": "testprov:model-a",
+ "stream": true,
+ "finish_reason": "stop",
+ "ttft_ms": 5,
+ "context_window": 100000,
+ "usage": {
+ "reported": true,
+ "input_fresh": 1000,
+ "cache_read": 0,
+ "cache_write": null,
+ "output": 100,
+ "reasoning": null
+ },
+ "tools_offered": 3,
+ "max_output": 4096
+ },
+ "content": {}
+ }
+ ]
+}
diff --git a/tests/corpora/observability/golden-phase2/S2-02b/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-02b/expected-issues.json
new file mode 100644
index 00000000..52a7068a
--- /dev/null
+++ b/tests/corpora/observability/golden-phase2/S2-02b/expected-issues.json
@@ -0,0 +1,31 @@
+{
+ "signals_version": "0.3",
+ "warn": 1,
+ "fired": {
+ "off_plan": [
+ 7
+ ]
+ },
+ "fine": [
+ "context_full",
+ "cut_off",
+ "made_up_tool",
+ "model_failed",
+ "refused",
+ "repeated_step",
+ "retried",
+ "step_limit",
+ "tool_failed",
+ "waited_unwatched"
+ ],
+ "not_checked": [
+ "blocked_by_sandbox",
+ "overridden",
+ "ran_without_sandbox",
+ "reviewer_allowed_send",
+ "reviewer_could_not_run",
+ "reviewer_paused",
+ "sandbox_failed",
+ "shadow_disagreed"
+ ]
+}
diff --git a/tests/corpora/observability/golden-phase2/S2-02b/recording.json b/tests/corpora/observability/golden-phase2/S2-02b/recording.json
new file mode 100644
index 00000000..c7aaddc0
--- /dev/null
+++ b/tests/corpora/observability/golden-phase2/S2-02b/recording.json
@@ -0,0 +1,339 @@
+{
+ "turns": [
+ {
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-02b",
+ "seq": 1,
+ "status": "completed",
+ "approval_mode": "interactive",
+ "attended": true,
+ "reviewer": {
+ "active": false,
+ "shadow": false,
+ "kind": null,
+ "model": null
+ },
+ "stamp": {
+ "sandbox": {
+ "provider": "direct",
+ "enforcement": "none"
+ }
+ },
+ "max_iterations": 50,
+ "kind": "run",
+ "trigger": "user",
+ "started_at": 1793600000000,
+ "incomplete": false,
+ "first_message_id": "01JP2MSG000000000000000000",
+ "last_message_id": "01JP2MSG000000000000000000",
+ "schema_version": 1,
+ "duration_ms": 88
+ }
+ ],
+ "steps": [
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-02b",
+ "parent_step_id": null,
+ "seq": 1,
+ "kind": "model",
+ "name": "model",
+ "started_at": 1793600000000,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": null,
+ "call_index": null,
+ "message_id": "01JP2MSG000000000000000010",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "purpose": "main",
+ "provider": "testprov",
+ "provider_title": "Test provider",
+ "model": "testprov:model-a",
+ "stream": true,
+ "finish_reason": "stop",
+ "ttft_ms": 5,
+ "context_window": 100000,
+ "usage": {
+ "reported": true,
+ "input_fresh": 1000,
+ "cache_read": 0,
+ "cache_write": null,
+ "output": 100,
+ "reasoning": null
+ },
+ "tools_offered": 3,
+ "max_output": 4096
+ },
+ "content": {}
+ },
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-02b",
+ "parent_step_id": null,
+ "seq": 2,
+ "kind": "tool",
+ "name": "todo_write",
+ "started_at": 1793600000011,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": "call_2",
+ "call_index": 0,
+ "message_id": "01JP2MSG000000000000000020",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "risk_class": "read",
+ "decision": {
+ "path": [],
+ "origin": "low_risk"
+ },
+ "args_hash": "f7cc59",
+ "repeat_ok": false,
+ "result_error": false,
+ "error_kind": null,
+ "plan": {
+ "items": [
+ {
+ "h": "1c6e6c",
+ "s": "in_progress"
+ }
+ ]
+ },
+ "connector": null,
+ "result_bytes": 120,
+ "exit_code": null,
+ "sandbox": null,
+ "reaches_network": false
+ },
+ "content": {}
+ },
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-02b",
+ "parent_step_id": null,
+ "seq": 3,
+ "kind": "tool",
+ "name": "write_file",
+ "started_at": 1793600000022,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": "call_3",
+ "call_index": 0,
+ "message_id": "01JP2MSG000000000000000020",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "risk_class": "write_local",
+ "decision": {
+ "path": [],
+ "origin": "low_risk"
+ },
+ "args_hash": "363379",
+ "repeat_ok": false,
+ "result_error": false,
+ "error_kind": null,
+ "connector": null,
+ "result_bytes": 120,
+ "exit_code": null,
+ "sandbox": null,
+ "reaches_network": false
+ },
+ "content": {}
+ },
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-02b",
+ "parent_step_id": null,
+ "seq": 4,
+ "kind": "model",
+ "name": "model",
+ "started_at": 1793600000033,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": null,
+ "call_index": null,
+ "message_id": "01JP2MSG000000000000000010",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "purpose": "main",
+ "provider": "testprov",
+ "provider_title": "Test provider",
+ "model": "testprov:model-a",
+ "stream": true,
+ "finish_reason": "stop",
+ "ttft_ms": 5,
+ "context_window": 100000,
+ "usage": {
+ "reported": true,
+ "input_fresh": 1000,
+ "cache_read": 0,
+ "cache_write": null,
+ "output": 100,
+ "reasoning": null
+ },
+ "tools_offered": 3,
+ "max_output": 4096
+ },
+ "content": {}
+ },
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-02b",
+ "parent_step_id": null,
+ "seq": 5,
+ "kind": "tool",
+ "name": "todo_write",
+ "started_at": 1793600000044,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": "call_5",
+ "call_index": 0,
+ "message_id": "01JP2MSG000000000000000020",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "risk_class": "read",
+ "decision": {
+ "path": [],
+ "origin": "low_risk"
+ },
+ "args_hash": "383e55",
+ "repeat_ok": false,
+ "result_error": false,
+ "error_kind": null,
+ "plan": {
+ "items": [
+ {
+ "h": "1c6e6c",
+ "s": "done"
+ }
+ ]
+ },
+ "connector": null,
+ "result_bytes": 120,
+ "exit_code": null,
+ "sandbox": null,
+ "reaches_network": false
+ },
+ "content": {}
+ },
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-02b",
+ "parent_step_id": null,
+ "seq": 6,
+ "kind": "model",
+ "name": "model",
+ "started_at": 1793600000055,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": null,
+ "call_index": null,
+ "message_id": "01JP2MSG000000000000000010",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "purpose": "main",
+ "provider": "testprov",
+ "provider_title": "Test provider",
+ "model": "testprov:model-a",
+ "stream": true,
+ "finish_reason": "stop",
+ "ttft_ms": 5,
+ "context_window": 100000,
+ "usage": {
+ "reported": true,
+ "input_fresh": 1000,
+ "cache_read": 0,
+ "cache_write": null,
+ "output": 100,
+ "reasoning": null
+ },
+ "tools_offered": 3,
+ "max_output": 4096
+ },
+ "content": {}
+ },
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-02b",
+ "parent_step_id": null,
+ "seq": 7,
+ "kind": "tool",
+ "name": "run_shell",
+ "started_at": 1793600000066,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": "call_7",
+ "call_index": 0,
+ "message_id": "01JP2MSG000000000000000020",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "risk_class": "exec",
+ "decision": {
+ "path": [],
+ "origin": "low_risk"
+ },
+ "args_hash": "ade0be",
+ "repeat_ok": false,
+ "result_error": false,
+ "error_kind": null,
+ "connector": null,
+ "result_bytes": 120,
+ "exit_code": null,
+ "sandbox": null,
+ "reaches_network": false
+ },
+ "content": {}
+ },
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-02b",
+ "parent_step_id": null,
+ "seq": 8,
+ "kind": "model",
+ "name": "model",
+ "started_at": 1793600000077,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": null,
+ "call_index": null,
+ "message_id": "01JP2MSG000000000000000010",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "purpose": "main",
+ "provider": "testprov",
+ "provider_title": "Test provider",
+ "model": "testprov:model-a",
+ "stream": true,
+ "finish_reason": "stop",
+ "ttft_ms": 5,
+ "context_window": 100000,
+ "usage": {
+ "reported": true,
+ "input_fresh": 1000,
+ "cache_read": 0,
+ "cache_write": null,
+ "output": 100,
+ "reasoning": null
+ },
+ "tools_offered": 3,
+ "max_output": 4096
+ },
+ "content": {}
+ }
+ ]
+}
diff --git a/tests/corpora/observability/golden-phase2/S2-02c/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-02c/expected-issues.json
new file mode 100644
index 00000000..11d75035
--- /dev/null
+++ b/tests/corpora/observability/golden-phase2/S2-02c/expected-issues.json
@@ -0,0 +1,28 @@
+{
+ "signals_version": "0.3",
+ "warn": 0,
+ "fired": {},
+ "fine": [
+ "context_full",
+ "cut_off",
+ "made_up_tool",
+ "model_failed",
+ "off_plan",
+ "refused",
+ "repeated_step",
+ "retried",
+ "step_limit",
+ "tool_failed",
+ "waited_unwatched"
+ ],
+ "not_checked": [
+ "blocked_by_sandbox",
+ "overridden",
+ "ran_without_sandbox",
+ "reviewer_allowed_send",
+ "reviewer_could_not_run",
+ "reviewer_paused",
+ "sandbox_failed",
+ "shadow_disagreed"
+ ]
+}
diff --git a/tests/corpora/observability/golden-phase2/S2-02c/recording.json b/tests/corpora/observability/golden-phase2/S2-02c/recording.json
new file mode 100644
index 00000000..d8fa5a61
--- /dev/null
+++ b/tests/corpora/observability/golden-phase2/S2-02c/recording.json
@@ -0,0 +1,359 @@
+{
+ "turns": [
+ {
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-02c",
+ "seq": 1,
+ "status": "completed",
+ "approval_mode": "interactive",
+ "attended": true,
+ "reviewer": {
+ "active": false,
+ "shadow": false,
+ "kind": null,
+ "model": null
+ },
+ "stamp": {
+ "sandbox": {
+ "provider": "direct",
+ "enforcement": "none"
+ }
+ },
+ "max_iterations": 50,
+ "kind": "run",
+ "trigger": "user",
+ "started_at": 1793600000000,
+ "incomplete": false,
+ "first_message_id": "01JP2MSG000000000000000000",
+ "last_message_id": "01JP2MSG000000000000000000",
+ "schema_version": 1,
+ "duration_ms": 88
+ }
+ ],
+ "steps": [
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-02c",
+ "parent_step_id": null,
+ "seq": 1,
+ "kind": "model",
+ "name": "model",
+ "started_at": 1793600000000,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": null,
+ "call_index": null,
+ "message_id": "01JP2MSG000000000000000010",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "purpose": "main",
+ "provider": "testprov",
+ "provider_title": "Test provider",
+ "model": "testprov:model-a",
+ "stream": true,
+ "finish_reason": "stop",
+ "ttft_ms": 5,
+ "context_window": 100000,
+ "usage": {
+ "reported": true,
+ "input_fresh": 1000,
+ "cache_read": 0,
+ "cache_write": null,
+ "output": 100,
+ "reasoning": null
+ },
+ "tools_offered": 3,
+ "max_output": 4096
+ },
+ "content": {}
+ },
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-02c",
+ "parent_step_id": null,
+ "seq": 2,
+ "kind": "tool",
+ "name": "todo_write",
+ "started_at": 1793600000011,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": "call_2",
+ "call_index": 0,
+ "message_id": "01JP2MSG000000000000000020",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "risk_class": "read",
+ "decision": {
+ "path": [],
+ "origin": "low_risk"
+ },
+ "args_hash": "85d8b3",
+ "repeat_ok": false,
+ "result_error": false,
+ "error_kind": null,
+ "plan": {
+ "items": [
+ {
+ "h": "6b8bb0",
+ "s": "in_progress"
+ },
+ {
+ "h": "0db52f",
+ "s": "pending"
+ }
+ ]
+ },
+ "connector": null,
+ "result_bytes": 120,
+ "exit_code": null,
+ "sandbox": null,
+ "reaches_network": false
+ },
+ "content": {}
+ },
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-02c",
+ "parent_step_id": null,
+ "seq": 3,
+ "kind": "model",
+ "name": "model",
+ "started_at": 1793600000022,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": null,
+ "call_index": null,
+ "message_id": "01JP2MSG000000000000000010",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "purpose": "main",
+ "provider": "testprov",
+ "provider_title": "Test provider",
+ "model": "testprov:model-a",
+ "stream": true,
+ "finish_reason": "stop",
+ "ttft_ms": 5,
+ "context_window": 100000,
+ "usage": {
+ "reported": true,
+ "input_fresh": 1000,
+ "cache_read": 0,
+ "cache_write": null,
+ "output": 100,
+ "reasoning": null
+ },
+ "tools_offered": 3,
+ "max_output": 4096
+ },
+ "content": {}
+ },
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-02c",
+ "parent_step_id": null,
+ "seq": 4,
+ "kind": "tool",
+ "name": "todo_write",
+ "started_at": 1793600000033,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": "call_4",
+ "call_index": 0,
+ "message_id": "01JP2MSG000000000000000020",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "risk_class": "read",
+ "decision": {
+ "path": [],
+ "origin": "low_risk"
+ },
+ "args_hash": "c6554a",
+ "repeat_ok": false,
+ "result_error": false,
+ "error_kind": null,
+ "plan": {
+ "items": [
+ {
+ "h": "47c423",
+ "s": "done"
+ },
+ {
+ "h": "0db52f",
+ "s": "in_progress"
+ }
+ ]
+ },
+ "connector": null,
+ "result_bytes": 120,
+ "exit_code": null,
+ "sandbox": null,
+ "reaches_network": false
+ },
+ "content": {}
+ },
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-02c",
+ "parent_step_id": null,
+ "seq": 5,
+ "kind": "model",
+ "name": "model",
+ "started_at": 1793600000044,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": null,
+ "call_index": null,
+ "message_id": "01JP2MSG000000000000000010",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "purpose": "main",
+ "provider": "testprov",
+ "provider_title": "Test provider",
+ "model": "testprov:model-a",
+ "stream": true,
+ "finish_reason": "stop",
+ "ttft_ms": 5,
+ "context_window": 100000,
+ "usage": {
+ "reported": true,
+ "input_fresh": 1000,
+ "cache_read": 0,
+ "cache_write": null,
+ "output": 100,
+ "reasoning": null
+ },
+ "tools_offered": 3,
+ "max_output": 4096
+ },
+ "content": {}
+ },
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-02c",
+ "parent_step_id": null,
+ "seq": 6,
+ "kind": "tool",
+ "name": "todo_write",
+ "started_at": 1793600000055,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": "call_6",
+ "call_index": 0,
+ "message_id": "01JP2MSG000000000000000020",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "risk_class": "read",
+ "decision": {
+ "path": [],
+ "origin": "low_risk"
+ },
+ "args_hash": "705a00",
+ "repeat_ok": false,
+ "result_error": false,
+ "error_kind": null,
+ "plan": {
+ "items": [
+ {
+ "h": "47c423",
+ "s": "done"
+ },
+ {
+ "h": "0db52f",
+ "s": "done"
+ }
+ ]
+ },
+ "connector": null,
+ "result_bytes": 120,
+ "exit_code": null,
+ "sandbox": null,
+ "reaches_network": false
+ },
+ "content": {}
+ },
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-02c",
+ "parent_step_id": null,
+ "seq": 7,
+ "kind": "tool",
+ "name": "read_file",
+ "started_at": 1793600000066,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": "call_7",
+ "call_index": 0,
+ "message_id": "01JP2MSG000000000000000020",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "risk_class": "read",
+ "decision": {
+ "path": [],
+ "origin": "low_risk"
+ },
+ "args_hash": "ade0be",
+ "repeat_ok": false,
+ "result_error": false,
+ "error_kind": null,
+ "connector": null,
+ "result_bytes": 120,
+ "exit_code": null,
+ "sandbox": null,
+ "reaches_network": false
+ },
+ "content": {}
+ },
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-02c",
+ "parent_step_id": null,
+ "seq": 8,
+ "kind": "model",
+ "name": "model",
+ "started_at": 1793600000077,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": null,
+ "call_index": null,
+ "message_id": "01JP2MSG000000000000000010",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "purpose": "main",
+ "provider": "testprov",
+ "provider_title": "Test provider",
+ "model": "testprov:model-a",
+ "stream": true,
+ "finish_reason": "stop",
+ "ttft_ms": 5,
+ "context_window": 100000,
+ "usage": {
+ "reported": true,
+ "input_fresh": 1000,
+ "cache_read": 0,
+ "cache_write": null,
+ "output": 100,
+ "reasoning": null
+ },
+ "tools_offered": 3,
+ "max_output": 4096
+ },
+ "content": {}
+ }
+ ]
+}
diff --git a/tests/corpora/observability/golden-phase2/S2-02n/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-02n/expected-issues.json
new file mode 100644
index 00000000..49ff0804
--- /dev/null
+++ b/tests/corpora/observability/golden-phase2/S2-02n/expected-issues.json
@@ -0,0 +1,28 @@
+{
+ "signals_version": "0.3",
+ "warn": 0,
+ "fired": {},
+ "fine": [
+ "context_full",
+ "cut_off",
+ "made_up_tool",
+ "model_failed",
+ "refused",
+ "repeated_step",
+ "retried",
+ "step_limit",
+ "tool_failed",
+ "waited_unwatched"
+ ],
+ "not_checked": [
+ "blocked_by_sandbox",
+ "off_plan",
+ "overridden",
+ "ran_without_sandbox",
+ "reviewer_allowed_send",
+ "reviewer_could_not_run",
+ "reviewer_paused",
+ "sandbox_failed",
+ "shadow_disagreed"
+ ]
+}
diff --git a/tests/corpora/observability/golden-phase2/S2-02n/recording.json b/tests/corpora/observability/golden-phase2/S2-02n/recording.json
new file mode 100644
index 00000000..c8e86642
--- /dev/null
+++ b/tests/corpora/observability/golden-phase2/S2-02n/recording.json
@@ -0,0 +1,145 @@
+{
+ "turns": [
+ {
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-02n",
+ "seq": 1,
+ "status": "completed",
+ "approval_mode": "interactive",
+ "attended": true,
+ "reviewer": {
+ "active": false,
+ "shadow": false,
+ "kind": null,
+ "model": null
+ },
+ "stamp": {
+ "sandbox": {
+ "provider": "direct",
+ "enforcement": "none"
+ }
+ },
+ "max_iterations": 50,
+ "kind": "run",
+ "trigger": "user",
+ "started_at": 1793600000000,
+ "incomplete": false,
+ "first_message_id": "01JP2MSG000000000000000000",
+ "last_message_id": "01JP2MSG000000000000000000",
+ "schema_version": 1,
+ "duration_ms": 33
+ }
+ ],
+ "steps": [
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-02n",
+ "parent_step_id": null,
+ "seq": 1,
+ "kind": "model",
+ "name": "model",
+ "started_at": 1793600000000,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": null,
+ "call_index": null,
+ "message_id": "01JP2MSG000000000000000010",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "purpose": "main",
+ "provider": "testprov",
+ "provider_title": "Test provider",
+ "model": "testprov:model-a",
+ "stream": true,
+ "finish_reason": "stop",
+ "ttft_ms": 5,
+ "context_window": 100000,
+ "usage": {
+ "reported": true,
+ "input_fresh": 1000,
+ "cache_read": 0,
+ "cache_write": null,
+ "output": 100,
+ "reasoning": null
+ },
+ "tools_offered": 3,
+ "max_output": 4096
+ },
+ "content": {}
+ },
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-02n",
+ "parent_step_id": null,
+ "seq": 2,
+ "kind": "tool",
+ "name": "read_file",
+ "started_at": 1793600000011,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": "call_2",
+ "call_index": 0,
+ "message_id": "01JP2MSG000000000000000020",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "risk_class": "read",
+ "decision": {
+ "path": [],
+ "origin": "low_risk"
+ },
+ "args_hash": "2bfd14",
+ "repeat_ok": false,
+ "result_error": false,
+ "error_kind": null,
+ "connector": null,
+ "result_bytes": 120,
+ "exit_code": null,
+ "sandbox": null,
+ "reaches_network": false
+ },
+ "content": {}
+ },
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-02n",
+ "parent_step_id": null,
+ "seq": 3,
+ "kind": "model",
+ "name": "model",
+ "started_at": 1793600000022,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": null,
+ "call_index": null,
+ "message_id": "01JP2MSG000000000000000010",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "purpose": "main",
+ "provider": "testprov",
+ "provider_title": "Test provider",
+ "model": "testprov:model-a",
+ "stream": true,
+ "finish_reason": "stop",
+ "ttft_ms": 5,
+ "context_window": 100000,
+ "usage": {
+ "reported": true,
+ "input_fresh": 1000,
+ "cache_read": 0,
+ "cache_write": null,
+ "output": 100,
+ "reasoning": null
+ },
+ "tools_offered": 3,
+ "max_output": 4096
+ },
+ "content": {}
+ }
+ ]
+}
diff --git a/tests/corpora/observability/golden-phase2/S2-03/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-03/expected-issues.json
new file mode 100644
index 00000000..058d86ad
--- /dev/null
+++ b/tests/corpora/observability/golden-phase2/S2-03/expected-issues.json
@@ -0,0 +1,31 @@
+{
+ "signals_version": "0.3",
+ "warn": 1,
+ "fired": {
+ "refused": [
+ 2
+ ]
+ },
+ "fine": [
+ "context_full",
+ "cut_off",
+ "made_up_tool",
+ "model_failed",
+ "repeated_step",
+ "retried",
+ "step_limit",
+ "tool_failed",
+ "waited_unwatched"
+ ],
+ "not_checked": [
+ "blocked_by_sandbox",
+ "off_plan",
+ "overridden",
+ "ran_without_sandbox",
+ "reviewer_allowed_send",
+ "reviewer_could_not_run",
+ "reviewer_paused",
+ "sandbox_failed",
+ "shadow_disagreed"
+ ]
+}
diff --git a/tests/corpora/observability/golden-phase2/S2-03/recording.json b/tests/corpora/observability/golden-phase2/S2-03/recording.json
new file mode 100644
index 00000000..bc9745ac
--- /dev/null
+++ b/tests/corpora/observability/golden-phase2/S2-03/recording.json
@@ -0,0 +1,150 @@
+{
+ "turns": [
+ {
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-03",
+ "seq": 1,
+ "status": "completed",
+ "approval_mode": "interactive",
+ "attended": true,
+ "reviewer": {
+ "active": false,
+ "shadow": false,
+ "kind": null,
+ "model": null
+ },
+ "stamp": {
+ "sandbox": {
+ "provider": "direct",
+ "enforcement": "none"
+ }
+ },
+ "max_iterations": 50,
+ "kind": "run",
+ "trigger": "user",
+ "started_at": 1793600000000,
+ "incomplete": false,
+ "first_message_id": "01JP2MSG000000000000000000",
+ "last_message_id": "01JP2MSG000000000000000000",
+ "schema_version": 1,
+ "duration_ms": 23
+ }
+ ],
+ "steps": [
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-03",
+ "parent_step_id": null,
+ "seq": 1,
+ "kind": "model",
+ "name": "model",
+ "started_at": 1793600000000,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": null,
+ "call_index": null,
+ "message_id": "01JP2MSG000000000000000010",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "purpose": "main",
+ "provider": "testprov",
+ "provider_title": "Test provider",
+ "model": "testprov:model-a",
+ "stream": true,
+ "finish_reason": "stop",
+ "ttft_ms": 5,
+ "context_window": 100000,
+ "usage": {
+ "reported": true,
+ "input_fresh": 1000,
+ "cache_read": 0,
+ "cache_write": null,
+ "output": 100,
+ "reasoning": null
+ },
+ "tools_offered": 3,
+ "max_output": 4096
+ },
+ "content": {}
+ },
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-03",
+ "parent_step_id": null,
+ "seq": 2,
+ "kind": "tool",
+ "name": "run_shell",
+ "started_at": 1793600000011,
+ "duration_ms": 0,
+ "status": "denied",
+ "call_id": "call_2",
+ "call_index": 0,
+ "message_id": "01JP2MSG000000000000000020",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "risk_class": "exec",
+ "decision": {
+ "path": [
+ {
+ "stage": "resolved",
+ "outcome": "denied"
+ }
+ ],
+ "origin": "user"
+ },
+ "args_hash": "2bfd14",
+ "repeat_ok": false,
+ "result_error": false,
+ "error_kind": null,
+ "connector": null,
+ "result_bytes": null,
+ "exit_code": null,
+ "sandbox": null,
+ "reaches_network": false
+ },
+ "content": {}
+ },
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-03",
+ "parent_step_id": null,
+ "seq": 3,
+ "kind": "model",
+ "name": "model",
+ "started_at": 1793600000012,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": null,
+ "call_index": null,
+ "message_id": "01JP2MSG000000000000000010",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "purpose": "main",
+ "provider": "testprov",
+ "provider_title": "Test provider",
+ "model": "testprov:model-a",
+ "stream": true,
+ "finish_reason": "stop",
+ "ttft_ms": 5,
+ "context_window": 100000,
+ "usage": {
+ "reported": true,
+ "input_fresh": 1000,
+ "cache_read": 0,
+ "cache_write": null,
+ "output": 100,
+ "reasoning": null
+ },
+ "tools_offered": 3,
+ "max_output": 4096
+ },
+ "content": {}
+ }
+ ]
+}
diff --git a/tests/corpora/observability/golden-phase2/S2-03b/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-03b/expected-issues.json
new file mode 100644
index 00000000..49ff0804
--- /dev/null
+++ b/tests/corpora/observability/golden-phase2/S2-03b/expected-issues.json
@@ -0,0 +1,28 @@
+{
+ "signals_version": "0.3",
+ "warn": 0,
+ "fired": {},
+ "fine": [
+ "context_full",
+ "cut_off",
+ "made_up_tool",
+ "model_failed",
+ "refused",
+ "repeated_step",
+ "retried",
+ "step_limit",
+ "tool_failed",
+ "waited_unwatched"
+ ],
+ "not_checked": [
+ "blocked_by_sandbox",
+ "off_plan",
+ "overridden",
+ "ran_without_sandbox",
+ "reviewer_allowed_send",
+ "reviewer_could_not_run",
+ "reviewer_paused",
+ "sandbox_failed",
+ "shadow_disagreed"
+ ]
+}
diff --git a/tests/corpora/observability/golden-phase2/S2-03b/recording.json b/tests/corpora/observability/golden-phase2/S2-03b/recording.json
new file mode 100644
index 00000000..204817ad
--- /dev/null
+++ b/tests/corpora/observability/golden-phase2/S2-03b/recording.json
@@ -0,0 +1,107 @@
+{
+ "turns": [
+ {
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-03b",
+ "seq": 1,
+ "status": "interrupted",
+ "approval_mode": "interactive",
+ "attended": true,
+ "reviewer": {
+ "active": false,
+ "shadow": false,
+ "kind": null,
+ "model": null
+ },
+ "stamp": {
+ "sandbox": {
+ "provider": "direct",
+ "enforcement": "none"
+ }
+ },
+ "max_iterations": 50,
+ "kind": "run",
+ "trigger": "user",
+ "started_at": 1793600000000,
+ "incomplete": false,
+ "first_message_id": "01JP2MSG000000000000000000",
+ "last_message_id": "01JP2MSG000000000000000000",
+ "schema_version": 1,
+ "duration_ms": 12
+ }
+ ],
+ "steps": [
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-03b",
+ "parent_step_id": null,
+ "seq": 1,
+ "kind": "model",
+ "name": "model",
+ "started_at": 1793600000000,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": null,
+ "call_index": null,
+ "message_id": "01JP2MSG000000000000000010",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "purpose": "main",
+ "provider": "testprov",
+ "provider_title": "Test provider",
+ "model": "testprov:model-a",
+ "stream": true,
+ "finish_reason": "stop",
+ "ttft_ms": 5,
+ "context_window": 100000,
+ "usage": {
+ "reported": true,
+ "input_fresh": 1000,
+ "cache_read": 0,
+ "cache_write": null,
+ "output": 100,
+ "reasoning": null
+ },
+ "tools_offered": 3,
+ "max_output": 4096
+ },
+ "content": {}
+ },
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-03b",
+ "parent_step_id": null,
+ "seq": 2,
+ "kind": "tool",
+ "name": "run_shell",
+ "started_at": 1793600000011,
+ "duration_ms": 0,
+ "status": "denied",
+ "call_id": "call_2",
+ "call_index": 0,
+ "message_id": "01JP2MSG000000000000000020",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "risk_class": "exec",
+ "decision": {
+ "path": [],
+ "origin": null
+ },
+ "args_hash": "2bfd14",
+ "repeat_ok": false,
+ "result_error": false,
+ "error_kind": null,
+ "connector": null,
+ "result_bytes": null,
+ "exit_code": null,
+ "sandbox": null,
+ "reaches_network": false
+ },
+ "content": {}
+ }
+ ]
+}
diff --git a/tests/corpora/observability/golden-phase2/S2-04/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-04/expected-issues.json
new file mode 100644
index 00000000..fa30041d
--- /dev/null
+++ b/tests/corpora/observability/golden-phase2/S2-04/expected-issues.json
@@ -0,0 +1,31 @@
+{
+ "signals_version": "0.3",
+ "warn": 0,
+ "fired": {
+ "overridden": [
+ 3
+ ]
+ },
+ "fine": [
+ "context_full",
+ "cut_off",
+ "made_up_tool",
+ "model_failed",
+ "refused",
+ "repeated_step",
+ "retried",
+ "reviewer_allowed_send",
+ "reviewer_could_not_run",
+ "reviewer_paused",
+ "step_limit",
+ "tool_failed",
+ "waited_unwatched"
+ ],
+ "not_checked": [
+ "blocked_by_sandbox",
+ "off_plan",
+ "ran_without_sandbox",
+ "sandbox_failed",
+ "shadow_disagreed"
+ ]
+}
diff --git a/tests/corpora/observability/golden-phase2/S2-04/recording.json b/tests/corpora/observability/golden-phase2/S2-04/recording.json
new file mode 100644
index 00000000..454603b7
--- /dev/null
+++ b/tests/corpora/observability/golden-phase2/S2-04/recording.json
@@ -0,0 +1,189 @@
+{
+ "turns": [
+ {
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-04",
+ "seq": 1,
+ "status": "completed",
+ "approval_mode": "auto-approve",
+ "attended": true,
+ "reviewer": {
+ "active": true,
+ "shadow": false,
+ "kind": "session_model",
+ "model": "m"
+ },
+ "stamp": {
+ "sandbox": {
+ "provider": "direct",
+ "enforcement": "none"
+ }
+ },
+ "max_iterations": 50,
+ "kind": "run",
+ "trigger": "user",
+ "started_at": 1793600000000,
+ "incomplete": false,
+ "first_message_id": "01JP2MSG000000000000000000",
+ "last_message_id": "01JP2MSG000000000000000000",
+ "schema_version": 1,
+ "duration_ms": 44
+ }
+ ],
+ "steps": [
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-04",
+ "parent_step_id": null,
+ "seq": 1,
+ "kind": "model",
+ "name": "model",
+ "started_at": 1793600000000,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": null,
+ "call_index": null,
+ "message_id": "01JP2MSG000000000000000010",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "purpose": "main",
+ "provider": "testprov",
+ "provider_title": "Test provider",
+ "model": "testprov:model-a",
+ "stream": true,
+ "finish_reason": "stop",
+ "ttft_ms": 5,
+ "context_window": 100000,
+ "usage": {
+ "reported": true,
+ "input_fresh": 1000,
+ "cache_read": 0,
+ "cache_write": null,
+ "output": 100,
+ "reasoning": null
+ },
+ "tools_offered": 3,
+ "max_output": 4096
+ },
+ "content": {}
+ },
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-04",
+ "parent_step_id": null,
+ "seq": 2,
+ "kind": "auto_approver",
+ "name": "auto_approver",
+ "started_at": 1793600000011,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": "call_3",
+ "call_index": null,
+ "message_id": null,
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "reviewer": {
+ "kind": "session_model",
+ "model": "m",
+ "external": false
+ },
+ "mode": "live",
+ "result": "deny",
+ "score": null,
+ "cutoff": null,
+ "usage_reported": true,
+ "breaker": {
+ "denials_in_row": 0,
+ "tripped": false
+ },
+ "matched_user": null
+ },
+ "content": {}
+ },
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-04",
+ "parent_step_id": null,
+ "seq": 3,
+ "kind": "tool",
+ "name": "run_shell",
+ "started_at": 1793600000022,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": "call_3",
+ "call_index": 0,
+ "message_id": "01JP2MSG000000000000000020",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "risk_class": "exec",
+ "decision": {
+ "path": [
+ {
+ "stage": "auto_approver",
+ "outcome": "deny"
+ },
+ {
+ "stage": "grant",
+ "outcome": "allow_anyway"
+ }
+ ],
+ "origin": "allow_anyway"
+ },
+ "args_hash": "363379",
+ "repeat_ok": false,
+ "result_error": false,
+ "error_kind": null,
+ "connector": null,
+ "result_bytes": 120,
+ "exit_code": null,
+ "sandbox": null,
+ "reaches_network": false
+ },
+ "content": {}
+ },
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-04",
+ "parent_step_id": null,
+ "seq": 4,
+ "kind": "model",
+ "name": "model",
+ "started_at": 1793600000033,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": null,
+ "call_index": null,
+ "message_id": "01JP2MSG000000000000000010",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "purpose": "main",
+ "provider": "testprov",
+ "provider_title": "Test provider",
+ "model": "testprov:model-a",
+ "stream": true,
+ "finish_reason": "stop",
+ "ttft_ms": 5,
+ "context_window": 100000,
+ "usage": {
+ "reported": true,
+ "input_fresh": 1000,
+ "cache_read": 0,
+ "cache_write": null,
+ "output": 100,
+ "reasoning": null
+ },
+ "tools_offered": 3,
+ "max_output": 4096
+ },
+ "content": {}
+ }
+ ]
+}
diff --git a/tests/corpora/observability/golden-phase2/S2-04n/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-04n/expected-issues.json
new file mode 100644
index 00000000..49ff0804
--- /dev/null
+++ b/tests/corpora/observability/golden-phase2/S2-04n/expected-issues.json
@@ -0,0 +1,28 @@
+{
+ "signals_version": "0.3",
+ "warn": 0,
+ "fired": {},
+ "fine": [
+ "context_full",
+ "cut_off",
+ "made_up_tool",
+ "model_failed",
+ "refused",
+ "repeated_step",
+ "retried",
+ "step_limit",
+ "tool_failed",
+ "waited_unwatched"
+ ],
+ "not_checked": [
+ "blocked_by_sandbox",
+ "off_plan",
+ "overridden",
+ "ran_without_sandbox",
+ "reviewer_allowed_send",
+ "reviewer_could_not_run",
+ "reviewer_paused",
+ "sandbox_failed",
+ "shadow_disagreed"
+ ]
+}
diff --git a/tests/corpora/observability/golden-phase2/S2-04n/recording.json b/tests/corpora/observability/golden-phase2/S2-04n/recording.json
new file mode 100644
index 00000000..8374b903
--- /dev/null
+++ b/tests/corpora/observability/golden-phase2/S2-04n/recording.json
@@ -0,0 +1,145 @@
+{
+ "turns": [
+ {
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-04n",
+ "seq": 1,
+ "status": "completed",
+ "approval_mode": "interactive",
+ "attended": true,
+ "reviewer": {
+ "active": false,
+ "shadow": false,
+ "kind": null,
+ "model": null
+ },
+ "stamp": {
+ "sandbox": {
+ "provider": "direct",
+ "enforcement": "none"
+ }
+ },
+ "max_iterations": 50,
+ "kind": "run",
+ "trigger": "user",
+ "started_at": 1793600000000,
+ "incomplete": false,
+ "first_message_id": "01JP2MSG000000000000000000",
+ "last_message_id": "01JP2MSG000000000000000000",
+ "schema_version": 1,
+ "duration_ms": 33
+ }
+ ],
+ "steps": [
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-04n",
+ "parent_step_id": null,
+ "seq": 1,
+ "kind": "model",
+ "name": "model",
+ "started_at": 1793600000000,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": null,
+ "call_index": null,
+ "message_id": "01JP2MSG000000000000000010",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "purpose": "main",
+ "provider": "testprov",
+ "provider_title": "Test provider",
+ "model": "testprov:model-a",
+ "stream": true,
+ "finish_reason": "stop",
+ "ttft_ms": 5,
+ "context_window": 100000,
+ "usage": {
+ "reported": true,
+ "input_fresh": 1000,
+ "cache_read": 0,
+ "cache_write": null,
+ "output": 100,
+ "reasoning": null
+ },
+ "tools_offered": 3,
+ "max_output": 4096
+ },
+ "content": {}
+ },
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-04n",
+ "parent_step_id": null,
+ "seq": 2,
+ "kind": "tool",
+ "name": "run_shell",
+ "started_at": 1793600000011,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": "call_2",
+ "call_index": 0,
+ "message_id": "01JP2MSG000000000000000020",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "risk_class": "exec",
+ "decision": {
+ "path": [],
+ "origin": "low_risk"
+ },
+ "args_hash": "2bfd14",
+ "repeat_ok": false,
+ "result_error": false,
+ "error_kind": null,
+ "connector": null,
+ "result_bytes": 120,
+ "exit_code": null,
+ "sandbox": null,
+ "reaches_network": false
+ },
+ "content": {}
+ },
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-04n",
+ "parent_step_id": null,
+ "seq": 3,
+ "kind": "model",
+ "name": "model",
+ "started_at": 1793600000022,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": null,
+ "call_index": null,
+ "message_id": "01JP2MSG000000000000000010",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "purpose": "main",
+ "provider": "testprov",
+ "provider_title": "Test provider",
+ "model": "testprov:model-a",
+ "stream": true,
+ "finish_reason": "stop",
+ "ttft_ms": 5,
+ "context_window": 100000,
+ "usage": {
+ "reported": true,
+ "input_fresh": 1000,
+ "cache_read": 0,
+ "cache_write": null,
+ "output": 100,
+ "reasoning": null
+ },
+ "tools_offered": 3,
+ "max_output": 4096
+ },
+ "content": {}
+ }
+ ]
+}
diff --git a/tests/corpora/observability/golden-phase2/S2-05/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-05/expected-issues.json
new file mode 100644
index 00000000..25ba330d
--- /dev/null
+++ b/tests/corpora/observability/golden-phase2/S2-05/expected-issues.json
@@ -0,0 +1,31 @@
+{
+ "signals_version": "0.3",
+ "warn": 1,
+ "fired": {
+ "reviewer_could_not_run": [
+ 2
+ ]
+ },
+ "fine": [
+ "context_full",
+ "cut_off",
+ "made_up_tool",
+ "model_failed",
+ "overridden",
+ "refused",
+ "repeated_step",
+ "retried",
+ "reviewer_allowed_send",
+ "reviewer_paused",
+ "step_limit",
+ "tool_failed",
+ "waited_unwatched"
+ ],
+ "not_checked": [
+ "blocked_by_sandbox",
+ "off_plan",
+ "ran_without_sandbox",
+ "sandbox_failed",
+ "shadow_disagreed"
+ ]
+}
diff --git a/tests/corpora/observability/golden-phase2/S2-05/recording.json b/tests/corpora/observability/golden-phase2/S2-05/recording.json
new file mode 100644
index 00000000..6bddd797
--- /dev/null
+++ b/tests/corpora/observability/golden-phase2/S2-05/recording.json
@@ -0,0 +1,146 @@
+{
+ "turns": [
+ {
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-05",
+ "seq": 1,
+ "status": "completed",
+ "approval_mode": "auto-approve",
+ "attended": true,
+ "reviewer": {
+ "active": true,
+ "shadow": false,
+ "kind": "session_model",
+ "model": "m"
+ },
+ "stamp": {
+ "sandbox": {
+ "provider": "direct",
+ "enforcement": "none"
+ }
+ },
+ "max_iterations": 50,
+ "kind": "run",
+ "trigger": "user",
+ "started_at": 1793600000000,
+ "incomplete": false,
+ "first_message_id": "01JP2MSG000000000000000000",
+ "last_message_id": "01JP2MSG000000000000000000",
+ "schema_version": 1,
+ "duration_ms": 33
+ }
+ ],
+ "steps": [
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-05",
+ "parent_step_id": null,
+ "seq": 1,
+ "kind": "model",
+ "name": "model",
+ "started_at": 1793600000000,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": null,
+ "call_index": null,
+ "message_id": "01JP2MSG000000000000000010",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "purpose": "main",
+ "provider": "testprov",
+ "provider_title": "Test provider",
+ "model": "testprov:model-a",
+ "stream": true,
+ "finish_reason": "stop",
+ "ttft_ms": 5,
+ "context_window": 100000,
+ "usage": {
+ "reported": true,
+ "input_fresh": 1000,
+ "cache_read": 0,
+ "cache_write": null,
+ "output": 100,
+ "reasoning": null
+ },
+ "tools_offered": 3,
+ "max_output": 4096
+ },
+ "content": {}
+ },
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-05",
+ "parent_step_id": null,
+ "seq": 2,
+ "kind": "auto_approver",
+ "name": "auto_approver",
+ "started_at": 1793600000011,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": "call_3",
+ "call_index": null,
+ "message_id": null,
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "reviewer": {
+ "kind": "session_model",
+ "model": "m",
+ "external": false
+ },
+ "mode": "live",
+ "result": "could_not_run",
+ "score": null,
+ "cutoff": null,
+ "usage_reported": true,
+ "breaker": {
+ "denials_in_row": 0,
+ "tripped": false
+ },
+ "matched_user": null
+ },
+ "content": {}
+ },
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-05",
+ "parent_step_id": null,
+ "seq": 3,
+ "kind": "model",
+ "name": "model",
+ "started_at": 1793600000022,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": null,
+ "call_index": null,
+ "message_id": "01JP2MSG000000000000000010",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "purpose": "main",
+ "provider": "testprov",
+ "provider_title": "Test provider",
+ "model": "testprov:model-a",
+ "stream": true,
+ "finish_reason": "stop",
+ "ttft_ms": 5,
+ "context_window": 100000,
+ "usage": {
+ "reported": true,
+ "input_fresh": 1000,
+ "cache_read": 0,
+ "cache_write": null,
+ "output": 100,
+ "reasoning": null
+ },
+ "tools_offered": 3,
+ "max_output": 4096
+ },
+ "content": {}
+ }
+ ]
+}
diff --git a/tests/corpora/observability/golden-phase2/S2-05c/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-05c/expected-issues.json
new file mode 100644
index 00000000..b142c8d2
--- /dev/null
+++ b/tests/corpora/observability/golden-phase2/S2-05c/expected-issues.json
@@ -0,0 +1,28 @@
+{
+ "signals_version": "0.3",
+ "warn": 0,
+ "fired": {},
+ "fine": [
+ "context_full",
+ "cut_off",
+ "made_up_tool",
+ "model_failed",
+ "overridden",
+ "refused",
+ "repeated_step",
+ "retried",
+ "reviewer_allowed_send",
+ "reviewer_could_not_run",
+ "reviewer_paused",
+ "step_limit",
+ "tool_failed",
+ "waited_unwatched"
+ ],
+ "not_checked": [
+ "blocked_by_sandbox",
+ "off_plan",
+ "ran_without_sandbox",
+ "sandbox_failed",
+ "shadow_disagreed"
+ ]
+}
diff --git a/tests/corpora/observability/golden-phase2/S2-05c/recording.json b/tests/corpora/observability/golden-phase2/S2-05c/recording.json
new file mode 100644
index 00000000..a6aac4b2
--- /dev/null
+++ b/tests/corpora/observability/golden-phase2/S2-05c/recording.json
@@ -0,0 +1,146 @@
+{
+ "turns": [
+ {
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-05c",
+ "seq": 1,
+ "status": "completed",
+ "approval_mode": "auto-approve",
+ "attended": true,
+ "reviewer": {
+ "active": true,
+ "shadow": false,
+ "kind": "session_model",
+ "model": "m"
+ },
+ "stamp": {
+ "sandbox": {
+ "provider": "direct",
+ "enforcement": "none"
+ }
+ },
+ "max_iterations": 50,
+ "kind": "run",
+ "trigger": "user",
+ "started_at": 1793600000000,
+ "incomplete": false,
+ "first_message_id": "01JP2MSG000000000000000000",
+ "last_message_id": "01JP2MSG000000000000000000",
+ "schema_version": 1,
+ "duration_ms": 33
+ }
+ ],
+ "steps": [
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-05c",
+ "parent_step_id": null,
+ "seq": 1,
+ "kind": "model",
+ "name": "model",
+ "started_at": 1793600000000,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": null,
+ "call_index": null,
+ "message_id": "01JP2MSG000000000000000010",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "purpose": "main",
+ "provider": "testprov",
+ "provider_title": "Test provider",
+ "model": "testprov:model-a",
+ "stream": true,
+ "finish_reason": "stop",
+ "ttft_ms": 5,
+ "context_window": 100000,
+ "usage": {
+ "reported": true,
+ "input_fresh": 1000,
+ "cache_read": 0,
+ "cache_write": null,
+ "output": 100,
+ "reasoning": null
+ },
+ "tools_offered": 3,
+ "max_output": 4096
+ },
+ "content": {}
+ },
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-05c",
+ "parent_step_id": null,
+ "seq": 2,
+ "kind": "auto_approver",
+ "name": "auto_approver",
+ "started_at": 1793600000011,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": "call_3",
+ "call_index": null,
+ "message_id": null,
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "reviewer": {
+ "kind": "session_model",
+ "model": "m",
+ "external": false
+ },
+ "mode": "live",
+ "result": "allow",
+ "score": null,
+ "cutoff": null,
+ "usage_reported": true,
+ "breaker": {
+ "denials_in_row": 0,
+ "tripped": false
+ },
+ "matched_user": null
+ },
+ "content": {}
+ },
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-05c",
+ "parent_step_id": null,
+ "seq": 3,
+ "kind": "model",
+ "name": "model",
+ "started_at": 1793600000022,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": null,
+ "call_index": null,
+ "message_id": "01JP2MSG000000000000000010",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "purpose": "main",
+ "provider": "testprov",
+ "provider_title": "Test provider",
+ "model": "testprov:model-a",
+ "stream": true,
+ "finish_reason": "stop",
+ "ttft_ms": 5,
+ "context_window": 100000,
+ "usage": {
+ "reported": true,
+ "input_fresh": 1000,
+ "cache_read": 0,
+ "cache_write": null,
+ "output": 100,
+ "reasoning": null
+ },
+ "tools_offered": 3,
+ "max_output": 4096
+ },
+ "content": {}
+ }
+ ]
+}
diff --git a/tests/corpora/observability/golden-phase2/S2-06/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-06/expected-issues.json
new file mode 100644
index 00000000..0bc057dc
--- /dev/null
+++ b/tests/corpora/observability/golden-phase2/S2-06/expected-issues.json
@@ -0,0 +1,31 @@
+{
+ "signals_version": "0.3",
+ "warn": 1,
+ "fired": {
+ "reviewer_paused": [
+ 3
+ ]
+ },
+ "fine": [
+ "context_full",
+ "cut_off",
+ "made_up_tool",
+ "model_failed",
+ "overridden",
+ "refused",
+ "repeated_step",
+ "retried",
+ "reviewer_allowed_send",
+ "reviewer_could_not_run",
+ "step_limit",
+ "tool_failed",
+ "waited_unwatched"
+ ],
+ "not_checked": [
+ "blocked_by_sandbox",
+ "off_plan",
+ "ran_without_sandbox",
+ "sandbox_failed",
+ "shadow_disagreed"
+ ]
+}
diff --git a/tests/corpora/observability/golden-phase2/S2-06/recording.json b/tests/corpora/observability/golden-phase2/S2-06/recording.json
new file mode 100644
index 00000000..3db32238
--- /dev/null
+++ b/tests/corpora/observability/golden-phase2/S2-06/recording.json
@@ -0,0 +1,165 @@
+{
+ "turns": [
+ {
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-06",
+ "seq": 1,
+ "status": "completed",
+ "approval_mode": "auto-approve",
+ "attended": true,
+ "reviewer": {
+ "active": true,
+ "shadow": false,
+ "kind": "session_model",
+ "model": "m"
+ },
+ "stamp": {
+ "sandbox": {
+ "provider": "direct",
+ "enforcement": "none"
+ }
+ },
+ "max_iterations": 50,
+ "kind": "run",
+ "trigger": "user",
+ "started_at": 1793600000000,
+ "incomplete": false,
+ "first_message_id": "01JP2MSG000000000000000000",
+ "last_message_id": "01JP2MSG000000000000000000",
+ "schema_version": 1,
+ "duration_ms": 34
+ }
+ ],
+ "steps": [
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-06",
+ "parent_step_id": null,
+ "seq": 1,
+ "kind": "model",
+ "name": "model",
+ "started_at": 1793600000000,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": null,
+ "call_index": null,
+ "message_id": "01JP2MSG000000000000000010",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "purpose": "main",
+ "provider": "testprov",
+ "provider_title": "Test provider",
+ "model": "testprov:model-a",
+ "stream": true,
+ "finish_reason": "stop",
+ "ttft_ms": 5,
+ "context_window": 100000,
+ "usage": {
+ "reported": true,
+ "input_fresh": 1000,
+ "cache_read": 0,
+ "cache_write": null,
+ "output": 100,
+ "reasoning": null
+ },
+ "tools_offered": 3,
+ "max_output": 4096
+ },
+ "content": {}
+ },
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-06",
+ "parent_step_id": null,
+ "seq": 2,
+ "kind": "auto_approver",
+ "name": "auto_approver",
+ "started_at": 1793600000011,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": "call_3",
+ "call_index": null,
+ "message_id": null,
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "reviewer": {
+ "kind": "session_model",
+ "model": "m",
+ "external": false
+ },
+ "mode": "live",
+ "result": "deny",
+ "score": null,
+ "cutoff": null,
+ "usage_reported": true,
+ "breaker": {
+ "denials_in_row": 3,
+ "tripped": true
+ },
+ "matched_user": null
+ },
+ "content": {}
+ },
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-06",
+ "parent_step_id": null,
+ "seq": 3,
+ "kind": "notice",
+ "name": "reviewer_paused",
+ "started_at": 1793600000022,
+ "duration_ms": 0,
+ "status": "ok",
+ "call_id": null,
+ "call_index": null,
+ "message_id": null,
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {},
+ "content": {}
+ },
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-06",
+ "parent_step_id": null,
+ "seq": 4,
+ "kind": "model",
+ "name": "model",
+ "started_at": 1793600000023,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": null,
+ "call_index": null,
+ "message_id": "01JP2MSG000000000000000010",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "purpose": "main",
+ "provider": "testprov",
+ "provider_title": "Test provider",
+ "model": "testprov:model-a",
+ "stream": true,
+ "finish_reason": "stop",
+ "ttft_ms": 5,
+ "context_window": 100000,
+ "usage": {
+ "reported": true,
+ "input_fresh": 1000,
+ "cache_read": 0,
+ "cache_write": null,
+ "output": 100,
+ "reasoning": null
+ },
+ "tools_offered": 3,
+ "max_output": 4096
+ },
+ "content": {}
+ }
+ ]
+}
diff --git a/tests/corpora/observability/golden-phase2/S2-07/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-07/expected-issues.json
new file mode 100644
index 00000000..64d81e9e
--- /dev/null
+++ b/tests/corpora/observability/golden-phase2/S2-07/expected-issues.json
@@ -0,0 +1,31 @@
+{
+ "signals_version": "0.3",
+ "warn": 0,
+ "fired": {
+ "shadow_disagreed": [
+ 2
+ ]
+ },
+ "fine": [
+ "context_full",
+ "cut_off",
+ "made_up_tool",
+ "model_failed",
+ "refused",
+ "repeated_step",
+ "retried",
+ "reviewer_could_not_run",
+ "step_limit",
+ "tool_failed",
+ "waited_unwatched"
+ ],
+ "not_checked": [
+ "blocked_by_sandbox",
+ "off_plan",
+ "overridden",
+ "ran_without_sandbox",
+ "reviewer_allowed_send",
+ "reviewer_paused",
+ "sandbox_failed"
+ ]
+}
diff --git a/tests/corpora/observability/golden-phase2/S2-07/recording.json b/tests/corpora/observability/golden-phase2/S2-07/recording.json
new file mode 100644
index 00000000..26aa4a2d
--- /dev/null
+++ b/tests/corpora/observability/golden-phase2/S2-07/recording.json
@@ -0,0 +1,146 @@
+{
+ "turns": [
+ {
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-07",
+ "seq": 1,
+ "status": "completed",
+ "approval_mode": "interactive",
+ "attended": true,
+ "reviewer": {
+ "active": false,
+ "shadow": true,
+ "kind": "session_model",
+ "model": "m"
+ },
+ "stamp": {
+ "sandbox": {
+ "provider": "direct",
+ "enforcement": "none"
+ }
+ },
+ "max_iterations": 50,
+ "kind": "run",
+ "trigger": "user",
+ "started_at": 1793600000000,
+ "incomplete": false,
+ "first_message_id": "01JP2MSG000000000000000000",
+ "last_message_id": "01JP2MSG000000000000000000",
+ "schema_version": 1,
+ "duration_ms": 33
+ }
+ ],
+ "steps": [
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-07",
+ "parent_step_id": null,
+ "seq": 1,
+ "kind": "model",
+ "name": "model",
+ "started_at": 1793600000000,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": null,
+ "call_index": null,
+ "message_id": "01JP2MSG000000000000000010",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "purpose": "main",
+ "provider": "testprov",
+ "provider_title": "Test provider",
+ "model": "testprov:model-a",
+ "stream": true,
+ "finish_reason": "stop",
+ "ttft_ms": 5,
+ "context_window": 100000,
+ "usage": {
+ "reported": true,
+ "input_fresh": 1000,
+ "cache_read": 0,
+ "cache_write": null,
+ "output": 100,
+ "reasoning": null
+ },
+ "tools_offered": 3,
+ "max_output": 4096
+ },
+ "content": {}
+ },
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-07",
+ "parent_step_id": null,
+ "seq": 2,
+ "kind": "auto_approver",
+ "name": "auto_approver",
+ "started_at": 1793600000011,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": "call_3",
+ "call_index": null,
+ "message_id": null,
+ "shadow": true,
+ "after_turn": false,
+ "attrs": {
+ "reviewer": {
+ "kind": "session_model",
+ "model": "m",
+ "external": false
+ },
+ "mode": "shadow",
+ "result": "allow",
+ "score": null,
+ "cutoff": null,
+ "usage_reported": true,
+ "breaker": {
+ "denials_in_row": 0,
+ "tripped": false
+ },
+ "matched_user": false
+ },
+ "content": {}
+ },
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-07",
+ "parent_step_id": null,
+ "seq": 3,
+ "kind": "model",
+ "name": "model",
+ "started_at": 1793600000022,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": null,
+ "call_index": null,
+ "message_id": "01JP2MSG000000000000000010",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "purpose": "main",
+ "provider": "testprov",
+ "provider_title": "Test provider",
+ "model": "testprov:model-a",
+ "stream": true,
+ "finish_reason": "stop",
+ "ttft_ms": 5,
+ "context_window": 100000,
+ "usage": {
+ "reported": true,
+ "input_fresh": 1000,
+ "cache_read": 0,
+ "cache_write": null,
+ "output": 100,
+ "reasoning": null
+ },
+ "tools_offered": 3,
+ "max_output": 4096
+ },
+ "content": {}
+ }
+ ]
+}
diff --git a/tests/corpora/observability/golden-phase2/S2-08/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-08/expected-issues.json
new file mode 100644
index 00000000..271fef7b
--- /dev/null
+++ b/tests/corpora/observability/golden-phase2/S2-08/expected-issues.json
@@ -0,0 +1,31 @@
+{
+ "signals_version": "0.3",
+ "warn": 1,
+ "fired": {
+ "tool_failed": [
+ 2
+ ]
+ },
+ "fine": [
+ "context_full",
+ "cut_off",
+ "made_up_tool",
+ "model_failed",
+ "refused",
+ "repeated_step",
+ "retried",
+ "step_limit",
+ "waited_unwatched"
+ ],
+ "not_checked": [
+ "blocked_by_sandbox",
+ "off_plan",
+ "overridden",
+ "ran_without_sandbox",
+ "reviewer_allowed_send",
+ "reviewer_could_not_run",
+ "reviewer_paused",
+ "sandbox_failed",
+ "shadow_disagreed"
+ ]
+}
diff --git a/tests/corpora/observability/golden-phase2/S2-08/recording.json b/tests/corpora/observability/golden-phase2/S2-08/recording.json
new file mode 100644
index 00000000..e95f7e0e
--- /dev/null
+++ b/tests/corpora/observability/golden-phase2/S2-08/recording.json
@@ -0,0 +1,145 @@
+{
+ "turns": [
+ {
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-08",
+ "seq": 1,
+ "status": "completed",
+ "approval_mode": "interactive",
+ "attended": true,
+ "reviewer": {
+ "active": false,
+ "shadow": false,
+ "kind": null,
+ "model": null
+ },
+ "stamp": {
+ "sandbox": {
+ "provider": "direct",
+ "enforcement": "none"
+ }
+ },
+ "max_iterations": 50,
+ "kind": "run",
+ "trigger": "user",
+ "started_at": 1793600000000,
+ "incomplete": false,
+ "first_message_id": "01JP2MSG000000000000000000",
+ "last_message_id": "01JP2MSG000000000000000000",
+ "schema_version": 1,
+ "duration_ms": 33
+ }
+ ],
+ "steps": [
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-08",
+ "parent_step_id": null,
+ "seq": 1,
+ "kind": "model",
+ "name": "model",
+ "started_at": 1793600000000,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": null,
+ "call_index": null,
+ "message_id": "01JP2MSG000000000000000010",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "purpose": "main",
+ "provider": "testprov",
+ "provider_title": "Test provider",
+ "model": "testprov:model-a",
+ "stream": true,
+ "finish_reason": "stop",
+ "ttft_ms": 5,
+ "context_window": 100000,
+ "usage": {
+ "reported": true,
+ "input_fresh": 1000,
+ "cache_read": 0,
+ "cache_write": null,
+ "output": 100,
+ "reasoning": null
+ },
+ "tools_offered": 3,
+ "max_output": 4096
+ },
+ "content": {}
+ },
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-08",
+ "parent_step_id": null,
+ "seq": 2,
+ "kind": "tool",
+ "name": "run_shell",
+ "started_at": 1793600000011,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": "call_2",
+ "call_index": 0,
+ "message_id": "01JP2MSG000000000000000020",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "risk_class": "exec",
+ "decision": {
+ "path": [],
+ "origin": "low_risk"
+ },
+ "exit_code": 2,
+ "args_hash": "2bfd14",
+ "repeat_ok": false,
+ "result_error": false,
+ "error_kind": null,
+ "connector": null,
+ "result_bytes": 120,
+ "sandbox": null,
+ "reaches_network": false
+ },
+ "content": {}
+ },
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-08",
+ "parent_step_id": null,
+ "seq": 3,
+ "kind": "model",
+ "name": "model",
+ "started_at": 1793600000022,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": null,
+ "call_index": null,
+ "message_id": "01JP2MSG000000000000000010",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "purpose": "main",
+ "provider": "testprov",
+ "provider_title": "Test provider",
+ "model": "testprov:model-a",
+ "stream": true,
+ "finish_reason": "stop",
+ "ttft_ms": 5,
+ "context_window": 100000,
+ "usage": {
+ "reported": true,
+ "input_fresh": 1000,
+ "cache_read": 0,
+ "cache_write": null,
+ "output": 100,
+ "reasoning": null
+ },
+ "tools_offered": 3,
+ "max_output": 4096
+ },
+ "content": {}
+ }
+ ]
+}
diff --git a/tests/corpora/observability/golden-phase2/S2-08b/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-08b/expected-issues.json
new file mode 100644
index 00000000..271fef7b
--- /dev/null
+++ b/tests/corpora/observability/golden-phase2/S2-08b/expected-issues.json
@@ -0,0 +1,31 @@
+{
+ "signals_version": "0.3",
+ "warn": 1,
+ "fired": {
+ "tool_failed": [
+ 2
+ ]
+ },
+ "fine": [
+ "context_full",
+ "cut_off",
+ "made_up_tool",
+ "model_failed",
+ "refused",
+ "repeated_step",
+ "retried",
+ "step_limit",
+ "waited_unwatched"
+ ],
+ "not_checked": [
+ "blocked_by_sandbox",
+ "off_plan",
+ "overridden",
+ "ran_without_sandbox",
+ "reviewer_allowed_send",
+ "reviewer_could_not_run",
+ "reviewer_paused",
+ "sandbox_failed",
+ "shadow_disagreed"
+ ]
+}
diff --git a/tests/corpora/observability/golden-phase2/S2-08b/recording.json b/tests/corpora/observability/golden-phase2/S2-08b/recording.json
new file mode 100644
index 00000000..bf3fd34a
--- /dev/null
+++ b/tests/corpora/observability/golden-phase2/S2-08b/recording.json
@@ -0,0 +1,145 @@
+{
+ "turns": [
+ {
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-08b",
+ "seq": 1,
+ "status": "completed",
+ "approval_mode": "interactive",
+ "attended": true,
+ "reviewer": {
+ "active": false,
+ "shadow": false,
+ "kind": null,
+ "model": null
+ },
+ "stamp": {
+ "sandbox": {
+ "provider": "direct",
+ "enforcement": "none"
+ }
+ },
+ "max_iterations": 50,
+ "kind": "run",
+ "trigger": "user",
+ "started_at": 1793600000000,
+ "incomplete": false,
+ "first_message_id": "01JP2MSG000000000000000000",
+ "last_message_id": "01JP2MSG000000000000000000",
+ "schema_version": 1,
+ "duration_ms": 33
+ }
+ ],
+ "steps": [
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-08b",
+ "parent_step_id": null,
+ "seq": 1,
+ "kind": "model",
+ "name": "model",
+ "started_at": 1793600000000,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": null,
+ "call_index": null,
+ "message_id": "01JP2MSG000000000000000010",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "purpose": "main",
+ "provider": "testprov",
+ "provider_title": "Test provider",
+ "model": "testprov:model-a",
+ "stream": true,
+ "finish_reason": "stop",
+ "ttft_ms": 5,
+ "context_window": 100000,
+ "usage": {
+ "reported": true,
+ "input_fresh": 1000,
+ "cache_read": 0,
+ "cache_write": null,
+ "output": 100,
+ "reasoning": null
+ },
+ "tools_offered": 3,
+ "max_output": 4096
+ },
+ "content": {}
+ },
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-08b",
+ "parent_step_id": null,
+ "seq": 2,
+ "kind": "tool",
+ "name": "read_file",
+ "started_at": 1793600000011,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": "call_2",
+ "call_index": 0,
+ "message_id": "01JP2MSG000000000000000020",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "risk_class": "read",
+ "decision": {
+ "path": [],
+ "origin": "low_risk"
+ },
+ "args_hash": "2bfd14",
+ "repeat_ok": false,
+ "result_error": true,
+ "error_kind": null,
+ "connector": null,
+ "result_bytes": 120,
+ "exit_code": null,
+ "sandbox": null,
+ "reaches_network": false
+ },
+ "content": {}
+ },
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-08b",
+ "parent_step_id": null,
+ "seq": 3,
+ "kind": "model",
+ "name": "model",
+ "started_at": 1793600000022,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": null,
+ "call_index": null,
+ "message_id": "01JP2MSG000000000000000010",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "purpose": "main",
+ "provider": "testprov",
+ "provider_title": "Test provider",
+ "model": "testprov:model-a",
+ "stream": true,
+ "finish_reason": "stop",
+ "ttft_ms": 5,
+ "context_window": 100000,
+ "usage": {
+ "reported": true,
+ "input_fresh": 1000,
+ "cache_read": 0,
+ "cache_write": null,
+ "output": 100,
+ "reasoning": null
+ },
+ "tools_offered": 3,
+ "max_output": 4096
+ },
+ "content": {}
+ }
+ ]
+}
diff --git a/tests/corpora/observability/golden-phase2/S2-08c/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-08c/expected-issues.json
new file mode 100644
index 00000000..8d3822b3
--- /dev/null
+++ b/tests/corpora/observability/golden-phase2/S2-08c/expected-issues.json
@@ -0,0 +1,31 @@
+{
+ "signals_version": "0.3",
+ "warn": 1,
+ "fired": {
+ "made_up_tool": [
+ 3
+ ]
+ },
+ "fine": [
+ "context_full",
+ "cut_off",
+ "model_failed",
+ "refused",
+ "repeated_step",
+ "retried",
+ "step_limit",
+ "tool_failed",
+ "waited_unwatched"
+ ],
+ "not_checked": [
+ "blocked_by_sandbox",
+ "off_plan",
+ "overridden",
+ "ran_without_sandbox",
+ "reviewer_allowed_send",
+ "reviewer_could_not_run",
+ "reviewer_paused",
+ "sandbox_failed",
+ "shadow_disagreed"
+ ]
+}
diff --git a/tests/corpora/observability/golden-phase2/S2-08c/recording.json b/tests/corpora/observability/golden-phase2/S2-08c/recording.json
new file mode 100644
index 00000000..8c768aaa
--- /dev/null
+++ b/tests/corpora/observability/golden-phase2/S2-08c/recording.json
@@ -0,0 +1,179 @@
+{
+ "turns": [
+ {
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-08c",
+ "seq": 1,
+ "status": "completed",
+ "approval_mode": "interactive",
+ "attended": true,
+ "reviewer": {
+ "active": false,
+ "shadow": false,
+ "kind": null,
+ "model": null
+ },
+ "stamp": {
+ "sandbox": {
+ "provider": "direct",
+ "enforcement": "none"
+ }
+ },
+ "max_iterations": 50,
+ "kind": "run",
+ "trigger": "user",
+ "started_at": 1793600000000,
+ "incomplete": false,
+ "first_message_id": "01JP2MSG000000000000000000",
+ "last_message_id": "01JP2MSG000000000000000000",
+ "schema_version": 1,
+ "duration_ms": 34
+ }
+ ],
+ "steps": [
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-08c",
+ "parent_step_id": null,
+ "seq": 1,
+ "kind": "model",
+ "name": "model",
+ "started_at": 1793600000000,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": null,
+ "call_index": null,
+ "message_id": "01JP2MSG000000000000000010",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "purpose": "main",
+ "provider": "testprov",
+ "provider_title": "Test provider",
+ "model": "testprov:model-a",
+ "stream": true,
+ "finish_reason": "stop",
+ "ttft_ms": 5,
+ "context_window": 100000,
+ "usage": {
+ "reported": true,
+ "input_fresh": 1000,
+ "cache_read": 0,
+ "cache_write": null,
+ "output": 100,
+ "reasoning": null
+ },
+ "tools_offered": 3,
+ "max_output": 4096
+ },
+ "content": {}
+ },
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-08c",
+ "parent_step_id": null,
+ "seq": 2,
+ "kind": "tool",
+ "name": "run_shell",
+ "started_at": 1793600000011,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": "call_2",
+ "call_index": 0,
+ "message_id": "01JP2MSG000000000000000020",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "risk_class": "exec",
+ "decision": {
+ "path": [],
+ "origin": "low_risk"
+ },
+ "exit_code": 0,
+ "args_hash": "2bfd14",
+ "repeat_ok": false,
+ "result_error": false,
+ "error_kind": null,
+ "connector": null,
+ "result_bytes": 120,
+ "sandbox": null,
+ "reaches_network": false
+ },
+ "content": {}
+ },
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-08c",
+ "parent_step_id": null,
+ "seq": 3,
+ "kind": "tool",
+ "name": "nope",
+ "started_at": 1793600000022,
+ "duration_ms": 0,
+ "status": "error",
+ "call_id": "call_3",
+ "call_index": 0,
+ "message_id": "01JP2MSG000000000000000020",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "risk_class": "read",
+ "decision": {
+ "path": [],
+ "origin": "low_risk"
+ },
+ "args_hash": "363379",
+ "repeat_ok": false,
+ "result_error": false,
+ "error_kind": "unknown_tool",
+ "connector": null,
+ "result_bytes": null,
+ "exit_code": null,
+ "sandbox": null,
+ "reaches_network": false
+ },
+ "content": {}
+ },
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-08c",
+ "parent_step_id": null,
+ "seq": 4,
+ "kind": "model",
+ "name": "model",
+ "started_at": 1793600000023,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": null,
+ "call_index": null,
+ "message_id": "01JP2MSG000000000000000010",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "purpose": "main",
+ "provider": "testprov",
+ "provider_title": "Test provider",
+ "model": "testprov:model-a",
+ "stream": true,
+ "finish_reason": "stop",
+ "ttft_ms": 5,
+ "context_window": 100000,
+ "usage": {
+ "reported": true,
+ "input_fresh": 1000,
+ "cache_read": 0,
+ "cache_write": null,
+ "output": 100,
+ "reasoning": null
+ },
+ "tools_offered": 3,
+ "max_output": 4096
+ },
+ "content": {}
+ }
+ ]
+}
diff --git a/tests/corpora/observability/golden-phase2/S2-09/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-09/expected-issues.json
new file mode 100644
index 00000000..7c7c4519
--- /dev/null
+++ b/tests/corpora/observability/golden-phase2/S2-09/expected-issues.json
@@ -0,0 +1,31 @@
+{
+ "signals_version": "0.3",
+ "warn": 1,
+ "fired": {
+ "model_failed": [
+ 1
+ ]
+ },
+ "fine": [
+ "context_full",
+ "cut_off",
+ "made_up_tool",
+ "refused",
+ "repeated_step",
+ "retried",
+ "step_limit",
+ "tool_failed",
+ "waited_unwatched"
+ ],
+ "not_checked": [
+ "blocked_by_sandbox",
+ "off_plan",
+ "overridden",
+ "ran_without_sandbox",
+ "reviewer_allowed_send",
+ "reviewer_could_not_run",
+ "reviewer_paused",
+ "sandbox_failed",
+ "shadow_disagreed"
+ ]
+}
diff --git a/tests/corpora/observability/golden-phase2/S2-09/recording.json b/tests/corpora/observability/golden-phase2/S2-09/recording.json
new file mode 100644
index 00000000..1ca390d9
--- /dev/null
+++ b/tests/corpora/observability/golden-phase2/S2-09/recording.json
@@ -0,0 +1,91 @@
+{
+ "turns": [
+ {
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-09",
+ "seq": 1,
+ "status": "error",
+ "approval_mode": "interactive",
+ "attended": true,
+ "reviewer": {
+ "active": false,
+ "shadow": false,
+ "kind": null,
+ "model": null
+ },
+ "stamp": {
+ "sandbox": {
+ "provider": "direct",
+ "enforcement": "none"
+ }
+ },
+ "max_iterations": 50,
+ "kind": "run",
+ "trigger": "user",
+ "started_at": 1793600000000,
+ "incomplete": false,
+ "first_message_id": "01JP2MSG000000000000000000",
+ "last_message_id": "01JP2MSG000000000000000000",
+ "schema_version": 1,
+ "duration_ms": 12
+ }
+ ],
+ "steps": [
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-09",
+ "parent_step_id": null,
+ "seq": 1,
+ "kind": "model",
+ "name": "model",
+ "started_at": 1793600000000,
+ "duration_ms": 10,
+ "status": "error",
+ "call_id": null,
+ "call_index": null,
+ "message_id": null,
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "purpose": "main",
+ "provider": "testprov",
+ "provider_title": "Test provider",
+ "model": "testprov:model-a",
+ "stream": true,
+ "finish_reason": null,
+ "ttft_ms": null,
+ "context_window": 100000,
+ "usage": {
+ "reported": false
+ },
+ "tools_offered": 3,
+ "max_output": 4096,
+ "error": {
+ "type": "auth",
+ "text": null
+ }
+ },
+ "content": {}
+ },
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-09",
+ "parent_step_id": null,
+ "seq": 2,
+ "kind": "notice",
+ "name": "error",
+ "started_at": 1793600000011,
+ "duration_ms": 0,
+ "status": "ok",
+ "call_id": null,
+ "call_index": null,
+ "message_id": null,
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {},
+ "content": {}
+ }
+ ]
+}
diff --git a/tests/corpora/observability/golden-phase2/S2-09c/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-09c/expected-issues.json
new file mode 100644
index 00000000..49ff0804
--- /dev/null
+++ b/tests/corpora/observability/golden-phase2/S2-09c/expected-issues.json
@@ -0,0 +1,28 @@
+{
+ "signals_version": "0.3",
+ "warn": 0,
+ "fired": {},
+ "fine": [
+ "context_full",
+ "cut_off",
+ "made_up_tool",
+ "model_failed",
+ "refused",
+ "repeated_step",
+ "retried",
+ "step_limit",
+ "tool_failed",
+ "waited_unwatched"
+ ],
+ "not_checked": [
+ "blocked_by_sandbox",
+ "off_plan",
+ "overridden",
+ "ran_without_sandbox",
+ "reviewer_allowed_send",
+ "reviewer_could_not_run",
+ "reviewer_paused",
+ "sandbox_failed",
+ "shadow_disagreed"
+ ]
+}
diff --git a/tests/corpora/observability/golden-phase2/S2-09c/recording.json b/tests/corpora/observability/golden-phase2/S2-09c/recording.json
new file mode 100644
index 00000000..04188a33
--- /dev/null
+++ b/tests/corpora/observability/golden-phase2/S2-09c/recording.json
@@ -0,0 +1,110 @@
+{
+ "turns": [
+ {
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-09c",
+ "seq": 1,
+ "status": "completed",
+ "approval_mode": "interactive",
+ "attended": true,
+ "reviewer": {
+ "active": false,
+ "shadow": false,
+ "kind": null,
+ "model": null
+ },
+ "stamp": {
+ "sandbox": {
+ "provider": "direct",
+ "enforcement": "none"
+ }
+ },
+ "max_iterations": 50,
+ "kind": "run",
+ "trigger": "user",
+ "started_at": 1793600000000,
+ "incomplete": false,
+ "first_message_id": "01JP2MSG000000000000000000",
+ "last_message_id": "01JP2MSG000000000000000000",
+ "schema_version": 1,
+ "duration_ms": 22
+ }
+ ],
+ "steps": [
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-09c",
+ "parent_step_id": null,
+ "seq": 1,
+ "kind": "model",
+ "name": "model",
+ "started_at": 1793600000000,
+ "duration_ms": 10,
+ "status": "error",
+ "call_id": null,
+ "call_index": null,
+ "message_id": null,
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "purpose": "main",
+ "provider": "testprov",
+ "provider_title": "Test provider",
+ "model": "testprov:model-a",
+ "stream": true,
+ "finish_reason": null,
+ "ttft_ms": null,
+ "context_window": 100000,
+ "usage": {
+ "reported": false
+ },
+ "tools_offered": 3,
+ "max_output": 4096,
+ "error": {
+ "type": "context_length",
+ "text": null
+ }
+ },
+ "content": {}
+ },
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-09c",
+ "parent_step_id": null,
+ "seq": 2,
+ "kind": "model",
+ "name": "model",
+ "started_at": 1793600000011,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": null,
+ "call_index": null,
+ "message_id": "01JP2MSG000000000000000010",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "purpose": "main",
+ "provider": "testprov",
+ "provider_title": "Test provider",
+ "model": "testprov:model-a",
+ "stream": true,
+ "finish_reason": "stop",
+ "ttft_ms": 5,
+ "context_window": 100000,
+ "usage": {
+ "reported": true,
+ "input_fresh": 1000,
+ "cache_read": 0,
+ "cache_write": null,
+ "output": 100,
+ "reasoning": null
+ },
+ "tools_offered": 3,
+ "max_output": 4096
+ },
+ "content": {}
+ }
+ ]
+}
diff --git a/tests/corpora/observability/golden-phase2/S2-10/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-10/expected-issues.json
new file mode 100644
index 00000000..bf5dc859
--- /dev/null
+++ b/tests/corpora/observability/golden-phase2/S2-10/expected-issues.json
@@ -0,0 +1,31 @@
+{
+ "signals_version": "0.3",
+ "warn": 1,
+ "fired": {
+ "retried": [
+ 1
+ ]
+ },
+ "fine": [
+ "context_full",
+ "cut_off",
+ "made_up_tool",
+ "model_failed",
+ "refused",
+ "repeated_step",
+ "step_limit",
+ "tool_failed",
+ "waited_unwatched"
+ ],
+ "not_checked": [
+ "blocked_by_sandbox",
+ "off_plan",
+ "overridden",
+ "ran_without_sandbox",
+ "reviewer_allowed_send",
+ "reviewer_could_not_run",
+ "reviewer_paused",
+ "sandbox_failed",
+ "shadow_disagreed"
+ ]
+}
diff --git a/tests/corpora/observability/golden-phase2/S2-10/recording.json b/tests/corpora/observability/golden-phase2/S2-10/recording.json
new file mode 100644
index 00000000..0b98bff5
--- /dev/null
+++ b/tests/corpora/observability/golden-phase2/S2-10/recording.json
@@ -0,0 +1,95 @@
+{
+ "turns": [
+ {
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-10",
+ "seq": 1,
+ "status": "completed",
+ "approval_mode": "interactive",
+ "attended": true,
+ "reviewer": {
+ "active": false,
+ "shadow": false,
+ "kind": null,
+ "model": null
+ },
+ "stamp": {
+ "sandbox": {
+ "provider": "direct",
+ "enforcement": "none"
+ }
+ },
+ "max_iterations": 50,
+ "kind": "run",
+ "trigger": "user",
+ "started_at": 1793600000000,
+ "incomplete": false,
+ "first_message_id": "01JP2MSG000000000000000000",
+ "last_message_id": "01JP2MSG000000000000000000",
+ "schema_version": 1,
+ "duration_ms": 11
+ }
+ ],
+ "steps": [
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-10",
+ "parent_step_id": null,
+ "seq": 1,
+ "kind": "model",
+ "name": "model",
+ "started_at": 1793600000000,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": null,
+ "call_index": null,
+ "message_id": "01JP2MSG000000000000000010",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "purpose": "main",
+ "provider": "testprov",
+ "provider_title": "Test provider",
+ "model": "testprov:model-a",
+ "stream": true,
+ "finish_reason": "stop",
+ "ttft_ms": 5,
+ "context_window": 100000,
+ "usage": {
+ "reported": true,
+ "input_fresh": 1000,
+ "cache_read": 0,
+ "cache_write": null,
+ "output": 100,
+ "reasoning": null
+ },
+ "tools_offered": 3,
+ "max_output": 4096,
+ "provider_meta": [
+ {
+ "wire": "openai_chat",
+ "status": 529,
+ "request": {
+ "model": "model-a"
+ },
+ "headers": {},
+ "event_counts": {},
+ "events": {}
+ },
+ {
+ "wire": "openai_chat",
+ "status": 200,
+ "request": {
+ "model": "model-a"
+ },
+ "headers": {},
+ "event_counts": {},
+ "events": {}
+ }
+ ]
+ },
+ "content": {}
+ }
+ ]
+}
diff --git a/tests/corpora/observability/golden-phase2/S2-10b/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-10b/expected-issues.json
new file mode 100644
index 00000000..49ff0804
--- /dev/null
+++ b/tests/corpora/observability/golden-phase2/S2-10b/expected-issues.json
@@ -0,0 +1,28 @@
+{
+ "signals_version": "0.3",
+ "warn": 0,
+ "fired": {},
+ "fine": [
+ "context_full",
+ "cut_off",
+ "made_up_tool",
+ "model_failed",
+ "refused",
+ "repeated_step",
+ "retried",
+ "step_limit",
+ "tool_failed",
+ "waited_unwatched"
+ ],
+ "not_checked": [
+ "blocked_by_sandbox",
+ "off_plan",
+ "overridden",
+ "ran_without_sandbox",
+ "reviewer_allowed_send",
+ "reviewer_could_not_run",
+ "reviewer_paused",
+ "sandbox_failed",
+ "shadow_disagreed"
+ ]
+}
diff --git a/tests/corpora/observability/golden-phase2/S2-10b/recording.json b/tests/corpora/observability/golden-phase2/S2-10b/recording.json
new file mode 100644
index 00000000..1d27a523
--- /dev/null
+++ b/tests/corpora/observability/golden-phase2/S2-10b/recording.json
@@ -0,0 +1,133 @@
+{
+ "turns": [
+ {
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-10b",
+ "seq": 1,
+ "status": "completed",
+ "approval_mode": "interactive",
+ "attended": true,
+ "reviewer": {
+ "active": false,
+ "shadow": false,
+ "kind": null,
+ "model": null
+ },
+ "stamp": {
+ "sandbox": {
+ "provider": "direct",
+ "enforcement": "none"
+ }
+ },
+ "max_iterations": 50,
+ "kind": "run",
+ "trigger": "user",
+ "started_at": 1793600000000,
+ "incomplete": false,
+ "first_message_id": "01JP2MSG000000000000000000",
+ "last_message_id": "01JP2MSG000000000000000000",
+ "schema_version": 1,
+ "duration_ms": 22
+ }
+ ],
+ "steps": [
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-10b",
+ "parent_step_id": null,
+ "seq": 1,
+ "kind": "model",
+ "name": "model",
+ "started_at": 1793600000000,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": null,
+ "call_index": null,
+ "message_id": "01JP2MSG000000000000000010",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "purpose": "main",
+ "provider": "testprov",
+ "provider_title": "Test provider",
+ "model": "testprov:model-a",
+ "stream": true,
+ "finish_reason": "stop",
+ "ttft_ms": 5,
+ "context_window": 100000,
+ "usage": {
+ "reported": true,
+ "input_fresh": 1000,
+ "cache_read": 0,
+ "cache_write": null,
+ "output": 100,
+ "reasoning": null
+ },
+ "tools_offered": 3,
+ "max_output": 4096
+ },
+ "content": {}
+ },
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-10b",
+ "parent_step_id": null,
+ "seq": 2,
+ "kind": "model",
+ "name": "model",
+ "started_at": 1793600000011,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": null,
+ "call_index": null,
+ "message_id": "01JP2MSG000000000000000010",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "purpose": "title",
+ "provider": "testprov",
+ "provider_title": "Test provider",
+ "model": "testprov:model-a",
+ "stream": true,
+ "finish_reason": "stop",
+ "ttft_ms": 5,
+ "context_window": 100000,
+ "usage": {
+ "reported": true,
+ "input_fresh": 1000,
+ "cache_read": 0,
+ "cache_write": null,
+ "output": 100,
+ "reasoning": null
+ },
+ "tools_offered": 3,
+ "max_output": 4096,
+ "provider_meta": [
+ {
+ "wire": "openai_chat",
+ "status": 400,
+ "request": {
+ "model": "model-a"
+ },
+ "headers": {},
+ "event_counts": {},
+ "events": {}
+ },
+ {
+ "wire": "openai_chat",
+ "status": 200,
+ "request": {
+ "model": "model-a"
+ },
+ "headers": {},
+ "event_counts": {},
+ "events": {}
+ }
+ ]
+ },
+ "content": {}
+ }
+ ]
+}
diff --git a/tests/corpora/observability/golden-phase2/S2-11/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-11/expected-issues.json
new file mode 100644
index 00000000..0436ff42
--- /dev/null
+++ b/tests/corpora/observability/golden-phase2/S2-11/expected-issues.json
@@ -0,0 +1,31 @@
+{
+ "signals_version": "0.3",
+ "warn": 1,
+ "fired": {
+ "cut_off": [
+ 1
+ ]
+ },
+ "fine": [
+ "context_full",
+ "made_up_tool",
+ "model_failed",
+ "refused",
+ "repeated_step",
+ "retried",
+ "step_limit",
+ "tool_failed",
+ "waited_unwatched"
+ ],
+ "not_checked": [
+ "blocked_by_sandbox",
+ "off_plan",
+ "overridden",
+ "ran_without_sandbox",
+ "reviewer_allowed_send",
+ "reviewer_could_not_run",
+ "reviewer_paused",
+ "sandbox_failed",
+ "shadow_disagreed"
+ ]
+}
diff --git a/tests/corpora/observability/golden-phase2/S2-11/recording.json b/tests/corpora/observability/golden-phase2/S2-11/recording.json
new file mode 100644
index 00000000..71c5cf0a
--- /dev/null
+++ b/tests/corpora/observability/golden-phase2/S2-11/recording.json
@@ -0,0 +1,92 @@
+{
+ "turns": [
+ {
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-11",
+ "seq": 1,
+ "status": "truncated",
+ "approval_mode": "interactive",
+ "attended": true,
+ "reviewer": {
+ "active": false,
+ "shadow": false,
+ "kind": null,
+ "model": null
+ },
+ "stamp": {
+ "sandbox": {
+ "provider": "direct",
+ "enforcement": "none"
+ }
+ },
+ "max_iterations": 50,
+ "kind": "run",
+ "trigger": "user",
+ "started_at": 1793600000000,
+ "incomplete": false,
+ "first_message_id": "01JP2MSG000000000000000000",
+ "last_message_id": "01JP2MSG000000000000000000",
+ "schema_version": 1,
+ "duration_ms": 12
+ }
+ ],
+ "steps": [
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-11",
+ "parent_step_id": null,
+ "seq": 1,
+ "kind": "model",
+ "name": "model",
+ "started_at": 1793600000000,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": null,
+ "call_index": null,
+ "message_id": "01JP2MSG000000000000000010",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "purpose": "main",
+ "provider": "testprov",
+ "provider_title": "Test provider",
+ "model": "testprov:model-a",
+ "stream": true,
+ "finish_reason": "length",
+ "ttft_ms": 5,
+ "context_window": 100000,
+ "usage": {
+ "reported": true,
+ "input_fresh": 1000,
+ "cache_read": 0,
+ "cache_write": null,
+ "output": 100,
+ "reasoning": null
+ },
+ "tools_offered": 3,
+ "max_output": 4096
+ },
+ "content": {}
+ },
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-11",
+ "parent_step_id": null,
+ "seq": 2,
+ "kind": "notice",
+ "name": "truncated",
+ "started_at": 1793600000011,
+ "duration_ms": 0,
+ "status": "ok",
+ "call_id": null,
+ "call_index": null,
+ "message_id": null,
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {},
+ "content": {}
+ }
+ ]
+}
diff --git a/tests/corpora/observability/golden-phase2/S2-12/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-12/expected-issues.json
new file mode 100644
index 00000000..0ccfcaba
--- /dev/null
+++ b/tests/corpora/observability/golden-phase2/S2-12/expected-issues.json
@@ -0,0 +1,31 @@
+{
+ "signals_version": "0.3",
+ "warn": 1,
+ "fired": {
+ "step_limit": [
+ 4
+ ]
+ },
+ "fine": [
+ "context_full",
+ "cut_off",
+ "made_up_tool",
+ "model_failed",
+ "refused",
+ "repeated_step",
+ "retried",
+ "tool_failed",
+ "waited_unwatched"
+ ],
+ "not_checked": [
+ "blocked_by_sandbox",
+ "off_plan",
+ "overridden",
+ "ran_without_sandbox",
+ "reviewer_allowed_send",
+ "reviewer_could_not_run",
+ "reviewer_paused",
+ "sandbox_failed",
+ "shadow_disagreed"
+ ]
+}
diff --git a/tests/corpora/observability/golden-phase2/S2-12/recording.json b/tests/corpora/observability/golden-phase2/S2-12/recording.json
new file mode 100644
index 00000000..a24e5448
--- /dev/null
+++ b/tests/corpora/observability/golden-phase2/S2-12/recording.json
@@ -0,0 +1,164 @@
+{
+ "turns": [
+ {
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-12",
+ "seq": 1,
+ "status": "max_iterations_exceeded",
+ "approval_mode": "interactive",
+ "attended": true,
+ "reviewer": {
+ "active": false,
+ "shadow": false,
+ "kind": null,
+ "model": null
+ },
+ "stamp": {
+ "sandbox": {
+ "provider": "direct",
+ "enforcement": "none"
+ }
+ },
+ "max_iterations": 50,
+ "kind": "run",
+ "trigger": "user",
+ "started_at": 1793600000000,
+ "incomplete": false,
+ "first_message_id": "01JP2MSG000000000000000000",
+ "last_message_id": "01JP2MSG000000000000000000",
+ "schema_version": 1,
+ "duration_ms": 34
+ }
+ ],
+ "steps": [
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-12",
+ "parent_step_id": null,
+ "seq": 1,
+ "kind": "model",
+ "name": "model",
+ "started_at": 1793600000000,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": null,
+ "call_index": null,
+ "message_id": "01JP2MSG000000000000000010",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "purpose": "main",
+ "provider": "testprov",
+ "provider_title": "Test provider",
+ "model": "testprov:model-a",
+ "stream": true,
+ "finish_reason": "stop",
+ "ttft_ms": 5,
+ "context_window": 100000,
+ "usage": {
+ "reported": true,
+ "input_fresh": 1000,
+ "cache_read": 0,
+ "cache_write": null,
+ "output": 100,
+ "reasoning": null
+ },
+ "tools_offered": 3,
+ "max_output": 4096
+ },
+ "content": {}
+ },
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-12",
+ "parent_step_id": null,
+ "seq": 2,
+ "kind": "tool",
+ "name": "read_file",
+ "started_at": 1793600000011,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": "call_2",
+ "call_index": 0,
+ "message_id": "01JP2MSG000000000000000020",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "risk_class": "read",
+ "decision": {
+ "path": [],
+ "origin": "low_risk"
+ },
+ "args_hash": "2bfd14",
+ "repeat_ok": false,
+ "result_error": false,
+ "error_kind": null,
+ "connector": null,
+ "result_bytes": 120,
+ "exit_code": null,
+ "sandbox": null,
+ "reaches_network": false
+ },
+ "content": {}
+ },
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-12",
+ "parent_step_id": null,
+ "seq": 3,
+ "kind": "model",
+ "name": "model",
+ "started_at": 1793600000022,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": null,
+ "call_index": null,
+ "message_id": "01JP2MSG000000000000000010",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "purpose": "main",
+ "provider": "testprov",
+ "provider_title": "Test provider",
+ "model": "testprov:model-a",
+ "stream": true,
+ "finish_reason": "stop",
+ "ttft_ms": 5,
+ "context_window": 100000,
+ "usage": {
+ "reported": true,
+ "input_fresh": 1000,
+ "cache_read": 0,
+ "cache_write": null,
+ "output": 100,
+ "reasoning": null
+ },
+ "tools_offered": 3,
+ "max_output": 4096
+ },
+ "content": {}
+ },
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-12",
+ "parent_step_id": null,
+ "seq": 4,
+ "kind": "notice",
+ "name": "step_limit",
+ "started_at": 1793600000033,
+ "duration_ms": 0,
+ "status": "ok",
+ "call_id": null,
+ "call_index": null,
+ "message_id": null,
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {},
+ "content": {}
+ }
+ ]
+}
diff --git a/tests/corpora/observability/golden-phase2/S2-13/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-13/expected-issues.json
new file mode 100644
index 00000000..60324950
--- /dev/null
+++ b/tests/corpora/observability/golden-phase2/S2-13/expected-issues.json
@@ -0,0 +1,32 @@
+{
+ "signals_version": "0.3",
+ "warn": 2,
+ "fired": {
+ "made_up_tool": [
+ 2,
+ 3
+ ]
+ },
+ "fine": [
+ "context_full",
+ "cut_off",
+ "model_failed",
+ "refused",
+ "repeated_step",
+ "retried",
+ "step_limit",
+ "tool_failed",
+ "waited_unwatched"
+ ],
+ "not_checked": [
+ "blocked_by_sandbox",
+ "off_plan",
+ "overridden",
+ "ran_without_sandbox",
+ "reviewer_allowed_send",
+ "reviewer_could_not_run",
+ "reviewer_paused",
+ "sandbox_failed",
+ "shadow_disagreed"
+ ]
+}
diff --git a/tests/corpora/observability/golden-phase2/S2-13/recording.json b/tests/corpora/observability/golden-phase2/S2-13/recording.json
new file mode 100644
index 00000000..9f7c338c
--- /dev/null
+++ b/tests/corpora/observability/golden-phase2/S2-13/recording.json
@@ -0,0 +1,179 @@
+{
+ "turns": [
+ {
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-13",
+ "seq": 1,
+ "status": "completed",
+ "approval_mode": "interactive",
+ "attended": true,
+ "reviewer": {
+ "active": false,
+ "shadow": false,
+ "kind": null,
+ "model": null
+ },
+ "stamp": {
+ "sandbox": {
+ "provider": "direct",
+ "enforcement": "none"
+ }
+ },
+ "max_iterations": 50,
+ "kind": "run",
+ "trigger": "user",
+ "started_at": 1793600000000,
+ "incomplete": false,
+ "first_message_id": "01JP2MSG000000000000000000",
+ "last_message_id": "01JP2MSG000000000000000000",
+ "schema_version": 1,
+ "duration_ms": 24
+ }
+ ],
+ "steps": [
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-13",
+ "parent_step_id": null,
+ "seq": 1,
+ "kind": "model",
+ "name": "model",
+ "started_at": 1793600000000,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": null,
+ "call_index": null,
+ "message_id": "01JP2MSG000000000000000010",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "purpose": "main",
+ "provider": "testprov",
+ "provider_title": "Test provider",
+ "model": "testprov:model-a",
+ "stream": true,
+ "finish_reason": "stop",
+ "ttft_ms": 5,
+ "context_window": 100000,
+ "usage": {
+ "reported": true,
+ "input_fresh": 1000,
+ "cache_read": 0,
+ "cache_write": null,
+ "output": 100,
+ "reasoning": null
+ },
+ "tools_offered": 3,
+ "max_output": 4096
+ },
+ "content": {}
+ },
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-13",
+ "parent_step_id": null,
+ "seq": 2,
+ "kind": "tool",
+ "name": "search_web_pages",
+ "started_at": 1793600000011,
+ "duration_ms": 0,
+ "status": "error",
+ "call_id": "call_2",
+ "call_index": 0,
+ "message_id": "01JP2MSG000000000000000020",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "risk_class": "read",
+ "decision": {
+ "path": [],
+ "origin": "low_risk"
+ },
+ "args_hash": "2bfd14",
+ "repeat_ok": false,
+ "result_error": false,
+ "error_kind": "unknown_tool",
+ "connector": null,
+ "result_bytes": null,
+ "exit_code": null,
+ "sandbox": null,
+ "reaches_network": false
+ },
+ "content": {}
+ },
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-13",
+ "parent_step_id": null,
+ "seq": 3,
+ "kind": "tool",
+ "name": "write_file",
+ "started_at": 1793600000012,
+ "duration_ms": 0,
+ "status": "error",
+ "call_id": "call_3",
+ "call_index": 0,
+ "message_id": "01JP2MSG000000000000000020",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "risk_class": "read",
+ "decision": {
+ "path": [],
+ "origin": "low_risk"
+ },
+ "args_hash": "363379",
+ "repeat_ok": false,
+ "result_error": false,
+ "error_kind": "bad_arguments",
+ "connector": null,
+ "result_bytes": null,
+ "exit_code": null,
+ "sandbox": null,
+ "reaches_network": false
+ },
+ "content": {}
+ },
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-13",
+ "parent_step_id": null,
+ "seq": 4,
+ "kind": "model",
+ "name": "model",
+ "started_at": 1793600000013,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": null,
+ "call_index": null,
+ "message_id": "01JP2MSG000000000000000010",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "purpose": "main",
+ "provider": "testprov",
+ "provider_title": "Test provider",
+ "model": "testprov:model-a",
+ "stream": true,
+ "finish_reason": "stop",
+ "ttft_ms": 5,
+ "context_window": 100000,
+ "usage": {
+ "reported": true,
+ "input_fresh": 1000,
+ "cache_read": 0,
+ "cache_write": null,
+ "output": 100,
+ "reasoning": null
+ },
+ "tools_offered": 3,
+ "max_output": 4096
+ },
+ "content": {}
+ }
+ ]
+}
diff --git a/tests/corpora/observability/golden-phase2/S2-14/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-14/expected-issues.json
new file mode 100644
index 00000000..2407cc96
--- /dev/null
+++ b/tests/corpora/observability/golden-phase2/S2-14/expected-issues.json
@@ -0,0 +1,31 @@
+{
+ "signals_version": "0.3",
+ "warn": 1,
+ "fired": {
+ "context_full": [
+ 2
+ ]
+ },
+ "fine": [
+ "cut_off",
+ "made_up_tool",
+ "model_failed",
+ "refused",
+ "repeated_step",
+ "retried",
+ "step_limit",
+ "tool_failed",
+ "waited_unwatched"
+ ],
+ "not_checked": [
+ "blocked_by_sandbox",
+ "off_plan",
+ "overridden",
+ "ran_without_sandbox",
+ "reviewer_allowed_send",
+ "reviewer_could_not_run",
+ "reviewer_paused",
+ "sandbox_failed",
+ "shadow_disagreed"
+ ]
+}
diff --git a/tests/corpora/observability/golden-phase2/S2-14/recording.json b/tests/corpora/observability/golden-phase2/S2-14/recording.json
new file mode 100644
index 00000000..b0e69937
--- /dev/null
+++ b/tests/corpora/observability/golden-phase2/S2-14/recording.json
@@ -0,0 +1,135 @@
+{
+ "turns": [
+ {
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-14",
+ "seq": 1,
+ "status": "completed",
+ "approval_mode": "interactive",
+ "attended": true,
+ "reviewer": {
+ "active": false,
+ "shadow": false,
+ "kind": null,
+ "model": null
+ },
+ "stamp": {
+ "sandbox": {
+ "provider": "direct",
+ "enforcement": "none"
+ }
+ },
+ "max_iterations": 50,
+ "kind": "run",
+ "trigger": "user",
+ "started_at": 1793600000000,
+ "incomplete": false,
+ "first_message_id": "01JP2MSG000000000000000000",
+ "last_message_id": "01JP2MSG000000000000000000",
+ "schema_version": 1,
+ "duration_ms": 33
+ }
+ ],
+ "steps": [
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-14",
+ "parent_step_id": null,
+ "seq": 1,
+ "kind": "model",
+ "name": "model",
+ "started_at": 1793600000000,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": null,
+ "call_index": null,
+ "message_id": "01JP2MSG000000000000000010",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "purpose": "main",
+ "provider": "testprov",
+ "provider_title": "Test provider",
+ "model": "testprov:model-a",
+ "stream": true,
+ "finish_reason": "stop",
+ "ttft_ms": 5,
+ "context_window": 100000,
+ "usage": {
+ "reported": true,
+ "input_fresh": 1000,
+ "cache_read": 0,
+ "cache_write": null,
+ "output": 100,
+ "reasoning": null
+ },
+ "tools_offered": 3,
+ "max_output": 4096
+ },
+ "content": {}
+ },
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-14",
+ "parent_step_id": null,
+ "seq": 2,
+ "kind": "summary",
+ "name": "summary",
+ "started_at": 1793600000011,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": null,
+ "call_index": null,
+ "message_id": null,
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "reason": "overflow",
+ "tokens_before": 80000,
+ "tokens_after": 12000,
+ "model_step_id": null
+ },
+ "content": {}
+ },
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-14",
+ "parent_step_id": null,
+ "seq": 3,
+ "kind": "model",
+ "name": "model",
+ "started_at": 1793600000022,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": null,
+ "call_index": null,
+ "message_id": "01JP2MSG000000000000000010",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "purpose": "main",
+ "provider": "testprov",
+ "provider_title": "Test provider",
+ "model": "testprov:model-a",
+ "stream": true,
+ "finish_reason": "stop",
+ "ttft_ms": 5,
+ "context_window": 100000,
+ "usage": {
+ "reported": true,
+ "input_fresh": 1000,
+ "cache_read": 0,
+ "cache_write": null,
+ "output": 100,
+ "reasoning": null
+ },
+ "tools_offered": 3,
+ "max_output": 4096
+ },
+ "content": {}
+ }
+ ]
+}
diff --git a/tests/corpora/observability/golden-phase2/S2-14b/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-14b/expected-issues.json
new file mode 100644
index 00000000..49ff0804
--- /dev/null
+++ b/tests/corpora/observability/golden-phase2/S2-14b/expected-issues.json
@@ -0,0 +1,28 @@
+{
+ "signals_version": "0.3",
+ "warn": 0,
+ "fired": {},
+ "fine": [
+ "context_full",
+ "cut_off",
+ "made_up_tool",
+ "model_failed",
+ "refused",
+ "repeated_step",
+ "retried",
+ "step_limit",
+ "tool_failed",
+ "waited_unwatched"
+ ],
+ "not_checked": [
+ "blocked_by_sandbox",
+ "off_plan",
+ "overridden",
+ "ran_without_sandbox",
+ "reviewer_allowed_send",
+ "reviewer_could_not_run",
+ "reviewer_paused",
+ "sandbox_failed",
+ "shadow_disagreed"
+ ]
+}
diff --git a/tests/corpora/observability/golden-phase2/S2-14b/recording.json b/tests/corpora/observability/golden-phase2/S2-14b/recording.json
new file mode 100644
index 00000000..c18557dd
--- /dev/null
+++ b/tests/corpora/observability/golden-phase2/S2-14b/recording.json
@@ -0,0 +1,135 @@
+{
+ "turns": [
+ {
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-14b",
+ "seq": 1,
+ "status": "completed",
+ "approval_mode": "interactive",
+ "attended": true,
+ "reviewer": {
+ "active": false,
+ "shadow": false,
+ "kind": null,
+ "model": null
+ },
+ "stamp": {
+ "sandbox": {
+ "provider": "direct",
+ "enforcement": "none"
+ }
+ },
+ "max_iterations": 50,
+ "kind": "run",
+ "trigger": "user",
+ "started_at": 1793600000000,
+ "incomplete": false,
+ "first_message_id": "01JP2MSG000000000000000000",
+ "last_message_id": "01JP2MSG000000000000000000",
+ "schema_version": 1,
+ "duration_ms": 33
+ }
+ ],
+ "steps": [
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-14b",
+ "parent_step_id": null,
+ "seq": 1,
+ "kind": "model",
+ "name": "model",
+ "started_at": 1793600000000,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": null,
+ "call_index": null,
+ "message_id": "01JP2MSG000000000000000010",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "purpose": "main",
+ "provider": "testprov",
+ "provider_title": "Test provider",
+ "model": "testprov:model-a",
+ "stream": true,
+ "finish_reason": "stop",
+ "ttft_ms": 5,
+ "context_window": 100000,
+ "usage": {
+ "reported": true,
+ "input_fresh": 79000,
+ "cache_read": 0,
+ "cache_write": null,
+ "output": 100,
+ "reasoning": null
+ },
+ "tools_offered": 3,
+ "max_output": 4096
+ },
+ "content": {}
+ },
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-14b",
+ "parent_step_id": null,
+ "seq": 2,
+ "kind": "summary",
+ "name": "summary",
+ "started_at": 1793600000011,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": null,
+ "call_index": null,
+ "message_id": null,
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "reason": "threshold",
+ "tokens_before": 80000,
+ "tokens_after": 12000,
+ "model_step_id": null
+ },
+ "content": {}
+ },
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-14b",
+ "parent_step_id": null,
+ "seq": 3,
+ "kind": "model",
+ "name": "model",
+ "started_at": 1793600000022,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": null,
+ "call_index": null,
+ "message_id": "01JP2MSG000000000000000010",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "purpose": "main",
+ "provider": "testprov",
+ "provider_title": "Test provider",
+ "model": "testprov:model-a",
+ "stream": true,
+ "finish_reason": "stop",
+ "ttft_ms": 5,
+ "context_window": 100000,
+ "usage": {
+ "reported": true,
+ "input_fresh": 1000,
+ "cache_read": 0,
+ "cache_write": null,
+ "output": 100,
+ "reasoning": null
+ },
+ "tools_offered": 3,
+ "max_output": 4096
+ },
+ "content": {}
+ }
+ ]
+}
diff --git a/tests/corpora/observability/golden-phase2/S2-14d/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-14d/expected-issues.json
new file mode 100644
index 00000000..d0ab7d0a
--- /dev/null
+++ b/tests/corpora/observability/golden-phase2/S2-14d/expected-issues.json
@@ -0,0 +1,31 @@
+{
+ "signals_version": "0.3",
+ "warn": 1,
+ "fired": {
+ "context_full": [
+ 1
+ ]
+ },
+ "fine": [
+ "cut_off",
+ "made_up_tool",
+ "model_failed",
+ "refused",
+ "repeated_step",
+ "retried",
+ "step_limit",
+ "tool_failed",
+ "waited_unwatched"
+ ],
+ "not_checked": [
+ "blocked_by_sandbox",
+ "off_plan",
+ "overridden",
+ "ran_without_sandbox",
+ "reviewer_allowed_send",
+ "reviewer_could_not_run",
+ "reviewer_paused",
+ "sandbox_failed",
+ "shadow_disagreed"
+ ]
+}
diff --git a/tests/corpora/observability/golden-phase2/S2-14d/recording.json b/tests/corpora/observability/golden-phase2/S2-14d/recording.json
new file mode 100644
index 00000000..7b2a3ec7
--- /dev/null
+++ b/tests/corpora/observability/golden-phase2/S2-14d/recording.json
@@ -0,0 +1,73 @@
+{
+ "turns": [
+ {
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-14d",
+ "seq": 1,
+ "status": "completed",
+ "approval_mode": "interactive",
+ "attended": true,
+ "reviewer": {
+ "active": false,
+ "shadow": false,
+ "kind": null,
+ "model": null
+ },
+ "stamp": {
+ "sandbox": {
+ "provider": "direct",
+ "enforcement": "none"
+ }
+ },
+ "max_iterations": 50,
+ "kind": "run",
+ "trigger": "user",
+ "started_at": 1793600000000,
+ "incomplete": false,
+ "first_message_id": "01JP2MSG000000000000000000",
+ "last_message_id": "01JP2MSG000000000000000000",
+ "schema_version": 1,
+ "duration_ms": 11
+ }
+ ],
+ "steps": [
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-14d",
+ "parent_step_id": null,
+ "seq": 1,
+ "kind": "model",
+ "name": "model",
+ "started_at": 1793600000000,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": null,
+ "call_index": null,
+ "message_id": "01JP2MSG000000000000000010",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "purpose": "main",
+ "provider": "testprov",
+ "provider_title": "Test provider",
+ "model": "testprov:model-a",
+ "stream": true,
+ "finish_reason": "stop",
+ "ttft_ms": 5,
+ "context_window": 100000,
+ "usage": {
+ "reported": true,
+ "input_fresh": 20000,
+ "cache_read": 76000,
+ "cache_write": null,
+ "output": 100,
+ "reasoning": null
+ },
+ "tools_offered": 3,
+ "max_output": 4096
+ },
+ "content": {}
+ }
+ ]
+}
diff --git a/tests/corpora/observability/golden-phase2/S2-15/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-15/expected-issues.json
new file mode 100644
index 00000000..b1634142
--- /dev/null
+++ b/tests/corpora/observability/golden-phase2/S2-15/expected-issues.json
@@ -0,0 +1,31 @@
+{
+ "signals_version": "0.3",
+ "warn": 1,
+ "fired": {
+ "blocked_by_sandbox": [
+ 2
+ ]
+ },
+ "fine": [
+ "context_full",
+ "cut_off",
+ "made_up_tool",
+ "model_failed",
+ "refused",
+ "repeated_step",
+ "retried",
+ "step_limit",
+ "tool_failed",
+ "waited_unwatched"
+ ],
+ "not_checked": [
+ "off_plan",
+ "overridden",
+ "ran_without_sandbox",
+ "reviewer_allowed_send",
+ "reviewer_could_not_run",
+ "reviewer_paused",
+ "sandbox_failed",
+ "shadow_disagreed"
+ ]
+}
diff --git a/tests/corpora/observability/golden-phase2/S2-15/recording.json b/tests/corpora/observability/golden-phase2/S2-15/recording.json
new file mode 100644
index 00000000..53688fc1
--- /dev/null
+++ b/tests/corpora/observability/golden-phase2/S2-15/recording.json
@@ -0,0 +1,150 @@
+{
+ "turns": [
+ {
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-15",
+ "seq": 1,
+ "status": "completed",
+ "approval_mode": "interactive",
+ "attended": true,
+ "reviewer": {
+ "active": false,
+ "shadow": false,
+ "kind": null,
+ "model": null
+ },
+ "stamp": {
+ "sandbox": {
+ "provider": "openshell",
+ "enforcement": "full",
+ "profile": "allowlist"
+ }
+ },
+ "max_iterations": 50,
+ "kind": "run",
+ "trigger": "user",
+ "started_at": 1793600000000,
+ "incomplete": false,
+ "first_message_id": "01JP2MSG000000000000000000",
+ "last_message_id": "01JP2MSG000000000000000000",
+ "schema_version": 1,
+ "duration_ms": 33
+ }
+ ],
+ "steps": [
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-15",
+ "parent_step_id": null,
+ "seq": 1,
+ "kind": "model",
+ "name": "model",
+ "started_at": 1793600000000,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": null,
+ "call_index": null,
+ "message_id": "01JP2MSG000000000000000010",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "purpose": "main",
+ "provider": "testprov",
+ "provider_title": "Test provider",
+ "model": "testprov:model-a",
+ "stream": true,
+ "finish_reason": "stop",
+ "ttft_ms": 5,
+ "context_window": 100000,
+ "usage": {
+ "reported": true,
+ "input_fresh": 1000,
+ "cache_read": 0,
+ "cache_write": null,
+ "output": 100,
+ "reasoning": null
+ },
+ "tools_offered": 3,
+ "max_output": 4096
+ },
+ "content": {}
+ },
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-15",
+ "parent_step_id": null,
+ "seq": 2,
+ "kind": "tool",
+ "name": "run_shell",
+ "started_at": 1793600000011,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": "call_2",
+ "call_index": 0,
+ "message_id": "01JP2MSG000000000000000020",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "risk_class": "exec",
+ "decision": {
+ "path": [],
+ "origin": "low_risk"
+ },
+ "exit_code": 6,
+ "sandbox": {
+ "blocked_known": true,
+ "blocked_count": 2,
+ "notice": null
+ },
+ "args_hash": "2bfd14",
+ "repeat_ok": false,
+ "result_error": false,
+ "error_kind": null,
+ "connector": null,
+ "result_bytes": 120,
+ "reaches_network": false
+ },
+ "content": {}
+ },
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-15",
+ "parent_step_id": null,
+ "seq": 3,
+ "kind": "model",
+ "name": "model",
+ "started_at": 1793600000022,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": null,
+ "call_index": null,
+ "message_id": "01JP2MSG000000000000000010",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "purpose": "main",
+ "provider": "testprov",
+ "provider_title": "Test provider",
+ "model": "testprov:model-a",
+ "stream": true,
+ "finish_reason": "stop",
+ "ttft_ms": 5,
+ "context_window": 100000,
+ "usage": {
+ "reported": true,
+ "input_fresh": 1000,
+ "cache_read": 0,
+ "cache_write": null,
+ "output": 100,
+ "reasoning": null
+ },
+ "tools_offered": 3,
+ "max_output": 4096
+ },
+ "content": {}
+ }
+ ]
+}
diff --git a/tests/corpora/observability/golden-phase2/S2-15c/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-15c/expected-issues.json
new file mode 100644
index 00000000..b4d8c189
--- /dev/null
+++ b/tests/corpora/observability/golden-phase2/S2-15c/expected-issues.json
@@ -0,0 +1,28 @@
+{
+ "signals_version": "0.3",
+ "warn": 0,
+ "fired": {},
+ "fine": [
+ "blocked_by_sandbox",
+ "context_full",
+ "cut_off",
+ "made_up_tool",
+ "model_failed",
+ "refused",
+ "repeated_step",
+ "retried",
+ "step_limit",
+ "tool_failed",
+ "waited_unwatched"
+ ],
+ "not_checked": [
+ "off_plan",
+ "overridden",
+ "ran_without_sandbox",
+ "reviewer_allowed_send",
+ "reviewer_could_not_run",
+ "reviewer_paused",
+ "sandbox_failed",
+ "shadow_disagreed"
+ ]
+}
diff --git a/tests/corpora/observability/golden-phase2/S2-15c/recording.json b/tests/corpora/observability/golden-phase2/S2-15c/recording.json
new file mode 100644
index 00000000..1676e377
--- /dev/null
+++ b/tests/corpora/observability/golden-phase2/S2-15c/recording.json
@@ -0,0 +1,149 @@
+{
+ "turns": [
+ {
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-15c",
+ "seq": 1,
+ "status": "completed",
+ "approval_mode": "interactive",
+ "attended": true,
+ "reviewer": {
+ "active": false,
+ "shadow": false,
+ "kind": null,
+ "model": null
+ },
+ "stamp": {
+ "sandbox": {
+ "provider": "openshell",
+ "enforcement": "full",
+ "profile": "allowlist"
+ }
+ },
+ "max_iterations": 50,
+ "kind": "run",
+ "trigger": "user",
+ "started_at": 1793600000000,
+ "incomplete": false,
+ "first_message_id": "01JP2MSG000000000000000000",
+ "last_message_id": "01JP2MSG000000000000000000",
+ "schema_version": 1,
+ "duration_ms": 33
+ }
+ ],
+ "steps": [
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-15c",
+ "parent_step_id": null,
+ "seq": 1,
+ "kind": "model",
+ "name": "model",
+ "started_at": 1793600000000,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": null,
+ "call_index": null,
+ "message_id": "01JP2MSG000000000000000010",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "purpose": "main",
+ "provider": "testprov",
+ "provider_title": "Test provider",
+ "model": "testprov:model-a",
+ "stream": true,
+ "finish_reason": "stop",
+ "ttft_ms": 5,
+ "context_window": 100000,
+ "usage": {
+ "reported": true,
+ "input_fresh": 1000,
+ "cache_read": 0,
+ "cache_write": null,
+ "output": 100,
+ "reasoning": null
+ },
+ "tools_offered": 3,
+ "max_output": 4096
+ },
+ "content": {}
+ },
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-15c",
+ "parent_step_id": null,
+ "seq": 2,
+ "kind": "tool",
+ "name": "run_shell",
+ "started_at": 1793600000011,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": "call_2",
+ "call_index": 0,
+ "message_id": "01JP2MSG000000000000000020",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "risk_class": "exec",
+ "decision": {
+ "path": [],
+ "origin": "low_risk"
+ },
+ "sandbox": {
+ "blocked_known": true,
+ "blocked_count": 0
+ },
+ "args_hash": "2bfd14",
+ "repeat_ok": false,
+ "result_error": false,
+ "error_kind": null,
+ "connector": null,
+ "result_bytes": 120,
+ "exit_code": null,
+ "reaches_network": false
+ },
+ "content": {}
+ },
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-15c",
+ "parent_step_id": null,
+ "seq": 3,
+ "kind": "model",
+ "name": "model",
+ "started_at": 1793600000022,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": null,
+ "call_index": null,
+ "message_id": "01JP2MSG000000000000000010",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "purpose": "main",
+ "provider": "testprov",
+ "provider_title": "Test provider",
+ "model": "testprov:model-a",
+ "stream": true,
+ "finish_reason": "stop",
+ "ttft_ms": 5,
+ "context_window": 100000,
+ "usage": {
+ "reported": true,
+ "input_fresh": 1000,
+ "cache_read": 0,
+ "cache_write": null,
+ "output": 100,
+ "reasoning": null
+ },
+ "tools_offered": 3,
+ "max_output": 4096
+ },
+ "content": {}
+ }
+ ]
+}
diff --git a/tests/corpora/observability/golden-phase2/S2-16/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-16/expected-issues.json
new file mode 100644
index 00000000..cea4e470
--- /dev/null
+++ b/tests/corpora/observability/golden-phase2/S2-16/expected-issues.json
@@ -0,0 +1,31 @@
+{
+ "signals_version": "0.3",
+ "warn": 1,
+ "fired": {
+ "ran_without_sandbox": [
+ 2
+ ]
+ },
+ "fine": [
+ "context_full",
+ "cut_off",
+ "made_up_tool",
+ "model_failed",
+ "refused",
+ "repeated_step",
+ "retried",
+ "step_limit",
+ "tool_failed",
+ "waited_unwatched"
+ ],
+ "not_checked": [
+ "blocked_by_sandbox",
+ "off_plan",
+ "overridden",
+ "reviewer_allowed_send",
+ "reviewer_could_not_run",
+ "reviewer_paused",
+ "sandbox_failed",
+ "shadow_disagreed"
+ ]
+}
diff --git a/tests/corpora/observability/golden-phase2/S2-16/recording.json b/tests/corpora/observability/golden-phase2/S2-16/recording.json
new file mode 100644
index 00000000..9e9bd0ff
--- /dev/null
+++ b/tests/corpora/observability/golden-phase2/S2-16/recording.json
@@ -0,0 +1,146 @@
+{
+ "turns": [
+ {
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-16",
+ "seq": 1,
+ "status": "completed",
+ "approval_mode": "interactive",
+ "attended": true,
+ "reviewer": {
+ "active": false,
+ "shadow": false,
+ "kind": null,
+ "model": null
+ },
+ "stamp": {
+ "sandbox": {
+ "provider": "direct",
+ "enforcement": "none",
+ "expected": true
+ }
+ },
+ "max_iterations": 50,
+ "kind": "run",
+ "trigger": "user",
+ "started_at": 1793600000000,
+ "incomplete": false,
+ "first_message_id": "01JP2MSG000000000000000000",
+ "last_message_id": "01JP2MSG000000000000000000",
+ "schema_version": 1,
+ "duration_ms": 33
+ }
+ ],
+ "steps": [
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-16",
+ "parent_step_id": null,
+ "seq": 1,
+ "kind": "model",
+ "name": "model",
+ "started_at": 1793600000000,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": null,
+ "call_index": null,
+ "message_id": "01JP2MSG000000000000000010",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "purpose": "main",
+ "provider": "testprov",
+ "provider_title": "Test provider",
+ "model": "testprov:model-a",
+ "stream": true,
+ "finish_reason": "stop",
+ "ttft_ms": 5,
+ "context_window": 100000,
+ "usage": {
+ "reported": true,
+ "input_fresh": 1000,
+ "cache_read": 0,
+ "cache_write": null,
+ "output": 100,
+ "reasoning": null
+ },
+ "tools_offered": 3,
+ "max_output": 4096
+ },
+ "content": {}
+ },
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-16",
+ "parent_step_id": null,
+ "seq": 2,
+ "kind": "tool",
+ "name": "run_shell",
+ "started_at": 1793600000011,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": "call_2",
+ "call_index": 0,
+ "message_id": "01JP2MSG000000000000000020",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "risk_class": "exec",
+ "decision": {
+ "path": [],
+ "origin": "low_risk"
+ },
+ "args_hash": "2bfd14",
+ "repeat_ok": false,
+ "result_error": false,
+ "error_kind": null,
+ "connector": null,
+ "result_bytes": 120,
+ "exit_code": null,
+ "sandbox": null,
+ "reaches_network": false
+ },
+ "content": {}
+ },
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-16",
+ "parent_step_id": null,
+ "seq": 3,
+ "kind": "model",
+ "name": "model",
+ "started_at": 1793600000022,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": null,
+ "call_index": null,
+ "message_id": "01JP2MSG000000000000000010",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "purpose": "main",
+ "provider": "testprov",
+ "provider_title": "Test provider",
+ "model": "testprov:model-a",
+ "stream": true,
+ "finish_reason": "stop",
+ "ttft_ms": 5,
+ "context_window": 100000,
+ "usage": {
+ "reported": true,
+ "input_fresh": 1000,
+ "cache_read": 0,
+ "cache_write": null,
+ "output": 100,
+ "reasoning": null
+ },
+ "tools_offered": 3,
+ "max_output": 4096
+ },
+ "content": {}
+ }
+ ]
+}
diff --git a/tests/corpora/observability/golden-phase2/S2-16c/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-16c/expected-issues.json
new file mode 100644
index 00000000..c6e0bfe7
--- /dev/null
+++ b/tests/corpora/observability/golden-phase2/S2-16c/expected-issues.json
@@ -0,0 +1,28 @@
+{
+ "signals_version": "0.3",
+ "warn": 0,
+ "fired": {},
+ "fine": [
+ "context_full",
+ "cut_off",
+ "made_up_tool",
+ "model_failed",
+ "ran_without_sandbox",
+ "refused",
+ "repeated_step",
+ "retried",
+ "step_limit",
+ "tool_failed",
+ "waited_unwatched"
+ ],
+ "not_checked": [
+ "blocked_by_sandbox",
+ "off_plan",
+ "overridden",
+ "reviewer_allowed_send",
+ "reviewer_could_not_run",
+ "reviewer_paused",
+ "sandbox_failed",
+ "shadow_disagreed"
+ ]
+}
diff --git a/tests/corpora/observability/golden-phase2/S2-16c/recording.json b/tests/corpora/observability/golden-phase2/S2-16c/recording.json
new file mode 100644
index 00000000..73b2b1b0
--- /dev/null
+++ b/tests/corpora/observability/golden-phase2/S2-16c/recording.json
@@ -0,0 +1,146 @@
+{
+ "turns": [
+ {
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-16c",
+ "seq": 1,
+ "status": "completed",
+ "approval_mode": "interactive",
+ "attended": true,
+ "reviewer": {
+ "active": false,
+ "shadow": false,
+ "kind": null,
+ "model": null
+ },
+ "stamp": {
+ "sandbox": {
+ "provider": "direct",
+ "enforcement": "none",
+ "expected": true
+ }
+ },
+ "max_iterations": 50,
+ "kind": "run",
+ "trigger": "user",
+ "started_at": 1793600000000,
+ "incomplete": false,
+ "first_message_id": "01JP2MSG000000000000000000",
+ "last_message_id": "01JP2MSG000000000000000000",
+ "schema_version": 1,
+ "duration_ms": 33
+ }
+ ],
+ "steps": [
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-16c",
+ "parent_step_id": null,
+ "seq": 1,
+ "kind": "model",
+ "name": "model",
+ "started_at": 1793600000000,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": null,
+ "call_index": null,
+ "message_id": "01JP2MSG000000000000000010",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "purpose": "main",
+ "provider": "testprov",
+ "provider_title": "Test provider",
+ "model": "testprov:model-a",
+ "stream": true,
+ "finish_reason": "stop",
+ "ttft_ms": 5,
+ "context_window": 100000,
+ "usage": {
+ "reported": true,
+ "input_fresh": 1000,
+ "cache_read": 0,
+ "cache_write": null,
+ "output": 100,
+ "reasoning": null
+ },
+ "tools_offered": 3,
+ "max_output": 4096
+ },
+ "content": {}
+ },
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-16c",
+ "parent_step_id": null,
+ "seq": 2,
+ "kind": "tool",
+ "name": "read_file",
+ "started_at": 1793600000011,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": "call_2",
+ "call_index": 0,
+ "message_id": "01JP2MSG000000000000000020",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "risk_class": "read",
+ "decision": {
+ "path": [],
+ "origin": "low_risk"
+ },
+ "args_hash": "2bfd14",
+ "repeat_ok": false,
+ "result_error": false,
+ "error_kind": null,
+ "connector": null,
+ "result_bytes": 120,
+ "exit_code": null,
+ "sandbox": null,
+ "reaches_network": false
+ },
+ "content": {}
+ },
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-16c",
+ "parent_step_id": null,
+ "seq": 3,
+ "kind": "model",
+ "name": "model",
+ "started_at": 1793600000022,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": null,
+ "call_index": null,
+ "message_id": "01JP2MSG000000000000000010",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "purpose": "main",
+ "provider": "testprov",
+ "provider_title": "Test provider",
+ "model": "testprov:model-a",
+ "stream": true,
+ "finish_reason": "stop",
+ "ttft_ms": 5,
+ "context_window": 100000,
+ "usage": {
+ "reported": true,
+ "input_fresh": 1000,
+ "cache_read": 0,
+ "cache_write": null,
+ "output": 100,
+ "reasoning": null
+ },
+ "tools_offered": 3,
+ "max_output": 4096
+ },
+ "content": {}
+ }
+ ]
+}
diff --git a/tests/corpora/observability/golden-phase2/S2-17/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-17/expected-issues.json
new file mode 100644
index 00000000..a3eb80ff
--- /dev/null
+++ b/tests/corpora/observability/golden-phase2/S2-17/expected-issues.json
@@ -0,0 +1,31 @@
+{
+ "signals_version": "0.3",
+ "warn": 1,
+ "fired": {
+ "waited_unwatched": [
+ 2
+ ]
+ },
+ "fine": [
+ "context_full",
+ "cut_off",
+ "made_up_tool",
+ "model_failed",
+ "refused",
+ "repeated_step",
+ "retried",
+ "step_limit",
+ "tool_failed"
+ ],
+ "not_checked": [
+ "blocked_by_sandbox",
+ "off_plan",
+ "overridden",
+ "ran_without_sandbox",
+ "reviewer_allowed_send",
+ "reviewer_could_not_run",
+ "reviewer_paused",
+ "sandbox_failed",
+ "shadow_disagreed"
+ ]
+}
diff --git a/tests/corpora/observability/golden-phase2/S2-17/recording.json b/tests/corpora/observability/golden-phase2/S2-17/recording.json
new file mode 100644
index 00000000..1561a112
--- /dev/null
+++ b/tests/corpora/observability/golden-phase2/S2-17/recording.json
@@ -0,0 +1,171 @@
+{
+ "turns": [
+ {
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-17",
+ "seq": 1,
+ "status": "completed",
+ "approval_mode": "interactive",
+ "attended": true,
+ "reviewer": {
+ "active": false,
+ "shadow": false,
+ "kind": null,
+ "model": null
+ },
+ "stamp": {
+ "sandbox": {
+ "provider": "direct",
+ "enforcement": "none"
+ }
+ },
+ "max_iterations": 50,
+ "kind": "run",
+ "trigger": "user",
+ "started_at": 1793600000000,
+ "incomplete": false,
+ "first_message_id": "01JP2MSG000000000000000000",
+ "last_message_id": "01JP2MSG000000000000000000",
+ "schema_version": 1,
+ "duration_ms": 1200034
+ }
+ ],
+ "steps": [
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-17",
+ "parent_step_id": null,
+ "seq": 1,
+ "kind": "model",
+ "name": "model",
+ "started_at": 1793600000000,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": null,
+ "call_index": null,
+ "message_id": "01JP2MSG000000000000000010",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "purpose": "main",
+ "provider": "testprov",
+ "provider_title": "Test provider",
+ "model": "testprov:model-a",
+ "stream": true,
+ "finish_reason": "stop",
+ "ttft_ms": 5,
+ "context_window": 100000,
+ "usage": {
+ "reported": true,
+ "input_fresh": 1000,
+ "cache_read": 0,
+ "cache_write": null,
+ "output": 100,
+ "reasoning": null
+ },
+ "tools_offered": 3,
+ "max_output": 4096
+ },
+ "content": {}
+ },
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-17",
+ "parent_step_id": null,
+ "seq": 2,
+ "kind": "wait",
+ "name": "approval",
+ "started_at": 1793600000011,
+ "duration_ms": 1200000,
+ "status": "ok",
+ "call_id": "call_3",
+ "call_index": null,
+ "message_id": null,
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "for_call_id": "call_3",
+ "kind": "approval",
+ "where": "inbox",
+ "attended": false,
+ "outcome": "approved",
+ "inbox_at": 1793600000011
+ },
+ "content": {}
+ },
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-17",
+ "parent_step_id": null,
+ "seq": 3,
+ "kind": "tool",
+ "name": "run_shell",
+ "started_at": 1793601200012,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": "call_3",
+ "call_index": 0,
+ "message_id": "01JP2MSG000000000000000020",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "risk_class": "exec",
+ "decision": {
+ "path": [],
+ "origin": "low_risk"
+ },
+ "args_hash": "363379",
+ "repeat_ok": false,
+ "result_error": false,
+ "error_kind": null,
+ "connector": null,
+ "result_bytes": 120,
+ "exit_code": null,
+ "sandbox": null,
+ "reaches_network": false
+ },
+ "content": {}
+ },
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-17",
+ "parent_step_id": null,
+ "seq": 4,
+ "kind": "model",
+ "name": "model",
+ "started_at": 1793601200023,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": null,
+ "call_index": null,
+ "message_id": "01JP2MSG000000000000000010",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "purpose": "main",
+ "provider": "testprov",
+ "provider_title": "Test provider",
+ "model": "testprov:model-a",
+ "stream": true,
+ "finish_reason": "stop",
+ "ttft_ms": 5,
+ "context_window": 100000,
+ "usage": {
+ "reported": true,
+ "input_fresh": 1000,
+ "cache_read": 0,
+ "cache_write": null,
+ "output": 100,
+ "reasoning": null
+ },
+ "tools_offered": 3,
+ "max_output": 4096
+ },
+ "content": {}
+ }
+ ]
+}
diff --git a/tests/corpora/observability/golden-phase2/S2-17b/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-17b/expected-issues.json
new file mode 100644
index 00000000..49ff0804
--- /dev/null
+++ b/tests/corpora/observability/golden-phase2/S2-17b/expected-issues.json
@@ -0,0 +1,28 @@
+{
+ "signals_version": "0.3",
+ "warn": 0,
+ "fired": {},
+ "fine": [
+ "context_full",
+ "cut_off",
+ "made_up_tool",
+ "model_failed",
+ "refused",
+ "repeated_step",
+ "retried",
+ "step_limit",
+ "tool_failed",
+ "waited_unwatched"
+ ],
+ "not_checked": [
+ "blocked_by_sandbox",
+ "off_plan",
+ "overridden",
+ "ran_without_sandbox",
+ "reviewer_allowed_send",
+ "reviewer_could_not_run",
+ "reviewer_paused",
+ "sandbox_failed",
+ "shadow_disagreed"
+ ]
+}
diff --git a/tests/corpora/observability/golden-phase2/S2-17b/recording.json b/tests/corpora/observability/golden-phase2/S2-17b/recording.json
new file mode 100644
index 00000000..501cf794
--- /dev/null
+++ b/tests/corpora/observability/golden-phase2/S2-17b/recording.json
@@ -0,0 +1,171 @@
+{
+ "turns": [
+ {
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-17b",
+ "seq": 1,
+ "status": "completed",
+ "approval_mode": "interactive",
+ "attended": true,
+ "reviewer": {
+ "active": false,
+ "shadow": false,
+ "kind": null,
+ "model": null
+ },
+ "stamp": {
+ "sandbox": {
+ "provider": "direct",
+ "enforcement": "none"
+ }
+ },
+ "max_iterations": 50,
+ "kind": "run",
+ "trigger": "user",
+ "started_at": 1793600000000,
+ "incomplete": false,
+ "first_message_id": "01JP2MSG000000000000000000",
+ "last_message_id": "01JP2MSG000000000000000000",
+ "schema_version": 1,
+ "duration_ms": 1200034
+ }
+ ],
+ "steps": [
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-17b",
+ "parent_step_id": null,
+ "seq": 1,
+ "kind": "model",
+ "name": "model",
+ "started_at": 1793600000000,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": null,
+ "call_index": null,
+ "message_id": "01JP2MSG000000000000000010",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "purpose": "main",
+ "provider": "testprov",
+ "provider_title": "Test provider",
+ "model": "testprov:model-a",
+ "stream": true,
+ "finish_reason": "stop",
+ "ttft_ms": 5,
+ "context_window": 100000,
+ "usage": {
+ "reported": true,
+ "input_fresh": 1000,
+ "cache_read": 0,
+ "cache_write": null,
+ "output": 100,
+ "reasoning": null
+ },
+ "tools_offered": 3,
+ "max_output": 4096
+ },
+ "content": {}
+ },
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-17b",
+ "parent_step_id": null,
+ "seq": 2,
+ "kind": "wait",
+ "name": "approval",
+ "started_at": 1793600000011,
+ "duration_ms": 1200000,
+ "status": "ok",
+ "call_id": "call_3",
+ "call_index": null,
+ "message_id": null,
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "for_call_id": "call_3",
+ "kind": "approval",
+ "where": "inbox",
+ "attended": true,
+ "outcome": "approved",
+ "inbox_at": 1793601000011
+ },
+ "content": {}
+ },
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-17b",
+ "parent_step_id": null,
+ "seq": 3,
+ "kind": "tool",
+ "name": "run_shell",
+ "started_at": 1793601200012,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": "call_3",
+ "call_index": 0,
+ "message_id": "01JP2MSG000000000000000020",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "risk_class": "exec",
+ "decision": {
+ "path": [],
+ "origin": "low_risk"
+ },
+ "args_hash": "363379",
+ "repeat_ok": false,
+ "result_error": false,
+ "error_kind": null,
+ "connector": null,
+ "result_bytes": 120,
+ "exit_code": null,
+ "sandbox": null,
+ "reaches_network": false
+ },
+ "content": {}
+ },
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-17b",
+ "parent_step_id": null,
+ "seq": 4,
+ "kind": "model",
+ "name": "model",
+ "started_at": 1793601200023,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": null,
+ "call_index": null,
+ "message_id": "01JP2MSG000000000000000010",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "purpose": "main",
+ "provider": "testprov",
+ "provider_title": "Test provider",
+ "model": "testprov:model-a",
+ "stream": true,
+ "finish_reason": "stop",
+ "ttft_ms": 5,
+ "context_window": 100000,
+ "usage": {
+ "reported": true,
+ "input_fresh": 1000,
+ "cache_read": 0,
+ "cache_write": null,
+ "output": 100,
+ "reasoning": null
+ },
+ "tools_offered": 3,
+ "max_output": 4096
+ },
+ "content": {}
+ }
+ ]
+}
diff --git a/tests/corpora/observability/golden-phase2/S2-17c/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-17c/expected-issues.json
new file mode 100644
index 00000000..49ff0804
--- /dev/null
+++ b/tests/corpora/observability/golden-phase2/S2-17c/expected-issues.json
@@ -0,0 +1,28 @@
+{
+ "signals_version": "0.3",
+ "warn": 0,
+ "fired": {},
+ "fine": [
+ "context_full",
+ "cut_off",
+ "made_up_tool",
+ "model_failed",
+ "refused",
+ "repeated_step",
+ "retried",
+ "step_limit",
+ "tool_failed",
+ "waited_unwatched"
+ ],
+ "not_checked": [
+ "blocked_by_sandbox",
+ "off_plan",
+ "overridden",
+ "ran_without_sandbox",
+ "reviewer_allowed_send",
+ "reviewer_could_not_run",
+ "reviewer_paused",
+ "sandbox_failed",
+ "shadow_disagreed"
+ ]
+}
diff --git a/tests/corpora/observability/golden-phase2/S2-17c/recording.json b/tests/corpora/observability/golden-phase2/S2-17c/recording.json
new file mode 100644
index 00000000..730da203
--- /dev/null
+++ b/tests/corpora/observability/golden-phase2/S2-17c/recording.json
@@ -0,0 +1,171 @@
+{
+ "turns": [
+ {
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-17c",
+ "seq": 1,
+ "status": "completed",
+ "approval_mode": "interactive",
+ "attended": true,
+ "reviewer": {
+ "active": false,
+ "shadow": false,
+ "kind": null,
+ "model": null
+ },
+ "stamp": {
+ "sandbox": {
+ "provider": "direct",
+ "enforcement": "none"
+ }
+ },
+ "max_iterations": 50,
+ "kind": "run",
+ "trigger": "user",
+ "started_at": 1793600000000,
+ "incomplete": false,
+ "first_message_id": "01JP2MSG000000000000000000",
+ "last_message_id": "01JP2MSG000000000000000000",
+ "schema_version": 1,
+ "duration_ms": 1200034
+ }
+ ],
+ "steps": [
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-17c",
+ "parent_step_id": null,
+ "seq": 1,
+ "kind": "model",
+ "name": "model",
+ "started_at": 1793600000000,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": null,
+ "call_index": null,
+ "message_id": "01JP2MSG000000000000000010",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "purpose": "main",
+ "provider": "testprov",
+ "provider_title": "Test provider",
+ "model": "testprov:model-a",
+ "stream": true,
+ "finish_reason": "stop",
+ "ttft_ms": 5,
+ "context_window": 100000,
+ "usage": {
+ "reported": true,
+ "input_fresh": 1000,
+ "cache_read": 0,
+ "cache_write": null,
+ "output": 100,
+ "reasoning": null
+ },
+ "tools_offered": 3,
+ "max_output": 4096
+ },
+ "content": {}
+ },
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-17c",
+ "parent_step_id": null,
+ "seq": 2,
+ "kind": "wait",
+ "name": "approval",
+ "started_at": 1793600000011,
+ "duration_ms": 1200000,
+ "status": "ok",
+ "call_id": "call_3",
+ "call_index": null,
+ "message_id": null,
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "for_call_id": "call_3",
+ "kind": "approval",
+ "where": "session",
+ "attended": true,
+ "outcome": "approved",
+ "inbox_at": null
+ },
+ "content": {}
+ },
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-17c",
+ "parent_step_id": null,
+ "seq": 3,
+ "kind": "tool",
+ "name": "run_shell",
+ "started_at": 1793601200012,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": "call_3",
+ "call_index": 0,
+ "message_id": "01JP2MSG000000000000000020",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "risk_class": "exec",
+ "decision": {
+ "path": [],
+ "origin": "low_risk"
+ },
+ "args_hash": "363379",
+ "repeat_ok": false,
+ "result_error": false,
+ "error_kind": null,
+ "connector": null,
+ "result_bytes": 120,
+ "exit_code": null,
+ "sandbox": null,
+ "reaches_network": false
+ },
+ "content": {}
+ },
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-17c",
+ "parent_step_id": null,
+ "seq": 4,
+ "kind": "model",
+ "name": "model",
+ "started_at": 1793601200023,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": null,
+ "call_index": null,
+ "message_id": "01JP2MSG000000000000000010",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "purpose": "main",
+ "provider": "testprov",
+ "provider_title": "Test provider",
+ "model": "testprov:model-a",
+ "stream": true,
+ "finish_reason": "stop",
+ "ttft_ms": 5,
+ "context_window": 100000,
+ "usage": {
+ "reported": true,
+ "input_fresh": 1000,
+ "cache_read": 0,
+ "cache_write": null,
+ "output": 100,
+ "reasoning": null
+ },
+ "tools_offered": 3,
+ "max_output": 4096
+ },
+ "content": {}
+ }
+ ]
+}
diff --git a/tests/corpora/observability/golden-phase2/S2-18/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-18/expected-issues.json
new file mode 100644
index 00000000..541d78da
--- /dev/null
+++ b/tests/corpora/observability/golden-phase2/S2-18/expected-issues.json
@@ -0,0 +1,31 @@
+{
+ "signals_version": "0.3",
+ "warn": 1,
+ "fired": {
+ "sandbox_failed": [
+ 1
+ ]
+ },
+ "fine": [
+ "blocked_by_sandbox",
+ "context_full",
+ "cut_off",
+ "made_up_tool",
+ "model_failed",
+ "refused",
+ "repeated_step",
+ "retried",
+ "step_limit",
+ "tool_failed",
+ "waited_unwatched"
+ ],
+ "not_checked": [
+ "off_plan",
+ "overridden",
+ "ran_without_sandbox",
+ "reviewer_allowed_send",
+ "reviewer_could_not_run",
+ "reviewer_paused",
+ "shadow_disagreed"
+ ]
+}
diff --git a/tests/corpora/observability/golden-phase2/S2-18/recording.json b/tests/corpora/observability/golden-phase2/S2-18/recording.json
new file mode 100644
index 00000000..13493af1
--- /dev/null
+++ b/tests/corpora/observability/golden-phase2/S2-18/recording.json
@@ -0,0 +1,96 @@
+{
+ "turns": [
+ {
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-18",
+ "seq": 1,
+ "status": "error",
+ "approval_mode": "interactive",
+ "attended": true,
+ "reviewer": {
+ "active": false,
+ "shadow": false,
+ "kind": null,
+ "model": null
+ },
+ "stamp": {
+ "sandbox": {
+ "provider": "openshell",
+ "enforcement": "full",
+ "profile": "allowlist"
+ }
+ },
+ "max_iterations": 50,
+ "kind": "run",
+ "trigger": "user",
+ "started_at": 1793600000000,
+ "incomplete": false,
+ "first_message_id": "01JP2MSG000000000000000000",
+ "last_message_id": "01JP2MSG000000000000000000",
+ "schema_version": 1,
+ "duration_ms": 12
+ }
+ ],
+ "steps": [
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-18",
+ "parent_step_id": null,
+ "seq": 1,
+ "kind": "sandbox",
+ "name": "setup",
+ "started_at": 1793600000000,
+ "duration_ms": 10,
+ "status": "error",
+ "call_id": null,
+ "call_index": null,
+ "message_id": null,
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "provider": "openshell",
+ "stages": [
+ {
+ "name": "preparing",
+ "started_at": 1793600000000,
+ "duration_ms": 100,
+ "status": "ok"
+ },
+ {
+ "name": "create",
+ "started_at": 1793600000100,
+ "duration_ms": 100,
+ "status": "error"
+ }
+ ],
+ "failed_stage": "create",
+ "trigger": "turn_start",
+ "error": {
+ "type": "OpenShellImageMissing",
+ "text": null
+ }
+ },
+ "content": {}
+ },
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-18",
+ "parent_step_id": null,
+ "seq": 2,
+ "kind": "notice",
+ "name": "error",
+ "started_at": 1793600000011,
+ "duration_ms": 0,
+ "status": "ok",
+ "call_id": null,
+ "call_index": null,
+ "message_id": null,
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {},
+ "content": {}
+ }
+ ]
+}
diff --git a/tests/corpora/observability/golden-phase2/S2-18c/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-18c/expected-issues.json
new file mode 100644
index 00000000..ab3defd8
--- /dev/null
+++ b/tests/corpora/observability/golden-phase2/S2-18c/expected-issues.json
@@ -0,0 +1,28 @@
+{
+ "signals_version": "0.3",
+ "warn": 0,
+ "fired": {},
+ "fine": [
+ "blocked_by_sandbox",
+ "context_full",
+ "cut_off",
+ "made_up_tool",
+ "model_failed",
+ "refused",
+ "repeated_step",
+ "retried",
+ "sandbox_failed",
+ "step_limit",
+ "tool_failed",
+ "waited_unwatched"
+ ],
+ "not_checked": [
+ "off_plan",
+ "overridden",
+ "ran_without_sandbox",
+ "reviewer_allowed_send",
+ "reviewer_could_not_run",
+ "reviewer_paused",
+ "shadow_disagreed"
+ ]
+}
diff --git a/tests/corpora/observability/golden-phase2/S2-18c/recording.json b/tests/corpora/observability/golden-phase2/S2-18c/recording.json
new file mode 100644
index 00000000..416b4d72
--- /dev/null
+++ b/tests/corpora/observability/golden-phase2/S2-18c/recording.json
@@ -0,0 +1,136 @@
+{
+ "turns": [
+ {
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-18c",
+ "seq": 1,
+ "status": "completed",
+ "approval_mode": "interactive",
+ "attended": true,
+ "reviewer": {
+ "active": false,
+ "shadow": false,
+ "kind": null,
+ "model": null
+ },
+ "stamp": {
+ "sandbox": {
+ "provider": "openshell",
+ "enforcement": "full",
+ "profile": "allowlist"
+ }
+ },
+ "max_iterations": 50,
+ "kind": "run",
+ "trigger": "user",
+ "started_at": 1793600000000,
+ "incomplete": false,
+ "first_message_id": "01JP2MSG000000000000000000",
+ "last_message_id": "01JP2MSG000000000000000000",
+ "schema_version": 1,
+ "duration_ms": 41023
+ }
+ ],
+ "steps": [
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-18c",
+ "parent_step_id": null,
+ "seq": 1,
+ "kind": "sandbox",
+ "name": "setup",
+ "started_at": 1793600000000,
+ "duration_ms": 41000,
+ "status": "ok",
+ "call_id": null,
+ "call_index": null,
+ "message_id": null,
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "provider": "openshell",
+ "stages": [],
+ "failed_stage": null,
+ "trigger": "turn_start"
+ },
+ "content": {}
+ },
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-18c",
+ "parent_step_id": null,
+ "seq": 2,
+ "kind": "model",
+ "name": "model",
+ "started_at": 1793600041001,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": null,
+ "call_index": null,
+ "message_id": "01JP2MSG000000000000000010",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "purpose": "main",
+ "provider": "testprov",
+ "provider_title": "Test provider",
+ "model": "testprov:model-a",
+ "stream": true,
+ "finish_reason": "stop",
+ "ttft_ms": 5,
+ "context_window": 100000,
+ "usage": {
+ "reported": true,
+ "input_fresh": 1000,
+ "cache_read": 0,
+ "cache_write": null,
+ "output": 100,
+ "reasoning": null
+ },
+ "tools_offered": 3,
+ "max_output": 4096
+ },
+ "content": {}
+ },
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-18c",
+ "parent_step_id": null,
+ "seq": 3,
+ "kind": "model",
+ "name": "model",
+ "started_at": 1793600041012,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": null,
+ "call_index": null,
+ "message_id": "01JP2MSG000000000000000010",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "purpose": "main",
+ "provider": "testprov",
+ "provider_title": "Test provider",
+ "model": "testprov:model-a",
+ "stream": true,
+ "finish_reason": "stop",
+ "ttft_ms": 5,
+ "context_window": 100000,
+ "usage": {
+ "reported": true,
+ "input_fresh": 1000,
+ "cache_read": 0,
+ "cache_write": null,
+ "output": 100,
+ "reasoning": null
+ },
+ "tools_offered": 3,
+ "max_output": 4096
+ },
+ "content": {}
+ }
+ ]
+}
diff --git a/tests/corpora/observability/golden-phase2/S2-19/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-19/expected-issues.json
new file mode 100644
index 00000000..34c71169
--- /dev/null
+++ b/tests/corpora/observability/golden-phase2/S2-19/expected-issues.json
@@ -0,0 +1,31 @@
+{
+ "signals_version": "0.3",
+ "warn": 0,
+ "fired": {
+ "reviewer_allowed_send": [
+ 3
+ ]
+ },
+ "fine": [
+ "context_full",
+ "cut_off",
+ "made_up_tool",
+ "model_failed",
+ "overridden",
+ "refused",
+ "repeated_step",
+ "retried",
+ "reviewer_could_not_run",
+ "reviewer_paused",
+ "step_limit",
+ "tool_failed",
+ "waited_unwatched"
+ ],
+ "not_checked": [
+ "blocked_by_sandbox",
+ "off_plan",
+ "ran_without_sandbox",
+ "sandbox_failed",
+ "shadow_disagreed"
+ ]
+}
diff --git a/tests/corpora/observability/golden-phase2/S2-19/recording.json b/tests/corpora/observability/golden-phase2/S2-19/recording.json
new file mode 100644
index 00000000..3a097bb2
--- /dev/null
+++ b/tests/corpora/observability/golden-phase2/S2-19/recording.json
@@ -0,0 +1,185 @@
+{
+ "turns": [
+ {
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-19",
+ "seq": 1,
+ "status": "completed",
+ "approval_mode": "auto-approve",
+ "attended": true,
+ "reviewer": {
+ "active": true,
+ "shadow": false,
+ "kind": "session_model",
+ "model": "m"
+ },
+ "stamp": {
+ "sandbox": {
+ "provider": "direct",
+ "enforcement": "none"
+ }
+ },
+ "max_iterations": 50,
+ "kind": "run",
+ "trigger": "user",
+ "started_at": 1793600000000,
+ "incomplete": false,
+ "first_message_id": "01JP2MSG000000000000000000",
+ "last_message_id": "01JP2MSG000000000000000000",
+ "schema_version": 1,
+ "duration_ms": 44
+ }
+ ],
+ "steps": [
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-19",
+ "parent_step_id": null,
+ "seq": 1,
+ "kind": "model",
+ "name": "model",
+ "started_at": 1793600000000,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": null,
+ "call_index": null,
+ "message_id": "01JP2MSG000000000000000010",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "purpose": "main",
+ "provider": "testprov",
+ "provider_title": "Test provider",
+ "model": "testprov:model-a",
+ "stream": true,
+ "finish_reason": "stop",
+ "ttft_ms": 5,
+ "context_window": 100000,
+ "usage": {
+ "reported": true,
+ "input_fresh": 1000,
+ "cache_read": 0,
+ "cache_write": null,
+ "output": 100,
+ "reasoning": null
+ },
+ "tools_offered": 3,
+ "max_output": 4096
+ },
+ "content": {}
+ },
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-19",
+ "parent_step_id": null,
+ "seq": 2,
+ "kind": "auto_approver",
+ "name": "auto_approver",
+ "started_at": 1793600000011,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": "call_3",
+ "call_index": null,
+ "message_id": null,
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "reviewer": {
+ "kind": "session_model",
+ "model": "m",
+ "external": false
+ },
+ "mode": "live",
+ "result": "allow",
+ "score": null,
+ "cutoff": null,
+ "usage_reported": true,
+ "breaker": {
+ "denials_in_row": 0,
+ "tripped": false
+ },
+ "matched_user": null
+ },
+ "content": {}
+ },
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-19",
+ "parent_step_id": null,
+ "seq": 3,
+ "kind": "tool",
+ "name": "run_shell",
+ "started_at": 1793600000022,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": "call_3",
+ "call_index": 0,
+ "message_id": "01JP2MSG000000000000000020",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "risk_class": "exec",
+ "decision": {
+ "path": [
+ {
+ "stage": "auto_approver",
+ "outcome": "allow"
+ }
+ ],
+ "origin": "reviewer"
+ },
+ "args_hash": "363379",
+ "repeat_ok": false,
+ "result_error": false,
+ "error_kind": null,
+ "reaches_network": true,
+ "connector": null,
+ "result_bytes": 120,
+ "exit_code": null,
+ "sandbox": null
+ },
+ "content": {}
+ },
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-19",
+ "parent_step_id": null,
+ "seq": 4,
+ "kind": "model",
+ "name": "model",
+ "started_at": 1793600000033,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": null,
+ "call_index": null,
+ "message_id": "01JP2MSG000000000000000010",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "purpose": "main",
+ "provider": "testprov",
+ "provider_title": "Test provider",
+ "model": "testprov:model-a",
+ "stream": true,
+ "finish_reason": "stop",
+ "ttft_ms": 5,
+ "context_window": 100000,
+ "usage": {
+ "reported": true,
+ "input_fresh": 1000,
+ "cache_read": 0,
+ "cache_write": null,
+ "output": 100,
+ "reasoning": null
+ },
+ "tools_offered": 3,
+ "max_output": 4096
+ },
+ "content": {}
+ }
+ ]
+}
diff --git a/tests/corpora/observability/golden-phase2/S2-19c/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-19c/expected-issues.json
new file mode 100644
index 00000000..b142c8d2
--- /dev/null
+++ b/tests/corpora/observability/golden-phase2/S2-19c/expected-issues.json
@@ -0,0 +1,28 @@
+{
+ "signals_version": "0.3",
+ "warn": 0,
+ "fired": {},
+ "fine": [
+ "context_full",
+ "cut_off",
+ "made_up_tool",
+ "model_failed",
+ "overridden",
+ "refused",
+ "repeated_step",
+ "retried",
+ "reviewer_allowed_send",
+ "reviewer_could_not_run",
+ "reviewer_paused",
+ "step_limit",
+ "tool_failed",
+ "waited_unwatched"
+ ],
+ "not_checked": [
+ "blocked_by_sandbox",
+ "off_plan",
+ "ran_without_sandbox",
+ "sandbox_failed",
+ "shadow_disagreed"
+ ]
+}
diff --git a/tests/corpora/observability/golden-phase2/S2-19c/recording.json b/tests/corpora/observability/golden-phase2/S2-19c/recording.json
new file mode 100644
index 00000000..2c9d507e
--- /dev/null
+++ b/tests/corpora/observability/golden-phase2/S2-19c/recording.json
@@ -0,0 +1,214 @@
+{
+ "turns": [
+ {
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-19c",
+ "seq": 1,
+ "status": "completed",
+ "approval_mode": "auto-approve",
+ "attended": true,
+ "reviewer": {
+ "active": true,
+ "shadow": false,
+ "kind": "session_model",
+ "model": "m"
+ },
+ "stamp": {
+ "sandbox": {
+ "provider": "direct",
+ "enforcement": "none"
+ }
+ },
+ "max_iterations": 50,
+ "kind": "run",
+ "trigger": "user",
+ "started_at": 1793600000000,
+ "incomplete": false,
+ "first_message_id": "01JP2MSG000000000000000000",
+ "last_message_id": "01JP2MSG000000000000000000",
+ "schema_version": 1,
+ "duration_ms": 55
+ }
+ ],
+ "steps": [
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-19c",
+ "parent_step_id": null,
+ "seq": 1,
+ "kind": "model",
+ "name": "model",
+ "started_at": 1793600000000,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": null,
+ "call_index": null,
+ "message_id": "01JP2MSG000000000000000010",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "purpose": "main",
+ "provider": "testprov",
+ "provider_title": "Test provider",
+ "model": "testprov:model-a",
+ "stream": true,
+ "finish_reason": "stop",
+ "ttft_ms": 5,
+ "context_window": 100000,
+ "usage": {
+ "reported": true,
+ "input_fresh": 1000,
+ "cache_read": 0,
+ "cache_write": null,
+ "output": 100,
+ "reasoning": null
+ },
+ "tools_offered": 3,
+ "max_output": 4096
+ },
+ "content": {}
+ },
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-19c",
+ "parent_step_id": null,
+ "seq": 2,
+ "kind": "auto_approver",
+ "name": "auto_approver",
+ "started_at": 1793600000011,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": "call_3",
+ "call_index": null,
+ "message_id": null,
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "reviewer": {
+ "kind": "session_model",
+ "model": "m",
+ "external": false
+ },
+ "mode": "live",
+ "result": "allow",
+ "score": null,
+ "cutoff": null,
+ "usage_reported": true,
+ "breaker": {
+ "denials_in_row": 0,
+ "tripped": false
+ },
+ "matched_user": null
+ },
+ "content": {}
+ },
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-19c",
+ "parent_step_id": null,
+ "seq": 3,
+ "kind": "tool",
+ "name": "run_shell",
+ "started_at": 1793600000022,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": "call_3",
+ "call_index": 0,
+ "message_id": "01JP2MSG000000000000000020",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "risk_class": "exec",
+ "decision": {
+ "path": [],
+ "origin": "reviewer"
+ },
+ "args_hash": "363379",
+ "repeat_ok": false,
+ "result_error": false,
+ "error_kind": null,
+ "reaches_network": false,
+ "connector": null,
+ "result_bytes": 120,
+ "exit_code": null,
+ "sandbox": null
+ },
+ "content": {}
+ },
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-19c",
+ "parent_step_id": null,
+ "seq": 4,
+ "kind": "tool",
+ "name": "web_fetch",
+ "started_at": 1793600000033,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": "call_4",
+ "call_index": 0,
+ "message_id": "01JP2MSG000000000000000020",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "risk_class": "egress",
+ "decision": {
+ "path": [],
+ "origin": "user"
+ },
+ "args_hash": "215ddd",
+ "repeat_ok": false,
+ "result_error": false,
+ "error_kind": null,
+ "connector": null,
+ "result_bytes": 120,
+ "exit_code": null,
+ "sandbox": null,
+ "reaches_network": true
+ },
+ "content": {}
+ },
+ {
+ "step_id": "01JP2STP000000000000000000",
+ "turn_id": "01JP2TRN000000000000000000",
+ "session_id": "golden-s2-19c",
+ "parent_step_id": null,
+ "seq": 5,
+ "kind": "model",
+ "name": "model",
+ "started_at": 1793600000044,
+ "duration_ms": 10,
+ "status": "ok",
+ "call_id": null,
+ "call_index": null,
+ "message_id": "01JP2MSG000000000000000010",
+ "shadow": false,
+ "after_turn": false,
+ "attrs": {
+ "purpose": "main",
+ "provider": "testprov",
+ "provider_title": "Test provider",
+ "model": "testprov:model-a",
+ "stream": true,
+ "finish_reason": "stop",
+ "ttft_ms": 5,
+ "context_window": 100000,
+ "usage": {
+ "reported": true,
+ "input_fresh": 1000,
+ "cache_read": 0,
+ "cache_write": null,
+ "output": 100,
+ "reasoning": null
+ },
+ "tools_offered": 3,
+ "max_output": 4096
+ },
+ "content": {}
+ }
+ ]
+}
diff --git a/tests/corpora/observability/golden-phase2/index.json b/tests/corpora/observability/golden-phase2/index.json
new file mode 100644
index 00000000..8feb4310
--- /dev/null
+++ b/tests/corpora/observability/golden-phase2/index.json
@@ -0,0 +1,321 @@
+{
+ "format": "1.2",
+ "signals_version": "0.3",
+ "built": true,
+ "scenarios": [
+ {
+ "id": "S2-01",
+ "check": "repeated_step",
+ "should": "fire",
+ "also_fires": [],
+ "fired": [
+ "repeated_step"
+ ]
+ },
+ {
+ "id": "S2-01c",
+ "check": "repeated_step",
+ "should": "stay quiet",
+ "also_fires": [],
+ "fired": []
+ },
+ {
+ "id": "S2-02",
+ "check": "off_plan",
+ "should": "fire",
+ "also_fires": [],
+ "fired": [
+ "off_plan"
+ ]
+ },
+ {
+ "id": "S2-02b",
+ "check": "off_plan",
+ "should": "fire",
+ "also_fires": [],
+ "fired": [
+ "off_plan"
+ ]
+ },
+ {
+ "id": "S2-02c",
+ "check": "off_plan",
+ "should": "stay quiet",
+ "also_fires": [],
+ "fired": []
+ },
+ {
+ "id": "S2-02n",
+ "check": "off_plan",
+ "should": "not run",
+ "also_fires": [],
+ "fired": []
+ },
+ {
+ "id": "S2-03",
+ "check": "refused",
+ "should": "fire",
+ "also_fires": [],
+ "fired": [
+ "refused"
+ ]
+ },
+ {
+ "id": "S2-03b",
+ "check": "refused",
+ "should": "stay quiet",
+ "also_fires": [],
+ "fired": []
+ },
+ {
+ "id": "S2-04",
+ "check": "overridden",
+ "should": "fire",
+ "also_fires": [],
+ "fired": [
+ "overridden"
+ ]
+ },
+ {
+ "id": "S2-04n",
+ "check": "overridden",
+ "should": "not run",
+ "also_fires": [],
+ "fired": []
+ },
+ {
+ "id": "S2-05",
+ "check": "reviewer_could_not_run",
+ "should": "fire",
+ "also_fires": [],
+ "fired": [
+ "reviewer_could_not_run"
+ ]
+ },
+ {
+ "id": "S2-05c",
+ "check": "reviewer_could_not_run",
+ "should": "stay quiet",
+ "also_fires": [],
+ "fired": []
+ },
+ {
+ "id": "S2-06",
+ "check": "reviewer_paused",
+ "should": "fire",
+ "also_fires": [],
+ "fired": [
+ "reviewer_paused"
+ ]
+ },
+ {
+ "id": "S2-19",
+ "check": "reviewer_allowed_send",
+ "should": "fire",
+ "also_fires": [],
+ "fired": [
+ "reviewer_allowed_send"
+ ]
+ },
+ {
+ "id": "S2-19c",
+ "check": "reviewer_allowed_send",
+ "should": "stay quiet",
+ "also_fires": [],
+ "fired": []
+ },
+ {
+ "id": "S2-07",
+ "check": "shadow_disagreed",
+ "should": "fire",
+ "also_fires": [],
+ "fired": [
+ "shadow_disagreed"
+ ]
+ },
+ {
+ "id": "S2-08",
+ "check": "tool_failed",
+ "should": "fire",
+ "also_fires": [],
+ "fired": [
+ "tool_failed"
+ ]
+ },
+ {
+ "id": "S2-08b",
+ "check": "tool_failed",
+ "should": "fire",
+ "also_fires": [],
+ "fired": [
+ "tool_failed"
+ ]
+ },
+ {
+ "id": "S2-08c",
+ "check": "tool_failed",
+ "should": "stay quiet",
+ "also_fires": [
+ "made_up_tool"
+ ],
+ "fired": [
+ "made_up_tool"
+ ]
+ },
+ {
+ "id": "S2-09",
+ "check": "model_failed",
+ "should": "fire",
+ "also_fires": [],
+ "fired": [
+ "model_failed"
+ ]
+ },
+ {
+ "id": "S2-09c",
+ "check": "model_failed",
+ "should": "stay quiet",
+ "also_fires": [],
+ "fired": []
+ },
+ {
+ "id": "S2-10",
+ "check": "retried",
+ "should": "fire",
+ "also_fires": [],
+ "fired": [
+ "retried"
+ ]
+ },
+ {
+ "id": "S2-10b",
+ "check": "retried",
+ "should": "stay quiet",
+ "also_fires": [],
+ "fired": []
+ },
+ {
+ "id": "S2-11",
+ "check": "cut_off",
+ "should": "fire",
+ "also_fires": [],
+ "fired": [
+ "cut_off"
+ ]
+ },
+ {
+ "id": "S2-12",
+ "check": "step_limit",
+ "should": "fire",
+ "also_fires": [],
+ "fired": [
+ "step_limit"
+ ]
+ },
+ {
+ "id": "S2-13",
+ "check": "made_up_tool",
+ "should": "fire",
+ "also_fires": [],
+ "fired": [
+ "made_up_tool"
+ ]
+ },
+ {
+ "id": "S2-14",
+ "check": "context_full",
+ "should": "fire",
+ "also_fires": [],
+ "fired": [
+ "context_full"
+ ]
+ },
+ {
+ "id": "S2-14b",
+ "check": "context_full",
+ "should": "stay quiet",
+ "also_fires": [],
+ "fired": []
+ },
+ {
+ "id": "S2-14d",
+ "check": "context_full",
+ "should": "fire",
+ "also_fires": [],
+ "fired": [
+ "context_full"
+ ]
+ },
+ {
+ "id": "S2-15",
+ "check": "blocked_by_sandbox",
+ "should": "fire",
+ "also_fires": [],
+ "fired": [
+ "blocked_by_sandbox"
+ ]
+ },
+ {
+ "id": "S2-15c",
+ "check": "blocked_by_sandbox",
+ "should": "stay quiet",
+ "also_fires": [],
+ "fired": []
+ },
+ {
+ "id": "S2-18",
+ "check": "sandbox_failed",
+ "should": "fire",
+ "also_fires": [],
+ "fired": [
+ "sandbox_failed"
+ ]
+ },
+ {
+ "id": "S2-18c",
+ "check": "sandbox_failed",
+ "should": "stay quiet",
+ "also_fires": [],
+ "fired": []
+ },
+ {
+ "id": "S2-16",
+ "check": "ran_without_sandbox",
+ "should": "fire",
+ "also_fires": [],
+ "fired": [
+ "ran_without_sandbox"
+ ]
+ },
+ {
+ "id": "S2-16c",
+ "check": "ran_without_sandbox",
+ "should": "stay quiet",
+ "also_fires": [],
+ "fired": []
+ },
+ {
+ "id": "S2-17",
+ "check": "waited_unwatched",
+ "should": "fire",
+ "also_fires": [],
+ "fired": [
+ "waited_unwatched"
+ ]
+ },
+ {
+ "id": "S2-17b",
+ "check": "waited_unwatched",
+ "should": "stay quiet",
+ "also_fires": [],
+ "fired": []
+ },
+ {
+ "id": "S2-17c",
+ "check": "waited_unwatched",
+ "should": "stay quiet",
+ "also_fires": [],
+ "fired": []
+ }
+ ]
+}
diff --git a/tests/corpora/observability/golden-phase2/vectors/hashes.json b/tests/corpora/observability/golden-phase2/vectors/hashes.json
new file mode 100644
index 00000000..5c6a4dc5
--- /dev/null
+++ b/tests/corpora/observability/golden-phase2/vectors/hashes.json
@@ -0,0 +1,101 @@
+{
+ "rule": "args_hash = first 6 hex of SHA-256 over the RFC 8785 (JCS) canonical JSON, UTF-8. plan item h = first 6 hex of SHA-256 over the item's text, UTF-8.",
+ "args": [
+ {
+ "name": "plain",
+ "value": {
+ "path": "notes.txt"
+ },
+ "canonical": "{\"path\":\"notes.txt\"}",
+ "args_hash": "327e09"
+ },
+ {
+ "name": "key order",
+ "value": {
+ "b": 1,
+ "a": 2
+ },
+ "canonical": "{\"a\":2,\"b\":1}",
+ "args_hash": "d3626a"
+ },
+ {
+ "name": "same, other order",
+ "value": {
+ "a": 2,
+ "b": 1
+ },
+ "canonical": "{\"a\":2,\"b\":1}",
+ "args_hash": "d3626a"
+ },
+ {
+ "name": "nested",
+ "value": {
+ "cmd": {
+ "args": [
+ "-x",
+ "tests"
+ ],
+ "env": {
+ "Z": "1",
+ "A": "2"
+ }
+ }
+ },
+ "canonical": "{\"cmd\":{\"args\":[\"-x\",\"tests\"],\"env\":{\"A\":\"2\",\"Z\":\"1\"}}}",
+ "args_hash": "0c3387"
+ },
+ {
+ "name": "non-ASCII",
+ "value": {
+ "query": "café 日本 Ω"
+ },
+ "canonical": "{\"query\":\"café 日本 Ω\"}",
+ "args_hash": "e7452a"
+ },
+ {
+ "name": "numbers",
+ "value": {
+ "n": [
+ 0,
+ -0.0,
+ 1,
+ 100.0,
+ 1.5,
+ 1e-07,
+ 1e+21,
+ 123456789012
+ ]
+ },
+ "canonical": "{\"n\":[0,0,1,100,1.5,1e-7,1e+21,123456789012]}",
+ "args_hash": "4840a3"
+ },
+ {
+ "name": "empty",
+ "value": {},
+ "canonical": "{}",
+ "args_hash": "44136f"
+ },
+ {
+ "name": "escapes",
+ "value": {
+ "text": "line\nnext \"quoted\" \\ tab\t"
+ },
+ "canonical": "{\"text\":\"line\\nnext \\\"quoted\\\" \\\\ tab\\t\"}",
+ "args_hash": "0094d3"
+ }
+ ],
+ "plan_items": [
+ {
+ "text": "read notes.txt",
+ "h": "47c423"
+ },
+ {
+ "text": "Answer who owns the docs",
+ "h": "606752"
+ },
+ {
+ "text": "修正テスト",
+ "h": "323a85"
+ }
+ ]
+}
diff --git a/tests/corpora/observability/golden-phase2/vectors/masking.json b/tests/corpora/observability/golden-phase2/vectors/masking.json
new file mode 100644
index 00000000..6defc337
--- /dev/null
+++ b/tests/corpora/observability/golden-phase2/vectors/masking.json
@@ -0,0 +1,47 @@
+{
+ "rule": "Rule R4: mask (stored secret values, longest first, then the patterns in attributes.yaml export.content_switches.masking), then cut at 8192 bytes of UTF-8 on a character boundary, then the marker.",
+ "mask": [
+ {
+ "text": "use sk-abcdefghijklmnopqrstu now",
+ "secrets": [],
+ "expected": "use [secret masked] now"
+ },
+ {
+ "text": "aws AKIAABCDEFGHIJKLMNOP done",
+ "secrets": [],
+ "expected": "aws [secret masked] done"
+ },
+ {
+ "text": "token ghp_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx and xoxb-1234567890-abc",
+ "secrets": [],
+ "expected": "token [secret masked] and [secret masked]"
+ },
+ {
+ "text": "-----BEGIN RSA PRIVATE KEY-----\nMIIE\n-----END RSA PRIVATE KEY----- kept",
+ "secrets": [],
+ "expected": "[secret masked] kept"
+ },
+ {
+ "text": "my key is hunter2-correct-horse here",
+ "secrets": [
+ "hunter2-correct-horse"
+ ],
+ "expected": "my key is [secret masked] here"
+ },
+ {
+ "text": "short pw abc is kept",
+ "secrets": [
+ "abc"
+ ],
+ "expected": "short pw abc is kept"
+ }
+ ],
+ "cut": [
+ {
+ "text_bytes": 8203,
+ "text": "'a' * 8191 + 'é' + 'b' * 10",
+ "expected_kept_bytes": 8191,
+ "expected_tail": "aaaa[cut: 12 more bytes]"
+ }
+ ]
+}
diff --git a/tests/corpora/observability/recording-format.schema.json b/tests/corpora/observability/recording-format.schema.json
index 12984ea8..bf42c3fa 100644
--- a/tests/corpora/observability/recording-format.schema.json
+++ b/tests/corpora/observability/recording-format.schema.json
@@ -1,6 +1,6 @@
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
- "$id": "urn:openworker:observability:recording-format:1.1",
+ "$id": "urn:openworker:observability:recording-format:1.2",
"title": "OpenWorker observability recording format 1.0",
"description": "Turns, steps, recording events and exchange pieces, as defined in recording-format.md (version 1.0, frozen 2026-10-31). Field names are those of phase-1/technical-design.md (Draft 5). Validate a recording file against #/$defs/recording, one recording event against #/$defs/event, one line of an exchange piece store against #/$defs/piece, one live message against #/$defs/live_message. Wherever a field may be null, its description says what null means.",
"$ref": "#/$defs/recording",
@@ -13,7 +13,7 @@
"count_or_null": {"type": ["integer", "null"], "minimum": 0},
"sha256": {"type": "string", "pattern": "^[0-9a-f]{64}$"},
"short_hash": {"type": ["string", "null"], "pattern": "^[0-9a-f]{6}$", "description": "First 6 hex characters of a SHA-256. null = nothing to hash."},
- "step_kind": {"enum": ["context", "model", "tool", "wait", "auto_approver", "summary", "notice"]},
+ "step_kind": {"enum": ["context", "model", "tool", "wait", "auto_approver", "summary", "notice", "sandbox"], "description": "sandbox: added in 1.2 (sandbox setup, Phase 2 OB-13)."},
"step_status": {"enum": ["running", "ok", "error", "denied", "interrupted", "skipped", "unknown"]},
"turn_status": {"enum": ["completed", "max_iterations_exceeded", "truncated", "sleeping", "interrupted", "error", "unknown"]},
"turn_kind": {"enum": ["run", "resume", "retry"]},
@@ -78,7 +78,8 @@
"profile": {"enum": ["allowlist", "open", null], "description": "null = no network profile (direct)."},
"sites": {"type": ["integer", "null"], "minimum": 0, "description": "null = no allowlist."},
"runner_version": {"type": ["string", "null"]},
- "image": {"type": ["string", "null"]}
+ "image": {"type": ["string", "null"]},
+ "expected": {"type": "boolean", "description": "1.2. This machine has a sandbox switched on in Settings, whatever this session used."}
}
},
"persona_fp": {"type": "string", "pattern": "^[0-9a-f]{6}$", "description": "Coworker fingerprint: first 6 hex characters of the section 9.4 hash. The session record keeps the same value."}
@@ -108,7 +109,9 @@
"stamp": {"$ref": "#/$defs/stamp"},
"first_message_id": {"$ref": "#/$defs/id_or_null", "description": "mid of the message that opened the turn. null = empty: a retry that appended no message, or a message written before Phase 1."},
"last_message_id": {"$ref": "#/$defs/id_or_null", "description": "mid of the last message appended before the turn ended. null = no message was appended."},
- "schema_version": {"const": 1, "description": "The record layout version: 1 for every 1.x version of this format. It goes up only when old readers cannot read the new shape."}
+ "schema_version": {"const": 1, "description": "The record layout version: 1 for every 1.x version of this format. It goes up only when old readers cannot read the new shape."},
+ "max_iterations": {"type": ["integer", "null"], "minimum": 1, "description": "1.2. The step limit in force for this turn. null or absent = not recorded (before 1.2)."},
+ "woken_by": {"$ref": "#/$defs/woken_by"}
}
},
@@ -160,7 +163,11 @@
"status": {"enum": ["running", "ok", "error", "interrupted", "unknown"]}}}},
{"if": {"properties": {"kind": {"const": "notice"}}},
"then": {"properties": {"name": {"$ref": "#/$defs/notice_name"}, "attrs": {"$ref": "#/$defs/notice_attrs"},
- "content": {"$ref": "#/$defs/no_content"}, "status": {"const": "ok"}}}}
+ "content": {"$ref": "#/$defs/no_content"}, "status": {"const": "ok"}}}},
+ {"if": {"properties": {"kind": {"const": "sandbox"}}},
+ "then": {"properties": {"name": {"const": "setup"}, "attrs": {"$ref": "#/$defs/sandbox_attrs"},
+ "content": {"$ref": "#/$defs/no_content"},
+ "status": {"enum": ["running", "ok", "error", "interrupted", "unknown"]}}}}
]
},
@@ -307,6 +314,7 @@
"usage": {"$ref": "#/$defs/usage"},
"tools_offered": {"$ref": "#/$defs/count"},
"tools_hash": {"$ref": "#/$defs/short_hash"},
+ "max_output": {"type": ["integer", "null"], "minimum": 1, "description": "1.2. The output limit the request asked for (max_tokens, max_output_tokens). null = none was sent."},
"error": {"$ref": "#/$defs/model_error"},
"ended_after_turn_ms": {"$ref": "#/$defs/ended_after_turn_ms"}
}
@@ -363,6 +371,20 @@
"notice": {"enum": ["restart", "regrant", null]}
}
},
+ "args_hash": {"oneOf": [{"$ref": "#/$defs/short_hash"}, {"type": "null"}], "description": "1.2. Short code of the call's arguments: SHA-256 over canonical JSON, first 6 hex. null = the arguments never parsed."},
+ "repeat_ok": {"type": "boolean", "description": "1.2. The tool declares that calling it again with the same arguments is normal (polling)."},
+ "result_error": {"type": "boolean", "description": "1.2. The tool ran and returned a result with a non-empty error key instead of raising."},
+ "error_kind": {"enum": ["unknown_tool", "bad_arguments", "raised", null], "description": "1.2. Why a tool step has status error. null = it did not."},
+ "reaches_network": {"type": ["boolean", "null"], "description": "1.2. The call reaches the network: risk class egress, or a shell command that names a network client or a URL (review_facts.shell_reaches_network). null = not known (the call never ran)."},
+ "plan": {"type": "object", "additionalProperties": false, "required": ["items"], "description": "1.2. todo_write only: the list as written.",
+ "properties": {"items": {"type": "array", "items": {"type": "object", "additionalProperties": false, "required": ["h", "s"],
+ "properties": {"h": {"type": "string", "pattern": "^[0-9a-f]{6}$", "description": "6 hex of the item's text."}, "s": {"enum": ["pending", "in_progress", "done"]}}}}}},
+ "helper": {"type": "object", "additionalProperties": false, "required": ["index", "status", "model_calls", "tool_calls", "tokens"], "description": "1.2. explore only, for display: what its helper did. The pill adds up the helper's own model steps, never this.",
+ "properties": {"index": {"type": "integer", "minimum": 1}, "status": {"enum": ["ok", "step_limit", "error", "interrupted"]},
+ "model_calls": {"$ref": "#/$defs/count"}, "tool_calls": {"$ref": "#/$defs/count"}, "tokens": {"$ref": "#/$defs/count"}}},
+ "team": {"type": "object", "additionalProperties": false, "required": ["team_id"], "description": "1.2. Team tools only: ids, no text.",
+ "properties": {"team_id": {"type": "string"}, "item_id": {"type": ["string", "null"]}, "worker_session_id": {"type": ["string", "null"]},
+ "worker_session_ids": {"type": "array", "items": {"type": "string"}, "description": "propose_team only: the sessions it created."}}},
"ended_after_turn_ms": {"$ref": "#/$defs/ended_after_turn_ms"}
}
},
@@ -394,6 +416,7 @@
"where": {"enum": ["session", "inbox"]},
"attended": {"type": "boolean"},
"outcome": {"enum": ["approved", "denied", "answered", "interrupted", null], "description": "null or absent = still waiting (live only)."},
+ "inbox_at": {"type": ["integer", "null"], "description": "1.2. Epoch ms when the card reached the Inbox: the step's start if it began there, else the step_updated that moved it. null = it never did."},
"ended_after_turn_ms": {"$ref": "#/$defs/ended_after_turn_ms"}
}
},
@@ -468,7 +491,7 @@
}
},
- "notice_name": {"enum": ["interrupted", "error", "reviewer_paused", "step_limit", "truncated", "context_changed", "stamp_changed", "summary_failed", "recording_gap"]},
+ "notice_name": {"enum": ["interrupted", "error", "reviewer_paused", "step_limit", "truncated", "context_changed", "stamp_changed", "summary_failed", "recording_gap", "controller_offline", "nobody_watching", "clock_jump", "sandbox_restarted"], "description": "The last four: added in 1.2. controller_offline, nobody_watching and clock_jump have a duration (a gap); the others are zero-length."},
"notice_attrs": {
"description": "{} for every notice except two, which carry what changed, under names already defined elsewhere, with the new value and only the keys that changed. context_changed: the per-turn context block as it is now, under the context step's names (per_turn, size_chars, block_hash). stamp_changed: attended, reviewer or stamp, under the turn's names (stamp holding only the changed groups).",
"type": "object",
@@ -479,9 +502,37 @@
"block_hash": {"$ref": "#/$defs/short_hash"},
"attended": {"type": "boolean"},
"reviewer": {"$ref": "#/$defs/reviewer_settings"},
- "stamp": {"$ref": "#/$defs/stamp"}
+ "stamp": {"$ref": "#/$defs/stamp"},
+ "reason": {"enum": ["closed", "failed", "ping_timeout", "folders_changed", "runner_restarted"], "description": "1.2. controller_offline: closed, failed, ping_timeout. sandbox_restarted: folders_changed, runner_restarted."},
+ "skipped_ms": {"type": "integer", "minimum": 0, "description": "1.2. clock_jump: the time the machine slept."},
+ "cause": {"enum": ["restart", "stopped", "unknown"], "description": "1.2. recording_gap: why the turn was found open."},
+ "until": {"type": "integer", "description": "1.2. recording_gap: epoch ms of the app start that found it."},
+ "provider": {"type": "string", "description": "1.2. sandbox_restarted: which sandbox."},
+ "how": {"enum": ["restart", "regrant"], "description": "1.2. sandbox_restarted: a new sandbox, or folder grants changed in place (Windows)."}
+ }
+ },
+ "sandbox_attrs": {
+ "description": "1.2. A sandbox setup step (kind sandbox, name setup): its stages in the order run.",
+ "type": "object",
+ "additionalProperties": false,
+ "required": ["provider", "stages", "failed_stage", "trigger"],
+ "properties": {
+ "provider": {"type": "string", "description": "openshell, seatbelt, windows, runner-local."},
+ "stages": {"type": "array", "items": {"type": "object", "additionalProperties": false, "required": ["name", "started_at", "duration_ms", "status"],
+ "properties": {"name": {"enum": ["preparing", "download_image", "copy_logins", "create", "connect", "verify", "ready"]},
+ "started_at": {"$ref": "#/$defs/epoch_ms"}, "duration_ms": {"type": ["integer", "null"], "minimum": 0},
+ "status": {"enum": ["ok", "error"]}, "uncertain": {"type": "boolean", "description": "download_image only: the provider cannot say how long the pull took."}}}},
+ "failed_stage": {"enum": ["preparing", "download_image", "copy_logins", "create", "connect", "verify", "ready", null], "description": "null = it worked."},
+ "error": {"$ref": "#/$defs/sandbox_error"},
+ "trigger": {"enum": ["turn_start", "first_tool"]},
+ "ended_after_turn_ms": {"$ref": "#/$defs/ended_after_turn_ms"}
}
},
+ "sandbox_error": {"type": "object", "additionalProperties": false, "required": ["type", "text"], "description": "1.2. A sandbox failure: the exception's class and its short text.",
+ "properties": {"type": {"type": "string"}, "text": {"type": ["string", "null"]}}},
+ "woken_by": {"type": ["object", "null"], "additionalProperties": false, "description": "1.2. A team worker's turn: who woke it. null or absent = not a worker turn, or not recorded.",
+ "properties": {"lead_session_id": {"type": "string"}, "team_id": {"type": "string"}, "item_ids": {"type": "array", "items": {"type": "string"}},
+ "via": {"enum": ["board", "steer", "mention", "timer"]}}},
"no_content": {"type": "object", "additionalProperties": false, "maxProperties": 0},
"live_message": {
diff --git a/tests/test_observability_issues.py b/tests/test_observability_issues.py
new file mode 100644
index 00000000..a144cf5d
--- /dev/null
+++ b/tests/test_observability_issues.py
@@ -0,0 +1,194 @@
+"""Observability Phase 2, OB-3: the Issues checks (E8b) and the 1.2 fields they read (E8a).
+
+The scenarios in `tests/corpora/observability/golden-phase2/` are the shared test of the checks:
+a copy of ocw-context docs/observability/golden/phase2 (signals.yaml 0.2, recording format 1.2).
+Every recording must give exactly its expected issues, and the hash vectors must hash the same,
+in this engine and in the Rust harness."""
+
+from __future__ import annotations
+
+import asyncio
+import json
+from pathlib import Path
+
+import pytest
+
+from coworker.engine import TurnEngine
+from coworker.observability import Recorder, signals
+from coworker.observability.canonical import jcs, short_hash, text_hash
+from coworker.permissions import Mode, PermissionEngine
+from coworker.providers import AssistantTurn, ModelCapabilities, ProviderClient, ToolCall
+from coworker.tools import ToolRegistry
+
+GOLDEN = Path(__file__).parent / "corpora" / "observability" / "golden-phase2"
+INDEX = json.loads((GOLDEN / "index.json").read_text(encoding="utf-8"))
+
+
+def _expected(rec: dict) -> dict:
+ res = signals.check_turn(rec["turns"][0], rec["steps"])
+ fired: dict[str, list[int]] = {}
+ for i in res["issues"]:
+ fired.setdefault(i["check"], []).append(i["step_seq"])
+ return {"signals_version": res["signals_version"], "warn": res["warn"],
+ "fired": {k: sorted(x for x in v if x is not None) for k, v in sorted(fired.items())},
+ "fine": sorted(f["check"] for f in res["fine"]),
+ "not_checked": sorted(n["check"] for n in res["not_checked"])}
+
+
+def test_the_shipped_signals_file_is_the_golden_tests_version():
+ assert signals.version() == INDEX["signals_version"]
+ assert set(signals.signals()["checks"]) <= set(signals.CHECKS)
+
+
[email protected]("sid", [s["id"] for s in INDEX["scenarios"]])
+def test_golden_scenario_gives_exactly_its_issues(sid):
+ rec = json.loads((GOLDEN / sid / "recording.json").read_text(encoding="utf-8"))
+ want = json.loads((GOLDEN / sid / "expected-issues.json").read_text(encoding="utf-8"))
+ assert _expected(rec) == want
+
+
+def test_hash_vectors():
+ v = json.loads((GOLDEN / "vectors" / "hashes.json").read_text(encoding="utf-8"))
+ for case in v["args"]:
+ assert jcs(case["value"]) == case["canonical"], case["name"]
+ assert short_hash(case["value"]) == case["args_hash"], case["name"]
+ for item in v["plan_items"]:
+ assert text_hash(item["text"]) == item["h"]
+
+
+# -- E8a: what the engine records for the checks ---------------------------------------------
+
+class _Scripted(ProviderClient):
+ def __init__(self, turns):
+ self._turns = list(turns)
+
+ def complete(self, *, model, messages, tools=None, **settings):
+ return self._turns.pop(0)
+
+ def capabilities(self, model):
+ return ModelCapabilities()
+
+
+class _Meta:
+ category: str = ""
+ risk_level: str = "low"
+ requires_approval: bool = False
+
+
+def read_file(path: str) -> dict:
+ """Read a file.
+
+ Args:
+ path: the file
+ """
+ return {"error": "file not found"} if path == "missing.txt" else {"text": "hi"}
+
+
+def todo_write(todos: list = None) -> dict:
+ """Replace the task list.
+
+ Args:
+ todos: the items
+ """
+ return {"count": len(todos or [])}
+
+
+def _call(cid, name, args):
+ return AssistantTurn(tool_calls=[ToolCall(id=cid, name=name, arguments=args)], finish_reason="tool_calls")
+
+
+def _run(tmp_path, turns, *, max_iterations=12):
+ registry = ToolRegistry()
+ registry.register(read_file, metadata=_Meta())
+ registry.register(todo_write, metadata=_Meta())
+ engine = TurnEngine(
+ provider=_Scripted(turns), registry=registry,
+ permissions=PermissionEngine(workspace_root=tmp_path, mode=Mode.BYPASS_APPROVALS),
+ model="openai:model-a", instructions="You are a test coworker.", max_iterations=max_iterations,
+ )
+ engine.audit_context = {"session_id": "sess_i"}
+ rec = Recorder()
+ engine.recorder = rec
+
+ async def go():
+ return [ev async for ev in engine.run("go", trigger="user")]
+
+ asyncio.run(go())
+ return rec.as_dict()
+
+
+def test_tool_steps_carry_the_12_fields(tmp_path):
+ d = _run(tmp_path, [
+ _call("c1", "todo_write", {"todos": [{"content": "read the notes", "status": "in_progress"},
+ {"content": "answer", "status": "completed"}]}),
+ _call("c2", "read_file", {"path": "missing.txt"}),
+ _call("c3", "nope_tool", {"x": 1}),
+ AssistantTurn(text="done"),
+ ])
+ assert d["turns"][0]["max_iterations"] == 12
+ tools = {s["call_id"]: s for s in d["steps"] if s["kind"] == "tool"}
+ plan = tools["c1"]["attrs"]
+ assert plan["args_hash"] == short_hash({"todos": [{"content": "read the notes", "status": "in_progress"},
+ {"content": "answer", "status": "completed"}]})
+ assert plan["plan"] == {"items": [{"h": text_hash("read the notes"), "s": "in_progress"},
+ {"h": text_hash("answer"), "s": "done"}]}
+ assert plan["reaches_network"] is False and plan["repeat_ok"] is False and plan["error_kind"] is None
+ missing = tools["c2"]["attrs"]
+ assert tools["c2"]["status"] == "ok" and missing["result_error"] is True
+ unknown = tools["c3"]
+ assert unknown["status"] == "error" and unknown["attrs"]["error_kind"] == "unknown_tool"
+ assert unknown["attrs"]["reaches_network"] is None # it never ran
+
+
+def test_a_recorded_turn_gives_its_issues(tmp_path):
+ d = _run(tmp_path, [
+ _call("c1", "read_file", {"path": "a"}), _call("c2", "read_file", {"path": "a"}),
+ _call("c3", "read_file", {"path": "a"}), _call("c4", "nope_tool", {}),
+ AssistantTurn(text="done"),
+ ])
+ res = signals.check_turn(d["turns"][0], d["steps"])
+ fired = {i["check"] for i in res["issues"]}
+ assert {"repeated_step", "made_up_tool"} <= fired
+ assert "tool_failed" not in fired # the unknown tool is made_up_tool's, not a tool failure
+ assert any(f["check"] == "step_limit" and "of 12" in f["text"] for f in res["fine"])
+
+
+# -- E8c: stored with the turn and served ------------------------------------------------------
+
+def test_issues_are_stored_at_turn_end_and_served(tmp_path):
+ from fastapi.testclient import TestClient
+
+ from coworker.server import SessionManager, create_app
+
+ (tmp_path / "notes.txt").write_text("Friday", encoding="utf-8")
+
+ class _Provider(_Scripted):
+ def complete(self, *, model, messages, tools=None, **settings):
+ if str(messages[0].get("content") or "").startswith("You title"):
+ return AssistantTurn(text="Notes")
+ return super().complete(model=model, messages=messages, tools=tools, **settings)
+
+ provider = _Provider([
+ _call("c1", "read_file", {"path": "missing.txt"}),
+ AssistantTurn(text="It is missing.", finish_reason="stop"),
+ ])
+ manager = SessionManager(workspace=tmp_path, provider=provider)
+ client = TestClient(create_app(manager))
+ with client.websocket_connect("/ws/session/i1") as ws:
+ assert ws.receive_json()["type"] == "ready"
+ ws.send_json({"type": "user_message", "text": "read missing.txt"})
+ while ws.receive_json()["type"] != "turn_done":
+ pass
+
+ [turn] = client.get("/v1/sessions/i1/turns").json()["turns"]
+ assert turn["issues"] == {"warn": 1, "info": 0}
+ got = client.get(f"/v1/sessions/i1/turns/{turn['turn_id']}/issues").json()
+ assert got["signals_version"] == signals.version() and got["seen"] is False
+ [failed] = got["issues"]
+ assert failed["check"] == "tool_failed" and failed["facts"]["how"] == "result_error"
+ assert {f["check"] for f in got["fine"]} >= {"repeated_step", "cut_off", "step_limit"}
+ row = next(s for s in client.get("/v1/sessions").json()["sessions"] if s["session_id"] == "i1")
+ assert row["issues_unseen"] == 1
+ assert client.post("/v1/sessions/i1/issues/seen").json() == {"ok": True}
+ row = next(s for s in client.get("/v1/sessions").json()["sessions"] if s["session_id"] == "i1")
+ assert row["issues_unseen"] == 0
diff --git a/tests/test_observability_waits.py b/tests/test_observability_waits.py
index 37d1b04f..ef20d359 100644
--- a/tests/test_observability_waits.py
+++ b/tests/test_observability_waits.py
@@ -107,7 +107,7 @@ def test_an_approved_card_is_a_wait_step_and_the_call_records_its_path(tmp_path)
[wait] = _steps(rec, "wait")
assert (wait["name"], wait["call_id"], wait["status"]) == ("approval", "c1", "ok")
assert wait["attrs"] == {"for_call_id": "c1", "kind": "approval", "attended": True,
- "where": "session", "outcome": "approved"}
+ "where": "session", "outcome": "approved", "inbox_at": None}
assert wait["content"] == {"resolved_by": "devika"}
[tool] = _steps(rec, "tool")
assert tool["status"] == "ok"