Python
"""The engine's own recordings fit the shared schema (recording-format.schema.json).
The golden test checks the recorder against the shared scenarios; this checks the other
half: that what the real engine records (turns, model, tool, wait, auto-approver, summary,
context and notice steps, stamps, decision paths, exchanges) has exactly the shape of
recording format 1.x, the same file the Rust SDK is built against.
"""
from __future__ import annotations
import asyncio
import importlib
import json
from pathlib import Path
from types import SimpleNamespace
import jsonschema
import pytest
from coworker import compaction as _compaction
from coworker.engine import ApprovalOutcome
from coworker.providers import AssistantTurn
from coworker.providers.base import TokenUsage
SCHEMA = json.loads((Path(__file__).parent / "corpora/observability/recording-format.schema.json").read_text("utf-8"))
waits = importlib.import_module("test_observability_waits")
reviews = importlib.import_module("test_observability_reviewer_helper")
stamps = importlib.import_module("test_observability_stamps_context")
def _check(recording):
validator = jsonschema.validators.validator_for(SCHEMA)(SCHEMA)
errors = sorted(validator.iter_errors(recording), key=lambda e: list(e.absolute_path))
assert not errors, "\n".join(f"{list(e.absolute_path)}: {e.message[:300]}" for e in errors[:8])
def test_approved_card_session_grant_and_low_risk(tmp_path):
engine, rec = waits._engine(tmp_path, [
waits._call("c1", "run_shell", {"command": "pytest"}),
waits._call("c2", "run_shell", {"command": "pytest"}),
waits._call("c3", "look", {"path": "x"}),
AssistantTurn(text="done", usage=TokenUsage(input=10, output=2)),
], answer=ApprovalOutcome.ALWAYS_COMMAND)
waits._run(engine)
_check(rec.as_dict())
def test_denied_card(tmp_path):
engine, rec = waits._engine(tmp_path, [waits._call("c1", "run_shell", {"command": "rm -rf b"}),
AssistantTurn(text="ok")], answer=ApprovalOutcome.DENY)
waits._run(engine)
_check(rec.as_dict())
def test_summary_step(tmp_path, monkeypatch):
engine, rec = waits._engine(tmp_path, [AssistantTurn(text="sum", usage=TokenUsage(input=9, output=1)),
AssistantTurn(text="hi", usage=TokenUsage(input=5, output=1)),
AssistantTurn(text="next")])
def fake(messages, *, provider, model, **kw):
provider.complete(model=model, messages=[{"role": "user", "content": "summarize"}])
return SimpleNamespace(boundary_index=0, carries_outside=False, transcript_path=None)
monkeypatch.setattr(_compaction, "build_state", fake)
waits._run(engine) # a turn, so the summary has one to belong to
async def go():
await engine._compact_now()
asyncio.run(go())
waits._run(engine, "again")
_check(rec.as_dict())
@pytest.mark.parametrize("verdict", ["allow", "deny"])
def test_live_auto_approver_checks(tmp_path, verdict):
engine, rec = reviews._engine(tmp_path, [reviews._call("c1", "run_shell", {"command": "make"}),
AssistantTurn(text="ok")],
[json.dumps({"verdict": verdict, "reason": "r"})])
reviews._run(engine)
_check(rec.as_dict())
def test_shadow_check_and_could_not_run(tmp_path):
engine, rec = reviews._engine(tmp_path, [reviews._call("c1", "run_shell", {"command": "make"}),
AssistantTurn(text="ok")],
[json.dumps({"verdict": "allow", "reason": "fine"})], shadow=True)
reviews._run(engine)
_check(rec.as_dict())
engine, rec = reviews._engine(tmp_path, [reviews._call("c1", "run_shell", {"command": "make"}),
AssistantTurn(text="ok")], [RuntimeError("no key")])
reviews._run(engine)
_check(rec.as_dict())
def test_explore_helper(tmp_path):
rec_holder = {}
original = reviews.Recorder
class Keep(original):
def __init__(self, *a, **k):
super().__init__(*a, **k)
rec_holder["rec"] = self
reviews.Recorder = Keep
try:
reviews.test_explore_helper_steps_are_nested_under_the_explore_step(tmp_path)
finally:
reviews.Recorder = original
_check(rec_holder["rec"].as_dict())
def test_context_notices_stamps_and_sandbox_facts(tmp_path):
engine, rec = stamps._engine(tmp_path, [stamps._call("c1", "look", {"path": "x"}), AssistantTurn(text="ok")],
blocks=["Folders: /w"])
stamps._run(engine)
_check(rec.as_dict())
engine, rec = stamps._engine(tmp_path, [RuntimeError("boom")])
stamps._run(engine)
_check(rec.as_dict())
def test_interactive_tools_and_steps_closed_by_a_stop_fit_the_schema(tmp_path):
# Round-2 in-app check: interactive tools (plan, question, folder) had no `decision`, and
# a step closed at its turn's end kept none of its kind's end fields.
import asyncio as _asyncio
from coworker.permissions import Mode
engine, rec = waits._engine(tmp_path, [waits._call("p1", "propose_plan", {"plan": "do it"}),
AssistantTurn(text="x")], mode=Mode.PLAN)
async def never(*a):
await _asyncio.sleep(30)
engine.plan_approver = never
async def go():
async def stop_soon():
await _asyncio.sleep(0.3)
engine.request_interrupt()
task = _asyncio.create_task(stop_soon())
[ev async for ev in engine.run("plan", trigger="user")]
await task
_asyncio.run(go())
[plan] = [s for s in rec.as_dict()["steps"] if s["name"] == "propose_plan"]
assert plan["attrs"]["decision"] == {"path": [], "origin": None}
_check(rec.as_dict())
def test_a_model_step_still_open_at_the_turns_end_fits_the_schema():
from coworker.observability import Recorder
rec = Recorder()
base = {"v": 1, "session_id": "s", "turn_id": "t"}
rec.emit({**base, "type": "turn_started", "at_ms": 1000, "mono_ms": 0, "kind": "run", "trigger": "user",
"first_message_id": "m1", "stamp": {},
"attrs": {"approval_mode": "interactive", "attended": True,
"reviewer": {"active": False, "shadow": False, "kind": None, "model": None}}})
rec.emit({**base, "type": "step_started", "step_id": "x", "at_ms": 1010, "mono_ms": 10, "kind": "model",
"name": "model", "parent_step_id": None, "call_id": None, "call_index": None, "shadow": False,
"after_turn": False, "content": {},
"attrs": {"purpose": "title", "provider": "openai", "provider_title": "OpenAI", "model": "openai:model-a",
"stream": False, "context_window": None, "tools_offered": 0, "tools_hash": None}})
rec.emit({**base, "type": "turn_ended", "at_ms": 1100, "mono_ms": 100, "status": "completed", "last_message_id": "m2"})
[step] = rec.as_dict()["steps"]
assert step["status"] == "interrupted" and step["attrs"]["usage"] == {"reported": False}
_check(rec.as_dict())