Skip to content
Python
7.0 KB
Raw Blame
"""The engine's own recordings fit the shared schema (recording-format.schema.json).

The golden test checks the recorder against the shared scenarios; this checks the other
half: that what the real engine records (turns, model, tool, wait, auto-approver, summary,
context and notice steps, stamps, decision paths, exchanges) has exactly the shape of
recording format 1.x, the same file the Rust SDK is built against.
"""

from __future__ import annotations

import asyncio
import importlib
import json
from pathlib import Path
from types import SimpleNamespace

import jsonschema
import pytest

from coworker import compaction as _compaction
from coworker.engine import ApprovalOutcome
from coworker.providers import AssistantTurn
from coworker.providers.base import TokenUsage

SCHEMA = json.loads((Path(__file__).parent / "corpora/observability/recording-format.schema.json").read_text("utf-8"))
waits = importlib.import_module("test_observability_waits")
reviews = importlib.import_module("test_observability_reviewer_helper")
stamps = importlib.import_module("test_observability_stamps_context")


def _check(recording):
    validator = jsonschema.validators.validator_for(SCHEMA)(SCHEMA)
    errors = sorted(validator.iter_errors(recording), key=lambda e: list(e.absolute_path))
    assert not errors, "\n".join(f"{list(e.absolute_path)}: {e.message[:300]}" for e in errors[:8])


def test_approved_card_session_grant_and_low_risk(tmp_path):
    engine, rec = waits._engine(tmp_path, [
        waits._call("c1", "run_shell", {"command": "pytest"}),
        waits._call("c2", "run_shell", {"command": "pytest"}),
        waits._call("c3", "look", {"path": "x"}),
        AssistantTurn(text="done", usage=TokenUsage(input=10, output=2)),
    ], answer=ApprovalOutcome.ALWAYS_COMMAND)
    waits._run(engine)
    _check(rec.as_dict())


def test_denied_card(tmp_path):
    engine, rec = waits._engine(tmp_path, [waits._call("c1", "run_shell", {"command": "rm -rf b"}),
                                           AssistantTurn(text="ok")], answer=ApprovalOutcome.DENY)
    waits._run(engine)
    _check(rec.as_dict())


def test_summary_step(tmp_path, monkeypatch):
    engine, rec = waits._engine(tmp_path, [AssistantTurn(text="sum", usage=TokenUsage(input=9, output=1)),
                                           AssistantTurn(text="hi", usage=TokenUsage(input=5, output=1)),
                                           AssistantTurn(text="next")])

    def fake(messages, *, provider, model, **kw):
        provider.complete(model=model, messages=[{"role": "user", "content": "summarize"}])
        return SimpleNamespace(boundary_index=0, carries_outside=False, transcript_path=None)

    monkeypatch.setattr(_compaction, "build_state", fake)
    waits._run(engine)  # a turn, so the summary has one to belong to

    async def go():
        await engine._compact_now()

    asyncio.run(go())
    waits._run(engine, "again")
    _check(rec.as_dict())


@pytest.mark.parametrize("verdict", ["allow", "deny"])
def test_live_auto_approver_checks(tmp_path, verdict):
    engine, rec = reviews._engine(tmp_path, [reviews._call("c1", "run_shell", {"command": "make"}),
                                             AssistantTurn(text="ok")],
                                  [json.dumps({"verdict": verdict, "reason": "r"})])
    reviews._run(engine)
    _check(rec.as_dict())


def test_shadow_check_and_could_not_run(tmp_path):
    engine, rec = reviews._engine(tmp_path, [reviews._call("c1", "run_shell", {"command": "make"}),
                                             AssistantTurn(text="ok")],
                                  [json.dumps({"verdict": "allow", "reason": "fine"})], shadow=True)
    reviews._run(engine)
    _check(rec.as_dict())
    engine, rec = reviews._engine(tmp_path, [reviews._call("c1", "run_shell", {"command": "make"}),
                                             AssistantTurn(text="ok")], [RuntimeError("no key")])
    reviews._run(engine)
    _check(rec.as_dict())


def test_explore_helper(tmp_path):
    rec_holder = {}
    original = reviews.Recorder

    class Keep(original):
        def __init__(self, *a, **k):
            super().__init__(*a, **k)
            rec_holder["rec"] = self

    reviews.Recorder = Keep
    try:
        reviews.test_explore_helper_steps_are_nested_under_the_explore_step(tmp_path)
    finally:
        reviews.Recorder = original
    _check(rec_holder["rec"].as_dict())


def test_context_notices_stamps_and_sandbox_facts(tmp_path):
    engine, rec = stamps._engine(tmp_path, [stamps._call("c1", "look", {"path": "x"}), AssistantTurn(text="ok")],
                                 blocks=["Folders: /w"])
    stamps._run(engine)
    _check(rec.as_dict())
    engine, rec = stamps._engine(tmp_path, [RuntimeError("boom")])
    stamps._run(engine)
    _check(rec.as_dict())


def test_interactive_tools_and_steps_closed_by_a_stop_fit_the_schema(tmp_path):
    # Round-2 in-app check: interactive tools (plan, question, folder) had no `decision`, and
    # a step closed at its turn's end kept none of its kind's end fields.
    import asyncio as _asyncio
    from coworker.permissions import Mode

    engine, rec = waits._engine(tmp_path, [waits._call("p1", "propose_plan", {"plan": "do it"}),
                                           AssistantTurn(text="x")], mode=Mode.PLAN)

    async def never(*a):
        await _asyncio.sleep(30)

    engine.plan_approver = never

    async def go():
        async def stop_soon():
            await _asyncio.sleep(0.3)
            engine.request_interrupt()
        task = _asyncio.create_task(stop_soon())
        [ev async for ev in engine.run("plan", trigger="user")]
        await task

    _asyncio.run(go())
    [plan] = [s for s in rec.as_dict()["steps"] if s["name"] == "propose_plan"]
    assert plan["attrs"]["decision"] == {"path": [], "origin": None}
    _check(rec.as_dict())


def test_a_model_step_still_open_at_the_turns_end_fits_the_schema():
    from coworker.observability import Recorder

    rec = Recorder()
    base = {"v": 1, "session_id": "s", "turn_id": "t"}
    rec.emit({**base, "type": "turn_started", "at_ms": 1000, "mono_ms": 0, "kind": "run", "trigger": "user",
              "first_message_id": "m1", "stamp": {},
              "attrs": {"approval_mode": "interactive", "attended": True,
                        "reviewer": {"active": False, "shadow": False, "kind": None, "model": None}}})
    rec.emit({**base, "type": "step_started", "step_id": "x", "at_ms": 1010, "mono_ms": 10, "kind": "model",
              "name": "model", "parent_step_id": None, "call_id": None, "call_index": None, "shadow": False,
              "after_turn": False, "content": {},
              "attrs": {"purpose": "title", "provider": "openai", "provider_title": "OpenAI", "model": "openai:model-a",
                        "stream": False, "context_window": None, "tools_offered": 0, "tools_hash": None}})
    rec.emit({**base, "type": "turn_ended", "at_ms": 1100, "mono_ms": 100, "status": "completed", "last_message_id": "m2"})
    [step] = rec.as_dict()["steps"]
    assert step["status"] == "interrupted" and step["attrs"]["usage"] == {"reported": False}
    _check(rec.as_dict())