Python
"""OPE-238: at the end of a turn the engine says which actions ran without asking.
An action "ran without asking" when auto-approve let it through with no approval card:
a gate tier (OPE-233 inert writes) or a reviewer "allow". The engine emits one
`turn_summary` event listing them, just before the event that ends the turn, so the
event that ends a turn is still the last one. Actions the user approved on a card, or
that the mode allows outright, are not in it.
Each item names its tool call, and the audit log can be filtered by that call's id,
so a surface can link an item to its audit rows.
"""
from __future__ import annotations
import asyncio
from coworker.audit import AuditStore
from coworker.events import EventType
from coworker.providers import AssistantTurn
from test_auto_approve import _engine, _FakeReviewer, _run, _tool_turn
def _summaries(events):
return [ev for ev in events if ev.type == EventType.TURN_SUMMARY]
def test_lists_gate_tier_and_reviewer_allowed_actions_before_turn_end(tmp_path):
engine, rows, approvals = _engine(
tmp_path,
[
_tool_turn(("write_file", {"path": "summary.txt", "content": "3 tasks done"})),
_tool_turn(("run_shell", {"command": "pytest -q"})),
_tool_turn(("write_file", {"path": "tool.py", "content": "print(1)"})),
AssistantTurn(text="done", finish_reason="stop"),
],
)
engine.permissions.inert_writes = "on"
engine.reviewer = _FakeReviewer({"run_shell": "allow", "write_file": "unsure"})
events = _run(engine, "do the tasks")
assert approvals == ["write_file"] # tool.py: the reviewer was unsure, the user approved
summaries = _summaries(events)
assert len(summaries) == 1
# Right before the event that ends the turn.
assert events[-1].type == EventType.TURN_END
assert events[-2] is summaries[0]
items = summaries[0].data["actions"]
assert [(i["name"], i["origin"]) for i in items] == [
("write_file", "gate_tier"),
("run_shell", "reviewer"),
]
assert all(i["status"] == "ok" for i in items)
assert all(i["tool_call_id"] for i in items)
assert items[1]["note"] == "scripted allow" # the reviewer's one-line reason rides along
assert items[0]["tier"] == "inert_write" and "tier" not in items[1]
assert all(i["effect"] == "change" for i in items)
assert summaries[0].data["count"] == 2
assert summaries[0].data["failed"] == 0 and summaries[0].data["reads"] == 0
def test_no_event_when_everything_was_asked(tmp_path):
engine, rows, approvals = _engine(
tmp_path,
[_tool_turn(("run_shell", {"command": "make deploy"})), AssistantTurn(text="done", finish_reason="stop")],
)
engine.reviewer = _FakeReviewer({"run_shell": "unsure"})
events = _run(engine, "deploy")
assert approvals == ["run_shell"]
assert _summaries(events) == []
def test_no_event_for_a_turn_without_tools(tmp_path):
engine, _rows, _approvals = _engine(tmp_path, [AssistantTurn(text="hi", finish_reason="stop")])
assert _summaries(_run(engine, "hello")) == []
def test_a_reviewer_deny_is_not_listed(tmp_path):
engine, _rows, _approvals = _engine(
tmp_path,
[_tool_turn(("run_shell", {"command": "rm -rf build"})), AssistantTurn(text="ok", finish_reason="stop")],
)
engine.reviewer = _FakeReviewer({"run_shell": "deny"})
assert _summaries(_run(engine, "clean up")) == []
def test_each_turn_gets_its_own_summary(tmp_path):
engine, _rows, _approvals = _engine(
tmp_path,
[
_tool_turn(("run_shell", {"command": "pytest -q"})),
AssistantTurn(text="first", finish_reason="stop"),
_tool_turn(("run_shell", {"command": "ruff check"})),
_tool_turn(("run_shell", {"command": "mypy ."})),
AssistantTurn(text="second", finish_reason="stop"),
],
)
engine.reviewer = _FakeReviewer({"run_shell": "allow"})
# Both requests in ONE event loop, as in the app. The engine's stop signal is an
# asyncio.Event; a second asyncio.run() would wait on it from a different loop, and
# that wait failing reads as "stopped", dropping a model turn at random.
async def _both():
first = [ev async for ev in engine.run("run the tests")]
second = [ev async for ev in engine.run("lint and type-check")]
return _summaries(first), _summaries(second)
first, second = asyncio.run(_both())
assert [s.data["count"] for s in first] == [1]
assert [s.data["count"] for s in second] == [2]
def test_a_failed_action_is_listed_with_its_status(tmp_path):
engine, _rows, _approvals = _engine(
tmp_path,
[_tool_turn(("run_shell", {"command": "pytest -q"})), AssistantTurn(text="done", finish_reason="stop")],
)
engine.reviewer = _FakeReviewer({"run_shell": "allow"})
def run_shell(command: str) -> str:
"""Run a command.
Args:
command: the command line
"""
raise RuntimeError("exit 1")
engine.registry.register(run_shell, metadata=engine.registry.get("run_shell").metadata)
items = _summaries(_run(engine, "run the tests"))[0].data["actions"]
assert items[0]["name"] == "run_shell"
assert items[0]["status"] != "ok"
def test_the_summary_carries_no_arguments(tmp_path):
# The tool events already carry what ran; the summary stays a short index of it.
engine, _rows, _approvals = _engine(
tmp_path,
[_tool_turn(("run_shell", {"command": "echo SECRET_SHAPED"})), AssistantTurn(text="done", finish_reason="stop")],
)
engine.reviewer = _FakeReviewer({"run_shell": "allow"})
summary = _summaries(_run(engine, "echo"))[0]
assert "SECRET_SHAPED" not in repr(summary.data)
def test_closing_the_turn_early_still_closes_the_inner_run(tmp_path):
# A surface that stops reading mid-turn (a disconnect) must still end the run, so the
# run grants are cleared as before.
engine, _rows, _approvals = _engine(
tmp_path,
[_tool_turn(("run_shell", {"command": "pytest -q"})), AssistantTurn(text="done", finish_reason="stop")],
)
engine.reviewer = _FakeReviewer({"run_shell": "allow"})
cleared: list[bool] = []
original = engine.permissions.clear_run_allowances
def spy():
cleared.append(True)
original()
engine.permissions.clear_run_allowances = spy
async def _go():
gen = engine.run("run the tests")
async for ev in gen:
if ev.type == EventType.TOOL_FINISHED:
break
await gen.aclose()
asyncio.run(_go())
assert len(cleared) >= 2 # once at the start, once in the run's finally
# -- the audit log can be filtered by tool call ------------------------------------------
def test_audit_store_filters_by_call_id(tmp_path):
store = AuditStore(tmp_path / "audit.db")
for call_id, stage in (("a1", "started"), ("a1", "finished"), ("b2", "finished")):
store.append({"session_id": "s", "tool": "run_shell", "stage": stage, "call_id": call_id, "arguments": {}})
rows = store.list(call_id="a1")
assert sorted(r["stage"] for r in rows) == ["finished", "started"]
assert {r["call_id"] for r in rows} == {"a1"}
assert len(store.list()) == 3
# -- the count means "changed state, and nobody approved it" ---------------------------
def _shell_returning(engine, results: dict[str, dict]):
"""Replace the test shell with one that answers each command from `results`."""
def run_shell(command: str) -> dict:
"""Run a command.
Args:
command: the command line
"""
return results[command]
engine.registry.register(run_shell, metadata=engine.registry.get("run_shell").metadata)
def test_the_proposed_event_carries_the_call_id(tmp_path):
# Two same-named calls in one batch must pair with their own finished events.
engine, _rows, _approvals = _engine(
tmp_path,
[
_tool_turn(("run_shell", {"command": "pytest -q"}), ("run_shell", {"command": "ruff check"})),
AssistantTurn(text="done", finish_reason="stop"),
],
)
engine.reviewer = _FakeReviewer({"run_shell": "allow"})
events = _run(engine, "check")
proposed = [ev.data["tool_call_id"] for ev in events if ev.type == EventType.TOOL_PROPOSED]
finished = [ev.data["tool_call_id"] for ev in events if ev.type == EventType.TOOL_FINISHED]
assert len(proposed) == 2 and all(proposed)
assert proposed == finished
def test_a_shell_commands_outcome_rides_on_the_finished_event_not_the_preview(tmp_path):
long_command = "python -c '" + "x = 1; " * 80 + "'" # pushes exit_code past the 300-char preview
engine, _rows, _approvals = _engine(
tmp_path,
[
_tool_turn(("run_shell", {"command": long_command})),
_tool_turn(("run_shell", {"command": "sleep 999"})),
_tool_turn(("run_shell", {"command": "npm run dev"})),
AssistantTurn(text="done", finish_reason="stop"),
],
)
engine.reviewer = _FakeReviewer({"run_shell": "allow"})
_shell_returning(
engine,
{
long_command: {"command": long_command, "cwd": "/w", "exit_code": 1, "output": "", "timed_out": False},
"sleep 999": {"command": "sleep 999", "cwd": "/w", "exit_code": None, "output": "", "timed_out": True},
"npm run dev": {"task_id": "bg-1", "command": "npm run dev", "status": "running"},
},
)
events = _run(engine, "run things")
finished = [ev.data for ev in events if ev.type == EventType.TOOL_FINISHED]
assert '"exit_code"' not in finished[0]["result_preview"] # the preview alone would say nothing
assert finished[0]["exit_code"] == 1 and "timed_out" not in finished[0]
assert finished[1]["exit_code"] is None and finished[1]["timed_out"] is True
assert finished[2]["background"] is True and "exit_code" not in finished[2]
# The same facts are persisted with the result, so a reload shows the same thing.
sidecars = [m["_display"] for m in engine.messages if m.get("role") == "tool"]
assert sidecars[0]["exit_code"] == 1
assert sidecars[1]["timed_out"] is True
assert sidecars[2]["background"] is True
# The summary: three changes attempted, two failed, one started in the background.
summary = _summaries(events)[0].data
assert (summary["count"], summary["failed"], summary["reads"]) == (1, 2, 0)
assert summary["actions"][0]["exit_code"] == 1
assert summary["actions"][1]["timed_out"] is True
assert summary["actions"][2]["background"] is True
def test_a_failed_call_keeps_its_status_on_the_persisted_message(tmp_path):
engine, _rows, _approvals = _engine(
tmp_path,
[_tool_turn(("run_shell", {"command": "pytest -q"})), AssistantTurn(text="done", finish_reason="stop")],
)
engine.reviewer = _FakeReviewer({"run_shell": "allow"})
def run_shell(command: str) -> str:
"""Run a command.
Args:
command: the command line
"""
raise RuntimeError("exit 1")
engine.registry.register(run_shell, metadata=engine.registry.get("run_shell").metadata)
events = _run(engine, "run the tests")
tool_msg = next(m for m in engine.messages if m.get("role") == "tool")
assert tool_msg["_display"]["status"] == "error"
assert tool_msg["_display"]["approval_origin"] == "reviewer"
summary = _summaries(events)[0].data
assert (summary["count"], summary["failed"]) == (0, 1)
def test_effect_follows_the_risk_class_and_the_read_only_tier(tmp_path):
from coworker.providers import ToolCall
engine, _rows, _approvals = _engine(tmp_path, [])
meta = engine.registry.get("run_shell").metadata
shell = ToolCall(id="c1", name="run_shell", arguments={"command": "cat x"})
assert engine._effect(shell, meta, "") == "change" # a command is a change unless classified
assert engine._effect(shell, meta, "readonly_command") == "read" # OPE-234's tier
write = ToolCall(id="c2", name="write_file", arguments={"path": "a.txt", "content": ""})
assert engine._effect(write, engine.registry.get("write_file").metadata, "inert_write") == "change"
fetch = ToolCall(id="c3", name="web_fetch", arguments={"url": "https://example.com"})
assert engine._effect(fetch, None, "") == "read" # a web read is a read
mcp = ToolCall(id="c4", name="mcp__jira__create_issue", arguments={})
assert engine._effect(mcp, None, "") == "change" # third-party tools that gate are writes or sends