AI-powered virtual executive team — a single coherent executive persona backed by 8 specialist agents (FastAPI + Next.js).
"""Unit tests for per-call model usage recording (audit.usage)."""
from __future__ import annotations
from pathlib import Path
from types import SimpleNamespace
import pytest
from openexecutive.audit import usage as au
from openexecutive.audit.logger import AuditLogger, set_audit_logger
@pytest.fixture
def audit(tmp_path: Path) -> AuditLogger:
logger = AuditLogger(tmp_path / "audit.db")
set_audit_logger(logger)
yield logger # type: ignore[misc]
set_audit_logger(None)
def _message(**usage: object) -> SimpleNamespace:
return SimpleNamespace(usage=SimpleNamespace(**usage), stop_reason="end_turn")
def test_usage_counts_reads_tokens_cost_and_server_searches() -> None:
msg = _message(
input_tokens=120, output_tokens=30, cache_creation_input_tokens=10,
cache_read_input_tokens=900, cost="0.04",
server_tool_use=SimpleNamespace(web_search_requests=3),
)
counts = au.usage_counts(msg)
assert counts == {
"input_tokens": 120, "output_tokens": 30,
"cache_creation_input_tokens": 10, "cache_read_input_tokens": 900,
"cost_usd": 0.04, "web_search_requests": 3,
}
def test_usage_counts_tolerates_missing_fields_and_no_usage() -> None:
assert au.usage_counts(SimpleNamespace(content=[])) is None
# A mock (or a usage object without integer counts) is not usage.
from unittest.mock import MagicMock
assert au.usage_counts(MagicMock()) is None
assert au.usage_counts(SimpleNamespace(usage=SimpleNamespace(cost="0.1"))) is None
counts = au.usage_counts(_message(input_tokens=5, cost="not-a-number"))
assert counts is not None
assert counts["output_tokens"] == 0 and counts["web_search_requests"] == 0
assert counts["cost_usd"] is None
def test_log_model_usage_writes_one_cache_event_row(audit: AuditLogger) -> None:
msg = _message(input_tokens=50, output_tokens=7,
server_tool_use=SimpleNamespace(web_search_requests=2))
counts = au.log_model_usage(msg, model="claude-sonnet-5", actor="specialist_research",
iteration=1, session_id="s1", turn_id="t1")
assert counts is not None and counts["web_search_requests"] == 2
rows = audit.query(event_type="cache_event")
assert len(rows) == 1
row = rows[0]
assert row.actor == "specialist_research" and row.session_id == "s1"
assert row.details["model"] == "claude-sonnet-5"
assert row.details["input_tokens"] == 50 and row.details["web_search_requests"] == 2
assert "run_id" not in row.details
def test_log_model_usage_tags_rows_with_the_bound_research_run(audit: AuditLogger) -> None:
with au.bind_research_run("run-abc") as rollup:
assert au.get_research_run_id() == "run-abc"
au.log_model_usage(_message(input_tokens=1), model="m", actor="research_synthesis")
au.log_model_usage(
_message(input_tokens=4, server_tool_use=SimpleNamespace(web_search_requests=2)),
model="m", actor="specialist_research",
)
assert au.get_research_run_id() is None
# The bound rollup summed both calls; a call outside the block does not count.
au.log_model_usage(_message(input_tokens=100), model="m", actor="executive")
out = rollup.as_dict()
assert out["calls"] == 2 and out["input_tokens"] == 5 and out["web_search_requests"] == 2
assert set(out["by_source"]) == {"research_synthesis", "specialist_research"}
au.log_model_usage(_message(input_tokens=1), model="m", actor="executive")
rows = {r.actor: r for r in audit.query(event_type="cache_event")}
assert rows["research_synthesis"].details["run_id"] == "run-abc"
assert rows["specialist_research"].details["run_id"] == "run-abc"
assert "run_id" not in rows["executive"].details
@pytest.mark.asyncio
async def test_rollup_collects_calls_made_in_gathered_tasks(audit: AuditLogger) -> None:
"""The specialist fan-out runs in asyncio.gather sub-tasks; they inherit
the binding and share the rollup object."""
import asyncio
async def one_call(i: int) -> None:
au.log_model_usage(_message(input_tokens=i), model="m", actor="specialist_research")
with au.bind_research_run("run-gather") as rollup:
await asyncio.gather(one_call(1), one_call(2), one_call(3))
assert rollup.as_dict()["calls"] == 3 and rollup.as_dict()["input_tokens"] == 6
def test_log_model_usage_never_raises_on_a_malformed_usage_block(audit: AuditLogger) -> None:
msg = _message(input_tokens=float("inf"), output_tokens=3)
assert au.log_model_usage(msg, model="m", actor="triage") is not None
row = audit.query(event_type="cache_event")[0]
assert row.details["input_tokens"] == 0 and row.details["output_tokens"] == 3
def test_log_model_usage_returns_none_without_usage(audit: AuditLogger) -> None:
assert au.log_model_usage(SimpleNamespace(content=[]), model="m", actor="triage") is None
assert audit.query(event_type="cache_event") == []
def test_usage_rollup_sums_per_source() -> None:
roll = au.UsageRollup()
roll.add("specialist_research", {"input_tokens": 10, "output_tokens": 2, "web_search_requests": 3})
roll.add("specialist_research", {"input_tokens": 5, "output_tokens": 1, "web_search_requests": 1})
roll.add("research_synthesis", {"input_tokens": 7, "cache_read_input_tokens": 100})
roll.add("research_watchlist", None) # a call without usage still counts as a call
out = roll.as_dict()
assert out["calls"] == 4
assert out["by_source"]["research_watchlist"]["calls"] == 1
assert out["input_tokens"] == 22 and out["web_search_requests"] == 4
assert out["cache_read_input_tokens"] == 100
assert out["by_source"]["specialist_research"] == {
"calls": 2, "input_tokens": 15, "cache_read_input_tokens": 0,
"cache_creation_input_tokens": 0, "output_tokens": 3, "web_search_requests": 4,
}
assert list(out["by_source"]) == ["research_synthesis", "research_watchlist", "specialist_research"]
def test_log_model_usage_counts_a_call_without_usage_in_the_rollup(audit: AuditLogger) -> None:
with au.bind_research_run("run-x") as rollup:
assert au.log_model_usage(SimpleNamespace(content=[]), model="m", actor="triage") is None
assert rollup.as_dict()["calls"] == 1
assert audit.query(event_type="cache_event") == [] # no row without counts