Skip to content

AI-powered virtual executive team — a single coherent executive persona backed by 8 specialist agents (FastAPI + Next.js).

fix(audit): log model usage from every call site (#427)

maarten-nem37d97b336b57
diff --git a/packages/core/openexecutive/agents/alert_review.py b/packages/core/openexecutive/agents/alert_review.py
index 66ad8726..bbf62303 100644
--- a/packages/core/openexecutive/agents/alert_review.py
+++ b/packages/core/openexecutive/agents/alert_review.py
@@ -12,6 +12,7 @@ from typing import Any
from pydantic import BaseModel, ConfigDict, Field
from openexecutive.agents.base import BaseAgent
+from openexecutive.audit.usage import log_model_usage
from openexecutive.providers import get_provider
logger = logging.getLogger(__name__)
@@ -183,10 +184,11 @@ class AlertReviewAgent(BaseAgent):
async def review(self, batch_context: str) -> list[AlertVerdict]:
"""Return verdicts for one batch; empty on any failure (never raises)."""
- provider = get_provider(self.effective_model())
+ model = self.effective_model()
+ provider = get_provider(model)
try:
message = await provider.messages_create(
- model=self.effective_model(),
+ model=model,
max_tokens=_MAX_TOKENS,
timeout=_REVIEW_TIMEOUT,
system=self.effective_system_prompt(),
@@ -197,6 +199,7 @@ class AlertReviewAgent(BaseAgent):
except Exception:
logger.exception("alert_review: model call failed — no verdicts this batch")
return []
+ log_model_usage(message, model=model, actor=self.name)
for block in getattr(message, "content", []) or []:
if getattr(block, "type", "") == "tool_use" and getattr(block, "name", "") == "emit_alert_reviews":
return parse_verdicts(getattr(block, "input", {}))
diff --git a/packages/core/openexecutive/api/routes/chat.py b/packages/core/openexecutive/api/routes/chat.py
index b14c821b..619c65fe 100644
--- a/packages/core/openexecutive/api/routes/chat.py
+++ b/packages/core/openexecutive/api/routes/chat.py
@@ -1850,6 +1850,7 @@ async def _fast_json_call(
malformed or non-object JSON) — callers fall back rather than raise.
"""
from openexecutive.agents.utility_fast import get_fast_model
+ from openexecutive.audit.usage import log_model_usage
from openexecutive.config import get_settings
from openexecutive.providers import get_provider
@@ -1864,6 +1865,7 @@ async def _fast_json_call(
),
timeout=get_settings().utility_fast_timeout_s,
)
+ log_model_usage(response, model=model, actor=log_tag)
text_blocks = [b for b in response.content if getattr(b, "type", "") == "text"]
raw = text_blocks[0].text.strip() if text_blocks else ""
if raw.startswith("```"):
diff --git a/packages/core/openexecutive/architecture/architecture-facts.yaml b/packages/core/openexecutive/architecture/architecture-facts.yaml
index d66d2bc5..11b1bf95 100644
--- a/packages/core/openexecutive/architecture/architecture-facts.yaml
+++ b/packages/core/openexecutive/architecture/architecture-facts.yaml
@@ -4911,7 +4911,7 @@ audit:
- alert — alert created by the triage pipeline
- integration_inbound — Slack / Discord / email / Telegram / Google Chat message received. The chat adapters put the first 160 characters of the message in the summary; the email poller keeps the summary to sender and subject and stores the text in the full payload (`message` — the sender's new text, cut at 20,000 escaped JSON chars, the measure the 64 KB cap uses; `attachments` — filenames, 8,000 escaped chars; `body` — the whole body with its quoted chain, only on a row private to the principal, since the chain quotes other people's mail and a public row is every signed-in user's) with `details.preview` (120 chars of the new text) and `details.body_len`, so /audit shows the request next to the reply's `chat_turn` row. `details.outcome` distinguishes the non-reply paths — `rejected_unknown_sender` (not on the People roster), `skipped_gate` (response gate said no), `dropped_thread_unreadable` (Slack wouldn't return a thread the bot is in, so a continuation went unanswered; the sender was told), and `handler_error` (the turn raised; the user got the generic apology). The error row carries the same `session_id` as the inbound row, so /audit can join a failed turn to the message that caused it.
- knowledge_retrieval — one row per `knowledge.retriever.retrieve()` call; details carry chunk counts + domains, full payload carries each chunk's source, distance, and text preview so the session view shows "which RAG chunks influenced this response"
- - cache_event — one row per model call, from every call site (`audit.usage.log_model_usage`): Executive iterations and the committee revision pass (`actor` executive / committee_revision), `BaseAgent.analyze` callers — chat-turn consults via `route_parallel` (`actor` specialist), workflow steps and other direct `route_to_specialist` callers (specialist_workflow), the MCP `consult_specialist` tool (specialist_mcp), and both Council test boxes (agent_test) — `BaseAgent.analyze_with_tools` callers (specialist_research, query_watch, signal_enrichment), the research synthesis and watchlist passes (research_synthesis / research_watchlist), triage, and the chat memory extractor (memory_extractor). Captures `input_tokens`, `output_tokens`, `cache_creation_input_tokens`, `cache_read_input_tokens`, `cost_usd`, `web_search_requests` (from `usage.server_tool_use`), `stop_reason`, and — inside a research run — the run's `run_id`, so prompt-cache hits are visible per turn, plus `cost_usd` — the actual USD OpenRouter charged for the call (read from `usage.cost`; null on the Anthropic-direct path or for rows logged before cost capture shipped). This is the token-usage + cost source of truth aggregated by both `/audit/sessions/{id}` (per session) and `/audit/usage` (cross-session).
+ - cache_event — one row per model call, from every call site (`audit.usage.log_model_usage`): Executive iterations and the committee revision pass (`actor` executive / committee_revision), `BaseAgent.analyze` callers — chat-turn consults via `route_parallel` (`actor` specialist), workflow steps and other direct `route_to_specialist` callers (specialist_workflow), the MCP `consult_specialist` tool (specialist_mcp), and both Council test boxes (agent_test) — `BaseAgent.analyze_with_tools` callers (specialist_research, query_watch, signal_enrichment), the research synthesis and watchlist passes (research_synthesis / research_watchlist), triage, the chat memory extractor (memory_extractor), alert review (alert_review), the brief's narrative and grounding repair (briefing_narrative / briefing_grounding), the end_of_day_digest, executive_reflection and weekly_review workflows, wait_for_human reply parsing (workflow replies and roster answers) and the inbound_resolver, committee reviewers (committee_reviewer), the fixture and engagement generators (their agent names), and the utility calls — session_title, response_gate, person_insight, initiatives_consolidation, pdf_reader, the chat page's suggested_prompts / followup_suggestion, and the eval runner's eval_judge. `tests/unit/test_usage_recording.py::test_every_model_call_records_usage` walks the package's AST and fails on any function that calls `messages_create` / `messages_stream` without calling `log_model_usage` (directly, through a same-module wrapper such as the Executive's `_emit_cache_event`, or from an enclosing function), so a new call site cannot skip the row silently; `providers/` and the runtime-dead `architecture/generator.py` are exempt, and the repo-root `evals/` harness (a separate client process with its own SDK client) is out of scope. Captures `input_tokens`, `output_tokens`, `cache_creation_input_tokens`, `cache_read_input_tokens`, `cost_usd`, `web_search_requests` (from `usage.server_tool_use`), `stop_reason`, and — inside a research run — the run's `run_id`, so prompt-cache hits are visible per turn, plus `cost_usd` — the actual USD OpenRouter charged for the call (read from `usage.cost`; null on the Anthropic-direct path or for rows logged before cost capture shipped). This is the token-usage + cost source of truth aggregated by both `/audit/sessions/{id}` (per session) and `/audit/usage` (cross-session).
- memory_snapshot — one row at Executive turn entry; captures the system-block layout, conversation history length, episodic context (full text in `full_json`), and a company-profile hash so the session view can render "what the agent saw" before any API call
- committee_review — one row per committee-reviewed turn; carries draft length, final length, per-reviewer critiques and timings (previously emitted but newly declared as a recognised event_type)
- peer_memory — one row per `openexecutive.memory.honcho_client` call (prefetch / sync_turn / seed_identity / directional_chat / prefetch_department / sync_department_turn / append_department_note); carries `op`, `person_id`, optional `department_slug`, `outcome` (ok / timeout / error / disabled / no_person / no_department / no_loop / empty), `duration_ms`, plus op-specific details (query preview, reasoning_level, peer count, co_present_person_ids, co_present_department_slugs, originating_person_id, target_person_id, session_id, kind) so the flow-chart view renders the peer_memory path alongside `<retrieved_context>` and `<past_decisions>`. Disabled / no-Person / no-Department turns still emit (outcome marker) so audit absence equals "code path never ran" instead of "the provider was down."
diff --git a/packages/core/openexecutive/architecture/prebuilt/audit.json b/packages/core/openexecutive/architecture/prebuilt/audit.json
index 8db9791e..7cfe8019 100644
--- a/packages/core/openexecutive/architecture/prebuilt/audit.json
+++ b/packages/core/openexecutive/architecture/prebuilt/audit.json
@@ -1,7 +1,7 @@
{
"section_id": "audit",
"title": "Audit Log",
- "markdown": "The audit log is a persistent, searchable, append-only record of what the system did — distinct from episodic memory (which biases future LLM behavior). Every event flows through `audit/logger.log_event`, a synchronous SQLite writer (WAL + 5s busy_timeout) that appends one row to the `audit_log` table in `episodic_memory.db`.\n\n### Event types\nRows are tagged by `event_type`, including `chat_turn`, `specialist_consult`, `tool_invocation`, `scheduled_action`, `alert`, `integration_inbound` (Slack / Discord / email / Telegram / Google Chat), plus `knowledge_retrieval`, `fact_confirmation` (an emailed standing-fact change held, applied, cancelled or refused; private), `fact_reviewed` (the principal approved or declined a teammate's proposed standing fact; private), `fact_approval_changed` (the principal turned \"needs my approval\" on or off for a teammate; private), `fact_retired` (the principal, or the teammate who recorded it, retired a standing fact from the Pulse page; private, since it names the fact a teammate can no longer read; the fact tools' own rows and their `skill:` dispatch rows are private `tool_invocation` rows, since they quote the principal and name their session and turn), `memory_extraction` (one row per episodic-extraction pass, written in a `finally` so a crashed pass is recorded too: how many items the model proposed, how many were stored, a `{kind, reason}` record per drop, a `malformed` count for payload shapes that were never usable items, and a `failure` naming the exception type when the pass raised — including `CancelledError`, which is a `BaseException` and so would otherwise be logged as a clean empty pass — a `proposed>0, stored=0` streak means the quote validator is rejecting everything, which is otherwise invisible), `cache_event` (one row per model call from every call site — Executive turns, specialist consults (chat turns and workflow steps), research specialists, the research routing and watchlist passes, triage, memory extraction — carrying tokens, cache hits, cost, server-side search count and, inside a research run, its `run_id`), `memory_snapshot`, `committee_review`, `peer_memory`, `attunement` (one row per open-loop extraction pass — opened / closed / dropped with a reason per drop and a `failure` type — plus a row per loop opened or closed and per 👍/👎 on a reply; see Attunement), `executive_paused` / `executive_resumed` (the operator pause switch; actor is the caller's email, details carry the reason, or on resume the held-action count), `scheduled_action_cancelled` (a pending scheduled action cancelled from the Pulse page or a script; actor is the signed-in email, `admin-token` or `local`), `workspace_settings_changed` (solo/team mode, the user's time zone or the principal's role changed through `PUT /workspace`; actor is the caller's email or `api`, details carry the mode's and zone's from/to, plus `role_fields_changed` — the names of the role fields that changed, never their text — when the role changed, and the new `company_domains` list when the company's email domains changed), `decision_class_mode_changed` (a decision class switched between `propose` and `auto_execute` through `PUT /decisions/classes/meeting_scheduling`; same actor rule, details carry the class and the mode's from/to), the Act as me events — `delegation_settings_changed` (a person turned it on or off for themselves), `delegation_reminder_set` and `delegation_reminder_sent` (a person set a reminder for themselves with `remind_me`, and it went to them; ids and times only, never the text; private to that person), `delegation_actions_proposed`, `delegation_actions_approved` and `delegation_actions_dismissed` (the Executive left a person an approval card, which they approved or dismissed; kinds and outcomes only, never the text; private to that person), `delegation_gmail_verified` (their own Gmail checked before turning it on), `delegation_voice_changed` (\"How I write\" learned, edited, locked or reset, or its signature read from Gmail again), `delegation_voice_described` (a style written from the person's own description, counts only; nothing saved until they save it as an edit), `delegation_drafted` (one per draft saved in their Gmail from chat), `delegation_inbox_changed` (Draft replies to my inbox turned on or off), `delegation_inbox_scanned` (one per inbox scan, with its counts and status), `delegation_reply_drafted` (the inbox watcher saved a reply and made its card), `delegation_reply_dismissed` (a card dismissed, and whether its draft was deleted or kept), `delegation_reply_closed` (a card Gmail already settled: sent or deleted there, answered, gone or expired) and `delegation_reply_sent` (the person tapped Send and the draft went, or the reconciler confirmed an unconfirmed send did) — always private and metadata only: ids, counts and flags, never the draft or the thread; the roster-request events (see Org) — `roster_request_created`, `roster_ack_sent`, `roster_ack_withheld` (an acknowledgement the adapter declined — an email Gmail did not authenticate), `roster_request_resolved` (actor is where it was answered: `web`, a chat channel, `email` or `people_page`), `roster_message_replayed` (one per held message: replayed, dropped, failed or unavailable) and `roster_email_answer_refused` (an emailed answer with no live token, or failing DMARC) — always private to the principal and never carrying what the new sender wrote, like the `integration_outbound` row (`kind: roster_ack`) for the one gateway send allowed to an address off the roster, the fixed acknowledgement; `grounding` (unattended prose that named people or figures its inputs don't hold — a brief or digest that held back lines or used its repair call, a reflection tool call refused or flag held back, a triage or review rewrite replaced; details carry the surface, the mode, the ungrounded names and figures and up to five held-back lines cut to 200 chars, private when the brief read the principal's private rows or the alert is private; in `GROUNDING_CHECKS=report` mode these rows are written and nothing is changed); and the external-signal/monitor events. An unknown chat sender's `integration_inbound` row still reads `rejected_unknown_sender` — it means the message did not reach the Executive, not that it was discarded: a DM or mention is held as a roster request.\n\n### Session & turn context\nEach row carries `session_id`, `turn_id`, `actor`, `summary`, `details_json`, `full_json`, `department`, `private_to_principal` and `private_to_person`. A `turn_id` (`f\"t-{uuid12}\"`) is set per Executive turn and propagated automatically via ContextVars to retriever / specialist / tool / cache emits, grouping one user-assistant exchange. `summary` is the searchable column (LIKE, ~300 char cap); `details_json` is structured and not searched. `department` is filterable in SQL (`AuditLogger.query(department=…)`) — the department check-in's skip rule uses it to ask whether anything was logged for a department since its Goals were last reviewed.\n\n### Rows private to the principal\nEvery signed-in user can read `/audit`, so a row about something private to the principal is marked `private_to_principal` as it is written, and only the principal reads it. `AuditLogger.log` marks a row private when its caller passes `private=True` — the email poller's two inbound rows for mail from one of the principal's contacts or mail they forwarded, and `create_alert`'s row for a private alert — or when `people_tools.audit_row_private_to_principal` says so at write time:\n- every row a turn private to the principal writes (`Session.private_to_principal`): its reply, each tool call, the model-call rows, and the background passes that inherit the turn;\n- on the principal's own verified turn (or inside `grant_contact_egress`), any row whose summary, details or full payload names one of their contacts — their email address or full name anywhere in the text, or their person id or chat id in a person or recipient field (`person_id`, `attendee_person_ids`, `user_id`, `chat_id`, `channel_ref`), JSON tool results included.\n- every row an Act as me turn writes once `ghostwrite_email` or a mailbox read (`search_my_email`, `read_my_email`, `read_my_email_attachment`, `my_email_awaiting_reply`) has touched the speaker's mailbox (`TurnDelegation.touched_mail`, set before the tool's first read): the tool row, the reply's `chat_turn` row carrying the draft's preview, the model-call rows, and anything it starts in the background, under `private_rows` (it runs no learning passes). Only the principal can have Act as me for now, so private to the principal is the right visibility; the tool's input and result are also redacted (`ghostwrite` and `my_email` are sensitive tool names).\n- every row the Act as me inbox watcher, a reply card's Send or Dismiss, an action card's Approve (or the card carried out on its own, in training), or a person's Act as me settings write: they run under `rows_for_person(person_id)` (or pass `private_to_person`), so their model-call rows and events are private, and the reply cards are their own person's alone. That includes what training records: `delegation_training_changed`, `delegation_reply_allowed`, `delegation_example_kept`, `delegation_actions_allowed`, `delegation_actions_handled`, `delegation_actions_held` and `delegation_learned_removed`.\n\n### Rows that belong to one team member\nAct as me can be on for team members as well as the principal (see Act as me). What a team member's own mailbox touches must not reach the principal either, so such a row also carries `private_to_person` — the team member's id — as it is written. `people_tools.audit_row_owner` decides it: the id a caller passes, else the `rows_for_person` scope, else the speaker of a turn whose `touched_mail` is set; the principal's own id is never stored, so their rows stay private to the principal exactly as above, and a principal lookup that fails keeps the owner. A row with an owner is always private, and `AuditLogger.query` and `count` leave it out in SQL unless the caller asks for that person's rows (`owned_by`, used only by Act as me's own reads) — so the log, a session's timeline, the brief's handled-overnight block and every other reader skip it, and `GET /audit/logs/{id}` answers 404 for it to everyone, the principal included. Only `usage_summary` counts it, so spend stays whole. `usage_summary` also adds up the sandboxed scripts (`run_script`) from their own rows — chat's `tool_invocation` with `kind: script` and a workflow step's `workflow_tool_call` with `tool: run_script`, each carrying `calls` and `duration_ms` — into `scripts` (scripts run and worked, tool calls, time, and `turns_avoided`, an upper bound on the model turns those calls would otherwise have needed); a script's individual calls are not counted twice. It also sums the `python_job` rows (one per `run_python_job`: `ok`, `duration_ms`, `cpu_ms`, `peak_mb`, files and bytes in and out, `attachments`, `saved_tool`) into `python_jobs`, with the `cache_event` usage of the turns that ran a job, each turn counted once. Rows that quote someone's own documents follow the same `rows_for_person` scope: a `draft_artifact` / `get_artifact` / `list_artifacts` dispatch row, the document tools' own rows, and a turn's `memory_snapshot` when its retrieval recalled a published document are private to the speaker (to the principal when they are the speaker).\n\nSome rows are written before their turn binds its session, so two scopes in `audit.context` count too. The email poller holds `private_rows` for the whole handling of a mail from a contact or a principal forward: every row is private, the knowledge retrieval (which quotes the mail) included. The chat adapters hold `principal_turn_rows` once they know the principal sent the message on a verified surface (`people_tools.audit_rows_on_senders_turn` on Slack, Discord and Telegram; the web chat route checks its caller), so the inbound row, the knowledge retrieval, alert triage and the start of the web stream follow the principal-turn rule above. Tasks started inside inherit the scope, except unattended work: a workflow run a chat reply resumes and each scheduled action start with neither (`unscoped_audit_rows`), so their rows follow the ordinary rule. A sender lookup that fails counts as not the principal, as the turn's own check does. Neither scope lets the turn reach anyone. A caller can add privacy but never remove it, and a check that cannot run keeps the row private. On anyone else's turn a contact is a stranger, so a row that names one keeps its visibility: hiding it would tell whoever wrote it that the address belongs to a contact. The poller handles each message with no session bound, so one message's private turn never marks the next message's rows.\n\nEvery read route resolves the caller as `/alerts` and `/decisions` do (`people.caller_is_principal`: the principal's signed-in email, or no `x-caller-email` — the CLI, direct curl, local login — once a principal is on the roster; before anyone is principal, nobody). For anyone else private rows are filtered in SQL (`include_private=False` on `query`, `count` and `usage_summary`), so `total` and pagination stay honest: `GET /audit/logs` leaves them out, `GET /audit/logs/{id}` answers 404 exactly as for a missing id, `GET /audit/sessions/{id}` shows only the other rows (404 when none are left), and `GET /audit/usage` leaves out their tokens and cost. Internal readers see every row unless they ask otherwise; the department check-in, whose report goes to the department, reads its recent activity without them, and its \"nothing new since the review\" skip rule ignores them too. Ids are one autoincrement sequence, so anyone else can still see a gap where a private row was written; that is accepted.\n\nAlways in the loop writes `history_notes_added` (count and source, never the note) and `history_settings_changed`, `history_note_changed`, `history_note_forgotten` and `history_conversation_forgotten` from its routes. Every one is private and belongs to its person alone (`private_to_person`). `recall_history` is a sensitive tool name, so its input and result are redacted.\n\n### Views\n- `GET /audit` — filterable list with a \"Group by session\" toggle and session pills.\n- `GET /audit/session/{id}` — full ordered timeline plus a server-derived node/edge graph rendered as a three-pane React Flow chart.\n\n### Invariant\nAudit writes never break a chat turn: all writes swallow exceptions and log a warning, and instrumentation observes only — it never touches request building or the cached prompt blocks. Audit is append-only: no UPDATE or DELETE paths exist in product code.",
+ "markdown": "The audit log is a persistent, searchable, append-only record of what the system did — distinct from episodic memory (which biases future LLM behavior). Every event flows through `audit/logger.log_event`, a synchronous SQLite writer (WAL + 5s busy_timeout) that appends one row to the `audit_log` table in `episodic_memory.db`.\n\n### Event types\nRows are tagged by `event_type`, including `chat_turn`, `specialist_consult`, `tool_invocation`, `scheduled_action`, `alert`, `integration_inbound` (Slack / Discord / email / Telegram / Google Chat), plus `knowledge_retrieval`, `fact_confirmation` (an emailed standing-fact change held, applied, cancelled or refused; private), `fact_reviewed` (the principal approved or declined a teammate's proposed standing fact; private), `fact_approval_changed` (the principal turned \"needs my approval\" on or off for a teammate; private), `fact_retired` (the principal, or the teammate who recorded it, retired a standing fact from the Pulse page; private, since it names the fact a teammate can no longer read; the fact tools' own rows and their `skill:` dispatch rows are private `tool_invocation` rows, since they quote the principal and name their session and turn), `memory_extraction` (one row per episodic-extraction pass, written in a `finally` so a crashed pass is recorded too: how many items the model proposed, how many were stored, a `{kind, reason}` record per drop, a `malformed` count for payload shapes that were never usable items, and a `failure` naming the exception type when the pass raised — including `CancelledError`, which is a `BaseException` and so would otherwise be logged as a clean empty pass — a `proposed>0, stored=0` streak means the quote validator is rejecting everything, which is otherwise invisible), `cache_event` (one row per model call from every call site — Executive turns, specialist consults (chat turns and workflow steps), research specialists, the research routing and watchlist passes, triage, memory extraction, alert review, the brief's narrative and its grounding repair, the digest, reflection and weekly-review workflows, committee reviewers, and the small utility calls (session titles, the chat-channel response gate, reply parsing, PDF transcription); a unit test fails any model call in the `openexecutive` package that records no row — carrying tokens, cache hits, cost, server-side search count and, inside a research run, its `run_id`), `memory_snapshot`, `committee_review`, `peer_memory`, `attunement` (one row per open-loop extraction pass — opened / closed / dropped with a reason per drop and a `failure` type — plus a row per loop opened or closed and per 👍/👎 on a reply; see Attunement), `executive_paused` / `executive_resumed` (the operator pause switch; actor is the caller's email, details carry the reason, or on resume the held-action count), `scheduled_action_cancelled` (a pending scheduled action cancelled from the Pulse page or a script; actor is the signed-in email, `admin-token` or `local`), `workspace_settings_changed` (solo/team mode, the user's time zone or the principal's role changed through `PUT /workspace`; actor is the caller's email or `api`, details carry the mode's and zone's from/to, plus `role_fields_changed` — the names of the role fields that changed, never their text — when the role changed, and the new `company_domains` list when the company's email domains changed), `decision_class_mode_changed` (a decision class switched between `propose` and `auto_execute` through `PUT /decisions/classes/meeting_scheduling`; same actor rule, details carry the class and the mode's from/to), the Act as me events — `delegation_settings_changed` (a person turned it on or off for themselves), `delegation_reminder_set` and `delegation_reminder_sent` (a person set a reminder for themselves with `remind_me`, and it went to them; ids and times only, never the text; private to that person), `delegation_actions_proposed`, `delegation_actions_approved` and `delegation_actions_dismissed` (the Executive left a person an approval card, which they approved or dismissed; kinds and outcomes only, never the text; private to that person), `delegation_gmail_verified` (their own Gmail checked before turning it on), `delegation_voice_changed` (\"How I write\" learned, edited, locked or reset, or its signature read from Gmail again), `delegation_voice_described` (a style written from the person's own description, counts only; nothing saved until they save it as an edit), `delegation_drafted` (one per draft saved in their Gmail from chat), `delegation_inbox_changed` (Draft replies to my inbox turned on or off), `delegation_inbox_scanned` (one per inbox scan, with its counts and status), `delegation_reply_drafted` (the inbox watcher saved a reply and made its card), `delegation_reply_dismissed` (a card dismissed, and whether its draft was deleted or kept), `delegation_reply_closed` (a card Gmail already settled: sent or deleted there, answered, gone or expired) and `delegation_reply_sent` (the person tapped Send and the draft went, or the reconciler confirmed an unconfirmed send did) — always private and metadata only: ids, counts and flags, never the draft or the thread; the roster-request events (see Org) — `roster_request_created`, `roster_ack_sent`, `roster_ack_withheld` (an acknowledgement the adapter declined — an email Gmail did not authenticate), `roster_request_resolved` (actor is where it was answered: `web`, a chat channel, `email` or `people_page`), `roster_message_replayed` (one per held message: replayed, dropped, failed or unavailable) and `roster_email_answer_refused` (an emailed answer with no live token, or failing DMARC) — always private to the principal and never carrying what the new sender wrote, like the `integration_outbound` row (`kind: roster_ack`) for the one gateway send allowed to an address off the roster, the fixed acknowledgement; `grounding` (unattended prose that named people or figures its inputs don't hold — a brief or digest that held back lines or used its repair call, a reflection tool call refused or flag held back, a triage or review rewrite replaced; details carry the surface, the mode, the ungrounded names and figures and up to five held-back lines cut to 200 chars, private when the brief read the principal's private rows or the alert is private; in `GROUNDING_CHECKS=report` mode these rows are written and nothing is changed); and the external-signal/monitor events. An unknown chat sender's `integration_inbound` row still reads `rejected_unknown_sender` — it means the message did not reach the Executive, not that it was discarded: a DM or mention is held as a roster request.\n\n### Session & turn context\nEach row carries `session_id`, `turn_id`, `actor`, `summary`, `details_json`, `full_json`, `department`, `private_to_principal` and `private_to_person`. A `turn_id` (`f\"t-{uuid12}\"`) is set per Executive turn and propagated automatically via ContextVars to retriever / specialist / tool / cache emits, grouping one user-assistant exchange. `summary` is the searchable column (LIKE, ~300 char cap); `details_json` is structured and not searched. `department` is filterable in SQL (`AuditLogger.query(department=…)`) — the department check-in's skip rule uses it to ask whether anything was logged for a department since its Goals were last reviewed.\n\n### Rows private to the principal\nEvery signed-in user can read `/audit`, so a row about something private to the principal is marked `private_to_principal` as it is written, and only the principal reads it. `AuditLogger.log` marks a row private when its caller passes `private=True` — the email poller's two inbound rows for mail from one of the principal's contacts or mail they forwarded, and `create_alert`'s row for a private alert — or when `people_tools.audit_row_private_to_principal` says so at write time:\n- every row a turn private to the principal writes (`Session.private_to_principal`): its reply, each tool call, the model-call rows, and the background passes that inherit the turn;\n- on the principal's own verified turn (or inside `grant_contact_egress`), any row whose summary, details or full payload names one of their contacts — their email address or full name anywhere in the text, or their person id or chat id in a person or recipient field (`person_id`, `attendee_person_ids`, `user_id`, `chat_id`, `channel_ref`), JSON tool results included.\n- every row an Act as me turn writes once `ghostwrite_email` or a mailbox read (`search_my_email`, `read_my_email`, `read_my_email_attachment`, `my_email_awaiting_reply`) has touched the speaker's mailbox (`TurnDelegation.touched_mail`, set before the tool's first read): the tool row, the reply's `chat_turn` row carrying the draft's preview, the model-call rows, and anything it starts in the background, under `private_rows` (it runs no learning passes). Only the principal can have Act as me for now, so private to the principal is the right visibility; the tool's input and result are also redacted (`ghostwrite` and `my_email` are sensitive tool names).\n- every row the Act as me inbox watcher, a reply card's Send or Dismiss, an action card's Approve (or the card carried out on its own, in training), or a person's Act as me settings write: they run under `rows_for_person(person_id)` (or pass `private_to_person`), so their model-call rows and events are private, and the reply cards are their own person's alone. That includes what training records: `delegation_training_changed`, `delegation_reply_allowed`, `delegation_example_kept`, `delegation_actions_allowed`, `delegation_actions_handled`, `delegation_actions_held` and `delegation_learned_removed`.\n\n### Rows that belong to one team member\nAct as me can be on for team members as well as the principal (see Act as me). What a team member's own mailbox touches must not reach the principal either, so such a row also carries `private_to_person` — the team member's id — as it is written. `people_tools.audit_row_owner` decides it: the id a caller passes, else the `rows_for_person` scope, else the speaker of a turn whose `touched_mail` is set; the principal's own id is never stored, so their rows stay private to the principal exactly as above, and a principal lookup that fails keeps the owner. A row with an owner is always private, and `AuditLogger.query` and `count` leave it out in SQL unless the caller asks for that person's rows (`owned_by`, used only by Act as me's own reads) — so the log, a session's timeline, the brief's handled-overnight block and every other reader skip it, and `GET /audit/logs/{id}` answers 404 for it to everyone, the principal included. Only `usage_summary` counts it, so spend stays whole. `usage_summary` also adds up the sandboxed scripts (`run_script`) from their own rows — chat's `tool_invocation` with `kind: script` and a workflow step's `workflow_tool_call` with `tool: run_script`, each carrying `calls` and `duration_ms` — into `scripts` (scripts run and worked, tool calls, time, and `turns_avoided`, an upper bound on the model turns those calls would otherwise have needed); a script's individual calls are not counted twice. It also sums the `python_job` rows (one per `run_python_job`: `ok`, `duration_ms`, `cpu_ms`, `peak_mb`, files and bytes in and out, `attachments`, `saved_tool`) into `python_jobs`, with the `cache_event` usage of the turns that ran a job, each turn counted once. Rows that quote someone's own documents follow the same `rows_for_person` scope: a `draft_artifact` / `get_artifact` / `list_artifacts` dispatch row, the document tools' own rows, and a turn's `memory_snapshot` when its retrieval recalled a published document are private to the speaker (to the principal when they are the speaker).\n\nSome rows are written before their turn binds its session, so two scopes in `audit.context` count too. The email poller holds `private_rows` for the whole handling of a mail from a contact or a principal forward: every row is private, the knowledge retrieval (which quotes the mail) included. The chat adapters hold `principal_turn_rows` once they know the principal sent the message on a verified surface (`people_tools.audit_rows_on_senders_turn` on Slack, Discord and Telegram; the web chat route checks its caller), so the inbound row, the knowledge retrieval, alert triage and the start of the web stream follow the principal-turn rule above. Tasks started inside inherit the scope, except unattended work: a workflow run a chat reply resumes and each scheduled action start with neither (`unscoped_audit_rows`), so their rows follow the ordinary rule. A sender lookup that fails counts as not the principal, as the turn's own check does. Neither scope lets the turn reach anyone. A caller can add privacy but never remove it, and a check that cannot run keeps the row private. On anyone else's turn a contact is a stranger, so a row that names one keeps its visibility: hiding it would tell whoever wrote it that the address belongs to a contact. The poller handles each message with no session bound, so one message's private turn never marks the next message's rows.\n\nEvery read route resolves the caller as `/alerts` and `/decisions` do (`people.caller_is_principal`: the principal's signed-in email, or no `x-caller-email` — the CLI, direct curl, local login — once a principal is on the roster; before anyone is principal, nobody). For anyone else private rows are filtered in SQL (`include_private=False` on `query`, `count` and `usage_summary`), so `total` and pagination stay honest: `GET /audit/logs` leaves them out, `GET /audit/logs/{id}` answers 404 exactly as for a missing id, `GET /audit/sessions/{id}` shows only the other rows (404 when none are left), and `GET /audit/usage` leaves out their tokens and cost. Internal readers see every row unless they ask otherwise; the department check-in, whose report goes to the department, reads its recent activity without them, and its \"nothing new since the review\" skip rule ignores them too. Ids are one autoincrement sequence, so anyone else can still see a gap where a private row was written; that is accepted.\n\nAlways in the loop writes `history_notes_added` (count and source, never the note) and `history_settings_changed`, `history_note_changed`, `history_note_forgotten` and `history_conversation_forgotten` from its routes. Every one is private and belongs to its person alone (`private_to_person`). `recall_history` is a sensitive tool name, so its input and result are redacted.\n\n### Views\n- `GET /audit` — filterable list with a \"Group by session\" toggle and session pills.\n- `GET /audit/session/{id}` — full ordered timeline plus a server-derived node/edge graph rendered as a three-pane React Flow chart.\n\n### Invariant\nAudit writes never break a chat turn: all writes swallow exceptions and log a warning, and instrumentation observes only — it never touches request building or the cached prompt blocks. Audit is append-only: no UPDATE or DELETE paths exist in product code.",
"mermaid": "flowchart TD\n Chat[\"Chat turn / consult / tool call\"]\n Sched[\"Scheduled action\"]\n Inbound[\"Inbound integration\"]\n Log[\"audit/logger.log_event\"]\n Priv[\"private_to_principal? private turn, or names a contact on the principal's turn\"]\n DB[\"audit_log table (append-only)\"]\n Gate[\"read routes: private rows for the principal only\"]\n ListView[\"/audit list view\"]\n SessView[\"/audit/session/{id} flow chart\"]\n Chat -->|event + turn_id| Log\n Sched --> Log\n Inbound --> Log\n Log --> Priv\n Priv -->|swallow errors, never block| DB\n DB --> Gate\n Gate --> ListView\n Gate --> SessView\n class Chat entry\n class Sched entry\n class Inbound entry\n class Log compute\n class Priv compute\n class DB storage\n class Gate compute\n class ListView hot\n class SessView hot",
- "generated_at": "2026-10-09T12:22:35Z"
+ "generated_at": "2026-10-11T08:51:44Z"
}
diff --git a/packages/core/openexecutive/audit/usage.py b/packages/core/openexecutive/audit/usage.py
index 5243ad36..9259969b 100644
--- a/packages/core/openexecutive/audit/usage.py
+++ b/packages/core/openexecutive/audit/usage.py
@@ -2,19 +2,26 @@
Every model call the system makes (Executive chat turns, specialist consults
and workflow specialist steps, specialist research, the research synthesis
-and watchlist passes, triage, the chat memory extractor, the Council test
-boxes) records one ``cache_event`` audit row through
+and watchlist passes, triage, alert review, the brief and its grounding
+repair, the digest, reflection and weekly-review workflows, the committee
+reviewers, the chat memory extractor, the Council test boxes and the small
+utility calls) records one ``cache_event`` audit row through
:func:`log_model_usage` — tokens, cache hits, provider-reported cost, and the
server-side web searches the call made. ``GET /audit/usage`` and the
per-session cost view sum those rows, so a run's weight can be read from the
-audit log instead of guessed.
+audit log instead of guessed. Each call site logs for itself, right after
+its call, because it alone knows its ``actor``;
+``tests/unit/test_usage_recording.py`` fails any model call in this package
+that does not. (The repo-root ``evals/`` harness is a separate client
+process with its own SDK client, so it writes no rows here.)
The ``actor`` column names the source of the call (``executive``,
``specialist`` for chat-turn consults, ``specialist_workflow`` for workflow
steps, ``specialist_mcp`` for the MCP consult tool, ``specialist_research``,
-``research_synthesis``, ``research_watchlist``, ``triage``,
-``memory_extractor``, ``agent_test`` for the Council test boxes), which is
-what the per-source breakdown groups on. A research run binds its
+``research_synthesis``, ``research_watchlist``, ``triage``, ``alert_review``,
+``briefing_narrative``, ``briefing_grounding``, a workflow's own name,
+``committee_reviewer``, ``memory_extractor``, ``agent_test`` for the Council
+test boxes, …), which is what the per-source breakdown groups on. A research run binds its
``run_id`` in a ContextVar for its
duration so every row the run produces carries it in ``details``; the
workflow then rolls those rows up into the ``usage`` block on its result.
diff --git a/packages/core/openexecutive/briefing/grounding.py b/packages/core/openexecutive/briefing/grounding.py
index 2be07ce7..06147688 100644
--- a/packages/core/openexecutive/briefing/grounding.py
+++ b/packages/core/openexecutive/briefing/grounding.py
@@ -764,6 +764,7 @@ _REPAIR_INSTRUCTION = (
async def _repair(draft: str, context: str, system: str, report: GroundingReport) -> str:
from openexecutive.agents.utility_fast import get_fast_model
+ from openexecutive.audit.usage import log_model_usage
from openexecutive.providers import get_provider
model = get_fast_model()
@@ -777,6 +778,7 @@ async def _repair(draft: str, context: str, system: str, report: GroundingReport
system=system,
messages=[{"role": "user", "content": content}],
)
+ log_model_usage(response, model=model, actor="briefing_grounding")
blocks = [b for b in response.content if getattr(b, "type", "") == "text"]
return blocks[0].text.strip() if blocks else ""
diff --git a/packages/core/openexecutive/briefing/narrative.py b/packages/core/openexecutive/briefing/narrative.py
index 9bdbb5c2..68ba5ec8 100644
--- a/packages/core/openexecutive/briefing/narrative.py
+++ b/packages/core/openexecutive/briefing/narrative.py
@@ -636,6 +636,7 @@ async def synthesize_briefing_narrative(
brief workflow yields an error event; the page regen logs and moves on).
"""
from openexecutive.agents.utility_fast import get_fast_model
+ from openexecutive.audit.usage import log_model_usage
from openexecutive.providers import get_provider
solo = mode == "solo"
@@ -672,6 +673,7 @@ async def synthesize_briefing_narrative(
system=system,
messages=[{"role": "user", "content": user_content}],
)
+ log_model_usage(response, model=model, actor="briefing_narrative")
text_blocks = [b for b in response.content if getattr(b, "type", "") == "text"]
return text_blocks[0].text.strip() if text_blocks else ""
diff --git a/packages/core/openexecutive/evals/judges.py b/packages/core/openexecutive/evals/judges.py
index c949a56b..07e4b71e 100644
--- a/packages/core/openexecutive/evals/judges.py
+++ b/packages/core/openexecutive/evals/judges.py
@@ -4,6 +4,7 @@ from __future__ import annotations
import json
from typing import Any
+from openexecutive.audit.usage import log_model_usage
from openexecutive.providers import get_provider
# The judge routes through the provider abstraction (like every other LLM call),
@@ -194,6 +195,7 @@ Respond in JSON format:
max_tokens=_JUDGE_MAX_TOKENS,
messages=[{"role": "user", "content": judge_prompt}],
)
+ log_model_usage(message, model=_JUDGE_MODEL, actor="eval_judge")
text = _response_text(message)
try:
start = text.find("{")
@@ -243,6 +245,7 @@ Respond in JSON:
max_tokens=_JUDGE_MAX_TOKENS,
messages=[{"role": "user", "content": judge_prompt}],
)
+ log_model_usage(message, model=_JUDGE_MODEL, actor="eval_judge")
text = _response_text(message)
try:
start = text.find("{")
@@ -305,6 +308,7 @@ Respond in JSON:
max_tokens=_JUDGE_MAX_TOKENS,
messages=[{"role": "user", "content": judge_prompt}],
)
+ log_model_usage(message, model=_JUDGE_MODEL, actor="eval_judge")
text = _response_text(message)
try:
start = text.find("{")
@@ -384,6 +388,7 @@ Respond in JSON:
max_tokens=_JUDGE_MAX_TOKENS,
messages=[{"role": "user", "content": judge_prompt}],
)
+ log_model_usage(message, model=_JUDGE_MODEL, actor="eval_judge")
text = _response_text(message)
try:
start = text.find("{")
diff --git a/packages/core/openexecutive/fixtures/generator.py b/packages/core/openexecutive/fixtures/generator.py
index e394c3ee..d003825a 100644
--- a/packages/core/openexecutive/fixtures/generator.py
+++ b/packages/core/openexecutive/fixtures/generator.py
@@ -718,6 +718,7 @@ async def _generate_bundle(
if not description or not description.strip():
raise GenerationError("scenario description is empty")
+ from openexecutive.audit.usage import log_model_usage
from openexecutive.providers.registry import get_provider
resolved_model = model if model is not None else agent.effective_model()
@@ -744,6 +745,7 @@ async def _generate_bundle(
tool_choice={"type": "tool", "name": "emit_fixture"},
messages=messages,
)
+ log_model_usage(response, model=resolved_model, actor=agent.name, iteration=attempt)
raw = _extract_tool_input(response)
try:
bundle = FixtureBundle.model_validate(raw)
diff --git a/packages/core/openexecutive/integrations/response_gate.py b/packages/core/openexecutive/integrations/response_gate.py
index 16a09c54..301e13a2 100644
--- a/packages/core/openexecutive/integrations/response_gate.py
+++ b/packages/core/openexecutive/integrations/response_gate.py
@@ -139,6 +139,7 @@ async def should_respond(
"""
try:
from openexecutive.agents.utility_fast import get_fast_model
+ from openexecutive.audit.usage import log_model_usage
from openexecutive.config import get_settings
from openexecutive.providers import get_provider
@@ -186,6 +187,7 @@ async def should_respond(
),
timeout=gate_timeout,
)
+ log_model_usage(response, model=model, actor="response_gate")
raw = "".join(
getattr(b, "text", "") for b in response.content
if getattr(b, "type", "") == "text"
diff --git a/packages/core/openexecutive/knowledge/pdf_reader.py b/packages/core/openexecutive/knowledge/pdf_reader.py
index d5c4fb5c..45245e2e 100644
--- a/packages/core/openexecutive/knowledge/pdf_reader.py
+++ b/packages/core/openexecutive/knowledge/pdf_reader.py
@@ -254,6 +254,8 @@ async def _model_slice(
provider: Any, model: str, data: bytes, start: int, end: int
) -> tuple[str, bool]:
"""Transcribe pages [start, end): ``(text, cut off at max_tokens)``."""
+ from openexecutive.audit.usage import log_model_usage
+
chunk = await asyncio.to_thread(slice_pdf, data, start, end)
response = await provider.messages_create(
model=model,
@@ -281,6 +283,7 @@ async def _model_slice(
}
],
)
+ log_model_usage(response, model=model, actor="pdf_reader")
if getattr(response, "stop_reason", None) == "refusal":
raise RuntimeError("model declined to transcribe the pages")
text = "".join(
diff --git a/packages/core/openexecutive/memory/initiatives_consolidation.py b/packages/core/openexecutive/memory/initiatives_consolidation.py
index ac1896f7..97f34604 100644
--- a/packages/core/openexecutive/memory/initiatives_consolidation.py
+++ b/packages/core/openexecutive/memory/initiatives_consolidation.py
@@ -90,6 +90,7 @@ async def propose_clusters(
Returns (all_initiatives, clusters). Empty cluster list means nothing
worth merging.
"""
+ from openexecutive.audit.usage import log_model_usage
from openexecutive.config import get_settings
from openexecutive.providers import get_provider
@@ -116,6 +117,7 @@ async def propose_clusters(
}
],
)
+ log_model_usage(response, model=routing_model, actor="initiatives_consolidation")
clusters: list[Cluster] = []
valid_ids = {i.id for i in initiatives}
diff --git a/packages/core/openexecutive/orchestrator/committee_reviewers.py b/packages/core/openexecutive/orchestrator/committee_reviewers.py
index 67ac25b0..70a41c5a 100644
--- a/packages/core/openexecutive/orchestrator/committee_reviewers.py
+++ b/packages/core/openexecutive/orchestrator/committee_reviewers.py
@@ -13,6 +13,7 @@ import json
import logging
from dataclasses import dataclass
+from openexecutive.audit.usage import log_model_usage
from openexecutive.orchestrator.router import SPECIALIST_DESCRIPTIONS
from openexecutive.prompts.committee_prompts import DOMAIN_REVIEWER_SYSTEM_TEMPLATE
from openexecutive.providers import get_provider
@@ -121,6 +122,7 @@ class Reviewer:
],
messages=[{"role": "user", "content": user_content}],
)
+ log_model_usage(msg, model=self.model, actor="committee_reviewer")
text_blocks = [b for b in msg.content if b.type == "text"]
text = text_blocks[0].text if text_blocks else ""
except Exception:
diff --git a/packages/core/openexecutive/people/insights.py b/packages/core/openexecutive/people/insights.py
index 188bcdc2..fdf22701 100644
--- a/packages/core/openexecutive/people/insights.py
+++ b/packages/core/openexecutive/people/insights.py
@@ -158,6 +158,7 @@ async def generate_person_insight(
"""
try:
from openexecutive.agents.utility_fast import get_fast_model
+ from openexecutive.audit.usage import log_model_usage
from openexecutive.providers import get_provider
honcho = (await _honcho_snippet(person)).strip()
@@ -173,6 +174,7 @@ async def generate_person_insight(
system=_INSIGHT_SYSTEM,
messages=[{"role": "user", "content": content}],
)
+ log_model_usage(response, model=model, actor="person_insight")
raw = "".join(
getattr(b, "text", "") for b in response.content
if getattr(b, "type", "") == "text"
diff --git a/packages/core/openexecutive/utils/session_title.py b/packages/core/openexecutive/utils/session_title.py
index c8eeddbe..ace0527d 100644
--- a/packages/core/openexecutive/utils/session_title.py
+++ b/packages/core/openexecutive/utils/session_title.py
@@ -40,6 +40,7 @@ async def generate_session_title(
"""
try:
from openexecutive.agents.utility_fast import get_fast_model
+ from openexecutive.audit.usage import log_model_usage
from openexecutive.providers import get_provider
model = get_fast_model()
@@ -58,6 +59,7 @@ async def generate_session_title(
}
],
)
+ log_model_usage(response, model=model, actor="session_title")
raw = "".join(
getattr(b, "text", "") for b in response.content
if getattr(b, "type", "") == "text"
diff --git a/packages/core/openexecutive/workflows/end_of_day_digest.py b/packages/core/openexecutive/workflows/end_of_day_digest.py
index e4bdca55..93277b8d 100644
--- a/packages/core/openexecutive/workflows/end_of_day_digest.py
+++ b/packages/core/openexecutive/workflows/end_of_day_digest.py
@@ -379,6 +379,7 @@ class EndOfDayDigestWorkflow(Workflow):
)
from openexecutive.agents.utility_fast import get_fast_model
+ from openexecutive.audit.usage import log_model_usage
from openexecutive.memory.facts import with_standing_facts
from openexecutive.providers import get_provider
@@ -397,6 +398,7 @@ class EndOfDayDigestWorkflow(Workflow):
system=system,
messages=[{"role": "user", "content": user_content}],
)
+ log_model_usage(response, model=model, actor=self.name)
text_blocks = [b for b in response.content if getattr(b, "type", "") == "text"]
artifact_text = text_blocks[0].text.strip() if text_blocks else ""
except Exception as exc:
diff --git a/packages/core/openexecutive/workflows/executive_reflection.py b/packages/core/openexecutive/workflows/executive_reflection.py
index 7f08f493..a1495207 100644
--- a/packages/core/openexecutive/workflows/executive_reflection.py
+++ b/packages/core/openexecutive/workflows/executive_reflection.py
@@ -713,6 +713,7 @@ class ExecutiveReflectionWorkflow(Workflow):
)
# The Python job tool as this instance describes it (a runner's libraries).
_ALL_SKILL_TOOLS = _offered_skill_tools()
+ from openexecutive.audit.usage import log_model_usage
from openexecutive.orchestrator.knowledge_tools import (
KNOWLEDGE_TOOL_HANDLERS,
KNOWLEDGE_TOOLS,
@@ -890,6 +891,7 @@ class ExecutiveReflectionWorkflow(Workflow):
type="error", message=f"Reflection LLM call failed: {exc}"
)
return
+ log_model_usage(response, model=model, actor=self.name, iteration=iteration)
# Only the await sits inside the tag — this is an async
# generator, and a ContextVar set across a `yield` would leak.
diff --git a/packages/core/openexecutive/workflows/inbound_resolver.py b/packages/core/openexecutive/workflows/inbound_resolver.py
index be34ec3c..dd0b721d 100644
--- a/packages/core/openexecutive/workflows/inbound_resolver.py
+++ b/packages/core/openexecutive/workflows/inbound_resolver.py
@@ -469,6 +469,7 @@ async def _llm_disambiguate(
import asyncio
from openexecutive.agents.utility_fast import get_fast_model
+ from openexecutive.audit.usage import log_model_usage
from openexecutive.config import get_settings
from openexecutive.providers import get_provider
@@ -481,6 +482,7 @@ async def _llm_disambiguate(
),
timeout=get_settings().utility_fast_timeout_s,
)
+ log_model_usage(response, model=model, actor="inbound_resolver")
raw_text_blocks = [b for b in response.content if getattr(b, "type", "") == "text"]
raw = raw_text_blocks[0].text.strip() if raw_text_blocks else ""
if raw.startswith("```"):
diff --git a/packages/core/openexecutive/workflows/wait_for_human.py b/packages/core/openexecutive/workflows/wait_for_human.py
index 17e3394e..1fe5e971 100644
--- a/packages/core/openexecutive/workflows/wait_for_human.py
+++ b/packages/core/openexecutive/workflows/wait_for_human.py
@@ -312,6 +312,7 @@ async def parse_decision(
fallback = _FALLBACKS.get(expected_shape, {"text": ""})
try:
+ from openexecutive.audit.usage import log_model_usage
from openexecutive.config import get_settings
from openexecutive.providers import get_provider
@@ -339,6 +340,7 @@ async def parse_decision(
}
],
)
+ log_model_usage(response, model=model, actor="wait_for_human")
# The SDK only emits text blocks for this prompt (no tools, no
# thinking). The union-attr complaint mypy raises here is a false
# positive at runtime; suppress rather than narrowing because the
diff --git a/packages/core/openexecutive/workflows/weekly_review.py b/packages/core/openexecutive/workflows/weekly_review.py
index e12b206e..23db7fa2 100644
--- a/packages/core/openexecutive/workflows/weekly_review.py
+++ b/packages/core/openexecutive/workflows/weekly_review.py
@@ -503,6 +503,7 @@ async def _next_weeks_top_three(body: str, period: str) -> str:
"""Next week's top three, as a numbered list, from one fast-model call
that follows the weekly-review playbook. "" on any failure."""
from openexecutive.agents.utility_fast import get_fast_model
+ from openexecutive.audit.usage import log_model_usage
from openexecutive.memory.facts import with_standing_facts
from openexecutive.providers import get_provider
from openexecutive.workflows.playbooks import load_playbook, playbook_clause
@@ -524,6 +525,7 @@ async def _next_weeks_top_three(body: str, period: str) -> str:
except Exception:
logger.exception("weekly_review: top three call failed — using the ranked pick")
return ""
+ log_model_usage(response, model=model, actor="weekly_review")
text = "".join(
getattr(b, "text", "") for b in getattr(response, "content", []) or []
if getattr(b, "type", "") == "text"
diff --git a/packages/core/tests/unit/test_usage_recording.py b/packages/core/tests/unit/test_usage_recording.py
index 9d5c714a..556f2825 100644
--- a/packages/core/tests/unit/test_usage_recording.py
+++ b/packages/core/tests/unit/test_usage_recording.py
@@ -1,9 +1,11 @@
"""Every model call site records a ``cache_event`` usage row: specialist
-consults, specialist tool calls, triage, and the chat memory extractor (the
-Executive's own turns and the research loops are covered by their module
-tests)."""
+consults, specialist tool calls, triage, the chat memory extractor, the
+brief and its grounding repair, and alert review (the Executive's own turns
+and the research loops are covered by their module tests), plus a guard that
+fails any model call in the package that records no row."""
from __future__ import annotations
+import ast
import asyncio
from pathlib import Path
from types import SimpleNamespace
@@ -11,6 +13,7 @@ from unittest.mock import AsyncMock
import pytest
+import openexecutive
from openexecutive.agents.base import BaseAgent
from openexecutive.audit.logger import AuditLogger, set_audit_logger
@@ -214,3 +217,209 @@ def test_memory_extractor_retry_records_its_own_usage(
rows = audit.query(event_type="cache_event")
assert len(rows) == 2 and {r.actor for r in rows} == {"memory_extractor"}
assert sorted(r.details["input_tokens"] for r in rows) == [12, 30]
+
+
+def _text_response(text: str, **usage: object) -> SimpleNamespace:
+ response = _response(**usage)
+ response.content = [SimpleNamespace(type="text", text=text)]
+ return response
+
+
+def fast_provider(monkeypatch: pytest.MonkeyPatch) -> AsyncMock:
+ """A utility-fast provider answering with one text block."""
+ create = AsyncMock(return_value=_text_response("brief", input_tokens=50, output_tokens=7))
+ monkeypatch.setattr(
+ "openexecutive.providers.get_provider",
+ lambda _m: SimpleNamespace(messages_create=create),
+ )
+ monkeypatch.setattr("openexecutive.agents.utility_fast.get_fast_model", lambda: "claude-fast")
+ return create
+
+
+def test_briefing_narrative_records_usage(audit: AuditLogger, fast_provider: AsyncMock) -> None:
+ """The morning brief and the /today header both synthesize through here."""
+ from openexecutive.briefing.narrative import synthesize_briefing_narrative
+
+ text = asyncio.run(synthesize_briefing_narrative(
+ today_data={}, activity=[], period_label="Monday", rendered_context="ctx",
+ ))
+
+ assert text == "brief"
+ rows = audit.query(event_type="cache_event")
+ assert [r.actor for r in rows] == ["briefing_narrative"]
+ assert rows[0].details["model"] == "claude-fast"
+ assert rows[0].details["input_tokens"] == 50
+
+
+def test_briefing_grounding_repair_records_usage(
+ audit: AuditLogger, fast_provider: AsyncMock,
+) -> None:
+ from openexecutive.briefing.grounding import GroundingReport, _repair
+
+ asyncio.run(_repair("draft", "context", "system", GroundingReport(lines=[])))
+
+ rows = audit.query(event_type="cache_event")
+ assert [r.actor for r in rows] == ["briefing_grounding"]
+ assert rows[0].details["output_tokens"] == 7
+
+
+def test_alert_review_records_usage(audit: AuditLogger, monkeypatch: pytest.MonkeyPatch) -> None:
+ from openexecutive.agents.alert_review import AlertReviewAgent
+
+ monkeypatch.setattr("openexecutive.agents.overrides.get_override", lambda _name: None)
+ block = SimpleNamespace(type="tool_use", name="emit_alert_reviews", input={"reviews": []})
+ response = _response(input_tokens=400, output_tokens=60, cache_read_input_tokens=300)
+ response.content = [block]
+ provider = SimpleNamespace(messages_create=AsyncMock(return_value=response))
+ monkeypatch.setattr("openexecutive.agents.alert_review.get_provider", lambda _m: provider)
+
+ agent = AlertReviewAgent()
+ asyncio.run(agent.review("batch"))
+
+ rows = audit.query(event_type="cache_event")
+ assert [r.actor for r in rows] == ["alert_review"]
+ assert rows[0].details["model"] == agent.effective_model()
+ assert rows[0].details["cache_read_input_tokens"] == 300
+
+
+def test_alert_review_failed_call_records_nothing(
+ audit: AuditLogger, monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """A call that raised has no response to read usage from."""
+ from openexecutive.agents.alert_review import AlertReviewAgent
+
+ monkeypatch.setattr("openexecutive.agents.overrides.get_override", lambda _name: None)
+ provider = SimpleNamespace(messages_create=AsyncMock(side_effect=RuntimeError("down")))
+ monkeypatch.setattr("openexecutive.agents.alert_review.get_provider", lambda _m: provider)
+
+ assert asyncio.run(AlertReviewAgent().review("batch")) == []
+ assert audit.query(event_type="cache_event") == []
+
+
+PACKAGE = Path(openexecutive.__file__).resolve().parent
+
+# Modules exempt from the guard below, each with the reason. Keep this short:
+# an entry is a place the usage view can silently under-count.
+_EXEMPT: dict[str, str] = {
+ "architecture/generator.py": (
+ "no runtime caller: architecture/__init__.py exports it for re-authoring, "
+ "but the /architecture page serves the hand-authored prebuilt JSON and "
+ "nothing calls generate_section outside tests"
+ ),
+}
+
+
+def _is_model_call(node: ast.Call) -> bool:
+ func = node.func
+ if not isinstance(func, ast.Attribute):
+ return False
+ if func.attr in {"messages_create", "messages_stream"}:
+ return True
+ # A raw SDK client (triage's injectable one) spells it messages.create.
+ return (
+ func.attr in {"create", "stream"}
+ and isinstance(func.value, ast.Attribute)
+ and func.value.attr == "messages"
+ )
+
+
+def _called_name(node: ast.Call) -> str | None:
+ func = node.func
+ return func.attr if isinstance(func, ast.Attribute) else getattr(func, "id", None)
+
+
+_FUNCTION = (ast.FunctionDef, ast.AsyncFunctionDef)
+
+
+def _own_calls(fn: ast.AST) -> list[ast.Call]:
+ """The calls in ``fn``'s own body — a nested def is its own function."""
+ calls: list[ast.Call] = []
+ stack = list(ast.iter_child_nodes(fn))
+ while stack:
+ node = stack.pop()
+ if isinstance(node, (*_FUNCTION, ast.Lambda, ast.ClassDef)):
+ continue
+ if isinstance(node, ast.Call):
+ calls.append(node)
+ stack.extend(ast.iter_child_nodes(node))
+ return calls
+
+
+def _unlogged_model_calls(tree: ast.Module) -> list[tuple[int, str]]:
+ """``(line, function)`` for each model call whose function records no
+ usage. A function records usage when it calls ``log_model_usage``, or a
+ function of the same module that does (the Executive's
+ ``_emit_cache_event``), or is nested in one that does (a retry closure
+ whose caller logs each response).
+
+ The check is per function and by name, so a green run is not proof:
+ it misses a call at module level or in a lambda, a call through
+ ``getattr`` or an alias, and a second model call in a function that
+ logs one, and it lets two same-named functions in a module share one
+ logging status."""
+ loggers = {"log_model_usage"} | {
+ fn.name
+ for fn in ast.walk(tree)
+ if isinstance(fn, _FUNCTION)
+ and any(_called_name(c) == "log_model_usage" for c in _own_calls(fn))
+ }
+ found: list[tuple[int, str]] = []
+
+ def visit(node: ast.AST, enclosing_logs: bool) -> None:
+ for child in ast.iter_child_nodes(node):
+ if not isinstance(child, _FUNCTION):
+ visit(child, enclosing_logs)
+ continue
+ calls = _own_calls(child)
+ logs = enclosing_logs or any(_called_name(c) in loggers for c in calls)
+ if not logs:
+ found.extend((c.lineno, child.name) for c in calls if _is_model_call(c))
+ visit(child, logs)
+
+ visit(tree, False)
+ return found
+
+
+def test_every_model_call_records_usage() -> None:
+ """Every model call in the ``openexecutive`` package records a
+ ``cache_event`` row, so ``/audit/usage`` and the session cost view count
+ it. A new call site that skips ``log_model_usage`` fails here instead of
+ quietly under-counting. The providers implement the call and are
+ skipped; the repo-root ``evals/`` harness is a separate client process
+ and out of scope."""
+ offenders = [
+ f"{rel}:{line} ({name})"
+ for path in sorted(PACKAGE.rglob("*.py"))
+ if not (rel := path.relative_to(PACKAGE).as_posix()).startswith("providers/")
+ and rel not in _EXEMPT
+ for line, name in _unlogged_model_calls(ast.parse(path.read_text(), filename=str(path)))
+ ]
+ assert offenders == []
+ assert sorted(m for m in _EXEMPT if not (PACKAGE / m).exists()) == []
+
+
+def test_the_usage_guard_catches_an_unlogged_call() -> None:
+ """Negative control: the guard flags a bare call and accepts each logged
+ shape, so a clean run above means something."""
+ source = (
+ "async def bare(p):\n"
+ " return await p.messages_create(model='m')\n"
+ "async def direct(p):\n"
+ " r = await p.messages_create(model='m')\n"
+ " log_model_usage(r, model='m', actor='a')\n"
+ "def _wrapper(r):\n"
+ " log_model_usage(r, model='m', actor='a')\n"
+ "async def via_wrapper(p):\n"
+ " async with p.messages_stream(model='m') as s:\n"
+ " _wrapper(await s.get_final_message())\n"
+ "async def outer(p):\n"
+ " async def _call():\n"
+ " return await p.messages_create(model='m')\n"
+ " log_model_usage(await _call(), model='m', actor='a')\n"
+ "async def raw(client):\n"
+ " return await client.messages.create(model='m')\n"
+ )
+ assert _unlogged_model_calls(ast.parse(source)) == [(2, "bare"), (16, "raw")]
+
+