AI-powered virtual executive team — a single coherent executive persona backed by 8 specialist agents (FastAPI + Next.js).
feat(evals): pick the judge model with EVAL_JUDGE_MODEL (#446)
Rufus096452239c78
diff --git a/.env.example b/.env.exampleindex 84896393..9091ffbb 100644--- a/.env.example+++ b/.env.example@@ -158,6 +158,9 @@ LOCAL_INCLUDE_USAGE_ACCOUNTING=false# ROUTING_MODEL=llama3.3# (Server-side web search has no local equivalent and is disabled for local# models; tool-use quality depends on the model you choose.)+# The eval judge (`make eval`, the Evals page) is Claude Opus unless this+# names another model; a slug from LOCAL_MODELS judges on the local server.+# EVAL_JUDGE_MODEL=llama3.3# --- Chat turn timeouts -------------------------------------------------------# Whole-turn wall-clock ceiling for a streaming /chat turn, in seconds. A turndiff --git a/evals/judges/triage_judge.py b/evals/judges/triage_judge.pyindex 318c25b6..9d58c02e 100644--- a/evals/judges/triage_judge.py+++ b/evals/judges/triage_judge.py@@ -9,10 +9,6 @@ from __future__ import annotationsimport jsonfrom typing import Any-import anthropic--JUDGE_MODEL = "claude-opus-4-7"-def _format_decision(decision: dict[str, Any]) -> str:return (@@ -61,11 +57,8 @@ def _format_expected(expected: dict[str, Any]) -> str:async def judge_triage(scenario: dict[str, Any],decision: dict[str, Any],- client: anthropic.AsyncAnthropic | None = None,) -> dict[str, Any]:- from openexecutive.providers.anthropic_provider import configured_async_client-- cli = client or configured_async_client()+ from openexecutive.evals.judges import judge_messagejudge_prompt = f"""You are an evaluator for a Triage agent that decides whether incomingevents (emails, Slack messages, documents) should fire proactive alerts to a@@ -99,13 +92,7 @@ Respond in JSON:{{"severity_accuracy": N, "channel_appropriateness": N, "dedup_correctness": N, "overall": N, "notes": "brief explanation"}}"""- message = await cli.messages.create(- model=JUDGE_MODEL,- max_tokens=500,- messages=[{"role": "user", "content": judge_prompt}],- )-- text = message.content[0].text # type: ignore[union-attr]+ text = await judge_message(judge_prompt)try:start = text.find("{")end = text.rfind("}") + 1diff --git a/evals/run_evals.py b/evals/run_evals.pyindex 066a2016..e1610727 100644--- a/evals/run_evals.py+++ b/evals/run_evals.py@@ -62,7 +62,7 @@ async def run_eval(scenario: dict, executive, session_factory) -> dict:}-async def judge_response(scenario: dict, response: str, client) -> dict:+async def judge_response(scenario: dict, response: str) -> dict:judge_prompt = f"""You are an evaluator for an AI executive advisory system.Evaluate this response to an executive question on a 1-5 scale for each dimension.@@ -85,13 +85,9 @@ Rate each dimension (1=poor, 3=acceptable, 5=excellent):Respond in JSON format:{{"persona_coherence": N, "domain_accuracy": N, "actionability": N, "topic_coverage": N, "specificity": N, "concision": N, "overall": N, "notes": "brief explanation"}}"""- message = await client.messages.create(- model="claude-opus-4-7",- max_tokens=500,- messages=[{"role": "user", "content": judge_prompt}],- )+ from openexecutive.evals.judges import judge_message- text = message.content[0].text+ text = await judge_message(judge_prompt)try:start = text.find("{")end = text.rfind("}") + 1@@ -133,7 +129,7 @@ async def run_workflow_eval(scenario: dict, store) -> dict:}-async def judge_workflow(scenario: dict, artifact: str, client) -> dict:+async def judge_workflow(scenario: dict, artifact: str) -> dict:expected_sections = scenario.get("expected_artifact_sections", []) or []quality_criteria = scenario.get("quality_criteria", {}) or {}@@ -161,13 +157,9 @@ Rate each dimension 1-5 (1=poor, 3=acceptable, 5=excellent):Respond in JSON:{{"structure": N, "specificity": N, "actionability": N, "coherence": N, "completeness": N, "overall": N, "notes": "brief"}}"""- message = await client.messages.create(- model="claude-opus-4-7",- max_tokens=500,- messages=[{"role": "user", "content": judge_prompt}],- )+ from openexecutive.evals.judges import judge_message- text = message.content[0].text+ text = await judge_message(judge_prompt)try:start = text.find("{")end = text.rfind("}") + 1@@ -176,7 +168,7 @@ Respond in JSON:return {"overall": 0, "notes": "Failed to parse judge response"}-async def run_triage_eval(scenario: dict, client) -> dict:+async def run_triage_eval(scenario: dict) -> dict:"""Runs a triage scenario directly against the TriageAgent (no Executive loop).Scenario YAML must include `event` + `context` (with `recent_alerts`,@@ -262,12 +254,10 @@ async def main() -> None:initialize_overrides_db()ReviewStore.initialize_db()- # Built through the app's factory so an organisation-scoped key gets its- # `anthropic-workspace-id` header here too — a bare AsyncAnthropic() 400s- # on every judge call for those keys (#128).- from openexecutive.providers.anthropic_provider import configured_async_client-- client = configured_async_client()+ # Judges go through the provider registry (openexecutive.evals.judges),+ # which builds the Anthropic client with the `anthropic-workspace-id`+ # header an organisation-scoped key needs (#128) and picks the judge+ # model from EVAL_JUDGE_MODEL.scenario_dir = Path(args.scenarios)output_dir = Path(args.output)@@ -315,8 +305,8 @@ async def main() -> None:for scenario in scenarios:print(f" [{scenario['id']}] {scenario['description']}")try:- result = await run_triage_eval(scenario, client)- scores = await judge_triage(scenario, result["decision"], client)+ result = await run_triage_eval(scenario)+ scores = await judge_triage(scenario, result["decision"])result["scores"] = scoresresult["passed"] = scores.get("overall", 0) >= 3.5results.append(result)@@ -348,7 +338,7 @@ async def main() -> None:print(f" [{scenario['id']}] {scenario['description']}")try:result = await run_workflow_eval(scenario, store)- scores = await judge_workflow(scenario, result["artifact"], client)+ scores = await judge_workflow(scenario, result["artifact"])result["scores"] = scoresresult["passed"] = scores.get("overall", 0) >= 3.5results.append(result)@@ -366,7 +356,7 @@ async def main() -> None:print(f" [{scenario['id']}] {scenario['description']}")try:result = await run_eval(scenario, executive, None)- scores = await judge_response(scenario, result["response"], client)+ scores = await judge_response(scenario, result["response"])result["scores"] = scoresresult["passed"] = scores.get("overall", 0) >= 3.5results.append(result)diff --git a/packages/core/openexecutive/architecture/architecture-facts.yaml b/packages/core/openexecutive/architecture/architecture-facts.yamlindex 6c8081b1..7d7c9b8c 100644--- a/packages/core/openexecutive/architecture/architecture-facts.yaml+++ b/packages/core/openexecutive/architecture/architecture-facts.yaml@@ -4936,7 +4936,7 @@ audit:- alert — alert created by the triage pipeline- integration_inbound — Slack / Discord / email / Telegram / Google Chat message received. The chat adapters put the first 160 characters of the message in the summary; the email poller keeps the summary to sender and subject and stores the text in the full payload (`message` — the sender's new text, cut at 20,000 escaped JSON chars, the measure the 64 KB cap uses; `attachments` — filenames, 8,000 escaped chars; `body` — the whole body with its quoted chain, only on a row private to the principal, since the chain quotes other people's mail and a public row is every signed-in user's) with `details.preview` (120 chars of the new text) and `details.body_len`, so /audit shows the request next to the reply's `chat_turn` row. `details.outcome` distinguishes the non-reply paths — `rejected_unknown_sender` (not on the People roster), `skipped_gate` (response gate said no), `dropped_thread_unreadable` (Slack wouldn't return a thread the bot is in, so a continuation went unanswered; the sender was told), and `handler_error` (the turn raised; the user got the generic apology). The error row carries the same `session_id` as the inbound row, so /audit can join a failed turn to the message that caused it.- knowledge_retrieval — one row per `knowledge.retriever.retrieve()` call; details carry chunk counts + domains, full payload carries each chunk's source, distance, and text preview so the session view shows "which RAG chunks influenced this response"- - cache_event — one row per model call, from every call site (`audit.usage.log_model_usage`): Executive iterations and the committee revision pass (`actor` executive / committee_revision), `BaseAgent.analyze` callers — chat-turn consults via `route_parallel` (`actor` specialist), workflow steps and other direct `route_to_specialist` callers (specialist_workflow), the MCP `consult_specialist` tool (specialist_mcp), and both Council test boxes (agent_test) — `BaseAgent.analyze_with_tools` callers (specialist_research, query_watch, signal_enrichment), the research synthesis and watchlist passes (research_synthesis / research_watchlist), triage, the chat memory extractor (memory_extractor), alert review (alert_review), the brief's narrative and grounding repair (briefing_narrative / briefing_grounding), the end_of_day_digest, executive_reflection and weekly_review workflows, wait_for_human reply parsing (workflow replies and roster answers) and the inbound_resolver, committee reviewers (committee_reviewer), the fixture and engagement generators (their agent names), and the utility calls — session_title, response_gate, person_insight, initiatives_consolidation, pdf_reader, the chat page's suggested_prompts / followup_suggestion, and the eval runner's eval_judge. `tests/unit/test_usage_recording.py::test_every_model_call_records_usage` walks the package's AST and fails on any function that calls `messages_create` / `messages_stream` without calling `log_model_usage` (directly, through a same-module wrapper such as the Executive's `_emit_cache_event`, or from an enclosing function), so a new call site cannot skip the row silently; `providers/` and the runtime-dead `architecture/generator.py` are exempt, and the repo-root `evals/` harness (a separate client process with its own SDK client) is out of scope. Captures `input_tokens`, `output_tokens`, `cache_creation_input_tokens`, `cache_read_input_tokens`, `cost_usd`, `web_search_requests` (from `usage.server_tool_use`), `stop_reason`, and — inside a research run — the run's `run_id`, so prompt-cache hits are visible per turn, plus `cost_usd` — the actual USD OpenRouter charged for the call (read from `usage.cost`; null on the Anthropic-direct path or for rows logged before cost capture shipped). This is the token-usage + cost source of truth aggregated by both `/audit/sessions/{id}` (per session) and `/audit/usage` (cross-session).+ - cache_event — one row per model call, from every call site (`audit.usage.log_model_usage`): Executive iterations and the committee revision pass (`actor` executive / committee_revision), `BaseAgent.analyze` callers — chat-turn consults via `route_parallel` (`actor` specialist), workflow steps and other direct `route_to_specialist` callers (specialist_workflow), the MCP `consult_specialist` tool (specialist_mcp), and both Council test boxes (agent_test) — `BaseAgent.analyze_with_tools` callers (specialist_research, query_watch, signal_enrichment), the research synthesis and watchlist passes (research_synthesis / research_watchlist), triage, the chat memory extractor (memory_extractor), alert review (alert_review), the brief's narrative and grounding repair (briefing_narrative / briefing_grounding), the end_of_day_digest, executive_reflection and weekly_review workflows, wait_for_human reply parsing (workflow replies and roster answers) and the inbound_resolver, committee reviewers (committee_reviewer), the fixture and engagement generators (their agent names), and the utility calls — session_title, response_gate, person_insight, initiatives_consolidation, pdf_reader, the chat page's suggested_prompts / followup_suggestion, and the eval runner's eval_judge. `tests/unit/test_usage_recording.py::test_every_model_call_records_usage` walks the package's AST and fails on any function that calls `messages_create` / `messages_stream` without calling `log_model_usage` (directly, through a same-module wrapper such as the Executive's `_emit_cache_event`, or from an enclosing function), so a new call site cannot skip the row silently; `providers/` and the runtime-dead `architecture/generator.py` are exempt, and the repo-root `evals/` harness is out of scope (its judges call the package's `evals.judges.judge_message`, which records the eval_judge row and uses `EVAL_JUDGE_MODEL`, default Opus). Captures `input_tokens`, `output_tokens`, `cache_creation_input_tokens`, `cache_read_input_tokens`, `cost_usd`, `web_search_requests` (from `usage.server_tool_use`), `stop_reason`, and — inside a research run — the run's `run_id`, so prompt-cache hits are visible per turn, plus `cost_usd` — the actual USD OpenRouter charged for the call (read from `usage.cost`; null on the Anthropic-direct path or for rows logged before cost capture shipped). This is the token-usage + cost source of truth aggregated by both `/audit/sessions/{id}` (per session) and `/audit/usage` (cross-session).- memory_snapshot — one row at Executive turn entry; captures the system-block layout, conversation history length, episodic context (full text in `full_json`), and a company-profile hash so the session view can render "what the agent saw" before any API call- committee_review — one row per committee-reviewed turn; carries draft length, final length, per-reviewer critiques and timings (previously emitted but newly declared as a recognised event_type)- peer_memory — one row per `openexecutive.memory.honcho_client` call (prefetch / sync_turn / seed_identity / directional_chat / prefetch_department / sync_department_turn / append_department_note); carries `op`, `person_id`, optional `department_slug`, `outcome` (ok / timeout / error / disabled / no_person / no_department / no_loop / empty), `duration_ms`, plus op-specific details (query preview, reasoning_level, peer count, co_present_person_ids, co_present_department_slugs, originating_person_id, target_person_id, session_id, kind) so the flow-chart view renders the peer_memory path alongside `<retrieved_context>` and `<past_decisions>`. Disabled / no-Person / no-Department turns still emit (outcome marker) so audit absence equals "code path never ran" instead of "the provider was down."diff --git a/packages/core/openexecutive/config.py b/packages/core/openexecutive/config.pyindex 4868dda8..f30b1fe7 100644--- a/packages/core/openexecutive/config.py+++ b/packages/core/openexecutive/config.py@@ -320,6 +320,12 @@ class Settings(BaseSettings):Literal["none", "minimal", "low", "medium", "high"] | None) = Field(None, alias="LOCAL_REASONING_EFFORT")+ # The model that scores eval answers (`make eval` and the Evals page).+ # Unset = Claude Opus. Any slug the registry routes works, so a slug in+ # LOCAL_MODELS runs the judge on the local server too, for a run with no+ # hosted API at all.+ eval_judge_model: str | None = Field(None, alias="EVAL_JUDGE_MODEL")+@field_validator("local_models", mode="before")@classmethoddef _parse_local_models(cls, v: Any) -> list[str]:diff --git a/packages/core/openexecutive/evals/judges.py b/packages/core/openexecutive/evals/judges.pyindex 07e4b71e..44e0e6a6 100644--- a/packages/core/openexecutive/evals/judges.py+++ b/packages/core/openexecutive/evals/judges.py@@ -8,13 +8,34 @@ from openexecutive.audit.usage import log_model_usagefrom openexecutive.providers import get_provider# The judge routes through the provider abstraction (like every other LLM call),-# so when OPENROUTER_ENABLED is on, judging bills the OpenRouter account too.+# so when OPENROUTER_ENABLED is on, judging bills the OpenRouter account too,+# and a slug listed in LOCAL_MODELS is judged by the local server.+# EVAL_JUDGE_MODEL picks the model; unset, it is this one._JUDGE_MODEL = "claude-opus-5-5"# Room for the judge's adaptive thinking (always on for Opus 5.5) before the# JSON verdict; 500 tokens could be spent thinking with nothing left to answer._JUDGE_MAX_TOKENS = 4096+def judge_model() -> str:+ """The model that scores eval answers: ``EVAL_JUDGE_MODEL``, else Opus."""+ from openexecutive.config import get_settings++ return get_settings().eval_judge_model or _JUDGE_MODEL+++async def judge_message(prompt: str, *, max_tokens: int = _JUDGE_MAX_TOKENS) -> str:+ """Send one judge prompt to the judge model and return the reply's text."""+ model = judge_model()+ message = await get_provider(model).messages_create(+ model=model,+ max_tokens=max_tokens,+ messages=[{"role": "user", "content": prompt}],+ )+ log_model_usage(message, model=model, actor="eval_judge")+ return _response_text(message)++def _response_text(message: Any) -> str:"""The reply's text, skipping thinking blocks that may come first."""return "".join(@@ -190,13 +211,7 @@ Rate each dimension (1=poor, 3=acceptable, 5=excellent):Respond in JSON format:{{"persona_coherence": N, "domain_accuracy": N, "actionability": N, "topic_coverage": N, "specificity": N, "overall": N, "notes": "brief explanation"}}"""- message = await get_provider(_JUDGE_MODEL).messages_create(- model=_JUDGE_MODEL,- max_tokens=_JUDGE_MAX_TOKENS,- messages=[{"role": "user", "content": judge_prompt}],- )- log_model_usage(message, model=_JUDGE_MODEL, actor="eval_judge")- text = _response_text(message)+ text = await judge_message(judge_prompt)try:start = text.find("{")end = text.rfind("}") + 1@@ -240,13 +255,7 @@ Rate each dimension 1-5 (1=poor, 3=acceptable, 5=excellent):Respond in JSON:{{"structure": N, "specificity": N, "actionability": N, "coherence": N, "completeness": N, "overall": N, "notes": "brief"}}"""- message = await get_provider(_JUDGE_MODEL).messages_create(- model=_JUDGE_MODEL,- max_tokens=_JUDGE_MAX_TOKENS,- messages=[{"role": "user", "content": judge_prompt}],- )- log_model_usage(message, model=_JUDGE_MODEL, actor="eval_judge")- text = _response_text(message)+ text = await judge_message(judge_prompt)try:start = text.find("{")end = text.rfind("}") + 1@@ -303,13 +312,7 @@ If any quality criterion is not met, overall must be 2 or lower and notes must nRespond in JSON:{{"faithfulness": N, "safety": N, "open_questions": N, "voice": N, "overall": N, "notes": "brief"}}"""- message = await get_provider(_JUDGE_MODEL).messages_create(- model=_JUDGE_MODEL,- max_tokens=_JUDGE_MAX_TOKENS,- messages=[{"role": "user", "content": judge_prompt}],- )- log_model_usage(message, model=_JUDGE_MODEL, actor="eval_judge")- text = _response_text(message)+ text = await judge_message(judge_prompt)try:start = text.find("{")end = text.rfind("}") + 1@@ -383,13 +386,7 @@ ACTUAL DECISION:Respond in JSON:{{"severity_accuracy": N, "channel_appropriateness": N, "dedup_correctness": N, "overall": N, "notes": "brief"}}"""- message = await get_provider(_JUDGE_MODEL).messages_create(- model=_JUDGE_MODEL,- max_tokens=_JUDGE_MAX_TOKENS,- messages=[{"role": "user", "content": judge_prompt}],- )- log_model_usage(message, model=_JUDGE_MODEL, actor="eval_judge")- text = _response_text(message)+ text = await judge_message(judge_prompt)try:start = text.find("{")end = text.rfind("}") + 1diff --git a/packages/core/tests/unit/test_eval_judge_model.py b/packages/core/tests/unit/test_eval_judge_model.pynew file mode 100644index 00000000..c9192407--- /dev/null+++ b/packages/core/tests/unit/test_eval_judge_model.py@@ -0,0 +1,69 @@+"""EVAL_JUDGE_MODEL picks the model that scores eval answers.++Unset, the judge stays on Claude Opus. Set, every judge call goes to that+model through the provider registry, so a slug listed in LOCAL_MODELS is+judged by the local server (an eval run with no hosted API at all).+"""+from __future__ import annotations++import asyncio+from typing import Any++import pytest++from openexecutive.evals import judges as judges_module+++class _Msg:+ content = [type("T", (), {"type": "text", "text": '{"overall": 4, "notes": "ok"}'})()]+++def calls(monkeypatch: pytest.MonkeyPatch) -> list[tuple[str, dict[str, Any]]]:+ seen: list[tuple[str, dict[str, Any]]] = []++ class _Provider:+ def __init__(self, routed: str) -> None:+ self.routed = routed++ async def messages_create(self, **kwargs: Any) -> Any:+ seen.append((self.routed, kwargs))+ return _Msg()++ monkeypatch.setattr(judges_module, "get_provider", _Provider)+ monkeypatch.setattr(judges_module, "log_model_usage", lambda *a, **k: None)+ return seen+++def test_unset_judges_with_opus(+ monkeypatch: pytest.MonkeyPatch, calls: list[tuple[str, dict[str, Any]]]+) -> None:+ monkeypatch.delenv("EVAL_JUDGE_MODEL", raising=False)++ scores = asyncio.run(judges_module.judge_chat({"query": "q", "expected_topics": []}, "r"))++ assert scores["overall"] == 4+ assert calls == [("claude-opus-5-5", calls[0][1])]+ assert calls[0][1]["model"] == "claude-opus-5-5"+++def test_set_routes_every_judge_to_that_model(+ monkeypatch: pytest.MonkeyPatch, calls: list[tuple[str, dict[str, Any]]]+) -> None:+ monkeypatch.setenv("EVAL_JUDGE_MODEL", "qwen3:4b")++ asyncio.run(judges_module.judge_chat({"query": "q", "expected_topics": []}, "r"))+ asyncio.run(+ judges_module.judge_workflow(+ {"workflow": "w", "description": "d", "expected_sections": []}, "artifact"+ )+ )++ assert [routed for routed, _ in calls] == ["qwen3:4b", "qwen3:4b"]+ assert all(kwargs["model"] == "qwen3:4b" for _, kwargs in calls)+++def test_blank_value_reads_as_unset(monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setenv("EVAL_JUDGE_MODEL", "")++ assert judges_module.judge_model() == "claude-opus-5-5"