Skip to content

AI-powered virtual executive team — a single coherent executive persona backed by 8 specialist agents (FastAPI + Next.js).

feat(evals): pick the judge model with EVAL_JUDGE_MODEL (#446)

Rufus096452239c78
diff --git a/.env.example b/.env.example
index 84896393..9091ffbb 100644
--- a/.env.example
+++ b/.env.example
@@ -158,6 +158,9 @@ LOCAL_INCLUDE_USAGE_ACCOUNTING=false
# ROUTING_MODEL=llama3.3
# (Server-side web search has no local equivalent and is disabled for local
# models; tool-use quality depends on the model you choose.)
+# The eval judge (`make eval`, the Evals page) is Claude Opus unless this
+# names another model; a slug from LOCAL_MODELS judges on the local server.
+# EVAL_JUDGE_MODEL=llama3.3
# --- Chat turn timeouts -------------------------------------------------------
# Whole-turn wall-clock ceiling for a streaming /chat turn, in seconds. A turn
diff --git a/evals/judges/triage_judge.py b/evals/judges/triage_judge.py
index 318c25b6..9d58c02e 100644
--- a/evals/judges/triage_judge.py
+++ b/evals/judges/triage_judge.py
@@ -9,10 +9,6 @@ from __future__ import annotations
import json
from typing import Any
-import anthropic
-
-JUDGE_MODEL = "claude-opus-4-7"
-
def _format_decision(decision: dict[str, Any]) -> str:
return (
@@ -61,11 +57,8 @@ def _format_expected(expected: dict[str, Any]) -> str:
async def judge_triage(
scenario: dict[str, Any],
decision: dict[str, Any],
- client: anthropic.AsyncAnthropic | None = None,
) -> dict[str, Any]:
- from openexecutive.providers.anthropic_provider import configured_async_client
-
- cli = client or configured_async_client()
+ from openexecutive.evals.judges import judge_message
judge_prompt = f"""You are an evaluator for a Triage agent that decides whether incoming
events (emails, Slack messages, documents) should fire proactive alerts to a
@@ -99,13 +92,7 @@ Respond in JSON:
{{"severity_accuracy": N, "channel_appropriateness": N, "dedup_correctness": N, "overall": N, "notes": "brief explanation"}}
"""
- message = await cli.messages.create(
- model=JUDGE_MODEL,
- max_tokens=500,
- messages=[{"role": "user", "content": judge_prompt}],
- )
-
- text = message.content[0].text # type: ignore[union-attr]
+ text = await judge_message(judge_prompt)
try:
start = text.find("{")
end = text.rfind("}") + 1
diff --git a/evals/run_evals.py b/evals/run_evals.py
index 066a2016..e1610727 100644
--- a/evals/run_evals.py
+++ b/evals/run_evals.py
@@ -62,7 +62,7 @@ async def run_eval(scenario: dict, executive, session_factory) -> dict:
}
-async def judge_response(scenario: dict, response: str, client) -> dict:
+async def judge_response(scenario: dict, response: str) -> dict:
judge_prompt = f"""You are an evaluator for an AI executive advisory system.
Evaluate this response to an executive question on a 1-5 scale for each dimension.
@@ -85,13 +85,9 @@ Rate each dimension (1=poor, 3=acceptable, 5=excellent):
Respond in JSON format:
{{"persona_coherence": N, "domain_accuracy": N, "actionability": N, "topic_coverage": N, "specificity": N, "concision": N, "overall": N, "notes": "brief explanation"}}"""
- message = await client.messages.create(
- model="claude-opus-4-7",
- max_tokens=500,
- messages=[{"role": "user", "content": judge_prompt}],
- )
+ from openexecutive.evals.judges import judge_message
- text = message.content[0].text
+ text = await judge_message(judge_prompt)
try:
start = text.find("{")
end = text.rfind("}") + 1
@@ -133,7 +129,7 @@ async def run_workflow_eval(scenario: dict, store) -> dict:
}
-async def judge_workflow(scenario: dict, artifact: str, client) -> dict:
+async def judge_workflow(scenario: dict, artifact: str) -> dict:
expected_sections = scenario.get("expected_artifact_sections", []) or []
quality_criteria = scenario.get("quality_criteria", {}) or {}
@@ -161,13 +157,9 @@ Rate each dimension 1-5 (1=poor, 3=acceptable, 5=excellent):
Respond in JSON:
{{"structure": N, "specificity": N, "actionability": N, "coherence": N, "completeness": N, "overall": N, "notes": "brief"}}"""
- message = await client.messages.create(
- model="claude-opus-4-7",
- max_tokens=500,
- messages=[{"role": "user", "content": judge_prompt}],
- )
+ from openexecutive.evals.judges import judge_message
- text = message.content[0].text
+ text = await judge_message(judge_prompt)
try:
start = text.find("{")
end = text.rfind("}") + 1
@@ -176,7 +168,7 @@ Respond in JSON:
return {"overall": 0, "notes": "Failed to parse judge response"}
-async def run_triage_eval(scenario: dict, client) -> dict:
+async def run_triage_eval(scenario: dict) -> dict:
"""Runs a triage scenario directly against the TriageAgent (no Executive loop).
Scenario YAML must include `event` + `context` (with `recent_alerts`,
@@ -262,12 +254,10 @@ async def main() -> None:
initialize_overrides_db()
ReviewStore.initialize_db()
- # Built through the app's factory so an organisation-scoped key gets its
- # `anthropic-workspace-id` header here too — a bare AsyncAnthropic() 400s
- # on every judge call for those keys (#128).
- from openexecutive.providers.anthropic_provider import configured_async_client
-
- client = configured_async_client()
+ # Judges go through the provider registry (openexecutive.evals.judges),
+ # which builds the Anthropic client with the `anthropic-workspace-id`
+ # header an organisation-scoped key needs (#128) and picks the judge
+ # model from EVAL_JUDGE_MODEL.
scenario_dir = Path(args.scenarios)
output_dir = Path(args.output)
@@ -315,8 +305,8 @@ async def main() -> None:
for scenario in scenarios:
print(f" [{scenario['id']}] {scenario['description']}")
try:
- result = await run_triage_eval(scenario, client)
- scores = await judge_triage(scenario, result["decision"], client)
+ result = await run_triage_eval(scenario)
+ scores = await judge_triage(scenario, result["decision"])
result["scores"] = scores
result["passed"] = scores.get("overall", 0) >= 3.5
results.append(result)
@@ -348,7 +338,7 @@ async def main() -> None:
print(f" [{scenario['id']}] {scenario['description']}")
try:
result = await run_workflow_eval(scenario, store)
- scores = await judge_workflow(scenario, result["artifact"], client)
+ scores = await judge_workflow(scenario, result["artifact"])
result["scores"] = scores
result["passed"] = scores.get("overall", 0) >= 3.5
results.append(result)
@@ -366,7 +356,7 @@ async def main() -> None:
print(f" [{scenario['id']}] {scenario['description']}")
try:
result = await run_eval(scenario, executive, None)
- scores = await judge_response(scenario, result["response"], client)
+ scores = await judge_response(scenario, result["response"])
result["scores"] = scores
result["passed"] = scores.get("overall", 0) >= 3.5
results.append(result)
diff --git a/packages/core/openexecutive/architecture/architecture-facts.yaml b/packages/core/openexecutive/architecture/architecture-facts.yaml
index 6c8081b1..7d7c9b8c 100644
--- a/packages/core/openexecutive/architecture/architecture-facts.yaml
+++ b/packages/core/openexecutive/architecture/architecture-facts.yaml
@@ -4936,7 +4936,7 @@ audit:
- alert — alert created by the triage pipeline
- integration_inbound — Slack / Discord / email / Telegram / Google Chat message received. The chat adapters put the first 160 characters of the message in the summary; the email poller keeps the summary to sender and subject and stores the text in the full payload (`message` — the sender's new text, cut at 20,000 escaped JSON chars, the measure the 64 KB cap uses; `attachments` — filenames, 8,000 escaped chars; `body` — the whole body with its quoted chain, only on a row private to the principal, since the chain quotes other people's mail and a public row is every signed-in user's) with `details.preview` (120 chars of the new text) and `details.body_len`, so /audit shows the request next to the reply's `chat_turn` row. `details.outcome` distinguishes the non-reply paths — `rejected_unknown_sender` (not on the People roster), `skipped_gate` (response gate said no), `dropped_thread_unreadable` (Slack wouldn't return a thread the bot is in, so a continuation went unanswered; the sender was told), and `handler_error` (the turn raised; the user got the generic apology). The error row carries the same `session_id` as the inbound row, so /audit can join a failed turn to the message that caused it.
- knowledge_retrieval — one row per `knowledge.retriever.retrieve()` call; details carry chunk counts + domains, full payload carries each chunk's source, distance, and text preview so the session view shows "which RAG chunks influenced this response"
- - cache_event — one row per model call, from every call site (`audit.usage.log_model_usage`): Executive iterations and the committee revision pass (`actor` executive / committee_revision), `BaseAgent.analyze` callers — chat-turn consults via `route_parallel` (`actor` specialist), workflow steps and other direct `route_to_specialist` callers (specialist_workflow), the MCP `consult_specialist` tool (specialist_mcp), and both Council test boxes (agent_test) — `BaseAgent.analyze_with_tools` callers (specialist_research, query_watch, signal_enrichment), the research synthesis and watchlist passes (research_synthesis / research_watchlist), triage, the chat memory extractor (memory_extractor), alert review (alert_review), the brief's narrative and grounding repair (briefing_narrative / briefing_grounding), the end_of_day_digest, executive_reflection and weekly_review workflows, wait_for_human reply parsing (workflow replies and roster answers) and the inbound_resolver, committee reviewers (committee_reviewer), the fixture and engagement generators (their agent names), and the utility calls — session_title, response_gate, person_insight, initiatives_consolidation, pdf_reader, the chat page's suggested_prompts / followup_suggestion, and the eval runner's eval_judge. `tests/unit/test_usage_recording.py::test_every_model_call_records_usage` walks the package's AST and fails on any function that calls `messages_create` / `messages_stream` without calling `log_model_usage` (directly, through a same-module wrapper such as the Executive's `_emit_cache_event`, or from an enclosing function), so a new call site cannot skip the row silently; `providers/` and the runtime-dead `architecture/generator.py` are exempt, and the repo-root `evals/` harness (a separate client process with its own SDK client) is out of scope. Captures `input_tokens`, `output_tokens`, `cache_creation_input_tokens`, `cache_read_input_tokens`, `cost_usd`, `web_search_requests` (from `usage.server_tool_use`), `stop_reason`, and — inside a research run — the run's `run_id`, so prompt-cache hits are visible per turn, plus `cost_usd` — the actual USD OpenRouter charged for the call (read from `usage.cost`; null on the Anthropic-direct path or for rows logged before cost capture shipped). This is the token-usage + cost source of truth aggregated by both `/audit/sessions/{id}` (per session) and `/audit/usage` (cross-session).
+ - cache_event — one row per model call, from every call site (`audit.usage.log_model_usage`): Executive iterations and the committee revision pass (`actor` executive / committee_revision), `BaseAgent.analyze` callers — chat-turn consults via `route_parallel` (`actor` specialist), workflow steps and other direct `route_to_specialist` callers (specialist_workflow), the MCP `consult_specialist` tool (specialist_mcp), and both Council test boxes (agent_test) — `BaseAgent.analyze_with_tools` callers (specialist_research, query_watch, signal_enrichment), the research synthesis and watchlist passes (research_synthesis / research_watchlist), triage, the chat memory extractor (memory_extractor), alert review (alert_review), the brief's narrative and grounding repair (briefing_narrative / briefing_grounding), the end_of_day_digest, executive_reflection and weekly_review workflows, wait_for_human reply parsing (workflow replies and roster answers) and the inbound_resolver, committee reviewers (committee_reviewer), the fixture and engagement generators (their agent names), and the utility calls — session_title, response_gate, person_insight, initiatives_consolidation, pdf_reader, the chat page's suggested_prompts / followup_suggestion, and the eval runner's eval_judge. `tests/unit/test_usage_recording.py::test_every_model_call_records_usage` walks the package's AST and fails on any function that calls `messages_create` / `messages_stream` without calling `log_model_usage` (directly, through a same-module wrapper such as the Executive's `_emit_cache_event`, or from an enclosing function), so a new call site cannot skip the row silently; `providers/` and the runtime-dead `architecture/generator.py` are exempt, and the repo-root `evals/` harness is out of scope (its judges call the package's `evals.judges.judge_message`, which records the eval_judge row and uses `EVAL_JUDGE_MODEL`, default Opus). Captures `input_tokens`, `output_tokens`, `cache_creation_input_tokens`, `cache_read_input_tokens`, `cost_usd`, `web_search_requests` (from `usage.server_tool_use`), `stop_reason`, and — inside a research run — the run's `run_id`, so prompt-cache hits are visible per turn, plus `cost_usd` — the actual USD OpenRouter charged for the call (read from `usage.cost`; null on the Anthropic-direct path or for rows logged before cost capture shipped). This is the token-usage + cost source of truth aggregated by both `/audit/sessions/{id}` (per session) and `/audit/usage` (cross-session).
- memory_snapshot — one row at Executive turn entry; captures the system-block layout, conversation history length, episodic context (full text in `full_json`), and a company-profile hash so the session view can render "what the agent saw" before any API call
- committee_review — one row per committee-reviewed turn; carries draft length, final length, per-reviewer critiques and timings (previously emitted but newly declared as a recognised event_type)
- peer_memory — one row per `openexecutive.memory.honcho_client` call (prefetch / sync_turn / seed_identity / directional_chat / prefetch_department / sync_department_turn / append_department_note); carries `op`, `person_id`, optional `department_slug`, `outcome` (ok / timeout / error / disabled / no_person / no_department / no_loop / empty), `duration_ms`, plus op-specific details (query preview, reasoning_level, peer count, co_present_person_ids, co_present_department_slugs, originating_person_id, target_person_id, session_id, kind) so the flow-chart view renders the peer_memory path alongside `<retrieved_context>` and `<past_decisions>`. Disabled / no-Person / no-Department turns still emit (outcome marker) so audit absence equals "code path never ran" instead of "the provider was down."
diff --git a/packages/core/openexecutive/config.py b/packages/core/openexecutive/config.py
index 4868dda8..f30b1fe7 100644
--- a/packages/core/openexecutive/config.py
+++ b/packages/core/openexecutive/config.py
@@ -320,6 +320,12 @@ class Settings(BaseSettings):
Literal["none", "minimal", "low", "medium", "high"] | None
) = Field(None, alias="LOCAL_REASONING_EFFORT")
+ # The model that scores eval answers (`make eval` and the Evals page).
+ # Unset = Claude Opus. Any slug the registry routes works, so a slug in
+ # LOCAL_MODELS runs the judge on the local server too, for a run with no
+ # hosted API at all.
+ eval_judge_model: str | None = Field(None, alias="EVAL_JUDGE_MODEL")
+
@field_validator("local_models", mode="before")
@classmethod
def _parse_local_models(cls, v: Any) -> list[str]:
diff --git a/packages/core/openexecutive/evals/judges.py b/packages/core/openexecutive/evals/judges.py
index 07e4b71e..44e0e6a6 100644
--- a/packages/core/openexecutive/evals/judges.py
+++ b/packages/core/openexecutive/evals/judges.py
@@ -8,13 +8,34 @@ from openexecutive.audit.usage import log_model_usage
from openexecutive.providers import get_provider
# The judge routes through the provider abstraction (like every other LLM call),
-# so when OPENROUTER_ENABLED is on, judging bills the OpenRouter account too.
+# so when OPENROUTER_ENABLED is on, judging bills the OpenRouter account too,
+# and a slug listed in LOCAL_MODELS is judged by the local server.
+# EVAL_JUDGE_MODEL picks the model; unset, it is this one.
_JUDGE_MODEL = "claude-opus-5-5"
# Room for the judge's adaptive thinking (always on for Opus 5.5) before the
# JSON verdict; 500 tokens could be spent thinking with nothing left to answer.
_JUDGE_MAX_TOKENS = 4096
+def judge_model() -> str:
+ """The model that scores eval answers: ``EVAL_JUDGE_MODEL``, else Opus."""
+ from openexecutive.config import get_settings
+
+ return get_settings().eval_judge_model or _JUDGE_MODEL
+
+
+async def judge_message(prompt: str, *, max_tokens: int = _JUDGE_MAX_TOKENS) -> str:
+ """Send one judge prompt to the judge model and return the reply's text."""
+ model = judge_model()
+ message = await get_provider(model).messages_create(
+ model=model,
+ max_tokens=max_tokens,
+ messages=[{"role": "user", "content": prompt}],
+ )
+ log_model_usage(message, model=model, actor="eval_judge")
+ return _response_text(message)
+
+
def _response_text(message: Any) -> str:
"""The reply's text, skipping thinking blocks that may come first."""
return "".join(
@@ -190,13 +211,7 @@ Rate each dimension (1=poor, 3=acceptable, 5=excellent):
Respond in JSON format:
{{"persona_coherence": N, "domain_accuracy": N, "actionability": N, "topic_coverage": N, "specificity": N, "overall": N, "notes": "brief explanation"}}"""
- message = await get_provider(_JUDGE_MODEL).messages_create(
- model=_JUDGE_MODEL,
- max_tokens=_JUDGE_MAX_TOKENS,
- messages=[{"role": "user", "content": judge_prompt}],
- )
- log_model_usage(message, model=_JUDGE_MODEL, actor="eval_judge")
- text = _response_text(message)
+ text = await judge_message(judge_prompt)
try:
start = text.find("{")
end = text.rfind("}") + 1
@@ -240,13 +255,7 @@ Rate each dimension 1-5 (1=poor, 3=acceptable, 5=excellent):
Respond in JSON:
{{"structure": N, "specificity": N, "actionability": N, "coherence": N, "completeness": N, "overall": N, "notes": "brief"}}"""
- message = await get_provider(_JUDGE_MODEL).messages_create(
- model=_JUDGE_MODEL,
- max_tokens=_JUDGE_MAX_TOKENS,
- messages=[{"role": "user", "content": judge_prompt}],
- )
- log_model_usage(message, model=_JUDGE_MODEL, actor="eval_judge")
- text = _response_text(message)
+ text = await judge_message(judge_prompt)
try:
start = text.find("{")
end = text.rfind("}") + 1
@@ -303,13 +312,7 @@ If any quality criterion is not met, overall must be 2 or lower and notes must n
Respond in JSON:
{{"faithfulness": N, "safety": N, "open_questions": N, "voice": N, "overall": N, "notes": "brief"}}"""
- message = await get_provider(_JUDGE_MODEL).messages_create(
- model=_JUDGE_MODEL,
- max_tokens=_JUDGE_MAX_TOKENS,
- messages=[{"role": "user", "content": judge_prompt}],
- )
- log_model_usage(message, model=_JUDGE_MODEL, actor="eval_judge")
- text = _response_text(message)
+ text = await judge_message(judge_prompt)
try:
start = text.find("{")
end = text.rfind("}") + 1
@@ -383,13 +386,7 @@ ACTUAL DECISION:
Respond in JSON:
{{"severity_accuracy": N, "channel_appropriateness": N, "dedup_correctness": N, "overall": N, "notes": "brief"}}"""
- message = await get_provider(_JUDGE_MODEL).messages_create(
- model=_JUDGE_MODEL,
- max_tokens=_JUDGE_MAX_TOKENS,
- messages=[{"role": "user", "content": judge_prompt}],
- )
- log_model_usage(message, model=_JUDGE_MODEL, actor="eval_judge")
- text = _response_text(message)
+ text = await judge_message(judge_prompt)
try:
start = text.find("{")
end = text.rfind("}") + 1
diff --git a/packages/core/tests/unit/test_eval_judge_model.py b/packages/core/tests/unit/test_eval_judge_model.py
new file mode 100644
index 00000000..c9192407
--- /dev/null
+++ b/packages/core/tests/unit/test_eval_judge_model.py
@@ -0,0 +1,69 @@
+"""EVAL_JUDGE_MODEL picks the model that scores eval answers.
+
+Unset, the judge stays on Claude Opus. Set, every judge call goes to that
+model through the provider registry, so a slug listed in LOCAL_MODELS is
+judged by the local server (an eval run with no hosted API at all).
+"""
+from __future__ import annotations
+
+import asyncio
+from typing import Any
+
+import pytest
+
+from openexecutive.evals import judges as judges_module
+
+
+class _Msg:
+ content = [type("T", (), {"type": "text", "text": '{"overall": 4, "notes": "ok"}'})()]
+
+
+def calls(monkeypatch: pytest.MonkeyPatch) -> list[tuple[str, dict[str, Any]]]:
+ seen: list[tuple[str, dict[str, Any]]] = []
+
+ class _Provider:
+ def __init__(self, routed: str) -> None:
+ self.routed = routed
+
+ async def messages_create(self, **kwargs: Any) -> Any:
+ seen.append((self.routed, kwargs))
+ return _Msg()
+
+ monkeypatch.setattr(judges_module, "get_provider", _Provider)
+ monkeypatch.setattr(judges_module, "log_model_usage", lambda *a, **k: None)
+ return seen
+
+
+def test_unset_judges_with_opus(
+ monkeypatch: pytest.MonkeyPatch, calls: list[tuple[str, dict[str, Any]]]
+) -> None:
+ monkeypatch.delenv("EVAL_JUDGE_MODEL", raising=False)
+
+ scores = asyncio.run(judges_module.judge_chat({"query": "q", "expected_topics": []}, "r"))
+
+ assert scores["overall"] == 4
+ assert calls == [("claude-opus-5-5", calls[0][1])]
+ assert calls[0][1]["model"] == "claude-opus-5-5"
+
+
+def test_set_routes_every_judge_to_that_model(
+ monkeypatch: pytest.MonkeyPatch, calls: list[tuple[str, dict[str, Any]]]
+) -> None:
+ monkeypatch.setenv("EVAL_JUDGE_MODEL", "qwen3:4b")
+
+ asyncio.run(judges_module.judge_chat({"query": "q", "expected_topics": []}, "r"))
+ asyncio.run(
+ judges_module.judge_workflow(
+ {"workflow": "w", "description": "d", "expected_sections": []}, "artifact"
+ )
+ )
+
+ assert [routed for routed, _ in calls] == ["qwen3:4b", "qwen3:4b"]
+ assert all(kwargs["model"] == "qwen3:4b" for _, kwargs in calls)
+
+
+def test_blank_value_reads_as_unset(monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setenv("EVAL_JUDGE_MODEL", "")
+
+ assert judges_module.judge_model() == "claude-opus-5-5"