Skip to content

AI-powered virtual executive team — a single coherent executive persona backed by 8 specialist agents (FastAPI + Next.js).

fix(evals): give MCP scenarios a gateway so they can reach their tools (#563)

ladislav-urband9c629e6ba24
diff --git a/evals/run_evals.py b/evals/run_evals.py
index e1610727..6276f342 100644
--- a/evals/run_evals.py
+++ b/evals/run_evals.py
@@ -8,6 +8,7 @@ from __future__ import annotations
import argparse
import asyncio
+import contextlib
import json
import os
import sys
@@ -200,6 +201,34 @@ async def run_triage_eval(scenario: dict) -> dict:
}
+async def _eval_gateway(enabled: bool):
+ """For an mcp run, an MCP gateway from the configured mcp_servers.json,
+ made the active one (approval cards and the brief read the active
+ gateway, as in the API); ``None`` otherwise. Exits when there is no
+ config, since an mcp run without tools would judge nothing real. The
+ gateway is closed and the active one reset on every way out, including a
+ start that fails or times out after spawning some servers."""
+ if not enabled:
+ yield None
+ return
+ from openexecutive.config import get_settings
+ from openexecutive.orchestrator.mcp_gateway import MCPGateway, set_active_gateway
+
+ config_path = get_settings().mcp_servers_config_path
+ if not config_path.exists():
+ sys.exit(f"--mcp needs an MCP config at {config_path} (MCP_SERVERS_CONFIG_PATH)")
+ gateway = MCPGateway()
+ try:
+ await asyncio.wait_for(gateway.start(config_path), timeout=120)
+ set_active_gateway(gateway)
+ print(f"MCP gateway started from {config_path}")
+ yield gateway
+ finally:
+ set_active_gateway(None)
+ await gateway.close()
+
+
async def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument(
@@ -348,23 +377,27 @@ async def main() -> None:
print(f" → ERROR: {e}")
results.append({"id": scenario["id"], "error": str(e), "passed": False})
else:
- # "chat" (default) and "mcp" both go through the Executive.
+ # "chat" (default) and "mcp" both go through the Executive; mcp with
+ # a gateway started from the configured mcp_servers.json, as the API
+ # starts it, so its scenarios can reach their tools (without one the
+ # Executive offers no MCP tools at all).
from openexecutive.orchestrator.executive import Executive
- executive = Executive()
- for scenario in scenarios:
- print(f" [{scenario['id']}] {scenario['description']}")
- try:
- result = await run_eval(scenario, executive, None)
- scores = await judge_response(scenario, result["response"])
- result["scores"] = scores
- result["passed"] = scores.get("overall", 0) >= 3.5
- results.append(result)
- status = "PASS" if result["passed"] else "FAIL"
- print(f" → {status} (overall: {scores.get('overall', 0):.1f}/5)")
- except Exception as e:
- print(f" → ERROR: {e}")
- results.append({"id": scenario["id"], "error": str(e), "passed": False})
+ async with _eval_gateway(target_kind == "mcp") as gateway:
+ executive = Executive(mcp_gateway=gateway)
+ for scenario in scenarios:
+ print(f" [{scenario['id']}] {scenario['description']}")
+ try:
+ result = await run_eval(scenario, executive, None)
+ scores = await judge_response(scenario, result["response"])
+ result["scores"] = scores
+ result["passed"] = scores.get("overall", 0) >= 3.5
+ results.append(result)
+ status = "PASS" if result["passed"] else "FAIL"
+ print(f" → {status} (overall: {scores.get('overall', 0):.1f}/5)")
+ except Exception as e:
+ print(f" → ERROR: {e}")
+ results.append({"id": scenario["id"], "error": str(e), "passed": False})
timestamp = datetime.utcnow().strftime("%Y%m%d_%H%M%S")
output_file = output_dir / f"eval_results_{target_kind}_{timestamp}.json"
diff --git a/packages/core/openexecutive/evals/runner.py b/packages/core/openexecutive/evals/runner.py
index 9c1599b4..250c5db5 100644
--- a/packages/core/openexecutive/evals/runner.py
+++ b/packages/core/openexecutive/evals/runner.py
@@ -181,8 +181,9 @@ def _make_run_one(
return _make_inbox_runner(sem, queue, passed, total, cancel_event)
if kind == "workflow":
return _make_workflow_runner(store, sem, queue, passed, total, cancel_event)
- # chat (default) and mcp both go through Executive.chat
- return _make_chat_runner(sem, queue, passed, total, cancel_event)
+ # chat (default) and mcp both go through Executive.chat; mcp with the
+ # API's running gateway, so its scenarios can reach their tools.
+ return _make_chat_runner(sem, queue, passed, total, cancel_event, with_mcp=kind == "mcp")
def _is_canceled(cancel_event: asyncio.Event | None) -> bool:
@@ -467,21 +468,47 @@ def _make_workflow_runner(
return run_one
+# Why an mcp scenario fails when the API has no running gateway (no
+# mcp_servers.json, or it failed to start): judged without tools, it would
+# measure nothing.
+NO_MCP_GATEWAY = "no MCP gateway is running, so this mcp scenario can't reach its tools"
+
+
def _make_chat_runner(
sem: asyncio.Semaphore,
queue: asyncio.Queue[dict[str, Any] | None],
passed: list[int],
total: int,
cancel_event: asyncio.Event | None,
+ *,
+ with_mcp: bool = False,
) -> RunOne:
from openexecutive.memory.company_profile import CompanyProfile
from openexecutive.orchestrator.debug_events import DebugCollector
from openexecutive.orchestrator.executive import Executive
+ from openexecutive.orchestrator.mcp_gateway import get_active_gateway
from openexecutive.orchestrator.session import Session
# Shared Executive — confirmed safe: no mutable instance state on the
# hot path; chat()/stream_chat() route all per-call state through Session.
- executive = Executive()
+ # An mcp scenario needs the gateway: without one the Executive offers no
+ # MCP tools, and the scenario would be judged on a turn that can't reach
+ # them. The gateway is read again for each scenario, so a long run follows
+ # a gateway the API replaced, and a scenario with none fails instead.
+ current_gateway = get_active_gateway() if with_mcp else None
+ executive = Executive(mcp_gateway=current_gateway)
+
+ def executive_for_scenario() -> Executive:
+ nonlocal current_gateway, executive
+ if not with_mcp:
+ return executive
+ gateway = get_active_gateway()
+ if gateway is None:
+ raise RuntimeError(NO_MCP_GATEWAY)
+ if gateway is not current_gateway:
+ current_gateway = gateway
+ executive = Executive(mcp_gateway=gateway)
+ return executive
async def run_one(i: int, scenario: dict[str, Any]) -> None:
if _is_canceled(cancel_event):
@@ -500,6 +527,7 @@ def _make_chat_runner(
}
)
try:
+ scenario_executive = executive_for_scenario()
ctx = scenario.get("company_context", {})
profile = CompanyProfile(
name=ctx.get("name", "Eval Company"),
@@ -526,7 +554,7 @@ def _make_chat_runner(
# Records which specialists the turn consulted, for
# the scenario's routing expectation (routing_check).
collector = DebugCollector()
- response = await executive.chat(
+ response = await scenario_executive.chat(
user_message=query,
session=session,
debug_collector=collector,
diff --git a/packages/core/tests/unit/test_evals_mcp_gateway.py b/packages/core/tests/unit/test_evals_mcp_gateway.py
new file mode 100644
index 00000000..0e886799
--- /dev/null
+++ b/packages/core/tests/unit/test_evals_mcp_gateway.py
@@ -0,0 +1,218 @@
+"""An mcp eval run reaches the MCP tools: the in-app runner gives its
+Executive the API's running gateway (evals/runner.py), reads it again for each
+scenario and fails one when there is none, and a chat run keeps running
+without one. evals/run_evals.py --mcp starts its own gateway, makes it the
+active one, and closes it and resets the active one on every way out."""
+from __future__ import annotations
+
+import asyncio
+import importlib.util
+from pathlib import Path
+from types import ModuleType, SimpleNamespace
+
+import pytest
+
+
+def test_an_mcp_run_gets_the_running_gateway_and_a_chat_run_none(monkeypatch: pytest.MonkeyPatch) -> None:
+ # Without a gateway the Executive offers no MCP tools, so an mcp
+ # scenario would be judged on a turn that can't reach them.
+ from openexecutive.evals import runner
+ from openexecutive.orchestrator import executive as executive_module
+
+ given: list[object] = []
+
+ class Recording:
+ def __init__(self, mcp_gateway: object = None) -> None:
+ given.append(mcp_gateway)
+
+ gateway = object()
+ monkeypatch.setattr(executive_module, "Executive", Recording)
+ monkeypatch.setattr("openexecutive.orchestrator.mcp_gateway.get_active_gateway", lambda: gateway)
+ queue: asyncio.Queue[dict[str, object] | None] = asyncio.Queue()
+ for with_mcp in (True, False):
+ runner._make_chat_runner(asyncio.Semaphore(1), queue, [0], 1, None, with_mcp=with_mcp)
+ assert given == [gateway, None]
+
+
+def test_the_run_picks_the_runner_by_kind(monkeypatch: pytest.MonkeyPatch) -> None:
+ from openexecutive.evals import runner
+
+ asked: list[bool] = []
+ monkeypatch.setattr(
+ runner, "_make_chat_runner", lambda *_a, with_mcp=False, **_k: asked.append(with_mcp)
+ )
+ queue: asyncio.Queue[dict[str, object] | None] = asyncio.Queue()
+ for kind in ("mcp", "chat"):
+ runner._make_run_one(
+ kind=kind, store=None, sem=asyncio.Semaphore(1), queue=queue, passed=[0], total=1, cancel_event=None
+ )
+ assert asked == [True, False]
+
+
+def _recording_executive(made: list[object]) -> type:
+ """A fake Executive that records the gateway each one was built with."""
+
+ class Recording:
+ def __init__(self, mcp_gateway: object = None) -> None:
+ made.append(mcp_gateway)
+
+ async def chat(self, **_kwargs: object) -> str:
+ return "answer"
+
+ return Recording
+
+
+def _drain(queue: asyncio.Queue[dict[str, object] | None]) -> list[dict[str, object]]:
+ events = []
+ while not queue.empty():
+ event = queue.get_nowait()
+ if event is not None and event["type"] != "scenario_start":
+ events.append(event)
+ return events
+
+
+async def test_an_mcp_scenario_with_no_running_gateway_fails(monkeypatch: pytest.MonkeyPatch) -> None:
+ # Judged without tools, it would measure nothing.
+ from openexecutive.evals import runner
+ from openexecutive.orchestrator import executive as executive_module
+
+ monkeypatch.setattr(executive_module, "Executive", _recording_executive([]))
+ monkeypatch.setattr("openexecutive.orchestrator.mcp_gateway.get_active_gateway", lambda: None)
+ queue: asyncio.Queue[dict[str, object] | None] = asyncio.Queue()
+ run_one = runner._make_chat_runner(asyncio.Semaphore(1), queue, [0], 1, None, with_mcp=True)
+ await run_one(0, {"id": "s0", "query": "q"})
+ assert [(e["type"], e["error"]) for e in _drain(queue)] == [("scenario_error", runner.NO_MCP_GATEWAY)]
+
+
+async def test_each_mcp_scenario_reads_the_running_gateway_again(monkeypatch: pytest.MonkeyPatch) -> None:
+ # A gateway the API replaces during a long run is the one the next scenario uses.
+ from openexecutive.evals import runner
+ from openexecutive.orchestrator import executive as executive_module
+
+ async def judged(*_args: object, **_kwargs: object) -> dict[str, float]:
+ return {"overall": 5.0}
+
+ first, second = object(), object()
+ active = [first]
+ made: list[object] = []
+ monkeypatch.setattr(executive_module, "Executive", _recording_executive(made))
+ monkeypatch.setattr("openexecutive.orchestrator.mcp_gateway.get_active_gateway", lambda: active[0])
+ monkeypatch.setattr(runner, "judge_chat", judged)
+ queue: asyncio.Queue[dict[str, object] | None] = asyncio.Queue()
+ run_one = runner._make_chat_runner(asyncio.Semaphore(1), queue, [0], 3, None, with_mcp=True)
+ await run_one(0, {"id": "s0", "query": "q"})
+ active[0] = second
+ await run_one(1, {"id": "s1", "query": "q"})
+ await run_one(2, {"id": "s2", "query": "q"})
+ # Built for the first gateway, again when it was replaced, and reused after.
+ assert made == [first, second]
+ assert [e["type"] for e in _drain(queue)] == ["scenario_done"] * 3
+
+
+class _Gateway:
+ """A fake MCPGateway: records start and close, and can fail to start."""
+
+ def __init__(self, fail: BaseException | None = None) -> None:
+ self.fail = fail
+ self.started = False
+ self.closed = False
+
+ async def start(self, config_path: object) -> None:
+ if self.fail is not None:
+ raise self.fail
+ self.started = True
+
+ async def close(self) -> None:
+ self.closed = True
+
+
+def _run_evals(monkeypatch: pytest.MonkeyPatch, tmp_path: Path, *, config: bool) -> ModuleType:
+ """evals/run_evals.py, with the MCP config at a path that exists only when ``config``."""
+ config_path = tmp_path / "mcp_servers.json"
+ if config:
+ config_path.write_text("{}")
+ monkeypatch.setattr(
+ "openexecutive.config.get_settings", lambda: SimpleNamespace(mcp_servers_config_path=config_path)
+ )
+ script = Path(__file__).resolve().parents[4] / "evals" / "run_evals.py"
+ spec = importlib.util.spec_from_file_location("run_evals_under_test", script)
+ assert spec is not None and spec.loader is not None
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+ return module
+
+
+async def test_the_mcp_run_starts_the_gateway_makes_it_active_and_closes_it(
+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+) -> None:
+ from openexecutive.orchestrator import mcp_gateway
+
+ gateway = _Gateway()
+ monkeypatch.setattr(mcp_gateway, "MCPGateway", lambda: gateway)
+ run_evals = _run_evals(monkeypatch, tmp_path, config=True)
+ async with run_evals._eval_gateway(True) as given:
+ assert given is gateway and gateway.started
+ assert mcp_gateway.get_active_gateway() is gateway
+ assert gateway.closed
+ assert mcp_gateway.get_active_gateway() is None
+
+
+async def test_a_failing_run_still_closes_the_gateway(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
+ from openexecutive.orchestrator import mcp_gateway
+
+ gateway = _Gateway()
+ monkeypatch.setattr(mcp_gateway, "MCPGateway", lambda: gateway)
+ run_evals = _run_evals(monkeypatch, tmp_path, config=True)
+ with pytest.raises(RuntimeError, match="scenario blew up"):
+ async with run_evals._eval_gateway(True):
+ raise RuntimeError("scenario blew up")
+ assert gateway.closed
+ assert mcp_gateway.get_active_gateway() is None
+
+
[email protected]("failure", [RuntimeError("server 3 failed"), TimeoutError()])
+async def test_a_start_that_fails_part_way_closes_what_it_spawned(
+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path, failure: BaseException
+) -> None:
+ # Servers spawned before the failure or the timeout would otherwise leak.
+ from openexecutive.orchestrator import mcp_gateway
+
+ gateway = _Gateway(fail=failure)
+ monkeypatch.setattr(mcp_gateway, "MCPGateway", lambda: gateway)
+ run_evals = _run_evals(monkeypatch, tmp_path, config=True)
+ with pytest.raises(type(failure)):
+ async with run_evals._eval_gateway(True):
+ pytest.fail("the run must not start")
+ assert gateway.closed
+ assert mcp_gateway.get_active_gateway() is None
+
+
+async def test_an_mcp_run_with_no_config_exits_before_any_gateway(
+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+) -> None:
+ from openexecutive.orchestrator import mcp_gateway
+
+ made: list[_Gateway] = []
+
+ def make() -> _Gateway:
+ made.append(_Gateway())
+ return made[-1]
+
+ monkeypatch.setattr(mcp_gateway, "MCPGateway", make)
+ run_evals = _run_evals(monkeypatch, tmp_path, config=False)
+ with pytest.raises(SystemExit, match="--mcp needs an MCP config"):
+ async with run_evals._eval_gateway(True):
+ pytest.fail("the run must not start")
+ assert made == []
+
+
+async def test_a_chat_run_gets_no_gateway(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
+ from openexecutive.orchestrator import mcp_gateway
+
+ def make() -> None:
+ pytest.fail("a chat run starts no gateway")
+
+ monkeypatch.setattr(mcp_gateway, "MCPGateway", make)
+ run_evals = _run_evals(monkeypatch, tmp_path, config=True)
+ async with run_evals._eval_gateway(False) as given:
+ assert given is None
diff --git a/packages/core/tests/unit/test_principal_role.py b/packages/core/tests/unit/test_principal_role.py
index 8feae9f3..308acf70 100644
--- a/packages/core/tests/unit/test_principal_role.py
+++ b/packages/core/tests/unit/test_principal_role.py
@@ -924,7 +924,7 @@ def test_chat_runner_puts_the_role_on_the_session(monkeypatch: pytest.MonkeyPatc
async def fake_judge(_scenario: Any, _response: str) -> dict[str, Any]:
return {"overall": 5}
- monkeypatch.setattr(Executive, "__init__", lambda self: None)
+ monkeypatch.setattr(Executive, "__init__", lambda self, mcp_gateway=None: None)
monkeypatch.setattr(Executive, "chat", fake_chat)
monkeypatch.setattr(runner, "judge_chat", fake_judge)
queue: asyncio.Queue[Any] = asyncio.Queue()