AI-powered virtual executive team — a single coherent executive persona backed by 8 specialist agents (FastAPI + Next.js).
fix(evals): give MCP scenarios a gateway so they can reach their tools (#563)
ladislav-urband9c629e6ba24
diff --git a/evals/run_evals.py b/evals/run_evals.pyindex e1610727..6276f342 100644--- a/evals/run_evals.py+++ b/evals/run_evals.py@@ -8,6 +8,7 @@ from __future__ import annotationsimport argparseimport asyncio+import contextlibimport jsonimport osimport sys@@ -200,6 +201,34 @@ async def run_triage_eval(scenario: dict) -> dict:}+async def _eval_gateway(enabled: bool):+ """For an mcp run, an MCP gateway from the configured mcp_servers.json,+ made the active one (approval cards and the brief read the active+ gateway, as in the API); ``None`` otherwise. Exits when there is no+ config, since an mcp run without tools would judge nothing real. The+ gateway is closed and the active one reset on every way out, including a+ start that fails or times out after spawning some servers."""+ if not enabled:+ yield None+ return+ from openexecutive.config import get_settings+ from openexecutive.orchestrator.mcp_gateway import MCPGateway, set_active_gateway++ config_path = get_settings().mcp_servers_config_path+ if not config_path.exists():+ sys.exit(f"--mcp needs an MCP config at {config_path} (MCP_SERVERS_CONFIG_PATH)")+ gateway = MCPGateway()+ try:+ await asyncio.wait_for(gateway.start(config_path), timeout=120)+ set_active_gateway(gateway)+ print(f"MCP gateway started from {config_path}")+ yield gateway+ finally:+ set_active_gateway(None)+ await gateway.close()++async def main() -> None:parser = argparse.ArgumentParser()parser.add_argument(@@ -348,23 +377,27 @@ async def main() -> None:print(f" → ERROR: {e}")results.append({"id": scenario["id"], "error": str(e), "passed": False})else:- # "chat" (default) and "mcp" both go through the Executive.+ # "chat" (default) and "mcp" both go through the Executive; mcp with+ # a gateway started from the configured mcp_servers.json, as the API+ # starts it, so its scenarios can reach their tools (without one the+ # Executive offers no MCP tools at all).from openexecutive.orchestrator.executive import Executive- executive = Executive()- for scenario in scenarios:- print(f" [{scenario['id']}] {scenario['description']}")- try:- result = await run_eval(scenario, executive, None)- scores = await judge_response(scenario, result["response"])- result["scores"] = scores- result["passed"] = scores.get("overall", 0) >= 3.5- results.append(result)- status = "PASS" if result["passed"] else "FAIL"- print(f" → {status} (overall: {scores.get('overall', 0):.1f}/5)")- except Exception as e:- print(f" → ERROR: {e}")- results.append({"id": scenario["id"], "error": str(e), "passed": False})+ async with _eval_gateway(target_kind == "mcp") as gateway:+ executive = Executive(mcp_gateway=gateway)+ for scenario in scenarios:+ print(f" [{scenario['id']}] {scenario['description']}")+ try:+ result = await run_eval(scenario, executive, None)+ scores = await judge_response(scenario, result["response"])+ result["scores"] = scores+ result["passed"] = scores.get("overall", 0) >= 3.5+ results.append(result)+ status = "PASS" if result["passed"] else "FAIL"+ print(f" → {status} (overall: {scores.get('overall', 0):.1f}/5)")+ except Exception as e:+ print(f" → ERROR: {e}")+ results.append({"id": scenario["id"], "error": str(e), "passed": False})timestamp = datetime.utcnow().strftime("%Y%m%d_%H%M%S")output_file = output_dir / f"eval_results_{target_kind}_{timestamp}.json"diff --git a/packages/core/openexecutive/evals/runner.py b/packages/core/openexecutive/evals/runner.pyindex 9c1599b4..250c5db5 100644--- a/packages/core/openexecutive/evals/runner.py+++ b/packages/core/openexecutive/evals/runner.py@@ -181,8 +181,9 @@ def _make_run_one(return _make_inbox_runner(sem, queue, passed, total, cancel_event)if kind == "workflow":return _make_workflow_runner(store, sem, queue, passed, total, cancel_event)- # chat (default) and mcp both go through Executive.chat- return _make_chat_runner(sem, queue, passed, total, cancel_event)+ # chat (default) and mcp both go through Executive.chat; mcp with the+ # API's running gateway, so its scenarios can reach their tools.+ return _make_chat_runner(sem, queue, passed, total, cancel_event, with_mcp=kind == "mcp")def _is_canceled(cancel_event: asyncio.Event | None) -> bool:@@ -467,21 +468,47 @@ def _make_workflow_runner(return run_one+# Why an mcp scenario fails when the API has no running gateway (no+# mcp_servers.json, or it failed to start): judged without tools, it would+# measure nothing.+NO_MCP_GATEWAY = "no MCP gateway is running, so this mcp scenario can't reach its tools"++def _make_chat_runner(sem: asyncio.Semaphore,queue: asyncio.Queue[dict[str, Any] | None],passed: list[int],total: int,cancel_event: asyncio.Event | None,+ *,+ with_mcp: bool = False,) -> RunOne:from openexecutive.memory.company_profile import CompanyProfilefrom openexecutive.orchestrator.debug_events import DebugCollectorfrom openexecutive.orchestrator.executive import Executive+ from openexecutive.orchestrator.mcp_gateway import get_active_gatewayfrom openexecutive.orchestrator.session import Session# Shared Executive — confirmed safe: no mutable instance state on the# hot path; chat()/stream_chat() route all per-call state through Session.- executive = Executive()+ # An mcp scenario needs the gateway: without one the Executive offers no+ # MCP tools, and the scenario would be judged on a turn that can't reach+ # them. The gateway is read again for each scenario, so a long run follows+ # a gateway the API replaced, and a scenario with none fails instead.+ current_gateway = get_active_gateway() if with_mcp else None+ executive = Executive(mcp_gateway=current_gateway)++ def executive_for_scenario() -> Executive:+ nonlocal current_gateway, executive+ if not with_mcp:+ return executive+ gateway = get_active_gateway()+ if gateway is None:+ raise RuntimeError(NO_MCP_GATEWAY)+ if gateway is not current_gateway:+ current_gateway = gateway+ executive = Executive(mcp_gateway=gateway)+ return executiveasync def run_one(i: int, scenario: dict[str, Any]) -> None:if _is_canceled(cancel_event):@@ -500,6 +527,7 @@ def _make_chat_runner(})try:+ scenario_executive = executive_for_scenario()ctx = scenario.get("company_context", {})profile = CompanyProfile(name=ctx.get("name", "Eval Company"),@@ -526,7 +554,7 @@ def _make_chat_runner(# Records which specialists the turn consulted, for# the scenario's routing expectation (routing_check).collector = DebugCollector()- response = await executive.chat(+ response = await scenario_executive.chat(user_message=query,session=session,debug_collector=collector,diff --git a/packages/core/tests/unit/test_evals_mcp_gateway.py b/packages/core/tests/unit/test_evals_mcp_gateway.pynew file mode 100644index 00000000..0e886799--- /dev/null+++ b/packages/core/tests/unit/test_evals_mcp_gateway.py@@ -0,0 +1,218 @@+"""An mcp eval run reaches the MCP tools: the in-app runner gives its+Executive the API's running gateway (evals/runner.py), reads it again for each+scenario and fails one when there is none, and a chat run keeps running+without one. evals/run_evals.py --mcp starts its own gateway, makes it the+active one, and closes it and resets the active one on every way out."""+from __future__ import annotations++import asyncio+import importlib.util+from pathlib import Path+from types import ModuleType, SimpleNamespace++import pytest+++def test_an_mcp_run_gets_the_running_gateway_and_a_chat_run_none(monkeypatch: pytest.MonkeyPatch) -> None:+ # Without a gateway the Executive offers no MCP tools, so an mcp+ # scenario would be judged on a turn that can't reach them.+ from openexecutive.evals import runner+ from openexecutive.orchestrator import executive as executive_module++ given: list[object] = []++ class Recording:+ def __init__(self, mcp_gateway: object = None) -> None:+ given.append(mcp_gateway)++ gateway = object()+ monkeypatch.setattr(executive_module, "Executive", Recording)+ monkeypatch.setattr("openexecutive.orchestrator.mcp_gateway.get_active_gateway", lambda: gateway)+ queue: asyncio.Queue[dict[str, object] | None] = asyncio.Queue()+ for with_mcp in (True, False):+ runner._make_chat_runner(asyncio.Semaphore(1), queue, [0], 1, None, with_mcp=with_mcp)+ assert given == [gateway, None]+++def test_the_run_picks_the_runner_by_kind(monkeypatch: pytest.MonkeyPatch) -> None:+ from openexecutive.evals import runner++ asked: list[bool] = []+ monkeypatch.setattr(+ runner, "_make_chat_runner", lambda *_a, with_mcp=False, **_k: asked.append(with_mcp)+ )+ queue: asyncio.Queue[dict[str, object] | None] = asyncio.Queue()+ for kind in ("mcp", "chat"):+ runner._make_run_one(+ kind=kind, store=None, sem=asyncio.Semaphore(1), queue=queue, passed=[0], total=1, cancel_event=None+ )+ assert asked == [True, False]+++def _recording_executive(made: list[object]) -> type:+ """A fake Executive that records the gateway each one was built with."""++ class Recording:+ def __init__(self, mcp_gateway: object = None) -> None:+ made.append(mcp_gateway)++ async def chat(self, **_kwargs: object) -> str:+ return "answer"++ return Recording+++def _drain(queue: asyncio.Queue[dict[str, object] | None]) -> list[dict[str, object]]:+ events = []+ while not queue.empty():+ event = queue.get_nowait()+ if event is not None and event["type"] != "scenario_start":+ events.append(event)+ return events+++async def test_an_mcp_scenario_with_no_running_gateway_fails(monkeypatch: pytest.MonkeyPatch) -> None:+ # Judged without tools, it would measure nothing.+ from openexecutive.evals import runner+ from openexecutive.orchestrator import executive as executive_module++ monkeypatch.setattr(executive_module, "Executive", _recording_executive([]))+ monkeypatch.setattr("openexecutive.orchestrator.mcp_gateway.get_active_gateway", lambda: None)+ queue: asyncio.Queue[dict[str, object] | None] = asyncio.Queue()+ run_one = runner._make_chat_runner(asyncio.Semaphore(1), queue, [0], 1, None, with_mcp=True)+ await run_one(0, {"id": "s0", "query": "q"})+ assert [(e["type"], e["error"]) for e in _drain(queue)] == [("scenario_error", runner.NO_MCP_GATEWAY)]+++async def test_each_mcp_scenario_reads_the_running_gateway_again(monkeypatch: pytest.MonkeyPatch) -> None:+ # A gateway the API replaces during a long run is the one the next scenario uses.+ from openexecutive.evals import runner+ from openexecutive.orchestrator import executive as executive_module++ async def judged(*_args: object, **_kwargs: object) -> dict[str, float]:+ return {"overall": 5.0}++ first, second = object(), object()+ active = [first]+ made: list[object] = []+ monkeypatch.setattr(executive_module, "Executive", _recording_executive(made))+ monkeypatch.setattr("openexecutive.orchestrator.mcp_gateway.get_active_gateway", lambda: active[0])+ monkeypatch.setattr(runner, "judge_chat", judged)+ queue: asyncio.Queue[dict[str, object] | None] = asyncio.Queue()+ run_one = runner._make_chat_runner(asyncio.Semaphore(1), queue, [0], 3, None, with_mcp=True)+ await run_one(0, {"id": "s0", "query": "q"})+ active[0] = second+ await run_one(1, {"id": "s1", "query": "q"})+ await run_one(2, {"id": "s2", "query": "q"})+ # Built for the first gateway, again when it was replaced, and reused after.+ assert made == [first, second]+ assert [e["type"] for e in _drain(queue)] == ["scenario_done"] * 3+++class _Gateway:+ """A fake MCPGateway: records start and close, and can fail to start."""++ def __init__(self, fail: BaseException | None = None) -> None:+ self.fail = fail+ self.started = False+ self.closed = False++ async def start(self, config_path: object) -> None:+ if self.fail is not None:+ raise self.fail+ self.started = True++ async def close(self) -> None:+ self.closed = True+++def _run_evals(monkeypatch: pytest.MonkeyPatch, tmp_path: Path, *, config: bool) -> ModuleType:+ """evals/run_evals.py, with the MCP config at a path that exists only when ``config``."""+ config_path = tmp_path / "mcp_servers.json"+ if config:+ config_path.write_text("{}")+ monkeypatch.setattr(+ "openexecutive.config.get_settings", lambda: SimpleNamespace(mcp_servers_config_path=config_path)+ )+ script = Path(__file__).resolve().parents[4] / "evals" / "run_evals.py"+ spec = importlib.util.spec_from_file_location("run_evals_under_test", script)+ assert spec is not None and spec.loader is not None+ module = importlib.util.module_from_spec(spec)+ spec.loader.exec_module(module)+ return module+++async def test_the_mcp_run_starts_the_gateway_makes_it_active_and_closes_it(+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path+) -> None:+ from openexecutive.orchestrator import mcp_gateway++ gateway = _Gateway()+ monkeypatch.setattr(mcp_gateway, "MCPGateway", lambda: gateway)+ run_evals = _run_evals(monkeypatch, tmp_path, config=True)+ async with run_evals._eval_gateway(True) as given:+ assert given is gateway and gateway.started+ assert mcp_gateway.get_active_gateway() is gateway+ assert gateway.closed+ assert mcp_gateway.get_active_gateway() is None+++async def test_a_failing_run_still_closes_the_gateway(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:+ from openexecutive.orchestrator import mcp_gateway++ gateway = _Gateway()+ monkeypatch.setattr(mcp_gateway, "MCPGateway", lambda: gateway)+ run_evals = _run_evals(monkeypatch, tmp_path, config=True)+ with pytest.raises(RuntimeError, match="scenario blew up"):+ async with run_evals._eval_gateway(True):+ raise RuntimeError("scenario blew up")+ assert gateway.closed+ assert mcp_gateway.get_active_gateway() is None++[email protected]("failure", [RuntimeError("server 3 failed"), TimeoutError()])+async def test_a_start_that_fails_part_way_closes_what_it_spawned(+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path, failure: BaseException+) -> None:+ # Servers spawned before the failure or the timeout would otherwise leak.+ from openexecutive.orchestrator import mcp_gateway++ gateway = _Gateway(fail=failure)+ monkeypatch.setattr(mcp_gateway, "MCPGateway", lambda: gateway)+ run_evals = _run_evals(monkeypatch, tmp_path, config=True)+ with pytest.raises(type(failure)):+ async with run_evals._eval_gateway(True):+ pytest.fail("the run must not start")+ assert gateway.closed+ assert mcp_gateway.get_active_gateway() is None+++async def test_an_mcp_run_with_no_config_exits_before_any_gateway(+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path+) -> None:+ from openexecutive.orchestrator import mcp_gateway++ made: list[_Gateway] = []++ def make() -> _Gateway:+ made.append(_Gateway())+ return made[-1]++ monkeypatch.setattr(mcp_gateway, "MCPGateway", make)+ run_evals = _run_evals(monkeypatch, tmp_path, config=False)+ with pytest.raises(SystemExit, match="--mcp needs an MCP config"):+ async with run_evals._eval_gateway(True):+ pytest.fail("the run must not start")+ assert made == []+++async def test_a_chat_run_gets_no_gateway(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:+ from openexecutive.orchestrator import mcp_gateway++ def make() -> None:+ pytest.fail("a chat run starts no gateway")++ monkeypatch.setattr(mcp_gateway, "MCPGateway", make)+ run_evals = _run_evals(monkeypatch, tmp_path, config=True)+ async with run_evals._eval_gateway(False) as given:+ assert given is Nonediff --git a/packages/core/tests/unit/test_principal_role.py b/packages/core/tests/unit/test_principal_role.pyindex 8feae9f3..308acf70 100644--- a/packages/core/tests/unit/test_principal_role.py+++ b/packages/core/tests/unit/test_principal_role.py@@ -924,7 +924,7 @@ def test_chat_runner_puts_the_role_on_the_session(monkeypatch: pytest.MonkeyPatcasync def fake_judge(_scenario: Any, _response: str) -> dict[str, Any]:return {"overall": 5}- monkeypatch.setattr(Executive, "__init__", lambda self: None)+ monkeypatch.setattr(Executive, "__init__", lambda self, mcp_gateway=None: None)monkeypatch.setattr(Executive, "chat", fake_chat)monkeypatch.setattr(runner, "judge_chat", fake_judge)queue: asyncio.Queue[Any] = asyncio.Queue()