Skip to content

AI-powered virtual executive team — a single coherent executive persona backed by 8 specialist agents (FastAPI + Next.js).

4.2 KB
Raw Blame
"""A chat scenario's routing expectation is scored, not just listed.

``required_routing`` names the specialists the Executive must consult and
``no_routing: true`` says it must consult none. The chat runner reads which
specialists the turn consulted off a ``DebugCollector`` and fails the
scenario when the expectation is not met, whatever the judge's score.
"""
from __future__ import annotations

import asyncio
from pathlib import Path
from typing import Any

import pytest

from openexecutive.evals import runner as runner_module
from openexecutive.evals.scenarios import (
    load_scenarios,
    routing_check,
    scenario_routing,
    validate_scenario_yaml,
)
from openexecutive.orchestrator.router import SPECIALIST_REGISTRY


def test_routing_check_needs_every_required_specialist() -> None:
    s = {"required_routing": ["cfo", "cso"]}
    assert routing_check(s, ["cso", "cfo", "cfo"]) == {
        "consulted": ["cfo", "cso"], "missing": [], "ok": True,
    }
    assert routing_check(s, ["cfo", "gc"])["missing"] == ["cso"]
    assert routing_check(s, ["cfo", "gc"])["ok"] is False


def test_no_routing_fails_on_any_consult() -> None:
    s = {"no_routing": True}
    assert routing_check(s, [])["ok"] is True
    assert routing_check(s, ["cmo"])["ok"] is False


def test_no_expectation_returns_none() -> None:
    assert routing_check({}, ["cfo"]) is None


@pytest.mark.parametrize(
    "body",
    [
        "required_routing: cfo\n",
        "required_routing: [cfo]\nno_routing: true\n",
        "no_routing: yes please\n",
    ],
)
def test_malformed_routing_is_rejected(body: str) -> None:
    with pytest.raises(ValueError):
        validate_scenario_yaml(f"id: x\ndomain: finance\nquery: q\n{body}")


def test_builtin_scenarios_name_real_specialists() -> None:
    for s in load_scenarios():
        required, _ = scenario_routing(s)
        assert set(required) <= set(SPECIALIST_REGISTRY), s["id"]


def test_runner_fails_a_scenario_whose_routing_is_not_met(
    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
    (tmp_path / "needs_cfo.yaml").write_text(
        "id: needs_cfo\ndomain: finance\nquery: runway?\nrequired_routing: [cfo]\n"
    )
    (tmp_path / "skips.yaml").write_text(
        "id: skips\ndomain: routing\nquery: thank-you note\nno_routing: true\n"
    )
    monkeypatch.setenv("EVAL_SCENARIOS_PATH", str(tmp_path))

    async def fake_chat(self: Any, **kwargs: Any) -> str:
        # Both turns consult the CFO: right for one, wrong for the other.
        kwargs["debug_collector"].emit("specialist_start", {"specialist": "cfo"})
        return "a response"

    async def fake_judge(scenario: dict[str, Any], response: str) -> dict[str, Any]:
        return {"overall": 5}

    from openexecutive.orchestrator.executive import Executive

    monkeypatch.setattr(Executive, "chat", fake_chat)
    monkeypatch.setattr(runner_module, "judge_chat", fake_judge)

    async def drive() -> dict[str, dict[str, Any]]:
        queue: asyncio.Queue[dict[str, Any] | None] = asyncio.Queue()
        run_one = runner_module._make_run_one(
            kind="chat", store=None, sem=asyncio.Semaphore(2), queue=queue,
            passed=[0], total=2, cancel_event=None,
        )
        scenarios = [s for s in load_scenarios(kind="chat") if s["id"] in ("needs_cfo", "skips")]
        for i, s in enumerate(scenarios):
            await run_one(i, s)
        done: dict[str, dict[str, Any]] = {}
        while not queue.empty():
            ev = await queue.get()
            if ev is not None and ev["type"] == "scenario_done":
                done[ev["scenario_id"]] = ev
        return done

    done = asyncio.run(drive())
    assert done["needs_cfo"]["passed"] is True
    assert done["needs_cfo"]["scores"]["routing"]["consulted"] == ["cfo"]
    assert done["skips"]["passed"] is False
    assert done["skips"]["scores"]["routing"]["ok"] is False


def test_both_personas_say_when_to_consult() -> None:
    from openexecutive.prompts.executive_persona import (
        EXECUTIVE_PERSONA_PROMPT,
        EXECUTIVE_PERSONA_SOLO_PROMPT,
    )

    for prompt in (EXECUTIVE_PERSONA_PROMPT, EXECUTIVE_PERSONA_SOLO_PROMPT):
        assert "call `consult_specialist` for that area before you answer" in prompt
        assert "Skip it for lookups" in prompt