Skip to content

AI-powered virtual executive team — a single coherent executive persona backed by 8 specialist agents (FastAPI + Next.js).

13.0 KB
Raw Blame
"""Act as me in the eval runner: a ``delegation`` block runs the turn against
an in-memory mailbox, and the drafts it saves reach the judge."""
from __future__ import annotations

import asyncio
from pathlib import Path
from typing import Any

import pytest

from openexecutive.delegation.gmail import DraftSpec
from openexecutive.delegation.settings import DelegationOverride
from openexecutive.evals import judges as judges_module
from openexecutive.evals import runner as runner_module
from openexecutive.evals.scenarios import (
    load_scenarios,
    scenario_delegation,
    validate_scenario_yaml,
)

_BLOCK = """
delegation:
  person: {full_name: Olivia Owner, email: [email protected]}
  thread:
    id: t-pilot
    subject: Brand refresh pilot
    messages:
      - {from: "Dana Prospect <[email protected]>", text: "Can we start Oct 5?", reply_to: [email protected]}
      - {from: "[email protected]", text: "Checking my calendar."}
"""


def test_a_block_becomes_a_fresh_mailbox_each_time() -> None:
    scenario = validate_scenario_yaml("id: s1\nquery: q\n" + _BLOCK)
    first, second = scenario_delegation(scenario), scenario_delegation(scenario)
    assert isinstance(first, DelegationOverride) and first.enabled
    assert first.gmail is not second.gmail
    assert first.person.email == "[email protected]" and first.person.is_principal
    thread = first.gmail.threads["t-pilot"]
    assert [m.labels for m in thread.messages] == [["INBOX"], ["SENT"]]
    assert thread.messages[0].reply_to == "[email protected]"
    assert scenario_delegation({"id": "s2"}) is None


@pytest.mark.parametrize("block", [
    "delegation: yes\n",
    "delegation:\n  person: {full_name: X}\n",
    "delegation:\n  person: {full_name: X, email: [email protected]}\n  thread: {id: '../x', messages: []}\n",
    "delegation:\n  person: {full_name: X, email: [email protected]}\n  thread: {id: t1, messages: [{text: hi}]}\n",
])
def test_a_malformed_block_fails_the_scenario(block: str) -> None:
    with pytest.raises(ValueError, match="delegation"):
        validate_scenario_yaml("id: s1\nquery: q\n" + block)


def test_the_judge_sees_the_drafts_only_for_these_scenarios() -> None:
    section = judges_module._delegation_section(
        {"delegation": {}, "quality_criteria": {"never_claims_the_email_was_sent": True}},
        [{"to": ["[email protected]"], "cc": [], "subject": "Re: Pilot", "body": "Yes to Oct 5."}],
    )
    assert "ACT AS ME" in section and "Yes to Oct 5." in section and "To: [email protected]" in section
    assert "never claims the email was sent" in section
    assert "(none)" in judges_module._delegation_section({"delegation": {}}, [])
    assert judges_module._delegation_section({"id": "x"}, None) == ""


def test_the_runner_runs_the_turn_with_the_mailbox_and_judges_its_drafts(
    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
    (tmp_path / "d.yaml").write_text("id: d\ndomain: delegation\ndescription: d\nquery: reply as me\n" + _BLOCK)
    monkeypatch.setenv("EVAL_SCENARIOS_PATH", str(tmp_path))
    judged: dict[str, Any] = {}

    async def fake_chat(self: Any, **kwargs: Any) -> str:
        override = kwargs["session"].delegation_override
        assert isinstance(override, DelegationOverride)
        await override.gmail.create_draft(DraftSpec(to=["[email protected]"], subject="Re: x", body="Yes."))
        return "Your draft is waiting in your Gmail Drafts."

    async def fake_judge(scenario: dict[str, Any], response: str, drafts: Any = None) -> dict[str, Any]:
        judged["drafts"] = drafts
        return {"overall": 5}

    from openexecutive.orchestrator.executive import Executive

    monkeypatch.setattr(Executive, "chat", fake_chat)
    monkeypatch.setattr(runner_module, "judge_chat", fake_judge)

    async def drive() -> None:
        queue: asyncio.Queue[dict[str, Any] | None] = asyncio.Queue()
        run_one = runner_module._make_run_one(
            kind="chat", store=None, sem=asyncio.Semaphore(1), queue=queue,
            passed=[0], total=1, cancel_event=None,
        )
        scenario = next(s for s in load_scenarios(kind="chat") if s["id"] == "d")
        await run_one(0, scenario)

    asyncio.run(drive())
    assert judged["drafts"][0]["body"] == "Yes." and judged["drafts"][0]["to"] == ["[email protected]"]


def test_the_shipped_scenarios_load() -> None:
    ids = {s["id"] for s in load_scenarios(kind="chat")}
    assert {"delegation_001", "delegation_002", "delegation_003"} <= ids


# ── The inbox watcher's scenarios (type: inbox) ─────────────────────────────

_INBOX = """
id: i1
type: inbox
description: d
inbox:
  person: {full_name: Olivia Owner, email: [email protected]}
  relation: contact
  expect: draft
  thread:
    id: t-kickoff
    subject: Kickoff
    messages:
      - {from: "Dana Park <[email protected]>", text: "Next week?", cc: [[email protected]]}
      - {from: "[email protected]", text: "Tuesday works."}
      - {from: "Dana Park <[email protected]>", text: "2pm or 4pm?", verified: false}
"""


def test_the_inbox_scenarios_are_their_own_kind() -> None:
    from openexecutive.evals.scenarios import scenario_inbox, scenario_kind

    scenario = validate_scenario_yaml(_INBOX)
    assert scenario_kind(scenario) == "inbox"
    case = scenario_inbox(scenario)
    # The message answered is the newest one from someone else.
    assert case.message.text == "2pm or 4pm?" and case.message.sender_authenticated is False
    assert case.thread.messages[0].sender_authenticated is True
    assert case.thread.messages[0].cc == ["[email protected]"]
    assert [m.labels for m in case.thread.messages] == [["INBOX"], ["SENT"], ["INBOX"]]
    assert case.person.email == "[email protected]" and case.person.is_principal
    assert case.relation == "contact" and case.expect_draft is True
    shipped = {s["id"] for s in load_scenarios(kind="inbox")}
    assert {f"delegation_inbox_00{i}" for i in range(1, 8)} <= shipped
    assert not shipped & {s["id"] for s in load_scenarios(kind="chat")}


@pytest.mark.parametrize(("change", "error"), [
    ("  relation: contact\n", "  relation: boss\n"),
    ("  expect: draft\n", "  expect: maybe\n"),
    ("  person: {full_name: Olivia Owner, email: [email protected]}\n", "  person: {full_name: X}\n"),
])
def test_a_malformed_inbox_block_fails_the_scenario(change: str, error: str) -> None:
    with pytest.raises(ValueError, match="inbox"):
        validate_scenario_yaml(_INBOX.replace(change, error))


def test_an_inbox_scenario_needs_someone_elses_message() -> None:
    only_hers = _INBOX.split("    messages:\n")[0] + (
        "    messages:\n      - {from: \"[email protected]\", text: \"Tuesday works.\"}\n"
    )
    with pytest.raises(ValueError, match="someone else"):
        validate_scenario_yaml(only_hers)
    with pytest.raises(ValueError, match="inbox"):
        validate_scenario_yaml("id: i2\ntype: inbox\n")


def _reply(**kw: Any) -> Any:
    from openexecutive.delegation.inbox import Reply

    base: dict[str, Any] = {
        "to": ["[email protected]"], "subject": "Re: Kickoff", "body": "Tuesday still works.",
        "open_questions": ["2pm or 4pm?"], "flags": [], "in_reply_to": None, "references": None,
    }
    return Reply(**{**base, **kw})


@pytest.mark.parametrize(("expect", "wants", "reply", "judged", "overall", "passed"), [
    ("draft", True, "reply", True, 4, True),
    ("draft", False, None, False, 1, False),
    ("no_draft", True, "reply", False, 1, False),
    ("no_draft", False, None, False, 5, True),
    # A draft was wanted but composing failed: that's a failure either way.
    ("draft", True, "compose_failed", False, 1, False),
    ("no_draft", True, "compose_failed", False, 1, False),
    # No verdict is a failed call, never a pass for "no draft".
    ("no_draft", None, None, False, 0, False),
])
def test_the_inbox_runner_checks_the_verdict_before_judging(
    monkeypatch: pytest.MonkeyPatch, expect: str, wants: bool | None, reply: str | None, judged: bool,
    overall: int, passed: bool,
) -> None:
    from openexecutive.delegation import inbox
    from openexecutive.delegation.inbox_classifier import Verdict

    seen: dict[str, Any] = {}
    verdict = None if wants is None else (
        Verdict(True, "scheduling", 0.9) if wants else Verdict(False, "thanks", 0.9)
    )

    async def fake_reply_for(person: Any, message: Any, thread: Any, *, relation: str, own: set[str]) -> Any:
        seen["args"] = (person.email, message.text, relation, own)
        return verdict, _reply() if reply == "reply" else reply

    async def fake_judge(scenario: dict[str, Any], outcome: dict[str, Any]) -> dict[str, Any]:
        seen["outcome"] = outcome
        return {"overall": 4}

    monkeypatch.setattr(inbox, "reply_for", fake_reply_for)
    monkeypatch.setattr(runner_module, "judge_inbox", fake_judge)
    scenario = validate_scenario_yaml(_INBOX.replace("expect: draft", f"expect: {expect}"))
    result = asyncio.run(runner_module.run_inbox_scenario(scenario))
    assert seen["args"] == ("[email protected]", "2pm or 4pm?", "contact", {"[email protected]"})
    assert ("outcome" in seen) is judged
    assert result["scores"]["overall"] == overall and result["passed"] is passed
    # Its newest message isn't verified: handled as a stranger, and said so.
    assert result["outcome"]["handled_as"] == "stranger"
    assert result["outcome"]["drafted"] is (reply == "reply")
    if reply == "reply":
        assert result["outcome"]["reply"]["open_questions"] == ["2pm or 4pm?"]
    assert result["outcome"]["no_reply_because"] == (reply if reply not in ("reply", None) else None)


def test_the_inbox_runner_scores_a_group_email_as_the_scan_decides(monkeypatch: pytest.MonkeyPatch) -> None:
    from openexecutive.delegation import inbox
    from openexecutive.delegation.inbox_classifier import Verdict

    async def fake_reply_for(*a: Any, **kw: Any) -> Any:
        # Wants a reply, but not from this person: the scan drafts nothing.
        return Verdict(True, "question", 0.95, asked_of_them=False), None

    monkeypatch.setattr(inbox, "reply_for", fake_reply_for)
    group = _INBOX.replace(
        'text: "2pm or 4pm?", verified: false}',
        'text: "2pm or 4pm?", verified: false, to: [[email protected], [email protected]]}',
    ).replace("expect: draft", "expect: no_draft")
    result = asyncio.run(runner_module.run_inbox_scenario(validate_scenario_yaml(group)))
    assert result["passed"] is True and result["scores"]["notes"] == "drafted nothing, as expected"


def test_the_inbox_kind_streams_through_the_suite_runner(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
    from openexecutive.delegation import inbox
    from openexecutive.delegation.inbox_classifier import Verdict

    (tmp_path / "i.yaml").write_text(_INBOX)
    monkeypatch.setenv("EVAL_SCENARIOS_PATH", str(tmp_path))

    async def fake_reply_for(*a: Any, **kw: Any) -> Any:
        return Verdict(True, "scheduling", 0.9), _reply()

    async def fake_judge(scenario: dict[str, Any], outcome: dict[str, Any]) -> dict[str, Any]:
        return {"overall": 5}

    monkeypatch.setattr(inbox, "reply_for", fake_reply_for)
    monkeypatch.setattr(runner_module, "judge_inbox", fake_judge)

    async def drive() -> list[dict[str, Any]]:
        queue: asyncio.Queue[dict[str, Any] | None] = asyncio.Queue()
        passed = [0]
        run_one = runner_module._make_run_one(
            kind="inbox", store=None, sem=asyncio.Semaphore(1), queue=queue,
            passed=passed, total=1, cancel_event=None,
        )
        scenario = next(s for s in load_scenarios(kind="inbox") if s["id"] == "i1")
        await run_one(0, scenario)
        assert passed == [1]
        events = []
        while not queue.empty():
            event = await queue.get()
            assert event is not None
            events.append(event)
        return events

    events = asyncio.run(drive())
    assert [e["type"] for e in events] == ["scenario_start", "scenario_done"]
    assert events[1]["outcome"]["reply"]["body"] == "Tuesday still works."


def test_the_inbox_judge_sees_the_thread_the_draft_and_its_questions(monkeypatch: pytest.MonkeyPatch) -> None:
    from dataclasses import asdict
    from types import SimpleNamespace

    prompts: list[str] = []

    class Provider:
        async def messages_create(self, **kw: Any) -> Any:
            prompts.append(kw["messages"][0]["content"])
            return SimpleNamespace(content=[SimpleNamespace(text='{"overall": 4, "notes": "fine"}')])

    monkeypatch.setattr(judges_module, "get_provider", lambda model: Provider())
    scenario = validate_scenario_yaml(_INBOX)
    outcome = {"verdict": {"kind": "scheduling"}, "reply": asdict(_reply(flags=["others_on_thread"]))}
    assert asyncio.run(judges_module.judge_inbox(scenario, outcome)) == {"overall": 4, "notes": "fine"}
    prompt = prompts[0]
    assert "2pm or 4pm?" in prompt and "Tuesday works." in prompt
    assert "To: [email protected]" in prompt and "Tuesday still works." in prompt
    assert '["2pm or 4pm?"]' in prompt and "others_on_thread" in prompt
    assert "THE SENDER IS: contact" in prompt