Skip to content

AI-powered virtual executive team — a single coherent executive persona backed by 8 specialist agents (FastAPI + Next.js).

7.4 KB
Raw Blame
"""Do what it says (orchestrator.claim_check): a reply that claims a change no
tool call made gets one more round."""
from __future__ import annotations

import asyncio
from typing import Any
from unittest.mock import patch

import pytest

from openexecutive.orchestrator import claim_check
from openexecutive.orchestrator.executive import Executive
from tests.unit._agent_loop_fakes import FinalMsg, ScriptedProvider, TextBlock, ToolUseBlock


@pytest.fixture(autouse=True)
def _claim_check_on(monkeypatch: pytest.MonkeyPatch) -> None:
    monkeypatch.setenv("CLAIM_CHECK_ENABLED", "true")
    monkeypatch.setattr("openexecutive.orchestrator.executive.audit_log", lambda *a, **k: None)


def _run(provider: ScriptedProvider, message: str = "Remove it from the brief") -> list[Any]:
    async def _go() -> list[Any]:
        items: list[Any] = []
        with patch("openexecutive.orchestrator.executive.get_provider", return_value=provider):
            async for item in Executive()._stream_agent_loop(
                system_blocks=[],
                messages=[{"role": "user", "content": message}],
                model="claude-test",
            ):
                items.append(item)
        return items

    return asyncio.run(_go())


def _verdict(payload: dict[str, Any]):
    calls: list[str] = []

    async def fake(model: str, turn: str) -> dict[str, Any]:
        calls.append(turn)
        return payload

    return fake, calls


# --- the pure parts ----------------------------------------------------------


def test_word_check_skips_replies_that_claim_nothing() -> None:
    assert not claim_check.may_claim("Revenue was up 4% on last quarter.")
    assert not claim_check.may_claim("   ")
    assert claim_check.may_claim("Done.")
    assert claim_check.may_claim("I'll treat the draft as closed.")
    assert claim_check.may_claim("Removed it from the brief")


def test_claim_must_be_in_the_reply_word_for_word() -> None:
    reply = "Done — I’ll treat the draft as closed."
    assert claim_check.accept_claim({"unbacked": True, "claim": "I'll treat the draft as closed"}, reply)
    assert claim_check.accept_claim({"unbacked": True, "claim": "I sent the email"}, reply) is None
    assert claim_check.accept_claim({"unbacked": False, "claim": "Done"}, reply) is None
    assert claim_check.accept_claim({"unbacked": True, "claim": ""}, reply) is None
    assert claim_check.accept_claim({"unbacked": True, "claim": "x" * 300}, "x" * 300) is None


def test_last_person_message_skips_tool_result_turns() -> None:
    messages = [
        {"role": "user", "content": "I sent it"},
        {"role": "assistant", "content": [{"type": "tool_use", "id": "t", "name": "x", "input": {}}]},
        {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "t", "content": "ok"}]},
    ]
    assert claim_check.last_person_message(messages) == "I sent it"


def test_check_is_off_without_the_setting(monkeypatch: pytest.MonkeyPatch) -> None:
    monkeypatch.setenv("CLAIM_CHECK_ENABLED", "false")
    fake, calls = _verdict({"unbacked": True, "claim": "Done"})
    with patch.object(claim_check, "_call_model", fake):
        assert asyncio.run(claim_check.unbacked_claim("x", "Done", [])) is None
    assert calls == []


def test_a_failed_check_lets_the_reply_through() -> None:
    async def boom(model: str, turn: str) -> dict[str, Any]:
        raise RuntimeError("provider down")

    with patch.object(claim_check, "_call_model", boom):
        assert asyncio.run(claim_check.unbacked_claim("x", "Done", [])) is None


def test_the_check_sees_the_turns_calls() -> None:
    fake, calls = _verdict({"unbacked": False, "claim": ""})
    with patch.object(claim_check, "_call_model", fake):
        asyncio.run(claim_check.unbacked_claim(
            "close it", "Closed it.", [("close_open_loop", '{"status": "closed"}')],
        ))
    assert "close_open_loop" in calls[0]
    assert "<assistant_reply>\nClosed it." in calls[0]


# --- in the agent loop -------------------------------------------------------


def test_unbacked_claim_gets_one_more_round_with_the_nudge() -> None:
    provider = ScriptedProvider([
        FinalMsg([TextBlock("Done.")], "end_turn"),
        FinalMsg([TextBlock("I can't remove items from the brief yet.")], "end_turn"),
    ])
    fake, calls = _verdict({"unbacked": True, "claim": "Done"})
    with patch.object(claim_check, "_call_model", fake):
        items = _run(provider)

    assert len(provider.calls) == 2
    second = provider.calls[1]["messages"]
    assert second[-2] == {"role": "assistant", "content": [{"type": "text", "text": "Done."}]}
    nudge = second[-1]["content"][0]["text"]
    assert "“Done”" in nudge and "no tool call" in nudge
    # The break between the two replies comes from the next round's text
    # (test_reply_rounds.py); the retry itself streams nothing of its own.
    assert "\n\n" not in items
    # Checked once a turn: the second reply ends it without another check.
    assert len(calls) == 1


def test_backed_or_plain_reply_ends_the_turn() -> None:
    provider = ScriptedProvider([FinalMsg([TextBlock("Done.")], "end_turn")])
    fake, calls = _verdict({"unbacked": False, "claim": ""})
    with patch.object(claim_check, "_call_model", fake):
        _run(provider)
    assert len(provider.calls) == 1
    assert len(calls) == 1


def test_reply_without_claim_words_makes_no_check_call() -> None:
    provider = ScriptedProvider([FinalMsg([TextBlock("Revenue was up 4%.")], "end_turn")])
    fake, calls = _verdict({"unbacked": True, "claim": "Revenue"})
    with patch.object(claim_check, "_call_model", fake):
        _run(provider, "How did revenue do?")
    assert calls == []
    assert len(provider.calls) == 1


def test_tool_calls_this_turn_reach_the_check() -> None:
    provider = ScriptedProvider([
        FinalMsg([ToolUseBlock("t1", "list_open_loops", {})], "tool_use"),
        FinalMsg([TextBlock("Closed it.")], "end_turn"),
    ])
    fake, calls = _verdict({"unbacked": False, "claim": ""})

    async def handler(_input: dict[str, Any]) -> str:
        return '{"loops": []}'

    with (
        patch.object(claim_check, "_call_model", fake),
        patch.dict(
            "openexecutive.orchestrator.executive._ALL_SKILL_HANDLERS",
            {"list_open_loops": handler},
        ),
    ):
        _run(provider, "I sent it")
    assert len(calls) == 1
    assert "- list_open_loops:" in calls[0]


def test_the_turns_calls_reach_the_caller(monkeypatch: pytest.MonkeyPatch) -> None:
    """``calls_out`` hands the turn's calls to the conversation review's record."""
    monkeypatch.setenv("CLAIM_CHECK_ENABLED", "false")
    provider = ScriptedProvider([
        FinalMsg([ToolUseBlock("t1", "list_open_loops", {})], "tool_use"),
        FinalMsg([TextBlock("Nothing is open.")], "end_turn"),
    ])

    async def handler(_input: dict[str, Any]) -> str:
        return '{"loops": []}'

    calls: list[tuple[str, str]] = []

    async def _go() -> None:
        with patch("openexecutive.orchestrator.executive.get_provider", return_value=provider):
            async for _ in Executive()._stream_agent_loop(
                system_blocks=[], messages=[{"role": "user", "content": "anything open?"}],
                model="claude-test", calls_out=calls,
            ):
                pass

    with patch.dict("openexecutive.orchestrator.executive._ALL_SKILL_HANDLERS",
                    {"list_open_loops": handler}):
        asyncio.run(_go())
    assert calls == [("list_open_loops", '{"loops": []}')]