AI-powered virtual executive team — a single coherent executive persona backed by 8 specialist agents (FastAPI + Next.js).
"""The less common tools sit behind open_tools / use_tool (orchestrator.tool_groups).
The tool list is the start of the cached prefix, so it must be the same on
every call of every turn: a group the model opens comes back as a tool
result, never as a change to the list. A use_tool call is rewritten into the
tool's own call before dispatch, so it runs the same handler, guards and
audit rows as a direct call would.
Also pinned here: the specialist lookups take their result counts from
settings, and a zero synced count skips the synced-source queries.
"""
from __future__ import annotations
import asyncio
import json
from types import SimpleNamespace
from typing import Any
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from openexecutive.orchestrator import tool_groups
from tests.unit.offered_tools import capture_offered
@pytest.fixture(autouse=True)
def _no_audit_writes(monkeypatch: pytest.MonkeyPatch) -> list[tuple[Any, ...]]:
rows: list[tuple[Any, ...]] = []
monkeypatch.setattr(
"openexecutive.orchestrator.executive.audit_log",
lambda *a, **_k: rows.append(a),
)
return rows
# --------------------------------------------------------------------- #
# The groups
# --------------------------------------------------------------------- #
def test_every_deferred_name_is_a_real_tool_with_a_handler() -> None:
from openexecutive.orchestrator.executive import _ALL_SKILL_HANDLERS, _ALL_SKILL_TOOLS
names = {t["name"] for t in _ALL_SKILL_TOOLS}
assert set(tool_groups.DEFERRED) <= names
assert set(tool_groups.DEFERRED) <= set(_ALL_SKILL_HANDLERS)
def test_each_tool_sits_in_one_group() -> None:
listed = [t for _, tools in tool_groups.GROUPS.values() for t in tools]
assert len(listed) == len(set(listed))
@pytest.mark.parametrize(
"name", ["consult_specialist", "propose_form_values", "message_person", "read_user_guide"]
)
def test_the_everyday_tools_stay_direct(name: str) -> None:
assert name not in tool_groups.DEFERRED
@pytest.mark.parametrize(
("name", "group"),
[("ask_about_person", "people"), ("forget_fact", "memory"), ("update_company_profile", "memory")],
)
def test_the_rarely_used_tools_sit_in_a_group(name: str, group: str) -> None:
assert tool_groups.DEFERRED[name] == group
def test_open_tools_lists_every_group() -> None:
desc = tool_groups.open_tools_tool(tool_groups.DEFERRED)["description"]
for group, (_, tools) in tool_groups.GROUPS.items():
assert group in desc
assert all(t in desc for t in tools)
def test_open_tools_names_only_the_offered_tools() -> None:
tool = tool_groups.open_tools_tool(["create_goal", "send_slack_dm"])
assert "create_goal" in tool["description"] and "send_slack_dm" in tool["description"]
assert "send_company_broadcast" not in tool["description"]
assert "calendar" not in tool["description"]
assert tool["input_schema"]["properties"]["group"]["enum"] == ["goals", "messaging"]
# The same offer gives the same definition (one cached list per kind of turn).
assert tool == tool_groups.open_tools_tool(["send_slack_dm", "create_goal"])
def test_split_keeps_the_direct_list_the_same_whatever_is_opened() -> None:
tools = [{"name": n, "input_schema": {}} for n in ("consult_specialist", "create_goal", "list_watchlist")]
direct, deferred = tool_groups.split(tools)
assert [t["name"] for t in direct] == ["consult_specialist", "open_tools", "use_tool"]
assert set(deferred) == {"create_goal", "list_watchlist"}
# Nothing deferred on the turn: no open_tools / use_tool either.
direct, deferred = tool_groups.split(tools[:1])
assert [t["name"] for t in direct] == ["consult_specialist"] and deferred == {}
def test_open_result_gives_only_the_offered_tools() -> None:
deferred = {"create_goal": {"name": "create_goal", "description": "d", "input_schema": {"type": "object"}}}
out = json.loads(tool_groups.open_result({"group": "goals"}, deferred))
assert out["call_with"] == "use_tool"
assert [t["name"] for t in out["tools"]] == ["create_goal"]
assert "error" in json.loads(tool_groups.open_result({"group": "calendar"}, deferred))
assert "error" in json.loads(tool_groups.open_result({"group": "nope"}, deferred))
assert "error" in json.loads(tool_groups.open_result("goals", deferred))
def test_unwrap() -> None:
call, err = tool_groups.unwrap(
{"id": "tu-1", "name": "use_tool", "input": {"name": "create_goal", "input": {"title": "t"}}}
)
assert err is None and call == {"id": "tu-1", "name": "create_goal", "input": {"title": "t"}}
# JSON text for the input is accepted.
call, _ = tool_groups.unwrap({"id": "tu-2", "name": "use_tool",
"input": {"name": "create_goal", "input": '{"title": "t"}'}})
assert call is not None and call["input"] == {"title": "t"}
# A direct tool, an unknown name, or a missing input object is refused.
for bad in (
{"name": "consult_specialist", "input": {}},
{"name": "use_tool", "input": {}},
{"name": "create_goal", "input": "not json"},
{"name": "create_goal"},
"create_goal",
):
call, err = tool_groups.unwrap({"id": "tu-3", "name": "use_tool", "input": bad})
assert call is None and err and "error" in json.loads(err)
# --------------------------------------------------------------------- #
# In the loop
# --------------------------------------------------------------------- #
class _Provider:
def __init__(self, finals: list[Any]) -> None:
self._finals = list(finals)
self.calls: list[dict[str, Any]] = []
def messages_stream(self, **kwargs: Any) -> Any:
import copy
self.calls.append(copy.deepcopy(kwargs))
final = self._finals.pop(0)
class _Stream:
async def __aenter__(self) -> _Stream:
return self
async def __aexit__(self, *_a: Any) -> None:
return None
def __aiter__(self) -> _Stream:
return self
async def __anext__(self) -> Any:
raise StopAsyncIteration
async def get_final_message(self) -> Any:
return final
return _Stream()
def _use(id_: str, name: str, input_: Any) -> Any:
return SimpleNamespace(type="tool_use", id=id_, name=name, input=input_)
def _run(rounds: list[list[Any]]) -> tuple[_Provider, list[Any], list[list[str]]]:
from openexecutive.orchestrator.executive import Executive
from openexecutive.orchestrator.schedule_tools import set_session
from openexecutive.orchestrator.session import Session
finals = [SimpleNamespace(content=r, stop_reason="tool_use", usage=None) for r in rounds]
finals.append(SimpleNamespace(content=[SimpleNamespace(type="text", text="ok")],
stop_reason="end_turn", usage=None))
provider = _Provider(finals)
items: list[Any] = []
async def _go() -> list[list[str]]:
with (
capture_offered() as offered,
patch("openexecutive.orchestrator.executive.get_provider", return_value=provider),
set_session(Session(session_id="web:groups")),
):
async for item in Executive()._stream_agent_loop(
system_blocks=[], messages=[{"role": "user", "content": "x"}],
model="claude-test", workspace_mode="team", principal_role_tag="",
turn_id="t-groups",
):
items.append(item)
return offered
offered = asyncio.run(_go())
return provider, items, offered
def _results(provider: _Provider, call: int) -> dict[str, str]:
return {b["tool_use_id"]: b["content"] for b in provider.calls[call]["messages"][-1]["content"]}
def test_the_request_carries_open_tools_and_no_deferred_tool() -> None:
provider, _, offered = _run([])
names = [t["name"] for t in provider.calls[0]["tools"] if "input_schema" in t]
assert {"open_tools", "use_tool"} <= set(names)
assert not set(names) & set(tool_groups.DEFERRED)
assert names == sorted(names)
# ...while the turn still offers them behind open_tools.
assert {"create_goal", "list_watchlist"} <= set(offered[0])
def test_opening_a_group_returns_its_tools_and_keeps_the_list_unchanged() -> None:
provider, _, _ = _run([[_use("tu-1", "open_tools", {"group": "goals"})]])
out = json.loads(_results(provider, 1)["tu-1"])
assert out["group"] == "goals"
names = {t["name"] for t in out["tools"]}
assert names == set(tool_groups.GROUPS["goals"][1])
assert all(t["input_schema"] for t in out["tools"])
# The cached prefix is byte-for-byte the same on the next call.
assert provider.calls[1]["tools"] == provider.calls[0]["tools"]
def test_use_tool_runs_the_tools_own_handler(_no_audit_writes: list[tuple[Any, ...]]) -> None:
from openexecutive.orchestrator import executive as executive_module
handler = AsyncMock(return_value=json.dumps({"status": "ok"}))
with patch.dict(executive_module._ALL_SKILL_HANDLERS, {"list_watchlist": handler}):
provider, items, _ = _run(
[[_use("tu-1", "use_tool", {"name": "list_watchlist", "input": {"limit": 5}})]]
)
handler.assert_awaited_once_with({"limit": 5})
assert json.loads(_results(provider, 1)["tu-1"]) == {"status": "ok"}
# The assistant turn keeps the model's own use_tool block, so the result
# pairs with the id it sent.
assistant = provider.calls[1]["messages"][-2]["content"]
assert any(b.get("name") == "use_tool" and b.get("id") == "tu-1" for b in assistant)
# The audit row and progress event name the tool itself.
assert any("list_watchlist" in str(row[1]) for row in _no_audit_writes if len(row) > 1)
assert "use_tool" not in json.dumps([i for i in items if isinstance(i, dict)], default=str)
def test_use_tool_on_a_direct_tool_or_without_input_is_an_error() -> None:
provider, _, _ = _run([[
_use("tu-1", "use_tool", {"name": "consult_specialist", "input": {}}),
_use("tu-2", "use_tool", {"name": "create_goal"}),
]])
results = _results(provider, 1)
assert "error" in json.loads(results["tu-1"])
assert "error" in json.loads(results["tu-2"])
# --------------------------------------------------------------------- #
# Specialist lookups
# --------------------------------------------------------------------- #
def test_specialist_retrieval_uses_the_settings_counts(monkeypatch: pytest.MonkeyPatch) -> None:
from openexecutive.orchestrator import router
monkeypatch.setattr(
"openexecutive.config.get_settings",
lambda: SimpleNamespace(specialist_builtin_n_results=4, specialist_company_n_results=1),
)
seen: dict[str, Any] = {}
def fake_retrieve(**kw: Any) -> str:
seen.update(kw)
return "ctx"
monkeypatch.setattr("openexecutive.knowledge.retriever.retrieve", fake_retrieve)
out = asyncio.run(router._retrieve_for_call({"query": "q", "specialist": "cfo"}))
assert out == "ctx"
assert (seen["n_builtin"], seen["n_company"], seen["n_synced"]) == (4, 1, 1)
def test_the_specialist_defaults_are_small() -> None:
from openexecutive.config import Settings
fields = Settings.model_fields
assert fields["specialist_builtin_n_results"].default == 3
assert fields["specialist_company_n_results"].default == 2
def test_no_synced_count_skips_the_synced_sources(monkeypatch: pytest.MonkeyPatch) -> None:
import openexecutive.knowledge.retriever as retriever_mod
from openexecutive.knowledge.store import ChromaDBStore
monkeypatch.setattr(retriever_mod, "_audit_log", lambda *a, **k: None)
review_store = SimpleNamespace(
get_withheld_keys=lambda _ct: set(),
get_withheld_source_ids=lambda: set(),
get_priority_map=lambda _ct: {},
list_annotations=lambda domains=None, active_only=True: [],
)
store = MagicMock()
store.query.return_value = []
def collections() -> set[str]:
return {c.kwargs["collection"] for c in store.query.call_args_list}
synced = {
ChromaDBStore.NOTION_COLLECTION,
ChromaDBStore.DRIVE_COLLECTION,
ChromaDBStore.ONEDRIVE_COLLECTION,
ChromaDBStore.CONFLUENCE_COLLECTION,
}
retriever_mod.retrieve(query="How should we price the renewal?", store=store,
review_store=review_store, n_builtin=1, n_company=1, n_synced=0)
assert not collections() & synced
store.query.reset_mock()
retriever_mod.retrieve(query="How should we price the renewal?", store=store,
review_store=review_store, n_builtin=1, n_company=1, n_synced=2)
assert collections() >= synced
assert all(c.kwargs["n_results"] == 2 for c in store.query.call_args_list
if c.kwargs["collection"] in synced)