Skip to content

Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.

26.3 KB
Raw Blame
import argparse
import ipaddress
import json
import os
import socket
import subprocess
import sys
import threading
import time
from copy import deepcopy
from pathlib import Path

import httpx
import pytest
import uvicorn
from fastapi import FastAPI, Request
from fastapi.testclient import TestClient
from fastapi.responses import JSONResponse

sys.path.insert(0, str(Path(__file__).resolve().parents[1]))

from src import main


class FakeMessage:
    def model_dump(self):
        return {"role": "assistant", "content": "probe reply"}


class FakeChoice:
    message = FakeMessage()


class FakeCompletion:
    choices = [FakeChoice()]


def run_input(messages):
    return {
        "threadId": "thread-1",
        "runId": "run-1",
        "state": {},
        "messages": messages,
        "tools": [],
        "context": [],
        "forwardedProps": {},
    }


@pytest.mark.parametrize(
    ("provider", "model", "expected"),
    [
        (None, None, "openai/gpt-5.5"),
        ("", "", "openai/gpt-5.5"),
        ("   ", "gpt-4o", "openai/gpt-4o"),
        ("openai", "   ", "openai/gpt-5.5"),
        ("anthropic", "claude-3-5-sonnet-latest", "anthropic/claude-3-5-sonnet-latest"),
        ("custom-provider", "custom-model", "custom-provider/custom-model"),
        ("   ", "azure/gpt-4o", "openai/azure/gpt-4o"),
        ("anthropic", "openai/gpt-4o", "anthropic/openai/gpt-4o"),
        ("openai", "qwen/qwen3-8b", "openai/qwen/qwen3-8b"),
        ("", "openai/gpt-5.6-terra", "openai/openai/gpt-5.6-terra"),
    ],
)
def test_model_normalizes_blank_provider_and_model_before_defaults(
    monkeypatch, provider, model, expected
):
    if provider is None:
        monkeypatch.delenv("BOT_PROVIDER", raising=False)
    else:
        monkeypatch.setenv("BOT_PROVIDER", provider)
    if model is None:
        monkeypatch.delenv("BOT_MODEL", raising=False)
    else:
        monkeypatch.setenv("BOT_MODEL", model)

    assert main._model() == expected


def test_crewai_endpoint_preserves_leading_bot_role_for_provider(monkeypatch):
    monkeypatch.setenv("MANAGED_AGENT_TOKEN", "test-token")
    provider_messages = []
    provider_tools = []

    async def record_completion(*, model, messages, tools, stream):
        provider_messages.append(deepcopy(messages))
        provider_tools.append(deepcopy(tools))
        return FakeCompletion()

    monkeypatch.setattr(main, "acompletion", record_completion)

    client = TestClient(main.app)
    response = client.post(
        "/",
        headers={"x-openbot-agent-token": "test-token"},
        json=run_input(
            [
                {
                    "id": "system-1",
                    "role": "system",
                    "content": "You are Ada, a Bot-specific finance analyst.",
                },
                {
                    "id": "user-1",
                    "role": "user",
                    "content": "What should I review first?",
                },
            ]
        ),
    )

    assert response.status_code == 200
    assert provider_messages == [
        [
            {
                "role": "system",
                "content": "You are Ada, a Bot-specific finance analyst.",
            },
            {
                "role": "user",
                "content": "What should I review first?",
            },
        ]
    ]
    assert provider_tools == [None]
    snapshots = [
        event["messages"]
        for event in agui_events(response.text)
        if event.get("type") == "MESSAGES_SNAPSHOT"
    ]
    assert snapshots
    assert snapshots[-1][:2] == [
        {
            "id": "system-1",
            "role": "system",
            "content": "You are Ada, a Bot-specific finance analyst.",
        },
        {
            "id": "user-1",
            "role": "user",
            "content": "What should I review first?",
        },
    ]


def test_provider_message_projection_keeps_supported_fields_without_mutating_state():
    original_messages = [
        {
            "id": "system-1",
            "role": "system",
            "content": "System instructions",
            "name": "system_name",
            "metadata": {"transport": "ag-ui"},
            "encrypted_value": "system-secret",
        },
        {
            "id": "user-1",
            "role": "user",
            "content": [
                {"type": "text", "text": "What is in this image?"},
                {"type": "image_url", "image_url": {"url": "data:image/png;base64,AAAA"}},
            ],
            "name": "user_name",
            "subagent_run_id": "run-user",
        },
        {
            "id": "assistant-1",
            "role": "assistant",
            "content": None,
            "name": "assistant_name",
            "tool_calls": [
                {
                    "id": "call-1",
                    "type": "function",
                    "function": {"name": "lookup", "arguments": "{\"q\":\"x\"}"},
                    "metadata": {"ui": True},
                }
            ],
            "metadata": {"transport": "ag-ui"},
        },
        {
            "id": "tool-1",
            "role": "tool",
            "content": "Tool result",
            "tool_call_id": "call-1",
            "error": None,
            "metadata": {"transport": "ag-ui"},
        },
    ]
    state_messages = deepcopy(original_messages)

    projected = main._provider_messages(state_messages)

    assert projected == [
        {
            "role": "system",
            "content": "System instructions",
            "name": "system_name",
        },
        {
            "role": "user",
            "content": [
                {"type": "text", "text": "What is in this image?"},
                {"type": "image_url", "image_url": {"url": "data:image/png;base64,AAAA"}},
            ],
            "name": "user_name",
        },
        {
            "role": "assistant",
            "name": "assistant_name",
            "tool_calls": [
                {
                    "id": "call-1",
                    "type": "function",
                    "function": {"name": "lookup", "arguments": "{\"q\":\"x\"}"},
                }
            ],
        },
        {
            "role": "tool",
            "content": "Tool result",
            "tool_call_id": "call-1",
        },
    ]
    assert state_messages == original_messages


def isolated_environment(directory):
    directory.mkdir(parents=True, exist_ok=True)
    inherited = (
        "HOME", "USERPROFILE", "HOMEDRIVE", "HOMEPATH", "CODEX_HOME",
        "PATH", "LANG", "SYSTEMROOT", "WINDIR",
    )
    environment = {name: os.environ[name] for name in inherited if name in os.environ}
    environment.update(
        MANAGED_AGENT_TOKEN="synthetic-openbot-token",
        OPENAI_API_KEY="sk-synthetic-openbot-key",
        OTEL_SDK_DISABLED="true",
        CREWAI_DISABLE_TELEMETRY="true",
        CREWAI_DISABLE_TRACKING="true",
        CREWAI_TELEMETRY_DISABLED="true",
        CREWAI_STORAGE_DIR=str(directory / "crewai"),
        LITELLM_LOCAL_MODEL_COST_MAP="True",
        PYTHONPYCACHEPREFIX=str(directory / "pycache"),
        TMPDIR=str(directory),
    )
    return environment


def prohibit_external_connections(event, arguments):
    if event == "socket.getaddrinfo":
        host = arguments[0]
    elif event in ("socket.connect", "socket.sendto"):
        address = arguments[1]
        if not isinstance(address, tuple):
            raise RuntimeError("Only loopback TCP/IP is allowed in this proof")
        host = address[0]
    else:
        return
    if host != "localhost" and not ipaddress.ip_address(host).is_loopback:
        raise RuntimeError("External network access is prohibited in this proof")


def free_loopback_socket():
    listener = socket.socket()
    listener.bind(("127.0.0.1", 0))
    return listener


def start_loopback_app(app):
    listener = free_loopback_socket()
    port = listener.getsockname()[1]
    server = uvicorn.Server(uvicorn.Config(app, log_level="error", lifespan="off"))
    thread = threading.Thread(target=server.run, kwargs={"sockets": [listener]})
    thread.start()
    deadline = time.monotonic() + 10
    while not server.started and thread.is_alive() and time.monotonic() < deadline:
        time.sleep(0.01)
    assert server.started, "loopback server did not start"
    return server, thread, f"http://127.0.0.1:{port}"


def stop_loopback_server(server, thread):
    server.should_exit = True
    thread.join(timeout=10)
    assert not thread.is_alive(), "loopback server did not stop"


def loopback_openai_receiver(records, strict_messages=False):
    app = FastAPI()

    @app.post("/chat/completions")
    async def chat_completions(request: Request):
        body = await request.json()
        records.append(
            {
                "model": body.get("model"),
                "messages": body.get("messages"),
                "tools": body.get("tools"),
                "authorization": request.headers.get("authorization"),
            }
        )
        if not body.get("model"):
            return JSONResponse({"error": {"message": "empty model rejected"}}, status_code=400)
        if strict_messages:
            allowed = {
                "system": {"role", "content", "name"},
                "user": {"role", "content", "name"},
                "assistant": {
                    "role",
                    "audio",
                    "content",
                    "function_call",
                    "name",
                    "refusal",
                    "tool_calls",
                },
                "tool": {"role", "content", "tool_call_id"},
            }
            for index, message in enumerate(body.get("messages") or []):
                role = message.get("role")
                extra = sorted(set(message) - allowed.get(role, set()))
                if extra:
                    return JSONResponse(
                        {
                            "error": {
                                "message": f"message {index} role {role} had unsupported fields: {extra}"
                            }
                        },
                        status_code=400,
                    )
                for tool_call in message.get("tool_calls") or []:
                    extra_tool_call = sorted(set(tool_call) - {"id", "type", "function"})
                    if extra_tool_call:
                        return JSONResponse(
                            {
                                "error": {
                                    "message": (
                                        f"message {index} tool call had unsupported fields: "
                                        f"{extra_tool_call}"
                                    )
                                }
                            },
                            status_code=400,
                        )
        if body.get("tools") == [show_note_provider_tool()] and not any(
            message.get("role") == "tool" for message in body.get("messages") or []
        ):
            return {
                "id": "chatcmpl-openbot-loopback-tool-call",
                "object": "chat.completion",
                "created": 1,
                "model": body["model"],
                "choices": [
                    {
                        "index": 0,
                        "message": {
                            "role": "assistant",
                            "content": None,
                            "tool_calls": [
                                {
                                    "id": "call_show_note_1",
                                    "type": "function",
                                    "function": {
                                        "name": "show_note",
                                        "arguments": "{\"title\":\"Quarterly plan\"}",
                                    },
                                }
                            ],
                        },
                        "finish_reason": "tool_calls",
                    }
                ],
                "usage": {
                    "prompt_tokens": 1,
                    "completion_tokens": 1,
                    "total_tokens": 2,
                },
            }
        if any(message.get("role") == "tool" for message in body.get("messages") or []):
            return {
                "id": "chatcmpl-openbot-loopback-tool-result",
                "object": "chat.completion",
                "created": 1,
                "model": body["model"],
                "choices": [
                    {
                        "index": 0,
                        "message": {
                            "role": "assistant",
                            "content": "Saved note Quarterly plan.",
                        },
                        "finish_reason": "stop",
                    }
                ],
                "usage": {
                    "prompt_tokens": 1,
                    "completion_tokens": 1,
                    "total_tokens": 2,
                },
            }
        return {
            "id": "chatcmpl-openbot-loopback",
            "object": "chat.completion",
            "created": 1,
            "model": body["model"],
            "choices": [
                {
                    "index": 0,
                    "message": {
                        "role": "assistant",
                        "content": "loopback response",
                    },
                    "finish_reason": "stop",
                }
            ],
            "usage": {
                "prompt_tokens": 1,
                "completion_tokens": 1,
                "total_tokens": 2,
            },
        }

    return app


def show_note_agui_tool():
    return {
        "name": "show_note",
        "description": "Show a note title to the caller.",
        "parameters": {
            "type": "object",
            "properties": {
                "title": {
                    "type": "string",
                    "description": "Title to show.",
                }
            },
            "required": ["title"],
            "additionalProperties": False,
        },
    }


def show_note_provider_tool():
    return {
        "type": "function",
        "function": show_note_agui_tool(),
    }


def agui_events(response_text):
    events = []
    for line in response_text.splitlines():
        if line.startswith("data: "):
            events.append(json.loads(line.removeprefix("data: ")))
    return events


def run_litellm_loopback_proof(proof_case, output):
    sys.addaudithook(prohibit_external_connections)
    provider, model, expected_provider_model, expected_receiver_model = {
        "blank-provider": ("   ", "gpt-4o", "openai/gpt-4o", "gpt-4o"),
        "blank-model": ("openai", "   ", "openai/gpt-5.5", "gpt-5.5"),
        "strict-projection": ("openai", "gpt-4o", "openai/gpt-4o", "gpt-4o"),
        "caller-tool": ("openai", "gpt-4o", "openai/gpt-4o", "gpt-4o"),
        "namespaced-model": (
            "openai",
            "qwen/qwen3-8b",
            "openai/qwen/qwen3-8b",
            "qwen/qwen3-8b",
        ),
        "gateway-model": (
            "openai",
            "openai/gpt-5.6-terra",
            "openai/openai/gpt-5.6-terra",
            "openai/gpt-5.6-terra",
        ),
    }[proof_case]
    os.environ["BOT_PROVIDER"] = provider
    os.environ["BOT_MODEL"] = model

    records = []
    receiver, receiver_thread, base_url = start_loopback_app(
        loopback_openai_receiver(records, strict_messages=proof_case == "strict-projection")
    )
    os.environ["OPENAI_BASE_URL"] = base_url
    os.environ["OPENAI_API_BASE"] = base_url
    harness, harness_thread, harness_url = start_loopback_app(main.app)
    try:
        with httpx.Client(base_url=harness_url, timeout=20, trust_env=False) as client:
            response = client.post(
                "/",
                headers={main.TOKEN_HEADER: "synthetic-openbot-token"},
                json={
                    **run_input(
                        strict_projection_messages()
                        if proof_case == "strict-projection"
                        else basic_messages()
                    ),
                    "tools": [show_note_agui_tool()] if proof_case == "caller-tool" else [],
                },
            )
            if proof_case == "caller-tool":
                first_events = agui_events(response.text)
                first_snapshot = [
                    event.get("messages")
                    for event in first_events
                    if event.get("type") == "MESSAGES_SNAPSHOT"
                ][-1]
                tool_call = first_snapshot[-1]["toolCalls"][0]
                second_response = client.post(
                    "/",
                    headers={main.TOKEN_HEADER: "synthetic-openbot-token"},
                    json={
                        **run_input(
                            [
                                *basic_messages(),
                                {
                                    "id": first_snapshot[-1]["id"],
                                    "role": "assistant",
                                    "content": None,
                                    "tool_calls": [
                                        {
                                            "id": tool_call["id"],
                                            "type": "function",
                                            "function": {
                                                "name": tool_call["function"]["name"],
                                                "arguments": tool_call["function"]["arguments"],
                                            },
                                        }
                                    ],
                                },
                                {
                                    "id": "tool-result-1",
                                    "role": "tool",
                                    "content": "{\"ok\":true}",
                                    "tool_call_id": tool_call["id"],
                                },
                            ]
                        ),
                        "runId": "run-2",
                        "tools": [show_note_agui_tool()],
                    },
                )
            else:
                first_events = []
                first_snapshot = []
                tool_call = None
                second_response = None
        events = agui_events(response.text)
        second_events = agui_events(second_response.text) if second_response else []
        result = {
            "proofCase": proof_case,
            "statusCode": response.status_code,
            "secondStatusCode": second_response.status_code if second_response else None,
            "normalizedModel": main._model(),
            "receiverRecords": records,
            "eventTypes": [event.get("type") for event in events],
            "secondEventTypes": [event.get("type") for event in second_events],
            "decodedToolCall": tool_call,
            "messagesSnapshots": [
                event.get("messages")
                for event in events
                if event.get("type") == "MESSAGES_SNAPSHOT"
            ],
            "runFinished": any(event.get("type") == "RUN_FINISHED" for event in events),
            "runError": any(event.get("type") == "RUN_ERROR" for event in events),
            "teardown": "pending",
        }
    finally:
        stop_loopback_server(harness, harness_thread)
        stop_loopback_server(receiver, receiver_thread)

    result["teardown"] = "stopped"
    output.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n")
    assert result["statusCode"] == 200, result
    assert result["normalizedModel"] == expected_provider_model, result
    assert result["runFinished"], result
    assert not result["runError"], result
    assert len(records) == (2 if proof_case == "caller-tool" else 1), result
    assert all(record["model"] == expected_receiver_model for record in records), result
    assert records[0]["messages"][0] == {
        "role": "system",
        "content": "You are Ada, a Bot-specific finance analyst.",
    }, result
    if proof_case == "caller-tool":
        assert records[0]["tools"] == [show_note_provider_tool()], result
        assert result["decodedToolCall"] == {
            "id": "call_show_note_1",
            "type": "function",
            "function": {
                "name": "show_note",
                "arguments": "{\"title\":\"Quarterly plan\"}",
            },
        }, result
        assert result["secondStatusCode"] == 200, result
        assert "RUN_FINISHED" in result["secondEventTypes"], result
        assert records[1]["tools"] == [show_note_provider_tool()], result
        assert records[1]["messages"][-2:] == [
            {
                "role": "assistant",
                "tool_calls": [
                    {
                        "id": "call_show_note_1",
                        "type": "function",
                        "function": {
                            "name": "show_note",
                            "arguments": "{\"title\":\"Quarterly plan\"}",
                        },
                    }
                ],
            },
            {
                "role": "tool",
                "content": "{\"ok\":true}",
                "tool_call_id": "call_show_note_1",
            },
        ], result
        assert result["messagesSnapshots"][-1][-1]["toolCalls"][0]["id"] == "call_show_note_1"
    elif proof_case == "strict-projection":
        assert_provider_messages_are_projected(records[0]["messages"])
        assert records[0]["messages"][1] == {
            "role": "user",
            "content": [
                {"type": "text", "text": "What should I review first?"},
                {"type": "image_url", "image_url": {"url": "data:image/png;base64,AAAA"}},
            ],
            "name": "analyst",
        }, result
        snapshot = result["messagesSnapshots"][-1]
        assert [message["id"] for message in snapshot[:4]] == [
            "system-1",
            "user-1",
            "assistant-1",
            "tool-1",
        ], result
        assert snapshot[2]["toolCalls"][0]["id"] == "call-1", result
        assert snapshot[3]["toolCallId"] == "call-1", result
    else:
        assert records[0]["messages"][1] == {
            "role": "user",
            "content": "What should I review first?",
        }, result


def basic_messages():
    return [
        {
            "id": "system-1",
            "role": "system",
            "content": "You are Ada, a Bot-specific finance analyst.",
        },
        {
            "id": "user-1",
            "role": "user",
            "content": "What should I review first?",
        },
    ]


def strict_projection_messages():
    return [
        basic_messages()[0],
        {
            "id": "user-1",
            "role": "user",
            "content": [
                {"type": "text", "text": "What should I review first?"},
                {
                    "type": "image",
                    "source": {
                        "type": "url",
                        "value": "data:image/png;base64,AAAA",
                        "mime_type": "image/png",
                    },
                },
            ],
            "name": "analyst",
            "metadata": {"client": "ag-ui"},
            "subagent_run_id": "run-user",
        },
        {
            "id": "assistant-1",
            "role": "assistant",
            "content": None,
            "tool_calls": [
                {
                    "id": "call-1",
                    "type": "function",
                    "function": {"name": "lookup", "arguments": "{\"symbol\":\"ACME\"}"},
                    "metadata": {"client": "ag-ui"},
                }
            ],
            "metadata": {"client": "ag-ui"},
        },
        {
            "id": "tool-1",
            "role": "tool",
            "content": "Synthetic result",
            "tool_call_id": "call-1",
            "metadata": {"client": "ag-ui"},
            "error": None,
        },
    ]


def assert_provider_messages_are_projected(messages):
    allowed = {
        "system": {"role", "content", "name"},
        "user": {"role", "content", "name"},
        "assistant": {
            "role",
            "audio",
            "content",
            "function_call",
            "name",
            "refusal",
            "tool_calls",
        },
        "tool": {"role", "content", "tool_call_id"},
    }
    for message in messages:
        assert set(message) <= allowed[message["role"]]
        assert "id" not in message
        assert "metadata" not in message
        for tool_call in message.get("tool_calls") or []:
            assert set(tool_call) <= {"id", "type", "function"}
            assert tool_call["id"] == "call-1"


@pytest.mark.parametrize(
    "proof_case",
    ["blank-provider", "blank-model", "strict-projection", "namespaced-model", "gateway-model"],
)
def test_crewai_endpoint_uses_normalized_model_with_real_litellm_loopback(
    tmp_path, proof_case
):
    output = tmp_path / f"{proof_case}.json"
    completed = subprocess.run(
        [
            sys.executable,
            str(Path(__file__).resolve()),
            "--proof-case", proof_case,
            "--output", str(output),
        ],
        cwd=tmp_path,
        env=isolated_environment(tmp_path),
        text=True,
        capture_output=True,
        timeout=90,
    )
    assert completed.returncode == 0, completed.stdout + completed.stderr


def test_crewai_endpoint_forwards_caller_tools_and_accepts_tool_result_continuation(
    tmp_path,
):
    output = tmp_path / "caller-tool.json"
    completed = subprocess.run(
        [
            sys.executable,
            str(Path(__file__).resolve()),
            "--proof-case",
            "caller-tool",
            "--output",
            str(output),
        ],
        cwd=tmp_path,
        env=isolated_environment(tmp_path),
        text=True,
        capture_output=True,
        timeout=90,
    )
    assert completed.returncode == 0, completed.stdout + completed.stderr


if __name__ == "__main__":
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument(
        "--proof-case",
        choices=[
            "blank-provider",
            "blank-model",
            "strict-projection",
            "caller-tool",
            "namespaced-model",
            "gateway-model",
        ],
        required=True,
    )
    parser.add_argument("--output", type=Path, required=True)
    arguments = parser.parse_args()
    run_litellm_loopback_proof(arguments.proof_case, arguments.output)