Skip to content

Enterprise agent platform for building, deploying, and operating AI teammates.

30.5 KB
Raw Blame
"""Real HTTP web host and standalone workers with only the LLM boundary replaced."""

from __future__ import annotations

import ast
import asyncio
import fcntl
import json
import multiprocessing
import os
import signal
import socket
import time
from collections.abc import Iterator
from contextlib import contextmanager
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any

import httpx
import pytest

from tests.e2e.app_harness import build_access_token

# A provider fault the hosts' model boundary injects: ``{"kind": "unavailable"
# | "invalid_output", "remaining": int | null}``. ``remaining`` counts provider
# attempts (each tier-1 retry is one); ``null`` keeps the fault until the file
# is removed. Counting attempts rather than time makes the moment the provider
# "recovers" deterministic.
MODEL_FAULT_FILE = "model-fault.json"


def _take_model_fault(root: Path, tool_names: set[str]) -> str | None:
    """Consume one attempt of the current fault, if it applies to this call."""
    try:
        handle = (root / MODEL_FAULT_FILE).open("r+")
    except FileNotFoundError:
        return None
    with handle:
        fcntl.flock(handle, fcntl.LOCK_EX)
        spec = json.load(handle)
        kind = spec["kind"]
        # Unusable output only matters where the model must answer.
        if kind == "invalid_output" and "final_answer" not in tool_names:
            return None
        remaining = spec.get("remaining")
        if remaining is not None:
            if remaining <= 0:
                return None
            spec["remaining"] = remaining - 1
            handle.seek(0)
            handle.truncate()
            json.dump(spec, handle)
        return kind


def _install_model_boundary(root: Path, role: str) -> None:
    import openai

    from tests.e2e.scripted_llm import ScriptedLLM
    from xagent.web.api import chat
    from xagent.web.services import agent_service_manager

    class RecordingLLM(ScriptedLLM):
        def __init__(self):
            super().__init__([])

        async def chat(self, messages, **kwargs):
            tool_names = {t["function"]["name"] for t in kwargs.get("tools", []) or []}
            fault = _take_model_fault(root, tool_names)
            with (root / f"model-{os.getpid()}.jsonl").open("a") as log:
                log.write(
                    json.dumps(
                        {
                            "role": role,
                            "messages": messages,
                            "tools": [
                                t["function"]["name"]
                                for t in kwargs.get("tools", []) or []
                            ],
                            "fault": fault,
                        }
                    )
                    + "\n"
                )
            if fault == "unavailable":
                # A real provider 5xx; the text must not read as a capacity,
                # quota or credential refusal, which are classified apart.
                request = httpx.Request(
                    "POST", "http://scripted-provider.invalid/v1/chat/completions"
                )
                raise openai.InternalServerError(
                    "Service Unavailable",
                    response=httpx.Response(503, request=request),
                    body=None,
                )
            if fault == "invalid_output":
                return {
                    "tool_calls": [
                        {
                            "id": "e2e-empty-final",
                            "function": {
                                "name": "final_answer",
                                "arguments": json.dumps(
                                    {"response_language": "English", "answer": ""}
                                ),
                            },
                        }
                    ]
                }
            text = json.dumps(messages)
            if "generate_execution_plan" in tool_names:
                return {
                    "tool_calls": [
                        {
                            "id": "e2e-plan",
                            "function": {
                                "name": "generate_execution_plan",
                                "arguments": json.dumps(
                                    {
                                        "response_language": "English",
                                        "steps": [
                                            {
                                                "id": "answer",
                                                "task": "Answer the question in English",
                                                "dependencies": [],
                                                "description": "Produce the answer",
                                                "termination_condition": "The answer is ready",
                                                "completion_evidence": "A final answer",
                                                "tool_names": [],
                                            }
                                        ],
                                    }
                                ),
                            },
                        }
                    ]
                }
            if "assess_dag_completion" in tool_names:
                return {
                    "tool_calls": [
                        {
                            "id": "e2e-assess",
                            "function": {
                                "name": "assess_dag_completion",
                                "arguments": json.dumps(
                                    {
                                        "status": "completed",
                                        "reason": "The answer is ready",
                                        "answer": "Shared E2E answer",
                                        "missing_work": "",
                                        "replan_instruction": "",
                                    }
                                ),
                            },
                        }
                    ]
                }
            if any(
                t["function"]["name"] == "select_execution_pattern"
                for t in kwargs.get("tools", []) or []
            ):
                return {
                    "tool_calls": [
                        {
                            "id": "e2e-route",
                            "function": {
                                "name": "select_execution_pattern",
                                "arguments": json.dumps(
                                    {
                                        "action": "final_answer"
                                        if "e2e:route-final" in text
                                        else "plan_execute"
                                        if "e2e:route-dag" in text
                                        else "react",
                                        "reason": "Exercise the agent runtime",
                                        "answer": "Shared E2E answer"
                                        if "e2e:route-final" in text
                                        else "",
                                        "requires_current_or_external_facts": False,
                                        "existing_context_sufficient": True,
                                        "evidence_basis": "Test question",
                                        "missing_verification": "",
                                    }
                                ),
                            },
                        }
                    ]
                }
            delegated = [
                t["function"]["name"]
                for t in kwargs.get("tools", []) or []
                if t["function"]["name"].startswith("worker_")
            ]
            if (
                "e2e:delegate" in text
                and delegated
                and not any(m.get("role") == "tool" for m in messages)
            ):
                return {
                    "tool_calls": [
                        {
                            "id": "e2e-delegation",
                            "function": {
                                "name": delegated[0],
                                "arguments": json.dumps(
                                    {"task": "Answer the delegated question"}
                                ),
                            },
                        }
                    ]
                }
            results = {
                m.get("tool_call_id"): m.get("content")
                for m in messages
                if m.get("role") == "tool"
            }
            if "e2e:files" in text and "e2e:ask" not in text:
                if "e2e-read" not in results:
                    return {
                        "tool_calls": [
                            {
                                "id": "e2e-read",
                                "function": {
                                    "name": "read_file",
                                    "arguments": json.dumps(
                                        {"file_path": "source.txt"}
                                    ),
                                },
                            }
                        ]
                    }
                if "e2e-write" not in results:
                    return {
                        "tool_calls": [
                            {
                                "id": "e2e-write",
                                "function": {
                                    "name": "write_file",
                                    "arguments": json.dumps(
                                        {
                                            "file_path": "derived.txt",
                                            "content": results["e2e-read"]
                                            .removeprefix("Tool read_file returned: ")
                                            .upper(),
                                        }
                                    ),
                                },
                            }
                        ]
                    }
                written = ast.literal_eval(
                    results["e2e-write"].removeprefix("Tool write_file returned: ")
                )
                return "Shared E2E answer: " + written["markdown_link"]
            if "e2e:gate" in text:
                (root / "model-entered").touch()
                while not (root / "model-release").exists():
                    await asyncio.sleep(0.02)
            if "e2e:ask" in text and "e2e:answer" not in text:
                return {
                    "tool_calls": [
                        {
                            "id": "e2e-question",
                            "function": {
                                "name": "ask_user_question",
                                "arguments": json.dumps(
                                    {"message": "Which choice?", "interactions": []}
                                ),
                            },
                        }
                    ]
                }
            if any(
                t["function"]["name"] == "final_answer"
                for t in kwargs.get("tools", []) or []
            ):
                return {
                    "tool_calls": [
                        {
                            "id": "e2e-final",
                            "function": {
                                "name": "final_answer",
                                "arguments": json.dumps(
                                    {
                                        "response_language": "English",
                                        "answer": "Shared E2E answer",
                                    }
                                ),
                            },
                        }
                    ]
                }
            return "Shared E2E answer"

    def boundary_llm():
        # Test-only knob: put the production tier-1 retry layer in front of
        # the scripted provider, so a fault is retried as a real one would be.
        llm = RecordingLLM()
        attempts = os.environ.get("E2E_MODEL_RETRY_ATTEMPTS")
        if not attempts:
            return llm
        from xagent.core.model.chat.basic.adapter import attach_chat_retry_wrapper

        return attach_chat_retry_wrapper(llm, max_retries=int(attempts))

    agent_service_manager.create_default_llm = boundary_llm
    chat.resolve_llms_from_names = lambda *a, **kw: (boundary_llm(), None, None, None)


def _host(pipe, environment: dict[str, str], role: str, root: str) -> None:
    os.environ.update(environment)
    os.environ.pop("PYTEST_CURRENT_TEST", None)
    os.environ["XAGENT_TASK_EXECUTION_ROLE"] = role
    root_path = Path(root)
    # Each host's diagnostics remain available when a bounded wait fails.
    log = (root_path / f"{role}-{os.getpid()}.log").open("w")
    os.dup2(log.fileno(), 1)
    os.dup2(log.fileno(), 2)
    _install_model_boundary(root_path, role)
    if (root_path / "event-read-acceptance").exists():
        import sqlalchemy as sa

        from tests.shared.execution_event_read_guard import reject_legacy_content

        sa.event.listen(
            sa.engine.Engine, "before_cursor_execute", reject_legacy_content
        )
    if (root_path / "gmail-public-key.pem").exists():
        from tests.e2e.test_shared_gmail import install_gmail_network_boundary

        install_gmail_network_boundary(root_path)
    from xagent.web import sandbox_manager

    sandbox_manager.get_sandbox_manager = lambda: None
    if role in {"web", "combined"}:
        import uvicorn

        from xagent.web.app import app

        with socket.socket() as listener:
            listener.bind(("127.0.0.1", 0))
            port = listener.getsockname()[1]
            listener.listen()
            server = uvicorn.Server(
                uvicorn.Config(app, log_level="warning", timeout_graceful_shutdown=20)
            )

            async def serve():
                serving = asyncio.create_task(server.serve(sockets=[listener]))
                while not server.started:
                    if serving.done():
                        await serving
                        raise RuntimeError("Web startup failed")
                    await asyncio.sleep(0.02)
                pipe.send({"port": port})
                await asyncio.to_thread(pipe.recv)
                server.should_exit = True
                await serving

            asyncio.run(serve())
    else:
        from xagent.web import worker
        from xagent.web.services.task_event_bridge import get_task_event_bridge

        async def consume():
            running = asyncio.create_task(worker._main())
            while True:
                if running.done():
                    await running
                    raise RuntimeError("Worker startup failed")
                try:
                    if get_task_event_bridge().ready.is_set():
                        break
                except ConnectionError:
                    pass
                await asyncio.sleep(0.02)
            pipe.send({"ready": True})
            await asyncio.to_thread(pipe.recv)
            os.kill(os.getpid(), signal.SIGTERM)
            await running

        asyncio.run(consume())


@dataclass
class SharedExecutionApp:
    root: Path
    environment: dict[str, str]
    token: str
    user_id: int
    processes: list[Any] = field(default_factory=list)
    pipes: list[Any] = field(default_factory=list)
    client: httpx.Client | None = None

    @property
    def headers(self):
        return {"Authorization": f"Bearer {self.token}"}

    def start(self, role: str):
        return self.start_all(role)[0]

    def start_all(self, *roles: str):
        """Spawn hosts together, then wait; each host pays a cold import."""
        context = multiprocessing.get_context("spawn")
        launched = []
        for role in roles:
            parent, child = context.Pipe()
            process = context.Process(
                target=_host, args=(child, self.environment, role, str(self.root))
            )
            process.start()
            child.close()
            self.processes.append(process)
            self.pipes.append(parent)
            launched.append((role, process, parent))
        for role, _, parent in launched:
            assert parent.poll(60), self.diagnostics()
            try:
                ready = parent.recv()
            except EOFError:
                pytest.fail(self.diagnostics())
            if role in {"web", "combined"}:
                if self.client is not None:
                    self.client.close()
                self.client = httpx.Client(
                    base_url=f"http://127.0.0.1:{ready['port']}", timeout=30
                )
        return [(process, parent) for _, process, parent in launched]

    def diagnostics(self):
        return "\n".join(
            f"{p.name}:\n{p.read_text()[-12000:]}" for p in self.root.glob("*.log")
        )

    def wait_task(
        self,
        task_id: int,
        *,
        status: str = "completed",
        run_id: str | None = None,
        timeout: float = 30,
    ):
        from xagent.web.models.database import get_session_local
        from xagent.web.models.task import Task

        deadline = time.monotonic() + timeout
        state = None
        while time.monotonic() < deadline:
            with get_session_local()() as db:
                task = db.get(Task, task_id)
                state = {
                    "status": task.status.value,
                    "run_id": task.run_id,
                    "runner_id": task.runner_id,
                    "output": task.output,
                    "error": task.error_message,
                }
            if (
                state["status"] == status
                and state["runner_id"] is None
                and (run_id is None or state["run_id"] == run_id)
            ):
                return state
            if state["status"] == "failed" and status != "failed":
                pytest.fail(f"Task failed: {state}\n{self.diagnostics()}")
            time.sleep(0.03)
        pytest.fail(f"Task did not reach {status}: {state}\n{self.diagnostics()}")

    def create_agent(self, name="Shared E2E agent", *, tool_categories=None):
        assert self.client is not None
        response = self.client.post(
            "/api/agents",
            headers=self.headers,
            json={
                "name": name,
                "instructions": "Answer the request.",
                "execution_mode": "balanced",
                "tool_categories": tool_categories or [],
            },
        )
        assert response.status_code == 200, response.text
        agent_id = response.json()["id"]
        response = self.client.post(
            f"/api/agents/{agent_id}/api-key", headers=self.headers
        )
        assert response.status_code == 200, response.text
        return agent_id, {"Authorization": f"Bearer {response.json()['full_key']}"}

    def close(self):
        if self.client is not None:
            self.client.close()
        for pipe in reversed(self.pipes):
            try:
                pipe.send("stop")
            except (BrokenPipeError, EOFError, OSError):
                pass
        failures = []
        for process in reversed(self.processes):
            process.join(30)
            if process.is_alive():
                failures.append(f"Host {process.pid} did not shut down")
                process.terminate()
                process.join(10)
            if process.exitcode != 0:
                failures.append(f"Host {process.pid} exited {process.exitcode}")
        for pipe in self.pipes:
            pipe.close()
        assert not failures, "\n".join(failures) + "\n" + self.diagnostics()


@contextmanager
def open_shared_app(
    tmp_path: Path,
    monkeypatch,
    *,
    database_url: str,
    host_database_url: str | None = None,
    event_read_acceptance: bool = False,
) -> Iterator[SharedExecutionApp]:
    """A web host and a worker on ``database_url``, closed on exit.

    The hosts reach the database through ``host_database_url`` when given;
    this process always uses ``database_url`` directly.
    """
    from xagent.web.models.database import get_engine, get_session_local, init_db
    from xagent.web.models.user import User

    root = tmp_path / "hosts"
    root.mkdir()
    if event_read_acceptance:
        (root / "event-read-acceptance").touch()
    monkeypatch.setenv("DATABASE_URL", database_url)
    monkeypatch.setenv("XAGENT_UPLOADS_DIR", str(tmp_path / "uploads"))
    monkeypatch.setenv("LANGFUSE_TRACING_ENABLED", "false")
    monkeypatch.setenv("XAGENT_TASK_LEASE_TTL_SECONDS", "10")
    monkeypatch.setenv("XAGENT_TASK_LEASE_HEARTBEAT_SECONDS", "2")
    monkeypatch.setenv("XAGENT_TASK_LEASE_RECOVERY_INTERVAL_SECONDS", "1")
    monkeypatch.setenv("XAGENT_TRIGGER_DISPATCHER_INTERVAL_SECONDS", "1")
    monkeypatch.setenv("XAGENT_TRIGGER_DISPATCHER_STARTUP_JITTER_SECONDS", "0")
    monkeypatch.setenv("XAGENT_FILE_STORAGE_STARTUP_SYNC_ENABLED", "false")
    init_db()
    try:
        with get_session_local()() as db:
            user = User(username="e2e-owner", password_hash="unused", is_admin=True)
            db.add(user)
            db.commit()
            user_id = user.id
        environment = dict(os.environ)
        if event_read_acceptance:
            # The production selector, not a test hook, must choose V2.
            from xagent.config import EXECUTION_EVENT_STORAGE_ENTRIES
            from xagent.web.services.conversation_storage_rollout import (
                STORAGE_ENTRIES,
            )

            environment[EXECUTION_EVENT_STORAGE_ENTRIES] = ",".join(
                sorted(STORAGE_ENTRIES)
            )
        if host_database_url is not None:
            environment["DATABASE_URL"] = host_database_url
        app = SharedExecutionApp(
            root,
            environment,
            build_access_token(username="e2e-owner", user_id=user_id),
            user_id,
        )
        try:
            app.start_all("web", "worker")
            yield app
        finally:
            app.close()
    finally:
        get_engine().dispose()


@pytest.fixture
def shared_app(tmp_path, monkeypatch, request):
    with open_shared_app(
        tmp_path,
        monkeypatch,
        database_url=f"sqlite:///{tmp_path / 'shared.db'}",
        event_read_acceptance=getattr(request, "param", 1) == 2,
    ) as app:
        yield app


def receive_event(ws, event_type):
    events = []
    deadline = time.monotonic() + 30
    for _ in range(200):
        remaining = deadline - time.monotonic()
        assert remaining > 0, events
        try:
            event = json.loads(ws.recv(timeout=remaining))
        except TimeoutError:
            pytest.fail(f"Missing {event_type}: {events}")
        events.append(event)
        if event["type"] == event_type or event.get("event_type") == event_type:
            return event, events
        assert event["type"] != "error", event
    pytest.fail(f"Missing {event_type}: {events}")


# ---------------------------------------------------------------- crash helpers

# Lease recovery takes the lease TTL (10s) plus a few polls; leave room for a
# cold worker start.
SETTLE_TIMEOUT = 60


def restart_hosts(app, *roles: str, **environment: str) -> None:
    """Replace every host, with ``environment`` applied to the new ones."""

    app.close()
    app.processes.clear()
    app.pipes.clear()
    app.client = None
    app.environment.update(environment)
    app.start_all(*roles)


def wait_for(predicate, app, what: str, timeout: float = SETTLE_TIMEOUT) -> Any:
    deadline = time.monotonic() + timeout
    while time.monotonic() < deadline:
        value = predicate()
        if value:
            return value
        time.sleep(0.05)
    pytest.fail(f"Timed out waiting for {what}\n{app.diagnostics()}")


def kill_host(app, process) -> None:
    """SIGKILL one host and forget it."""
    process.kill()
    process.join(10)
    assert process.exitcode == -signal.SIGKILL
    index = app.processes.index(process)
    app.processes.pop(index)
    app.pipes.pop(index).close()


def model_calls(app) -> list[dict[str, Any]]:
    return [
        json.loads(line)
        for path in app.root.glob("model-*.jsonl")
        for line in path.read_text().splitlines()
    ]


def task_state(task_id: int) -> dict[str, Any]:
    from xagent.web.models.database import get_session_local
    from xagent.web.models.task import Task

    with get_session_local()() as db:
        task = db.get(Task, task_id)
        return {
            "status": task.status.value,
            "control_state": task.control_state,
            "run_id": task.run_id,
            "runner_id": task.runner_id,
            "state_version": task.state_version,
        }


def recovery_state(task_id: int) -> tuple[dict[str, Any] | None, list[str]]:
    """The task's recovery row and its event names, in order."""
    import sqlalchemy as sa

    from xagent.web.models.database import get_session_local
    from xagent.web.models.task_auto_recovery import (
        TaskAutoRecovery,
        TaskRecoveryEvent,
    )

    with get_session_local()() as db:
        row = db.get(TaskAutoRecovery, task_id)
        events = [
            event.event
            for event in db.scalars(
                sa.select(TaskRecoveryEvent)
                .where(TaskRecoveryEvent.task_id == task_id)
                .order_by(TaskRecoveryEvent.id)
            )
        ]
        if row is None:
            return None, events
        return {
            "state": row.state,
            "state_detail": row.state_detail,
            "reason": row.reason,
            "run_id": row.run_id,
            "total_resumes": row.total_resumes,
            "no_progress_resumes": row.no_progress_resumes,
        }, events


def auto_resume_commands(task_id: int) -> list[str]:
    from xagent.web.models.database import get_session_local
    from xagent.web.models.task_command import TaskExecutionCommand

    with get_session_local()() as db:
        return [
            command.command_id
            for command in db.query(TaskExecutionCommand)
            .filter(TaskExecutionCommand.task_id == task_id)
            .order_by(TaskExecutionCommand.id)
            if command.command_id.startswith("auto-resume:")
        ]


def ws_url(app, task_id: int) -> str:
    return (
        str(app.client.base_url).replace("http://", "ws://").rstrip("/")
        + f"/ws/chat/{task_id}"
    )


def create_task(app, title: str, description: str) -> int:
    """An owner's internal balanced task, not yet executed."""
    response = app.client.post(
        "/api/chat/task/create",
        headers=app.headers,
        json={
            "title": title,
            "description": description,
            "execution_mode": "balanced",
        },
    )
    assert response.status_code == 200, response.text
    return response.json()["task_id"]


def create_sdk_task(app, content: str) -> tuple[int, str, dict[str, str]]:
    """An agent's SDK task: its id, run id and the caller's headers."""
    agent_id, headers = app.create_agent()
    response = app.client.post(
        "/v1/chat/tasks",
        headers=headers,
        json={"agent_id": agent_id, "message": {"role": "user", "content": content}},
    )
    assert response.status_code == 202, response.text
    accepted = response.json()
    return accepted["task_id"], accepted["run_id"], headers


def start_task(app, description: str = "e2e:gate") -> int:
    """An owner's internal balanced task, executed over the WebSocket.

    A gated description (``e2e:gate``) makes the first model call wait for
    ``model-release``; this then returns once that call has started.
    """
    from websockets.sync.client import connect

    task_id = create_task(app, "Auto resume", description)
    with connect(ws_url(app, task_id), additional_headers=app.headers) as ws:
        receive_event(ws, "historical_data_complete")
        ws.send(json.dumps({"type": "execute_task"}))
        receive_event(ws, "execution_started")
    if "e2e:gate" in description:
        wait_for(lambda: (app.root / "model-entered").exists(), app, "model call")
    return task_id


# Long enough for several automatic resumes with their backoff and jitter.
COLLECT_EVENTS_TIMEOUT = 45


def collect_events(app, ws, until) -> list[dict[str, Any]]:
    """Events up to and including the first one ``until`` accepts.

    Unlike :func:`receive_event` this tolerates ``error`` events: a failed
    model call may surface one before the run pauses.
    """
    events: list[dict[str, Any]] = []
    deadline = time.monotonic() + COLLECT_EVENTS_TIMEOUT
    while (remaining := deadline - time.monotonic()) > 0:
        try:
            event = json.loads(ws.recv(timeout=remaining))
        except TimeoutError:
            break
        events.append(event)
        if until(event):
            return events
    pytest.fail(f"Timed out collecting events: {events}\n{app.diagnostics()}")


# ---------------------------------------------------------------- model faults


def set_model_fault(app, kind: str, *, count: int | None) -> None:
    """Fail the next ``count`` provider attempts (``None``: until cleared).

    Change it only while no model call is in flight.
    """
    path = app.root / MODEL_FAULT_FILE
    staged = path.with_suffix(".tmp")
    staged.write_text(json.dumps({"kind": kind, "remaining": count}))
    os.replace(staged, path)


def clear_model_fault(app) -> None:
    (app.root / MODEL_FAULT_FILE).unlink(missing_ok=True)


def model_faults(app, kind: str | None = None) -> list[dict[str, Any]]:
    return [
        call
        for call in model_calls(app)
        if call.get("fault") is not None and kind in (None, call["fault"])
    ]


def answered_model_calls(app) -> list[dict[str, Any]]:
    return [call for call in model_calls(app) if call.get("fault") is None]


def task_detail(app, task_id: int) -> dict[str, Any]:
    response = app.client.get(f"/api/chat/task/{task_id}", headers=app.headers)
    assert response.status_code == 200, response.text
    return response.json()


def assistant_messages(task_id: int) -> list[tuple[str, str]]:
    """``(message_type, content)`` of the task's persisted assistant messages."""
    from xagent.web.models.chat_message import TaskChatMessage
    from xagent.web.models.database import get_session_local

    with get_session_local()() as db:
        return [
            (message.message_type, message.content)
            for message in db.query(TaskChatMessage)
            .filter(
                TaskChatMessage.task_id == task_id,
                TaskChatMessage.role == "assistant",
            )
            .order_by(TaskChatMessage.id)
        ]