Skip to content

Enterprise agent platform for building, deploying, and operating AI teammates.

13.5 KB
Raw Blame
from __future__ import annotations

from collections.abc import Iterator
from pathlib import Path
from typing import Any
from uuid import UUID

import pytest
from fastapi.testclient import TestClient
from sqlalchemy.orm import Session, sessionmaker

from tests.e2e.minio_harness import (
    MinioStorage,
    PersistenceApp,
    run_file_persistence_app,
    run_minio_storage,
)
from xagent.web.models.task import Task, TaskStatus
from xagent.web.models.uploaded_file import UploadedFile
from xagent.web.services.client_error_messages import CLIENT_SAFE_TASK_FAILURE

pytestmark = [pytest.mark.e2e, pytest.mark.docker]


LLM_RESPONSES_PATH = (
    Path(__file__).parent / "fixtures" / "file_persistence_minio_llm_responses.json"
)
INPUT_MATERIALIZATION_LLM_RESPONSES_PATH = (
    Path(__file__).parent / "fixtures" / "file_input_materialization_llm_responses.json"
)


@pytest.fixture
def minio_storage(monkeypatch: pytest.MonkeyPatch) -> Iterator[MinioStorage]:
    yield from run_minio_storage(monkeypatch)


@pytest.fixture
def persistence_app(
    monkeypatch: pytest.MonkeyPatch, tmp_path: Path, minio_storage: MinioStorage
) -> Iterator[PersistenceApp]:
    del minio_storage
    yield from run_file_persistence_app(
        monkeypatch,
        tmp_path,
        llm_responses_path=LLM_RESPONSES_PATH,
    )


def _fetch_file_record(session_factory: sessionmaker[Session], file_id: str) -> Any:
    db = session_factory()
    try:
        return db.query(UploadedFile).filter(UploadedFile.file_id == file_id).one()
    finally:
        db.close()


def _assert_task_output_generation_key(
    storage_key: str,
    *,
    user_id: int,
    task_id: int,
    file_id: str,
    relative_path: str,
) -> None:
    prefix = f"users/{user_id}/tasks/{task_id}/outputs/{file_id}/_versions/"
    assert storage_key.startswith(prefix)
    generation, actual_relative_path = storage_key.removeprefix(prefix).split("/", 1)
    assert UUID(generation)
    assert actual_relative_path == relative_path


def _wait_for_startup_file_storage_sync(client: TestClient) -> None:
    response = client.get("/api/auth/setup-status")
    assert response.status_code == 200


def _receive_until_task_completed(websocket: Any) -> dict[str, Any]:
    for _ in range(200):
        message = websocket.receive_json()
        if message.get("type") == "task_completed":
            return message
        if message.get("type") in {
            "task_error",
            "agent_error",
            "error",
        }:
            raise AssertionError(f"Task execution failed over websocket: {message}")
    raise AssertionError("Timed out waiting for task_completed websocket event")


def _receive_until_type(websocket: Any, event_type: str) -> dict[str, Any]:
    for _ in range(50):
        message = websocket.receive_json()
        if message.get("type") == event_type:
            return message
        if message.get("type") in {
            "task_error",
            "agent_error",
            "error",
        }:
            raise AssertionError(f"Unexpected websocket error: {message}")
    raise AssertionError(f"Timed out waiting for {event_type} websocket event")


def _receive_until_execution_error(websocket: Any) -> dict[str, Any]:
    for _ in range(200):
        message = websocket.receive_json()
        if message.get("type") in {"task_error", "agent_error", "error"}:
            return message
        if message.get("type") == "task_completed":
            raise AssertionError(f"Expected task_error, got completion: {message}")
    raise AssertionError("Timed out waiting for websocket execution error")


def test_task_uploads_agent_outputs_and_startup_sync_persist_to_minio(
    persistence_app: PersistenceApp,
    minio_storage: MinioStorage,
) -> None:
    client: TestClient = persistence_app.client
    headers = persistence_app.headers
    session_factory = persistence_app.session_factory
    user_id = persistence_app.user_id
    startup_repair_file_id = persistence_app.startup_repair_file_id
    token = persistence_app.token

    _wait_for_startup_file_storage_sync(client)
    startup_repair_record = _fetch_file_record(session_factory, startup_repair_file_id)
    assert startup_repair_record.storage_backend == "s3"
    assert startup_repair_record.storage_status == "available"
    assert startup_repair_record.storage_key == (
        f"users/{user_id}/uploads/{startup_repair_file_id}/startup-repair.txt"
    )
    assert minio_storage.object_bytes(startup_repair_record.storage_key) == (
        b"startup repair content\n"
    )

    create_response = client.post(
        "/api/chat/task/create",
        json={
            "title": "MinIO output task",
            "description": "Create static-output.txt",
            "execution_mode": "balanced",
        },
        headers=headers,
    )

    assert create_response.status_code == 200
    task_id = create_response.json()["task_id"]
    db = session_factory()
    try:
        task = db.query(Task).filter(Task.id == task_id).one()
        assert task.status == TaskStatus.PENDING
        assert task.execution_mode == "balanced"
    finally:
        db.close()

    response = client.post(
        "/api/files/upload",
        files=[
            ("files", ("alpha.txt", b"alpha upload\n", "text/plain")),
            ("files", ("beta.txt", b"beta upload\n", "text/plain")),
        ],
        data={"task_type": "general", "task_id": str(task_id)},
        headers=headers,
    )

    assert response.status_code == 200
    uploaded = response.json()["files"]
    assert len(uploaded) == 2
    for item, expected_content in zip(uploaded, [b"alpha upload\n", b"beta upload\n"]):
        record = _fetch_file_record(session_factory, item["file_id"])
        assert record.task_id == task_id
        assert record.storage_backend == "s3"
        assert record.storage_status == "available"
        assert record.storage_key
        assert record.file_size == len(expected_content)
        assert minio_storage.object_bytes(record.storage_key) == expected_content

    with client.websocket_connect(
        f"/ws/chat/{task_id}", headers={"Authorization": f"Bearer {token}"}
    ) as websocket:
        _receive_until_type(websocket, "trace_event")
        websocket.send_json({"type": "execute_task"})
        execution_started = _receive_until_type(websocket, "execution_started")
        assert execution_started["type"] == "execution_started"
        completion_message = _receive_until_task_completed(websocket)

    assert completion_message["success"] is True
    db = session_factory()
    try:
        task = db.query(Task).filter(Task.id == task_id).one()
        assert task.status == TaskStatus.COMPLETED
        output_record = (
            db.query(UploadedFile)
            .filter(
                UploadedFile.user_id == user_id,
                UploadedFile.task_id == task_id,
                UploadedFile.filename == "static-output.txt",
            )
            .one()
        )
        assert output_record.storage_backend == "s3"
        assert output_record.storage_status == "available"
        assert output_record.workspace_category == "output"
        assert output_record.workspace_relative_path == "output/static-output.txt"
        _assert_task_output_generation_key(
            output_record.storage_key,
            user_id=user_id,
            task_id=task_id,
            file_id=output_record.file_id,
            relative_path="output/static-output.txt",
        )
        assert minio_storage.object_bytes(output_record.storage_key) == (
            b"minio persistence e2e output\n"
        )
    finally:
        db.close()


def test_websocket_output_persistence_sends_error_and_rolls_back_when_minio_write_fails(
    persistence_app: PersistenceApp,
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    client: TestClient = persistence_app.client
    headers = persistence_app.headers
    session_factory = persistence_app.session_factory
    token = persistence_app.token

    create_response = client.post(
        "/api/chat/task/create",
        json={
            "title": "MinIO output outage task",
            "description": "Create static-output.txt while MinIO output write fails",
            "execution_mode": "balanced",
        },
        headers=headers,
    )

    assert create_response.status_code == 200
    task_id = create_response.json()["task_id"]

    from xagent.core.file_storage.storage import FsspecFileStorage

    real_put_file = FsspecFileStorage.put_file
    failed_output_keys: list[str] = []

    def fail_task_output_put_file(
        self: FsspecFileStorage,
        source: Path,
        key: str,
        content_type: str | None = None,
    ) -> Any:
        if f"/tasks/{task_id}/outputs/" in f"/{key}":
            failed_output_keys.append(key)
            raise RuntimeError("simulated MinIO output write outage")
        return real_put_file(self, source, key, content_type)

    monkeypatch.setattr(FsspecFileStorage, "put_file", fail_task_output_put_file)

    with client.websocket_connect(
        f"/ws/chat/{task_id}", headers={"Authorization": f"Bearer {token}"}
    ) as websocket:
        _receive_until_type(websocket, "trace_event")
        websocket.send_json({"type": "execute_task"})
        execution_started = _receive_until_type(websocket, "execution_started")
        assert execution_started["type"] == "execution_started"
        error_message = _receive_until_execution_error(websocket)

    assert error_message["message"] == CLIENT_SAFE_TASK_FAILURE
    assert error_message["error"] == CLIENT_SAFE_TASK_FAILURE
    assert "simulated MinIO output write outage" not in repr(error_message)
    assert failed_output_keys
    assert all(f"/tasks/{task_id}/outputs/" in f"/{key}" for key in failed_output_keys)

    db = session_factory()
    try:
        task = db.get(Task, task_id)
        assert task is not None
        assert task.status == TaskStatus.FAILED
        assert "durable object" in str(task.error_message).lower()
        assert (
            db.query(UploadedFile)
            .filter(
                UploadedFile.task_id == task_id,
                UploadedFile.filename == "static-output.txt",
            )
            .first()
            is None
        )
    finally:
        db.close()


def test_chat_task_materializes_missing_local_input_from_minio_before_agent_reads_it(
    monkeypatch: pytest.MonkeyPatch,
    tmp_path: Path,
    minio_storage: MinioStorage,
) -> None:
    app_generator = run_file_persistence_app(
        monkeypatch,
        tmp_path,
        llm_responses_path=INPUT_MATERIALIZATION_LLM_RESPONSES_PATH,
    )
    app = next(app_generator)
    try:
        client: TestClient = app.client
        headers = app.headers
        session_factory = app.session_factory
        user_id = app.user_id
        token = app.token

        create_response = client.post(
            "/api/chat/task/create",
            json={
                "title": "MinIO input task",
                "description": "Read source.txt and create a derived output",
                "execution_mode": "balanced",
            },
            headers=headers,
        )
        assert create_response.status_code == 200
        task_id = create_response.json()["task_id"]

        upload_response = client.post(
            "/api/files/upload",
            files={"file": ("source.txt", b"source from minio\n", "text/plain")},
            data={"task_type": "general", "task_id": str(task_id)},
            headers=headers,
        )
        assert upload_response.status_code == 200
        uploaded_file_id = upload_response.json()["file_id"]
        uploaded_record = _fetch_file_record(session_factory, uploaded_file_id)
        uploaded_local_path = Path(str(uploaded_record.storage_path))
        uploaded_storage_key = str(uploaded_record.storage_key)
        assert minio_storage.object_bytes(uploaded_storage_key) == (
            b"source from minio\n"
        )
        uploaded_local_path.unlink()

        with client.websocket_connect(
            f"/ws/chat/{task_id}", headers={"Authorization": f"Bearer {token}"}
        ) as websocket:
            _receive_until_type(websocket, "trace_event")
            websocket.send_json(
                {
                    "type": "chat",
                    "message": "Read source.txt and create derived-from-input.txt",
                    "files": [
                        {
                            "file_id": uploaded_file_id,
                            "name": "source.txt",
                            "size": len(b"source from minio\n"),
                            "type": "text/plain",
                        }
                    ],
                }
            )
            completion_message = _receive_until_task_completed(websocket)

        assert completion_message["success"] is True
        assert uploaded_local_path.read_bytes() == b"source from minio\n"

        db = session_factory()
        try:
            task = db.query(Task).filter(Task.id == task_id).one()
            assert task.status == TaskStatus.COMPLETED
            output_record = (
                db.query(UploadedFile)
                .filter(
                    UploadedFile.user_id == user_id,
                    UploadedFile.task_id == task_id,
                    UploadedFile.filename == "derived-from-input.txt",
                )
                .one()
            )
            assert output_record.storage_backend == "s3"
            assert output_record.storage_status == "available"
            assert minio_storage.object_bytes(str(output_record.storage_key)) == (
                b"derived from uploaded input: source from minio\n"
            )
        finally:
            db.close()
    finally:
        app_generator.close()