Skip to content

Enterprise agent platform for building, deploying, and operating AI teammates.

13.0 KB
Raw Blame
from __future__ import annotations

import queue
import threading
from collections.abc import Iterator
from pathlib import Path
from typing import Any
from uuid import uuid4

import pytest
from sqlalchemy.orm import Session, sessionmaker

from tests.e2e.app_harness import (
    configure_e2e_app_environment,
    create_e2e_user,
    disable_external_app_services,
    init_e2e_db,
    run_e2e_app_client,
    seed_registered_local_file,
)
from tests.e2e.minio_harness import MinioStorage, run_minio_storage
from tests.shared.async_waits import DB_PROGRESS_TIMEOUT
from xagent.core.file_storage.factory import get_unscoped_file_storage
from xagent.web.models.uploaded_file import UploadedFile

pytestmark = [pytest.mark.e2e, pytest.mark.docker]


@pytest.fixture
def minio_storage(monkeypatch: pytest.MonkeyPatch) -> Iterator[MinioStorage]:
    yield from run_minio_storage(monkeypatch)


def _record(session_factory: sessionmaker[Session], file_id: str) -> UploadedFile:
    db = session_factory()
    try:
        return db.query(UploadedFile).filter(UploadedFile.file_id == file_id).one()
    finally:
        db.close()


def test_startup_sync_repairs_only_files_that_need_durable_storage(
    monkeypatch: pytest.MonkeyPatch,
    tmp_path: Path,
    minio_storage: MinioStorage,
) -> None:
    uploads_dir = configure_e2e_app_environment(monkeypatch, tmp_path=tmp_path)
    disable_external_app_services(monkeypatch)
    SessionLocal = init_e2e_db()

    db = SessionLocal()
    try:
        user = create_e2e_user(db, username="startup-sync-user")
        user_id = user.id

        legacy_file_id = str(uuid4())
        existing_file_id = str(uuid4())
        missing_remote_file_id = str(uuid4())
        missing_local_file_id = str(uuid4())

        existing_key = f"users/{user_id}/uploads/{existing_file_id}/existing.txt"
        missing_remote_key = (
            f"users/{user_id}/uploads/{missing_remote_file_id}/missing-remote.txt"
        )
        missing_local_key = (
            f"users/{user_id}/uploads/{missing_local_file_id}/missing-local.txt"
        )
        existing_object = get_unscoped_file_storage().put_bytes(
            b"already durable\n",
            existing_key,
            content_type="text/plain",
        )

        legacy = seed_registered_local_file(
            db,
            uploads_dir=uploads_dir,
            user_id=user_id,
            filename="legacy.txt",
            content=b"legacy needs upload\n",
            file_id=legacy_file_id,
            mime_type="text/plain",
            storage_status="legacy",
        )
        seed_registered_local_file(
            db,
            uploads_dir=uploads_dir,
            user_id=user_id,
            filename="existing.txt",
            content=b"local should not overwrite remote\n",
            file_id=existing_file_id,
            mime_type="text/plain",
            storage_backend=existing_object.backend,
            storage_key=existing_object.key,
            storage_uri=existing_object.uri,
            checksum=existing_object.checksum,
            etag=existing_object.etag,
            storage_status="available",
        )
        seed_registered_local_file(
            db,
            uploads_dir=uploads_dir,
            user_id=user_id,
            filename="missing-remote.txt",
            content=b"remote should be repaired\n",
            file_id=missing_remote_file_id,
            mime_type="text/plain",
            storage_backend="s3",
            storage_key=missing_remote_key,
            storage_status="available",
        )
        missing_local = seed_registered_local_file(
            db,
            uploads_dir=uploads_dir,
            user_id=user_id,
            filename="missing-local.txt",
            content=b"this file disappears before startup\n",
            file_id=missing_local_file_id,
            mime_type="text/plain",
            storage_backend="s3",
            storage_key=missing_local_key,
            storage_status="available",
        )
        missing_local.path.unlink()
    finally:
        db.close()

    with run_e2e_app_client(
        monkeypatch,
        username=user.username,
        user_id=user_id,
    ) as app:
        setup_response = app.client.get("/api/auth/setup-status")
        assert setup_response.status_code == 200

        legacy_record = _record(app.session_factory, legacy_file_id)
        existing_record = _record(app.session_factory, existing_file_id)
        missing_remote_record = _record(app.session_factory, missing_remote_file_id)
        missing_local_record = _record(app.session_factory, missing_local_file_id)

        assert legacy_record.storage_backend == "s3"
        assert legacy_record.storage_status == "available"
        assert legacy_record.storage_key == (
            f"users/{user_id}/uploads/{legacy_file_id}/legacy.txt"
        )
        assert minio_storage.object_bytes(str(legacy_record.storage_key)) == (
            b"legacy needs upload\n"
        )
        assert legacy.path.exists()

        assert existing_record.storage_key == existing_key
        assert minio_storage.object_bytes(existing_key) == b"already durable\n"

        assert missing_remote_record.storage_key == missing_remote_key
        assert minio_storage.object_bytes(missing_remote_key) == (
            b"remote should be repaired\n"
        )

        assert missing_local_record.storage_key == missing_local_key
        assert not minio_storage.exists(missing_local_key)


def test_startup_file_storage_sync_runs_after_startup_but_gates_client_service(
    monkeypatch: pytest.MonkeyPatch,
    tmp_path: Path,
    minio_storage: MinioStorage,
) -> None:
    uploads_dir = configure_e2e_app_environment(monkeypatch, tmp_path=tmp_path)
    disable_external_app_services(monkeypatch)
    SessionLocal = init_e2e_db()

    db = SessionLocal()
    try:
        user = create_e2e_user(db, username="startup-gate-user")
        user_id = user.id
        legacy_file_id = str(uuid4())
        seed_registered_local_file(
            db,
            uploads_dir=uploads_dir,
            user_id=user_id,
            filename="startup-gate.txt",
            content=b"startup gate content\n",
            file_id=legacy_file_id,
            mime_type="text/plain",
            storage_status="legacy",
        )
    finally:
        db.close()

    sync_started = threading.Event()
    release_sync = threading.Event()
    close_app = threading.Event()
    client_request_started = threading.Event()
    client_request_completed = threading.Event()
    app_queue: queue.Queue[Any] = queue.Queue(maxsize=1)
    app_errors: queue.Queue[BaseException] = queue.Queue()
    request_errors: queue.Queue[BaseException] = queue.Queue()
    request_responses: queue.Queue[Any] = queue.Queue(maxsize=1)

    def _configure_app_module(app_module: Any) -> None:
        original_sync = app_module.run_startup_file_storage_sync

        def _delayed_startup_sync() -> None:
            sync_started.set()
            assert release_sync.wait(timeout=10)
            original_sync()

        app_module.run_startup_file_storage_sync = _delayed_startup_sync

    def _run_app() -> None:
        try:
            with run_e2e_app_client(
                monkeypatch,
                username=user.username,
                user_id=user_id,
                configure_app_module=_configure_app_module,
            ) as app:
                app_queue.put(app)
                assert close_app.wait(timeout=10)
        except BaseException as exc:
            app_errors.put(exc)

    app_thread = threading.Thread(target=_run_app, daemon=True)
    app_thread.start()

    try:
        app = app_queue.get(timeout=5)
        assert sync_started.wait(timeout=DB_PROGRESS_TIMEOUT)

        health_response = app.client.get("/health")
        assert health_response.status_code == 200

        ready_response = app.client.get("/ready")
        assert ready_response.status_code == 503
        assert ready_response.json()["status"] == "starting"

        def _make_client_request() -> None:
            try:
                client_request_started.set()
                response = app.client.get("/api/auth/setup-status")
                request_responses.put(response)
            except BaseException as exc:
                request_errors.put(exc)
            finally:
                client_request_completed.set()

        request_thread = threading.Thread(target=_make_client_request, daemon=True)
        request_thread.start()

        assert client_request_started.wait(timeout=5)
        assert not client_request_completed.wait(timeout=0.2)

        release_sync.set()
        request_thread.join(timeout=10)
        assert not request_thread.is_alive()
        assert request_errors.empty()

        client_response = request_responses.get_nowait()
        assert client_response.status_code == 200

        ready_response = app.client.get("/ready")
        assert ready_response.status_code == 200
        assert ready_response.json()["status"] == "ready"

        legacy_record = _record(app.session_factory, legacy_file_id)
        assert legacy_record.storage_backend == "s3"
        assert legacy_record.storage_status == "available"
        assert legacy_record.storage_key == (
            f"users/{user_id}/uploads/{legacy_file_id}/startup-gate.txt"
        )
        assert minio_storage.object_bytes(str(legacy_record.storage_key)) == (
            b"startup gate content\n"
        )
    finally:
        release_sync.set()
        close_app.set()
        app_thread.join(timeout=10)

    assert not app_thread.is_alive()
    assert app_errors.empty()


def test_startup_file_storage_sync_recovers_after_initial_storage_failure(
    monkeypatch: pytest.MonkeyPatch,
    tmp_path: Path,
    minio_storage: MinioStorage,
) -> None:
    uploads_dir = configure_e2e_app_environment(monkeypatch, tmp_path=tmp_path)
    disable_external_app_services(monkeypatch)
    SessionLocal = init_e2e_db()

    db = SessionLocal()
    try:
        user = create_e2e_user(db, username="startup-retry-user")
        user_id = user.id
        legacy_file_id = str(uuid4())
        seed_registered_local_file(
            db,
            uploads_dir=uploads_dir,
            user_id=user_id,
            filename="startup-retry.txt",
            content=b"startup retry content\n",
            file_id=legacy_file_id,
            mime_type="text/plain",
            storage_status="legacy",
        )
    finally:
        db.close()

    first_attempt_failed = threading.Event()
    sync_completed = threading.Event()
    close_app = threading.Event()
    app_queue: queue.Queue[Any] = queue.Queue(maxsize=1)
    app_errors: queue.Queue[BaseException] = queue.Queue()

    def _configure_app_module(app_module: Any) -> None:
        original_sync = app_module.run_startup_file_storage_sync
        attempts = {"count": 0}

        def _fail_once_then_sync() -> None:
            attempts["count"] += 1
            if attempts["count"] == 1:
                first_attempt_failed.set()
                raise RuntimeError("s3 unavailable")
            original_sync()
            sync_completed.set()

        app_module.run_startup_file_storage_sync = _fail_once_then_sync
        app_module.FILE_STORAGE_STARTUP_SYNC_RETRY_INTERVAL_SECONDS = 0.01

    def _run_app() -> None:
        try:
            with run_e2e_app_client(
                monkeypatch,
                username=user.username,
                user_id=user_id,
                configure_app_module=_configure_app_module,
            ) as app:
                app_queue.put(app)
                assert close_app.wait(timeout=10)
        except BaseException as exc:
            app_errors.put(exc)

    app_thread = threading.Thread(target=_run_app, daemon=True)
    app_thread.start()

    try:
        app = app_queue.get(timeout=5)
        assert first_attempt_failed.wait(timeout=DB_PROGRESS_TIMEOUT)

        ready_response = app.client.get("/ready")
        assert ready_response.status_code == 503
        assert ready_response.json()["status"] == "error"

        failed_client_response = app.client.get("/api/auth/setup-status")
        assert failed_client_response.status_code == 503
        assert failed_client_response.json()["detail"] == (
            "Startup file storage sync failed"
        )

        assert sync_completed.wait(timeout=DB_PROGRESS_TIMEOUT)

        recovered_client_response = app.client.get("/api/auth/setup-status")
        assert recovered_client_response.status_code == 200

        ready_response = app.client.get("/ready")
        assert ready_response.status_code == 200
        assert ready_response.json()["status"] == "ready"

        legacy_record = _record(app.session_factory, legacy_file_id)
        assert legacy_record.storage_backend == "s3"
        assert legacy_record.storage_status == "available"
        assert legacy_record.storage_key == (
            f"users/{user_id}/uploads/{legacy_file_id}/startup-retry.txt"
        )
        assert minio_storage.object_bytes(str(legacy_record.storage_key)) == (
            b"startup retry content\n"
        )
    finally:
        close_app.set()
        app_thread.join(timeout=10)

    assert not app_thread.is_alive()
    assert app_errors.empty()