Skip to content

Enterprise agent platform for building, deploying, and operating AI teammates.

23.6 KB
Raw Blame
from __future__ import annotations

import hashlib
from collections.abc import Iterator
from datetime import datetime, timedelta, timezone
from pathlib import Path
from unittest.mock import AsyncMock
from uuid import uuid4

import pytest
from fastapi.testclient import TestClient
from sqlalchemy.orm import Session, sessionmaker

from tests.e2e.app_harness import (
    SeededLocalFile,
    build_access_token,
    configure_e2e_app_environment,
    create_e2e_user,
    disable_external_app_services,
    init_e2e_db,
    run_e2e_app_client,
)
from tests.e2e.minio_harness import MinioStorage, run_minio_storage
from xagent.core.tools.adapters.vibe.file_ingestion_tool import (
    CreateKnowledgeBaseFromFileTool,
)
from xagent.core.tools.core.RAG_tools.core.schemas import (
    IngestionResult,
    IngestionStepResult,
)
from xagent.core.workspace import TaskWorkspace
from xagent.web.models.task import Task
from xagent.web.models.uploaded_file import UploadedFile
from xagent.web.services.kb_file_service import reconcile_uploaded_files

pytestmark = [pytest.mark.e2e, pytest.mark.docker]


@pytest.fixture
def minio_storage(monkeypatch: pytest.MonkeyPatch) -> Iterator[MinioStorage]:
    yield from run_minio_storage(monkeypatch)


def _upload_text_file(client: TestClient, headers: dict[str, str]) -> dict[str, str]:
    response = client.post(
        "/api/files/upload",
        files={"file": ("source.txt", b"source from minio\n", "text/plain")},
        data={"task_type": "general"},
        headers=headers,
    )
    assert response.status_code == 200
    return response.json()


def _record(session_factory: sessionmaker[Session], file_id: str) -> UploadedFile:
    db = session_factory()
    try:
        return db.query(UploadedFile).filter(UploadedFile.file_id == file_id).one()
    finally:
        db.close()


def _seed_durable_uploaded_file(
    session_factory: sessionmaker[Session],
    minio_storage: MinioStorage,
    *,
    user_id: int,
    filename: str,
    content: bytes,
    local_path: Path,
    task_id: int | None = None,
    file_id: str | None = None,
    mime_type: str = "text/plain",
    workspace_relative_path: str | None = None,
    workspace_category: str | None = None,
) -> SeededLocalFile:
    resolved_file_id = file_id or str(uuid4())
    storage_key = f"users/{user_id}/uploads/{resolved_file_id}/{filename}"
    minio_storage.put_object(storage_key, content, mime_type)
    db = session_factory()
    try:
        record = UploadedFile(
            file_id=resolved_file_id,
            user_id=user_id,
            task_id=task_id,
            filename=filename,
            storage_path=str(local_path),
            storage_backend="s3",
            storage_key=storage_key,
            storage_uri=f"s3://{minio_storage.bucket}/{minio_storage.prefix}/{storage_key}",
            storage_status="available",
            mime_type=mime_type,
            file_size=len(content),
            workspace_relative_path=workspace_relative_path,
            workspace_category=workspace_category,
        )
        db.add(record)
        db.commit()
    finally:
        db.close()
    return SeededLocalFile(
        file_id=resolved_file_id,
        path=local_path,
        filename=filename,
    )


def test_download_and_preview_materialize_uploaded_file_from_minio(
    monkeypatch: pytest.MonkeyPatch,
    tmp_path: Path,
    minio_storage: MinioStorage,
) -> None:
    uploads_dir = configure_e2e_app_environment(monkeypatch, tmp_path=tmp_path)
    del uploads_dir
    disable_external_app_services(monkeypatch)
    SessionLocal = init_e2e_db()
    db = SessionLocal()
    try:
        user = create_e2e_user(db, username="file-api-user")
    finally:
        db.close()

    with run_e2e_app_client(
        monkeypatch,
        username=user.username,
        user_id=user.id,
    ) as app:
        uploaded = _upload_text_file(app.client, app.headers)
        file_id = uploaded["file_id"]
        record = _record(app.session_factory, file_id)
        local_path = Path(str(record.storage_path))
        storage_key = str(record.storage_key)
        durable_sha256 = str(record.checksum)
        assert minio_storage.object_info(storage_key)["Metadata"] == {
            "xagent-sha256": durable_sha256
        }
        assert local_path.exists()
        local_path.unlink()

        download = app.client.get(
            f"/api/files/download/{file_id}",
            headers=app.headers,
        )
        assert download.status_code == 200
        assert download.content == b"source from minio\n"
        assert local_path.read_bytes() == b"source from minio\n"

        local_path.unlink()
        preview = app.client.get(
            f"/api/files/preview/{file_id}",
            headers=app.headers,
        )
        assert preview.status_code == 200
        assert preview.content == b"source from minio\n"
        assert not local_path.exists()
        expected_materialized = (
            tmp_path
            / "materialized"
            / hashlib.sha256(storage_key.encode("utf-8")).hexdigest()[:16]
            / durable_sha256
            / "source.txt"
        )
        assert expected_materialized.read_bytes() == b"source from minio\n"
        assert not list((tmp_path / "materialized").rglob("*.metadata.json"))


def test_public_preview_relative_asset_restores_durable_only_file_from_minio(
    monkeypatch: pytest.MonkeyPatch,
    tmp_path: Path,
    minio_storage: MinioStorage,
) -> None:
    configure_e2e_app_environment(monkeypatch, tmp_path=tmp_path)
    disable_external_app_services(monkeypatch)
    SessionLocal = init_e2e_db()
    db = SessionLocal()
    try:
        user = create_e2e_user(db, username="public-preview-asset-user")
        task = Task(id=2811, user_id=user.id, title="Public preview asset")
        db.add(task)
        db.commit()
    finally:
        db.close()

    with run_e2e_app_client(
        monkeypatch,
        username=user.username,
        user_id=user.id,
    ) as app:
        html_upload = app.client.post(
            "/api/files/upload",
            files={
                "file": (
                    "index.html",
                    b"<script src='assets/app.js'></script>",
                    "text/html",
                )
            },
            data={"task_type": "general", "task_id": "2811"},
            headers=app.headers,
        )
        assert html_upload.status_code == 200
        html_file_id = html_upload.json()["file_id"]
        html_record = _record(app.session_factory, html_file_id)
        asset_path = Path(str(html_record.storage_path)).parent / "assets" / "app.js"
        seeded_asset = _seed_durable_uploaded_file(
            app.session_factory,
            minio_storage,
            user_id=user.id,
            task_id=2811,
            filename="app.js",
            content=b"console.log('from minio');",
            local_path=asset_path,
            file_id="44444444-4444-4444-8444-444444444444",
            mime_type="application/javascript",
            workspace_relative_path="output/assets/app.js",
            workspace_category="output",
        )
        assert not seeded_asset.path.exists()

        preview_asset = app.client.get(
            f"/api/files/public/preview/{html_file_id}",
            params={"relative_path": "assets/app.js"},
        )

        assert preview_asset.status_code == 200
        assert preview_asset.content == b"console.log('from minio');"
        assert seeded_asset.path.read_bytes() == b"console.log('from minio');"


def test_workspace_user_file_listing_includes_durable_only_minio_file(
    monkeypatch: pytest.MonkeyPatch,
    tmp_path: Path,
    minio_storage: MinioStorage,
) -> None:
    configure_e2e_app_environment(monkeypatch, tmp_path=tmp_path)
    disable_external_app_services(monkeypatch)
    SessionLocal = init_e2e_db()
    db = SessionLocal()
    try:
        user = create_e2e_user(db, username="workspace-list-minio-user")
        task = Task(id=2711, user_id=user.id, title="Workspace durable listing")
        db.add(task)
        db.commit()
    finally:
        db.close()

    missing_local_path = (
        tmp_path
        / "uploads"
        / f"user_{user.id}"
        / "web_task_2711"
        / "input"
        / "durable-only.txt"
    )
    _seed_durable_uploaded_file(
        SessionLocal,
        minio_storage,
        user_id=user.id,
        task_id=2711,
        filename="durable-only.txt",
        content=b"durable only listing\n",
        local_path=missing_local_path,
        file_id="11111111-1111-4111-8111-111111111111",
    )

    workspace = TaskWorkspace(id="web_task_2711", base_dir=str(tmp_path / "uploads"))
    workspace.db_session = SessionLocal()
    try:
        result = workspace.list_all_user_files(include_workspace_files=False)
    finally:
        workspace.db_session.close()

    assert result["success"] is True
    files_by_id = {item["file_id"]: item for item in result["files"]}
    assert "11111111-1111-4111-8111-111111111111" in files_by_id
    listed = files_by_id["11111111-1111-4111-8111-111111111111"]
    assert listed["filename"] == "durable-only.txt"
    assert listed["size"] == len(b"durable only listing\n")
    assert listed["in_current_workspace"] is False


def test_upload_returns_503_and_rolls_back_when_minio_write_fails(
    monkeypatch: pytest.MonkeyPatch,
    tmp_path: Path,
    minio_storage: MinioStorage,
) -> None:
    configure_e2e_app_environment(monkeypatch, tmp_path=tmp_path)
    del minio_storage
    disable_external_app_services(monkeypatch)
    SessionLocal = init_e2e_db()
    db = SessionLocal()
    try:
        user = create_e2e_user(db, username="file-upload-outage-user")
    finally:
        db.close()

    with run_e2e_app_client(
        monkeypatch,
        username=user.username,
        user_id=user.id,
    ) as app:
        from xagent.core.file_storage.storage import FsspecFileStorage

        def fail_put_file(
            self: FsspecFileStorage,
            source: Path,
            key: str,
            content_type: str | None = None,
        ) -> None:
            del source, key, content_type
            raise RuntimeError("simulated MinIO write outage")

        monkeypatch.setattr(FsspecFileStorage, "put_file", fail_put_file)

        upload = app.client.post(
            "/api/files/upload",
            files={"file": ("outage.txt", b"outage content\n", "text/plain")},
            data={"task_type": "general"},
            headers=app.headers,
        )

        assert upload.status_code == 503
        assert "durable storage" in upload.json()["detail"].lower()
        assert not list((tmp_path / "uploads").rglob("outage.txt"))

        db = app.session_factory()
        try:
            assert (
                db.query(UploadedFile)
                .filter(
                    UploadedFile.user_id == user.id,
                    UploadedFile.filename == "outage.txt",
                )
                .first()
                is None
            )
        finally:
            db.close()


def test_download_and_preview_return_503_when_minio_read_fails_without_local_copy(
    monkeypatch: pytest.MonkeyPatch,
    tmp_path: Path,
    minio_storage: MinioStorage,
) -> None:
    configure_e2e_app_environment(monkeypatch, tmp_path=tmp_path)
    del minio_storage
    disable_external_app_services(monkeypatch)
    SessionLocal = init_e2e_db()
    db = SessionLocal()
    try:
        user = create_e2e_user(db, username="file-read-outage-user")
    finally:
        db.close()

    with run_e2e_app_client(
        monkeypatch,
        username=user.username,
        user_id=user.id,
    ) as app:
        uploaded = _upload_text_file(app.client, app.headers)
        file_id = uploaded["file_id"]
        record = _record(app.session_factory, file_id)
        local_path = Path(str(record.storage_path))
        assert local_path.exists()
        local_path.unlink()

        from xagent.core.file_storage.storage import FsspecFileStorage

        def fail_open_read(self: FsspecFileStorage, key: str) -> None:
            del key
            raise RuntimeError("simulated MinIO read outage")

        monkeypatch.setattr(FsspecFileStorage, "open_read", fail_open_read)

        download = app.client.get(
            f"/api/files/download/{file_id}",
            headers=app.headers,
        )
        preview = app.client.get(
            f"/api/files/preview/{file_id}",
            headers=app.headers,
        )

        assert download.status_code == 503
        assert "durable storage" in download.json()["detail"].lower()
        assert preview.status_code == 503
        assert "durable storage" in preview.json()["detail"].lower()
        assert not local_path.exists()


@pytest.mark.asyncio
async def test_create_kb_from_file_tool_restores_durable_only_upload_from_minio(
    monkeypatch: pytest.MonkeyPatch,
    tmp_path: Path,
    minio_storage: MinioStorage,
) -> None:
    configure_e2e_app_environment(monkeypatch, tmp_path=tmp_path)
    disable_external_app_services(monkeypatch)
    SessionLocal = init_e2e_db()
    db = SessionLocal()
    try:
        user = create_e2e_user(db, username="kb-tool-minio-user")
    finally:
        db.close()

    local_path = tmp_path / "uploads" / f"user_{user.id}" / "kb-tool.txt"
    seeded = _seed_durable_uploaded_file(
        SessionLocal,
        minio_storage,
        user_id=user.id,
        filename="kb-tool.txt",
        content=b"kb tool durable content\n",
        local_path=local_path,
        file_id="33333333-3333-4333-8333-333333333333",
        mime_type="text/plain",
    )
    assert not seeded.path.exists()

    ingest_result = IngestionResult(
        status="success",
        doc_id="doc-1",
        parse_hash="parse-1",
        chunk_count=1,
        embedding_count=1,
        vector_count=1,
        completed_steps=[IngestionStepResult(name="register_document")],
        failed_step=None,
        message="ok",
        warnings=[],
        file_id=seeded.file_id,
    )
    observed_source_paths: list[str] = []

    def fake_run_document_ingestion(**kwargs: object) -> IngestionResult:
        source_path = str(kwargs["source_path"])
        observed_source_paths.append(source_path)
        assert Path(source_path).read_bytes() == b"kb tool durable content\n"
        return ingest_result

    service = AsyncMock()
    service.prepare_collection.return_value = "agent_file_kb"

    monkeypatch.setattr(
        "xagent.core.tools.adapters.vibe.agent_kb_service.AgentKnowledgeBaseService",
        lambda user_id: service,
    )
    monkeypatch.setattr(
        "xagent.core.tools.core.RAG_tools.pipelines.document_ingestion.run_document_ingestion",
        fake_run_document_ingestion,
    )

    tool = CreateKnowledgeBaseFromFileTool(user_id=user.id, is_admin=False)
    result = await tool.run_json_async(
        {"file_ids": [seeded.file_id], "collection_name": "agent_file_kb"}
    )

    assert result["success"] is True
    assert result["files_ingested"] == 1
    assert observed_source_paths == [str(seeded.path)]
    assert seeded.path.read_bytes() == b"kb tool durable content\n"


def test_download_serves_local_copy_when_minio_read_fails(
    monkeypatch: pytest.MonkeyPatch,
    tmp_path: Path,
    minio_storage: MinioStorage,
) -> None:
    configure_e2e_app_environment(monkeypatch, tmp_path=tmp_path)
    del minio_storage
    disable_external_app_services(monkeypatch)
    SessionLocal = init_e2e_db()
    db = SessionLocal()
    try:
        user = create_e2e_user(db, username="file-local-read-outage-user")
    finally:
        db.close()

    with run_e2e_app_client(
        monkeypatch,
        username=user.username,
        user_id=user.id,
    ) as app:
        uploaded = _upload_text_file(app.client, app.headers)
        file_id = uploaded["file_id"]
        record = _record(app.session_factory, file_id)
        local_path = Path(str(record.storage_path))
        assert local_path.exists()

        from xagent.core.file_storage.storage import FsspecFileStorage

        def fail_open_read(self: FsspecFileStorage, key: str) -> None:
            del key
            raise RuntimeError("simulated MinIO read outage")

        monkeypatch.setattr(FsspecFileStorage, "open_read", fail_open_read)

        download = app.client.get(
            f"/api/files/download/{file_id}",
            headers=app.headers,
        )

        assert download.status_code == 200
        assert download.content == b"source from minio\n"


def test_delete_removes_uploaded_file_from_db_local_disk_and_minio(
    monkeypatch: pytest.MonkeyPatch,
    tmp_path: Path,
    minio_storage: MinioStorage,
) -> None:
    configure_e2e_app_environment(monkeypatch, tmp_path=tmp_path)
    disable_external_app_services(monkeypatch)
    SessionLocal = init_e2e_db()
    db = SessionLocal()
    try:
        user = create_e2e_user(db, username="file-delete-user")
    finally:
        db.close()

    with run_e2e_app_client(
        monkeypatch,
        username=user.username,
        user_id=user.id,
    ) as app:
        uploaded = _upload_text_file(app.client, app.headers)
        file_id = uploaded["file_id"]
        record = _record(app.session_factory, file_id)
        storage_key = str(record.storage_key)
        local_path = Path(str(record.storage_path))

        assert local_path.exists()
        assert minio_storage.exists(storage_key)

        delete = app.client.delete(f"/api/files/{file_id}", headers=app.headers)
        assert delete.status_code == 200
        assert not local_path.exists()
        assert not minio_storage.exists(storage_key)

        db = app.session_factory()
        try:
            assert (
                db.query(UploadedFile).filter(UploadedFile.file_id == file_id).first()
                is None
            )
        finally:
            db.close()


def test_delete_keeps_db_row_when_durable_cleanup_fails(
    monkeypatch: pytest.MonkeyPatch,
    tmp_path: Path,
    minio_storage: MinioStorage,
) -> None:
    configure_e2e_app_environment(monkeypatch, tmp_path=tmp_path)
    disable_external_app_services(monkeypatch)
    SessionLocal = init_e2e_db()
    db = SessionLocal()
    try:
        user = create_e2e_user(db, username="file-delete-failure-user")
    finally:
        db.close()

    with run_e2e_app_client(
        monkeypatch,
        username=user.username,
        user_id=user.id,
    ) as app:
        uploaded = _upload_text_file(app.client, app.headers)
        file_id = uploaded["file_id"]
        record = _record(app.session_factory, file_id)
        storage_key = str(record.storage_key)
        local_path = Path(str(record.storage_path))

        from xagent.core.file_storage.storage import FsspecFileStorage

        real_delete = FsspecFileStorage.delete

        def fail_target_delete(self: FsspecFileStorage, key: str) -> None:
            if key == storage_key:
                raise RuntimeError("simulated durable delete failure")
            real_delete(self, key)

        monkeypatch.setattr(FsspecFileStorage, "delete", fail_target_delete)

        assert local_path.exists()
        assert minio_storage.exists(storage_key)

        delete = app.client.delete(f"/api/files/{file_id}", headers=app.headers)
        assert delete.status_code == 503

        assert local_path.exists()
        assert minio_storage.exists(storage_key)
        assert minio_storage.object_bytes(storage_key) == b"source from minio\n"

        db = app.session_factory()
        try:
            assert (
                db.query(UploadedFile).filter(UploadedFile.file_id == file_id).first()
                is not None
            )
        finally:
            db.close()


def test_stale_kb_reconcile_retries_minio_object_cleanup_after_delete_failure(
    monkeypatch: pytest.MonkeyPatch,
    tmp_path: Path,
    minio_storage: MinioStorage,
) -> None:
    configure_e2e_app_environment(monkeypatch, tmp_path=tmp_path)
    disable_external_app_services(monkeypatch)
    SessionLocal = init_e2e_db()
    db = SessionLocal()
    try:
        user = create_e2e_user(db, username="kb-reconcile-minio-user")
    finally:
        db.close()

    local_path = tmp_path / "uploads" / f"user_{user.id}" / "kb" / "stale-failed.md"
    seeded = _seed_durable_uploaded_file(
        SessionLocal,
        minio_storage,
        user_id=user.id,
        filename="stale-failed.md",
        content=b"stale failed kb file\n",
        local_path=local_path,
        file_id="22222222-2222-4222-8222-222222222222",
        mime_type="text/markdown",
    )
    db = SessionLocal()
    try:
        record = (
            db.query(UploadedFile).filter(UploadedFile.file_id == seeded.file_id).one()
        )
        record.created_at = datetime.now(timezone.utc) - timedelta(days=10)
        db.commit()
    finally:
        db.close()

    from xagent.core.file_storage.storage import FsspecFileStorage

    real_delete = FsspecFileStorage.delete
    attempts = 0

    def fail_first_target_delete(self: FsspecFileStorage, key: str) -> None:
        nonlocal attempts
        if key == f"users/{user.id}/uploads/{seeded.file_id}/stale-failed.md":
            attempts += 1
            if attempts == 1:
                raise RuntimeError("simulated MinIO delete outage")
        real_delete(self, key)

    monkeypatch.setattr(FsspecFileStorage, "delete", fail_first_target_delete)

    db = SessionLocal()
    try:
        first = reconcile_uploaded_files(
            db,
            user_id=user.id,
            is_admin=False,
            stale_ttl_hours=1,
            delete_stale=True,
            deletable_statuses={"UNKNOWN"},
        )
        assert first["cleanup_errors"] == 1
        assert first["deleted"] == 0
        assert (
            db.query(UploadedFile)
            .filter(UploadedFile.file_id == seeded.file_id)
            .first()
            is not None
        )
        assert minio_storage.exists(
            f"users/{user.id}/uploads/{seeded.file_id}/stale-failed.md"
        )

        second = reconcile_uploaded_files(
            db,
            user_id=user.id,
            is_admin=False,
            stale_ttl_hours=1,
            delete_stale=True,
            deletable_statuses={"UNKNOWN"},
        )
        assert second["cleanup_errors"] == 0
        assert second["deleted"] == 1
        assert (
            db.query(UploadedFile)
            .filter(UploadedFile.file_id == seeded.file_id)
            .first()
            is None
        )
        assert not minio_storage.exists(
            f"users/{user.id}/uploads/{seeded.file_id}/stale-failed.md"
        )
        assert attempts == 2
    finally:
        db.close()


def test_file_routes_reject_cross_user_access_and_keep_minio_object(
    monkeypatch: pytest.MonkeyPatch,
    tmp_path: Path,
    minio_storage: MinioStorage,
) -> None:
    configure_e2e_app_environment(monkeypatch, tmp_path=tmp_path)
    disable_external_app_services(monkeypatch)
    SessionLocal = init_e2e_db()
    db = SessionLocal()
    try:
        owner = create_e2e_user(db, username="owner-user")
        other = create_e2e_user(db, username="other-user")
    finally:
        db.close()

    with run_e2e_app_client(
        monkeypatch,
        username=owner.username,
        user_id=owner.id,
    ) as app:
        uploaded = _upload_text_file(app.client, app.headers)
        file_id = uploaded["file_id"]
        record = _record(app.session_factory, file_id)
        storage_key = str(record.storage_key)
        other_headers = {
            "Authorization": (
                f"Bearer {build_access_token(username=other.username, user_id=other.id)}"
            )
        }

        for method, path in [
            ("GET", f"/api/files/download/{file_id}"),
            ("GET", f"/api/files/preview/{file_id}"),
            ("DELETE", f"/api/files/{file_id}"),
        ]:
            response = app.client.request(method, path, headers=other_headers)
            assert response.status_code == 403

        assert minio_storage.exists(storage_key)
        assert minio_storage.object_bytes(storage_key) == b"source from minio\n"