Skip to content

Enterprise agent platform for building, deploying, and operating AI teammates.

18.2 KB
Raw Blame
"""Row deletes commit before their file bytes are deleted (#2664)."""

from __future__ import annotations

import asyncio
import threading
from contextlib import ExitStack
from pathlib import Path
from types import SimpleNamespace
from typing import Any
from unittest.mock import AsyncMock, MagicMock, patch

import pytest
from fastapi.testclient import TestClient
from sqlalchemy.exc import OperationalError

from tests.web.api import test_kb_dir as kb_dir
from tests.web.api import test_kb_local_rollback_contract as local
from xagent.core.file_storage.factory import get_unscoped_file_storage
from xagent.core.file_storage.storage import FsspecFileStorage
from xagent.core.tools.core.RAG_tools.core.schemas import (
    CollectionOperationResult,
    IngestionResult,
)
from xagent.core.tools.core.RAG_tools.storage.contracts import DocumentRecord
from xagent.web.api import kb as kb_module
from xagent.web.models.uploaded_file import UploadedFile
from xagent.web.services.managed_file_ref import ManagedFileRef
from xagent.web.services.uploaded_file_store import UploadedFileStore

test_env = kb_dir.test_env
temp_uploads = kb_dir.temp_uploads

FILE_A = {"provider": "google-drive", "fileId": "drive-A", "fileName": "a.csv"}
FILE_B = {"provider": "google-drive", "fileId": "drive-B", "fileName": "b.csv"}
METADATA_STORE = "xagent.core.tools.core.RAG_tools.storage.factory.get_metadata_store"


def _durable_storage(monkeypatch, tmp_path: Path) -> None:
    monkeypatch.setenv("XAGENT_FILE_STORAGE_URI", (tmp_path / "objects").as_uri())
    get_unscoped_file_storage.cache_clear()


def _rows(sessions) -> dict[str, tuple[str, str, str]]:
    db = sessions()
    try:
        return {
            str(r.filename): (str(r.file_id), str(r.storage_key), str(r.storage_path))
            for r in db.query(UploadedFile).all()
        }
    finally:
        db.close()


def _rows_without_bytes(sessions) -> list[str]:
    storage = get_unscoped_file_storage()
    return sorted(
        name for name, (_, key, _) in _rows(sessions).items() if not storage.exists(key)
    )


def _byte_delete_warnings(caplog) -> list[str]:
    return [
        r.getMessage()
        for r in caplog.records
        if r.name == "xagent.web.api.kb"
        and r.exc_info
        and r.getMessage().startswith("Failed to delete file bytes")
    ]


class _MetadataStore:
    """Config read hops to a worker thread, like the LanceDB store, and can be held."""

    def __init__(self, hold=None) -> None:
        self.hold = hold
        self.delete_collection_metadata = AsyncMock(return_value={})
        self.save_collection_config = AsyncMock()

    async def get_collection_config(self, **_kw: Any) -> None:
        def _read() -> None:
            if self.hold is not None:
                self.hold()

        return await asyncio.to_thread(_read)


def _post_cloud(test_env, files, ingest, *extra, downloader_gate=None):
    app, headers, _user, _ = test_env

    class _Files:
        def get(self, fileId, **_kw):
            name = {"drive-A": "a.csv", "drive-B": "b.csv"}[fileId]
            return kb_dir._fake_drive_metadata_request(fileId, name, "text/csv")

        def get_media(self, fileId, **_kw):
            return {"fileId": fileId}

    class _Downloader:
        def __init__(self, fh, request):
            self._fh = fh
            self._file_id = request["fileId"]

        def next_chunk(self):
            if downloader_gate is not None:
                downloader_gate(self._file_id)
            self._fh.write(self._file_id.encode())
            return None, True

    with ExitStack() as stack:
        for p in (
            patch("xagent.web.api.kb.get_google_credentials", return_value=object()),
            patch(
                "xagent.web.api.kb.build", return_value=SimpleNamespace(files=_Files)
            ),
            patch("xagent.web.api.kb.MediaIoBaseDownload", _Downloader),
            patch("xagent.web.api.kb.run_document_ingestion", side_effect=ingest),
            patch(
                "xagent.web.api.kb._ensure_collection_access", new_callable=AsyncMock
            ),
            *extra,
        ):
            stack.enter_context(p)
        return TestClient(app).post(
            "/api/kb/ingest-cloud",
            json={"collection": "cloud_coll", "files": files},
            headers=headers,
        )


def _broken_commit() -> None:
    raise OperationalError("COMMIT", {}, Exception("disk I/O error"))


def _is_file(kw: dict[str, Any], prefix: str) -> bool:
    return Path(kw["source_path"]).name.startswith(prefix)


def _failed(**_kw: Any) -> IngestionResult:
    return IngestionResult(status="error", message="embedding down")


def test_cloud_collection_failure_after_file_step_leaves_no_row_without_bytes(
    test_env, temp_uploads, monkeypatch, tmp_path
) -> None:
    _durable_storage(monkeypatch, tmp_path)
    sessions = test_env[3]
    seen: dict[str, Any] = {}

    class _BrokenStore:
        def list_document_records(self, **_kw):
            raise RuntimeError("lance down")

    def _ingest(**kw):
        seen.update(_rows(sessions))
        return _failed()

    response = _post_cloud(
        test_env,
        [FILE_A],
        _ingest,
        patch("xagent.web.api.kb.get_collection_sync", return_value=object()),
        patch("xagent.web.api.kb.get_vector_index_store", _BrokenStore),
    )

    assert response.status_code == 200
    assert response.json()[0]["message"].endswith(
        "lance down. Original ingestion error: embedding down"
    )
    assert "a.csv" not in _rows(sessions)
    assert not get_unscoped_file_storage().exists(seen["a.csv"][1])


def test_cloud_failed_row_delete_commit_keeps_row_and_bytes(
    test_env, temp_uploads, monkeypatch, tmp_path
) -> None:
    _durable_storage(monkeypatch, tmp_path)
    sessions = test_env[3]
    seen: dict[str, Any] = {}
    real_orphan = kb_module._delete_uploaded_file_if_orphaned

    def _orphan_then_break_commit(db, **kwargs):
        deleted = real_orphan(db, **kwargs)
        db.commit = _broken_commit
        return deleted

    def _ingest(**kw):
        seen.update(_rows(sessions))
        return _failed()

    response = _post_cloud(
        test_env,
        [FILE_A],
        _ingest,
        patch("xagent.web.api.kb.get_collection_sync", return_value=object()),
        patch(
            "xagent.web.api.kb._delete_uploaded_file_if_orphaned",
            side_effect=_orphan_then_break_commit,
        ),
    )

    assert "disk I/O error" in response.json()[0]["message"]
    assert _rows(sessions) == seen
    assert get_unscoped_file_storage().exists(seen["a.csv"][1])


def test_cloud_sibling_rollback_cannot_undo_this_files_row_delete(
    test_env, temp_uploads, monkeypatch, tmp_path
) -> None:
    _durable_storage(monkeypatch, tmp_path)
    sessions = test_env[3]
    a_in_collection_step = threading.Event()
    b_rolled_back = threading.Event()
    waited: list[bool] = []
    original_restore = kb_module._restore_ingest_file_backup

    def _restore(**kw):
        try:
            return original_restore(**kw)
        finally:
            if Path(kw["file_path"]).name.startswith("b__"):
                b_rolled_back.set()

    def _ingest(**kw):
        if _is_file(kw, "a__"):
            return _failed()
        waited.append(a_in_collection_step.wait(8))
        return IngestionResult(
            status="partial",
            doc_id="doc-b",
            completed_steps=[
                {"name": "register_document", "metadata": {"created": True}}
            ],
            message="embedding failed",
        )

    def _hold() -> None:
        a_in_collection_step.set()
        waited.append(b_rolled_back.wait(8))

    response = _post_cloud(
        test_env,
        [FILE_A, FILE_B],
        _ingest,
        patch("xagent.web.api.kb.get_collection_sync", side_effect=ValueError("new")),
        patch(
            "xagent.web.api.kb.delete_collection",
            return_value=SimpleNamespace(status="success", message=""),
        ),
        patch(
            "xagent.web.api.kb.delete_document",
            return_value=SimpleNamespace(status="error", message="lance busy"),
        ),
        patch(METADATA_STORE, return_value=_MetadataStore(_hold)),
        patch("xagent.web.api.kb._restore_ingest_file_backup", side_effect=_restore),
    )

    a_entry, b_entry = response.json()
    assert a_entry["message"] == "embedding down"
    assert b_entry["message"].startswith("Failed to fully roll back cloud ingest")
    assert len(waited) >= 2 and all(waited)
    assert "a.csv" not in _rows(sessions)
    assert _rows_without_bytes(sessions) == []


def test_cloud_sibling_upsert_is_not_blocked_by_a_rollback_in_progress(
    test_env, temp_uploads, monkeypatch, tmp_path
) -> None:
    _durable_storage(monkeypatch, tmp_path)
    a_in_collection_step = threading.Event()
    b_ingested = threading.Event()
    waited: list[bool] = []

    def _gate(file_id: str) -> None:
        if file_id == "drive-B":
            waited.append(a_in_collection_step.wait(8))

    def _hold() -> None:
        a_in_collection_step.set()
        waited.append(b_ingested.wait(8))

    def _ingest(**kw):
        if _is_file(kw, "a__"):
            return _failed()
        b_ingested.set()
        return IngestionResult(status="success", doc_id="doc-b", message="ok")

    response = _post_cloud(
        test_env,
        [FILE_A, FILE_B],
        _ingest,
        patch("xagent.web.api.kb.get_collection_sync", side_effect=ValueError("new")),
        patch(
            "xagent.web.api.kb.delete_collection",
            return_value=SimpleNamespace(status="success", message=""),
        ),
        patch(METADATA_STORE, return_value=_MetadataStore(_hold)),
        downloader_gate=_gate,
    )

    a_entry, b_entry = response.json()
    assert a_entry["message"] == "embedding down"
    assert (b_entry["status"], b_entry["message"]) == ("success", "ok")
    assert len(waited) >= 2 and all(waited)


def test_local_whole_collection_metadata_failure_leaves_no_row_without_bytes(
    test_env, temp_uploads, monkeypatch, tmp_path
) -> None:
    _durable_storage(monkeypatch, tmp_path)

    response = local._post_ingest(
        test_env,
        "x.txt",
        "fresh",
        patch("xagent.web.api.kb.run_document_ingestion", side_effect=_failed),
        patch(
            "xagent.web.api.kb.delete_collection",
            return_value=SimpleNamespace(status="success", message=""),
        ),
        patch(METADATA_STORE, return_value=_MetadataStore()),
        patch(
            "xagent.web.api.kb._cleanup_failed_new_collection_metadata",
            AsyncMock(side_effect=RuntimeError("metadata store down")),
        ),
    )

    assert response.status_code == 500
    assert "metadata store down" in response.json()["detail"]
    assert _rows(test_env[3]) == {}


def _seed(test_env, uploads, durable_by_name, collection="", owner_id=None):
    _app, _headers, user, sessions = test_env
    owner_id = owner_id or int(user.id)
    db = sessions()
    try:
        for name, durable in durable_by_name.items():
            path = uploads / f"user_{owner_id}" / collection / name
            path.parent.mkdir(parents=True, exist_ok=True)
            path.write_text(name)
            record = UploadedFile(
                user_id=owner_id,
                filename=name,
                # Resolved, as the directory pass compares against a resolved dir.
                storage_path=str(path.resolve()),
                mime_type="text/plain",
                file_size=path.stat().st_size,
            )
            db.add(record)
            db.flush()
            if durable:
                ManagedFileRef(record).sync_to_durable()
        db.commit()
    finally:
        db.close()
    return {n: row for n, row in _rows(sessions).items() if n in durable_by_name}


def _collection_store(documents: list[DocumentRecord]) -> list[Any]:
    store = MagicMock()
    store.list_document_records.side_effect = lambda *_a, **_kw: list(documents)

    def _delete_collection(*_args, **_kwargs):
        documents.clear()
        return CollectionOperationResult(
            status="success",
            collection="demo",
            message="deleted",
            affected_documents=[],
            deleted_counts={},
        )

    return [
        patch("xagent.web.api.kb.get_vector_index_store", return_value=store),
        patch(
            "xagent.core.tools.core.RAG_tools.storage.factory.get_vector_index_store",
            return_value=store,
        ),
        patch("xagent.web.api.kb.delete_collection", side_effect=_delete_collection),
    ]


def _delete_collection_api(test_env, documented, *extra):
    app, headers, _user, _ = test_env
    documents = [
        DocumentRecord(doc_id=name, file_id=file_id, source_path="x")
        for name, (file_id, _, _) in documented.items()
    ]
    with ExitStack() as stack:
        for p in (
            *_collection_store(documents),
            patch(
                "xagent.web.api.kb._ensure_collection_access", new_callable=AsyncMock
            ),
            *extra,
        ):
            stack.enter_context(p)
        return TestClient(app).delete("/api/kb/collections/demo", headers=headers)


def test_collection_delete_byte_failure_leaves_no_row_without_bytes(
    test_env, temp_uploads, monkeypatch, tmp_path, caplog
) -> None:
    _durable_storage(monkeypatch, tmp_path)
    rows = _seed(test_env, temp_uploads, {"one.txt": True, "two.txt": True})
    real_delete = FsspecFileStorage.delete
    failed: list[str] = []

    def _first_delete_fails(self, key: str) -> None:
        if not failed:
            failed.append(key)
            raise RuntimeError("object store 503")
        real_delete(self, key)

    monkeypatch.setattr(FsspecFileStorage, "delete", _first_delete_fails)
    response = _delete_collection_api(test_env, rows)

    assert response.status_code == 200
    assert _rows(test_env[3]) == {}
    storage = get_unscoped_file_storage()
    assert [key for _, key, _ in rows.values() if storage.exists(key)] == failed
    assert _byte_delete_warnings(caplog) == [
        f"Failed to delete file bytes for {failed[0]} after commit"
    ]


def test_collection_delete_failed_commit_keeps_every_file_byte(
    test_env, temp_uploads, monkeypatch, tmp_path
) -> None:
    _durable_storage(monkeypatch, tmp_path)
    rows = _seed(test_env, temp_uploads, {"legacy.txt": False, "durable.txt": True})
    # No document names this row; the directory pass finds it by path.
    in_dir = _seed(test_env, temp_uploads, {"in_dir.txt": True}, collection="demo")
    real_cleanup = kb_module.delete_collection_uploaded_files

    def _cleanup_then_fail_commit(db, **kwargs):
        db.commit = _broken_commit
        return real_cleanup(db, **kwargs)

    response = _delete_collection_api(
        test_env,
        rows,
        patch(
            "xagent.web.api.kb.delete_collection_uploaded_files",
            side_effect=_cleanup_then_fail_commit,
        ),
    )

    assert response.status_code == 500
    assert _rows(test_env[3]) == {**rows, **in_dir}
    storage = get_unscoped_file_storage()
    assert storage.exists(in_dir["in_dir.txt"][1])
    assert storage.exists(rows["durable.txt"][1])
    assert Path(rows["legacy.txt"][2]).exists()


def test_collection_delete_commits_every_owners_row_deletes(
    test_env, temp_uploads, monkeypatch, tmp_path
) -> None:
    _durable_storage(monkeypatch, tmp_path)
    user_id = int(test_env[2].id)
    for owner_id in (user_id, user_id + 1):
        _seed(test_env, temp_uploads, {f"{owner_id}.txt": True}, "demo", owner_id)
    db = test_env[3]()
    try:
        with ExitStack() as stack:
            for p in _collection_store([]):
                stack.enter_context(p)
            kb_module._perform_kb_collection_delete("demo", user_id, True, db)
    finally:
        db.close()

    assert _rows(test_env[3]) == {}


def test_run_after_commit_runs_every_delete_and_logs_each_failure(caplog) -> None:
    ran: list[str] = []

    def _fail(name: str) -> Any:
        def _action() -> None:
            ran.append(name)
            raise OSError(name)

        return (f"key-{name}", _action)

    with pytest.raises(OSError, match="first"):
        kb_module._run_after_commit([_fail("first"), _fail("second")])

    assert ran == ["first", "second"]
    assert _byte_delete_warnings(caplog) == [
        "Failed to delete file bytes for key-first after commit",
        "Failed to delete file bytes for key-second after commit",
    ]


@pytest.mark.parametrize("helper", ["store", "orphan"])
def test_queued_local_unlink_waits_for_the_commit(
    helper, test_env, temp_uploads, monkeypatch, tmp_path
) -> None:
    _durable_storage(monkeypatch, tmp_path)
    file_id, key, path = _seed(test_env, temp_uploads, {"x.txt": True})["x.txt"]
    storage = get_unscoped_file_storage()
    after_commit: list[Any] = []
    db = test_env[3]()
    try:
        if helper == "store":
            UploadedFileStore(db).delete(
                db.query(UploadedFile).one(), after_commit=after_commit
            )
        else:
            kb_module._delete_uploaded_file_if_orphaned(
                db,
                file_id=file_id,
                user_id=int(test_env[2].id),
                remaining_file_ids=set(),
                after_commit=after_commit,
            )
        assert Path(path).exists()
        db.commit()
        assert Path(path).exists() and storage.exists(key)
        assert sorted(label for label, _ in after_commit) == sorted([key, file_id])
        kb_module._run_after_commit(after_commit)
    finally:
        db.close()

    assert not Path(path).exists()
    assert not storage.exists(key)


def test_cloud_byte_delete_failure_after_commit_reports_incomplete_rollback(
    test_env, temp_uploads, monkeypatch, tmp_path
) -> None:
    _durable_storage(monkeypatch, tmp_path)
    sessions = test_env[3]
    seen: dict[str, Any] = {}

    def _ingest(**kw):
        seen.update(_rows(sessions))
        return _failed()

    def _broken_delete(self, key: str) -> None:
        raise RuntimeError("object store 503")

    monkeypatch.setattr(FsspecFileStorage, "delete", _broken_delete)
    response = _post_cloud(
        test_env,
        [FILE_A],
        _ingest,
        patch("xagent.web.api.kb.get_collection_sync", return_value=object()),
    )

    assert "object store 503" in response.json()[0]["message"]
    assert "a.csv" not in _rows(sessions)
    assert get_unscoped_file_storage().exists(seen["a.csv"][1])