Skip to content

Build and run agents you can see, understand and trust.

fix(app): clean up source blob when document registration is cancelled (#3657)

rome-xidff94ffd9a51
diff --git a/src/agentscope/app/_service/_knowledge_base.py b/src/agentscope/app/_service/_knowledge_base.py
index 7856c1f71..091f5a21a 100644
--- a/src/agentscope/app/_service/_knowledge_base.py
+++ b/src/agentscope/app/_service/_knowledge_base.py
@@ -17,6 +17,7 @@ with service-mode listing would leave records out of sync, and the
project's stance is that a knowledge base is managed end-to-end in one
mode.
"""
+import asyncio
import uuid
from collections import Counter
from typing import IO, TYPE_CHECKING, AsyncIterator
@@ -428,10 +429,15 @@ class KnowledgeBaseService:
owner_id,
record,
)
- except Exception:
- # Storage write failed — drop the blob so the orphan
- # sweeper doesn't later see a referenced-by-nobody file.
- await self._delete_blob_quietly(blob_uri)
+ except (Exception, asyncio.CancelledError):
+ # Storage write failed or cancelled — drop the blob only if
+ # the record was not persisted, keeping it if already committed.
+ await self._cleanup_unpersisted_blob(
+ owner_id,
+ knowledge_base_id,
+ document_id,
+ blob_uri,
+ )
raise
await enqueue_index_task(
@@ -885,6 +891,46 @@ class KnowledgeBaseService:
detail=str(exc),
) from exc
+ async def _cleanup_unpersisted_blob(
+ self,
+ user_id: str,
+ knowledge_base_id: str,
+ document_id: str,
+ blob_uri: str,
+ ) -> None:
+ """Clean up the blob only if storage persistence did not commit.
+
+ Preserve the source file if a persisted record references it.
+ If the lookup fails, log the error and keep the file.
+ """
+
+ async def _check_and_delete() -> None:
+ try:
+ persisted = await self._storage.get_knowledge_document(
+ user_id,
+ knowledge_base_id,
+ document_id,
+ )
+ except Exception:
+ logger.exception(
+ "Failed to verify document %s persistence; keeping blob "
+ "%s",
+ document_id,
+ blob_uri,
+ )
+ return
+
+ if persisted is None:
+ try:
+ await self._blob_store.delete(blob_uri)
+ except Exception:
+ logger.exception(
+ "Failed to delete blob %s",
+ blob_uri,
+ )
+
+ await asyncio.shield(_check_and_delete())
+
async def _delete_blob_quietly(self, blob_uri: str) -> None:
"""Best-effort blob delete — swallow backend errors.
@@ -892,10 +938,14 @@ class KnowledgeBaseService:
record/vector-store state is still consistent and a future
sweep can reclaim the disk space. Surface only via logs.
"""
- try:
- await self._blob_store.delete(blob_uri)
- except Exception: # noqa: BLE001 — cleanup only
- logger.exception(
- "Failed to delete blob %s",
- blob_uri,
- )
+
+ async def _delete() -> None:
+ try:
+ await self._blob_store.delete(blob_uri)
+ except Exception:
+ logger.exception(
+ "Failed to delete blob %s",
+ blob_uri,
+ )
+
+ await asyncio.shield(_delete())
diff --git a/tests/service_knowledge_base_upload_test.py b/tests/service_knowledge_base_upload_test.py
index 9fc5f2638..2b9ccfbe5 100644
--- a/tests/service_knowledge_base_upload_test.py
+++ b/tests/service_knowledge_base_upload_test.py
@@ -18,7 +18,8 @@ delete flow through ``TestClient``. Verifies that:
import asyncio
import io
import tempfile
-from typing import Any
+from contextlib import asynccontextmanager
+from typing import Any, AsyncIterator
from unittest.async_case import IsolatedAsyncioTestCase
from unittest.mock import patch
@@ -26,6 +27,7 @@ import fakeredis.aioredis
from fastapi.testclient import TestClient
from agentscope.app import create_app
+from agentscope.app._service import KnowledgeBaseService, ResourceAccessService
from agentscope.app.rag.blob_store import LocalBlobStore
from agentscope.app.rag.knowledge_base_manager import (
KnowledgeBaseManagerBase,
@@ -607,3 +609,177 @@ class KnowledgeBaseUploadFlowTest(IsolatedAsyncioTestCase):
document.data.blob_uri,
),
)
+
+ @asynccontextmanager
+ async def _paused_upload(
+ self,
+ target: Any,
+ method: str,
+ *,
+ after: bool = False,
+ ) -> AsyncIterator[
+ tuple[asyncio.Task[KnowledgeDocumentRecord], KnowledgeDocumentRecord]
+ ]:
+ """Pause a real registration before or after a storage operation."""
+ state = self._app.state
+ storage = state.storage
+ service = KnowledgeBaseService(
+ storage,
+ state.knowledge_base_manager,
+ state.blob_store,
+ state.message_bus,
+ ResourceAccessService(storage, state.resource_access_policy),
+ )
+ reached = asyncio.Event()
+ original = getattr(target, method)
+
+ async def pause(*args: Any, **kwargs: Any) -> None:
+ if after:
+ await original(*args, **kwargs)
+ reached.set()
+ await asyncio.Event().wait()
+
+ async with storage:
+ with (
+ patch.object(target, method, side_effect=pause),
+ patch.object(
+ storage,
+ "upsert_knowledge_document",
+ wraps=storage.upsert_knowledge_document,
+ ) as write,
+ ):
+ task = asyncio.create_task(
+ service.register_document(
+ "user-1",
+ self._kb_id,
+ "cancel.txt",
+ io.BytesIO(b"abc"),
+ 3,
+ ),
+ )
+ try:
+ await asyncio.wait_for(reached.wait(), timeout=5)
+ record = write.await_args.args[1]
+ self.assertTrue(
+ await state.blob_store.exists(record.data.blob_uri),
+ )
+ yield task, record
+ finally:
+ task.cancel()
+ await asyncio.gather(task, return_exceptions=True)
+
+ async def _cancel_upload(
+ self,
+ task: asyncio.Task[KnowledgeDocumentRecord],
+ ) -> None:
+ task.cancel()
+ with self.assertRaises(asyncio.CancelledError):
+ await asyncio.wait_for(asyncio.shield(task), timeout=5)
+
+ async def test_cancelled_registration_cleans_up_source_blob(self) -> None:
+ """Cancellation before persistence deletes the source blob."""
+ state = self._app.state
+ async with self._paused_upload(
+ state.storage,
+ "upsert_knowledge_document",
+ ) as (task, record):
+ await self._cancel_upload(task)
+ self.assertFalse(
+ await state.blob_store.exists(record.data.blob_uri),
+ )
+
+ async def test_repeated_cancellation_during_delete_cleans_up_source_blob(
+ self,
+ ) -> None:
+ """A second cancellation during delete must not interrupt cleanup."""
+ state = self._app.state
+ started = asyncio.Event()
+ resume = asyncio.Event()
+ done = asyncio.Event()
+ delete = state.blob_store.delete
+
+ async def slow_delete(uri: str) -> None:
+ started.set()
+ try:
+ await resume.wait()
+ await delete(uri)
+ finally:
+ done.set()
+
+ async with self._paused_upload(
+ state.storage,
+ "upsert_knowledge_document",
+ ) as (task, record):
+ with patch.object(
+ state.blob_store,
+ "delete",
+ side_effect=slow_delete,
+ ):
+ task.cancel()
+ try:
+ await asyncio.wait_for(started.wait(), timeout=5)
+ await self._cancel_upload(task)
+ finally:
+ resume.set()
+ await asyncio.wait_for(done.wait(), timeout=5)
+ self.assertFalse(
+ await state.blob_store.exists(record.data.blob_uri),
+ )
+
+ async def test_cancellation_after_storage_upsert_preserves_source_blob(
+ self,
+ ) -> None:
+ """A fully persisted record retains its source file."""
+ await self._assert_persisted_blob(
+ self._app.state.storage,
+ "upsert_knowledge_document",
+ after=True,
+ )
+
+ async def test_cancellation_after_redis_set_preserves_source_blob(
+ self,
+ ) -> None:
+ """The document SET can commit before the first index SADD."""
+ await self._assert_persisted_blob(self._fr, "sadd")
+
+ async def _assert_persisted_blob(
+ self,
+ target: Any,
+ method: str,
+ *,
+ after: bool = False,
+ ) -> None:
+ state = self._app.state
+ async with self._paused_upload(target, method, after=after) as (
+ task,
+ record,
+ ):
+ await self._cancel_upload(task)
+ stored = await state.storage.get_knowledge_document(
+ "user-1",
+ self._kb_id,
+ record.id,
+ )
+ self.assertEqual(stored, record)
+ self.assertTrue(
+ await state.blob_store.exists(record.data.blob_uri),
+ )
+
+ async def test_cancellation_lookup_failure_preserves_source_blob(
+ self,
+ ) -> None:
+ """An inconclusive lookup must leave the file intact."""
+ state = self._app.state
+ async with self._paused_upload(
+ state.storage,
+ "upsert_knowledge_document",
+ ) as (task, record):
+ with patch.object(
+ state.storage,
+ "get_knowledge_document",
+ side_effect=RuntimeError("lookup failed"),
+ ):
+ await self._cancel_upload(task)
+ self.assertTrue(
+ await state.blob_store.exists(record.data.blob_uri),
+ )