Build and run agents you can see, understand and trust.
Python agentchatbotlarge-language-modelsllmllm-agentmcpmulti-agentmulti-modalreact-agentrealtime-agentvoice-agentvoice-assistant
fix(rag): load milvus collection before reads (#3433)
aqi27c52cc65088
diff --git a/src/agentscope/rag/_vdb/_milvus_lite.py b/src/agentscope/rag/_vdb/_milvus_lite.pyindex 7f344b10a..f901e7b9c 100644--- a/src/agentscope/rag/_vdb/_milvus_lite.py+++ b/src/agentscope/rag/_vdb/_milvus_lite.py@@ -194,6 +194,31 @@ class MilvusLiteStore(VectorStoreBase):collection_name=name,)+ async def _ensure_loaded(+ self,+ client: "MilvusClient",+ collection: str,+ ) -> None:+ """Make sure a collection is loaded before reading from it.++ Milvus may leave a collection in the ``released`` state — for+ example after the server (or, for Milvus Lite, the host process)+ restarts. Searching or querying a released collection fails with+ ``MilvusException: Collection ... is in state 'released'; call+ load() before search/get/query``. ``load_collection`` is+ idempotent, so calling it before every read is safe.++ Args:+ client (`MilvusClient`):+ The client bound to this store.+ collection (`str`):+ The collection that is about to be read.+ """+ await asyncio.to_thread(+ client.load_collection,+ collection_name=collection,+ )+# ------------------------------------------------------------------# Data operations# ------------------------------------------------------------------@@ -321,8 +346,11 @@ class MilvusLiteStore(VectorStoreBase):cosine / inner product metrics, or ascending distancesemantics for L2 as exposed by Milvus."""+ client = self.get_client()+ await self._ensure_loaded(client, collection)+response = await asyncio.to_thread(- self.get_client().search,+ client.search,collection_name=collection,data=[query_vector],anns_field="vector",@@ -370,6 +398,7 @@ class MilvusLiteStore(VectorStoreBase):One summary per distinct ``document_id``."""client = self.get_client()+ await self._ensure_loaded(client, collection)rows = await asyncio.to_thread(self._query_all_rows,client,@@ -503,8 +532,10 @@ class MilvusLiteStore(VectorStoreBase):# deterministic entity IDs may hold duplicates from indexing# retries, and truncating at `limit` raw rows could then drop a# low index while returning a duplicate of a higher one.+ client = self.get_client()+ await self._ensure_loaded(client, collection)rows = await asyncio.to_thread(- self.get_client().query,+ client.query,collection_name=collection,filter=" and ".join(clauses),output_fields=["chunk"],diff --git a/tests/rag_vdb_milvus_lite_test.py b/tests/rag_vdb_milvus_lite_test.pyindex 2f15405f5..5b9a68014 100644--- a/tests/rag_vdb_milvus_lite_test.py+++ b/tests/rag_vdb_milvus_lite_test.py@@ -7,13 +7,15 @@ from importlib.util import find_specfrom pathlib import Pathfrom types import SimpleNamespacefrom unittest.async_case import IsolatedAsyncioTestCase-from unittest.mock import Mock, patch+from unittest.mock import AsyncMock, Mock, call, patchfrom utils import AnyString+from agentscope.embedding import EmbeddingModelBase, EmbeddingResponsefrom agentscope.message import TextBlockfrom agentscope.rag import (Chunk,+ KnowledgeBase,MilvusLiteStore,VectorRecord,VectorSearchResult,@@ -401,31 +403,66 @@ class MilvusLiteStoreTest(IsolatedAsyncioTestCase):self.assertEqual([s.document_id for s in summaries], ["doc-2"])async def test_persists_records_after_reopen(self) -> None:- """Records remain available after closing and reopening the DB."""+ """KnowledgeBase reads load existing collections after reopening."""db_path = self._tmpdir / "persistent.db"+ record = _make_record(+ "Persisted",+ [1.0, 0.0, 0.0],+ document_id="doc-1",+ )+ embedding = AsyncMock(spec=EmbeddingModelBase)+ embedding.dimensions = 3+ embedding.supports_multimodal = False+ embedding.return_value = EmbeddingResponse(+ embeddings=[record.vector],+ )async with MilvusLiteStore(uri=str(db_path)) as first_store:await first_store.create_collection("kb_persistent", dimensions=3)- await first_store.insert(- "kb_persistent",+ await first_store.insert("kb_persistent", [record])++ cases = [+ (+ "search",+ (["Persisted"],),[- _make_record(- "Persisted",- [1.0, 0.0, 0.0],- document_id="doc-1",- ),+ {+ "score": 1.0,+ "document_id": "doc-1",+ "chunk": record.chunk.model_dump(),+ },],- )-- async with MilvusLiteStore(uri=str(db_path)) as second_store:- await second_store.create_collection("kb_persistent", dimensions=3)- results = await second_store.search(- "kb_persistent",- query_vector=[1.0, 0.0, 0.0],- top_k=1,- )-- self.assertEqual([r.document_id for r in results], ["doc-1"])+ ),+ (+ "list_documents",+ (),+ [+ {+ "document_id": "doc-1",+ "source": "doc-1.txt",+ "chunk_count": 1,+ "metadata": {},+ },+ ],+ ),+ ("list_chunks", ("doc-1",), [record.chunk.model_dump()]),+ ]+ for method, args, expected in cases:+ with self.subTest(method=method):+ # Reopen for each read so earlier reads cannot load it.+ async with MilvusLiteStore(uri=str(db_path)) as second_store:+ knowledge = KnowledgeBase(+ name="persistent",+ description="Persisted test documents",+ embedding_model=embedding,+ vector_store=second_store,+ collection="kb_persistent",+ )+ results = await getattr(knowledge, method)(*args)+ self.assertEqual(+ [result.model_dump() for result in results],+ expected,+ )async def test_list_chunks_pagination_and_isolation(self) -> None:"""list_chunks pages by chunk_index and isolates documents."""@@ -615,3 +652,80 @@ class MilvusDeleteFilterTest(IsolatedAsyncioTestCase):collection_name="shared",filter='document_id == "doc"',)+++class MilvusReadLoadsCollectionTest(IsolatedAsyncioTestCase):+ """Read paths load the collection before touching Milvus.++ Milvus can leave a collection in the ``released`` state, e.g. after the+ host process restarts; searching or querying it then raises+ ``Collection ... is in state 'released'``. The read helpers must+ therefore call ``load_collection`` first. A mocked client keeps these+ tests independent of a running Milvus service.+ """++ async def asyncSetUp(self) -> None:+ """Create a store without connecting to Milvus."""+ self.store = MilvusLiteStore()+ self.client = Mock()++ async def test_search_loads_collection_first(self) -> None:+ """search() loads the collection before issuing the search."""+ self.client.search.return_value = [[]]+ with patch.object(self.store, "get_client", return_value=self.client):+ await self.store.search("kb_1", query_vector=[1.0, 0.0, 0.0])++ self.assertEqual(+ [call[0] for call in self.client.method_calls],+ ["load_collection", "search"],+ )+ self.client.load_collection.assert_called_once_with(+ collection_name="kb_1",+ )++ async def test_list_documents_loads_collection_first(self) -> None:+ """list_documents() loads the collection before querying it."""+ self.client.query_rows.return_value = []+ with (+ patch.object(self.store, "get_client", return_value=self.client),+ patch.object(+ self.store,+ "_query_all_rows",+ self.client.query_rows,+ ),+ ):+ self.assertEqual(await self.store.list_documents("kb_1"), [])++ self.assertEqual(+ self.client.method_calls,+ [+ call.load_collection(collection_name="kb_1"),+ call.query_rows(self.client, "kb_1", None),+ ],+ )++ async def test_list_chunks_loads_collection_first(self) -> None:+ """list_chunks() loads the collection before querying its page."""+ self.client.query.return_value = []+ with patch.object(self.store, "get_client", return_value=self.client):+ self.assertEqual(+ await self.store.list_chunks("kb_1", "doc-1"),+ [],+ )++ self.assertEqual(+ self.client.method_calls,+ [+ call.load_collection(collection_name="kb_1"),+ call.query(+ collection_name="kb_1",+ filter=(+ 'document_id == "doc-1" and '+ 'chunk["chunk_index"] >= 0 and '+ 'chunk["chunk_index"] < 30'+ ),+ output_fields=["chunk"],+ limit=16_384,+ ),+ ],+ )