Build and run agents you can see, understand and trust.
Python agentchatbotlarge-language-modelsllmllm-agentmcpmulti-agentmulti-modalreact-agentrealtime-agentvoice-agentvoice-assistant
feat: Add support for milvus lite vector database in the RAG module of AgentScope (#825)
AIGCoole900d9cbf7c7
diff --git a/examples/functionality/vector_store/milvus_lite/README.md b/examples/functionality/vector_store/milvus_lite/README.mdnew file mode 100644index 000000000..af220cb47--- /dev/null+++ b/examples/functionality/vector_store/milvus_lite/README.md@@ -0,0 +1,128 @@+# MilvusLite Vector Store++This example demonstrates how to use **MilvusLiteStore** for vector storage and semantic search in AgentScope.+It includes four test scenarios covering CRUD operations, metadata filtering, document chunking, and distance metrics.++### Quick Start++Install agentscope first, and then the MilvusLite dependency:++```bash+# In MacOS/Linux+pip install pymilvus\[milvus_lite\]++# In Windows+pip install pymilvus[milvus_lite]+```++Run the example script, which showcases adding, searching with/without filters in MilvusLite vector store:++```bash+python milvuslite_store.py+```++> **Note:** The script creates `.db` files in the current directory. You can delete them after testing.++## Usage++### Initialize Store+```python+from agentscope.rag import MilvusLiteStore++store = MilvusLiteStore(+ uri="./milvus_test.db",+ collection_name="test_collection",+ dimensions=768, # Match your embedding model+ distance="COSINE", # COSINE, L2, or IP+)+```++### Add Documents++```python+from agentscope.rag import Document, DocMetadata+from agentscope.message import TextBlock++doc = Document(+ metadata=DocMetadata(+ content=TextBlock(type="text", text="Your document text"),+ doc_id="doc_1",+ chunk_id=0,+ total_chunks=1,+ ),+ embedding=[0.1, 0.2, ...], # Your embedding vector+)++await store.add([doc])+```++### Search++```python+results = await store.search(+ query_embedding=[0.15, 0.25, ...],+ limit=5,+ score_threshold=0.9, # Optional+ filter='doc_id like "prefix%"', # Optional+)+```++### Delete++```python+await store.delete(filter_expr='doc_id == "doc_1"')+```++## Distance Metrics++| Metric | Description | Best For |+|--------|-------------|----------|+| **COSINE** | Cosine similarity | Text embeddings (recommended) |+| **L2** | Euclidean distance | Spatial data |+| **IP** | Inner Product | Recommendation systems |++## Filter Expressions++```python+# Exact match+filter='doc_id == "doc_1"'++# Pattern matching+filter='doc_id like "prefix%"'++# Numeric and logical operators+filter='chunk_id >= 0 and total_chunks > 1'+```++## Advanced Usage++### Access Underlying Client+```python+client = store.get_client()+stats = client.get_collection_stats(collection_name="test_collection")+```++### Document Metadata+- `content`: Text content (TextBlock)+- `doc_id`: Unique document identifier+- `chunk_id`: Chunk position (0-indexed)+- `total_chunks`: Total chunks in document++## FAQ++**What embedding dimension should I use?**+Match your embedding model's output dimension (e.g., 768 for BERT, 1536 for OpenAI ada-002).++**Can I change the distance metric after creation?**+No, create a new collection with the desired metric.++**How do I delete the database?**+Delete the `.db` file specified in the `uri` parameter.++**Is this suitable for production?**+MilvusLite works well for development and small-scale applications. For production at scale, consider Milvus standalone or cluster mode.++## References++- [Milvus Documentation](https://milvus.io/docs)+- [AgentScope RAG Tutorial](https://doc.agentscope.io/tutorial/task_rag.html)diff --git a/examples/functionality/vector_store/milvus_lite/main.py b/examples/functionality/vector_store/milvus_lite/main.pynew file mode 100644index 000000000..57cbe7faa--- /dev/null+++ b/examples/functionality/vector_store/milvus_lite/main.py@@ -0,0 +1,327 @@+# -*- coding: utf-8 -*-+"""Example of using MilvusLiteStore in AgentScope RAG system."""+import asyncio+from agentscope.rag import (+ MilvusLiteStore,+ Document,+ DocMetadata,+)+from agentscope.message import TextBlock+++async def example_basic_operations() -> None:+ """The example of basic CRUD operations with MilvusLiteStore."""+ print("\n" + "=" * 60)+ print("Test 1: Basic CRUD Operations")+ print("=" * 60)++ # Initialize MilvusLiteStore with a local file+ store = MilvusLiteStore(+ uri="./milvus_test.db",+ collection_name="test_collection",+ dimensions=4, # Small dimension for testing+ distance="COSINE",+ )++ print("✓ MilvusLiteStore initialized")++ # Create test documents with embeddings+ test_docs = [+ Document(+ metadata=DocMetadata(+ content=TextBlock(+ text="Artificial Intelligence is the future",+ ),+ doc_id="doc_1",+ chunk_id=0,+ total_chunks=1,+ ),+ embedding=[0.1, 0.2, 0.3, 0.4],+ ),+ Document(+ metadata=DocMetadata(+ content=TextBlock(text="Machine Learning is a subset of AI"),+ doc_id="doc_2",+ chunk_id=0,+ total_chunks=1,+ ),+ embedding=[0.2, 0.3, 0.4, 0.5],+ ),+ Document(+ metadata=DocMetadata(+ content=TextBlock(text="Deep Learning uses neural networks"),+ doc_id="doc_3",+ chunk_id=0,+ total_chunks=1,+ ),+ embedding=[0.3, 0.4, 0.5, 0.6],+ ),+ ]++ # Test add operation+ await store.add(test_docs)+ print(f"✓ Added {len(test_docs)} documents to the store")++ # Test search operation+ query_embedding = [0.15, 0.25, 0.35, 0.45]+ results = await store.search(+ query_embedding=query_embedding,+ limit=2,+ )++ print(f"\n✓ Search completed, found {len(results)} results:")+ for i, result in enumerate(results, 1):+ print(f" {i}. Score: {result.score:.4f}")+ print(f" Content: {result.metadata.content}")+ print(f" Doc ID: {result.metadata.doc_id}")++ # Test search with score threshold+ results_filtered = await store.search(+ query_embedding=query_embedding,+ limit=5,+ score_threshold=0.9,+ )+ print(f"\n✓ Search with threshold (>0.9): {len(results_filtered)} results")++ # Test delete operation+ # Note: We need to use filter expression to delete by doc_id+ await store.delete(filter='doc_id == "doc_2"')+ print("\n✓ Deleted document with doc_id='doc_2'")++ # Verify deletion+ results_after_delete = await store.search(+ query_embedding=query_embedding,+ limit=5,+ )+ print(f"✓ After deletion: {len(results_after_delete)} documents remain")++ # Get client for advanced operations+ client = store.get_client()+ print(f"\n✓ Got MilvusClient: {type(client).__name__}")+++async def example_filter_search() -> None:+ """The example of search with metadata filtering."""+ print("\n" + "=" * 60)+ print("Test 2: Search with Metadata Filtering")+ print("=" * 60)++ store = MilvusLiteStore(+ uri="./milvus_filter_test.db",+ collection_name="filter_collection",+ dimensions=4,+ distance="COSINE",+ )++ # Create documents with different categories+ docs = [+ Document(+ metadata=DocMetadata(+ content=TextBlock(text="Python is a programming language"),+ doc_id="prog_1",+ chunk_id=0,+ total_chunks=1,+ ),+ embedding=[0.1, 0.2, 0.3, 0.4],+ ),+ Document(+ metadata=DocMetadata(+ content=TextBlock(+ text="Java is used for enterprise applications",+ ),+ doc_id="prog_2",+ chunk_id=0,+ total_chunks=1,+ ),+ embedding=[0.2, 0.3, 0.4, 0.5],+ ),+ Document(+ metadata=DocMetadata(+ content=TextBlock(text="Neural networks are used in AI"),+ doc_id="ai_1",+ chunk_id=0,+ total_chunks=1,+ ),+ embedding=[0.3, 0.4, 0.5, 0.6],+ ),+ Document(+ metadata=DocMetadata(+ content=TextBlock(text="Deep learning requires GPUs"),+ doc_id="ai_2",+ chunk_id=0,+ total_chunks=1,+ ),+ embedding=[0.4, 0.5, 0.6, 0.7],+ ),+ ]++ await store.add(docs)+ print(f"✓ Added {len(docs)} documents with different doc_id prefixes")++ # Search without filter+ query_embedding = [0.25, 0.35, 0.45, 0.55]+ all_results = await store.search(+ query_embedding=query_embedding,+ limit=4,+ )+ print(f"\n✓ Search without filter: {len(all_results)} results")+ for i, result in enumerate(all_results, 1):+ doc_id = result.metadata.doc_id+ score = result.score+ print(f" {i}. Doc ID: {doc_id}, Score: {score:.4f}")++ # Search with filter for programming docs+ prog_results = await store.search(+ query_embedding=query_embedding,+ limit=4,+ filter='doc_id like "prog%"',+ )+ filter_msg = "doc_id like 'prog%'"+ print(f"\n✓ Search with filter ({filter_msg}): {len(prog_results)}")+ for i, result in enumerate(prog_results, 1):+ doc_id = result.metadata.doc_id+ score = result.score+ print(f" {i}. Doc ID: {doc_id}, Score: {score:.4f}")++ # Search with filter for AI docs+ ai_results = await store.search(+ query_embedding=query_embedding,+ limit=4,+ filter='doc_id like "ai%"',+ )+ filter_msg = "doc_id like 'ai%'"+ print(f"\n✓ Search with filter ({filter_msg}): {len(ai_results)}")+ for i, result in enumerate(ai_results, 1):+ doc_id = result.metadata.doc_id+ score = result.score+ print(f" {i}. Doc ID: {doc_id}, Score: {score:.4f}")+++async def example_multiple_chunks() -> None:+ """The example of documents with multiple chunks."""+ print("\n" + "=" * 60)+ print("Test 3: Documents with Multiple Chunks")+ print("=" * 60)++ store = MilvusLiteStore(+ uri="./milvus_chunks_test.db",+ collection_name="chunks_collection",+ dimensions=4,+ distance="COSINE",+ )++ # Create a document split into multiple chunks+ chunks = [+ Document(+ metadata=DocMetadata(+ content=TextBlock(text="Chapter 1: Introduction to AI"),+ doc_id="book_1",+ chunk_id=0,+ total_chunks=3,+ ),+ embedding=[0.1, 0.2, 0.3, 0.4],+ ),+ Document(+ metadata=DocMetadata(+ content=TextBlock(text="Chapter 2: Machine Learning Basics"),+ doc_id="book_1",+ chunk_id=1,+ total_chunks=3,+ ),+ embedding=[0.2, 0.3, 0.4, 0.5],+ ),+ Document(+ metadata=DocMetadata(+ content=TextBlock(text="Chapter 3: Deep Learning Advanced"),+ doc_id="book_1",+ chunk_id=2,+ total_chunks=3,+ ),+ embedding=[0.3, 0.4, 0.5, 0.6],+ ),+ ]++ await store.add(chunks)+ print(f"✓ Added document with {len(chunks)} chunks")++ # Search and verify chunk information+ query_embedding = [0.2, 0.3, 0.4, 0.5]+ results = await store.search(+ query_embedding=query_embedding,+ limit=3,+ )++ print("\n✓ Search results for multi-chunk document:")+ for i, result in enumerate(results, 1):+ chunk_info = (+ f"{result.metadata.chunk_id}/{result.metadata.total_chunks}"+ )+ print(f" {i}. Chunk {chunk_info}")+ print(f" Content: {result.metadata.content}")+ print(f" Score: {result.score:.4f}")+++async def example_distance_metrics() -> None:+ """The example of different distance metrics."""+ print("\n" + "=" * 60)+ print("Test 4: Different Distance Metrics")+ print("=" * 60)++ # Test with different metrics+ metrics = ["COSINE", "L2", "IP"]++ for metric in metrics:+ print(f"\n--- Testing {metric} metric ---")+ store = MilvusLiteStore(+ uri=f"./milvus_{metric.lower()}_test.db",+ collection_name=f"{metric.lower()}_collection",+ dimensions=4,+ distance=metric,+ )++ docs = [+ Document(+ metadata=DocMetadata(+ content=TextBlock(text=f"Test doc for {metric}"),+ doc_id=f"doc_{metric}_1",+ chunk_id=0,+ total_chunks=1,+ ),+ embedding=[0.1, 0.2, 0.3, 0.4],+ ),+ ]++ await store.add(docs)+ results = await store.search(+ query_embedding=[0.1, 0.2, 0.3, 0.4],+ limit=1,+ )++ print(f"✓ {metric} metric: Score = {results[0].score:.4f}")+++async def main() -> None:+ """Run all example."""+ print("\n" + "=" * 60)+ print("MilvusLiteStore Comprehensive Test Suite")+ print("=" * 60)++ try:+ await example_basic_operations()+ await example_filter_search()+ await example_multiple_chunks()+ await example_distance_metrics()++ print("\n" + "=" * 60)+ print("✓ All tests completed successfully!")+ print("=" * 60)++ except Exception as e:+ print(f"\n✗ Test failed with error: {e}")+ import traceback++ traceback.print_exc()+++if __name__ == "__main__":+ asyncio.run(main())diff --git a/pyproject.toml b/pyproject.tomlindex 6f552fdff..abfe2c924 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -67,6 +67,7 @@ dev = ["furo","myst_parser","matplotlib",+ "pymilvus[milvus_lite]",][project.urls]diff --git a/src/agentscope/rag/__init__.py b/src/agentscope/rag/__init__.pyindex 08fd67bd7..80a58abee 100644--- a/src/agentscope/rag/__init__.py+++ b/src/agentscope/rag/__init__.py@@ -14,6 +14,7 @@ from ._reader import (from ._store import (VDBStoreBase,QdrantStore,+ MilvusLiteStore,)from ._knowledge_base import KnowledgeBasefrom ._simple_knowledge import SimpleKnowledge@@ -28,6 +29,7 @@ __all__ = ["Document","VDBStoreBase","QdrantStore",+ "MilvusLiteStore","KnowledgeBase","SimpleKnowledge",]diff --git a/src/agentscope/rag/_store/__init__.py b/src/agentscope/rag/_store/__init__.pyindex 8b1f3c2b5..ceebe517e 100644--- a/src/agentscope/rag/_store/__init__.py+++ b/src/agentscope/rag/_store/__init__.py@@ -5,8 +5,10 @@ from ._store_base import (VDBStoreBase,)from ._qdrant_store import QdrantStore+from ._milvuslite_store import MilvusLiteStore__all__ = ["VDBStoreBase","QdrantStore",+ "MilvusLiteStore",]diff --git a/src/agentscope/rag/_store/_milvuslite_store.py b/src/agentscope/rag/_store/_milvuslite_store.pynew file mode 100644index 000000000..b5601dceb--- /dev/null+++ b/src/agentscope/rag/_store/_milvuslite_store.py@@ -0,0 +1,257 @@+# -*- coding: utf-8 -*-+"""The Milvus Lite vector store implementation."""+import json+from typing import Any, Literal, TYPE_CHECKING++from .._reader import Document+from ._store_base import VDBStoreBase+from .._document import DocMetadata++from ..._utils._common import _map_text_to_uuid+from ...types import Embedding++if TYPE_CHECKING:+ from pymilvus import MilvusClient+else:+ MilvusClient = "pymilvus.MilvusClient"+++class MilvusLiteStore(VDBStoreBase):+ """The Milvus Lite vector store implementation, supporting both local and+ remote Milvus instances.++ .. note:: In Milvus Lite, we use the scalar fields to store the metadata,+ including the document ID, chunk ID, and original content. The new+ MilvusClient API is used for simplified operations.++ .. note:: Milvus Lite is not supported on Windows OS for now (2025-10-21).++ """++ def __init__(+ self,+ uri: str,+ collection_name: str,+ dimensions: int,+ distance: Literal["COSINE", "L2", "IP"] = "COSINE",+ token: str = "",+ client_kwargs: dict[str, Any] | None = None,+ collection_kwargs: dict[str, Any] | None = None,+ ) -> None:+ """Initialize the Milvus Lite vector store.++ Args:+ uri (`str`):+ The URI of the Milvus instance. For Milvus Lite, use a local+ file path like "./milvus_demo.db". For remote Milvus server,+ use URI like "http://localhost:19530".+ collection_name (`str`):+ The name of the collection to store the embeddings.+ dimensions (`int`):+ The dimension of the embeddings.+ distance (`Literal["COSINE", "L2", "IP"]`, default to "COSINE"):+ The distance metric to use for the collection. Can be one of+ "COSINE", "L2", or "IP". Defaults to "COSINE".+ token (`str`, defaults to ""):+ The token for authentication when connecting to remote Milvus.+ Format: "username:password". Not needed for Milvus Lite.+ client_kwargs (`dict[str, Any] | None`, optional):+ Other keyword arguments for the Milvus client.+ collection_kwargs (`dict[str, Any] | None`, optional):+ Other keyword arguments for creating the collection.+ """++ try:+ from pymilvus import MilvusClient+ except ImportError as e:+ raise ImportError(+ "Milvus client is not installed. Please install it with "+ "`pip install pymilvus[milvus_lite]`.",+ ) from e++ client_kwargs = client_kwargs or {}++ # Initialize MilvusClient with uri and optional token+ init_params = {"uri": uri, **client_kwargs}+ if token:+ init_params["token"] = token++ self._client = MilvusClient(**init_params)++ self.collection_name = collection_name+ self.dimensions = dimensions+ self.distance = distance+ self.collection_kwargs = collection_kwargs or {}++ async def _validate_collection(self) -> None:+ """Validate the collection exists, if not, create it."""+ if not self._client.has_collection(self.collection_name):+ # Create collection with the new MilvusClient API+ # By default, it creates an auto-incrementing integer ID field+ kwargs = {+ "collection_name": self.collection_name,+ "dimension": self.dimensions,+ "metric_type": self.distance,+ **self.collection_kwargs,+ }++ self._client.create_collection(**kwargs)++ async def add(self, documents: list[Document], **kwargs: Any) -> None:+ """Add embeddings to the Milvus vector store.++ Args:+ documents (`list[Document]`):+ A list of embedding records to be recorded in the Milvus store.+ **kwargs (`Any`):+ Additional arguments for the insert operation.+ """+ await self._validate_collection()++ # Prepare data for insertion using the new MilvusClient API+ data = []+ for doc in documents:+ # Generate a unique integer ID based on hash+ unique_string = json.dumps(+ {+ "doc_id": doc.metadata.doc_id,+ "chunk_id": doc.metadata.chunk_id,+ "content": doc.metadata.content,+ },+ ensure_ascii=False,+ )++ id_type = self.collection_kwargs.get("id_type", "int")+ if id_type == "string":+ unique_id = _map_text_to_uuid(unique_string)[:6]+ else:+ unique_id = abs(hash(unique_string)) % (10**10)++ # Prepare data entry with vector and metadata+ entry = {+ # Fixed fields for Milvus+ "id": unique_id,+ "vector": doc.embedding,+ # fields that will be returned in the "entity" field during+ # search+ "doc_id": doc.metadata.doc_id,+ "chunk_id": doc.metadata.chunk_id,+ "content": doc.metadata.content,+ "total_chunks": doc.metadata.total_chunks,+ }+ data.append(entry)++ # Insert data using MilvusClient+ self._client.insert(+ collection_name=self.collection_name,+ data=data,+ )++ async def search(+ self,+ query_embedding: Embedding,+ limit: int,+ score_threshold: float | None = None,+ **kwargs: Any,+ ) -> list[Document]:+ """Search relevant documents from the Milvus vector store.++ Args:+ query_embedding (`Embedding`):+ The embedding of the query text.+ limit (`int`):+ The number of relevant documents to retrieve.+ score_threshold (`float | None`, optional):+ The threshold of the score to filter the results.+ **kwargs (`Any`):+ Additional arguments for the Milvus client search API.+ - filter (`str`): Expression to filter the search results.+ - output_fields (`list[str]`): Fields to include in results.+ """++ # Get output fields if specified+ if "output_fields" not in kwargs:+ kwargs["output_fields"] = [+ "doc_id",+ "chunk_id",+ "content",+ "total_chunks",+ ]++ # Execute search using MilvusClient+ results = self._client.search(+ collection_name=self.collection_name,+ data=[query_embedding],+ limit=limit,+ **kwargs,+ )++ # Process results+ collected_res = []+ for hits in results:+ for hit in hits:+ # Check score threshold+ if (+ score_threshold is not None+ and hit["distance"] < score_threshold+ ):+ continue++ # Get metadata from entity+ entity = hit["entity"]++ doc_metadata = DocMetadata(+ content=entity.get("content", ""),+ doc_id=entity.get("doc_id", ""),+ chunk_id=entity.get("chunk_id", 0),+ total_chunks=entity.get("total_chunks", 0),+ )++ # Create Document+ collected_res.append(+ Document(+ embedding=None, # Vector not returned by default+ score=hit["distance"],+ metadata=doc_metadata,+ ),+ )++ return collected_res++ async def delete(+ self,+ ids: list[str] | None = None,+ filter: str | None = None, # pylint: disable=redefined-builtin+ **kwargs: Any,+ ) -> None:+ """Delete documents from the Milvus vector store.++ Args:+ ids (`list[str] | None`, optional):+ List of entity IDs to delete.+ filter (`str | None`, optional):+ Expression to filter documents to delete.+ **kwargs (`Any`):+ Additional arguments for the delete operation.+ """+ if ids is None and filter is None:+ raise ValueError(+ "Either ids or filter_expr must be provided for deletion.",+ )++ # Delete data using MilvusClient+ self._client.delete(+ collection_name=self.collection_name,+ ids=ids,+ filter=filter,+ )++ def get_client(self) -> MilvusClient:+ """Get the underlying Milvus client, so that developers can access+ the full functionality of Milvus.++ Returns:+ `MilvusClient`:+ The underlying Milvus client.+ """+ return self._clientdiff --git a/tests/rag_store_test.py b/tests/rag_store_test.pyindex ec9beaaaa..afbd08372 100644--- a/tests/rag_store_test.py+++ b/tests/rag_store_test.py@@ -1,5 +1,6 @@# -*- coding: utf-8 -*-"""Test the RAG store implementations."""+import osfrom unittest import IsolatedAsyncioTestCasefrom agentscope.message import TextBlock@@ -7,12 +8,19 @@ from agentscope.rag import (QdrantStore,Document,DocMetadata,+ MilvusLiteStore,)class RAGStoreTest(IsolatedAsyncioTestCase):"""Test cases for RAG store implementations."""+ async def asyncSetUp(self) -> None:+ """Set up before each test."""+ # Remove the test database file after the test+ if os.path.exists("./milvus_demo.db"):+ os.remove("./milvus_demo.db")+async def test_qdrant_store(self) -> None:"""Test the QdrantStore implementation."""store = QdrantStore(@@ -64,3 +72,64 @@ class RAGStoreTest(IsolatedAsyncioTestCase):res[0].metadata.content["text"],"This is a test document.",)++ async def test_milvus_lite_store(self) -> None:+ """Test the MilvusLiteStore implementation."""+ if os.name == "nt":+ self.skipTest("Milvus Lite is not supported on Windows.")++ store = MilvusLiteStore(+ uri="./milvus_demo.db",+ collection_name="test_milvus",+ dimensions=3,+ )++ await store.add(+ [+ Document(+ embedding=[0.1, 0.2, 0.3],+ metadata=DocMetadata(+ content=TextBlock(+ type="text",+ text="This is a test document.",+ ),+ doc_id="doc1",+ chunk_id=0,+ total_chunks=2,+ ),+ ),+ Document(+ embedding=[0.9, 0.1, 0.4],+ metadata=DocMetadata(+ content=TextBlock(+ type="text",+ text="This is another test document.",+ ),+ doc_id="doc1",+ chunk_id=1,+ total_chunks=2,+ ),+ ),+ ],+ )++ res = await store.search(+ query_embedding=[0.15, 0.25, 0.35],+ limit=3,+ score_threshold=0.8,+ )+ self.assertEqual(len(res), 1)+ self.assertEqual(+ round(res[0].score, 4),+ 0.9974,+ )+ self.assertEqual(+ res[0].metadata.content["text"],+ "This is a test document.",+ )++ async def asyncTearDown(self) -> None:+ """Clean up after tests."""+ # Remove Milvus Lite database file+ if os.path.exists("./milvus_demo.db"):+ os.remove("./milvus_demo.db")