Build and run agents you can see, understand and trust.
Python agentchatbotlarge-language-modelsllmllm-agentmcpmulti-agentmulti-modalreact-agentrealtime-agentvoice-agentvoice-assistant
feat(rag): add OceanBaseStore as a vector database choice (#1078)
Chojan Shang9745c3087407
diff --git a/examples/functionality/vector_store/oceanbase/README.md b/examples/functionality/vector_store/oceanbase/README.mdnew file mode 100644index 000000000..d8d30424f--- /dev/null+++ b/examples/functionality/vector_store/oceanbase/README.md@@ -0,0 +1,164 @@+# OceanBase Vector Store++This example demonstrates how to use **OceanBaseStore** for vector storage and semantic search in AgentScope.+It includes CRUD operations, metadata filtering, document chunking, and distance metric tests.++### Quick Start++Install dependencies (including `pyobvector`):++```bash+pip install -e .[full]+```++Start seekdb (a minimal OceanBase-compatible instance):++```bash+docker run -d -p 2881:2881 oceanbase/seekdb+```++Run the example script:++```bash+python main.py+```++> **Note:** The script defaults to `127.0.0.1:2881`, user `root`, database `test`.+> If you use a multi-tenant OceanBase account (e.g., `root@test`), override via environment variables.++## Usage++### Initialize Store++```python+from agentscope.rag import OceanBaseStore++store = OceanBaseStore(+ collection_name="test_collection",+ dimensions=768,+ distance="COSINE",+ uri="127.0.0.1:2881",+ user="root",+ password="",+ db_name="test",+)+```++### Add Documents++```python+from agentscope.rag import Document, DocMetadata+from agentscope.message import TextBlock++doc = Document(+ metadata=DocMetadata(+ content=TextBlock(type="text", text="Your document text"),+ doc_id="doc_1",+ chunk_id=0,+ total_chunks=1,+ ),+ embedding=[0.1, 0.2, 0.3],+)++await store.add([doc])+```++### Search++```python+results = await store.search(+ query_embedding=[0.1, 0.2, 0.3],+ limit=5,+ score_threshold=0.9,+)+```++### Filter Search++```python+client = store.get_client()+table = client.load_table(collection_name="test_collection")++results = await store.search(+ query_embedding=[0.1, 0.2, 0.3],+ limit=5,+ flter=[table.c["doc_id"].like("doc%")],+)+```++> Note: The parameter name is `flter` (missing the "i") to avoid clashing with+> Python's built-in `filter` and follows the underlying library's convention.++### Delete++```python+client = store.get_client()+table = client.load_table(collection_name="test_collection")++await store.delete(where=[table.c["doc_id"] == "doc_1"])+```++## Distance Metrics++| Metric | Description | Best For |+|--------|-------------|----------|+| **COSINE** | Cosine similarity | Text embeddings (recommended) |+| **L2** | Euclidean distance | Spatial data |+| **IP** | Inner product | Recommendation systems |++## Filter Expressions++Build filters using SQLAlchemy expressions and pass them via `flter`:++```python+table = store.get_client().load_table("test_collection")++filters = [+ table.c["doc_id"] == "doc_1",+ table.c["doc_id"].like("prefix%"),+ table.c["chunk_id"] >= 0,+]+```++## Advanced Usage++### Access Underlying Client++```python+client = store.get_client()+stats = client.get_collection_stats(collection_name="test_collection")+```++### Document Metadata++- `content`: Text content (TextBlock)+- `doc_id`: Unique document identifier+- `chunk_id`: Chunk position (0-indexed)+- `total_chunks`: Total chunks in document++## FAQ++**What embedding dimension should I use?**+Match your embedding model's output dimension (e.g., 768 for BERT, 1536 for OpenAI ada-002).++**Can I change the distance metric after creation?**+No, create a new collection with the desired metric.++**How do I clean up test data?**+Drop the collection via the underlying client or remove the seekdb container volume.++## Environment Variables++The script supports the following environment variables to override connection settings:++```bash+export OCEANBASE_URI="127.0.0.1:2881"+export OCEANBASE_USER="root"+export OCEANBASE_PASSWORD=""+export OCEANBASE_DB="test"+```++## References++- [OceanBase Vector Store](https://github.com/oceanbase/pyobvector)+- [AgentScope RAG Tutorial](https://doc.agentscope.io/tutorial/task_rag.html)diff --git a/examples/functionality/vector_store/oceanbase/main.py b/examples/functionality/vector_store/oceanbase/main.pynew file mode 100644index 000000000..41097484f--- /dev/null+++ b/examples/functionality/vector_store/oceanbase/main.py@@ -0,0 +1,350 @@+# -*- coding: utf-8 -*-+"""Example of using OceanBaseStore in AgentScope RAG system."""+import asyncio+import os++from agentscope.rag import (+ OceanBaseStore,+ Document,+ DocMetadata,+)+from agentscope.message import TextBlock+++def _create_store(+ collection_name: str,+ dimensions: int = 4,+ distance: str = "COSINE",+) -> OceanBaseStore:+ return OceanBaseStore(+ collection_name=collection_name,+ dimensions=dimensions,+ distance=distance,+ uri=os.getenv("OCEANBASE_URI", "127.0.0.1:2881"),+ user=os.getenv("OCEANBASE_USER", "root"),+ password=os.getenv("OCEANBASE_PASSWORD", ""),+ db_name=os.getenv("OCEANBASE_DB", "test"),+ )+++async def example_basic_operations() -> None:+ """The example of basic CRUD operations with OceanBaseStore."""+ print("\n" + "=" * 60)+ print("Test 1: Basic CRUD Operations")+ print("=" * 60)++ store = _create_store(collection_name="ob_basic_collection")+ store.get_client().drop_collection("ob_basic_collection")++ print("✓ OceanBaseStore initialized")++ test_docs = [+ Document(+ metadata=DocMetadata(+ content=TextBlock(+ type="text",+ text="Artificial Intelligence is the future",+ ),+ doc_id="doc_1",+ chunk_id=0,+ total_chunks=1,+ ),+ embedding=[0.1, 0.2, 0.3, 0.4],+ ),+ Document(+ metadata=DocMetadata(+ content=TextBlock(+ type="text",+ text="Machine Learning is a subset of AI",+ ),+ doc_id="doc_2",+ chunk_id=0,+ total_chunks=1,+ ),+ embedding=[0.2, 0.3, 0.4, 0.5],+ ),+ Document(+ metadata=DocMetadata(+ content=TextBlock(+ type="text",+ text="Deep Learning uses neural networks",+ ),+ doc_id="doc_3",+ chunk_id=0,+ total_chunks=1,+ ),+ embedding=[0.3, 0.4, 0.5, 0.6],+ ),+ ]++ await store.add(test_docs)+ print(f"✓ Added {len(test_docs)} documents to the store")++ query_embedding = [0.15, 0.25, 0.35, 0.45]+ results = await store.search(+ query_embedding=query_embedding,+ limit=2,+ )++ print(f"\n✓ Search completed, found {len(results)} results:")+ for i, result in enumerate(results, 1):+ print(f" {i}. Score: {result.score:.4f}")+ print(f" Content: {result.metadata.content}")+ print(f" Doc ID: {result.metadata.doc_id}")++ results_filtered = await store.search(+ query_embedding=query_embedding,+ limit=5,+ score_threshold=0.9,+ )+ print(f"\n✓ Search with threshold (>0.9): {len(results_filtered)} results")++ client = store.get_client()+ table = client.load_table(collection_name="ob_basic_collection")+ await store.delete(where=[table.c["doc_id"] == "doc_2"])+ print("\n✓ Deleted document with doc_id='doc_2'")++ results_after_delete = await store.search(+ query_embedding=query_embedding,+ limit=5,+ )+ print(f"✓ After deletion: {len(results_after_delete)} documents remain")++ print(f"\n✓ Got MilvusLikeClient: {type(client).__name__}")+++async def example_filter_search() -> None:+ """The example of search with metadata filtering."""+ print("\n" + "=" * 60)+ print("Test 2: Search with Metadata Filtering")+ print("=" * 60)++ store = _create_store(collection_name="ob_filter_collection")+ client = store.get_client()+ client.drop_collection("ob_filter_collection")++ docs = [+ Document(+ metadata=DocMetadata(+ content=TextBlock(+ type="text",+ text="Python is a programming language",+ ),+ doc_id="prog_1",+ chunk_id=0,+ total_chunks=1,+ ),+ embedding=[0.1, 0.2, 0.3, 0.4],+ ),+ Document(+ metadata=DocMetadata(+ content=TextBlock(+ type="text",+ text="Java is used for enterprise applications",+ ),+ doc_id="prog_2",+ chunk_id=0,+ total_chunks=1,+ ),+ embedding=[0.2, 0.3, 0.4, 0.5],+ ),+ Document(+ metadata=DocMetadata(+ content=TextBlock(+ type="text",+ text="Neural networks are used in AI",+ ),+ doc_id="ai_1",+ chunk_id=0,+ total_chunks=1,+ ),+ embedding=[0.3, 0.4, 0.5, 0.6],+ ),+ Document(+ metadata=DocMetadata(+ content=TextBlock(+ type="text",+ text="Deep learning requires GPUs",+ ),+ doc_id="ai_2",+ chunk_id=0,+ total_chunks=1,+ ),+ embedding=[0.4, 0.5, 0.6, 0.7],+ ),+ ]++ await store.add(docs)+ print(f"✓ Added {len(docs)} documents with different doc_id prefixes")++ query_embedding = [0.25, 0.35, 0.45, 0.55]+ all_results = await store.search(+ query_embedding=query_embedding,+ limit=4,+ )+ print(f"\n✓ Search without filter: {len(all_results)} results")+ for i, result in enumerate(all_results, 1):+ print(+ f" {i}. Doc ID: {result.metadata.doc_id}, "+ f"Score: {result.score:.4f}",+ )++ table = client.load_table(collection_name="ob_filter_collection")+ prog_results = await store.search(+ query_embedding=query_embedding,+ limit=4,+ flter=[table.c["doc_id"].like("prog%")],+ )+ print("\n✓ Search with filter (doc_id like 'prog%'):")+ for i, result in enumerate(prog_results, 1):+ print(+ f" {i}. Doc ID: {result.metadata.doc_id}, "+ f"Score: {result.score:.4f}",+ )++ ai_results = await store.search(+ query_embedding=query_embedding,+ limit=4,+ flter=[table.c["doc_id"].like("ai%")],+ )+ print("\n✓ Search with filter (doc_id like 'ai%'):")+ for i, result in enumerate(ai_results, 1):+ print(+ f" {i}. Doc ID: {result.metadata.doc_id}, "+ f"Score: {result.score:.4f}",+ )+++async def example_multiple_chunks() -> None:+ """The example of documents with multiple chunks."""+ print("\n" + "=" * 60)+ print("Test 3: Documents with Multiple Chunks")+ print("=" * 60)++ store = _create_store(collection_name="ob_chunks_collection")+ store.get_client().drop_collection("ob_chunks_collection")++ chunks = [+ Document(+ metadata=DocMetadata(+ content=TextBlock(+ type="text",+ text="Chapter 1: Introduction to AI",+ ),+ doc_id="book_1",+ chunk_id=0,+ total_chunks=3,+ ),+ embedding=[0.1, 0.2, 0.3, 0.4],+ ),+ Document(+ metadata=DocMetadata(+ content=TextBlock(+ type="text",+ text="Chapter 2: Machine Learning Basics",+ ),+ doc_id="book_1",+ chunk_id=1,+ total_chunks=3,+ ),+ embedding=[0.2, 0.3, 0.4, 0.5],+ ),+ Document(+ metadata=DocMetadata(+ content=TextBlock(+ type="text",+ text="Chapter 3: Deep Learning Advanced",+ ),+ doc_id="book_1",+ chunk_id=2,+ total_chunks=3,+ ),+ embedding=[0.3, 0.4, 0.5, 0.6],+ ),+ ]++ await store.add(chunks)+ print(f"✓ Added document with {len(chunks)} chunks")++ query_embedding = [0.2, 0.3, 0.4, 0.5]+ results = await store.search(+ query_embedding=query_embedding,+ limit=3,+ )++ print("\n✓ Search results for multi-chunk document:")+ for i, result in enumerate(results, 1):+ chunk_info = (+ f"{result.metadata.chunk_id}/{result.metadata.total_chunks}"+ )+ print(f" {i}. Chunk {chunk_info}")+ print(f" Content: {result.metadata.content}")+ print(f" Score: {result.score:.4f}")+++async def example_distance_metrics() -> None:+ """The example of different distance metrics."""+ print("\n" + "=" * 60)+ print("Test 4: Different Distance Metrics")+ print("=" * 60)++ metrics = ["COSINE", "L2", "IP"]++ for metric in metrics:+ print(f"\n--- Testing {metric} metric ---")+ collection_name = f"ob_{metric}_collection"+ store = _create_store(+ collection_name=collection_name,+ distance=metric,+ )+ store.get_client().drop_collection(collection_name)++ docs = [+ Document(+ metadata=DocMetadata(+ content=TextBlock(+ type="text",+ text=f"Test doc for {metric}",+ ),+ doc_id=f"doc_{metric}_1",+ chunk_id=0,+ total_chunks=1,+ ),+ embedding=[0.1, 0.2, 0.3, 0.4],+ ),+ ]++ await store.add(docs)+ results = await store.search(+ query_embedding=[0.1, 0.2, 0.3, 0.4],+ limit=1,+ )++ print(f"✓ {metric} metric: Score = {results[0].score:.4f}")+++async def main() -> None:+ """Run all example."""+ print("\n" + "=" * 60)+ print("OceanBaseStore Comprehensive Test Suite")+ print("=" * 60)++ try:+ await example_basic_operations()+ await example_filter_search()+ await example_multiple_chunks()+ await example_distance_metrics()++ print("\n" + "=" * 60)+ print("✓ All tests completed successfully!")+ print("=" * 60)++ except Exception as e:+ print(f"\n✗ Test failed with error: {e}")+ import traceback++ traceback.print_exc()+++if __name__ == "__main__":+ asyncio.run(main())diff --git a/pyproject.toml b/pyproject.tomlindex b658327c7..17db9a330 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -118,11 +118,13 @@ qdrant = ["qdrant-client==1.15.1"]milvus = ["pymilvus[milvus_lite]"]ali_mysql = ["mysql-connector-python"]mongodb = ["pymongo"]+oceanbase = ["pyobvector>=0.2.0,<0.3.0"]vdbs = ["agentscope[ali_mysql]","agentscope[qdrant]","agentscope[milvus]","agentscope[mongodb]",+ "agentscope[oceanbase]",]rag = [diff --git a/src/agentscope/rag/__init__.py b/src/agentscope/rag/__init__.pyindex 00c360b9d..2c0d5909f 100644--- a/src/agentscope/rag/__init__.py+++ b/src/agentscope/rag/__init__.py@@ -18,6 +18,7 @@ from ._store import (VDBStoreBase,QdrantStore,MilvusLiteStore,+ OceanBaseStore,MongoDBStore,AlibabaCloudMySQLStore,)@@ -38,6 +39,7 @@ __all__ = ["VDBStoreBase","QdrantStore","MilvusLiteStore",+ "OceanBaseStore","MongoDBStore","AlibabaCloudMySQLStore","KnowledgeBase",diff --git a/src/agentscope/rag/_store/__init__.py b/src/agentscope/rag/_store/__init__.pyindex 96e0ce323..67951ace4 100644--- a/src/agentscope/rag/_store/__init__.py+++ b/src/agentscope/rag/_store/__init__.py@@ -6,6 +6,7 @@ from ._store_base import ()from ._qdrant_store import QdrantStorefrom ._milvuslite_store import MilvusLiteStore+from ._oceanbase_store import OceanBaseStorefrom ._mongodb_store import MongoDBStorefrom ._alibabacloud_mysql_store import AlibabaCloudMySQLStore@@ -13,6 +14,7 @@ __all__ = ["VDBStoreBase","QdrantStore","MilvusLiteStore",+ "OceanBaseStore","MongoDBStore","AlibabaCloudMySQLStore",]diff --git a/src/agentscope/rag/_store/_oceanbase_store.py b/src/agentscope/rag/_store/_oceanbase_store.pynew file mode 100644index 000000000..3ace3eeb4--- /dev/null+++ b/src/agentscope/rag/_store/_oceanbase_store.py@@ -0,0 +1,552 @@+# -*- coding: utf-8 -*-+"""The OceanBase vector store implementation."""+import asyncio+import json+from typing import Any, Callable, Literal, TYPE_CHECKING++from .._reader import Document+from ._store_base import VDBStoreBase+from .._document import DocMetadata+from ..._utils._common import _map_text_to_uuid+from ...message import TextBlock+from ...types import Embedding++if TYPE_CHECKING:+ from pyobvector import MilvusLikeClient+else:+ MilvusLikeClient = "pyobvector.MilvusLikeClient"+++# Metric configuration: data-driven approach focusing on exceptions+# Base metric names for pyobvector+_METRIC_NAMES = {+ "COSINE": "cosine",+ "L2": "l2",+ "IP": "inner_product",+}++# Only configure special cases that differ from the base+_SEARCH_METRIC_OVERRIDES = {+ "IP": "ip", # IP uses "ip" for search to get positive inner product+}++_SCORE_CONVERTERS: dict[str, Callable[[float], float]] = {+ "COSINE": lambda d: 1.0 - d, # COSINE converts distance to similarity+}+++class OceanBaseStore(VDBStoreBase):+ """The OceanBase vector store implementation, supporting OceanBase and+ seekdb via pyobvector."""++ # Field names - using descriptive constants to avoid magic strings+ PRIMARY_FIELD = "id"+ VECTOR_FIELD = "embedding"+ DOC_ID_FIELD = "doc_id"+ CHUNK_ID_FIELD = "chunk_id"+ TOTAL_CHUNKS_FIELD = "total_chunks"+ CONTENT_FIELD = "content"++ # Index configuration+ INDEX_NAME = "vidx"+ INDEX_TYPE = "hnsw"++ def __init__(+ self,+ collection_name: str,+ dimensions: int,+ uri: str = "127.0.0.1:2881",+ user: str = "root@test",+ password: str = "",+ db_name: str = "test",+ distance: Literal["COSINE", "L2", "IP"] = "COSINE",+ client_kwargs: dict[str, Any] | None = None,+ collection_kwargs: dict[str, Any] | None = None,+ ) -> None:+ """Initialize the OceanBase vector store.++ Args:+ collection_name (`str`):+ The name of the collection to store the embeddings.+ dimensions (`int`):+ The dimension of the embeddings.+ uri (`str`, defaults to `"127.0.0.1:2881"`):+ The OceanBase server URI, such as "127.0.0.1:2881".+ user (`str`, defaults to `"root@test"`):+ The username for authentication.+ password (`str`, defaults to `""`):+ The password for authentication.+ db_name (`str`, defaults to `"test"`):+ The database name to connect to.+ distance (`Literal["COSINE", "L2", "IP"]`, defaults to `"COSINE"`):+ The distance metric to use for the collection. Can be one of+ "COSINE", "L2", or "IP".+ client_kwargs (`dict[str, Any] | None`, optional):+ Keyword arguments passed to `pyobvector.MilvusLikeClient`.+ Explicit connection arguments override matching keys here.+ collection_kwargs (`dict[str, Any] | None`, optional):+ Keyword arguments passed to `create_collection`.+ """+ try:+ import pyobvector+ except ImportError as e:+ raise ImportError(+ "OceanBase client is not installed. Please install it with "+ "`pip install pyobvector`.",+ ) from e++ self._pyobvector = pyobvector+ client_kwargs = dict(client_kwargs or {})++ self._client = pyobvector.MilvusLikeClient(+ uri=uri,+ user=user,+ password=password,+ db_name=db_name,+ **client_kwargs,+ )++ self.collection_name = collection_name+ self.dimensions = dimensions+ self.distance = distance+ self.collection_kwargs = collection_kwargs or {}+ self._collection_ready = False++ def _get_metric_type(self) -> str:+ """Get the metric type for index creation."""+ return _METRIC_NAMES[self.distance]++ def _get_search_metric_type(self) -> str:+ """Get the search metric type for queries.++ Returns the override value if exists, otherwise uses index metric.+ This allows special handling (e.g., IP uses "ip" for positive values).+ """+ return _SEARCH_METRIC_OVERRIDES.get(+ self.distance,+ self._get_metric_type(),+ )++ async def _validate_collection(self) -> None:+ """Validate the collection exists, if not, create it."""+ if self._collection_ready:+ return++ if await asyncio.to_thread(+ self._client.has_collection,+ self.collection_name,+ ):+ self._collection_ready = True+ return++ collection_kwargs = dict(self.collection_kwargs)++ if "schema" not in collection_kwargs:+ collection_kwargs["schema"] = self._create_schema()++ if "index_params" not in collection_kwargs:+ collection_kwargs["index_params"] = self._create_index_params()++ await asyncio.to_thread(+ self._client.create_collection,+ collection_name=self.collection_name,+ **collection_kwargs,+ )+ self._collection_ready = True++ def _create_schema(self) -> Any:+ """Create the collection schema with all required fields.++ Returns:+ Schema object with primary, vector, and metadata fields configured.+ """+ schema = self._client.create_schema()++ # Primary key field+ schema.add_field(+ field_name=self.PRIMARY_FIELD,+ datatype=self._pyobvector.DataType.VARCHAR,+ is_primary=True,+ auto_id=False,+ max_length=36,+ )++ # Vector field+ schema.add_field(+ field_name=self.VECTOR_FIELD,+ datatype=self._pyobvector.DataType.FLOAT_VECTOR,+ dim=self.dimensions,+ )++ # Metadata fields+ schema.add_field(+ field_name=self.DOC_ID_FIELD,+ datatype=self._pyobvector.DataType.STRING,+ )+ schema.add_field(+ field_name=self.CHUNK_ID_FIELD,+ datatype=self._pyobvector.DataType.INT64,+ )+ schema.add_field(+ field_name=self.TOTAL_CHUNKS_FIELD,+ datatype=self._pyobvector.DataType.INT64,+ )+ schema.add_field(+ field_name=self.CONTENT_FIELD,+ datatype=self._pyobvector.DataType.JSON,+ nullable=True,+ )++ return schema++ def _create_index_params(self) -> Any:+ """Create index parameters for vector search.++ Returns:+ Index parameters configured with HNSW index and appropriate metric.+ """+ index_params = self._client.prepare_index_params()+ index_params.add_index(+ field_name=self.VECTOR_FIELD,+ index_type=self.INDEX_TYPE,+ index_name=self.INDEX_NAME,+ metric_type=self._get_metric_type(),+ )+ return index_params++ @staticmethod+ def _content_to_text(content: Any) -> str:+ """Extract text string from content of various formats."""+ if isinstance(content, str):+ return content+ if isinstance(content, dict) and content.get("type") == "text":+ text = content.get("text", "")+ return text if isinstance(text, str) else ""+ return ""++ @staticmethod+ def _normalize_content(content: Any, fallback_text: str) -> Any:+ """Normalize content to TextBlock format."""+ # Already in correct format+ if isinstance(content, dict) and content.get("type"):+ return content+ # Convert string to TextBlock+ if isinstance(content, str):+ return TextBlock(type="text", text=content)+ # Use fallback+ return TextBlock(type="text", text=fallback_text or "")++ def _document_to_dict(self, doc: Document) -> dict[str, Any]:+ """Convert a Document to a dictionary for insertion.++ Args:+ doc (`Document`):+ Document to convert++ Returns:+ `dict[str, Any]`:+ Dictionary with fields ready for database insertion++ Raises:+ `ValueError`:+ If document embedding is None+ """+ if doc.embedding is None:+ raise ValueError(+ "Document embedding is required for OceanBaseStore.add.",+ )++ # Create unique ID from document metadata+ unique_string = json.dumps(+ {+ "doc_id": doc.metadata.doc_id,+ "chunk_id": doc.metadata.chunk_id,+ "content": doc.metadata.content,+ },+ ensure_ascii=True,+ sort_keys=True,+ )++ return {+ self.PRIMARY_FIELD: _map_text_to_uuid(unique_string),+ self.VECTOR_FIELD: doc.embedding,+ self.DOC_ID_FIELD: doc.metadata.doc_id,+ self.CHUNK_ID_FIELD: doc.metadata.chunk_id,+ self.TOTAL_CHUNKS_FIELD: doc.metadata.total_chunks,+ self.CONTENT_FIELD: doc.metadata.content,+ }++ async def add(self, documents: list[Document], **kwargs: Any) -> None:+ """Add embeddings to the OceanBase vector store.++ Args:+ documents (`list[Document]`):+ A list of embedding records to be recorded in the store.+ """+ await self._validate_collection()++ data = [self._document_to_dict(doc) for doc in documents]++ await asyncio.to_thread(+ self._client.insert,+ collection_name=self.collection_name,+ data=data,+ **kwargs,+ )++ @staticmethod+ def _extract_distance(+ row: dict[str, Any],+ output_fields: list[str],+ ) -> float | None:+ """Extract distance value from search result row.++ The distance is stored in an extra field that's not in output_fields.+ """+ extra_keys = [key for key in row if key not in output_fields]+ return row.get(extra_keys[-1]) if extra_keys else None++ def _create_document_from_row(+ self,+ row: dict[str, Any],+ output_fields: list[str],+ ) -> tuple[Document, float | None]:+ """Create a Document from a search result row.++ Args:+ row (`dict[str, Any]`):+ Search result row containing document data+ output_fields (`list[str]`):+ List of fields requested in output++ Returns:+ `tuple[Document, float | None]`:+ Tuple of (Document, score)+ """+ distance = self._extract_distance(row, output_fields)+ score = self._convert_distance_to_score(distance)++ content_value = row.get(self.CONTENT_FIELD)+ content_text = self._content_to_text(content_value)+ content = self._normalize_content(content_value, content_text)++ doc_metadata = DocMetadata(+ content=content,+ doc_id=str(row.get(self.DOC_ID_FIELD, "")),+ chunk_id=int(row.get(self.CHUNK_ID_FIELD) or 0),+ total_chunks=int(row.get(self.TOTAL_CHUNKS_FIELD) or 0),+ )++ return (+ Document(+ embedding=None,+ score=score,+ metadata=doc_metadata,+ ),+ score,+ )++ def _convert_distance_to_score(+ self,+ distance: float | None,+ ) -> float | None:+ """Convert distance value to score based on metric type.++ Args:+ distance (`float | None`):+ Raw distance value from database++ Returns:+ `float | None`:+ Converted score (similarity for COSINE, raw value for others)++ """+ if distance is None:+ return None++ # Apply converter if defined, otherwise return raw value (identity)+ converter = _SCORE_CONVERTERS.get(self.distance)+ return converter(distance) if converter else distance++ async def search(+ self,+ query_embedding: Embedding,+ limit: int,+ score_threshold: float | None = None,+ **kwargs: Any,+ ) -> list[Document]:+ """Search relevant documents from the OceanBase vector store.++ Args:+ query_embedding (`Embedding`):+ The embedding of the query text.+ limit (`int`):+ The number of relevant documents to retrieve.+ score_threshold (`float | None`, optional):+ The threshold of the score to filter the results.+ Note: Score semantics (aligned with Milvus):+ - COSINE: similarity [0,1], higher = more similar+ - L2: Euclidean distance, smaller = more similar+ - IP: inner product, larger = more similar+ **kwargs (`Any`):+ Additional arguments for the search API.+ - flter (`list`): Filter conditions.+ - partition_names (`list[str]`): Partition filter.+ - output_fields (`list[str]`): Fields to include in results.+ - search_params (`dict`): Search parameters.+ """+ await self._validate_collection()++ # Remove unsupported parameter+ kwargs.pop("with_dist", None)++ # Prepare output fields and search parameters+ output_fields = self._prepare_output_fields(+ kwargs.pop("output_fields", None),+ )+ search_params = self._prepare_search_params(+ kwargs.pop("search_params", None),+ )++ results = await asyncio.to_thread(+ self._client.search,+ collection_name=self.collection_name,+ data=query_embedding,+ anns_field=self.VECTOR_FIELD,+ with_dist=True,+ limit=limit,+ output_fields=output_fields,+ search_params=search_params,+ **kwargs,+ )++ # Process results and filter by score threshold+ return self._filter_results_by_threshold(+ results,+ output_fields,+ score_threshold,+ )++ def _prepare_output_fields(+ self,+ output_fields: list[str] | None,+ ) -> list[str]:+ """Prepare output fields ensuring all required fields are included.++ Args:+ output_fields (`list[str] | None`):+ User-specified output fields or None++ Returns:+ `list[str]`:+ List of output fields with required fields included+ """+ required_fields = [+ self.DOC_ID_FIELD,+ self.CHUNK_ID_FIELD,+ self.TOTAL_CHUNKS_FIELD,+ self.CONTENT_FIELD,+ ]++ output_fields = output_fields or required_fields++ # Use dict.fromkeys to preserve order while removing duplicates+ return list(dict.fromkeys(output_fields + required_fields))++ def _prepare_search_params(+ self,+ search_params: dict[str, Any] | None,+ ) -> dict[str, Any]:+ """Prepare search parameters with appropriate metric type.++ Args:+ search_params (`dict[str, Any] | None`):+ User-specified search parameters or None++ Returns:+ `dict[str, Any]`:+ Search parameters dict with metric_type set+ """+ search_params = dict(search_params or {}) # Create a copy+ search_params.setdefault("metric_type", self._get_search_metric_type())+ return search_params++ def _filter_results_by_threshold(+ self,+ results: list[dict[str, Any]],+ output_fields: list[str],+ score_threshold: float | None,+ ) -> list[Document]:+ """Filter search results by score threshold.++ Args:+ results (`list[dict[str, Any]]`):+ Raw search results from database+ output_fields (`list[str]`):+ List of output fields+ score_threshold (`float | None`):+ Minimum score threshold or None++ Returns:+ `list[Document]`:+ List of filtered Document objects+ """+ documents = []+ for row in results:+ doc, score = self._create_document_from_row(row, output_fields)++ if score_threshold is not None and (+ score is None or score < score_threshold+ ):+ continue++ documents.append(doc)++ return documents++ async def delete(+ self,+ ids: list[str] | None = None,+ where: Any | None = None,+ where_document: Any | None = None,+ **kwargs: Any,+ ) -> None:+ """Delete documents from the OceanBase vector store.++ Args:+ ids (`list[str] | None`, optional):+ List of entity IDs to delete.+ where (`Any | None`, optional):+ Filter conditions for deletion.+ where_document (`Any | None`, optional):+ Unsupported in OceanBaseStore.+ """+ await self._validate_collection()++ if where_document is not None:+ raise ValueError(+ "where_document is not supported for OceanBaseStore.delete.",+ )++ if ids is None and where is None:+ raise ValueError(+ "At least one of ids or where must be provided for deletion.",+ )++ await asyncio.to_thread(+ self._client.delete,+ collection_name=self.collection_name,+ ids=ids,+ flter=where,+ **kwargs,+ )++ def get_client(self) -> MilvusLikeClient:+ """Get the underlying OceanBase client, so that developers can access+ the full functionality of OceanBase.++ Returns:+ `MilvusLikeClient`:+ The underlying OceanBase client.+ """+ return self._clientdiff --git a/tests/rag_store_test.py b/tests/rag_store_test.pyindex b7db30f3a..72e53ad69 100644--- a/tests/rag_store_test.py+++ b/tests/rag_store_test.py@@ -1,6 +1,8 @@# -*- coding: utf-8 -*-"""Test the RAG store implementations."""import os+import types+import uuidfrom typing import AsyncGeneratorfrom unittest import IsolatedAsyncioTestCasefrom unittest.mock import MagicMock, patch, AsyncMock@@ -11,6 +13,7 @@ from agentscope.rag import (Document,DocMetadata,MilvusLiteStore,+ OceanBaseStore,AlibabaCloudMySQLStore,MongoDBStore,)@@ -132,6 +135,160 @@ class RAGStoreTest(IsolatedAsyncioTestCase):"This is a test document.",)+ async def test_oceanbase_store(self) -> None:+ """Use real OceanBase when env is provided, otherwise use a mock."""++ def _make_mock_pyobvector(+ search_rows: list[dict],+ ) -> tuple[types.SimpleNamespace, MagicMock]:+ """Create a minimal pyobvector mock aligned with the+ existing style."""+ mock_client = MagicMock()+ mock_client.has_collection.return_value = False+ mock_client.create_schema.return_value = MagicMock()+ mock_client.prepare_index_params.return_value = MagicMock()+ mock_client.search.return_value = search_rows++ mock_pyobvector = types.SimpleNamespace(+ MilvusLikeClient=MagicMock(return_value=mock_client),+ DataType=types.SimpleNamespace(+ VARCHAR="VARCHAR",+ FLOAT_VECTOR="FLOAT_VECTOR",+ STRING="STRING",+ INT64="INT64",+ JSON="JSON",+ ),+ )+ return mock_pyobvector, mock_client++ required_vars = [+ "OCEANBASE_URI",+ "OCEANBASE_USER",+ "OCEANBASE_DB",+ ]+ missing_vars = [var for var in required_vars if not os.getenv(var)]+ if missing_vars:+ # COSINE distance = 1 - similarity+ mock_rows = [+ {+ "doc_id": "doc1",+ "chunk_id": 0,+ "total_chunks": 2,+ "content": {+ "type": "text",+ "text": "This is a test document.",+ },+ "distance": 1.0 - 0.9974,+ },+ ]+ mock_pyobvector, mock_client = _make_mock_pyobvector(mock_rows)++ with patch.dict("sys.modules", {"pyobvector": mock_pyobvector}):+ store = OceanBaseStore(+ collection_name=f"test_ob_{uuid.uuid4().hex[:8]}",+ dimensions=3,+ uri="127.0.0.1:2881",+ user="root@test",+ password="",+ db_name="test",+ )++ await store.add(+ [+ Document(+ embedding=[0.1, 0.2, 0.3],+ metadata=DocMetadata(+ content=TextBlock(+ type="text",+ text="This is a test document.",+ ),+ doc_id="doc1",+ chunk_id=0,+ total_chunks=2,+ ),+ ),+ ],+ )++ self.assertTrue(mock_client.insert.called)+ self.assertTrue(mock_client.create_collection.called)++ res = await store.search(+ query_embedding=[0.15, 0.25, 0.35],+ limit=3,+ score_threshold=0.8,+ )+ self.assertEqual(len(res), 1)+ self.assertEqual(round(res[0].score, 4), 0.9974)+ self.assertEqual(+ res[0].metadata.content["text"],+ "This is a test document.",+ )++ await store.delete(ids=["dummy-id"])+ self.assertTrue(mock_client.delete.called)+ return++ collection_name = f"test_ob_{uuid.uuid4().hex[:8]}"+ store = OceanBaseStore(+ collection_name=collection_name,+ dimensions=3,+ uri=os.getenv("OCEANBASE_URI", ""),+ user=os.getenv("OCEANBASE_USER", ""),+ password=os.getenv("OCEANBASE_PASSWORD", ""),+ db_name=os.getenv("OCEANBASE_DB", ""),+ )++ client = store.get_client()+ client.drop_collection(collection_name)++ try:+ await store.add(+ [+ Document(+ embedding=[0.1, 0.2, 0.3],+ metadata=DocMetadata(+ content=TextBlock(+ type="text",+ text="This is a test document.",+ ),+ doc_id="doc1",+ chunk_id=0,+ total_chunks=2,+ ),+ ),+ Document(+ embedding=[0.9, 0.1, 0.4],+ metadata=DocMetadata(+ content=TextBlock(+ type="text",+ text="This is another test document.",+ ),+ doc_id="doc1",+ chunk_id=1,+ total_chunks=2,+ ),+ ),+ ],+ )++ res = await store.search(+ query_embedding=[0.15, 0.25, 0.35],+ limit=3,+ score_threshold=0.8,+ )+ self.assertEqual(len(res), 1)+ self.assertEqual(+ round(res[0].score, 4),+ 0.9974,+ )+ self.assertEqual(+ res[0].metadata.content["text"],+ "This is a test document.",+ )+ finally:+ client.drop_collection(collection_name)+async def test_alibabacloud_mysql_store(self) -> None:"""Test the AlibabaCloudMySQLStore implementation using mocks."""# Create mock MySQL module and connector