Skip to content

Build and run agents you can see, understand and trust.

feat(rag): add OceanBaseStore as a vector database choice (#1078)

Chojan Shang9745c3087407
diff --git a/examples/functionality/vector_store/oceanbase/README.md b/examples/functionality/vector_store/oceanbase/README.md
new file mode 100644
index 000000000..d8d30424f
--- /dev/null
+++ b/examples/functionality/vector_store/oceanbase/README.md
@@ -0,0 +1,164 @@
+# OceanBase Vector Store
+
+This example demonstrates how to use **OceanBaseStore** for vector storage and semantic search in AgentScope.
+It includes CRUD operations, metadata filtering, document chunking, and distance metric tests.
+
+### Quick Start
+
+Install dependencies (including `pyobvector`):
+
+```bash
+pip install -e .[full]
+```
+
+Start seekdb (a minimal OceanBase-compatible instance):
+
+```bash
+docker run -d -p 2881:2881 oceanbase/seekdb
+```
+
+Run the example script:
+
+```bash
+python main.py
+```
+
+> **Note:** The script defaults to `127.0.0.1:2881`, user `root`, database `test`.
+> If you use a multi-tenant OceanBase account (e.g., `root@test`), override via environment variables.
+
+## Usage
+
+### Initialize Store
+
+```python
+from agentscope.rag import OceanBaseStore
+
+store = OceanBaseStore(
+ collection_name="test_collection",
+ dimensions=768,
+ distance="COSINE",
+ uri="127.0.0.1:2881",
+ user="root",
+ password="",
+ db_name="test",
+)
+```
+
+### Add Documents
+
+```python
+from agentscope.rag import Document, DocMetadata
+from agentscope.message import TextBlock
+
+doc = Document(
+ metadata=DocMetadata(
+ content=TextBlock(type="text", text="Your document text"),
+ doc_id="doc_1",
+ chunk_id=0,
+ total_chunks=1,
+ ),
+ embedding=[0.1, 0.2, 0.3],
+)
+
+await store.add([doc])
+```
+
+### Search
+
+```python
+results = await store.search(
+ query_embedding=[0.1, 0.2, 0.3],
+ limit=5,
+ score_threshold=0.9,
+)
+```
+
+### Filter Search
+
+```python
+client = store.get_client()
+table = client.load_table(collection_name="test_collection")
+
+results = await store.search(
+ query_embedding=[0.1, 0.2, 0.3],
+ limit=5,
+ flter=[table.c["doc_id"].like("doc%")],
+)
+```
+
+> Note: The parameter name is `flter` (missing the "i") to avoid clashing with
+> Python's built-in `filter` and follows the underlying library's convention.
+
+### Delete
+
+```python
+client = store.get_client()
+table = client.load_table(collection_name="test_collection")
+
+await store.delete(where=[table.c["doc_id"] == "doc_1"])
+```
+
+## Distance Metrics
+
+| Metric | Description | Best For |
+|--------|-------------|----------|
+| **COSINE** | Cosine similarity | Text embeddings (recommended) |
+| **L2** | Euclidean distance | Spatial data |
+| **IP** | Inner product | Recommendation systems |
+
+## Filter Expressions
+
+Build filters using SQLAlchemy expressions and pass them via `flter`:
+
+```python
+table = store.get_client().load_table("test_collection")
+
+filters = [
+ table.c["doc_id"] == "doc_1",
+ table.c["doc_id"].like("prefix%"),
+ table.c["chunk_id"] >= 0,
+]
+```
+
+## Advanced Usage
+
+### Access Underlying Client
+
+```python
+client = store.get_client()
+stats = client.get_collection_stats(collection_name="test_collection")
+```
+
+### Document Metadata
+
+- `content`: Text content (TextBlock)
+- `doc_id`: Unique document identifier
+- `chunk_id`: Chunk position (0-indexed)
+- `total_chunks`: Total chunks in document
+
+## FAQ
+
+**What embedding dimension should I use?**
+Match your embedding model's output dimension (e.g., 768 for BERT, 1536 for OpenAI ada-002).
+
+**Can I change the distance metric after creation?**
+No, create a new collection with the desired metric.
+
+**How do I clean up test data?**
+Drop the collection via the underlying client or remove the seekdb container volume.
+
+## Environment Variables
+
+The script supports the following environment variables to override connection settings:
+
+```bash
+export OCEANBASE_URI="127.0.0.1:2881"
+export OCEANBASE_USER="root"
+export OCEANBASE_PASSWORD=""
+export OCEANBASE_DB="test"
+```
+
+## References
+
+- [OceanBase Vector Store](https://github.com/oceanbase/pyobvector)
+- [AgentScope RAG Tutorial](https://doc.agentscope.io/tutorial/task_rag.html)
diff --git a/examples/functionality/vector_store/oceanbase/main.py b/examples/functionality/vector_store/oceanbase/main.py
new file mode 100644
index 000000000..41097484f
--- /dev/null
+++ b/examples/functionality/vector_store/oceanbase/main.py
@@ -0,0 +1,350 @@
+# -*- coding: utf-8 -*-
+"""Example of using OceanBaseStore in AgentScope RAG system."""
+import asyncio
+import os
+
+from agentscope.rag import (
+ OceanBaseStore,
+ Document,
+ DocMetadata,
+)
+from agentscope.message import TextBlock
+
+
+def _create_store(
+ collection_name: str,
+ dimensions: int = 4,
+ distance: str = "COSINE",
+) -> OceanBaseStore:
+ return OceanBaseStore(
+ collection_name=collection_name,
+ dimensions=dimensions,
+ distance=distance,
+ uri=os.getenv("OCEANBASE_URI", "127.0.0.1:2881"),
+ user=os.getenv("OCEANBASE_USER", "root"),
+ password=os.getenv("OCEANBASE_PASSWORD", ""),
+ db_name=os.getenv("OCEANBASE_DB", "test"),
+ )
+
+
+async def example_basic_operations() -> None:
+ """The example of basic CRUD operations with OceanBaseStore."""
+ print("\n" + "=" * 60)
+ print("Test 1: Basic CRUD Operations")
+ print("=" * 60)
+
+ store = _create_store(collection_name="ob_basic_collection")
+ store.get_client().drop_collection("ob_basic_collection")
+
+ print("✓ OceanBaseStore initialized")
+
+ test_docs = [
+ Document(
+ metadata=DocMetadata(
+ content=TextBlock(
+ type="text",
+ text="Artificial Intelligence is the future",
+ ),
+ doc_id="doc_1",
+ chunk_id=0,
+ total_chunks=1,
+ ),
+ embedding=[0.1, 0.2, 0.3, 0.4],
+ ),
+ Document(
+ metadata=DocMetadata(
+ content=TextBlock(
+ type="text",
+ text="Machine Learning is a subset of AI",
+ ),
+ doc_id="doc_2",
+ chunk_id=0,
+ total_chunks=1,
+ ),
+ embedding=[0.2, 0.3, 0.4, 0.5],
+ ),
+ Document(
+ metadata=DocMetadata(
+ content=TextBlock(
+ type="text",
+ text="Deep Learning uses neural networks",
+ ),
+ doc_id="doc_3",
+ chunk_id=0,
+ total_chunks=1,
+ ),
+ embedding=[0.3, 0.4, 0.5, 0.6],
+ ),
+ ]
+
+ await store.add(test_docs)
+ print(f"✓ Added {len(test_docs)} documents to the store")
+
+ query_embedding = [0.15, 0.25, 0.35, 0.45]
+ results = await store.search(
+ query_embedding=query_embedding,
+ limit=2,
+ )
+
+ print(f"\n✓ Search completed, found {len(results)} results:")
+ for i, result in enumerate(results, 1):
+ print(f" {i}. Score: {result.score:.4f}")
+ print(f" Content: {result.metadata.content}")
+ print(f" Doc ID: {result.metadata.doc_id}")
+
+ results_filtered = await store.search(
+ query_embedding=query_embedding,
+ limit=5,
+ score_threshold=0.9,
+ )
+ print(f"\n✓ Search with threshold (>0.9): {len(results_filtered)} results")
+
+ client = store.get_client()
+ table = client.load_table(collection_name="ob_basic_collection")
+ await store.delete(where=[table.c["doc_id"] == "doc_2"])
+ print("\n✓ Deleted document with doc_id='doc_2'")
+
+ results_after_delete = await store.search(
+ query_embedding=query_embedding,
+ limit=5,
+ )
+ print(f"✓ After deletion: {len(results_after_delete)} documents remain")
+
+ print(f"\n✓ Got MilvusLikeClient: {type(client).__name__}")
+
+
+async def example_filter_search() -> None:
+ """The example of search with metadata filtering."""
+ print("\n" + "=" * 60)
+ print("Test 2: Search with Metadata Filtering")
+ print("=" * 60)
+
+ store = _create_store(collection_name="ob_filter_collection")
+ client = store.get_client()
+ client.drop_collection("ob_filter_collection")
+
+ docs = [
+ Document(
+ metadata=DocMetadata(
+ content=TextBlock(
+ type="text",
+ text="Python is a programming language",
+ ),
+ doc_id="prog_1",
+ chunk_id=0,
+ total_chunks=1,
+ ),
+ embedding=[0.1, 0.2, 0.3, 0.4],
+ ),
+ Document(
+ metadata=DocMetadata(
+ content=TextBlock(
+ type="text",
+ text="Java is used for enterprise applications",
+ ),
+ doc_id="prog_2",
+ chunk_id=0,
+ total_chunks=1,
+ ),
+ embedding=[0.2, 0.3, 0.4, 0.5],
+ ),
+ Document(
+ metadata=DocMetadata(
+ content=TextBlock(
+ type="text",
+ text="Neural networks are used in AI",
+ ),
+ doc_id="ai_1",
+ chunk_id=0,
+ total_chunks=1,
+ ),
+ embedding=[0.3, 0.4, 0.5, 0.6],
+ ),
+ Document(
+ metadata=DocMetadata(
+ content=TextBlock(
+ type="text",
+ text="Deep learning requires GPUs",
+ ),
+ doc_id="ai_2",
+ chunk_id=0,
+ total_chunks=1,
+ ),
+ embedding=[0.4, 0.5, 0.6, 0.7],
+ ),
+ ]
+
+ await store.add(docs)
+ print(f"✓ Added {len(docs)} documents with different doc_id prefixes")
+
+ query_embedding = [0.25, 0.35, 0.45, 0.55]
+ all_results = await store.search(
+ query_embedding=query_embedding,
+ limit=4,
+ )
+ print(f"\n✓ Search without filter: {len(all_results)} results")
+ for i, result in enumerate(all_results, 1):
+ print(
+ f" {i}. Doc ID: {result.metadata.doc_id}, "
+ f"Score: {result.score:.4f}",
+ )
+
+ table = client.load_table(collection_name="ob_filter_collection")
+ prog_results = await store.search(
+ query_embedding=query_embedding,
+ limit=4,
+ flter=[table.c["doc_id"].like("prog%")],
+ )
+ print("\n✓ Search with filter (doc_id like 'prog%'):")
+ for i, result in enumerate(prog_results, 1):
+ print(
+ f" {i}. Doc ID: {result.metadata.doc_id}, "
+ f"Score: {result.score:.4f}",
+ )
+
+ ai_results = await store.search(
+ query_embedding=query_embedding,
+ limit=4,
+ flter=[table.c["doc_id"].like("ai%")],
+ )
+ print("\n✓ Search with filter (doc_id like 'ai%'):")
+ for i, result in enumerate(ai_results, 1):
+ print(
+ f" {i}. Doc ID: {result.metadata.doc_id}, "
+ f"Score: {result.score:.4f}",
+ )
+
+
+async def example_multiple_chunks() -> None:
+ """The example of documents with multiple chunks."""
+ print("\n" + "=" * 60)
+ print("Test 3: Documents with Multiple Chunks")
+ print("=" * 60)
+
+ store = _create_store(collection_name="ob_chunks_collection")
+ store.get_client().drop_collection("ob_chunks_collection")
+
+ chunks = [
+ Document(
+ metadata=DocMetadata(
+ content=TextBlock(
+ type="text",
+ text="Chapter 1: Introduction to AI",
+ ),
+ doc_id="book_1",
+ chunk_id=0,
+ total_chunks=3,
+ ),
+ embedding=[0.1, 0.2, 0.3, 0.4],
+ ),
+ Document(
+ metadata=DocMetadata(
+ content=TextBlock(
+ type="text",
+ text="Chapter 2: Machine Learning Basics",
+ ),
+ doc_id="book_1",
+ chunk_id=1,
+ total_chunks=3,
+ ),
+ embedding=[0.2, 0.3, 0.4, 0.5],
+ ),
+ Document(
+ metadata=DocMetadata(
+ content=TextBlock(
+ type="text",
+ text="Chapter 3: Deep Learning Advanced",
+ ),
+ doc_id="book_1",
+ chunk_id=2,
+ total_chunks=3,
+ ),
+ embedding=[0.3, 0.4, 0.5, 0.6],
+ ),
+ ]
+
+ await store.add(chunks)
+ print(f"✓ Added document with {len(chunks)} chunks")
+
+ query_embedding = [0.2, 0.3, 0.4, 0.5]
+ results = await store.search(
+ query_embedding=query_embedding,
+ limit=3,
+ )
+
+ print("\n✓ Search results for multi-chunk document:")
+ for i, result in enumerate(results, 1):
+ chunk_info = (
+ f"{result.metadata.chunk_id}/{result.metadata.total_chunks}"
+ )
+ print(f" {i}. Chunk {chunk_info}")
+ print(f" Content: {result.metadata.content}")
+ print(f" Score: {result.score:.4f}")
+
+
+async def example_distance_metrics() -> None:
+ """The example of different distance metrics."""
+ print("\n" + "=" * 60)
+ print("Test 4: Different Distance Metrics")
+ print("=" * 60)
+
+ metrics = ["COSINE", "L2", "IP"]
+
+ for metric in metrics:
+ print(f"\n--- Testing {metric} metric ---")
+ collection_name = f"ob_{metric}_collection"
+ store = _create_store(
+ collection_name=collection_name,
+ distance=metric,
+ )
+ store.get_client().drop_collection(collection_name)
+
+ docs = [
+ Document(
+ metadata=DocMetadata(
+ content=TextBlock(
+ type="text",
+ text=f"Test doc for {metric}",
+ ),
+ doc_id=f"doc_{metric}_1",
+ chunk_id=0,
+ total_chunks=1,
+ ),
+ embedding=[0.1, 0.2, 0.3, 0.4],
+ ),
+ ]
+
+ await store.add(docs)
+ results = await store.search(
+ query_embedding=[0.1, 0.2, 0.3, 0.4],
+ limit=1,
+ )
+
+ print(f"✓ {metric} metric: Score = {results[0].score:.4f}")
+
+
+async def main() -> None:
+ """Run all example."""
+ print("\n" + "=" * 60)
+ print("OceanBaseStore Comprehensive Test Suite")
+ print("=" * 60)
+
+ try:
+ await example_basic_operations()
+ await example_filter_search()
+ await example_multiple_chunks()
+ await example_distance_metrics()
+
+ print("\n" + "=" * 60)
+ print("✓ All tests completed successfully!")
+ print("=" * 60)
+
+ except Exception as e:
+ print(f"\n✗ Test failed with error: {e}")
+ import traceback
+
+ traceback.print_exc()
+
+
+if __name__ == "__main__":
+ asyncio.run(main())
diff --git a/pyproject.toml b/pyproject.toml
index b658327c7..17db9a330 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -118,11 +118,13 @@ qdrant = ["qdrant-client==1.15.1"]
milvus = ["pymilvus[milvus_lite]"]
ali_mysql = ["mysql-connector-python"]
mongodb = ["pymongo"]
+oceanbase = ["pyobvector>=0.2.0,<0.3.0"]
vdbs = [
"agentscope[ali_mysql]",
"agentscope[qdrant]",
"agentscope[milvus]",
"agentscope[mongodb]",
+ "agentscope[oceanbase]",
]
rag = [
diff --git a/src/agentscope/rag/__init__.py b/src/agentscope/rag/__init__.py
index 00c360b9d..2c0d5909f 100644
--- a/src/agentscope/rag/__init__.py
+++ b/src/agentscope/rag/__init__.py
@@ -18,6 +18,7 @@ from ._store import (
VDBStoreBase,
QdrantStore,
MilvusLiteStore,
+ OceanBaseStore,
MongoDBStore,
AlibabaCloudMySQLStore,
)
@@ -38,6 +39,7 @@ __all__ = [
"VDBStoreBase",
"QdrantStore",
"MilvusLiteStore",
+ "OceanBaseStore",
"MongoDBStore",
"AlibabaCloudMySQLStore",
"KnowledgeBase",
diff --git a/src/agentscope/rag/_store/__init__.py b/src/agentscope/rag/_store/__init__.py
index 96e0ce323..67951ace4 100644
--- a/src/agentscope/rag/_store/__init__.py
+++ b/src/agentscope/rag/_store/__init__.py
@@ -6,6 +6,7 @@ from ._store_base import (
)
from ._qdrant_store import QdrantStore
from ._milvuslite_store import MilvusLiteStore
+from ._oceanbase_store import OceanBaseStore
from ._mongodb_store import MongoDBStore
from ._alibabacloud_mysql_store import AlibabaCloudMySQLStore
@@ -13,6 +14,7 @@ __all__ = [
"VDBStoreBase",
"QdrantStore",
"MilvusLiteStore",
+ "OceanBaseStore",
"MongoDBStore",
"AlibabaCloudMySQLStore",
]
diff --git a/src/agentscope/rag/_store/_oceanbase_store.py b/src/agentscope/rag/_store/_oceanbase_store.py
new file mode 100644
index 000000000..3ace3eeb4
--- /dev/null
+++ b/src/agentscope/rag/_store/_oceanbase_store.py
@@ -0,0 +1,552 @@
+# -*- coding: utf-8 -*-
+"""The OceanBase vector store implementation."""
+import asyncio
+import json
+from typing import Any, Callable, Literal, TYPE_CHECKING
+
+from .._reader import Document
+from ._store_base import VDBStoreBase
+from .._document import DocMetadata
+from ..._utils._common import _map_text_to_uuid
+from ...message import TextBlock
+from ...types import Embedding
+
+if TYPE_CHECKING:
+ from pyobvector import MilvusLikeClient
+else:
+ MilvusLikeClient = "pyobvector.MilvusLikeClient"
+
+
+# Metric configuration: data-driven approach focusing on exceptions
+# Base metric names for pyobvector
+_METRIC_NAMES = {
+ "COSINE": "cosine",
+ "L2": "l2",
+ "IP": "inner_product",
+}
+
+# Only configure special cases that differ from the base
+_SEARCH_METRIC_OVERRIDES = {
+ "IP": "ip", # IP uses "ip" for search to get positive inner product
+}
+
+_SCORE_CONVERTERS: dict[str, Callable[[float], float]] = {
+ "COSINE": lambda d: 1.0 - d, # COSINE converts distance to similarity
+}
+
+
+class OceanBaseStore(VDBStoreBase):
+ """The OceanBase vector store implementation, supporting OceanBase and
+ seekdb via pyobvector."""
+
+ # Field names - using descriptive constants to avoid magic strings
+ PRIMARY_FIELD = "id"
+ VECTOR_FIELD = "embedding"
+ DOC_ID_FIELD = "doc_id"
+ CHUNK_ID_FIELD = "chunk_id"
+ TOTAL_CHUNKS_FIELD = "total_chunks"
+ CONTENT_FIELD = "content"
+
+ # Index configuration
+ INDEX_NAME = "vidx"
+ INDEX_TYPE = "hnsw"
+
+ def __init__(
+ self,
+ collection_name: str,
+ dimensions: int,
+ uri: str = "127.0.0.1:2881",
+ user: str = "root@test",
+ password: str = "",
+ db_name: str = "test",
+ distance: Literal["COSINE", "L2", "IP"] = "COSINE",
+ client_kwargs: dict[str, Any] | None = None,
+ collection_kwargs: dict[str, Any] | None = None,
+ ) -> None:
+ """Initialize the OceanBase vector store.
+
+ Args:
+ collection_name (`str`):
+ The name of the collection to store the embeddings.
+ dimensions (`int`):
+ The dimension of the embeddings.
+ uri (`str`, defaults to `"127.0.0.1:2881"`):
+ The OceanBase server URI, such as "127.0.0.1:2881".
+ user (`str`, defaults to `"root@test"`):
+ The username for authentication.
+ password (`str`, defaults to `""`):
+ The password for authentication.
+ db_name (`str`, defaults to `"test"`):
+ The database name to connect to.
+ distance (`Literal["COSINE", "L2", "IP"]`, defaults to `"COSINE"`):
+ The distance metric to use for the collection. Can be one of
+ "COSINE", "L2", or "IP".
+ client_kwargs (`dict[str, Any] | None`, optional):
+ Keyword arguments passed to `pyobvector.MilvusLikeClient`.
+ Explicit connection arguments override matching keys here.
+ collection_kwargs (`dict[str, Any] | None`, optional):
+ Keyword arguments passed to `create_collection`.
+ """
+ try:
+ import pyobvector
+ except ImportError as e:
+ raise ImportError(
+ "OceanBase client is not installed. Please install it with "
+ "`pip install pyobvector`.",
+ ) from e
+
+ self._pyobvector = pyobvector
+ client_kwargs = dict(client_kwargs or {})
+
+ self._client = pyobvector.MilvusLikeClient(
+ uri=uri,
+ user=user,
+ password=password,
+ db_name=db_name,
+ **client_kwargs,
+ )
+
+ self.collection_name = collection_name
+ self.dimensions = dimensions
+ self.distance = distance
+ self.collection_kwargs = collection_kwargs or {}
+ self._collection_ready = False
+
+ def _get_metric_type(self) -> str:
+ """Get the metric type for index creation."""
+ return _METRIC_NAMES[self.distance]
+
+ def _get_search_metric_type(self) -> str:
+ """Get the search metric type for queries.
+
+ Returns the override value if exists, otherwise uses index metric.
+ This allows special handling (e.g., IP uses "ip" for positive values).
+ """
+ return _SEARCH_METRIC_OVERRIDES.get(
+ self.distance,
+ self._get_metric_type(),
+ )
+
+ async def _validate_collection(self) -> None:
+ """Validate the collection exists, if not, create it."""
+ if self._collection_ready:
+ return
+
+ if await asyncio.to_thread(
+ self._client.has_collection,
+ self.collection_name,
+ ):
+ self._collection_ready = True
+ return
+
+ collection_kwargs = dict(self.collection_kwargs)
+
+ if "schema" not in collection_kwargs:
+ collection_kwargs["schema"] = self._create_schema()
+
+ if "index_params" not in collection_kwargs:
+ collection_kwargs["index_params"] = self._create_index_params()
+
+ await asyncio.to_thread(
+ self._client.create_collection,
+ collection_name=self.collection_name,
+ **collection_kwargs,
+ )
+ self._collection_ready = True
+
+ def _create_schema(self) -> Any:
+ """Create the collection schema with all required fields.
+
+ Returns:
+ Schema object with primary, vector, and metadata fields configured.
+ """
+ schema = self._client.create_schema()
+
+ # Primary key field
+ schema.add_field(
+ field_name=self.PRIMARY_FIELD,
+ datatype=self._pyobvector.DataType.VARCHAR,
+ is_primary=True,
+ auto_id=False,
+ max_length=36,
+ )
+
+ # Vector field
+ schema.add_field(
+ field_name=self.VECTOR_FIELD,
+ datatype=self._pyobvector.DataType.FLOAT_VECTOR,
+ dim=self.dimensions,
+ )
+
+ # Metadata fields
+ schema.add_field(
+ field_name=self.DOC_ID_FIELD,
+ datatype=self._pyobvector.DataType.STRING,
+ )
+ schema.add_field(
+ field_name=self.CHUNK_ID_FIELD,
+ datatype=self._pyobvector.DataType.INT64,
+ )
+ schema.add_field(
+ field_name=self.TOTAL_CHUNKS_FIELD,
+ datatype=self._pyobvector.DataType.INT64,
+ )
+ schema.add_field(
+ field_name=self.CONTENT_FIELD,
+ datatype=self._pyobvector.DataType.JSON,
+ nullable=True,
+ )
+
+ return schema
+
+ def _create_index_params(self) -> Any:
+ """Create index parameters for vector search.
+
+ Returns:
+ Index parameters configured with HNSW index and appropriate metric.
+ """
+ index_params = self._client.prepare_index_params()
+ index_params.add_index(
+ field_name=self.VECTOR_FIELD,
+ index_type=self.INDEX_TYPE,
+ index_name=self.INDEX_NAME,
+ metric_type=self._get_metric_type(),
+ )
+ return index_params
+
+ @staticmethod
+ def _content_to_text(content: Any) -> str:
+ """Extract text string from content of various formats."""
+ if isinstance(content, str):
+ return content
+ if isinstance(content, dict) and content.get("type") == "text":
+ text = content.get("text", "")
+ return text if isinstance(text, str) else ""
+ return ""
+
+ @staticmethod
+ def _normalize_content(content: Any, fallback_text: str) -> Any:
+ """Normalize content to TextBlock format."""
+ # Already in correct format
+ if isinstance(content, dict) and content.get("type"):
+ return content
+ # Convert string to TextBlock
+ if isinstance(content, str):
+ return TextBlock(type="text", text=content)
+ # Use fallback
+ return TextBlock(type="text", text=fallback_text or "")
+
+ def _document_to_dict(self, doc: Document) -> dict[str, Any]:
+ """Convert a Document to a dictionary for insertion.
+
+ Args:
+ doc (`Document`):
+ Document to convert
+
+ Returns:
+ `dict[str, Any]`:
+ Dictionary with fields ready for database insertion
+
+ Raises:
+ `ValueError`:
+ If document embedding is None
+ """
+ if doc.embedding is None:
+ raise ValueError(
+ "Document embedding is required for OceanBaseStore.add.",
+ )
+
+ # Create unique ID from document metadata
+ unique_string = json.dumps(
+ {
+ "doc_id": doc.metadata.doc_id,
+ "chunk_id": doc.metadata.chunk_id,
+ "content": doc.metadata.content,
+ },
+ ensure_ascii=True,
+ sort_keys=True,
+ )
+
+ return {
+ self.PRIMARY_FIELD: _map_text_to_uuid(unique_string),
+ self.VECTOR_FIELD: doc.embedding,
+ self.DOC_ID_FIELD: doc.metadata.doc_id,
+ self.CHUNK_ID_FIELD: doc.metadata.chunk_id,
+ self.TOTAL_CHUNKS_FIELD: doc.metadata.total_chunks,
+ self.CONTENT_FIELD: doc.metadata.content,
+ }
+
+ async def add(self, documents: list[Document], **kwargs: Any) -> None:
+ """Add embeddings to the OceanBase vector store.
+
+ Args:
+ documents (`list[Document]`):
+ A list of embedding records to be recorded in the store.
+ """
+ await self._validate_collection()
+
+ data = [self._document_to_dict(doc) for doc in documents]
+
+ await asyncio.to_thread(
+ self._client.insert,
+ collection_name=self.collection_name,
+ data=data,
+ **kwargs,
+ )
+
+ @staticmethod
+ def _extract_distance(
+ row: dict[str, Any],
+ output_fields: list[str],
+ ) -> float | None:
+ """Extract distance value from search result row.
+
+ The distance is stored in an extra field that's not in output_fields.
+ """
+ extra_keys = [key for key in row if key not in output_fields]
+ return row.get(extra_keys[-1]) if extra_keys else None
+
+ def _create_document_from_row(
+ self,
+ row: dict[str, Any],
+ output_fields: list[str],
+ ) -> tuple[Document, float | None]:
+ """Create a Document from a search result row.
+
+ Args:
+ row (`dict[str, Any]`):
+ Search result row containing document data
+ output_fields (`list[str]`):
+ List of fields requested in output
+
+ Returns:
+ `tuple[Document, float | None]`:
+ Tuple of (Document, score)
+ """
+ distance = self._extract_distance(row, output_fields)
+ score = self._convert_distance_to_score(distance)
+
+ content_value = row.get(self.CONTENT_FIELD)
+ content_text = self._content_to_text(content_value)
+ content = self._normalize_content(content_value, content_text)
+
+ doc_metadata = DocMetadata(
+ content=content,
+ doc_id=str(row.get(self.DOC_ID_FIELD, "")),
+ chunk_id=int(row.get(self.CHUNK_ID_FIELD) or 0),
+ total_chunks=int(row.get(self.TOTAL_CHUNKS_FIELD) or 0),
+ )
+
+ return (
+ Document(
+ embedding=None,
+ score=score,
+ metadata=doc_metadata,
+ ),
+ score,
+ )
+
+ def _convert_distance_to_score(
+ self,
+ distance: float | None,
+ ) -> float | None:
+ """Convert distance value to score based on metric type.
+
+ Args:
+ distance (`float | None`):
+ Raw distance value from database
+
+ Returns:
+ `float | None`:
+ Converted score (similarity for COSINE, raw value for others)
+
+ """
+ if distance is None:
+ return None
+
+ # Apply converter if defined, otherwise return raw value (identity)
+ converter = _SCORE_CONVERTERS.get(self.distance)
+ return converter(distance) if converter else distance
+
+ async def search(
+ self,
+ query_embedding: Embedding,
+ limit: int,
+ score_threshold: float | None = None,
+ **kwargs: Any,
+ ) -> list[Document]:
+ """Search relevant documents from the OceanBase vector store.
+
+ Args:
+ query_embedding (`Embedding`):
+ The embedding of the query text.
+ limit (`int`):
+ The number of relevant documents to retrieve.
+ score_threshold (`float | None`, optional):
+ The threshold of the score to filter the results.
+ Note: Score semantics (aligned with Milvus):
+ - COSINE: similarity [0,1], higher = more similar
+ - L2: Euclidean distance, smaller = more similar
+ - IP: inner product, larger = more similar
+ **kwargs (`Any`):
+ Additional arguments for the search API.
+ - flter (`list`): Filter conditions.
+ - partition_names (`list[str]`): Partition filter.
+ - output_fields (`list[str]`): Fields to include in results.
+ - search_params (`dict`): Search parameters.
+ """
+ await self._validate_collection()
+
+ # Remove unsupported parameter
+ kwargs.pop("with_dist", None)
+
+ # Prepare output fields and search parameters
+ output_fields = self._prepare_output_fields(
+ kwargs.pop("output_fields", None),
+ )
+ search_params = self._prepare_search_params(
+ kwargs.pop("search_params", None),
+ )
+
+ results = await asyncio.to_thread(
+ self._client.search,
+ collection_name=self.collection_name,
+ data=query_embedding,
+ anns_field=self.VECTOR_FIELD,
+ with_dist=True,
+ limit=limit,
+ output_fields=output_fields,
+ search_params=search_params,
+ **kwargs,
+ )
+
+ # Process results and filter by score threshold
+ return self._filter_results_by_threshold(
+ results,
+ output_fields,
+ score_threshold,
+ )
+
+ def _prepare_output_fields(
+ self,
+ output_fields: list[str] | None,
+ ) -> list[str]:
+ """Prepare output fields ensuring all required fields are included.
+
+ Args:
+ output_fields (`list[str] | None`):
+ User-specified output fields or None
+
+ Returns:
+ `list[str]`:
+ List of output fields with required fields included
+ """
+ required_fields = [
+ self.DOC_ID_FIELD,
+ self.CHUNK_ID_FIELD,
+ self.TOTAL_CHUNKS_FIELD,
+ self.CONTENT_FIELD,
+ ]
+
+ output_fields = output_fields or required_fields
+
+ # Use dict.fromkeys to preserve order while removing duplicates
+ return list(dict.fromkeys(output_fields + required_fields))
+
+ def _prepare_search_params(
+ self,
+ search_params: dict[str, Any] | None,
+ ) -> dict[str, Any]:
+ """Prepare search parameters with appropriate metric type.
+
+ Args:
+ search_params (`dict[str, Any] | None`):
+ User-specified search parameters or None
+
+ Returns:
+ `dict[str, Any]`:
+ Search parameters dict with metric_type set
+ """
+ search_params = dict(search_params or {}) # Create a copy
+ search_params.setdefault("metric_type", self._get_search_metric_type())
+ return search_params
+
+ def _filter_results_by_threshold(
+ self,
+ results: list[dict[str, Any]],
+ output_fields: list[str],
+ score_threshold: float | None,
+ ) -> list[Document]:
+ """Filter search results by score threshold.
+
+ Args:
+ results (`list[dict[str, Any]]`):
+ Raw search results from database
+ output_fields (`list[str]`):
+ List of output fields
+ score_threshold (`float | None`):
+ Minimum score threshold or None
+
+ Returns:
+ `list[Document]`:
+ List of filtered Document objects
+ """
+ documents = []
+ for row in results:
+ doc, score = self._create_document_from_row(row, output_fields)
+
+ if score_threshold is not None and (
+ score is None or score < score_threshold
+ ):
+ continue
+
+ documents.append(doc)
+
+ return documents
+
+ async def delete(
+ self,
+ ids: list[str] | None = None,
+ where: Any | None = None,
+ where_document: Any | None = None,
+ **kwargs: Any,
+ ) -> None:
+ """Delete documents from the OceanBase vector store.
+
+ Args:
+ ids (`list[str] | None`, optional):
+ List of entity IDs to delete.
+ where (`Any | None`, optional):
+ Filter conditions for deletion.
+ where_document (`Any | None`, optional):
+ Unsupported in OceanBaseStore.
+ """
+ await self._validate_collection()
+
+ if where_document is not None:
+ raise ValueError(
+ "where_document is not supported for OceanBaseStore.delete.",
+ )
+
+ if ids is None and where is None:
+ raise ValueError(
+ "At least one of ids or where must be provided for deletion.",
+ )
+
+ await asyncio.to_thread(
+ self._client.delete,
+ collection_name=self.collection_name,
+ ids=ids,
+ flter=where,
+ **kwargs,
+ )
+
+ def get_client(self) -> MilvusLikeClient:
+ """Get the underlying OceanBase client, so that developers can access
+ the full functionality of OceanBase.
+
+ Returns:
+ `MilvusLikeClient`:
+ The underlying OceanBase client.
+ """
+ return self._client
diff --git a/tests/rag_store_test.py b/tests/rag_store_test.py
index b7db30f3a..72e53ad69 100644
--- a/tests/rag_store_test.py
+++ b/tests/rag_store_test.py
@@ -1,6 +1,8 @@
# -*- coding: utf-8 -*-
"""Test the RAG store implementations."""
import os
+import types
+import uuid
from typing import AsyncGenerator
from unittest import IsolatedAsyncioTestCase
from unittest.mock import MagicMock, patch, AsyncMock
@@ -11,6 +13,7 @@ from agentscope.rag import (
Document,
DocMetadata,
MilvusLiteStore,
+ OceanBaseStore,
AlibabaCloudMySQLStore,
MongoDBStore,
)
@@ -132,6 +135,160 @@ class RAGStoreTest(IsolatedAsyncioTestCase):
"This is a test document.",
)
+ async def test_oceanbase_store(self) -> None:
+ """Use real OceanBase when env is provided, otherwise use a mock."""
+
+ def _make_mock_pyobvector(
+ search_rows: list[dict],
+ ) -> tuple[types.SimpleNamespace, MagicMock]:
+ """Create a minimal pyobvector mock aligned with the
+ existing style."""
+ mock_client = MagicMock()
+ mock_client.has_collection.return_value = False
+ mock_client.create_schema.return_value = MagicMock()
+ mock_client.prepare_index_params.return_value = MagicMock()
+ mock_client.search.return_value = search_rows
+
+ mock_pyobvector = types.SimpleNamespace(
+ MilvusLikeClient=MagicMock(return_value=mock_client),
+ DataType=types.SimpleNamespace(
+ VARCHAR="VARCHAR",
+ FLOAT_VECTOR="FLOAT_VECTOR",
+ STRING="STRING",
+ INT64="INT64",
+ JSON="JSON",
+ ),
+ )
+ return mock_pyobvector, mock_client
+
+ required_vars = [
+ "OCEANBASE_URI",
+ "OCEANBASE_USER",
+ "OCEANBASE_DB",
+ ]
+ missing_vars = [var for var in required_vars if not os.getenv(var)]
+ if missing_vars:
+ # COSINE distance = 1 - similarity
+ mock_rows = [
+ {
+ "doc_id": "doc1",
+ "chunk_id": 0,
+ "total_chunks": 2,
+ "content": {
+ "type": "text",
+ "text": "This is a test document.",
+ },
+ "distance": 1.0 - 0.9974,
+ },
+ ]
+ mock_pyobvector, mock_client = _make_mock_pyobvector(mock_rows)
+
+ with patch.dict("sys.modules", {"pyobvector": mock_pyobvector}):
+ store = OceanBaseStore(
+ collection_name=f"test_ob_{uuid.uuid4().hex[:8]}",
+ dimensions=3,
+ uri="127.0.0.1:2881",
+ user="root@test",
+ password="",
+ db_name="test",
+ )
+
+ await store.add(
+ [
+ Document(
+ embedding=[0.1, 0.2, 0.3],
+ metadata=DocMetadata(
+ content=TextBlock(
+ type="text",
+ text="This is a test document.",
+ ),
+ doc_id="doc1",
+ chunk_id=0,
+ total_chunks=2,
+ ),
+ ),
+ ],
+ )
+
+ self.assertTrue(mock_client.insert.called)
+ self.assertTrue(mock_client.create_collection.called)
+
+ res = await store.search(
+ query_embedding=[0.15, 0.25, 0.35],
+ limit=3,
+ score_threshold=0.8,
+ )
+ self.assertEqual(len(res), 1)
+ self.assertEqual(round(res[0].score, 4), 0.9974)
+ self.assertEqual(
+ res[0].metadata.content["text"],
+ "This is a test document.",
+ )
+
+ await store.delete(ids=["dummy-id"])
+ self.assertTrue(mock_client.delete.called)
+ return
+
+ collection_name = f"test_ob_{uuid.uuid4().hex[:8]}"
+ store = OceanBaseStore(
+ collection_name=collection_name,
+ dimensions=3,
+ uri=os.getenv("OCEANBASE_URI", ""),
+ user=os.getenv("OCEANBASE_USER", ""),
+ password=os.getenv("OCEANBASE_PASSWORD", ""),
+ db_name=os.getenv("OCEANBASE_DB", ""),
+ )
+
+ client = store.get_client()
+ client.drop_collection(collection_name)
+
+ try:
+ await store.add(
+ [
+ Document(
+ embedding=[0.1, 0.2, 0.3],
+ metadata=DocMetadata(
+ content=TextBlock(
+ type="text",
+ text="This is a test document.",
+ ),
+ doc_id="doc1",
+ chunk_id=0,
+ total_chunks=2,
+ ),
+ ),
+ Document(
+ embedding=[0.9, 0.1, 0.4],
+ metadata=DocMetadata(
+ content=TextBlock(
+ type="text",
+ text="This is another test document.",
+ ),
+ doc_id="doc1",
+ chunk_id=1,
+ total_chunks=2,
+ ),
+ ),
+ ],
+ )
+
+ res = await store.search(
+ query_embedding=[0.15, 0.25, 0.35],
+ limit=3,
+ score_threshold=0.8,
+ )
+ self.assertEqual(len(res), 1)
+ self.assertEqual(
+ round(res[0].score, 4),
+ 0.9974,
+ )
+ self.assertEqual(
+ res[0].metadata.content["text"],
+ "This is a test document.",
+ )
+ finally:
+ client.drop_collection(collection_name)
+
async def test_alibabacloud_mysql_store(self) -> None:
"""Test the AlibabaCloudMySQLStore implementation using mocks."""
# Create mock MySQL module and connector