Skip to content

armgit / camel

Public Imported from GitHub
0

🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org

feat: add tidb vector storage (#1942)

Mini256a7440eedbea6
diff --git a/camel/retrievers/auto_retriever.py b/camel/retrievers/auto_retriever.py
index 13974b15f..a2111b101 100644
--- a/camel/retrievers/auto_retriever.py
+++ b/camel/retrievers/auto_retriever.py
@@ -29,6 +29,7 @@ from camel.storages import (
BaseVectorStorage,
MilvusStorage,
QdrantStorage,
+ TiDBStorage,
)
from camel.types import StorageType
from camel.utils import Constants
@@ -90,6 +91,19 @@ class AutoRetriever:
url_and_api_key=self.url_and_api_key,
)
+ if self.storage_type == StorageType.TIDB:
+ if self.url_and_api_key is None:
+ raise ValueError(
+ "URL (database url) and API key required for TiDB storage "
+ "are not provided. Format: "
+ "mysql+pymysql://<username>:<password>@<host>:4000/test"
+ )
+ return TiDBStorage(
+ vector_dim=self.embedding_model.get_output_dim(),
+ collection_name=collection_name,
+ url_and_api_key=self.url_and_api_key,
+ )
+
if self.storage_type == StorageType.QDRANT:
return QdrantStorage(
vector_dim=self.embedding_model.get_output_dim(),
diff --git a/camel/storages/__init__.py b/camel/storages/__init__.py
index 12f96cd08..7809a1c77 100644
--- a/camel/storages/__init__.py
+++ b/camel/storages/__init__.py
@@ -28,6 +28,7 @@ from .vectordb_storages.base import (
)
from .vectordb_storages.milvus import MilvusStorage
from .vectordb_storages.qdrant import QdrantStorage
+from .vectordb_storages.tidb import TiDBStorage
__all__ = [
'BaseKeyValueStorage',
@@ -40,6 +41,7 @@ __all__ = [
'VectorDBQueryResult',
'QdrantStorage',
'MilvusStorage',
+ "TiDBStorage",
'BaseGraphStorage',
'Neo4jGraph',
'NebulaGraph',
diff --git a/camel/storages/vectordb_storages/__init__.py b/camel/storages/vectordb_storages/__init__.py
index a4b5ead4c..4f44fd2e1 100644
--- a/camel/storages/vectordb_storages/__init__.py
+++ b/camel/storages/vectordb_storages/__init__.py
@@ -21,6 +21,7 @@ from .base import (
)
from .milvus import MilvusStorage
from .qdrant import QdrantStorage
+from .tidb import TiDBStorage
__all__ = [
'BaseVectorStorage',
@@ -28,6 +29,7 @@ __all__ = [
'VectorDBQueryResult',
'QdrantStorage',
'MilvusStorage',
+ "TiDBStorage",
'VectorRecord',
'VectorDBStatus',
]
diff --git a/camel/storages/vectordb_storages/tidb.py b/camel/storages/vectordb_storages/tidb.py
new file mode 100644
index 000000000..ff5259caf
--- /dev/null
+++ b/camel/storages/vectordb_storages/tidb.py
@@ -0,0 +1,329 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+import json
+import logging
+import re
+from datetime import datetime
+from enum import Enum
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple
+
+from pytidb import Table
+
+from camel.storages.vectordb_storages import (
+ BaseVectorStorage,
+ VectorDBQuery,
+ VectorDBQueryResult,
+ VectorDBStatus,
+ VectorRecord,
+)
+from camel.utils import dependencies_required
+
+if TYPE_CHECKING:
+ from pytidb import TiDBClient
+
+logger = logging.getLogger(__name__)
+
+
+class EnumEncoder(json.JSONEncoder):
+ def default(self, obj):
+ if isinstance(obj, Enum):
+ return obj.value
+ return super().default(obj)
+
+
+class TiDBStorage(BaseVectorStorage):
+ r"""An implementation of the `BaseVectorStorage` for interacting with TiDB.
+
+ The detailed information about TiDB is available at:
+ `TiDB Vector Search <https://ai.pingcap.com/>`_
+
+ Args:
+ vector_dim (int): The dimension of storing vectors.
+ url_and_api_key (Tuple[str, str] or str): A tuple containing
+ the database url and API key for connecting to a TiDB cluster.
+ The URL should be in the format:
+ "mysql+pymysql://<username>:<password>@<host>:<port>/<db_name>".
+
+ TiDB will not use the API Key, but retains the definition for
+ interface compatible.
+ collection_name (Optional[str], optional): Name of the collection.
+ The collection name will be used as the table name in TiDB.
+
+ If not provided, set it to the current time with iso format.
+ **kwargs (Any): Additional keyword arguments for initializing
+ TiDB connection.
+
+ Raises:
+ ImportError: If `pytidb` package is not installed.
+ """
+
+ @dependencies_required('pytidb')
+ def __init__(
+ self,
+ vector_dim: int,
+ collection_name: Optional[str] = None,
+ url_and_api_key: Optional[Tuple[str, str] | str] = None,
+ **kwargs: Any,
+ ) -> None:
+ from pytidb import TiDBClient
+
+ self._client: TiDBClient
+ database_url = None
+ if isinstance(url_and_api_key, str):
+ database_url = url_and_api_key
+ elif isinstance(url_and_api_key, tuple):
+ database_url = url_and_api_key[0]
+ self._create_client(database_url, **kwargs)
+ self.vector_dim = vector_dim
+ self.collection_name = collection_name or self._generate_table_name()
+ self._table = self._open_and_create_table()
+ self._table_model = self._table.table_model
+ self._check_table()
+
+ def _create_client(
+ self,
+ database_url: Optional[str] = None,
+ **kwargs: Any,
+ ) -> None:
+ r"""Initializes the TiDB client with the provided connection details.
+
+ Args:
+ database_url (str): The database connection string for the TiDB
+ server.
+ **kwargs: Additional keyword arguments passed to the TiDB client.
+ """
+ from pytidb import TiDBClient
+
+ self._client = TiDBClient.connect(
+ database_url,
+ **kwargs,
+ )
+
+ def _get_table_model(self, collection_name: str) -> Any:
+ from pytidb.schema import Field, TableModel, VectorField
+ from sqlalchemy import JSON
+
+ class VectorDBRecord(TableModel):
+ id: Optional[str] = Field(None, primary_key=True)
+ vector: list[float] = VectorField(self.vector_dim)
+ payload: Optional[dict[str, Any]] = Field(None, sa_type=JSON)
+
+ # Notice: Avoid repeated definition warnings by dynamically generating
+ # class names.
+ return type(
+ f"VectorDBRecord_{collection_name}",
+ (VectorDBRecord,),
+ {"__tablename__": collection_name},
+ table=True,
+ )
+
+ def _open_and_create_table(self) -> "Table[Any]":
+ """Opens an existing table or creates a new table in TiDB."""
+ table = self._client.open_table(self.collection_name)
+ if table is None:
+ table = self._client.create_table(
+ schema=self._get_table_model(self.collection_name)
+ )
+ return table
+
+ def _check_table(self):
+ r"""Ensuring the specified table matches the specified vector
+ dimensionality.
+ """
+ in_dim = self._get_table_info()["vector_dim"]
+ if in_dim != self.vector_dim:
+ raise ValueError(
+ "Vector dimension of the existing table "
+ f'"{self.collection_name}" ({in_dim}) is different from '
+ f"the given embedding dim ({self.vector_dim})."
+ )
+
+ def _generate_table_name(self) -> str:
+ r"""Generates a unique name for a new table based on the current
+ timestamp. TiDB table names can only contain alphanumeric
+ characters and underscores.
+
+ Returns:
+ str: A unique, valid table name.
+ """
+ timestamp = datetime.now().isoformat()
+ transformed_name = re.sub(r'[^a-zA-Z0-9_]', '_', timestamp)
+ valid_name = "vectors_" + transformed_name
+ return valid_name
+
+ def _get_table_info(self) -> Dict[str, Any]:
+ r"""Retrieves details of an existing table.
+
+ Returns:
+ Dict[str, Any]: A dictionary containing details about the
+ table.
+ """
+ vector_count = self._table.rows()
+ # Get vector dimension from table schema
+ columns = self._table.columns()
+ dim_value = None
+ for col in columns:
+ match = re.search(r'vector\((\d+)\)', col.column_type)
+ if match:
+ dim_value = int(match.group(1))
+ break
+
+ return {
+ "vector_count": vector_count,
+ "vector_dim": dim_value,
+ }
+
+ def _validate_and_convert_vectors(
+ self, records: List[VectorRecord]
+ ) -> List[Any]:
+ r"""Validates and converts VectorRecord instances to VectorDBRecord
+ instances.
+
+ Args:
+ records (List[VectorRecord]): List of vector records to validate
+ and convert.
+
+ Returns:
+ List[VectorDBRecord]: A list of VectorDBRecord instances.
+ """
+ db_records = []
+ for record in records:
+ payload = record.payload
+ if isinstance(payload, str):
+ payload = json.loads(payload)
+ elif isinstance(payload, dict):
+ payload = json.loads(json.dumps(payload, cls=EnumEncoder))
+ else:
+ payload = None
+
+ db_records.append(
+ self._table_model(
+ id=record.id,
+ vector=record.vector,
+ payload=payload,
+ )
+ )
+ return db_records
+
+ def add(
+ self,
+ records: List[VectorRecord],
+ **kwargs,
+ ) -> None:
+ r"""Adds a list of vectors to the specified table.
+
+ Args:
+ records (List[VectorRecord]): List of vectors to be added.
+ **kwargs (Any): Additional keyword arguments pass to insert.
+
+ Raises:
+ RuntimeError: If there was an error in the addition process.
+ """
+
+ db_records = self._validate_and_convert_vectors(records)
+ if len(db_records) == 0:
+ return
+ self._table.bulk_insert(db_records)
+
+ logger.debug(
+ f"Successfully added vectors to TiDB table: {self.collection_name}"
+ )
+
+ def delete(
+ self,
+ ids: List[str],
+ **kwargs: Any,
+ ) -> None:
+ r"""Deletes a list of vectors identified by their IDs from the
+ storage.
+
+ Args:
+ ids (List[str]): List of unique identifiers for the vectors to be
+ deleted.
+ **kwargs (Any): Additional keyword arguments passed to delete.
+
+ Raises:
+ RuntimeError: If there is an error during the deletion process.
+ """
+ self._table.delete({"id": {"$in": ids}})
+ logger.debug(
+ f"Successfully deleted vectors from TiDB table "
+ f"<{self.collection_name}>"
+ )
+
+ def status(self) -> VectorDBStatus:
+ r"""Retrieves the current status of the TiDB table.
+
+ Returns:
+ VectorDBStatus: An object containing information about the
+ table's status.
+ """
+ status = self._get_table_info()
+ return VectorDBStatus(
+ vector_dim=status["vector_dim"],
+ vector_count=status["vector_count"],
+ )
+
+ def query(
+ self,
+ query: VectorDBQuery,
+ **kwargs: Any,
+ ) -> List[VectorDBQueryResult]:
+ r"""Searches for similar vectors in the storage based on the provided
+ query.
+
+ Args:
+ query (VectorDBQuery): The query object containing the search
+ vector and the number of top similar vectors to retrieve.
+ **kwargs (Any): Additional keyword arguments passed to search.
+
+ Returns:
+ List[VectorDBQueryResult]: A list of vectors retrieved from the
+ storage based on similarity to the query vector.
+ """
+ rows = (
+ self._table.search(query.query_vector).limit(query.top_k).to_list()
+ )
+
+ query_results = []
+ for row in rows:
+ query_results.append(
+ VectorDBQueryResult.create(
+ similarity=float(row['similarity_score']),
+ id=str(row['id']),
+ payload=row['payload'],
+ vector=row['vector'],
+ )
+ )
+ return query_results
+
+ def clear(self) -> None:
+ r"""Removes all vectors from the TiDB table. This method
+ deletes the existing table and then recreates it with the same
+ schema to effectively remove all stored vectors.
+ """
+ self._table.truncate()
+
+ def load(self) -> None:
+ r"""Load the collection hosted on cloud service."""
+ pass
+
+ @property
+ def client(self) -> "TiDBClient":
+ r"""Provides direct access to the TiDB client.
+
+ Returns:
+ Any: The TiDB client instance.
+ """
+ return self._client
diff --git a/camel/types/enums.py b/camel/types/enums.py
index 2081f737d..5c107a8cf 100644
--- a/camel/types/enums.py
+++ b/camel/types/enums.py
@@ -882,6 +882,7 @@ class OpenAIVisionDetailType(Enum):
class StorageType(Enum):
MILVUS = "milvus"
QDRANT = "qdrant"
+ TIDB = "tidb"
class OpenAPIName(Enum):
diff --git a/docs/camel.storages.vectordb_storages.rst b/docs/camel.storages.vectordb_storages.rst
index d16150a68..b1c2af596 100644
--- a/docs/camel.storages.vectordb_storages.rst
+++ b/docs/camel.storages.vectordb_storages.rst
@@ -23,6 +23,14 @@ camel.storages.vectordb\_storages.milvus module
camel.storages.vectordb\_storages.qdrant module
-----------------------------------------------
+.. automodule:: camel.storages.vectordb_storages.tidb
+ :members:
+ :undoc-members:
+ :show-inheritance:
+
+camel.storages.vectordb\_storages.tidb module
+-----------------------------------------------
+
.. automodule:: camel.storages.vectordb_storages.qdrant
:members:
:undoc-members:
diff --git a/docs/get_started/installation.md b/docs/get_started/installation.md
index acb7a011f..4a951cf7a 100644
--- a/docs/get_started/installation.md
+++ b/docs/get_started/installation.md
@@ -19,7 +19,7 @@ Available extras:
- `all`: Includes all features below
- `model_platforms`: OpenAI, Google, Mistral, Anthropic Claude, Cohere etc.
- `huggingface`: Transformers, Diffusers, Accelerate, Datasets, PyTorch etc.
-- `rag`: Sentence Transformers, Qdrant, Milvus, BM25 etc.
+- `rag`: Sentence Transformers, Qdrant, Milvus, TiDB, BM25 etc.
- `storage`: Neo4j, Redis, Azure Blob, Google Cloud Storage, AWS S3 etc.
- `web_tools`: DuckDuckGo, Wikipedia, WolframAlpha, Google Maps, Weather API etc.
- `document_tools`: PDF, Word, OpenAPI, BeautifulSoup, Unstructured etc.
diff --git a/docs/key_modules/storages.md b/docs/key_modules/storages.md
index 363e355ee..38a38b928 100644
--- a/docs/key_modules/storages.md
+++ b/docs/key_modules/storages.md
@@ -44,6 +44,12 @@ The Storage module is a comprehensive framework designed for handling various ty
Reference: [Milvus](https://milvus.io/docs/overview.md/)
+**`TiDBStorage`**:
+
+- Description: A concrete implementation of `BaseVectorStorage`, tailored for interacting with TiDB, one database for all your AI ambitions: vector embeddings, knowledge graphs, and operational data.
+
+Reference: [TiDB](https://ai.pingcap.com/)
+
**`QdrantStorage`**:
- Description: A concrete implementation of `BaseVectorStorage`, tailored for interacting with Qdrant, a vector search engine.
@@ -154,7 +160,51 @@ milvus_storage.clear()
>>> {'key2': 'value2'} 0.5669466853141785
```
-### 3.4. Using `QdrantStorage`
+### 3.4. Using `TiDBStorage`
+
+If you use TiDB Serverless, you can redirect to [TiDB Cloud Web Console](https://tidbcloud.com/console/clusters) to get the database URL. (Select Connect with "SQLAlchemy" > "PyMySQL" on the connection panel)
+
+```python
+import os
+from camel.storages import TiDBStorage, VectorDBQuery, VectorRecord
+
+os.environ["TIDB_DATABASE_URL"] = "The database url of your TiDB cluster."
+
+# Create an instance of TiDBStorage with dimension = 4
+tidb_storage = TiDBStorage(
+ url_and_api_key=(os.getenv("DATABASE_URL"), ''),
+ vector_dim=4,
+ collection_name="my_collection"
+)
+
+# Add two vector records
+tidb_storage.add([
+ VectorRecord(
+ vector=[-0.1, 0.1, -0.1, 0.1],
+ payload={'key1': 'value1'},
+ ),
+ VectorRecord(
+ vector=[-0.1, 0.1, 0.1, 0.1],
+ payload={'key2': 'value2'},
+ ),
+])
+
+# Load the remote collection
+tidb_storage.load()
+
+# Query similar vectors
+query_results = tidb_storage.query(VectorDBQuery(query_vector=[0.1, 0.2, 0.1, 0.1], top_k=1))
+for result in query_results:
+ print(result.record.payload, result.similarity)
+
+# Clear all vectors
+tidb_storage.clear()
+```
+```markdown
+>>> {'key2': 'value2'} 0.5669466755703252
+```
+
+### 3.5. Using `QdrantStorage`
```python
from camel.storages import QdrantStorage, VectorDBQuery, VectorRecord
@@ -184,7 +234,7 @@ qdrant_storage.clear()
>>> {'key2': 'value2'} 0.5669467095138407
```
-### 3.5. Using `NebulaGraph`
+### 3.6. Using `NebulaGraph`
```python
from camel.storages.graph_storages import NebulaGraph
@@ -196,7 +246,7 @@ query = 'SHOW TAGS;'
print(nebula_graph.query(query))
```
-### 3.5. Using `Neo4jGraph`
+### 3.7. Using `Neo4jGraph`
```python
from camel.storages import Neo4jGraph
diff --git a/examples/storages/tidb_vector_storage.py b/examples/storages/tidb_vector_storage.py
new file mode 100644
index 000000000..ec811b17c
--- /dev/null
+++ b/examples/storages/tidb_vector_storage.py
@@ -0,0 +1,89 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+
+from camel.storages.vectordb_storages import (
+ TiDBStorage,
+ VectorDBQuery,
+ VectorRecord,
+)
+
+"""
+Before the DATABASE_URL, you can setup the a TiDB database cluster first:
+
+(Option 1): TiDB Serverless
+
+1. Go to [TiDB Cloud](https://tidbcloud.com/console/clusters) to create
+ a serverless cluster
+2. Click the **Connect** button
+3. Select "SQLAlchemy" > "PyMySQL" for the **Connect With** option, then
+ you can get the DATABASE_URL like:
+
+DATABASE_URL="mysql+pymysql://<USERNAME>:<PASSWORD>@<HOST>:4000/test&ssl_verify_cert=true&ssl_verify_identity=true"
+
+(Option 2): TiDB playground cluster on local
+
+1. Install TiUP via command:
+
+```
+curl --proto '=https' --tlsv1.2 -sSf \
+ https://tiup-mirrors.pingcap.com/install.sh | sh
+```
+
+2. Deploy a playground cluster via command: `tiup playground`
+3. The DATABASE_URL should be like: "mysql+pymysql://root:@localhost:4000/test"
+"""
+
+DATABASE_URL = "mysql+pymysql://root:@localhost:4000/test"
+
+
+def main():
+ # Create an instance of TiDBStorage with dimension = 4
+ tidb_storage = TiDBStorage(
+ url_and_api_key=(DATABASE_URL, ''),
+ vector_dim=4,
+ collection_name="my_collection",
+ )
+
+ # Add two vector records
+ tidb_storage.add(
+ [
+ VectorRecord(
+ vector=[-0.1, 0.1, -0.1, 0.1],
+ payload={'key1': 'value1'},
+ ),
+ VectorRecord(
+ vector=[-0.1, 0.1, 0.1, 0.1],
+ payload={'key2': 'value2'},
+ ),
+ ]
+ )
+
+ # Query similar vectors
+ query_results = tidb_storage.query(
+ VectorDBQuery(query_vector=[0.1, 0.2, 0.1, 0.1], top_k=1)
+ )
+ for result in query_results:
+ print(result.record.payload, result.similarity)
+
+ """
+ Output:
+ {'key2': 'value2'} 0.5669466755703252
+ """
+
+ # Clear all vectors
+ tidb_storage.clear()
+
+
+if __name__ == "__main__":
+ main()
diff --git a/pyproject.toml b/pyproject.toml
index e7b722fad..c63f6bb01 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -69,6 +69,7 @@ rag = [
"sentence-transformers>=3.0.1,<4",
"qdrant-client>=1.9.0,<2",
"pymilvus>=2.4.0,<3",
+ "pytidb-experimental==0.0.1.dev4",
"neo4j>=5.18.0,<6",
"nebula3-python==3.8.2",
"rank-bm25>=0.2.2,<0.3",
@@ -173,6 +174,7 @@ huggingface = [
storage = [
"qdrant-client>=1.9.0,<2",
"pymilvus>=2.4.0,<3",
+ "pytidb-experimental==0.0.1.dev4",
"neo4j>=5.18.0,<6",
"nebula3-python==3.8.2",
"redis>=5.0.6,<6",
@@ -243,6 +245,7 @@ all = [
"networkx>=3.4.2,<4",
"qdrant-client>=1.9.0,<2",
"pymilvus>=2.4.0,<3",
+ "pytidb-experimental==0.0.1.dev4",
"cohere>=5.11.0,<6",
"sentence-transformers>=3.0.1,<4",
"neo4j>=5.18.0,<6",
@@ -404,6 +407,7 @@ module = [
"fish_audio_sdk",
"sentence_transformers.*",
"pymilvus",
+ "pytidb",
"pillow",
"slack-sdk",
"slack-bolt",
diff --git a/test/storages/vector_storages/test_tidb.py b/test/storages/vector_storages/test_tidb.py
new file mode 100644
index 000000000..5ea89ddd0
--- /dev/null
+++ b/test/storages/vector_storages/test_tidb.py
@@ -0,0 +1,89 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+from unittest.mock import MagicMock, create_autospec, patch
+
+import pytest
+
+from camel.storages import TiDBStorage, VectorDBQuery, VectorRecord
+
+
+def mock_tidb_storage():
+ with patch('camel.storages.TiDBStorage') as MockTiDBStorage:
+ mock_storage1 = create_autospec(TiDBStorage)
+ mock_storage2 = create_autospec(TiDBStorage)
+ MockTiDBStorage.side_effect = [mock_storage1, mock_storage2]
+ yield mock_storage1, mock_storage2
+
+
+def setup_mock_storage(mock_storage, vectors, query_result_id, payload):
+ mock_query_result = MagicMock()
+ mock_query_result.record.id = query_result_id
+ mock_query_result.record.payload = payload
+ mock_storage.query.return_value = [mock_query_result]
+ mock_storage.add(vectors)
+ mock_storage.status.return_value = MagicMock(vector_count=0)
+
+
+def test_multiple_remote_clients(mock_tidb_storage):
+ mock_storage1, mock_storage2 = mock_tidb_storage
+
+ # Example vectors for testing
+ vectors1 = [
+ VectorRecord(
+ vector=[0.1, 0.1, 0.1, 0.1], payload={"message": "text1"}
+ ),
+ VectorRecord(
+ vector=[0.1, -0.1, -0.1, 0.1], payload={"message": "text2"}
+ ),
+ ]
+ vectors2 = [
+ VectorRecord(
+ vector=[-0.1, 0.1, -0.1, 0.1], payload={"message": "text3"}
+ ),
+ VectorRecord(
+ vector=[-0.1, 0.1, 0.1, 0.1],
+ payload={"message": "text4", "number": 1},
+ ),
+ ]
+ setup_mock_storage(
+ mock_storage1, vectors1, vectors1[0].id, {"message": "text1"}
+ )
+ setup_mock_storage(
+ mock_storage2, vectors2, vectors2[0].id, {"message": "text3"}
+ )
+
+ # Assert add method was called correctly
+ mock_storage1.add.assert_called_once_with(vectors1)
+ mock_storage2.add.assert_called_once_with(vectors2)
+
+ # Perform and verify queries
+ query1 = VectorDBQuery(query_vector=[0.1, 0.2, 0.1, 0.1], top_k=1)
+ result1 = mock_storage1.query(query1)
+ assert result1[0].record.id == vectors1[0].id
+ assert result1[0].record.payload == {"message": "text1"}
+
+ query2 = VectorDBQuery(query_vector=[-0.1, 0.2, -0.1, 0.1], top_k=1)
+ result2 = mock_storage2.query(query2)
+ assert result2[0].record.id == vectors2[0].id
+ assert result2[0].record.payload == {"message": "text3"}
+
+ # Clear and check status for each storage
+ mock_storage1.clear()
+ status1 = mock_storage1.status()
+ assert status1.vector_count == 0
+
+ mock_storage2.clear()
+ status2 = mock_storage2.status()
+ assert status2.vector_count == 0