Skip to content

armgit / camel

Public Imported from GitHub
0

🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org

chore: Enhance TiDB vector DB integration (#2014)

Wendong-Fance214d3db4f2
diff --git a/camel/storages/vectordb_storages/tidb.py b/camel/storages/vectordb_storages/tidb.py
index ff5259caf..bd8c40c7d 100644
--- a/camel/storages/vectordb_storages/tidb.py
+++ b/camel/storages/vectordb_storages/tidb.py
@@ -16,9 +16,7 @@ import logging
import re
from datetime import datetime
from enum import Enum
-from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple
-
-from pytidb import Table
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union
from camel.storages.vectordb_storages import (
BaseVectorStorage,
@@ -30,7 +28,7 @@ from camel.storages.vectordb_storages import (
from camel.utils import dependencies_required
if TYPE_CHECKING:
- from pytidb import TiDBClient
+ from pytidb import Table, TiDBClient
logger = logging.getLogger(__name__)
@@ -50,17 +48,15 @@ class TiDBStorage(BaseVectorStorage):
Args:
vector_dim (int): The dimension of storing vectors.
- url_and_api_key (Tuple[str, str] or str): A tuple containing
- the database url and API key for connecting to a TiDB cluster.
- The URL should be in the format:
+ url_and_api_key (Optional[Union[Tuple[str, str], str]]): A tuple
+ containing the database url and API key for connecting to a TiDB
+ cluster. The URL should be in the format:
"mysql+pymysql://<username>:<password>@<host>:<port>/<db_name>".
-
TiDB will not use the API Key, but retains the definition for
interface compatible.
- collection_name (Optional[str], optional): Name of the collection.
- The collection name will be used as the table name in TiDB.
-
- If not provided, set it to the current time with iso format.
+ collection_name (Optional[str]): Name of the collection.
+ The collection name will be used as the table name in TiDB. If not
+ provided, set it to the current time with iso format.
**kwargs (Any): Additional keyword arguments for initializing
TiDB connection.
@@ -73,7 +69,7 @@ class TiDBStorage(BaseVectorStorage):
self,
vector_dim: int,
collection_name: Optional[str] = None,
- url_and_api_key: Optional[Tuple[str, str] | str] = None,
+ url_and_api_key: Optional[Union[Tuple[str, str], str]] = None,
**kwargs: Any,
) -> None:
from pytidb import TiDBClient
@@ -99,8 +95,8 @@ class TiDBStorage(BaseVectorStorage):
r"""Initializes the TiDB client with the provided connection details.
Args:
- database_url (str): The database connection string for the TiDB
- server.
+ database_url (Optional[str]): The database connection string for
+ the TiDB server.
**kwargs: Additional keyword arguments passed to the TiDB client.
"""
from pytidb import TiDBClient
@@ -129,7 +125,7 @@ class TiDBStorage(BaseVectorStorage):
)
def _open_and_create_table(self) -> "Table[Any]":
- """Opens an existing table or creates a new table in TiDB."""
+ r"""Opens an existing table or creates a new table in TiDB."""
table = self._client.open_table(self.collection_name)
if table is None:
table = self._client.create_table(
@@ -179,6 +175,13 @@ class TiDBStorage(BaseVectorStorage):
dim_value = int(match.group(1))
break
+ # If no vector column found, log a warning
+ if dim_value is None:
+ logger.warning(
+ f"No vector column found in table {self.collection_name}. "
+ "This may indicate an incompatible table schema."
+ )
+
return {
"vector_count": vector_count,
"vector_dim": dim_value,
@@ -192,7 +195,7 @@ class TiDBStorage(BaseVectorStorage):
Args:
records (List[VectorRecord]): List of vector records to validate
- and convert.
+ and convert.
Returns:
List[VectorDBRecord]: A list of VectorDBRecord instances.
diff --git a/uv.lock b/uv.lock
index e0cbac7e1..00a5e51b6 100644
--- a/uv.lock
+++ b/uv.lock
@@ -617,6 +617,7 @@ all = [
{ name = "pytest" },
{ name = "pytest-asyncio" },
{ name = "pytest-cov" },
+ { name = "pytidb-experimental" },
{ name = "qdrant-client" },
{ name = "rank-bm25" },
{ name = "redis" },
@@ -754,6 +755,7 @@ rag = [
{ name = "neo4j" },
{ name = "pandasai" },
{ name = "pymilvus" },
+ { name = "pytidb-experimental" },
{ name = "qdrant-client" },
{ name = "rank-bm25" },
{ name = "sentence-transformers" },
@@ -772,6 +774,7 @@ storage = [
{ name = "nebula3-python" },
{ name = "neo4j" },
{ name = "pymilvus" },
+ { name = "pytidb-experimental" },
{ name = "qdrant-client" },
{ name = "redis" },
]
@@ -953,6 +956,9 @@ requires-dist = [
{ name = "pytest-asyncio", marker = "extra == 'test'", specifier = ">=0.23.0,<0.24" },
{ name = "pytest-cov", marker = "extra == 'all'", specifier = ">=4,<5" },
{ name = "pytest-cov", marker = "extra == 'dev'", specifier = ">=4,<5" },
+ { name = "pytidb-experimental", marker = "extra == 'all'", specifier = "==0.0.1.dev4" },
+ { name = "pytidb-experimental", marker = "extra == 'rag'", specifier = "==0.0.1.dev4" },
+ { name = "pytidb-experimental", marker = "extra == 'storage'", specifier = "==0.0.1.dev4" },
{ name = "pyyaml", specifier = ">=6.0.2,<7" },
{ name = "qdrant-client", marker = "extra == 'all'", specifier = ">=1.9.0,<2" },
{ name = "qdrant-client", marker = "extra == 'rag'", specifier = ">=1.9.0,<2" },
@@ -4860,6 +4866,15 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/c3/9f/2872a29766dfe65fb97268233503cc9f7ba6e776eedbdb88ed47216aef84/pymupdf-1.25.4-cp39-abi3-win_amd64.whl", hash = "sha256:af8fa1f7920b3b27343b2511cb9fc44baf2bad8b948db0b0f9d5b2fa589cb305", size = 16579269 },
]
+[[package]]
+name = "pymysql"
+version = "1.1.1"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/b3/8f/ce59b5e5ed4ce8512f879ff1fa5ab699d211ae2495f1adaa5fbba2a1eada/pymysql-1.1.1.tar.gz", hash = "sha256:e127611aaf2b417403c60bf4dc570124aeb4a57f5f37b8e95ae399a42f904cd0", size = 47678 }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/0c/94/e4181a1f6286f545507528c78016e00065ea913276888db2262507693ce5/PyMySQL-1.1.1-py3-none-any.whl", hash = "sha256:4de15da4c61dc132f4fb9ab763063e693d521a80fd0e87943b9a453dd4c19d6c", size = 44972 },
+]
+
[[package]]
name = "pynacl"
version = "1.5.0"
@@ -5049,6 +5064,20 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/53/67/f56c69a98c7eb244025845506387d0f961681657c9fcd8b2d2edd148f9d2/python_oxmsg-0.0.2-py3-none-any.whl", hash = "sha256:22be29b14c46016bcd05e34abddfd8e05ee82082f53b82753d115da3fc7d0355", size = 31455 },
]
+[[package]]
+name = "pytidb-experimental"
+version = "0.0.1.dev4"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "pymysql" },
+ { name = "sqlmodel" },
+ { name = "tidb-vector" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/32/b9/11fdb74a0cf645cb02ccff7a1f163333eedd461e364425caede13dba5d6b/pytidb_experimental-0.0.1.dev4.tar.gz", hash = "sha256:0161b77758af363a6d3f0c16df010effae2cea3b0f98ca08b6d0b9eb07760923", size = 15079 }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/8b/71/a531e6648455fd8a936530c67c619a33726f55331bacdac74aa6742d0baf/pytidb_experimental-0.0.1.dev4-py3-none-any.whl", hash = "sha256:5ba3f5bb533927b0f82421df4c6f379dafd4f2afe8cfaf15ec0f00833794f1d0", size = 17824 },
+]
+
[[package]]
name = "pytz"
version = "2024.2"
@@ -6133,6 +6162,19 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/6b/ed/8bc1d54387434f4c1b99a54721691444e9e249bb728a0da47b3150c756d6/sqlglotrs-0.3.0-cp312-none-win_amd64.whl", hash = "sha256:b9f308732f12331f06c53fcb1d7c2b135a43aa22486b4c88c26d42710f329448", size = 190557 },
]
+[[package]]
+name = "sqlmodel"
+version = "0.0.22"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "pydantic" },
+ { name = "sqlalchemy" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/b5/39/8641040ab0d5e1d8a1c2325ae89a01ae659fc96c61a43d158fb71c9a0bf0/sqlmodel-0.0.22.tar.gz", hash = "sha256:7d37c882a30c43464d143e35e9ecaf945d88035e20117bf5ec2834a23cbe505e", size = 116392 }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/dd/b1/3af5104b716c420e40a6ea1b09886cae3a1b9f4538343875f637755cae5b/sqlmodel-0.0.22-py3-none-any.whl", hash = "sha256:a1ed13e28a1f4057cbf4ff6cdb4fc09e85702621d3259ba17b3c230bfb2f941b", size = 28276 },
+]
+
[[package]]
name = "sse-starlette"
version = "2.2.1"
@@ -6256,6 +6298,18 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/32/d5/f9a850d79b0851d1d4ef6456097579a9005b31fea68726a4ae5f2d82ddd9/threadpoolctl-3.6.0-py3-none-any.whl", hash = "sha256:43a0b8fd5a2928500110039e43a5eed8480b918967083ea48dc3ab9f13c4a7fb", size = 18638 },
]
+[[package]]
+name = "tidb-vector"
+version = "0.0.14"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "numpy" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/d8/fc/039b05b6b9002e49e004958bc3fc9b4424bb5b44783da17ceae7d158d0dc/tidb_vector-0.0.14.tar.gz", hash = "sha256:a63f4af54bb91aea51ae55d8cb9528b7bd2d3838181d7978cf5fcadfa8ab2887", size = 20805 }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/28/83/41a82d7248f213ea800e65ab1733405070a8d13825f5756f1784e6553c3c/tidb_vector-0.0.14-py3-none-any.whl", hash = "sha256:19f25d5c3fd024d4dbabac54d4f2920c19235a678b832f7a1c46a00b1bc990a3", size = 21914 },
+]
+
[[package]]
name = "tiktoken"
version = "0.7.0"