Skip to content

armgit / camel

Public Imported from GitHub
0

🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org

fix: surrealdb integration (#2891)

Martin Schaer662ab3baaacf
diff --git a/camel/storages/vectordb_storages/__init__.py b/camel/storages/vectordb_storages/__init__.py
index 068789fc8..9cabdfaf4 100644
--- a/camel/storages/vectordb_storages/__init__.py
+++ b/camel/storages/vectordb_storages/__init__.py
@@ -25,6 +25,7 @@ from .milvus import MilvusStorage
from .oceanbase import OceanBaseStorage
from .pgvector import PgVectorStorage
from .qdrant import QdrantStorage
+from .surreal import SurrealStorage
from .tidb import TiDBStorage
from .weaviate import WeaviateStorage
diff --git a/camel/storages/vectordb_storages/surreal.py b/camel/storages/vectordb_storages/surreal.py
index cd0d7f82a..881515163 100644
--- a/camel/storages/vectordb_storages/surreal.py
+++ b/camel/storages/vectordb_storages/surreal.py
@@ -29,6 +29,7 @@ if TYPE_CHECKING:
from surrealdb import Surreal # type: ignore[import-not-found]
logger = get_logger(__name__)
+# logger.setLevel("DEBUG")
class SurrealStorage(BaseVectorStorage):
@@ -68,13 +69,16 @@ class SurrealStorage(BaseVectorStorage):
url: str = "ws://localhost:8000/rpc",
table: str = "vector_store",
vector_dim: int = 786,
+ vector_type: str = "F64",
distance: VectorDistance = VectorDistance.COSINE,
+ hnsw_effort: int = 40,
namespace: str = "default",
database: str = "demo",
user: str = "root",
password: str = "root",
) -> None:
- r"""Initialize SurrealStorage with connection settings and ensure
+ r"""
+ Initialize SurrealStorage with connection settings and ensure
the target table exists.
Args:
@@ -96,6 +100,8 @@ class SurrealStorage(BaseVectorStorage):
(default: :obj:`"root"`)
"""
+ from surrealdb import Surreal
+
self.url = url
self.table = table
self.ns = namespace
@@ -103,89 +109,91 @@ class SurrealStorage(BaseVectorStorage):
self.user = user
self.password = password
self.vector_dim = vector_dim
+ self.vector_type = vector_type
self.distance = distance
+ self._hnsw_effort = hnsw_effort
+ self._surreal_client = Surreal(self.url)
+ self._surreal_client.signin({"username": user, "password": password})
+ self._surreal_client.use(namespace, database)
+
self._check_and_create_table()
- self._surreal_client = None
def _table_exists(self) -> bool:
- r"""Check whether the target table exists in the database.
+ r"""
+ Check whether the target table exists in the database.
Returns:
bool: True if the table exists, False otherwise.
"""
- from surrealdb import Surreal # type: ignore[import-not-found]
+ res = self._surreal_client.query("INFO FOR DB;")
+ tables = res.get('tables', {})
+ logger.debug(f"_table_exists: {res}")
+ return self.table in tables
- with Surreal(self.url) as db:
- db.signin({"username": self.user, "password": self.password})
- db.use(self.ns, self.db)
- res = db.query_raw("INFO FOR DB;")
- tables = res['result'][0]['result'].get('tables', {})
- return self.table in tables
-
- def _get_table_info(self) -> Dict[str, int]:
- r"""Retrieve dimension and record count from the table metadata.
+ def _get_table_info(self) -> dict[str, int | None]:
+ r"""
+ Retrieve dimension and record count from the table metadata.
Returns:
Dict[str, int]: A dictionary with 'dim' and 'count' keys.
"""
- from surrealdb import Surreal # type: ignore[import-not-found]
-
if not self._table_exists():
return {"dim": self.vector_dim, "count": 0}
- with Surreal(self.url) as db:
- db.signin({"username": self.user, "password": self.password})
- db.use(self.ns, self.db)
- res = db.query_raw(f"INFO FOR TABLE {self.table};")
-
- indexes = res['result'][0]['result'].get("indexes", {})
-
- dim = self.vector_dim
- idx_def = indexes.get("hnsw_idx")
- if idx_def and isinstance(idx_def, str):
- m = re.search(r"DIMENSION\s+(\d+)", idx_def)
- if m:
- dim = int(m.group(1))
- cnt = db.query_raw(f"SELECT COUNT() AS count FROM {self.table};")
- try:
- count = cnt['result'][0]['result'][0]['count']
- except (KeyError, IndexError, TypeError):
- logger.warning(
- "Unexpected result format when counting records: %s", cnt
- )
- count = 0
-
- return {"dim": dim, "count": count}
+ res = self._surreal_client.query(f"INFO FOR TABLE {self.table};")
+ logger.debug(f"_get_table_info: {res}")
+ indexes = res.get("indexes", {})
+
+ dim = self.vector_dim
+ idx_def = indexes.get("hnsw_idx")
+ if idx_def and isinstance(idx_def, str):
+ m = re.search(r"DIMENSION\s+(\d+)", idx_def)
+ if m:
+ dim = int(m.group(1))
+ cnt = self._surreal_client.query(
+ f"SELECT COUNT() FROM ONLY {self.table} GROUP ALL LIMIT 1;"
+ )
+ count = cnt.get("count", 0)
+ return {"dim": dim, "count": count}
def _create_table(self):
- r"""Define and create the vector storage table with HNSW index."""
- from surrealdb import Surreal # type: ignore[import-not-found]
-
- with Surreal(self.url) as db:
- db.signin({"username": self.user, "password": self.password})
- db.use(self.ns, self.db)
- db.query_raw(
- f"""DEFINE TABLE {self.table} SCHEMALESS;
- DEFINE FIELD payload ON {self.table} TYPE object;
- DEFINE FIELD embedding ON {self.table} TYPE array;
- DEFINE INDEX hnsw_idx ON {self.table}
- FIELDS embedding HNSW DIMENSION {self.vector_dim};
- """
+ r"""
+ Define and create the vector storage table with HNSW index.
+
+ Documentation: https://surrealdb.com/docs/surrealdb/reference-guide/vector-search#vector-search-cheat-sheet
+ """
+ if self.distance.value not in ["cosine", "euclidean", "manhattan"]:
+ raise ValueError(
+ f"Unsupported distance metric: {self.distance.value}"
)
+ surql_query = f"""
+ DEFINE TABLE {self.table} SCHEMALESS;
+ DEFINE FIELD payload ON {self.table} FLEXIBLE TYPE object;
+ DEFINE FIELD embedding ON {self.table} TYPE array<float>;
+ DEFINE INDEX hnsw_idx ON {self.table}
+ FIELDS embedding
+ HNSW DIMENSION {self.vector_dim}
+ DIST {self.distance.value}
+ TYPE {self.vector_type}
+ EFC 150 M 12 M0 24;
+ """
+ logger.debug(f"_create_table query: {surql_query}")
+ res = self._surreal_client.query_raw(surql_query)
+ logger.debug(f"_create_table response: {res}")
+ if "error" in res:
+ raise ValueError(f"Failed to create table: {res['error']}")
logger.info(f"Table '{self.table}' created successfully.")
def _drop_table(self):
- r"""Drop the vector storage table if it exists."""
- from surrealdb import Surreal # type: ignore[import-not-found]
-
- with Surreal(self.url) as db:
- db.signin({"username": self.user, "password": self.password})
- db.use(self.ns, self.db)
- db.query_raw(f"REMOVE TABLE IF EXISTS {self.table};")
+ r"""
+ Drop the vector storage table if it exists.
+ """
+ self._surreal_client.query_raw(f"REMOVE TABLE IF EXISTS {self.table};")
logger.info(f"Table '{self.table}' deleted successfully.")
def _check_and_create_table(self):
- r"""Check if the table exists and matches the expected vector
- dimension. If not, create a new table.
+ r"""
+ Check if the table exists and matches the expected vector dimension.
+ If not, create a new table.
"""
if self._table_exists():
in_dim = self._get_table_info()["dim"]
@@ -200,7 +208,8 @@ class SurrealStorage(BaseVectorStorage):
def _validate_and_convert_records(
self, records: List[VectorRecord]
) -> List[Dict]:
- r"""Validate and convert VectorRecord instances into
+ r"""
+ Validate and convert VectorRecord instances into
SurrealDB-compatible dictionaries.
Args:
@@ -229,8 +238,8 @@ class SurrealStorage(BaseVectorStorage):
query: VectorDBQuery,
**kwargs: Any,
) -> List[VectorDBQueryResult]:
- r"""Perform a top-k similarity search using the configured distance
- metric.
+ r"""
+ Perform a top-k similarity search using the configured distance metric.
Args:
query (VectorDBQuery): Query containing the query vector
@@ -240,65 +249,38 @@ class SurrealStorage(BaseVectorStorage):
List[VectorDBQueryResult]: Ranked list of matching records
with similarity scores.
"""
- from surrealdb import Surreal # type: ignore[import-not-found]
-
- metric = {
- VectorDistance.COSINE: "cosine",
- VectorDistance.EUCLIDEAN: "euclidean",
- VectorDistance.DOT: "dot",
- }[self.distance]
-
- metric_func = {
- VectorDistance.COSINE: "vector::similarity::cosine",
- VectorDistance.EUCLIDEAN: "vector::distance::euclidean",
- VectorDistance.DOT: "vector::dot",
- }.get(self.distance)
-
- if not metric_func:
- raise ValueError(f"Unsupported distance metric: {self.distance}")
-
- with Surreal(self.url) as db:
- db.signin({"username": self.user, "password": self.password})
- db.use(self.ns, self.db)
-
- # Use parameterized query to prevent SQL injection
- sql_query = f"""SELECT payload, embedding,
- {metric_func}(embedding, $query_vec) AS score
- FROM {self.table}
- WHERE embedding <|{query.top_k},{metric}|> $query_vec
- ORDER BY score;
- """
-
- response = db.query_raw(
- sql_query, {"query_vec": query.query_vector}
- )
+ surql_query = f"""
+ SELECT id, embedding, payload, vector::distance::knn() AS dist
+ FROM {self.table}
+ WHERE embedding <|{query.top_k},{self._hnsw_effort}|> $vector
+ ORDER BY dist;
+ """
+ logger.debug(
+ f"query surql: {surql_query} with $vector = {query.query_vector}"
+ )
- if not response.get("result") or not response["result"]:
- return []
-
- results = response["result"][0]
-
- if "result" not in results:
- return []
-
- return [
- VectorDBQueryResult(
- record=VectorRecord(
- vector=row["embedding"], payload=row.get("payload", {})
- ),
- similarity=(
- 1.0 - row["score"]
- if self.distance == VectorDistance.COSINE
- else 1.0 / (1.0 + row["score"])
- if self.distance == VectorDistance.EUCLIDEAN
- else row["score"]
- ),
- )
- for row in results["result"]
- ]
+ response = self._surreal_client.query(
+ surql_query, {"vector": query.query_vector}
+ )
+ logger.debug(f"query response: {response}")
+
+ return [
+ VectorDBQueryResult(
+ record=VectorRecord(
+ id=row["id"].id,
+ vector=row["embedding"],
+ payload=row["payload"],
+ ),
+ similarity=1.0 - row["dist"]
+ if self.distance == VectorDistance.COSINE
+ else -row["score"],
+ )
+ for row in response
+ ]
def add(self, records: List[VectorRecord], **kwargs) -> None:
- r"""Insert validated vector records into the SurrealDB table.
+ r"""
+ Insert validated vector records into the SurrealDB table.
Args:
records (List[VectorRecord]): List of vector records to add.
@@ -306,16 +288,10 @@ class SurrealStorage(BaseVectorStorage):
logger.info(
"Adding %d records to table '%s'.", len(records), self.table
)
- from surrealdb import Surreal # type: ignore[import-not-found]
-
try:
- with Surreal(self.url) as db:
- db.signin({"username": self.user, "password": self.password})
- db.use(self.ns, self.db)
-
- validated_records = self._validate_and_convert_records(records)
- for record in validated_records:
- db.create(self.table, record)
+ validated_records = self._validate_and_convert_records(records)
+ for record in validated_records:
+ self._surreal_client.create(self.table, record)
logger.info(
"Successfully added %d records to table '%s'.",
@@ -334,46 +310,38 @@ class SurrealStorage(BaseVectorStorage):
def delete(
self, ids: Optional[List[str]] = None, if_all: bool = False, **kwargs
) -> None:
- r"""Delete specific records by ID or clear the entire table.
+ r"""
+ Delete specific records by ID or clear the entire table.
Args:
ids (Optional[List[str]]): List of record IDs to delete.
if_all (bool): Whether to delete all records in the table.
"""
- from surrealdb import Surreal # type: ignore[import-not-found]
- from surrealdb.data.types.record_id import ( # type: ignore[import-not-found]
- RecordID,
- )
+ from surrealdb.data.types.record_id import RecordID
try:
- with Surreal(self.url) as db:
- db.signin({"username": self.user, "password": self.password})
- db.use(self.ns, self.db)
-
- if if_all:
- db.delete(self.table, **kwargs)
- logger.info(
- f"Deleted all records from table '{self.table}'"
- )
- return
-
- if not ids:
- raise ValueError(
- "Either `ids` must be provided or `if_all=True`"
- )
-
- for id_str in ids:
- rec = RecordID(self.table, id_str)
- db.delete(rec, **kwargs)
- logger.info(f"Deleted record {rec}")
+ if if_all:
+ self._surreal_client.delete(self.table, **kwargs)
+ logger.info(f"Deleted all records from table '{self.table}'")
+ return
+
+ if not ids:
+ raise ValueError(
+ "Either `ids` must be provided or `if_all=True`"
+ )
+
+ for id_str in ids:
+ rec = RecordID(self.table, id_str)
+ self._surreal_client.delete(rec, **kwargs)
+ logger.info(f"Deleted record {rec}")
except Exception as e:
logger.exception("Error deleting records from SurrealDB")
raise RuntimeError(f"Failed to delete records {ids!r}") from e
def status(self) -> VectorDBStatus:
- r"""Retrieve the status of the vector table including dimension and
- count.
+ r"""
+ Retrieve the status of the vector table including dimension and count.
Returns:
VectorDBStatus: Object containing vector table metadata.
@@ -392,7 +360,9 @@ class SurrealStorage(BaseVectorStorage):
)
def clear(self) -> None:
- r"""Reset the vector table by dropping and recreating it."""
+ r"""
+ Reset the vector table by dropping and recreating it.
+ """
self._drop_table()
self._create_table()
@@ -404,12 +374,4 @@ class SurrealStorage(BaseVectorStorage):
@property
def client(self) -> "Surreal":
r"""Provides access to the underlying SurrealDB client."""
- if self._surreal_client is None:
- from surrealdb import Surreal # type: ignore[import-not-found]
-
- self._surreal_client = Surreal(self.url)
- self._surreal_client.signin( # type: ignore[attr-defined]
- {"username": self.user, "password": self.password}
- )
- self._surreal_client.use(self.ns, self.db) # type: ignore[attr-defined]
return self._surreal_client
diff --git a/examples/storages/surreal_vector_storage.py b/examples/storages/surreal_vector_storage.py
index b758996a7..866ff75aa 100644
--- a/examples/storages/surreal_vector_storage.py
+++ b/examples/storages/surreal_vector_storage.py
@@ -22,7 +22,7 @@ from camel.storages.vectordb_storages import (
def main():
url = os.getenv("SURREAL_URL", "ws://localhost:8000/rpc")
- table = os.getenv("SURREAL_TABLE", "tb")
+ table = os.getenv("SURREAL_TABLE", "documents")
vector_dim = int(os.getenv("SURREAL_VECTOR_DIM", 4))
namespace = os.getenv("SURREAL_NAMESPACE", "ns")
database = os.getenv("SURREAL_DATABASE", "db")
@@ -60,14 +60,10 @@ def main():
storage.add([vec1, vec2, vec3, vec4])
print("[Step 2] After add:", storage.status())
- surreal_client = storage.client
- with surreal_client as db:
- db.signin({"username": user, "password": password})
- db.use(namespace, database)
- res = db.query_raw(
- "SELECT * FROM lyz_tb WHERE payload.name = 'test_3';"
- )["result"][0]["result"][0]["id"].id
- print("[Step 3] Query Result ID for 'test_3':", res)
+ res = storage.client.query(
+ f"SELECT * FROM {table} WHERE payload.name = 'test_3'"
+ )[0]["id"].id
+ print("[Step 3] Query Result ID for 'test_3':", res)
storage.delete(ids=[res])
print("[Step 4] After delete 'test_3':", storage.status())
@@ -81,17 +77,27 @@ def main():
if __name__ == "__main__":
main()
-"""
-[Step 1] After clear: vector_dim=4 vector_count=0oceanbasech
-[Step 2] After add: vector_dim=4 vector_count=4
-[Step 3] Query Result ID for 'test_3': lov5h16x6uog7l2xtsqp
-[Step 4] After delete 'test_3': vector_dim=4 vector_count=3
-[Step 5] Vector Query Result:
-[VectorDBQueryResult(record=VectorRecord(vector=[],
-id='9803ae3a-18da-4152-a522-48e1939a3604',
-payload={'name': 'test_2'}), similarity=0.027665393972965968),
-VectorDBQueryResult(record=VectorRecord(vector=[],
-id='de3d085c-ed1b-4d23-9d12-d9fc64ae1e00',
-payload={'name': 'test_1'}), similarity=0.00010404203326297434)]
-
-"""
+# Example output:
+#
+# [Step 1] After clear: vector_dim=4 vector_count=0
+# [Step 2] After add: vector_dim=4 vector_count=4
+# [Step 3] Query Result ID for 'test_3': 1xujuq7atehbx140rm6l
+# [Step 4] After delete 'test_3': vector_dim=4 vector_count=3
+# [Step 5] Vector Query Result: [
+# VectorDBQueryResult(
+# record=VectorRecord(
+# vector=[1.0, 2.0, 3.0, 4.0],
+# id='v6ws0mh1vkus9e13c4wq',
+# payload={'name': 'test_1'}
+# ),
+# similarity=0.999895957966737
+# ),
+# VectorDBQueryResult(
+# record=VectorRecord(
+# vector=[5.0, 6.0, 7.0, 8.0],
+# id='ate88u7d19b8jbc3t7ux',
+# payload={'name': 'test_2'}
+# ),
+# similarity=0.972334606027034
+# )
+# ]
diff --git a/pyproject.toml b/pyproject.toml
index 6e087e475..c3c53a76e 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -35,11 +35,7 @@ dependencies = [
[project.dependency-groups]
-test = [
- "pytest>=7,<8",
- "mock>=5,<6",
- "pytest-asyncio>=0.23.0,<0.24",
-]
+test = ["pytest>=7,<8", "mock>=5,<6", "pytest-asyncio>=0.23.0,<0.24"]
[project.optional-dependencies]
dev = [
@@ -75,7 +71,7 @@ rag = [
"pymilvus>=2.4.0,<3",
"pytidb-experimental==0.0.1.dev4",
"faiss-cpu>=1.7.2,<2",
- "weaviate-client>=4.15.0",
+ "weaviate-client>=4.15.0",
"neo4j>=5.18.0,<6",
"nebula3-python==3.8.2",
"rank-bm25>=0.2.2,<0.3",
@@ -132,7 +128,7 @@ document_tools = [
"chunkr-ai>=0.0.50",
"python-pptx>=1.0.2",
"reportlab>=4.4.2",
- "onnxruntime<=1.20.1"
+ "onnxruntime<=1.20.1",
]
media_tools = [
"imageio[pyav]>=2.34.2,<3",
@@ -162,7 +158,7 @@ data_tools = [
"aiosqlite>=0.20.0,<0.21",
"stripe>=11.3.0,<12",
"networkx>=3.4.2,<4",
- "math-verify>=0.7.0,<0.8",
+ "math-verify>=0.7.0,<0.8",
]
research_tools = [
"arxiv>=2.1.3,<3",
@@ -217,6 +213,7 @@ storage = [
"weaviate-client>=4.15.0",
"psycopg[binary]>=3.1.18,<4",
"pgvector>=0.2.4,<0.3",
+ "surrealdb>=1.0.6",
"chromadb>=0.6.0,<1.0.0",
]
owl = [
@@ -270,7 +267,7 @@ owl = [
"markitdown>=0.1.1",
"exa-py>=1.10.0,<2",
"reportlab>=4.4.2",
- "onnxruntime<=1.20.1"
+ "onnxruntime<=1.20.1",
]
all = [
"numpy>=1.2,<=2.2",
@@ -394,10 +391,11 @@ all = [
"pytesseract>=0.3.13",
"psycopg[binary]>=3.1.18,<4",
"pgvector>=0.2.4,<0.3",
+ "surrealdb>=1.0.6",
"chromadb>=0.6.0,<1.0.0",
"traceroot==0.0.4a5",
"reportlab>=4.4.2",
- "onnxruntime<=1.20.1"
+ "onnxruntime<=1.20.1",
]
[project.urls]
@@ -409,29 +407,25 @@ Documentation = "https://docs.camel-ai.org"
include = [
"camel",
"camel/toolkits/hybrid_browser_toolkit/ts/package.json",
- "camel/toolkits/hybrid_browser_toolkit/ts/package-lock.json",
+ "camel/toolkits/hybrid_browser_toolkit/ts/package-lock.json",
"camel/toolkits/hybrid_browser_toolkit/ts/tsconfig.json",
"camel/toolkits/hybrid_browser_toolkit/ts/src/**/*",
"camel/toolkits/hybrid_browser_toolkit/ts/websocket-server.js",
- "hatch_build_npm.py"
-]
-exclude = [
- "camel/toolkits/hybrid_browser_toolkit/ts/node_modules/**/*"
+ "hatch_build_npm.py",
]
+exclude = ["camel/toolkits/hybrid_browser_toolkit/ts/node_modules/**/*"]
[tool.hatch.build.targets.wheel]
include = [
"camel",
"camel/toolkits/hybrid_browser_toolkit/ts/package.json",
"camel/toolkits/hybrid_browser_toolkit/ts/package-lock.json",
- "camel/toolkits/hybrid_browser_toolkit/ts/tsconfig.json",
+ "camel/toolkits/hybrid_browser_toolkit/ts/tsconfig.json",
"camel/toolkits/hybrid_browser_toolkit/ts/src/**/*",
"camel/toolkits/hybrid_browser_toolkit/ts/dist/**/*",
- "camel/toolkits/hybrid_browser_toolkit/ts/websocket-server.js"
-]
-exclude = [
- "camel/toolkits/hybrid_browser_toolkit/ts/node_modules/**/*"
+ "camel/toolkits/hybrid_browser_toolkit/ts/websocket-server.js",
]
+exclude = ["camel/toolkits/hybrid_browser_toolkit/ts/node_modules/**/*"]
[tool.hatch.build.targets.wheel.hooks.custom]
path = "hatch_build_npm.py"
@@ -452,9 +446,9 @@ quote-style = "preserve"
[tool.ruff.lint]
extend-select = [
- "I", # isort
- "B", # flake8-bugbear
- "C4", # flake8-comprehensions
+ "I", # isort
+ "B", # flake8-bugbear
+ "C4", # flake8-comprehensions
"PGH", # pygrep-hooks
"RUF", # ruff
"E",
@@ -476,7 +470,7 @@ convention = "google"
known-first-party = ["camel"]
[tool.ruff.lint.per-file-ignores]
-"camel/toolkits/search_toolkit.py" = ["E501"]
+"camel/toolkits/search_toolkit.py" = ["E501"]
[tool.pytest.ini_options]
pythonpath = ["."]
@@ -493,8 +487,8 @@ include_namespace_packages = true
[tool.mypy]
exclude = [
- '\.venv/.*', # exclude .venv directory
- 'site-packages/.*', # exclude site-packages
+ '\.venv/.*', # exclude .venv directory
+ 'site-packages/.*', # exclude site-packages
]
[[tool.mypy.overrides]]
@@ -629,7 +623,8 @@ module = [
"psycopg.*",
"pgvecto.*",
"pgvector.*",
- "reportlab.*"
+ "reportlab.*",
+ "surrealdb.*",
]
ignore_missing_imports = true
diff --git a/test/storages/vector_storages/test_surreal.py b/test/storages/vector_storages/test_surreal.py
index a6ffdc936..5ec6cb2eb 100644
--- a/test/storages/vector_storages/test_surreal.py
+++ b/test/storages/vector_storages/test_surreal.py
@@ -15,6 +15,7 @@ import sys
from unittest.mock import MagicMock, patch
import pytest
+from surrealdb import RecordID
from camel.storages.vectordb_storages.surreal import (
SurrealStorage,
@@ -35,8 +36,7 @@ sys.modules['surrealdb.data.types.record_id'] = MagicMock()
def storage():
with patch("surrealdb.Surreal") as mock_surreal:
mock_db = MagicMock()
- mock_surreal.return_value.__enter__ = MagicMock(return_value=mock_db)
- mock_surreal.return_value.__exit__ = MagicMock(return_value=None)
+ mock_surreal.return_value = mock_db
# Mock the methods that would be called during initialization
mock_db.signin = MagicMock()
@@ -69,24 +69,20 @@ def test_add(storage):
def test_query(storage):
storage_instance, mock_db = storage
- mock_db.query_raw.return_value = {
- "result": [
- {
- "result": [
- {
- "payload": {"id": "1"},
- "embedding": [0.1, 0.2, 0.3],
- "score": 0.1,
- },
- {
- "payload": {"id": "2"},
- "embedding": [0.4, 0.5, 0.6],
- "score": 0.2,
- },
- ]
- }
- ]
- }
+ mock_db.query.return_value = [
+ {
+ "id": RecordID(table_name="documents", identifier="1"),
+ "payload": {"id": "1"},
+ "embedding": [0.1, 0.2, 0.3],
+ "dist": 0.9,
+ },
+ {
+ "id": RecordID(table_name="documents", identifier="2"),
+ "payload": {"id": "2"},
+ "embedding": [0.4, 0.5, 0.6],
+ "dist": 0.8,
+ },
+ ]
query = VectorDBQuery(query_vector=[0.2, 0.3, 0.4], top_k=2)
results = storage_instance.query(query)
@@ -99,16 +95,28 @@ def test_query(storage):
def test_status(storage):
storage_instance, mock_db = storage
- mock_db.query_raw.side_effect = [
- {"result": [{"result": {"tables": {storage_instance.table: {}}}}]},
- {"result": [{"result": {"indexes": {"hnsw_idx": "DIMENSION 3"}}}]},
- {"result": [{"result": [{"count": 5}]}]},
+ mock_db.query.side_effect = [
+ {"tables": {storage_instance.table: {}}},
+ {
+ 'events': {},
+ 'fields': {
+ 'embedding': 'DEFINE FIELD embedding ON documents TYPE array<float> PERMISSIONS FULL', # noqa: E501
+ 'embedding[*]': 'DEFINE FIELD embedding[*] ON documents TYPE float PERMISSIONS FULL', # noqa: E501
+ 'payload': 'DEFINE FIELD payload ON documents FLEXIBLE TYPE object PERMISSIONS FULL', # noqa: E501
+ },
+ 'indexes': {
+ 'hnsw_idx': 'DEFINE INDEX hnsw_idx ON documents FIELDS embedding HNSW DIMENSION 4 DIST COSINE TYPE F64 EFC 150 M 12 M0 24 LM 0.40242960438184466f' # noqa: E501
+ },
+ 'lives': {},
+ 'tables': {},
+ },
+ {"count": 5},
]
status = storage_instance.status()
assert isinstance(status, VectorDBStatus)
- assert status.vector_dim == 3
+ assert status.vector_dim == 4
assert status.vector_count == 5
diff --git a/uv.lock b/uv.lock
index 3ae86106b..9d7259fee 100644
--- a/uv.lock
+++ b/uv.lock
@@ -768,6 +768,7 @@ all = [
{ name = "slack-sdk" },
{ name = "soundfile" },
{ name = "stripe" },
+ { name = "surrealdb" },
{ name = "sympy" },
{ name = "tabulate" },
{ name = "tavily-python" },
@@ -993,6 +994,7 @@ storage = [
{ name = "pytidb-experimental" },
{ name = "qdrant-client" },
{ name = "redis" },
+ { name = "surrealdb" },
{ name = "weaviate-client" },
]
web-tools = [
@@ -1300,6 +1302,8 @@ requires-dist = [
{ name = "sphinxext-rediraffe", marker = "extra == 'docs'", specifier = ">=0.2.7,<0.3" },
{ name = "stripe", marker = "extra == 'all'", specifier = ">=11.3.0,<12" },
{ name = "stripe", marker = "extra == 'data-tools'", specifier = ">=11.3.0,<12" },
+ { name = "surrealdb", marker = "extra == 'all'", specifier = ">=1.0.6" },
+ { name = "surrealdb", marker = "extra == 'storage'", specifier = ">=1.0.6" },
{ name = "sympy", marker = "extra == 'all'", specifier = ">=1.13.3,<2" },
{ name = "sympy", marker = "extra == 'web-tools'", specifier = ">=1.13.3,<2" },
{ name = "tabulate", marker = "extra == 'all'", specifier = ">=0.9.0" },
@@ -1362,6 +1366,15 @@ requires-dist = [
]
provides-extras = ["dev", "docs", "rag", "web-tools", "document-tools", "media-tools", "communication-tools", "data-tools", "research-tools", "dev-tools", "model-platforms", "huggingface", "storage", "owl", "all"]
+[[package]]
+name = "cerberus"
+version = "1.3.7"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/08/92/6d861524d97a2c4913816309ca12afe313b32c8efc3ec641de98b890834b/cerberus-1.3.7.tar.gz", hash = "sha256:ecf249665400a0b7a9d5e4ee1ffc234fd5d003186d3e1904f70bc14038642c13", size = 29651, upload-time = "2024-12-31T14:24:00.964Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/15/ce/e3abf3fd04da28978eefb06ea906549f20f23f2ec6df8873ede6b62c8a8c/Cerberus-1.3.7-py3-none-any.whl", hash = "sha256:180e7d1fa1a5765cbff7b5c716e52fddddfab859dc8f625b0d563ace4b7a7ab3", size = 30508, upload-time = "2024-12-31T14:23:58.745Z" },
+]
+
[[package]]
name = "certifi"
version = "2025.7.14"
@@ -7529,6 +7542,22 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/d8/67/e320a11da2049dfd50fd1726952093b173332563f32d1a94cc3758932147/stripe-11.6.0-py2.py3-none-any.whl", hash = "sha256:6e6cf09ebb6d5fc2d708401cb8868fd7bff987a6d09a0433caaa92c62f97dbc5", size = 1636766, upload-time = "2025-02-24T22:39:44.629Z" },
]
+[[package]]
+name = "surrealdb"
+version = "1.0.6"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "aiohttp" },
+ { name = "cerberus" },
+ { name = "requests" },
+ { name = "typing-extensions", marker = "python_full_version < '3.12'" },
+ { name = "websockets" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/ef/0a/98aa9b08476811459525e38cd36c43a748b9b87592d42c87a8fa3d024186/surrealdb-1.0.6.tar.gz", hash = "sha256:fb3536db53e839b557cb57ffac78a3e46ac2a853b27225d53f6d58f7bdeb751c", size = 66450, upload-time = "2025-07-21T13:10:46.993Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/50/29/a89324828c8b8f6cc43b275881a987205549f2c8216244f841ea5a42d2ac/surrealdb-1.0.6-py3-none-any.whl", hash = "sha256:b569e307d5835b6040325069ea768864092bd8c501da57fefc44f489ce985cae", size = 62606, upload-time = "2025-07-21T13:10:45.523Z" },
+]
+
[[package]]
name = "sympy"
version = "1.14.0"